diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml new file mode 100644 index 00000000000..72778445a91 --- /dev/null +++ b/.github/workflows/ci-nightly.yml @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +# Nightly CI pipeline that tests optional dependencies (PyTorch, numba-cuda) +# against the latest cuda-python wheels built on main. +# +# This workflow does NOT build wheels — it downloads them from the latest +# successful CI run on main and runs integration tests with optional deps. + +name: "CI: Nightly optional-deps" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +on: + push: + branches: + - "main" + - "pull-request/[0-9]+" + schedule: + # 2 AM UTC daily, after the midnight main CI build finishes + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + run-id: + description: > + Override the CI run ID to download artifacts from. + Leave empty to auto-detect the latest successful main run. + type: string + default: '' + +jobs: + find-wheels: + runs-on: ubuntu-latest + outputs: + RUN_ID: ${{ steps.find.outputs.run_id }} + HEAD_SHA: ${{ steps.find.outputs.head_sha }} + CUDA_BUILD_VER: ${{ steps.get-vars.outputs.cuda_build_ver }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + + - name: Get CUDA build versions + id: get-vars + run: | + cuda_build_ver=$(yq '.cuda.build.version' ci/versions.yml) + echo "cuda_build_ver=$cuda_build_ver" >> $GITHUB_OUTPUT + + - name: Find latest successful CI run on main + id: find + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [[ -n "${{ inputs.run-id }}" ]]; then + RUN_ID="${{ inputs.run-id }}" + echo "Using manually specified run ID: $RUN_ID" + else + RUN_ID=$(gh run list \ + -b main \ + -L 1 \ + -w "CI" \ + -s success \ + -R "${{ github.repository }}" \ + --json databaseId \ + | jq -r '.[0].databaseId') + + if [[ -z "$RUN_ID" || "$RUN_ID" == "null" ]]; then + echo "::error::No successful CI run found on main" + exit 1 + fi + echo "Using latest successful CI run: $RUN_ID" + fi + + # Resolve the head SHA from the CI run — artifact names embed this. + HEAD_SHA=$(gh run view "$RUN_ID" \ + -R "${{ github.repository }}" \ + --json headSha \ + | jq -r '.headSha') + + echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT + echo "head_sha=$HEAD_SHA" >> $GITHUB_OUTPUT + echo "Source commit: $HEAD_SHA" + + # ── PyTorch interop tests ── + + test-pytorch-linux: + name: "Nightly PyTorch (linux-64)" + if: ${{ github.repository_owner == 'nvidia' }} + needs: find-wheels + permissions: + contents: read + actions: read + secrets: inherit + uses: ./.github/workflows/test-wheel-linux.yml + with: + build-type: nightly + host-platform: linux-64 + build-ctk-ver: ${{ needs.find-wheels.outputs.CUDA_BUILD_VER }} + run-id: ${{ needs.find-wheels.outputs.RUN_ID }} + sha: ${{ needs.find-wheels.outputs.HEAD_SHA }} + test-mode: nightly-pytorch + matrix_filter: 'map(select(.MODE == "nightly-pytorch"))' + + test-pytorch-windows: + name: "Nightly PyTorch (win-64)" + if: ${{ github.repository_owner == 'nvidia' }} + needs: find-wheels + permissions: + contents: read + actions: read + secrets: inherit + uses: ./.github/workflows/test-wheel-windows.yml + with: + build-type: nightly + host-platform: win-64 + build-ctk-ver: ${{ needs.find-wheels.outputs.CUDA_BUILD_VER }} + run-id: ${{ needs.find-wheels.outputs.RUN_ID }} + sha: ${{ needs.find-wheels.outputs.HEAD_SHA }} + test-mode: nightly-pytorch + matrix_filter: 'map(select(.MODE == "nightly-pytorch"))' + + # ── numba-cuda tests ── + + test-numba-cuda-linux-64: + name: "Nightly numba-cuda (linux-64)" + if: ${{ github.repository_owner == 'nvidia' }} + needs: find-wheels + permissions: + contents: read + actions: read + secrets: inherit + uses: ./.github/workflows/test-wheel-linux.yml + with: + build-type: nightly + host-platform: linux-64 + build-ctk-ver: ${{ needs.find-wheels.outputs.CUDA_BUILD_VER }} + run-id: ${{ needs.find-wheels.outputs.RUN_ID }} + sha: ${{ needs.find-wheels.outputs.HEAD_SHA }} + test-mode: nightly-numba-cuda + matrix_filter: 'map(select(.MODE == "nightly-numba-cuda"))' + + test-numba-cuda-linux-aarch64: + name: "Nightly numba-cuda (linux-aarch64)" + if: ${{ github.repository_owner == 'nvidia' }} + needs: find-wheels + permissions: + contents: read + actions: read + secrets: inherit + uses: ./.github/workflows/test-wheel-linux.yml + with: + build-type: nightly + host-platform: linux-aarch64 + build-ctk-ver: ${{ needs.find-wheels.outputs.CUDA_BUILD_VER }} + run-id: ${{ needs.find-wheels.outputs.RUN_ID }} + sha: ${{ needs.find-wheels.outputs.HEAD_SHA }} + test-mode: nightly-numba-cuda + matrix_filter: 'map(select(.MODE == "nightly-numba-cuda"))' + + test-numba-cuda-windows: + name: "Nightly numba-cuda (win-64)" + if: ${{ github.repository_owner == 'nvidia' }} + needs: find-wheels + permissions: + contents: read + actions: read + secrets: inherit + uses: ./.github/workflows/test-wheel-windows.yml + with: + build-type: nightly + host-platform: win-64 + build-ctk-ver: ${{ needs.find-wheels.outputs.CUDA_BUILD_VER }} + run-id: ${{ needs.find-wheels.outputs.RUN_ID }} + sha: ${{ needs.find-wheels.outputs.HEAD_SHA }} + test-mode: nightly-numba-cuda + matrix_filter: 'map(select(.MODE == "nightly-numba-cuda"))' + + # ── Status check ── + + checks: + name: Nightly check status + if: always() + runs-on: ubuntu-latest + needs: + - test-pytorch-linux + - test-pytorch-windows + - test-numba-cuda-linux-64 + - test-numba-cuda-linux-aarch64 + - test-numba-cuda-windows + steps: + - name: Exit + run: | + # If any dependency was cancelled or failed, that's a failure. + # + # See ci.yml for the full rationale on why we must use always() + # and explicitly check each result rather than relying on the + # default behaviour. + if ${{ needs.test-pytorch-linux.result == 'cancelled' || + needs.test-pytorch-linux.result == 'failure' || + needs.test-pytorch-windows.result == 'cancelled' || + needs.test-pytorch-windows.result == 'failure' || + needs.test-numba-cuda-linux-64.result == 'cancelled' || + needs.test-numba-cuda-linux-64.result == 'failure' || + needs.test-numba-cuda-linux-aarch64.result == 'cancelled' || + needs.test-numba-cuda-linux-aarch64.result == 'failure' || + needs.test-numba-cuda-windows.result == 'cancelled' || + needs.test-numba-cuda-windows.result == 'failure' }}; then + exit 1 + fi + exit 0 diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 35c5e6c3734..e1a36bc086b 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -29,6 +29,24 @@ on: skip-bindings-test: type: boolean default: false + run-id: + description: > + Workflow run ID to download artifacts from. + Defaults to the current run when empty. + type: string + default: '' + test-mode: + description: > + Test mode: 'standard' (default), 'nightly-pytorch', or + 'nightly-numba-cuda'. + type: string + default: 'standard' + sha: + description: > + Commit SHA used to construct artifact names. + Defaults to github.sha (current run) when empty. + type: string + default: '' defaults: run: @@ -119,7 +137,7 @@ jobs: HOST_PLATFORM: ${{ inputs.host-platform }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} PY_VER: ${{ matrix.PY_VER }} - SHA: ${{ github.sha }} + SHA: ${{ inputs.sha || github.sha }} SKIP_BINDINGS_TEST_OVERRIDE: ${{ inputs.skip-bindings-test && '1' || '0' }} run: ./ci/tools/env-vars test @@ -128,6 +146,8 @@ jobs: with: name: cuda-pathfinder-wheel path: ./cuda_pathfinder + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python build artifacts if: ${{ env.USE_BACKPORT_BINDINGS == '0' }} @@ -135,6 +155,8 @@ jobs: with: name: cuda-python-wheel path: . + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda.bindings build artifacts if: ${{ env.USE_BACKPORT_BINDINGS == '0' }} @@ -142,6 +164,8 @@ jobs: with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python & cuda.bindings build artifacts from the prior branch if: ${{ env.USE_BACKPORT_BINDINGS == '1' }} @@ -181,7 +205,7 @@ jobs: - name: Display structure of downloaded cuda-python artifacts run: | pwd - ls -lahR . + ls -lah cuda_python*.whl cuda_pathfinder/ - name: Display structure of downloaded cuda.bindings artifacts run: | @@ -194,6 +218,8 @@ jobs: with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.bindings Cython tests if: ${{ env.SKIP_CYTHON_TEST == '0' }} @@ -206,6 +232,8 @@ jobs: with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts run: | @@ -218,6 +246,8 @@ jobs: with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core Cython tests if: ${{ env.SKIP_CYTHON_TEST == '0' }} @@ -256,7 +286,9 @@ jobs: - name: Set up test repetition on nightly runs run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV" + # ── Standard test steps (skipped for nightly modes) ── - name: Run cuda.pathfinder tests with see_what_works + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works @@ -264,14 +296,14 @@ jobs: run: run-tests pathfinder - name: Run cuda.bindings tests - if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} run: run-tests bindings - name: Run cuda.bindings benchmarks (smoke test) - if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} run: | pip install pyperf pushd benchmarks/cuda_bindings @@ -279,12 +311,14 @@ jobs: popd - name: Run cuda.core tests + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} run: run-tests core - name: Ensure cuda-python installable + if: ${{ inputs.test-mode == 'standard' }} run: | if [[ "${{ matrix.LOCAL_CTK }}" == 1 ]]; then pip install --only-binary=:all: cuda_python*.whl @@ -293,6 +327,7 @@ jobs: fi - name: Install cuda.pathfinder extra wheels for testing + if: ${{ inputs.test-mode == 'standard' }} run: | set -euo pipefail pushd cuda_pathfinder @@ -301,8 +336,38 @@ jobs: popd - name: Run cuda.pathfinder tests with all_must_work + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work run: run-tests pathfinder + + # ── Nightly: install wheels + optional dep together ── + - name: Install cuda-python wheels + PyTorch + if: ${{ inputs.test-mode == 'nightly-pytorch' }} + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + TORCH_VER: ${{ matrix.TORCH_VER }} + TORCH_CUDA: ${{ matrix.TORCH_CUDA }} + run: run-tests nightly-pytorch + + - name: Install cuda-python wheels + numba-cuda + if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + run: run-tests nightly-numba-cuda + + # ── Nightly: run tests ── + - name: Run PyTorch interop tests + if: ${{ inputs.test-mode == 'nightly-pytorch' }} + run: | + pushd cuda_core + pytest -rxXs -v --durations=0 tests/test_utils.py tests/example_tests/ + popd + + - name: Run numba-cuda tests + if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} + run: python -m numba.runtests numba.cuda.tests diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 765823c6bfc..eefc9273594 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -29,6 +29,24 @@ on: skip-bindings-test: type: boolean default: false + run-id: + description: > + Workflow run ID to download artifacts from. + Defaults to the current run when empty. + type: string + default: '' + test-mode: + description: > + Test mode: 'standard' (default), 'nightly-pytorch', or + 'nightly-numba-cuda'. + type: string + default: 'standard' + sha: + description: > + Commit SHA used to construct artifact names. + Defaults to github.sha (current run) when empty. + type: string + default: '' jobs: compute-matrix: @@ -113,7 +131,7 @@ jobs: HOST_PLATFORM: ${{ inputs.host-platform }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} PY_VER: ${{ matrix.PY_VER }} - SHA: ${{ github.sha }} + SHA: ${{ inputs.sha || github.sha }} SKIP_BINDINGS_TEST_OVERRIDE: ${{ inputs.skip-bindings-test && '1' || '0' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: ./ci/tools/env-vars test @@ -123,6 +141,8 @@ jobs: with: name: cuda-pathfinder-wheel path: ./cuda_pathfinder + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python build artifacts if: ${{ env.USE_BACKPORT_BINDINGS == '0' }} @@ -130,6 +150,8 @@ jobs: with: name: cuda-python-wheel path: . + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda.bindings build artifacts if: ${{ env.USE_BACKPORT_BINDINGS == '0' }} @@ -137,6 +159,8 @@ jobs: with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python & cuda.bindings build artifacts from the prior branch if: ${{ env.USE_BACKPORT_BINDINGS == '1' }} @@ -167,7 +191,8 @@ jobs: - name: Display structure of downloaded cuda-python artifacts run: | Get-Location - Get-ChildItem -Recurse -Force | Select-Object Mode, LastWriteTime, Length, FullName + Get-ChildItem cuda_python*.whl | Select-Object Mode, LastWriteTime, Length, FullName + Get-ChildItem cuda_pathfinder/ | Select-Object Mode, LastWriteTime, Length, FullName - name: Display structure of downloaded cuda.bindings artifacts run: | @@ -180,6 +205,8 @@ jobs: with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.bindings Cython tests if: ${{ env.SKIP_CYTHON_TEST == '0' }} @@ -192,6 +219,8 @@ jobs: with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts run: | @@ -204,6 +233,8 @@ jobs: with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} + run-id: ${{ inputs.run-id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core Cython tests if: ${{ env.SKIP_CYTHON_TEST == '0' }} @@ -237,7 +268,9 @@ jobs: shell: bash --noprofile --norc -xeuo pipefail {0} run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV" + # ── Standard test steps (skipped for nightly modes) ── - name: Run cuda.pathfinder tests with see_what_works + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works @@ -246,7 +279,7 @@ jobs: run: run-tests pathfinder - name: Run cuda.bindings tests - if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} @@ -254,6 +287,7 @@ jobs: run: run-tests bindings - name: Run cuda.core tests + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} @@ -261,6 +295,7 @@ jobs: run: run-tests core - name: Ensure cuda-python installable + if: ${{ inputs.test-mode == 'standard' }} run: | if ('${{ matrix.LOCAL_CTK }}' -eq '1') { pip install --only-binary=:all: (Get-ChildItem -Filter cuda_python*.whl).FullName @@ -269,6 +304,7 @@ jobs: } - name: Install cuda.pathfinder extra wheels for testing + if: ${{ inputs.test-mode == 'standard' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | pushd cuda_pathfinder @@ -277,9 +313,50 @@ jobs: popd - name: Run cuda.pathfinder tests with all_must_work + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work shell: bash --noprofile --norc -xeuo pipefail {0} run: run-tests pathfinder + + # ── Nightly: install wheels + optional dep together ── + - name: Install Visual C++ Redistributable (required by PyTorch on Windows) + if: ${{ inputs.test-mode == 'nightly-pytorch' }} + run: | + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath ".\vc_redist.x64.exe" -ArgumentList "/install", "/quiet", "/norestart" -Wait + Remove-Item "vc_redist.x64.exe" + + - name: Install cuda-python wheels + PyTorch + if: ${{ inputs.test-mode == 'nightly-pytorch' }} + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + TORCH_VER: ${{ matrix.TORCH_VER }} + TORCH_CUDA: ${{ matrix.TORCH_CUDA }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: run-tests nightly-pytorch + + - name: Install cuda-python wheels + numba-cuda + if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: run-tests nightly-numba-cuda + + # ── Nightly: run tests ── + - name: Run PyTorch interop tests + if: ${{ inputs.test-mode == 'nightly-pytorch' }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + pushd cuda_core + pytest -rxXs -v --durations=0 tests/test_utils.py tests/example_tests/ + popd + + - name: Run numba-cuda tests + if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: python -m numba.runtests numba.cuda.tests diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index a402e3e4cf7..19931c3943a 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -62,7 +62,17 @@ linux: - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.2.1', LOCAL_CTK: '1', GPU: 'h100', GPU_COUNT: '2', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest', FLAVOR: 'wsl' } - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.2.1', LOCAL_CTK: '0', GPU: 'rtx4090', GPU_COUNT: '1', DRIVER: 'latest', FLAVOR: 'wsl' } - nightly: [] + nightly: + # nightly-pytorch (amd64 only — PyTorch does not ship arm64 GPU wheels) + - { MODE: 'nightly-pytorch', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', TORCH_VER: 'latest', TORCH_CUDA: 'cu126' } + - { MODE: 'nightly-pytorch', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', TORCH_VER: 'latest', TORCH_CUDA: 'cu130' } + - { MODE: 'nightly-pytorch', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu126' } + - { MODE: 'nightly-pytorch', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } + # nightly-numba-cuda + - { MODE: 'nightly-numba-cuda', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } + - { MODE: 'nightly-numba-cuda', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.2.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } + - { MODE: 'nightly-numba-cuda', ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } + - { MODE: 'nightly-numba-cuda', ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.2.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } windows: pull-request: @@ -85,4 +95,12 @@ windows: - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.2.1', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - nightly: [] + nightly: + # nightly-pytorch + - { MODE: 'nightly-pytorch', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', TORCH_VER: 'latest', TORCH_CUDA: 'cu126' } + - { MODE: 'nightly-pytorch', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', TORCH_VER: 'latest', TORCH_CUDA: 'cu130' } + - { MODE: 'nightly-pytorch', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu126' } + - { MODE: 'nightly-pytorch', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } + # nightly-numba-cuda + - { MODE: 'nightly-numba-cuda', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { MODE: 'nightly-numba-cuda', ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.2.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } diff --git a/ci/tools/run-tests b/ci/tools/run-tests index d42634a7073..a4001b7a001 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -13,19 +13,22 @@ if [[ ${#} -ne 1 ]]; then echo "Error: This script requires exactly 1 argument. You provided ${#}" exit 1 fi -if [[ "${1}" != "bindings" && "${1}" != "core" && "${1}" != "pathfinder" ]]; then - echo "Error: Invalid test module '${1}'. Must be 'bindings', 'core', or 'pathfinder'" +if [[ "${1}" != "bindings" && "${1}" != "core" && "${1}" != "pathfinder" && "${1}" != "nightly-pytorch" && "${1}" != "nightly-numba-cuda" ]]; then + echo "Error: Invalid test module '${1}'. Must be 'bindings', 'core', 'pathfinder', 'nightly-pytorch', or 'nightly-numba-cuda'" exit 1 fi test_module=${1} -# Unconditionally install pathfinder wheel -# (it is a direct dependency of bindings, and a transitive dependency of core) -pushd ./cuda_pathfinder -echo "Installing pathfinder wheel" -pip install ./*.whl --group test -popd +# For standard modes, install pathfinder up front (it is a direct dependency +# of bindings, and a transitive dependency of core). Nightly modes install +# all wheels together in a single pip call further below. +if [[ "${test_module}" != "nightly-pytorch" && "${test_module}" != "nightly-numba-cuda" ]]; then + pushd ./cuda_pathfinder + echo "Installing pathfinder wheel" + pip install ./*.whl --group test + popd +fi if [[ "${test_module}" == "pathfinder" ]]; then pushd ./cuda_pathfinder @@ -91,4 +94,72 @@ elif [[ "${test_module}" == "core" ]]; then ${SANITIZER_CMD} pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/cython fi popd +elif [[ "${test_module}" == "nightly-pytorch" || "${test_module}" == "nightly-numba-cuda" ]]; then + # Nightly optional-dependency testing. + # Install ALL wheels (pathfinder + bindings + core) and the optional dep + # in a single pip call so pip resolves version constraints in one shot + # and avoids costly uninstall/reinstall cycles. + + TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})" + CUDA_VER_MINOR="$(cut -d '.' -f 1-2 <<< "${CUDA_VER}")" + + FREE_THREADING="" + if python -c 'import sys; assert not sys._is_gil_enabled()' 2> /dev/null; then + FREE_THREADING+="-ft" + fi + + # Resolve pathfinder wheel to absolute path before pushd. + # CUDA_BINDINGS_ARTIFACTS_DIR and CUDA_CORE_ARTIFACTS_DIR are already + # absolute (set via realpath in env-vars). + PATHFINDER_WHL=($(realpath ./cuda_pathfinder/*.whl)) + + BINDINGS_WHL=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl) + if [[ "${LOCAL_CTK}" != 1 ]]; then + BINDINGS_WHL=("${BINDINGS_WHL[0]}[all]") + fi + + CORE_WHL=("${CUDA_CORE_ARTIFACTS_DIR}"/*.whl) + if [[ "${LOCAL_CTK}" != 1 ]]; then + CORE_WHL=("${CORE_WHL[0]}[cu${TEST_CUDA_MAJOR}]") + fi + + # pushd so --group reads test dependency groups from cuda_core/pyproject.toml. + # The explicit cuda-toolkit[...]==X.Y.* pin overrides the group's looser ==X.*. + pushd ./cuda_core + + PIP_ARGS=( + "${PATHFINDER_WHL[@]}" + "${BINDINGS_WHL[@]}" + "${CORE_WHL[@]}" + --group "test-cu${TEST_CUDA_MAJOR}${FREE_THREADING}" + ) + + if [[ "${test_module}" == "nightly-pytorch" ]]; then + # TORCH_VER and TORCH_CUDA must be set by the caller. + # Use cuda-toolkit[cudart] only — torch brings its own nvcc/nvrtc/etc. + # This avoids version conflicts between our nvidia-* pins and torch's. + echo "Installing pathfinder + bindings + core + test deps + PyTorch ${TORCH_VER} (${TORCH_CUDA})" + PIP_ARGS+=("cuda-toolkit[cudart]==${CUDA_VER_MINOR}.*") + if [[ "${TORCH_VER}" == "latest" ]]; then + PIP_ARGS+=(torch) + else + PIP_ARGS+=("torch==${TORCH_VER}") + fi + PIP_ARGS+=(--extra-index-url "https://download.pytorch.org/whl/${TORCH_CUDA}") + elif [[ "${test_module}" == "nightly-numba-cuda" ]]; then + echo "Installing pathfinder + bindings + core + test deps + numba-cuda" + # numba-cuda's test-cuXX group deps (can't use --group for a wheel install): + PIP_ARGS+=( + "cuda-toolkit[curand,cublas]==${CUDA_VER_MINOR}.*" + "numba-cuda[cu${TEST_CUDA_MAJOR}]" + "cupy-cuda${TEST_CUDA_MAJOR}x" + psutil cffi pytest-xdist pytest-benchmark filecheck ml_dtypes statistics + ) + fi + + pip install "${PIP_ARGS[@]}" + popd + + echo "Nightly install complete — installed packages:" + pip list fi