From 4033cfde02f10ff9fda09a4622a2a023e7be967d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 02:36:22 +0000 Subject: [PATCH 01/12] Initial plan From 351539f27c5bfd53e9f756591cce921b6dab5b68 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 02:42:30 +0000 Subject: [PATCH 02/12] Migrate BACKPORT_BRANCH from file to versions.yml Co-authored-by: leofang <5534781+leofang@users.noreply.github.com> --- .github/BACKPORT_BRANCH | 1 - .github/workflows/backport.yml | 2 +- .github/workflows/build-wheel.yml | 2 +- .github/workflows/test-wheel-linux.yml | 2 +- .github/workflows/test-wheel-windows.yml | 2 +- .spdx-ignore | 1 - ci/versions.yml | 2 ++ 7 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 .github/BACKPORT_BRANCH diff --git a/.github/BACKPORT_BRANCH b/.github/BACKPORT_BRANCH deleted file mode 100644 index 1ba33f6aec..0000000000 --- a/.github/BACKPORT_BRANCH +++ /dev/null @@ -1 +0,0 @@ -12.9.x diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index a050aa1712..83413bf762 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -28,7 +28,7 @@ jobs: - name: Load branch name id: get-branch run: | - OLD_BRANCH=$(cat .github/BACKPORT_BRANCH) + OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) echo "OLD_BRANCH=${OLD_BRANCH}" >> $GITHUB_ENV - name: Create backport pull requests diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index cc44f5557b..f6f3817002 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -350,7 +350,7 @@ jobs: && apt install gh -y fi - OLD_BRANCH=$(cat .github/BACKPORT_BRANCH) + OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" LATEST_PRIOR_RUN_ID=$(gh run list -b ${OLD_BRANCH} -L 1 -w "ci.yml" -s completed -R NVIDIA/cuda-python --json databaseId | jq '.[]| .databaseId') if [[ "$LATEST_PRIOR_RUN_ID" == "" ]]; then diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index ba3152f140..57199ddb64 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -147,7 +147,7 @@ jobs: && apt update \ && apt install gh -y - OLD_BRANCH=$(cat .github/BACKPORT_BRANCH) + OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" LATEST_PRIOR_RUN_ID=$(gh run list -b ${OLD_BRANCH} -L 1 -w "ci.yml" -s completed -R NVIDIA/cuda-python --json databaseId | jq '.[]| .databaseId') if [[ "$LATEST_PRIOR_RUN_ID" == "" ]]; then diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 340f418b22..784837dd7c 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -145,7 +145,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - $OLD_BRANCH = Get-Content .github/BACKPORT_BRANCH + $OLD_BRANCH = yq '.backport_branch' ci/versions.yml $OLD_BASENAME = "cuda-bindings-python${env:PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" $runData = gh run list -b $OLD_BRANCH -L 1 -w "ci.yml" -s completed -R NVIDIA/cuda-python --json databaseId | ConvertFrom-Json if (-not $runData -or $runData.Length -eq 0 -or -not $runData[0].databaseId -or [string]::IsNullOrEmpty($runData[0].databaseId)) { diff --git a/.spdx-ignore b/.spdx-ignore index 84f051fafc..ee658d2559 100644 --- a/.spdx-ignore +++ b/.spdx-ignore @@ -5,7 +5,6 @@ LICENSE *.png .gitattributes .gitignore -.github/BACKPORT_BRANCH requirements*.txt cuda_bindings/examples/* diff --git a/ci/versions.yml b/ci/versions.yml index 6ce7e3a32f..5a5a9e586d 100644 --- a/ci/versions.yml +++ b/ci/versions.yml @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +backport_branch: "12.9.x" + cuda: build: version: "13.0.2" From 301772e477c69a7a5a3bd5f1b757ebfd8ff0278b Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Wed, 3 Dec 2025 22:18:56 -0500 Subject: [PATCH 03/12] remove the leftover from previous hack The version check is no longer needed, see context: https://github.com/NVIDIA/cuda-python/pull/1041#discussion_r2396216761. --- .github/workflows/build-wheel.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index f6f3817002..feb06868dc 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -334,7 +334,6 @@ jobs: cuda-path: "./cuda_toolkit_prev" - name: Download cuda.bindings build artifacts from the prior branch - if: startsWith(matrix.python-version, '3.14') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | From 0772426eeb03a789f6640132f4000522c07ce401 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Wed, 3 Dec 2025 22:48:13 -0500 Subject: [PATCH 04/12] install yq in GH-hosted Win runners --- .github/workflows/build-wheel.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index feb06868dc..0bac7bba39 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -75,6 +75,23 @@ jobs: if: ${{ startsWith(inputs.host-platform, 'win') }} uses: ilammy/msvc-dev-cmd@v1 # TODO: ask admin to allow pinning commits + - name: Set up yq + # GitHub made an unprofessional decision to not provide it in their Windows VMs, + # see https://github.com/actions/runner-images/issues/7443. + if: ${{ startsWith(inputs.host-platform, 'win') }} + env: + # doesn't seem there's an easy way to avoid hard-coding it? + YQ_URL: https://github.com/mikefarah/yq/releases/latest/download/yq_windows_amd64.exe + YQ_DIR: yq_latest + shell: pwsh -command ". '{0}'" + run: | + mkdir -Force -ErrorAction SilentlyContinue "${env:YQ_DIR}" | Out-Null + Invoke-WebRequest -UseBasicParsing -OutFile "${env:YQ_DIR}/yq.exe" -Uri "$env:YQ_URL" + ls -l $env:YQ_DIR + echo "$((Get-Location).Path)\\$env:YQ_DIR" >> $env:GITHUB_PATH + $env:Path += ";$((Get-Location).Path)\\$env:YQ_DIR" + yq --version + - name: Set environment variables env: CUDA_VER: ${{ inputs.cuda-version }} From 1a545d43566d424de59956e66b55049074b265aa Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Wed, 3 Dec 2025 22:52:23 -0500 Subject: [PATCH 05/12] remove outdated WAR --- .github/workflows/build-wheel.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 0bac7bba39..c53ca8954d 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -292,8 +292,7 @@ jobs: id: setup-python2 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - # workaround for actions/runner-images#12377 (the cached 3.13.4 is buggy on Windows) - python-version: ${{ matrix.python-version == '3.13' && '3.13.5' || matrix.python-version }} + python-version: ${{ matrix.python-version }} - name: verify free-threaded build if: endsWith(matrix.python-version, 't') From aa9a34fbe71bfb833f50b49b0faaabcdf977f06c Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Wed, 3 Dec 2025 22:54:32 -0500 Subject: [PATCH 06/12] zstd is now installed in self-hosted Windows runners --- .github/workflows/test-wheel-windows.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 784837dd7c..b98e9ba2d7 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -125,21 +125,6 @@ jobs: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - - name: Install zstd - # the GPU runner image does not have zstd pre-installed... and it's needed by actions/cache - if: ${{ matrix.LOCAL_CTK == '1' }} - env: - # doesn't seem there's an easy way to avoid hard-coding it? - ZSTD_URL: https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-v1.5.7-win64.zip - ZSTD_DIR: zstd-v1.5.7-win64 - run: | - Invoke-WebRequest -Uri "$env:ZSTD_URL" -OutFile "zstd-win64.zip" - Expand-Archive -Path "zstd-win64.zip" -DestinationPath . - ls -l $env:ZSTD_DIR - echo "$((Get-Location).Path)\\$env:ZSTD_DIR" >> $env:GITHUB_PATH - $env:Path += ";$((Get-Location).Path)\\$env:ZSTD_DIR" - zstd --version - - name: Download cuda-python & cuda.bindings build artifacts from the prior branch if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '1'}} env: From ca9a06f10292155255760fa44a41c20701836621 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Wed, 3 Dec 2025 22:56:52 -0500 Subject: [PATCH 07/12] install yq in self-hosted Linux runners --- .github/workflows/test-wheel-linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 57199ddb64..f28ca20692 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -99,8 +99,8 @@ jobs: continue-on-error: false with: # for artifact fetching, graphics libs - dependencies: "jq wget libgl1 libegl1" - dependent_exes: "jq wget" + dependencies: "jq yq wget libgl1 libegl1" + dependent_exes: "jq yq wget" - name: Set environment variables env: From 9fdfa6cdc510a5e3786e383920dff53f6c8791ef Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Wed, 3 Dec 2025 23:16:44 -0500 Subject: [PATCH 08/12] avoid installing yq on self-hosted Linux runners... the Debian package is unfortunately discontinued, and it is simpler if we consolidate the yq calls in the same job where yq is available --- .github/workflows/test-wheel-linux.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index f28ca20692..8f1151318f 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -33,6 +33,7 @@ jobs: (inputs.host-platform == 'linux-aarch64' && 'arm64') }} outputs: MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }} + OLD_BRANCH: ${{ steps.compute-matrix.outputs.OLD_BRANCH }} steps: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -67,6 +68,10 @@ jobs: echo "MATRIX=${MATRIX}" | tee --append "${GITHUB_OUTPUT}" + # This job has yq already installed, so let's do it here + OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) + echo "OLD_BRANCH=${OLD_BRANCH}" >> "$GITHUB_OUTPUT" + test: name: py${{ matrix.PY_VER }}, ${{ matrix.CUDA_VER }}, ${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}, ${{ matrix.GPU }} needs: compute-matrix @@ -99,8 +104,8 @@ jobs: continue-on-error: false with: # for artifact fetching, graphics libs - dependencies: "jq yq wget libgl1 libegl1" - dependent_exes: "jq yq wget" + dependencies: "jq wget libgl1 libegl1" + dependent_exes: "jq wget" - name: Set environment variables env: @@ -147,7 +152,7 @@ jobs: && apt update \ && apt install gh -y - OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) + OLD_BRANCH=${{ needs.compute-matrix.outputs.OLD_BRANCH }} OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" LATEST_PRIOR_RUN_ID=$(gh run list -b ${OLD_BRANCH} -L 1 -w "ci.yml" -s completed -R NVIDIA/cuda-python --json databaseId | jq '.[]| .databaseId') if [[ "$LATEST_PRIOR_RUN_ID" == "" ]]; then From f1af6cbdbc65f555d423cbfe23d2de9cdb54dccc Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 4 Dec 2025 00:06:07 -0500 Subject: [PATCH 09/12] CAN WE GO CONTAINERLESS?! --- .github/workflows/test-wheel-linux.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 8f1151318f..d0483a09eb 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -81,13 +81,6 @@ jobs: runs-on: "linux-${{ matrix.ARCH }}-gpu-${{ matrix.GPU }}-${{ matrix.DRIVER }}-1" # The build stage could fail but we want the CI to keep moving. if: ${{ github.repository_owner == 'nvidia' && !cancelled() }} - # Our self-hosted runners require a container - # TODO: use a different (nvidia?) container - container: - options: -u root --security-opt seccomp=unconfined --shm-size 16g - image: ubuntu:22.04 - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: - name: Ensure GPU is working run: nvidia-smi From a26bfc5875c866445db80f940157b94ca28cc6e8 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 4 Dec 2025 00:49:09 -0500 Subject: [PATCH 10/12] try this --- .github/workflows/test-wheel-linux.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index d0483a09eb..5926e56d8f 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -212,13 +212,15 @@ jobs: pwd ls -lahR $CUDA_CORE_CYTHON_TESTS_DIR + mkdir -p ${HOME}/setup_python + - name: Set up Python ${{ matrix.PY_VER }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.PY_VER }} env: - # we use self-hosted runners on which setup-python behaves weirdly... - AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache" + # we use self-hosted runners on which setup-python behaves weirdly (Python include can't be found)... + AGENT_TOOLSDIRECTORY: "${HOME}/setup_python" - name: Set up mini CTK if: ${{ matrix.LOCAL_CTK == '1' }} From e5fac2bf115f5afb24fabb8029f789a9f34facbd Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 4 Dec 2025 21:16:17 -0500 Subject: [PATCH 11/12] try this --- .github/workflows/test-wheel-linux.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 5926e56d8f..cce40f660e 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -212,15 +212,13 @@ jobs: pwd ls -lahR $CUDA_CORE_CYTHON_TESTS_DIR - mkdir -p ${HOME}/setup_python - - name: Set up Python ${{ matrix.PY_VER }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.PY_VER }} env: # we use self-hosted runners on which setup-python behaves weirdly (Python include can't be found)... - AGENT_TOOLSDIRECTORY: "${HOME}/setup_python" + AGENT_TOOLSDIRECTORY: "./setup_python" - name: Set up mini CTK if: ${{ matrix.LOCAL_CTK == '1' }} From 4011bb8e54b57b6138ce8da809bca606be4b9b21 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 4 Dec 2025 21:58:37 -0500 Subject: [PATCH 12/12] try this --- .github/workflows/test-wheel-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index cce40f660e..d5fc0dbc14 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -218,7 +218,7 @@ jobs: python-version: ${{ matrix.PY_VER }} env: # we use self-hosted runners on which setup-python behaves weirdly (Python include can't be found)... - AGENT_TOOLSDIRECTORY: "./setup_python" + AGENT_TOOLSDIRECTORY: ${{ runner.temp }}/setup_python - name: Set up mini CTK if: ${{ matrix.LOCAL_CTK == '1' }}