Release pypi prep change (#1587)

* [aarch64] Release pypi prep script change for aarch64 builds * Release versions for testing Testing calling version (#1588) Upstream/release validations (#1589) * Testing calling version * add release matrix Upstream/release validations (#1590) * Testing calling version * add release matrix * test test (#1591) test (#1592) Release v1 (#1595) * test * test Release v1 (#1596) * test * test * test test (#1597) Test versions validations (#1598) * test * basedir Test versions validations (#1599) * test * basedir * test test (#1600) * test * test Add release versions everywhere (#1601) * test * test * test * test test (#1602) Test version validations (#1603) * test * test Test version validations (#1604) * test * test * test tests (#1605) More tests nov16 (#1606) * tests * test More tests nov16 (#1607) * tests * test * test More tests nov16 (#1608) * tests * test * test * test More tests nov16 (#1609) * tests * test * test * test * test test poetry_fix test test Changes needed for core enablement of 3.12 binary wheels (#1586) Fix aarch64 build on 3.8 (#1593) Add some more validation checks for torch.linalg.eigh and torch.compile (#1580) * Add some more validation checks for torch.linalg.eigh and torch.compile * Update test * Also update smoke_test.py * Fix lint Revert "Add some more validation checks for torch.linalg.eigh and torch.compile (#1580)" (#1594) This reverts commit 4c7fa06. Release validations using release version matrix (#1611) * Release pypi prep change (#1587) * [aarch64] Release pypi prep script change for aarch64 builds * Release versions for testing Testing calling version (#1588) Upstream/release validations (#1589) * Testing calling version * add release matrix Upstream/release validations (#1590) * Testing calling version * add release matrix * test test (#1591) test (#1592) Release v1 (#1595) * test * test Release v1 (#1596) * test * test * test test (#1597) Test versions validations (#1598) * test * basedir Test versions validations (#1599) * test * basedir * test test (#1600) * test * test Add release versions everywhere (#1601) * test * test * test * test test (#1602) Test version validations (#1603) * test * test Test version validations (#1604) * test * test * test tests (#1605) More tests nov16 (#1606) * tests * test More tests nov16 (#1607) * tests * test * test More tests nov16 (#1608) * tests * test * test * test More tests nov16 (#1609) * tests * test * test * test * test * fix_lint fix: typo (#1581) desired_cuda -> DESIRED_CUDA (#1612) * desired_cuda -> DESIRED_CUDA Found with shellcheck * Update manywheel/build_cuda.sh Co-authored-by: Nikita Shulga <2453524+malfet@users.noreply.github.com> --------- Co-authored-by: Nikita Shulga <2453524+malfet@users.noreply.github.com> [BE] Cleanup build unused code (#1613) 1. Upload Scripts are not used anymore. We use Github Action upload workflows 2. M1 Builds are now automated 3. build_all.bat run git grep in pytorch and builder - No result Changes to pypi release promotion scripts introduced for 2.1.0 and 2.1.1 (#1614) * Changes topypi release promotion scripts introduced during 2.1.1 * typo Pin miniconda version for Windows To Miniconda3-py311_23.9.0-0-Windows-x86_64.exe
pytorch · Nov 21, 2023 · 97abda2 · 97abda2
1 parent 75a8770
commit 97abda2
Show file tree

Hide file tree

Showing 26 changed files with 251 additions and 265 deletions.
diff --git a/.github/scripts/validate_pipy.sh b/.github/scripts/validate_pipy.sh
@@ -2,15 +2,21 @@ conda create -yn ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
 conda activate ${ENV_NAME}_pypi
 
 TEST_SUFFIX=""
+RELEASE_SUFFIX=""
+# if RELESE version is passed as parameter - install speific version
+if [[ ! -z ${RELEASE_VERSION} ]]; then
+    RELEASE_SUFFIX="==${RELEASE_VERSION}"
+fi
+
 if [[ ${TORCH_ONLY} == 'true' ]]; then
     TEST_SUFFIX=" --package torchonly"
-    pip3 install --pre torch --extra-index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn"
+    pip3 install --pre torch${RELEASE_SUFFIX} --extra-index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn"
 else
     if [[ ${MATRIX_CHANNEL} != "release" ]]; then
-        pip3 install --pre torch --extra-index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn"
+        pip3 install --pre torch${RELEASE_SUFFIX}  --extra-index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn"
         pip3 install --pre torchvision torchaudio --extra-index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}"
     else
-        pip3 install torch torchvision torchaudio
+        pip3 install torch${RELEASE_SUFFIX}  torchvision torchaudio
     fi
 fi
 

diff --git a/.github/scripts/validate_poetry.sh b/.github/scripts/validate_poetry.sh
@@ -13,25 +13,17 @@ if [[ ${TORCH_ONLY} == 'true' ]]; then
     TEST_SUFFIX=" --package torchonly"
 fi
 
-if [[ ${MATRIX_CHANNEL} != "release" ]]; then
-    # Installing poetry from our custom repo. We need to configure it before use and disable authentication
-    export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
-    poetry source add --priority=explicit domains "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}"
-    poetry source add --priority=supplemental pytorch-channel "https://download.pytorch.org/whl/${MATRIX_CHANNEL}"
-    poetry source add --priority=supplemental pytorch "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn"
-    poetry --quiet add --source pytorch torch
+RELEASE_SUFFIX=""
+# if RELESE version is passed as parameter - install speific version
+if [[ ! -z ${RELEASE_VERSION} ]]; then
+    RELEASE_SUFFIX="@${RELEASE_VERSION}"
+fi
+
 
-    if [[ ${TORCH_ONLY} != 'true' ]]; then
-        poetry --quiet add --source domains torchvision torchaudio
-    fi
+if [[ ${TORCH_ONLY} == 'true' ]]; then
+    poetry --quiet add torch${RELEASE_SUFFIX}
 else
-    export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
-    poetry source add --priority=explicit pytorch "https://download.pytorch.org/whl/${MATRIX_DESIRED_CUDA}"
-    if [[ ${TORCH_ONLY} == 'true' ]]; then
-        poetry --quiet add torch
-    else
-        poetry --quiet add --source pytorch torch torchaudio torchvision
-    fi
+    poetry --quiet add torch${RELEASE_SUFFIX} torchaudio torchvision
 fi
 
 python ../test/smoke_test/smoke_test.py ${TEST_SUFFIX} --runtime-error-check disabled

diff --git a/.github/workflows/validate-aarch64-linux-binaries.yml b/.github/workflows/validate-aarch64-linux-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-aarch64-linux-matrix:
@@ -47,7 +67,6 @@ jobs:
       os: linux-aarch64
       channel: ${{ inputs.channel }}
       with-cuda: disable
-
   linux-aarch64:
     needs: generate-aarch64-linux-matrix
     strategy:
@@ -72,6 +91,8 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="linux-aarch64"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
         eval "$(conda shell.bash hook)"
 
         # Standart case: Validate binaries

diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml
@@ -60,44 +60,69 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate'
+        default: ""
+        required: false
+        type: string
+
 
 jobs:
+  generate-release-matrix:
+    uses: pytorch/test-infra/.github/workflows/generate_release_matrix.yml@main
+    with:
+      version: ${{ inputs.version }}
+
   win:
     if:  inputs.os == 'windows' || inputs.os == 'all'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-windows-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
 
   linux:
     if:  inputs.os == 'linux' || inputs.os == 'all'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-linux-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
 
   linux-aarch64:
     if:  inputs.os == 'linux-aarch64'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-aarch64-linux-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
-
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
   mac:
     if:  inputs.os == 'macos' || inputs.os == 'all'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-macos-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
 
   mac-arm64:
     if:  inputs.os == 'macos' || inputs.os == 'all'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-macos-arm64-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-linux-matrix:
@@ -64,8 +84,11 @@ jobs:
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
         export TARGET_OS="linux"
         eval "$(conda shell.bash hook)"
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
+        cat release_matrix.json
 
         # Special case PyPi installation package. And Install of PyPi package via poetry
         if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" && ${MATRIX_GPU_ARCH_VERSION} == "12.1" ]]; then

diff --git a/.github/workflows/validate-macos-arm64-binaries.yml b/.github/workflows/validate-macos-arm64-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-macos-arm64-matrix:
@@ -64,4 +84,6 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="macos-arm64"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
         source ./.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-macos-binaries.yml b/.github/workflows/validate-macos-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-macos-matrix:
@@ -64,4 +84,6 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="macos"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
         source ./.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-windows-matrix:
@@ -46,7 +66,6 @@ jobs:
       package-type: all
       os: windows
       channel: ${{ inputs.channel }}
-
   win:
     needs: generate-windows-matrix
     strategy:
@@ -66,9 +85,10 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="windows"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
         source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh
         if [[ ${MATRIX_GPU_ARCH_VERSION} == "12.1" ]]; then
           ./windows/internal/driver_update.bat
         fi
-
         source ./.github/scripts/validate_binaries.sh
diff --git a/aarch64_linux/aarch64_ci_setup.sh b/aarch64_linux/aarch64_ci_setup.sh
@@ -19,7 +19,17 @@ curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/late
 chmod +x /mambaforge.sh
 /mambaforge.sh -b -p /opt/conda
 rm /mambaforge.sh
-/opt/conda/bin/conda config --set ssl_verify False
-/opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf pygit2 openblas ninja scons
+source /opt/conda/etc/profile.d/conda.sh
+conda config --set ssl_verify False
+conda create -y -c conda-forge -n aarch64_env python=${DESIRED_PYTHON}
+conda activate aarch64_env
+
+if [[ "$DESIRED_PYTHON"  == "3.8" ]]; then
+    NUMPY_VERSION="1.24.4"
+else
+    NUMPY_VERSION="1.26.0"
+fi
+conda install -y -c conda-forge numpy==${NUMPY_VERSION} pyyaml==6.0.1 patchelf==0.17.2 pygit2==1.13.2 openblas==0.3.24 ninja==1.11.1 scons==4.5.2
+
 python --version
 conda --version