Use MATRIX_* variables instead of redeefining new var each time (#1265)

pytorch · Jan 11, 2023 · cd499c7 · cd499c7
1 parent 2a3f8b8
commit cd499c7
Show file tree

Hide file tree

Showing 9 changed files with 41 additions and 82 deletions.
diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
@@ -1,38 +1,31 @@
-#!/usr/bin/env bash
-set -ex
-
-if [[ ${TARGET_OS} == 'windows' ]]; then
-    source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh
-else
-    eval "$(conda shell.bash hook)"
-fi
-
-if [[ ${PACKAGE_TYPE} == "libtorch" ]]; then
-    curl ${INSTALLATION} -o libtorch.zip
+if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then
+    curl ${MATRIX_INSTALLATION} -o libtorch.zip
     unzip libtorch.zip
 else
-    if [ $DESIRED_PYTHON == '3.11' ]; then
+    if [ $MATRIX_PYTHON_VERSION == '3.11' ]; then
         export CPYTHON_VERSIONS=3.11.0
         sudo yum -y install openssl-devel libssl-dev bzip2-devel libffi-devel
         sudo yum -y groupinstall "Development Tools"
         export PYTHON_PATH="/opt/_internal/cpython-3.11.0/bin"
         export PIP_PATH="${PYTHON_PATH}/pip"
-        export PIP_INSTALLATION="${INSTALLATION/pip3/"$PIP_PATH"}"
+        export PIP_INSTALLATION="${MATRIX_INSTALLATION/pip3/"$PIP_PATH"}"
         export WITH_OPENSSL="/opt/openssl"
         ./common/install_cpython.sh
         eval ${PYTHON_PATH}/python --version
         eval ${PIP_INSTALLATION}
         eval ${PYTHON_PATH}/python ./test/smoke_test/smoke_test.py --package torchonly
     else
-        conda create -y -n ${ENV_NAME} python=${DESIRED_PYTHON} numpy pillow
+        conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy pillow
         conda activate ${ENV_NAME}
-        export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
-        export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH
-        INSTALLATION=${INSTALLATION/"conda install"/"conda install -y"}
+        INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"}
         eval $INSTALLATION
-        python  ./test/smoke_test/smoke_test.py
-        if [[ ${TARGET_OS} != 'macos' && ${TARGET_OS} != 'windows' ]]; then
+
+        if [[ ${TARGET_OS} == 'linux' ]]; then
+            export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
+            export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH
             ${PWD}/check_binary.sh
         fi
+
+        python  ./test/smoke_test/smoke_test.py
     fi
 fi
diff --git a/.github/workflows/validate-domain-library.yml b/.github/workflows/validate-domain-library.yml
@@ -89,13 +89,7 @@ jobs:
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
-        export DESIRED_PYTHON="${{ matrix.python_version }}"
-        export PACKAGE_TYPE="${{ matrix.package_type }}"
-        export CHANNEL="${{ matrix.channel }}"
         export SMOKE_TEST="${{ inputs.smoke_test }}"
-        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
-        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
-        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
         eval $SMOKE_TEST
   validate-windows:
     if:  (inputs.os == 'windows' || inputs.os == 'all')
@@ -113,13 +107,7 @@ jobs:
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
-        export DESIRED_PYTHON="${{ matrix.python_version }}"
-        export PACKAGE_TYPE="${{ matrix.package_type }}"
-        export CHANNEL="${{ matrix.channel }}"
         export SMOKE_TEST="${{ inputs.smoke_test }}"
-        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
-        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
-        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
         export TARGET_OS="windows"
         eval $SMOKE_TEST
   validate-macos:
@@ -138,13 +126,8 @@ jobs:
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
-        export DESIRED_PYTHON="${{ matrix.python_version }}"
-        export PACKAGE_TYPE="${{ matrix.package_type }}"
-        export CHANNEL="${{ matrix.channel }}"
+        export TARGET_OS="macos"
         export SMOKE_TEST="${{ inputs.smoke_test }}"
-        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
-        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
-        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
         eval $SMOKE_TEST
   validate-macos-arm64:
     if:  (inputs.os == 'macos-arm64' || inputs.os == 'all')
@@ -162,11 +145,5 @@ jobs:
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
-        export DESIRED_PYTHON="${{ matrix.python_version }}"
-        export PACKAGE_TYPE="${{ matrix.package_type }}"
-        export CHANNEL="${{ matrix.channel }}"
-        export SMOKE_TEST="${{ inputs.smoke_test }}"
-        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
-        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
-        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
+        export TARGET_OS="macos-arm64"
         eval $SMOKE_TEST
diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
@@ -50,16 +50,10 @@ jobs:
       repository: "pytorch/builder"
       ref: ${{ inputs.ref || github.ref }}
       job-name: ${{ matrix.build_name }}
+      binary-matrix: ${{ toJSON(matrix) }}
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
-        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
-        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
-        export INSTALLATION="${{ matrix.installation }}"
-        export CUDA_VER="${{ matrix.desired_cuda }}"
-        export DESIRED_PYTHON="${{ matrix.python_version }}"
-        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
-        export DESIRED_DEVTOOLSET="${{ matrix.devtoolset }}"
-        export PACKAGE_TYPE="${{ matrix.package_type }}"
         export TARGET_OS="linux"
-        ./.github/scripts/validate_binaries.sh
+        eval "$(conda shell.bash hook)"
+        source ./.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-macos-arm64-binaries.yml b/.github/workflows/validate-macos-arm64-binaries.yml
@@ -48,15 +48,9 @@ jobs:
       repository: "pytorch/builder"
       ref: ${{ inputs.ref || github.ref }}
       job-name: ${{ matrix.build_name }}
+      binary-matrix: ${{ toJSON(matrix) }}
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
-        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
-        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
-        export INSTALLATION="${{ matrix.installation }}"
-        export CUDA_VER="${{ matrix.desired_cuda }}"
-        export DESIRED_PYTHON="${{ matrix.python_version }}"
-        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
-        export PACKAGE_TYPE="${{ matrix.package_type }}"
         export TARGET_OS="macos-arm64"
-        ./.github/scripts/validate_binaries.sh
+        source ./.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-macos-binaries.yml b/.github/workflows/validate-macos-binaries.yml
@@ -48,16 +48,9 @@ jobs:
       repository: "pytorch/builder"
       ref: ${{ inputs.ref || github.ref }}
       job-name: ${{ matrix.build_name }}
+      binary-matrix: ${{ toJSON(matrix) }}
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
-        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
-        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
-        export INSTALLATION="${{ matrix.installation }}"
-        export CUDA_VER="${{ matrix.desired_cuda }}"
-        export DESIRED_PYTHON="${{ matrix.python_version }}"
-        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
-        export DESIRED_DEVTOOLSET="${{ matrix.devtoolset }}"
-        export PACKAGE_TYPE="${{ matrix.package_type }}"
         export TARGET_OS="macos"
-        ./.github/scripts/validate_binaries.sh
+        source ./.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml
@@ -24,6 +24,7 @@ on:
       - .github/workflows/validate-windows-binaries.yml
       - .github/workflows/validate-macos-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
+      - .github/scripts/validate_binaries.sh
       - test/smoke_test/*
 jobs:
   nightly:

diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml
@@ -49,17 +49,11 @@ jobs:
       repository: "pytorch/builder"
       ref: ${{ inputs.ref || github.ref }}
       job-name: ${{ matrix.build_name }}
+      binary-matrix: ${{ toJSON(matrix) }}
       timeout: 60
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
-        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
-        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
-        export INSTALLATION="${{ matrix.installation }}"
-        export CUDA_VER="${{ matrix.desired_cuda }}"
-        export DESIRED_PYTHON="${{ matrix.python_version }}"
-        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
-        export DESIRED_DEVTOOLSET="${{ matrix.devtoolset }}"
-        export PACKAGE_TYPE="${{ matrix.package_type }}"
         export TARGET_OS="windows"
-        ./.github/scripts/validate_binaries.sh
+        source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh
+        source ./.github/scripts/validate_binaries.sh
diff --git a/check_binary.sh b/check_binary.sh
@@ -22,6 +22,19 @@ set -eux -o pipefail
 # libtorch package.
 
 
+if [[ -z ${DESIRED_PYTHON:-} ]]; then
+  export DESIRED_PYTHON=${MATRIX_PYTHON_VERSION:-}
+fi
+if [[ -z ${DESIRED_CUDA:-} ]]; then
+  export DESIRED_CUDA=${MATRIX_DESIRED_CUDA:-}
+fi
+if [[ -z ${DESIRED_DEVTOOLSET:-} ]]; then
+  export DESIRED_DEVTOOLSET=${MATRIX_DESIRED_DEVTOOLSET:-}
+fi
+if [[ -z ${PACKAGE_TYPE:-} ]]; then
+  export PACKAGE_TYPE=${MATRIX_PACKAGE_TYPE:-}
+fi
+
 # The install root depends on both the package type and the os
 # All MacOS packages use conda, even for the wheel packages.
 if [[ "$PACKAGE_TYPE" == libtorch ]]; then

diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py
@@ -8,10 +8,10 @@
 import importlib
 import subprocess
 
-gpu_arch_ver = os.getenv("GPU_ARCH_VER")
-gpu_arch_type = os.getenv("GPU_ARCH_TYPE")
+gpu_arch_ver = os.getenv("MATRIX_GPU_ARCH_VERSION")
+gpu_arch_type = os.getenv("MATRIX_GPU_ARCH_TYPE")
 # use installation env variable to tell if it is nightly channel
-installation_str = os.getenv("INSTALLATION")
+installation_str = os.getenv("MATRIX_INSTALLATION")
 is_cuda_system = gpu_arch_type == "cuda"
 SCRIPT_DIR = Path(__file__).parent
 NIGHTLY_ALLOWED_DELTA = 3