Using windows runners from test-infra for validation workflows (#1188)

* Testing new windows runners test Testing Testing testing testing test Test Testing testing Testing Testing test Test test testing testing Test testing test testing testing testing testing testing testing test test testing testing testing testing Test test test testing testing testing testing testing testing testing testing testing Refactor code * Adding details for the test-infra issue
pytorch · Nov 9, 2022 · b290a75 · b290a75
1 parent 736bac6
commit b290a75
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 111 deletions.
diff --git a/.github/actions/validate-windows-binary/action.yml b/.github/actions/validate-windows-binary/action.yml
diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml
@@ -20,74 +20,37 @@ on:
           - all
 
 jobs:
-  generate-windows-conda-matrix:
+  generate-windows-matrix:
     uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
     with:
-      package-type: conda
+      package-type: all
       os: windows
       channel: ${{ inputs.channel }}
-  generate-windows-wheel-matrix:
-    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
-    with:
-      package-type: wheel
-      os: windows
-      channel: ${{ inputs.channel }}
-  generate-windows-libtorch-matrix:
-    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
-    with:
-      package-type: libtorch
-      os: windows
-      channel: ${{ inputs.channel }}
-
-  win-conda:
-    needs: generate-windows-conda-matrix
-    strategy:
-      matrix:
-        ${{ fromJson(needs.generate-windows-conda-matrix.outputs.matrix) }}
-      fail-fast: false
-    runs-on: ${{ matrix.validation_runner }}
-    steps:
-      - name: Checkout PyTorch builder
-        uses: actions/checkout@v2
-      - name: Validate binary conda
-        uses: ./.github/actions/validate-windows-binary
-        with:
-          gpu_arch_type: ${{ matrix.gpu_arch_type }}
-          gpu_arch_ver: ${{ matrix.gpu_arch_version }}
-          installation: ${{ matrix.installation }}
-          python_version: ${{ matrix.python_version }}
 
-  win-wheel:
-    needs: generate-windows-wheel-matrix
+  win:
+    needs: generate-windows-matrix
     strategy:
-      matrix:
-        ${{ fromJson(needs.generate-windows-wheel-matrix.outputs.matrix) }}
+      matrix: ${{ fromJson(needs.generate-windows-matrix.outputs.matrix) }}
       fail-fast: false
-    runs-on: ${{ matrix.validation_runner }}
-    steps:
-      - name: Checkout PyTorch builder
-        uses: actions/checkout@v2
-      - name: Validate binary wheel
-        uses: ./.github/actions/validate-windows-binary
-        with:
-          gpu_arch_type: ${{ matrix.gpu_arch_type }}
-          gpu_arch_ver: ${{ matrix.gpu_arch_version }}
-          installation: ${{ matrix.installation }}
-          python_version: ${{ matrix.python_version }}
+    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
+    with:
+      runner: ${{ matrix.validation_runner }}
+      repository: "pytorch/builder"
+      job-name: ${{ matrix.build_name }}
+      script: |
+        set -ex
+        export ENV_NAME="conda-env-${{ github.run_id }}"
+        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
+        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
+        export INSTALLATION="${{ matrix.installation }}"
+        export CUDA_VER="${{ matrix.desired_cuda }}"
 
-  win-libt:
-    needs: generate-windows-libtorch-matrix
-    strategy:
-      matrix:
-        ${{ fromJson(needs.generate-windows-libtorch-matrix.outputs.matrix) }}
-      fail-fast: false
-    runs-on: "windows-2019"
-    env:
-      PYTHON_VERSION: ${{ matrix.python_version }}
-    steps:
-      - name: Install pytorch and smoke test
-        shell: powershell
-        run: |
-          $install = '${{ matrix.installation }}'
-          Invoke-WebRequest -Uri $install -OutFile 'libtorch.zip'
-          Expand-Archive -Force libtorch.zip .
+        if [[ ${{ matrix.package_type }} == "libtorch" ]]; then
+          curl ${{ matrix.installation }} -o libtorch.zip
+          unzip libtorch.zip
+        else
+          conda create -y -n ${ENV_NAME} python=${{ matrix.python_version }} numpy pillow
+          conda activate ${ENV_NAME}
+          eval $INSTALLATION
+          python ./test/smoke_test/smoke_test.py
+        fi
diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py
@@ -4,6 +4,7 @@
 from pathlib import Path
 import argparse
 import torch
+import platform
 
 gpu_arch_ver = os.getenv("GPU_ARCH_VER")
 gpu_arch_type = os.getenv("GPU_ARCH_TYPE")
@@ -78,7 +79,9 @@ def smoke_test_cuda(package: str) -> None:
     if(package == 'all'):
         import torchaudio
         import torchvision
-        if installation_str.find("nightly") != -1:
+        # There is an issue with current windows runners calling conda from python
+        # https://github.com/pytorch/test-infra/issues/1054
+        if installation_str.find("nightly") != -1 or platform.system() == "Windows" :
             # just print out cuda version, as version check were already performed during import
             print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
             print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}")