Add test jobs on CPU agents for CUDA builds on Windows (#37904)

Summary: Targets #37811 (comment). Pull Request resolved: #37904 Differential Revision: D21484360 Pulled By: seemethere fbshipit-source-id: b25cbf35b8432a587bce86815c97ff444cab255c
pytorch · May 8, 2020 · 55de7c3 · 55de7c3
1 parent e84aa02
commit 55de7c3
Show file tree

Hide file tree

Showing 7 changed files with 154 additions and 1 deletion.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -284,6 +284,7 @@ pytorch_windows_params: &pytorch_windows_params
     USE_CUDA: <<parameters.use_cuda>>
     TORCH_CUDA_ARCH_LIST: "7.5"
     JOB_BASE_NAME: <<parameters.test_name>>
+    JOB_EXECUTOR: <<parameters.executor>>
 caffe2_params: &caffe2_params
   parameters:
     build_environment:
@@ -672,6 +673,45 @@ jobs:
             if [[ "${VC_YEAR}" == "2017" ]]; then
               powershell .circleci/scripts/vs_install.ps1
             fi
+      - run:
+          name: Install Cuda
+          no_output_timeout: 30m
+          command: |
+            if [[ "${CUDA_VERSION}" != "cpu" && "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
+              curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
+              7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
+              cd cuda_10.1.243_426.00_win10
+              mkdir cuda_install_logs
+              ./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
+              if [[ "${VC_YEAR}" == "2017" ]]; then
+                cp -r CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions/* "C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/Common7/IDE/VC/VCTargets/BuildCustomizations/"
+              else
+                cp -r CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions/* "C:/Program Files (x86)/Microsoft Visual Studio/2019/Community/MSBuild/Microsoft/VC/v160/BuildCustomizations/"
+              fi
+              curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
+              7z x NvToolsExt.7z -oNvToolsExt
+              mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
+              cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
+              export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
+              cat cuda_install_logs/LOG.setup.exe.log
+              if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
+              then
+                echo "CUDA installation failed"
+                exit 1
+              fi
+              cd ..
+              rm -rf ./cuda_10.1.243_426.00_win10
+              rm -f ./cuda_10.1.243_426.00_win10.exe
+            fi
+      - run:
+          name: Install Cudnn
+          command : |
+            if [[ "${CUDA_VERSION}" != "cpu" && "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
+              cd c:/
+              curl --retry 3 -O https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip
+              7z x cudnn-10.1-windows10-x64-v7.6.4.38.zip -ocudnn
+              cp -r cudnn/cuda/* "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/"
+            fi
       - run:
           name: Test
           no_output_timeout: "30m"
@@ -2323,6 +2363,32 @@ workflows:
           requires:
             - setup
             - pytorch_windows_vs2019_py36_cpu_build
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test1
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          test_name: pytorch-windows-test1
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: ""
+          vc_year: "2019"
+          vc_product: "Community"
+          use_cuda: "0"
+          requires:
+            - setup
+            - pytorch_windows_vs2019_py36_cuda10.1_build
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test2
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          test_name: pytorch-windows-test2
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: ""
+          vc_year: "2019"
+          vc_product: "Community"
+          use_cuda: "0"
+          requires:
+            - setup
+            - pytorch_windows_vs2019_py36_cuda10.1_build
       - pytorch_linux_build:
           name: pytorch_linux_xenial_pynightly_build
           requires:

diff --git a/.circleci/verbatim-sources/pytorch-build-params.yml b/.circleci/verbatim-sources/pytorch-build-params.yml
@@ -82,3 +82,4 @@ pytorch_windows_params: &pytorch_windows_params
     USE_CUDA: <<parameters.use_cuda>>
     TORCH_CUDA_ARCH_LIST: "7.5"
     JOB_BASE_NAME: <<parameters.test_name>>
+    JOB_EXECUTOR: <<parameters.executor>>
diff --git a/.circleci/verbatim-sources/pytorch-job-specs.yml b/.circleci/verbatim-sources/pytorch-job-specs.yml
@@ -278,6 +278,45 @@ jobs:
             if [[ "${VC_YEAR}" == "2017" ]]; then
               powershell .circleci/scripts/vs_install.ps1
             fi
+      - run:
+          name: Install Cuda
+          no_output_timeout: 30m
+          command: |
+            if [[ "${CUDA_VERSION}" != "cpu" && "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
+              curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
+              7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
+              cd cuda_10.1.243_426.00_win10
+              mkdir cuda_install_logs
+              ./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
+              if [[ "${VC_YEAR}" == "2017" ]]; then
+                cp -r CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions/* "C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/Common7/IDE/VC/VCTargets/BuildCustomizations/"
+              else
+                cp -r CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions/* "C:/Program Files (x86)/Microsoft Visual Studio/2019/Community/MSBuild/Microsoft/VC/v160/BuildCustomizations/"
+              fi
+              curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
+              7z x NvToolsExt.7z -oNvToolsExt
+              mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
+              cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
+              export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
+              cat cuda_install_logs/LOG.setup.exe.log
+              if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
+              then
+                echo "CUDA installation failed"
+                exit 1
+              fi
+              cd ..
+              rm -rf ./cuda_10.1.243_426.00_win10
+              rm -f ./cuda_10.1.243_426.00_win10.exe
+            fi
+      - run:
+          name: Install Cudnn
+          command : |
+            if [[ "${CUDA_VERSION}" != "cpu" && "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
+              cd c:/
+              curl --retry 3 -O https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip
+              7z x cudnn-10.1-windows10-x64-v7.6.4.38.zip -ocudnn
+              cp -r cudnn/cuda/* "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/"
+            fi
       - run:
           name: Test
           no_output_timeout: "30m"

diff --git a/.circleci/verbatim-sources/windows-build-test.yml b/.circleci/verbatim-sources/windows-build-test.yml
@@ -300,3 +300,29 @@
           requires:
             - setup
             - pytorch_windows_vs2019_py36_cpu_build
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test1
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          test_name: pytorch-windows-test1
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: ""
+          vc_year: "2019"
+          vc_product: "Community"
+          use_cuda: "0"
+          requires:
+            - setup
+            - pytorch_windows_vs2019_py36_cuda10.1_build
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test2
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          test_name: pytorch-windows-test2
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: ""
+          vc_year: "2019"
+          vc_product: "Community"
+          use_cuda: "0"
+          requires:
+            - setup
+            - pytorch_windows_vs2019_py36_cuda10.1_build
diff --git a/.jenkins/pytorch/win-test-helpers/run_python_nn_smoketests.py b/.jenkins/pytorch/win-test-helpers/run_python_nn_smoketests.py
@@ -6,6 +6,10 @@
 import os
 
 COMMON_TESTS = [
+    (
+        "Checking that torch is available",
+        "import torch",
+    ),
     (
         "Checking that caffe2.python is available",
         "from caffe2.python import core",
@@ -43,4 +47,15 @@
         command_args = ["python", "-c", python_commands]
         command_string = " ".join(command_args)
         print("Command:", command_string)
-        subprocess.check_call(command_args)
+        try:
+            subprocess.check_call(command_args)
+        except subprocess.CalledProcessError as e:
+            sdk_root = os.environ.get('WindowsSdkDir', 'C:\\Program Files (x86)\\Windows Kits\\10')
+            debugger = os.path.join(sdk_root, 'Debuggers', 'x64', 'cdb.exe')
+            if os.path.exists(debugger):
+                command_args = [debugger, "-o", "-c", "~*g; q"] + command_args
+                command_string = " ".join(command_args)
+                print("Reruning with traceback enabled")
+                print("Command:", command_string)
+                subprocess.run(command_args, check=False)
+            exit(e.returncode)
diff --git a/.jenkins/pytorch/win-test-helpers/test_libtorch.bat b/.jenkins/pytorch/win-test-helpers/test_libtorch.bat
@@ -1,3 +1,7 @@
+:: Skip LibTorch tests when building a GPU binary and testing on a CPU machine
+:: because LibTorch tests are not well designed for this use case.
+if "%USE_CUDA%" == "0" IF NOT "%CUDA_VERSION%" == "cpu" exit /b 0
+
 call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
 if errorlevel 1 exit /b 1
 

diff --git a/.jenkins/pytorch/win-test-helpers/test_python_nn.bat b/.jenkins/pytorch/win-test-helpers/test_python_nn.bat
@@ -4,7 +4,9 @@ call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
 pushd test
 
 echo Some smoke tests
+"C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\gflags.exe" /i python.exe +sls
 python %SCRIPT_HELPERS_DIR%\run_python_nn_smoketests.py
+"C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\gflags.exe" /i python.exe -sls
 if ERRORLEVEL 1 exit /b 1
 
 echo Run nn tests