From ecbc63277d7e1125793256c0bf8b5da05e7c37b5 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 29 Oct 2021 00:30:58 -0700 Subject: [PATCH 01/10] improve download nightly pytorch --- .circleci/unittest/windows/scripts/install.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index e4756179371..5275546adbb 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -12,8 +12,10 @@ this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" conda activate ./env +# TODO, refactor the below logic to make it easy to understand how to get correct cuda_version. if [ "${CU_VERSION:-}" == cpu ] ; then cudatoolkit="cpuonly" + wheeldir="cpu" else if [[ ${#CU_VERSION} -eq 4 ]]; then CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" @@ -23,11 +25,11 @@ else echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" cudatoolkit="cudatoolkit=${version}" + wheeldir="cu"${version/./} fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -# conda-forge channel is required for cudatoolkit 11.1 on Windows, see https://github.com/pytorch/vision/issues/4458 -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest +pip install torch --pre -f https://download.pytorch.org/whl/nightly/${wheeldir}/torch_nightly.html if [ $PYTHON_VERSION == "3.6" ]; then printf "Installing minimal PILLOW version\n" From d72561e06f69051349a0781159e1ecc15063b7c0 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 29 Oct 2021 02:28:32 -0700 Subject: [PATCH 02/10] update unitest cuda to 11.3 --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a17410b9a6f..7a67737ae06 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -761,7 +761,7 @@ jobs: executor: name: windows-gpu environment: - CUDA_VERSION: "11.1" + CUDA_VERSION: "11.3" PYTHON_VERSION: << parameters.python_version >> steps: - checkout diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 4b0bc8d7e1d..ce6a9bd8f8b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -761,7 +761,7 @@ jobs: executor: name: windows-gpu environment: - CUDA_VERSION: "11.1" + CUDA_VERSION: "11.3" PYTHON_VERSION: << parameters.python_version >> steps: - checkout From bb4a9315b73d82d63830241bdd3eba168be7e57c Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 29 Oct 2021 02:48:43 -0700 Subject: [PATCH 03/10] add nvjpeg in cuda 11.3 --- packaging/windows/internal/cuda_install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat index db181c62321..0b42a2cbba3 100644 --- a/packaging/windows/internal/cuda_install.bat +++ b/packaging/windows/internal/cuda_install.bat @@ -167,7 +167,7 @@ if not exist "%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" ( curl -k -L "https://ossci-windows.s3.amazonaws.com/%CUDA_INSTALL_EXE%" --output "%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" if errorlevel 1 exit /b 1 set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" - set "ARGS=thrust_11.3 nvcc_11.3 cuobjdump_11.3 nvprune_11.3 nvprof_11.3 cupti_11.3 cublas_11.3 cublas_dev_11.3 cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3" + set "ARGS=thrust_11.3 nvcc_11.3 cuobjdump_11.3 nvprune_11.3 nvprof_11.3 cupti_11.3 cublas_11.3 cublas_dev_11.3 cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvjpeg_11.3 nvjpeg_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3" ) From 297489c9df7b40c7ae832f5e00899329df5654f2 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 29 Oct 2021 03:15:28 -0700 Subject: [PATCH 04/10] avoid catching exception --- torchvision/io/image.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/torchvision/io/image.py b/torchvision/io/image.py index 6dba6a7b168..86291b3b0ac 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -5,12 +5,14 @@ from .._internally_replaced_utils import _get_extension_path -try: - lib_path = _get_extension_path("image") - torch.ops.load_library(lib_path) -except (ImportError, OSError): - pass - +#try: +# lib_path = _get_extension_path("image") +# torch.ops.load_library(lib_path) +#except (ImportError, OSError): +# pass + +lib_path = _get_extension_path('image') +torch.ops.load_library(lib_path) class ImageReadMode(Enum): """ From 9c734f271a70140da19493ce5533d082b48f7abb Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 29 Oct 2021 04:35:24 -0700 Subject: [PATCH 05/10] restore to cu111 --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a67737ae06..a17410b9a6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -761,7 +761,7 @@ jobs: executor: name: windows-gpu environment: - CUDA_VERSION: "11.3" + CUDA_VERSION: "11.1" PYTHON_VERSION: << parameters.python_version >> steps: - checkout diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index ce6a9bd8f8b..4b0bc8d7e1d 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -761,7 +761,7 @@ jobs: executor: name: windows-gpu environment: - CUDA_VERSION: "11.3" + CUDA_VERSION: "11.1" PYTHON_VERSION: << parameters.python_version >> steps: - checkout From 7cba686e0e724fde8482eb6f06bee7eb4b38207f Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 29 Oct 2021 20:10:47 +0800 Subject: [PATCH 06/10] Update install.sh --- .circleci/unittest/windows/scripts/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index 5275546adbb..b4e7defb062 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -29,6 +29,7 @@ else fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" +pip install pytest pip install torch --pre -f https://download.pytorch.org/whl/nightly/${wheeldir}/torch_nightly.html if [ $PYTHON_VERSION == "3.6" ]; then From 40af3a2294de6fdef41d26630b00197c33c3a936 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 1 Nov 2021 04:10:45 -0700 Subject: [PATCH 07/10] use conda MatchSpec string --- .circleci/unittest/windows/scripts/install.sh | 6 ++---- packaging/windows/internal/cuda_install.bat | 2 +- torchvision/io/image.py | 14 ++++++-------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index b4e7defb062..88f716ba599 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -15,7 +15,7 @@ conda activate ./env # TODO, refactor the below logic to make it easy to understand how to get correct cuda_version. if [ "${CU_VERSION:-}" == cpu ] ; then cudatoolkit="cpuonly" - wheeldir="cpu" + version="cpu" else if [[ ${#CU_VERSION} -eq 4 ]]; then CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" @@ -25,12 +25,10 @@ else echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" cudatoolkit="cudatoolkit=${version}" - wheeldir="cu"${version/./} fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -pip install pytest -pip install torch --pre -f https://download.pytorch.org/whl/nightly/${wheeldir}/torch_nightly.html +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" pytest if [ $PYTHON_VERSION == "3.6" ]; then printf "Installing minimal PILLOW version\n" diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat index 0b42a2cbba3..db181c62321 100644 --- a/packaging/windows/internal/cuda_install.bat +++ b/packaging/windows/internal/cuda_install.bat @@ -167,7 +167,7 @@ if not exist "%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" ( curl -k -L "https://ossci-windows.s3.amazonaws.com/%CUDA_INSTALL_EXE%" --output "%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" if errorlevel 1 exit /b 1 set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" - set "ARGS=thrust_11.3 nvcc_11.3 cuobjdump_11.3 nvprune_11.3 nvprof_11.3 cupti_11.3 cublas_11.3 cublas_dev_11.3 cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvjpeg_11.3 nvjpeg_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3" + set "ARGS=thrust_11.3 nvcc_11.3 cuobjdump_11.3 nvprune_11.3 nvprof_11.3 cupti_11.3 cublas_11.3 cublas_dev_11.3 cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3" ) diff --git a/torchvision/io/image.py b/torchvision/io/image.py index 86291b3b0ac..6dba6a7b168 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -5,14 +5,12 @@ from .._internally_replaced_utils import _get_extension_path -#try: -# lib_path = _get_extension_path("image") -# torch.ops.load_library(lib_path) -#except (ImportError, OSError): -# pass - -lib_path = _get_extension_path('image') -torch.ops.load_library(lib_path) +try: + lib_path = _get_extension_path("image") + torch.ops.load_library(lib_path) +except (ImportError, OSError): + pass + class ImageReadMode(Enum): """ From 26f8c7d592f77025119e432867d7ff34e698292b Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 1 Nov 2021 06:58:45 -0700 Subject: [PATCH 08/10] update linux --- .circleci/unittest/linux/scripts/install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index e9b9e80c077..c7d3ccfdf3f 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -12,6 +12,7 @@ conda activate ./env if [ "${CU_VERSION:-}" == cpu ] ; then cudatoolkit="cpuonly" + version="cpu" else if [[ ${#CU_VERSION} -eq 4 ]]; then CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" @@ -24,7 +25,7 @@ else fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" pytest if [ $PYTHON_VERSION == "3.6" ]; then printf "Installing minimal PILLOW version\n" From 9dc34dd80919a79b09b711c2bdabab0b1eddc439 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 1 Nov 2021 07:19:26 -0700 Subject: [PATCH 09/10] for linux --- .circleci/unittest/linux/scripts/install.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index c7d3ccfdf3f..1ba79756601 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -24,8 +24,17 @@ else cudatoolkit="cudatoolkit=${version}" fi +case "$(uname -s)" in + Darwin*) os=MacOSX;; + *) os=Linux +esac + printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" pytest +if [ "${os}" == "MacOSX" ]; then + conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest +else + conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" pytest +fi if [ $PYTHON_VERSION == "3.6" ]; then printf "Installing minimal PILLOW version\n" From 99b0be429b48df4a0069816668a51e406e592942 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 1 Nov 2021 22:25:58 +0800 Subject: [PATCH 10/10] Update install.sh --- .circleci/unittest/linux/scripts/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 1ba79756601..5493176f902 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -31,9 +31,9 @@ esac printf "Installing PyTorch with %s\n" "${cudatoolkit}" if [ "${os}" == "MacOSX" ]; then - conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest + conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest else - conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" pytest + conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" pytest fi if [ $PYTHON_VERSION == "3.6" ]; then