From 05fab1a9c8464a76176158c45ce13ea9d6e547df Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Tue, 20 Oct 2020 18:05:02 +0000 Subject: [PATCH 1/7] Dynamically change the base PyTorch version in unittest --- .circleci/config.yml | 15 +++++++++++++++ .circleci/config.yml.in | 15 +++++++++++++++ .circleci/unittest/linux/scripts/install.sh | 4 ++-- .circleci/unittest/windows/scripts/install.sh | 4 ++-- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac229fa9433..b91e9704fe0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -295,6 +295,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -313,6 +314,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -333,6 +335,7 @@ jobs: image_name: torchvision/smoke_test steps: - checkout + - designate_upload_channel - run: name: Build and push Docker image no_output_timeout: "1h" @@ -352,6 +355,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -377,6 +381,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -400,6 +405,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -440,6 +446,7 @@ jobs: image_name: "pytorch/manylinux-cuda101" steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -477,6 +484,7 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -516,6 +524,7 @@ jobs: CUDA_VERSION: "10.1" steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -554,6 +563,7 @@ jobs: resource_class: large steps: - checkout + - designate_upload_channel - run: name: Install wget command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget @@ -596,6 +606,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout_merge + - designate_upload_channel - run: name: Setup conda command: .circleci/unittest/linux/scripts/setup_env.sh @@ -613,6 +624,7 @@ jobs: CU_VERSION: << parameters.cu_version >> steps: - checkout_merge + - designate_upload_channel - run: name: Setup conda command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh @@ -626,6 +638,7 @@ jobs: xcode: "9.0" steps: - checkout_merge + - designate_upload_channel - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh @@ -640,6 +653,7 @@ jobs: name: windows-cpu steps: - checkout_merge + - designate_upload_channel - run: command: | set -ex @@ -652,6 +666,7 @@ jobs: name: windows-gpu steps: - checkout_merge + - designate_upload_channel - run: command: | set -ex diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 3b9c2a22376..5d08b53c01b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -295,6 +295,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -313,6 +314,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -333,6 +335,7 @@ jobs: image_name: torchvision/smoke_test steps: - checkout + - designate_upload_channel - run: name: Build and push Docker image no_output_timeout: "1h" @@ -352,6 +355,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -377,6 +381,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -400,6 +405,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -440,6 +446,7 @@ jobs: image_name: "pytorch/manylinux-cuda101" steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -477,6 +484,7 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -516,6 +524,7 @@ jobs: CUDA_VERSION: "10.1" steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -554,6 +563,7 @@ jobs: resource_class: large steps: - checkout + - designate_upload_channel - run: name: Install wget command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget @@ -596,6 +606,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout_merge + - designate_upload_channel - run: name: Setup conda command: .circleci/unittest/linux/scripts/setup_env.sh @@ -613,6 +624,7 @@ jobs: CU_VERSION: << parameters.cu_version >> steps: - checkout_merge + - designate_upload_channel - run: name: Setup conda command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh @@ -626,6 +638,7 @@ jobs: xcode: "9.0" steps: - checkout_merge + - designate_upload_channel - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh @@ -640,6 +653,7 @@ jobs: name: windows-cpu steps: - checkout_merge + - designate_upload_channel - run: command: | set -ex @@ -652,6 +666,7 @@ jobs: name: windows-gpu steps: - checkout_merge + - designate_upload_channel - run: command: | set -ex diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 65273c75152..2de17f12744 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -23,7 +23,7 @@ else cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" printf "* Installing torchvision\n" -python setup.py develop \ No newline at end of file +python setup.py develop diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index b0f585a5483..bdf2a869fe1 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -25,7 +25,7 @@ else cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" printf "* Installing torchvision\n" -"$this_dir/vc_env_helper.bat" python setup.py develop \ No newline at end of file +"$this_dir/vc_env_helper.bat" python setup.py develop From b48ec4190a87ea92987578d0ab96a2be9db04692 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Tue, 20 Oct 2020 18:08:10 +0000 Subject: [PATCH 2/7] Fix build_cmake --- packaging/build_cmake.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build_cmake.sh b/packaging/build_cmake.sh index ff384457e5e..55b47c4c502 100755 --- a/packaging/build_cmake.sh +++ b/packaging/build_cmake.sh @@ -23,7 +23,7 @@ fi setup_visual_studio_constraint setup_junit_results_folder -conda install -yq pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE -c pytorch-nightly +conda install -yq pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE -c "pytorch-${UPLOAD_CHANNEL}" TORCH_PATH=$(dirname $(python -c "import torch; print(torch.__file__)")) if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then From c2f698f999901e64a4097f70de7a97bec390a91d Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Tue, 20 Oct 2020 18:21:31 +0000 Subject: [PATCH 3/7] Fix GPU test --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b91e9704fe0..25ee499087e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -627,10 +627,10 @@ jobs: - designate_upload_channel - run: name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - run: name: Build torchvision C++ distribution and test - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh cmake_macos_cpu: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 5d08b53c01b..5ffef2e72ec 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -627,10 +627,10 @@ jobs: - designate_upload_channel - run: name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - run: name: Build torchvision C++ distribution and test - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh cmake_macos_cpu: <<: *binary_common From 7e3e7180b8514ff576f35915fdc5c830b1a74d29 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Tue, 20 Oct 2020 18:38:56 +0000 Subject: [PATCH 4/7] Fix linux GPU test --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 25ee499087e..09ec219aa60 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -468,7 +468,7 @@ jobs: - env - run: name: Install torchvision - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 5ffef2e72ec..664bb96d04d 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -468,7 +468,7 @@ jobs: - env - run: name: Install torchvision - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh From 581bb217934cdcaa78f294a377036d136c44f413 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Tue, 20 Oct 2020 18:57:38 +0000 Subject: [PATCH 5/7] Update pkg_helpers.bash --- packaging/pkg_helpers.bash | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 09d3ddbbecc..79dc409e0f5 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -235,8 +235,7 @@ setup_pip_pytorch_version() { else pip_install "torch==$PYTORCH_VERSION$PYTORCH_VERSION_SUFFIX" \ -f "https://download.pytorch.org/whl/${CU_VERSION}/torch_stable.html" \ - -f "https://download.pytorch.org/whl/test/${CU_VERSION}/torch_test.html" \ - -f "https://download.pytorch.org/whl/nightly/${CU_VERSION}/torch_nightly.html" + -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html" fi } @@ -261,7 +260,7 @@ setup_conda_pytorch_constraint() { exit 1 fi else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly -c pytorch-test" + export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-${UPLOAD_CHANNEL}" fi if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" From b6e2a98c1e0f11cd64977b4f6b66f00da8fdf890 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Tue, 20 Oct 2020 19:24:44 +0000 Subject: [PATCH 6/7] Fix binary builds --- .circleci/config.yml | 6 ++++++ .circleci/config.yml.in | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09ec219aa60..af7d890fd74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,6 +134,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout_merge + - designate_upload_channel - run: packaging/build_wheel.sh - store_artifacts: path: dist @@ -149,6 +150,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout_merge + - designate_upload_channel - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 @@ -164,6 +166,7 @@ jobs: executor: windows-cpu steps: - checkout_merge + - designate_upload_channel - run: name: Build conda packages command: | @@ -189,6 +192,7 @@ jobs: executor: windows-cpu steps: - checkout_merge + - designate_upload_channel - run: name: Build wheel packages command: | @@ -211,6 +215,7 @@ jobs: xcode: "9.4.1" steps: - checkout_merge + - designate_upload_channel - run: # Cannot easily deduplicate this as source'ing activate # will set environment variables which we need to propagate @@ -233,6 +238,7 @@ jobs: xcode: "9.4.1" steps: - checkout_merge + - designate_upload_channel - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 664bb96d04d..6a6177aef37 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -134,6 +134,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout_merge + - designate_upload_channel - run: packaging/build_wheel.sh - store_artifacts: path: dist @@ -149,6 +150,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout_merge + - designate_upload_channel - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 @@ -164,6 +166,7 @@ jobs: executor: windows-cpu steps: - checkout_merge + - designate_upload_channel - run: name: Build conda packages command: | @@ -189,6 +192,7 @@ jobs: executor: windows-cpu steps: - checkout_merge + - designate_upload_channel - run: name: Build wheel packages command: | @@ -211,6 +215,7 @@ jobs: xcode: "9.4.1" steps: - checkout_merge + - designate_upload_channel - run: # Cannot easily deduplicate this as source'ing activate # will set environment variables which we need to propagate @@ -233,6 +238,7 @@ jobs: xcode: "9.4.1" steps: - checkout_merge + - designate_upload_channel - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh From 6e92d9d8f7b34baa43427a2a25e36c47c6356466 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Tue, 20 Oct 2020 19:26:44 +0000 Subject: [PATCH 7/7] Tweak --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af7d890fd74..d378b3a9922 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -633,7 +633,7 @@ jobs: - designate_upload_channel - run: name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - run: name: Build torchvision C++ distribution and test command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 6a6177aef37..e6418ed7607 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -633,7 +633,7 @@ jobs: - designate_upload_channel - run: name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - run: name: Build torchvision C++ distribution and test command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh