From 4811dc7727702e72c2646d7f80faac5d21ec6af7 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 13:33:17 -0700 Subject: [PATCH 01/18] Add uploading support Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 47 +++++++++++++++++++++++++++++++++++++---- .circleci/config.yml.in | 47 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 222195c24c..95f061ffe5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,12 +2,13 @@ version: 2.1 # How to test the Linux jobs: # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel -# - Replace binary_linux_wheel with the name of the job you want to test +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 +# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. +# Job names are 'name:' key. binary_common: &binary_common parameters: - # Edit these defaults to do a release` + # Edit these defaults to do a release build_version: description: "version number of release binary; by default, build a nightly" type: string @@ -53,6 +54,10 @@ jobs: - run: packaging/build_wheel.sh - store_artifacts: path: dist + - persist_to_workspace: + root: dist + paths: + - "*" binary_linux_conda: <<: *binary_common @@ -65,6 +70,23 @@ jobs: - store_artifacts: path: /opt/conda/conda-bld/linux-64 + # Requires org-member context + binary_conda_upload: + # NB: We don't really need cuda, just a working anaconda-client + docker: + - image: "soumith/conda-cuda" + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + set -x + anaconda login \ + --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ + --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" + set +x + anaconda upload ~/workspace/* -u pytorch-nightly --label main --no-progress --force + binary_macos_wheel: <<: *binary_common macos: @@ -158,4 +180,21 @@ workflows: python_version: "3.6" - binary_macos_conda: name: binary_macos_conda_py3.7 - python_version: "3.7" \ No newline at end of file + python_version: "3.7" + + master: + triggers: + - schedule: + cron: "0 9 * * *" + filters: + branches: + only: + - master + jobs: + - binary_linux_conda: + name: master_binary_linux_conda_py3.7 + python_version: "3.7" + - binary_conda_upload: + context: org-member + requires: + - master_binary_linux_conda_py3.7 \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 47bf1ce630..386e412171 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -2,12 +2,13 @@ version: 2.1 # How to test the Linux jobs: # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel -# - Replace binary_linux_wheel with the name of the job you want to test +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 +# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. +# Job names are 'name:' key. binary_common: &binary_common parameters: - # Edit these defaults to do a release` + # Edit these defaults to do a release build_version: description: "version number of release binary; by default, build a nightly" type: string @@ -53,6 +54,10 @@ jobs: - run: packaging/build_wheel.sh - store_artifacts: path: dist + - persist_to_workspace: + root: dist + paths: + - "*" binary_linux_conda: <<: *binary_common @@ -65,6 +70,23 @@ jobs: - store_artifacts: path: /opt/conda/conda-bld/linux-64 + # Requires org-member context + binary_conda_upload: + # NB: We don't really need cuda, just a working anaconda-client + docker: + - image: "soumith/conda-cuda" + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + set -x + anaconda login \ + --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ + --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" + set +x + anaconda upload ~/workspace/* -u pytorch-nightly --label main --no-progress --force + binary_macos_wheel: <<: *binary_common macos: @@ -117,4 +139,21 @@ workflows: {%- endif -%} {%- endfor -%} {%- endfor -%} -{%- endfor -%} +{%- endfor %} + + master: + triggers: + - schedule: + cron: "0 9 * * *" + filters: + branches: + only: + - master + jobs: + - binary_linux_conda: + name: master_binary_linux_conda_py3.7 + python_version: "3.7" + - binary_conda_upload: + context: org-member + requires: + - master_binary_linux_conda_py3.7 From 51cc469ea2a9b9c18f633a19b3243d90c7d2306e Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 13:35:19 -0700 Subject: [PATCH 02/18] Run master workflow Signed-off-by: Edward Z. Yang --- .circleci/config.yml.in | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 386e412171..94be8b47d5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -142,13 +142,13 @@ workflows: {%- endfor %} master: - triggers: - - schedule: - cron: "0 9 * * *" - filters: - branches: - only: - - master +# triggers: +# - schedule: +# cron: "0 9 * * *" +# filters: +# branches: +# only: +# - master jobs: - binary_linux_conda: name: master_binary_linux_conda_py3.7 From aad29f9a63afa61364277e0d2c855b97f7d46187 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 13:36:08 -0700 Subject: [PATCH 03/18] regen Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 95f061ffe5..7eee212a94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -183,13 +183,13 @@ workflows: python_version: "3.7" master: - triggers: - - schedule: - cron: "0 9 * * *" - filters: - branches: - only: - - master +# triggers: +# - schedule: +# cron: "0 9 * * *" +# filters: +# branches: +# only: +# - master jobs: - binary_linux_conda: name: master_binary_linux_conda_py3.7 From c47870b56804856320879ed2165cd722bacd6a4d Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 13:55:59 -0700 Subject: [PATCH 04/18] Bugfix Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 8 ++++++-- .circleci/config.yml.in | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7eee212a94..dd39bd01c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,10 @@ jobs: - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" # Requires org-member context binary_conda_upload: @@ -80,11 +84,11 @@ jobs: at: ~/workspace - run: command: | - set -x + set +x anaconda login \ --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" - set +x + set -x anaconda upload ~/workspace/* -u pytorch-nightly --label main --no-progress --force binary_macos_wheel: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 94be8b47d5..3e4559a2e5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -69,6 +69,10 @@ jobs: - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" # Requires org-member context binary_conda_upload: @@ -80,11 +84,11 @@ jobs: at: ~/workspace - run: command: | - set -x + set +x anaconda login \ --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" - set +x + set -x anaconda upload ~/workspace/* -u pytorch-nightly --label main --no-progress --force binary_macos_wheel: From b0a9817e2be78f0064bf3b9f0ed3c71be96ff62c Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 14:41:56 -0700 Subject: [PATCH 05/18] More bits Signed-off-by: Edward Z. Yang --- .circleci/config.yml.in | 71 +++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 3e4559a2e5..1501d43763 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -74,23 +74,6 @@ jobs: paths: - "*" - # Requires org-member context - binary_conda_upload: - # NB: We don't really need cuda, just a working anaconda-client - docker: - - image: "soumith/conda-cuda" - steps: - - attach_workspace: - at: ~/workspace - - run: - command: | - set +x - anaconda login \ - --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ - --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" - set -x - anaconda upload ~/workspace/* -u pytorch-nightly --label main --no-progress --force - binary_macos_wheel: <<: *binary_common macos: @@ -108,6 +91,10 @@ jobs: packaging/build_wheel.sh - store_artifacts: path: dist + - persist_to_workspace: + root: dist + paths: + - "*" binary_macos_conda: <<: *binary_common @@ -124,6 +111,48 @@ jobs: packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" + + # Requires org-member context + binary_conda_upload: + # NB: We don't really need cuda, just a working anaconda-client + docker: + - image: "soumith/conda-cuda" + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + # Prevent credential from leaking + set +x + anaconda login \ + --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ + --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" + set -x + anaconda upload ~/workspace/*.tar.gz -u pytorch-nightly --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + docker: + - image: circleci/python:3.7 + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + pip install awscli + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/libtorch/nightly/" --acl public-read + done + workflows: build: @@ -161,3 +190,11 @@ workflows: context: org-member requires: - master_binary_linux_conda_py3.7 + + - binary_linux_wheel: + name: master_binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + context: org-member + requires: + - master_binary_linux_wheel_py3.7 From aa7016855e5bcfef9966474faad1b0548683195c Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 14:44:07 -0700 Subject: [PATCH 06/18] fixup Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 73 +++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd39bd01c0..dbe6abd758 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,23 +74,6 @@ jobs: paths: - "*" - # Requires org-member context - binary_conda_upload: - # NB: We don't really need cuda, just a working anaconda-client - docker: - - image: "soumith/conda-cuda" - steps: - - attach_workspace: - at: ~/workspace - - run: - command: | - set +x - anaconda login \ - --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ - --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" - set -x - anaconda upload ~/workspace/* -u pytorch-nightly --label main --no-progress --force - binary_macos_wheel: <<: *binary_common macos: @@ -108,6 +91,10 @@ jobs: packaging/build_wheel.sh - store_artifacts: path: dist + - persist_to_workspace: + root: dist + paths: + - "*" binary_macos_conda: <<: *binary_common @@ -124,6 +111,48 @@ jobs: packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" + + # Requires org-member context + binary_conda_upload: + # NB: We don't really need cuda, just a working anaconda-client + docker: + - image: "soumith/conda-cuda" + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + # Prevent credential from leaking + set +x + anaconda login \ + --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ + --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" + set -x + anaconda upload ~/workspace/*.tar.gz -u pytorch-nightly --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + docker: + - image: circleci/python:3.7 + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + pip install awscli + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/libtorch/nightly/" --acl public-read + done + workflows: build: @@ -201,4 +230,12 @@ workflows: - binary_conda_upload: context: org-member requires: - - master_binary_linux_conda_py3.7 \ No newline at end of file + - master_binary_linux_conda_py3.7 + + - binary_linux_wheel: + name: master_binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + context: org-member + requires: + - master_binary_linux_wheel_py3.7 \ No newline at end of file From f075829071ccd1119da2c7955de89b7fd2726433 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 14:49:06 -0700 Subject: [PATCH 07/18] Take updates from vision Signed-off-by: Edward Z. Yang --- packaging/pkg_helpers.bash | 51 +++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 0265d5bd60..6447bcc585 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -30,8 +30,8 @@ setup_python() { fi } -# Fill CUDA_SUFFIX with CUDA_VERSION, but don't fill it for the default -# CUDA version (that's a blank suffix) +# Fill CUDA_SUFFIX and CU_VERSION with CUDA_VERSION. CUDA_SUFFIX is +# left blank for the default CUDA version (that's a blank suffix) setup_cuda_suffix() { if [[ "$(uname)" == Darwin ]]; then if [[ "$CUDA_VERSION" != "cpu" ]]; then @@ -39,14 +39,21 @@ setup_cuda_suffix() { exit 1 fi export CPU_SUFFIX="" + export CU_VERSION="cpu" else case "$CUDA_VERSION" in 10.0) - export CUDA_SUFFIX="" ;; # default! + export CUDA_SUFFIX="" + export CU_VERSION="cu100" + ;; 9.2) - export CUDA_SUFFIX="+cu92" ;; + export CUDA_SUFFIX="+cu92" + export CU_VERSION="cu92" + ;; cpu) - export CUDA_SUFFIX="+cpu" ;; + export CUDA_SUFFIX="+cpu" + export CU_VERSION="cpu" + ;; *) echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" esac @@ -89,14 +96,19 @@ pip_install() { # version into PYTORCH_VERSION, if applicable setup_pip_pytorch_version() { if [[ -z "$PYTORCH_VERSION" ]]; then - # Install latest prerelease CPU version of torch, per our nightlies. - pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we - # strip off the local package qualifier. - export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" + # Install latest prerelease version of torch, per our nightlies, consistent + # with the requested cuda version + pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${CU_VERSION}/torch_nightly.html" + if [[ "$CUDA_VERSION" == "cpu" ]]; then + # CUDA and CPU are ABI compatible on the CPU-only parts, so strip + # in this case + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" + else + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')" + fi else # TODO: Maybe add staging too - pip_install "torch==$PYTORCH_VERSION" \ + pip_install "torch==$PYTORCH_VERSION$CUDA_SUFFIX" \ -f https://download.pytorch.org/whl/torch_stable.html fi } @@ -125,10 +137,21 @@ setup_conda_pytorch_constraint() { # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT setup_conda_cudatoolkit_constraint() { - if [[ "$CUDA_VERSION" == cpu ]]; then + export CONDA_CPUONLY_FEATURE="" + if [[ "$(uname)" == Darwin ]]; then export CONDA_CUDATOOLKIT_CONSTRAINT="" else - echo - # TODO + case "$CUDA_VERSION" in + 10.0) + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]" + ;; + 9.2) + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]" + ;; + cpu) + export CONDA_CUDATOOLKIT_CONSTRAINT="" + export CONDA_CPUONLY_FEATURE="- cpuonly" + ;; + esac fi } From 5f0ea3ae388e1b128c3f2acbd41208840aabca15 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 14:54:22 -0700 Subject: [PATCH 08/18] Some development Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- packaging/build_conda.sh | 2 +- packaging/build_wheel.sh | 2 +- packaging/pkg_helpers.bash | 6 +++++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbe6abd758..c6570a7090 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,7 +143,7 @@ jobs: at: ~/workspace - run: command: | - pip install awscli + pip install --user awscli # Prevent credential from leaking set +x export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 1501d43763..069418b620 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -143,7 +143,7 @@ jobs: at: ~/workspace - run: command: | - pip install awscli + pip install --user awscli # Prevent credential from leaking set +x export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index b785eeab62..1941fa1100 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" setup_build_version 0.4.0 -setup_cuda_suffix +setup_cpuonly_cuda_suffix setup_macos export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index ef3a0ba885..beeb45695d 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" setup_python -setup_cuda_suffix +setup_cpuonly_cuda_suffix setup_build_version 0.4.0 setup_macos "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 6447bcc585..b4c9cdcee7 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -40,6 +40,7 @@ setup_cuda_suffix() { fi export CPU_SUFFIX="" export CU_VERSION="cpu" + export WHEEL_DIR="" else case "$CUDA_VERSION" in 10.0) @@ -58,6 +59,7 @@ setup_cuda_suffix() { echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" esac export CPU_SUFFIX="+cpu" + export WHEEL_DIR="$CU_VERSION/" fi } @@ -65,6 +67,8 @@ setup_cuda_suffix() { setup_cpuonly_cuda_suffix() { export CUDA_SUFFIX="" export CPU_SUFFIX="" + export CU_VERSION="cpu" + export WHEEL_DIR="" } # Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string @@ -98,7 +102,7 @@ setup_pip_pytorch_version() { if [[ -z "$PYTORCH_VERSION" ]]; then # Install latest prerelease version of torch, per our nightlies, consistent # with the requested cuda version - pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${CU_VERSION}/torch_nightly.html" + pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${WHEEL_DIR}torch_nightly.html" if [[ "$CUDA_VERSION" == "cpu" ]]; then # CUDA and CPU are ABI compatible on the CPU-only parts, so strip # in this case From bbb532b766da3f4b1333b90700164e410f81fae8 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 14:56:58 -0700 Subject: [PATCH 09/18] bz2 Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 8 +++++--- .circleci/config.yml.in | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c6570a7090..36483b8621 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,21 +118,21 @@ jobs: # Requires org-member context binary_conda_upload: - # NB: We don't really need cuda, just a working anaconda-client docker: - - image: "soumith/conda-cuda" + - image: continuumio/miniconda steps: - attach_workspace: at: ~/workspace - run: command: | # Prevent credential from leaking + conda install -yq anaconda-client set +x anaconda login \ --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" set -x - anaconda upload ~/workspace/*.tar.gz -u pytorch-nightly --label main --no-progress --force + anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force # Requires org-member context binary_wheel_upload: @@ -141,6 +141,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - checkout - run: command: | pip install --user awscli @@ -149,6 +150,7 @@ jobs: export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" set -x + packaging/upload_wheel.sh for pkg in ~/workspace/*.whl; do aws s3 cp "$pkg" "s3://pytorch/libtorch/nightly/" --acl public-read done diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 069418b620..561d87c21b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -118,21 +118,21 @@ jobs: # Requires org-member context binary_conda_upload: - # NB: We don't really need cuda, just a working anaconda-client docker: - - image: "soumith/conda-cuda" + - image: continuumio/miniconda steps: - attach_workspace: at: ~/workspace - run: command: | # Prevent credential from leaking + conda install -yq anaconda-client set +x anaconda login \ --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" set -x - anaconda upload ~/workspace/*.tar.gz -u pytorch-nightly --label main --no-progress --force + anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force # Requires org-member context binary_wheel_upload: @@ -141,6 +141,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - checkout - run: command: | pip install --user awscli @@ -149,6 +150,7 @@ jobs: export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" set -x + packaging/upload_wheel.sh for pkg in ~/workspace/*.whl; do aws s3 cp "$pkg" "s3://pytorch/libtorch/nightly/" --acl public-read done From 5e4d729e9a36eaa3f0eee259aa8c141eb5d33fe3 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 15:18:22 -0700 Subject: [PATCH 10/18] fixup Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 3 +-- .circleci/config.yml.in | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 36483b8621..d4397a9909 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -150,9 +150,8 @@ jobs: export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" set -x - packaging/upload_wheel.sh for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/libtorch/nightly/" --acl public-read + aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read done diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 561d87c21b..453bde28c3 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -150,9 +150,8 @@ jobs: export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" set -x - packaging/upload_wheel.sh for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/libtorch/nightly/" --acl public-read + aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read done From 2384e8b0531a4f910ee57aab2fa038ce34d5d848 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 15:34:11 -0700 Subject: [PATCH 11/18] Turn on config for everything Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 151 +++++++++++++++++++++++++++++++++++++--- .circleci/config.yml.in | 61 ++++++++-------- 2 files changed, 174 insertions(+), 38 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d4397a9909..1bae2950a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -154,11 +154,11 @@ jobs: aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read done - workflows: build: jobs: - circleci_consistency + - binary_linux_wheel: name: binary_linux_wheel_py2.7 python_version: "2.7" @@ -216,7 +216,7 @@ workflows: name: binary_macos_conda_py3.7 python_version: "3.7" - master: + nightly: # triggers: # - schedule: # cron: "0 9 * * *" @@ -225,18 +225,151 @@ workflows: # only: # - master jobs: + - circleci_consistency + + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py2.7 + python_version: "2.7" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py2.7_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py2.7 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py2.7_unicode_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py2.7 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.5 + python_version: "3.5" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.5_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.5 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.6 + python_version: "3.6" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.6_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.6 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.7_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py2.7 + python_version: "2.7" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py2.7_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py2.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py2.7_unicode_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py2.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.5 + python_version: "3.5" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.5_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.5 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.6 + python_version: "3.6" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.6_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.6 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.7_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.7 + - binary_linux_conda: + name: nightly_binary_linux_conda_py2.7 + python_version: "2.7" + - binary_conda_upload: + name: nightly_binary_linux_conda_py2.7_upload + context: org-member + requires: + - nightly_binary_linux_conda_py2.7 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.5 + python_version: "3.5" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.5_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.5 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.6 + python_version: "3.6" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.6_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.6 - binary_linux_conda: - name: master_binary_linux_conda_py3.7 + name: nightly_binary_linux_conda_py3.7 python_version: "3.7" - binary_conda_upload: + name: nightly_binary_linux_conda_py3.7_upload context: org-member requires: - - master_binary_linux_conda_py3.7 - - - binary_linux_wheel: - name: master_binary_linux_wheel_py3.7 + - nightly_binary_linux_conda_py3.7 + - binary_macos_conda: + name: nightly_binary_macos_conda_py2.7 + python_version: "2.7" + - binary_conda_upload: + name: nightly_binary_macos_conda_py2.7_upload + context: org-member + requires: + - nightly_binary_macos_conda_py2.7 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.5 + python_version: "3.5" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.5_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.5 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.6 + python_version: "3.6" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.6_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.6 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.7 python_version: "3.7" - - binary_wheel_upload: + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.7_upload context: org-member requires: - - master_binary_linux_wheel_py3.7 \ No newline at end of file + - nightly_binary_macos_conda_py3.7 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 453bde28c3..53744a1e42 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -155,27 +155,42 @@ jobs: done -workflows: - build: - jobs: - - circleci_consistency -{%- for btype in ["wheel", "conda"] -%} -{%- for os in ["linux", "macos"] -%} -{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] %} +{%- macro workflow(btype, os, python_version, unicode, prefix='', upload=False) %} - binary_{{os}}_{{btype}}: - name: binary_{{os}}_{{btype}}_py{{python_version}} - python_version: "{{python_version}}" -{%- if btype == "wheel" and python_version == "2.7" %} - - binary_{{os}}_{{btype}}: - name: binary_{{os}}_{{btype}}_py{{python_version}}_unicode + name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }} python_version: "{{python_version}}" +{%- if unicode %} unicode_abi: "1" -{%- endif -%} +{%- endif %} + +{%- if upload %} + - binary_{{btype}}_upload: + name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_upload + context: org-member + requires: + - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}} +{%- endif %} +{%- endmacro %} + +{%- macro workflows(prefix='', upload=False) %} +{%- for btype in ["wheel", "conda"] -%} +{%- for os in ["linux", "macos"] -%} +{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] -%} +{%- for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]) -%} + {{ workflow(btype, os, python_version, unicode, prefix=prefix, upload=upload) }} +{%- endfor -%} {%- endfor -%} {%- endfor -%} -{%- endfor %} +{%- endfor %} +{%- endmacro %} + +workflows: + build: + jobs: + - circleci_consistency + {{ workflows() }} - master: + nightly: # triggers: # - schedule: # cron: "0 9 * * *" @@ -184,18 +199,6 @@ workflows: # only: # - master jobs: - - binary_linux_conda: - name: master_binary_linux_conda_py3.7 - python_version: "3.7" - - binary_conda_upload: - context: org-member - requires: - - master_binary_linux_conda_py3.7 + - circleci_consistency + {{ workflows(prefix="nightly_", upload=True) }} - - binary_linux_wheel: - name: master_binary_linux_wheel_py3.7 - python_version: "3.7" - - binary_wheel_upload: - context: org-member - requires: - - master_binary_linux_wheel_py3.7 From 9ac84f7cda87828c8c5a3aac6b7d2dfad41dfc42 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 15:36:24 -0700 Subject: [PATCH 12/18] Be a little less wasteful Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 8 ++++---- .circleci/config.yml.in | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bae2950a5..6c58186ef8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,10 +155,10 @@ jobs: done workflows: - build: - jobs: - - circleci_consistency - +# build: +# jobs: +# - circleci_consistency +# - binary_linux_wheel: name: binary_linux_wheel_py2.7 python_version: "2.7" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 53744a1e42..3d8d90ff1f 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -185,10 +185,10 @@ jobs: {%- endmacro %} workflows: - build: - jobs: - - circleci_consistency - {{ workflows() }} +# build: +# jobs: +# - circleci_consistency +# {{ workflows() }} nightly: # triggers: From f7ba5ad70b91223df9e52e331050b7bde771dc8a Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 15:42:11 -0700 Subject: [PATCH 13/18] test Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 70 +---------------------------------------- .circleci/config.yml.in | 24 +++++++------- 2 files changed, 14 insertions(+), 80 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c58186ef8..0da0f28c58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,75 +155,7 @@ jobs: done workflows: -# build: -# jobs: -# - circleci_consistency -# - - binary_linux_wheel: - name: binary_linux_wheel_py2.7 - python_version: "2.7" - - binary_linux_wheel: - name: binary_linux_wheel_py2.7_unicode - python_version: "2.7" - unicode_abi: "1" - - binary_linux_wheel: - name: binary_linux_wheel_py3.5 - python_version: "3.5" - - binary_linux_wheel: - name: binary_linux_wheel_py3.6 - python_version: "3.6" - - binary_linux_wheel: - name: binary_linux_wheel_py3.7 - python_version: "3.7" - - binary_macos_wheel: - name: binary_macos_wheel_py2.7 - python_version: "2.7" - - binary_macos_wheel: - name: binary_macos_wheel_py2.7_unicode - python_version: "2.7" - unicode_abi: "1" - - binary_macos_wheel: - name: binary_macos_wheel_py3.5 - python_version: "3.5" - - binary_macos_wheel: - name: binary_macos_wheel_py3.6 - python_version: "3.6" - - binary_macos_wheel: - name: binary_macos_wheel_py3.7 - python_version: "3.7" - - binary_linux_conda: - name: binary_linux_conda_py2.7 - python_version: "2.7" - - binary_linux_conda: - name: binary_linux_conda_py3.5 - python_version: "3.5" - - binary_linux_conda: - name: binary_linux_conda_py3.6 - python_version: "3.6" - - binary_linux_conda: - name: binary_linux_conda_py3.7 - python_version: "3.7" - - binary_macos_conda: - name: binary_macos_conda_py2.7 - python_version: "2.7" - - binary_macos_conda: - name: binary_macos_conda_py3.5 - python_version: "3.5" - - binary_macos_conda: - name: binary_macos_conda_py3.6 - python_version: "3.6" - - binary_macos_conda: - name: binary_macos_conda_py3.7 - python_version: "3.7" - - nightly: -# triggers: -# - schedule: -# cron: "0 9 * * *" -# filters: -# branches: -# only: -# - master + build: jobs: - circleci_consistency diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 3d8d90ff1f..d1314fd771 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -185,19 +185,21 @@ jobs: {%- endmacro %} workflows: -# build: -# jobs: -# - circleci_consistency -# {{ workflows() }} + build: +{%- if False %} + jobs: + - circleci_consistency + {{ workflows() }} nightly: -# triggers: -# - schedule: -# cron: "0 9 * * *" -# filters: -# branches: -# only: -# - master + triggers: + - schedule: + cron: "0 9 * * *" + filters: + branches: + only: + - master +{%- endif %} jobs: - circleci_consistency {{ workflows(prefix="nightly_", upload=True) }} From 87b4dc49332956e2a218e9ae0f1c5022bc165ea4 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 16:03:03 -0700 Subject: [PATCH 14/18] Unbork wheel Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 1 + .circleci/config.yml.in | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0da0f28c58..a74777131c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -145,6 +145,7 @@ jobs: - run: command: | pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" # Prevent credential from leaking set +x export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index d1314fd771..18c9434b23 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -145,6 +145,7 @@ jobs: - run: command: | pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" # Prevent credential from leaking set +x export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" From 33af5425e4dd01e494f17d6a58159cb00df30e4a Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 20:19:43 -0700 Subject: [PATCH 15/18] Refactor again Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- packaging/build_conda.sh | 5 +- packaging/build_wheel.sh | 4 +- packaging/pkg_helpers.bash | 183 ++++++++++++++++++++++++------------- 5 files changed, 122 insertions(+), 74 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a74777131c..1f953e9505 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,7 @@ binary_common: &binary_common BUILD_VERSION: << parameters.build_version >> PYTORCH_VERSION: << parameters.pytorch_version >> UNICODE_ABI: << parameters.unicode_abi >> - CUDA_VERSION: cpu + CU_VERSION: cpu jobs: circleci_consistency: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 18c9434b23..f7c4b35ad0 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -30,7 +30,7 @@ binary_common: &binary_common BUILD_VERSION: << parameters.build_version >> PYTORCH_VERSION: << parameters.pytorch_version >> UNICODE_ABI: << parameters.unicode_abi >> - CUDA_VERSION: cpu + CU_VERSION: cpu jobs: circleci_consistency: diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 1941fa1100..6d0492579f 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -4,10 +4,7 @@ set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" -setup_build_version 0.4.0 -setup_cpuonly_cuda_suffix -setup_macos +setup_env 0.4.0 export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio - diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index beeb45695d..ebfb5c6f79 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -4,10 +4,8 @@ set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" +setup_env 0.4.0 setup_python -setup_cpuonly_cuda_suffix -setup_build_version 0.4.0 -setup_macos "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies pip_install numpy future setup_pip_pytorch_version diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index b4c9cdcee7..57d14e6e93 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -1,81 +1,92 @@ # A set of useful bash functions for common functionality we need to do in # many build scripts -# Respecting PYTHON_VERSION and UNICODE_ABI, add (or install) the correct -# version of Python to perform a build. Relevant to wheel builds. -setup_python() { - if [[ "$(uname)" == Darwin ]]; then - eval "$(conda shell.bash hook)" - conda env remove -n "env$PYTHON_VERSION" || true - conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" - conda activate "env$PYTHON_VERSION" - else - case "$PYTHON_VERSION" in - 2.7) - if [[ -n "$UNICODE_ABI" ]]; then - python_abi=cp27-cp27mu - else - python_abi=cp27-cp27m - fi - ;; - 3.5) python_abi=cp35-cp35m ;; - 3.6) python_abi=cp36-cp36m ;; - 3.7) python_abi=cp37-cp37m ;; - *) - echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" - exit 1 - ;; - esac - export PATH="/opt/python/$python_abi/bin:$PATH" - fi -} -# Fill CUDA_SUFFIX and CU_VERSION with CUDA_VERSION. CUDA_SUFFIX is -# left blank for the default CUDA version (that's a blank suffix) -setup_cuda_suffix() { - if [[ "$(uname)" == Darwin ]]; then - if [[ "$CUDA_VERSION" != "cpu" ]]; then - echo "CUDA_VERSION on OS X must be cpu" +# Setup CUDA environment variables, based on CU_VERSION +# +# Inputs: +# CU_VERSION (cpu, cu92, cu100) +# NO_CUDA_PACKAGE (bool) +# +# Outputs: +# VERSION_SUFFIX (e.g., "") +# PYTORCH_VERSION_SUFFIX (e.g., +cpu) +# WHEEL_DIR (e.g., cu100/) +# CUDA_HOME (e.g., /usr/local/cuda-9.2, respected by torch.utils.cpp_extension) +# FORCE_CUDA (respected by torchvision setup.py) +# NVCC_FLAGS (respected by torchvision setup.py) +# +# Precondition: CUDA versions are installed in their conventional locations in +# /usr/local/cuda-* +# +# NOTE: Why VERSION_SUFFIX versus PYTORCH_VERSION_SUFFIX? If you're building +# a package with CUDA on a platform we support CUDA on, VERSION_SUFFIX == +# PYTORCH_VERSION_SUFFIX and everyone is happy. However, if you are building a +# package with only CPU bits (e.g., torchaudio), then VERSION_SUFFIX is always +# empty, but PYTORCH_VERSION_SUFFIX is +cpu (because that's how you get a CPU +# version of a Python package. But that doesn't apply if you're on OS X, +# since the default CU_VERSION on OS X is cpu. +setup_cuda() { + if [[ "$(uname)" == Darwin ]] || [[ -n "$NO_CUDA_PACKAGE" ]]; then + if [[ "$CU_VERSION" != "cpu" ]]; then + echo "CU_VERSION on OS X / package with no CUDA must be cpu" exit 1 fi - export CPU_SUFFIX="" - export CU_VERSION="cpu" + if [[ "$(uname)" == Darwin ]]; then + export PYTORCH_VERSION_SUFFIX="" + else + export PYTORCH_VERSION_SUFFIX="+cpu" + fi + export VERSION_SUFFIX="" + # NB: When there is no CUDA package available, we put these + # packages in the top-level directory, so they are eligible + # for selection even if you are otherwise trying to install + # a cu100 stack. This differs from when there ARE CUDA packages + # available; then we don't want the cpu package; we want + # to give you as much goodies as possible. export WHEEL_DIR="" else - case "$CUDA_VERSION" in - 10.0) - export CUDA_SUFFIX="" - export CU_VERSION="cu100" + case "$CU_VERSION" in + cu100) + export PYTORCH_VERSION_SUFFIX="" + export CUDA_HOME=/usr/local/cuda-10.0/ + export FORCE_CUDA=1 + # Hard-coding gencode flags is temporary situation until + # https://github.com/pytorch/pytorch/pull/23408 lands + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" ;; - 9.2) - export CUDA_SUFFIX="+cu92" - export CU_VERSION="cu92" + cu92) + export CUDA_HOME=/usr/local/cuda-9.2/ + export PYTORCH_VERSION_SUFFIX="+cu92" + export FORCE_CUDA=1 + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50" ;; cpu) - export CUDA_SUFFIX="+cpu" - export CU_VERSION="cpu" + export PYTORCH_VERSION_SUFFIX="+cpu" ;; *) - echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" + echo "Unrecognized CU_VERSION=$CU_VERSION" esac - export CPU_SUFFIX="+cpu" + export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX" export WHEEL_DIR="$CU_VERSION/" fi } -# If a package is cpu-only, we never provide a cuda suffix -setup_cpuonly_cuda_suffix() { - export CUDA_SUFFIX="" - export CPU_SUFFIX="" - export CU_VERSION="cpu" - export WHEEL_DIR="" -} - -# Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string -# Usage: setup_build_version 0.2 +# Populate build version if necessary, and add version suffix +# +# Inputs: +# BUILD_VERSION (e.g., 0.2.0 or empty) +# +# Outputs: +# BUILD_VERSION (e.g., 0.2.0.dev20190807+cpu) +# +# Fill BUILD_VERSION if it doesn't exist already with a nightly string +# Usage: setup_build_version 0.2.0 setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then - export BUILD_VERSION="$1.dev$(date "+%Y%m%d")" + export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + else + export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi } @@ -86,11 +97,55 @@ setup_macos() { fi } +# Top-level entry point for things every package will need to do +# +# Usage: setup_env 0.2.0 +setup_env() { + setup_cuda + setup_build_version "$1" + setup_macos +} + # Function to retry functions that sometimes timeout or have flaky failures retry () { $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) } +# Inputs: +# PYTHON_VERSION (2.7, 3.5, 3.6, 3.7) +# UNICODE_ABI (bool) +# +# Outputs: +# PATH modified to put correct Python version in PATH +# +# Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image +setup_wheel_python() { + if [[ "$(uname)" == Darwin ]]; then + eval "$(conda shell.bash hook)" + conda env remove -n "env$PYTHON_VERSION" || true + conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" + conda activate "env$PYTHON_VERSION" + else + case "$PYTHON_VERSION" in + 2.7) + if [[ -n "$UNICODE_ABI" ]]; then + python_abi=cp27-cp27mu + else + python_abi=cp27-cp27m + fi + ;; + 3.5) python_abi=cp35-cp35m ;; + 3.6) python_abi=cp36-cp36m ;; + 3.7) python_abi=cp37-cp37m ;; + *) + echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" + exit 1 + ;; + esac + export PATH="/opt/python/$python_abi/bin:$PATH" + fi +} + # Install with pip a bit more robustly than the default pip_install() { retry pip install --progress-bar off "$@" @@ -111,9 +166,9 @@ setup_pip_pytorch_version() { export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')" fi else - # TODO: Maybe add staging too pip_install "torch==$PYTORCH_VERSION$CUDA_SUFFIX" \ - -f https://download.pytorch.org/whl/torch_stable.html + -f https://download.pytorch.org/whl/torch_stable.html \ + -f https://download.pytorch.org/whl/torch_nightly.html fi } @@ -121,8 +176,6 @@ setup_pip_pytorch_version() { # CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions # # You MUST have populated CUDA_SUFFIX before hand. -# -# TODO: This is currently hard-coded for CPU-only case setup_conda_pytorch_constraint() { if [[ -z "$PYTORCH_VERSION" ]]; then export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" @@ -131,11 +184,11 @@ setup_conda_pytorch_constraint() { export CONDA_CHANNEL_FLAGS="-c pytorch" fi if [[ "$CUDA_VERSION" == cpu ]]; then - export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${CPU_SUFFIX}" + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" else - export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" - export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}" fi } From 7acb1c7a06fba6603ce656bb529fca3637d80c9d Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 20:22:06 -0700 Subject: [PATCH 16/18] bugfixes Signed-off-by: Edward Z. Yang --- packaging/build_wheel.sh | 2 +- packaging/pkg_helpers.bash | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index ebfb5c6f79..eeac98d3f6 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" setup_env 0.4.0 -setup_python +setup_wheel_python "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies pip_install numpy future setup_pip_pytorch_version diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 57d14e6e93..f5f473a758 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -168,7 +168,7 @@ setup_pip_pytorch_version() { else pip_install "torch==$PYTORCH_VERSION$CUDA_SUFFIX" \ -f https://download.pytorch.org/whl/torch_stable.html \ - -f https://download.pytorch.org/whl/torch_nightly.html + -f https://download.pytorch.org/whl/nightly/torch_nightly.html fi } @@ -181,7 +181,7 @@ setup_conda_pytorch_constraint() { export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else - export CONDA_CHANNEL_FLAGS="-c pytorch" + export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" fi if [[ "$CUDA_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" From d327902319d7e58e4b566e9b680d9cffc75635a9 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 20:46:24 -0700 Subject: [PATCH 17/18] bbugfixes Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 2 +- packaging/build_conda.sh | 1 + packaging/build_wheel.sh | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f953e9505..cf452255b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -176,7 +176,7 @@ workflows: name: nightly_binary_linux_wheel_py2.7_unicode_upload context: org-member requires: - - nightly_binary_linux_wheel_py2.7 + - nightly_binary_linux_wheel_py2.7_unicode - binary_linux_wheel: name: nightly_binary_linux_wheel_py3.5 python_version: "3.5" @@ -217,7 +217,7 @@ workflows: name: nightly_binary_macos_wheel_py2.7_unicode_upload context: org-member requires: - - nightly_binary_macos_wheel_py2.7 + - nightly_binary_macos_wheel_py2.7_unicode - binary_macos_wheel: name: nightly_binary_macos_wheel_py3.5 python_version: "3.5" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index f7c4b35ad0..9f2a7f14a6 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -169,7 +169,7 @@ jobs: name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_upload context: org-member requires: - - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}} + - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }} {%- endif %} {%- endmacro %} diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 6d0492579f..be00e74f3c 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -4,6 +4,7 @@ set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" +export NO_CUDA_PACKAGE=1 setup_env 0.4.0 export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index eeac98d3f6..60818f336b 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -4,6 +4,7 @@ set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" +export NO_CUDA_PACKAGE=1 setup_env 0.4.0 setup_wheel_python "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies From 1cc167b8d9156be6d7465da5fe74de072e20a6ae Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 8 Aug 2019 05:37:19 -0700 Subject: [PATCH 18/18] All ready now. Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 68 +++++++++++++++++++++++++++++++++++++++++ .circleci/config.yml.in | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf452255b1..c3b443d844 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,6 +160,74 @@ workflows: jobs: - circleci_consistency + - binary_linux_wheel: + name: binary_linux_wheel_py2.7 + python_version: "2.7" + - binary_linux_wheel: + name: binary_linux_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_linux_wheel: + name: binary_linux_wheel_py3.5 + python_version: "3.5" + - binary_linux_wheel: + name: binary_linux_wheel_py3.6 + python_version: "3.6" + - binary_linux_wheel: + name: binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_macos_wheel: + name: binary_macos_wheel_py2.7 + python_version: "2.7" + - binary_macos_wheel: + name: binary_macos_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_macos_wheel: + name: binary_macos_wheel_py3.5 + python_version: "3.5" + - binary_macos_wheel: + name: binary_macos_wheel_py3.6 + python_version: "3.6" + - binary_macos_wheel: + name: binary_macos_wheel_py3.7 + python_version: "3.7" + - binary_linux_conda: + name: binary_linux_conda_py2.7 + python_version: "2.7" + - binary_linux_conda: + name: binary_linux_conda_py3.5 + python_version: "3.5" + - binary_linux_conda: + name: binary_linux_conda_py3.6 + python_version: "3.6" + - binary_linux_conda: + name: binary_linux_conda_py3.7 + python_version: "3.7" + - binary_macos_conda: + name: binary_macos_conda_py2.7 + python_version: "2.7" + - binary_macos_conda: + name: binary_macos_conda_py3.5 + python_version: "3.5" + - binary_macos_conda: + name: binary_macos_conda_py3.6 + python_version: "3.6" + - binary_macos_conda: + name: binary_macos_conda_py3.7 + python_version: "3.7" + + nightly: + triggers: + - schedule: + cron: "0 9 * * *" + filters: + branches: + only: + - master + jobs: + - circleci_consistency + - binary_linux_wheel: name: nightly_binary_linux_wheel_py2.7 python_version: "2.7" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 9f2a7f14a6..771e17443b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -187,7 +187,7 @@ jobs: workflows: build: -{%- if False %} +{%- if True %} jobs: - circleci_consistency {{ workflows() }}