From 97dd37eaf7472e83f8d1e258ad26b88e9f0bbefb Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 13:39:58 -0700 Subject: [PATCH 01/27] Revert "Revert "Attempt OS X binaries"" This reverts commit 0e64ab6831a90dbd0101c66a0dba718531bd6c7c. --- .circleci/config.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01d3bef8b2..0f030e4d91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,6 +36,31 @@ jobs: - store_artifacts: path: /opt/conda/conda-bld/linux-64 + binary_macos_wheel: + parameters: + <<: *param_target_commit + macos: + xcode: "9.0" + resource_class: 2xlarge+ + steps: + - checkout + - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/wheel/osx_wheel.sh + - store_artifacts: + path: $HOME/torchaudio_wheels + + binary_macos_conda: + parameters: + <<: *param_target_commit + macos: + xcode: "9.0" + resource_class: 2xlarge+ + steps: + - checkout + # TODO: This needs a conda install... + - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/conda/build_audio.sh + - store_artifacts: + path: $HOME/miniconda3/conda-bld/osx-64 + workflows: build: jobs: From 9bd3fee79a8697d95211b66aa12122247c1a36ec Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 13:39:59 -0700 Subject: [PATCH 02/27] Revert "Revert "Add to workflow"" This reverts commit 5ad6d2ef7def3e9d8ddaa35c29f4c498f137d6fc. --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f030e4d91..2c3e42bc1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,3 +66,5 @@ workflows: jobs: - binary_linux_wheel - binary_linux_conda + - binary_macos_wheel + - binary_macos_conda From bb30476f4fcb73176c8b65737e5b5b1eed20b1e6 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 13:43:58 -0700 Subject: [PATCH 03/27] Put in portability thing for realpath Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 2 -- build_tools/packaging/conda/build_audio.sh | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c3e42bc1a..9893db7384 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,6 @@ jobs: <<: *param_target_commit macos: xcode: "9.0" - resource_class: 2xlarge+ steps: - checkout - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/wheel/osx_wheel.sh @@ -53,7 +52,6 @@ jobs: <<: *param_target_commit macos: xcode: "9.0" - resource_class: 2xlarge+ steps: - checkout # TODO: This needs a conda install... diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh index bfdf592de2..71935728ec 100755 --- a/build_tools/packaging/conda/build_audio.sh +++ b/build_tools/packaging/conda/build_audio.sh @@ -5,6 +5,10 @@ fi set -ex +command -v realpath >/dev/null 2>&1 || realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" +} + # Function to retry functions that sometimes timeout or have flaky failures retry () { $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) From b25eccc08673d3933b6bcbd03ce7eb31c12ea2a2 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 14:05:33 -0700 Subject: [PATCH 04/27] Replace wget with portable curl, add missing conda install Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 10 ++++++++-- build_tools/packaging/wheel/build_from_source.sh | 8 ++++---- build_tools/packaging/wheel/osx_wheel.sh | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9893db7384..bdf43cf87f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,8 +54,14 @@ jobs: xcode: "9.0" steps: - checkout - # TODO: This needs a conda install... - - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/conda/build_audio.sh + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + TARGET_COMMIT=<< parameters.target_commit >> \ + build_tools/packaging/conda/build_audio.sh - store_artifacts: path: $HOME/miniconda3/conda-bld/osx-64 diff --git a/build_tools/packaging/wheel/build_from_source.sh b/build_tools/packaging/wheel/build_from_source.sh index eacd890852..6748de5e46 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/build_tools/packaging/wheel/build_from_source.sh @@ -2,10 +2,10 @@ rm -rf source_code mkdir source_code pushd source_code -wget -q -O sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -wget -q -O lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -wget -q -O flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -wget -q -O libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +curl -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh index 6aaaf31c5c..cc7e4f6482 100755 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ b/build_tools/packaging/wheel/osx_wheel.sh @@ -8,7 +8,7 @@ if [[ ":$PATH:" == *"conda"* ]]; then fi # download and activate anaconda rm -rf ~/minconda_wheel_env_tmp -wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ +curl -o Miniconda3-latest-MacOSX-x86_64.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \ ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \ rm Miniconda3-latest-MacOSX-x86_64.sh From f208db7ad958c3d75bec4e6016f8cd4c2654c86f Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 14:07:34 -0700 Subject: [PATCH 05/27] Fix bad pwd assumption Signed-off-by: Edward Z. Yang --- build_tools/packaging/wheel/osx_wheel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh index cc7e4f6482..2e57d8a918 100755 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ b/build_tools/packaging/wheel/osx_wheel.sh @@ -21,7 +21,7 @@ export OUT_DIR=~/torchaudio_wheels export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ -CURR_PATH=$(pwd) +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd /tmp rm -rf audio @@ -29,7 +29,7 @@ git clone https://github.com/pytorch/audio mkdir audio/third_party export PREFIX="/tmp/audio" -. $CURR_PATH/build_from_source.sh +. "$script_dir/build_from_source.sh" cd /tmp/audio From 1e4f097481eb81171afbfb43b8fdb9702acb4027 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 14:11:12 -0700 Subject: [PATCH 06/27] Follow URLs in curl Signed-off-by: Edward Z. Yang --- .../packaging/wheel/build_from_source.sh | 8 +++---- build_tools/packaging/wheel/osx_wheel.sh | 22 ++++++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/build_tools/packaging/wheel/build_from_source.sh b/build_tools/packaging/wheel/build_from_source.sh index 6748de5e46..2914ff2e44 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/build_tools/packaging/wheel/build_from_source.sh @@ -2,10 +2,10 @@ rm -rf source_code mkdir source_code pushd source_code -curl -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -curl -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -curl -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -curl -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +curl -L -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -L -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -L -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -L -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh index 2e57d8a918..b2fa5d8f55 100755 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ b/build_tools/packaging/wheel/osx_wheel.sh @@ -23,15 +23,25 @@ export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd /tmp -rm -rf audio -git clone https://github.com/pytorch/audio -mkdir audio/third_party +if [[ "$TARGET_COMMIT" == HEAD ]]; then + # Assume that this script was called from a valid checkout + WORKDIR="$(realpath "$script_dir/../../..")" +else + WORKDIR="/tmp/audio" + cd /tmp + rm -rf audio + git clone https://github.com/pytorch/audio + cd audio + git checkout "$TARGET_COMMIT" + git submodule update --init --recursive +fi + +mkdir "$WORKDIR/third_party" -export PREFIX="/tmp/audio" +export PREFIX="$WORKDIR" . "$script_dir/build_from_source.sh" -cd /tmp/audio +cd "$WORKDIR" desired_pythons=( "2.7" "3.5" "3.6" "3.7" ) # for each python From 28c62b79fe9a1d2da061bcb9b090cf6a94de337a Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 14:15:16 -0700 Subject: [PATCH 07/27] Expunge more realpath use Signed-off-by: Edward Z. Yang --- build_tools/packaging/wheel/osx_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh index b2fa5d8f55..e9c2822d4d 100755 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ b/build_tools/packaging/wheel/osx_wheel.sh @@ -25,7 +25,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" if [[ "$TARGET_COMMIT" == HEAD ]]; then # Assume that this script was called from a valid checkout - WORKDIR="$(realpath "$script_dir/../../..")" + WORKDIR="$script_dir/../../.." else WORKDIR="/tmp/audio" cd /tmp From fc30f5862955b4f476a5fee7883001065a0e33e7 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 14:22:54 -0700 Subject: [PATCH 08/27] Disable coreaudio by default Signed-off-by: Edward Z. Yang --- build_tools/packaging/wheel/build_from_source.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/packaging/wheel/build_from_source.sh b/build_tools/packaging/wheel/build_from_source.sh index 2914ff2e44..44fb2b7ca8 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/build_tools/packaging/wheel/build_from_source.sh @@ -43,7 +43,7 @@ pushd sox-14.4.2 ./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/sox" \ LDFLAGS="-L$PREFIX/third_party/lame/lib -L$PREFIX/third_party/flac/lib -L$PREFIX/third_party/mad/lib" \ CPPFLAGS="-I$PREFIX/third_party/lame/include -I$PREFIX/third_party/flac/include -I$PREFIX/third_party/mad/include" \ - --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking + --with-lame --with-flac --with-mad --without-coreaudio --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd From 4e7f966b0a124bfae67f2f33174b9cd74cebc938 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 15:12:58 -0700 Subject: [PATCH 09/27] Get started on release bits Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 72 +++++++++++++------ .../packaging/wheel/linux_manywheel.sh | 7 +- build_tools/packaging/wheel/osx_wheel.sh | 28 ++++++-- 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bdf43cf87f..3bbc657dcc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,55 +1,69 @@ version: 2.1 -# How to test: +# How to test the Linux jobs: # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ # - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel # - Replace binary_linux_wheel with the name of the job you want to test -param_target_commit: ¶m_target_commit - target_commit: - description: "what hash or branch to build binaries for; by default, same as trigger" - type: string - default: HEAD +binary_common: &binary_common + parameters: + target_commit: + description: "what hash or branch to build binaries for; by default, same as trigger" + type: string + default: HEAD + version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + build_number: + description: "build number (if you need to rebuild a version with minor changes)" + type: integer + default: 1 + environment: + TARGET_COMMIT: << parameters.target_commit >> + TORCHAUDIO_BUILD_VERSION: << parameters.version >> + TORCHAUDIO_BUILD_NUMBER: << parameters.build_number >> + TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION: << parameters.pytorch_version >> jobs: binary_linux_wheel: - parameters: - <<: *param_target_commit + <<: *binary_common docker: - image: "soumith/manylinux-cuda100" resource_class: 2xlarge+ steps: - checkout - - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/wheel/linux_manywheel.sh + - run: build_tools/packaging/wheel/linux_manywheel.sh - store_artifacts: path: /remote/cpu binary_linux_conda: - parameters: - <<: *param_target_commit + <<: *binary_common docker: - image: "soumith/conda-cuda" resource_class: 2xlarge+ steps: - checkout - - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/conda/build_audio.sh + - run: build_tools/packaging/conda/build_audio.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 binary_macos_wheel: - parameters: - <<: *param_target_commit + <<: *binary_common macos: xcode: "9.0" steps: - checkout - - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/wheel/osx_wheel.sh + - run: build_tools/packaging/wheel/osx_wheel.sh - store_artifacts: - path: $HOME/torchaudio_wheels + path: /Users/distiller/torchaudio_wheels binary_macos_conda: - parameters: - <<: *param_target_commit + <<: *binary_common macos: xcode: "9.0" steps: @@ -60,10 +74,15 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build - TARGET_COMMIT=<< parameters.target_commit >> \ - build_tools/packaging/conda/build_audio.sh + build_tools/packaging/conda/build_audio.sh - store_artifacts: - path: $HOME/miniconda3/conda-bld/osx-64 + path: /Users/distiller/miniconda3/conda-bld/osx-64 + +release_parameters: &release_parameters + version: "0.3.0" + target_commit: "v0.3.0" + pytorch_version: "1.2.0" + build_number: 1 workflows: build: @@ -72,3 +91,14 @@ workflows: - binary_linux_conda - binary_macos_wheel - binary_macos_conda + + release: + jobs: + - binary_linux_wheel: + <<: *release_parameters + - binary_linux_conda: + <<: *release_parameters + - binary_macos_wheel: + <<: *release_parameters + - binary_macos_conda: + <<: *release_parameters diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh index 6b6979d0e4..7f12126d90 100755 --- a/build_tools/packaging/wheel/linux_manywheel.sh +++ b/build_tools/packaging/wheel/linux_manywheel.sh @@ -52,7 +52,12 @@ for PYDIR in "${python_installations[@]}"; do # the CPU build, because it takes less time to download. export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')" else - pip install "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" -f https://download.pytorch.org/whl/torch_stable.html + # NB: We include the nightly channel to, since sometimes we stage + # prereleases in it. Those releases should get moved to stable + # when they're ready + pip install "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" \ + -f https://download.pytorch.org/whl/torch_stable.html \ + -f https://download.pytorch.org/whl/nightly/torch_nightly.html fi echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh index e9c2822d4d..b69e1f020b 100755 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ b/build_tools/packaging/wheel/osx_wheel.sh @@ -15,8 +15,12 @@ curl -o Miniconda3-latest-MacOSX-x86_64.sh https://repo.anaconda.com/miniconda/M . ~/minconda_wheel_env_tmp/bin/activate -export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" -export TORCHAUDIO_BUILD_NUMBER="1" +if [[ -z "$TORCHAUDIO_BUILD_VERSION" ]]; then + export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" +fi +if [[ -z "$TORCHAUDIO_BUILD_NUMBER" ]]; then + export TORCHAUDIO_BUILD_NUMBER="1" +fi export OUT_DIR=~/torchaudio_wheels export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ @@ -43,6 +47,8 @@ export PREFIX="$WORKDIR" cd "$WORKDIR" +ORIG_TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" + desired_pythons=( "2.7" "3.5" "3.6" "3.7" ) # for each python for desired_python in "${desired_pythons[@]}" @@ -52,11 +58,19 @@ do conda create -yn $env_name python="$desired_python" conda activate $env_name - export TORCHAUDIO_PYTORCH_DEPENDENCY_NAME=torch_nightly - pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # NB: OS X builds don't have local package qualifiers - # NB: Don't use \+ here, it's not portable - export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: *//')" + if [[ -z "$ORIG_TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" ]]; then + pip install torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + # NB: OS X builds don't have local package qualifiers + # NB: Don't use \+ here, it's not portable + export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')" + else + # NB: We include the nightly channel to, since sometimes we stage + # prereleases in it. Those releases should get moved to stable + # when they're ready + pip install "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" \ + -f https://download.pytorch.org/whl/torch_stable.html \ + -f https://download.pytorch.org/whl/nightly/torch_nightly.html + fi echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" pip install numpy future From 81696fdc3a49de77865593afdca15b52f155187c Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 15:32:06 -0700 Subject: [PATCH 10/27] Refactor scripts Signed-off-by: Edward Z. Yang --- .../{wheel => }/build_from_source.sh | 12 +++-- build_tools/packaging/conda/build_audio.sh | 19 +------ .../packaging/conda/torchaudio/build.sh | 3 +- build_tools/packaging/setup_workdir | 24 +++++++++ build_tools/packaging/wheel/build_wheel.sh | 42 +++++++++++++++ .../packaging/wheel/linux_manywheel.sh | 54 +------------------ build_tools/packaging/wheel/osx_wheel.sh | 51 +----------------- 7 files changed, 81 insertions(+), 124 deletions(-) rename build_tools/packaging/{wheel => }/build_from_source.sh (93%) create mode 100644 build_tools/packaging/setup_workdir create mode 100755 build_tools/packaging/wheel/build_wheel.sh diff --git a/build_tools/packaging/wheel/build_from_source.sh b/build_tools/packaging/build_from_source.sh similarity index 93% rename from build_tools/packaging/wheel/build_from_source.sh rename to build_tools/packaging/build_from_source.sh index 44fb2b7ca8..62cfd7ea93 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/build_tools/packaging/build_from_source.sh @@ -1,6 +1,12 @@ -rm -rf source_code -mkdir source_code -pushd source_code +#!/bin/bash + +# Arguments: PREFIX, specifying where to install dependencies into + +PREFIX="$1" + +rm -rf /tmp/torchaudio-deps +mkdir /tmp/torchaudio-deps +pushd /tmp/torchaudio-deps curl -L -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" curl -L -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh index 71935728ec..2a8fcf613c 100755 --- a/build_tools/packaging/conda/build_audio.sh +++ b/build_tools/packaging/conda/build_audio.sh @@ -5,10 +5,6 @@ fi set -ex -command -v realpath >/dev/null 2>&1 || realpath() { - [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" -} - # Function to retry functions that sometimes timeout or have flaky failures retry () { $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) @@ -23,20 +19,7 @@ fi script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -if [[ "$TARGET_COMMIT" == HEAD ]]; then - # Assume that this script was called from a valid checkout - WORKDIR="$(realpath "$script_dir/../../..")" -else - WORKDIR="/tmp/audio" - cd /tmp - rm -rf audio - git clone https://github.com/pytorch/audio - cd audio - git checkout "$TARGET_COMMIT" - git submodule update --init --recursive - mkdir audio/third_party -fi - +. "$script_dir/../setup_workdir" export TORCHAUDIO_GITHUB_ROOT_DIR="$WORKDIR" cd "$script_dir" diff --git a/build_tools/packaging/conda/torchaudio/build.sh b/build_tools/packaging/conda/torchaudio/build.sh index 7e5917060f..138826b69a 100644 --- a/build_tools/packaging/conda/torchaudio/build.sh +++ b/build_tools/packaging/conda/torchaudio/build.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -ex -PREFIX="$(pwd)" -. build_tools/packaging/wheel/build_from_source.sh +build_tools/packaging/build_from_source.sh "$(pwd)" IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt diff --git a/build_tools/packaging/setup_workdir b/build_tools/packaging/setup_workdir new file mode 100644 index 0000000000..c995ca9513 --- /dev/null +++ b/build_tools/packaging/setup_workdir @@ -0,0 +1,24 @@ +# Checkout or reuse existing work directory based on TARGET_COMMIT, +# build our static dependencies, and cd into the workdir +# +# Requires script_dir to be set, to a directory one BELOW this +# script (satisfied by conda/ and wheel/) Yeah this is hella +# janky + +if [[ "$TARGET_COMMIT" == HEAD ]]; then + # Assume that this script was called from a valid checkout + WORKDIR="$script_dir/../../.." +else + WORKDIR="/tmp/audio" + cd /tmp + rm -rf audio + git clone https://github.com/pytorch/audio + cd audio + git checkout "$TARGET_COMMIT" + git submodule update --init --recursive +fi + +mkdir "$WORKDIR/third_party" +"$script_dir/../build_from_source.sh" "$WORKDIR" + +cd "$WORKDIR" diff --git a/build_tools/packaging/wheel/build_wheel.sh b/build_tools/packaging/wheel/build_wheel.sh new file mode 100755 index 0000000000..d42023a597 --- /dev/null +++ b/build_tools/packaging/wheel/build_wheel.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Assume we are in a valid Python environment that we want to build the wheel in + +if [[ -z "$OUT_DIR" ]]; then + export OUT_DIR="/tmp/torchaudio-build" +fi +if [[ -z "$TORCHAUDIO_BUILD_VERSION" ]]; then + export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" +fi +if [[ -z "$TORCHAUDIO_BUILD_NUMBER" ]]; then + export TORCHAUDIO_BUILD_NUMBER="1" +fi +if [[ "$(uname)" == Darwin ]]; then + export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ +fi + +if [[ -z "$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" ]]; then + is_nightly=1 # to unset later + pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we + # strip off the local package qualifier. Also, we choose to build against + # the CPU build, because it takes less time to download. + export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')" +else + is_nightly= + # NB: We include the nightly channel to, since sometimes we stage + # prereleases in it. Those releases should get moved to stable + # when they're ready + pip install "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" \ + -f https://download.pytorch.org/whl/torch_stable.html \ + -f https://download.pytorch.org/whl/nightly/torch_nightly.html +fi +echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" + +# NB: do not actually install requirements.txt; that is only needed for +# testing +pip install numpy future +IS_WHEEL=1 python setup.py clean +IS_WHEEL=1 python setup.py bdist_wheel +mkdir -p $OUT_DIR +cp dist/*.whl $OUT_DIR/ diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh index 7f12126d90..f18ac1099d 100755 --- a/build_tools/packaging/wheel/linux_manywheel.sh +++ b/build_tools/packaging/wheel/linux_manywheel.sh @@ -1,13 +1,6 @@ #!/bin/bash set -ex - -if [[ -z "$TORCHAUDIO_BUILD_VERSION" ]]; then - export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" -fi -if [[ -z "$TORCHAUDIO_BUILD_NUMBER" ]]; then - export TORCHAUDIO_BUILD_NUMBER="1" -fi export OUT_DIR="/remote/cpu" script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -18,54 +11,11 @@ for desired_py in "${DESIRED_PYTHON[@]}"; do python_installations+=("/opt/python/$desired_py") done -if [[ "$TARGET_COMMIT" == HEAD ]]; then - # Assume that this script was called from a valid checkout - WORKDIR="$(realpath "$script_dir/../../..")" -else - WORKDIR="/tmp/audio" - cd /tmp - rm -rf audio - git clone https://github.com/pytorch/audio - cd audio - git checkout "$TARGET_COMMIT" - git submodule update --init --recursive -fi - -mkdir "$WORKDIR/third_party" - -export PREFIX="$WORKDIR" -. "$script_dir/build_from_source.sh" - -cd "$WORKDIR" - -ORIG_TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" +. "$script_dir/../setup_workdir" OLD_PATH=$PATH for PYDIR in "${python_installations[@]}"; do export PATH="$PYDIR/bin:$OLD_PATH" pip install --upgrade pip - - if [[ -z "$ORIG_TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" ]]; then - pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we - # strip off the local package qualifier. Also, we choose to build against - # the CPU build, because it takes less time to download. - export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')" - else - # NB: We include the nightly channel to, since sometimes we stage - # prereleases in it. Those releases should get moved to stable - # when they're ready - pip install "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" \ - -f https://download.pytorch.org/whl/torch_stable.html \ - -f https://download.pytorch.org/whl/nightly/torch_nightly.html - fi - echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" - - # NB: do not actually install requirements.txt; that is only needed for - # testing - pip install numpy future - IS_WHEEL=1 python setup.py clean - IS_WHEEL=1 python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ + "$script_dir/build_wheel.sh" done diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh index b69e1f020b..0dd0335051 100755 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ b/build_tools/packaging/wheel/osx_wheel.sh @@ -15,39 +15,11 @@ curl -o Miniconda3-latest-MacOSX-x86_64.sh https://repo.anaconda.com/miniconda/M . ~/minconda_wheel_env_tmp/bin/activate -if [[ -z "$TORCHAUDIO_BUILD_VERSION" ]]; then - export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" -fi -if [[ -z "$TORCHAUDIO_BUILD_NUMBER" ]]; then - export TORCHAUDIO_BUILD_NUMBER="1" -fi export OUT_DIR=~/torchaudio_wheels -export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ - script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -if [[ "$TARGET_COMMIT" == HEAD ]]; then - # Assume that this script was called from a valid checkout - WORKDIR="$script_dir/../../.." -else - WORKDIR="/tmp/audio" - cd /tmp - rm -rf audio - git clone https://github.com/pytorch/audio - cd audio - git checkout "$TARGET_COMMIT" - git submodule update --init --recursive -fi - -mkdir "$WORKDIR/third_party" - -export PREFIX="$WORKDIR" -. "$script_dir/build_from_source.sh" - -cd "$WORKDIR" - -ORIG_TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" +. "$script_dir/../setup_workdir" desired_pythons=( "2.7" "3.5" "3.6" "3.7" ) # for each python @@ -58,24 +30,5 @@ do conda create -yn $env_name python="$desired_python" conda activate $env_name - if [[ -z "$ORIG_TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" ]]; then - pip install torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # NB: OS X builds don't have local package qualifiers - # NB: Don't use \+ here, it's not portable - export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')" - else - # NB: We include the nightly channel to, since sometimes we stage - # prereleases in it. Those releases should get moved to stable - # when they're ready - pip install "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" \ - -f https://download.pytorch.org/whl/torch_stable.html \ - -f https://download.pytorch.org/whl/nightly/torch_nightly.html - fi - echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" - - pip install numpy future - IS_WHEEL=1 python setup.py clean - IS_WHEEL=1 python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ + "$script_dir/build_wheel.sh" done From d60f7282ce146dca24ac23d356d71e2f3e35be59 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 16:03:29 -0700 Subject: [PATCH 11/27] Improve Signed-off-by: Edward Z. Yang --- build_tools/packaging/build_from_source.sh | 2 ++ build_tools/packaging/wheel/build_wheel.sh | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build_tools/packaging/build_from_source.sh b/build_tools/packaging/build_from_source.sh index 62cfd7ea93..3e16da5ede 100755 --- a/build_tools/packaging/build_from_source.sh +++ b/build_tools/packaging/build_from_source.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -ex + # Arguments: PREFIX, specifying where to install dependencies into PREFIX="$1" diff --git a/build_tools/packaging/wheel/build_wheel.sh b/build_tools/packaging/wheel/build_wheel.sh index d42023a597..b8d6c0021d 100755 --- a/build_tools/packaging/wheel/build_wheel.sh +++ b/build_tools/packaging/wheel/build_wheel.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -ex + # Assume we are in a valid Python environment that we want to build the wheel in if [[ -z "$OUT_DIR" ]]; then @@ -17,7 +19,7 @@ fi if [[ -z "$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" ]]; then is_nightly=1 # to unset later - pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip install --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we # strip off the local package qualifier. Also, we choose to build against # the CPU build, because it takes less time to download. @@ -27,7 +29,7 @@ else # NB: We include the nightly channel to, since sometimes we stage # prereleases in it. Those releases should get moved to stable # when they're ready - pip install "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" \ + pip install --progress-bar off "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" \ -f https://download.pytorch.org/whl/torch_stable.html \ -f https://download.pytorch.org/whl/nightly/torch_nightly.html fi @@ -35,7 +37,7 @@ echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" # NB: do not actually install requirements.txt; that is only needed for # testing -pip install numpy future +pip install --progress-bar off numpy future IS_WHEEL=1 python setup.py clean IS_WHEEL=1 python setup.py bdist_wheel mkdir -p $OUT_DIR From 196ed19b7854cebccc035031dd396c591b789a68 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 16:10:48 -0700 Subject: [PATCH 12/27] OS X build fix Signed-off-by: Edward Z. Yang --- build_tools/packaging/wheel/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/packaging/wheel/build_wheel.sh b/build_tools/packaging/wheel/build_wheel.sh index b8d6c0021d..b52fb7503d 100755 --- a/build_tools/packaging/wheel/build_wheel.sh +++ b/build_tools/packaging/wheel/build_wheel.sh @@ -23,7 +23,7 @@ if [[ -z "$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" ]]; then # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we # strip off the local package qualifier. Also, we choose to build against # the CPU build, because it takes less time to download. - export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')" + export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" else is_nightly= # NB: We include the nightly channel to, since sometimes we stage From fd297e1b2d7d4e2a4ebfef66505cb1b06ea31e3a Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 16:11:09 -0700 Subject: [PATCH 13/27] Disable release builds for now Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3bbc657dcc..515c0e269a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,13 +92,13 @@ workflows: - binary_macos_wheel - binary_macos_conda - release: - jobs: - - binary_linux_wheel: - <<: *release_parameters - - binary_linux_conda: - <<: *release_parameters - - binary_macos_wheel: - <<: *release_parameters - - binary_macos_conda: - <<: *release_parameters +# release: +# jobs: +# - binary_linux_wheel: +# <<: *release_parameters +# - binary_linux_conda: +# <<: *release_parameters +# - binary_macos_wheel: +# <<: *release_parameters +# - binary_macos_conda: +# <<: *release_parameters From 5deb45f42f149783d28d4dab418150747cdf9344 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 16:18:08 -0700 Subject: [PATCH 14/27] Minor improvements Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 16 ++++++++++++---- build_tools/packaging/conda/build_audio.sh | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 515c0e269a..8472e6d760 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,18 +87,26 @@ release_parameters: &release_parameters workflows: build: jobs: - - binary_linux_wheel - - binary_linux_conda - - binary_macos_wheel - - binary_macos_conda + - binary_linux_wheel: + name: nightly_binary_linux_wheel + - binary_linux_conda: + name: nightly_binary_linux_conda + - binary_macos_wheel: + name: nightly_binary_macos_wheel + - binary_macos_conda: + name: nightly_binary_macos_conda # release: # jobs: # - binary_linux_wheel: # <<: *release_parameters +# name: release_binary_linux_wheel # - binary_linux_conda: # <<: *release_parameters +# name: release_binary_linux_conda # - binary_macos_wheel: # <<: *release_parameters +# name: release_binary_macos_wheel # - binary_macos_conda: # <<: *release_parameters +# name: release_binary_macos_conda diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh index 2a8fcf613c..9afdd14475 100755 --- a/build_tools/packaging/conda/build_audio.sh +++ b/build_tools/packaging/conda/build_audio.sh @@ -26,6 +26,7 @@ cd "$script_dir" ANACONDA_USER=pytorch-nightly conda config --set anaconda_upload no +conda config --set channel_priority strict LATEST_PYTORCH_NIGHTLY_VERSION=$(conda search --json 'pytorch[channel=pytorch-nightly]' | python "$script_dir/get-latest.py") export CONDA_PYTORCH_CONSTRAINT=" - pytorch ==${LATEST_PYTORCH_NIGHTLY_VERSION}" From f76fa1828648694a4e33e2a5e2634bc62c964804 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 18:09:04 -0700 Subject: [PATCH 15/27] More severe rewrite Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 80 +++++++++++++------ .../packaging => packaging}/README.md | 0 .../build_from_source.sh | 0 packaging/build_wheel.sh | 14 ++++ .../conda/build_audio.sh | 0 .../conda/get-latest.py | 0 .../conda/torchaudio/build.sh | 0 .../conda/torchaudio/meta.yaml | 0 packaging/pkg_helpers.bash | 64 +++++++++++++++ .../packaging => packaging}/setup_workdir | 0 .../wheel/build_wheel.sh | 0 .../wheel/linux_manywheel.sh | 0 .../wheel/osx_wheel.sh | 0 setup.py | 15 ++-- 14 files changed, 138 insertions(+), 35 deletions(-) rename {build_tools/packaging => packaging}/README.md (100%) rename {build_tools/packaging => packaging}/build_from_source.sh (100%) create mode 100755 packaging/build_wheel.sh rename {build_tools/packaging => packaging}/conda/build_audio.sh (100%) rename {build_tools/packaging => packaging}/conda/get-latest.py (100%) rename {build_tools/packaging => packaging}/conda/torchaudio/build.sh (100%) rename {build_tools/packaging => packaging}/conda/torchaudio/meta.yaml (100%) create mode 100644 packaging/pkg_helpers.bash rename {build_tools/packaging => packaging}/setup_workdir (100%) rename {build_tools/packaging => packaging}/wheel/build_wheel.sh (100%) rename {build_tools/packaging => packaging}/wheel/linux_manywheel.sh (100%) rename {build_tools/packaging => packaging}/wheel/osx_wheel.sh (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8472e6d760..43cbc0b7aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,11 +7,7 @@ version: 2.1 binary_common: &binary_common parameters: - target_commit: - description: "what hash or branch to build binaries for; by default, same as trigger" - type: string - default: HEAD - version: + build_version: description: "version number of release binary; by default, build a nightly" type: string default: "" @@ -19,15 +15,18 @@ binary_common: &binary_common description: "PyTorch version to build against; by default, use a nightly" type: string default: "" - build_number: - description: "build number (if you need to rebuild a version with minor changes)" - type: integer - default: 1 + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" environment: - TARGET_COMMIT: << parameters.target_commit >> - TORCHAUDIO_BUILD_VERSION: << parameters.version >> - TORCHAUDIO_BUILD_NUMBER: << parameters.build_number >> - TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION: << parameters.pytorch_version >> + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> jobs: binary_linux_wheel: @@ -37,9 +36,9 @@ jobs: resource_class: 2xlarge+ steps: - checkout - - run: build_tools/packaging/wheel/linux_manywheel.sh + - run: packaging/build_wheel.sh - store_artifacts: - path: /remote/cpu + path: dist binary_linux_conda: <<: *binary_common @@ -48,7 +47,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout - - run: build_tools/packaging/conda/build_audio.sh + - run: packaging/conda/build_audio.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 @@ -58,7 +57,12 @@ jobs: xcode: "9.0" steps: - checkout - - run: build_tools/packaging/wheel/osx_wheel.sh + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/wheel/osx_wheel.sh - store_artifacts: path: /Users/distiller/torchaudio_wheels @@ -80,21 +84,47 @@ jobs: release_parameters: &release_parameters version: "0.3.0" - target_commit: "v0.3.0" pytorch_version: "1.2.0" - build_number: 1 workflows: build: jobs: - binary_linux_wheel: - name: nightly_binary_linux_wheel - - binary_linux_conda: - name: nightly_binary_linux_conda + name: nightly_binary_linux_wheel_cp27m + python_version: "2.7" + - binary_linux_wheel: + name: nightly_binary_linux_wheel_cp27mu + python_version: "2.7" + unicode_abi: "1" + - binary_linux_wheel: + name: nightly_binary_linux_wheel_cp35 + python_version: "3.5" + - binary_linux_wheel: + name: nightly_binary_linux_wheel_cp36 + python_version: "3.6" + - binary_linux_wheel: + name: nightly_binary_linux_wheel_cp37 + python_version: "3.7" +# - binary_linux_conda: +# name: nightly_binary_linux_conda + - binary_macos_wheel: + name: nightly_binary_macos_wheel_cp27m + python_version: "2.7" - binary_macos_wheel: - name: nightly_binary_macos_wheel - - binary_macos_conda: - name: nightly_binary_macos_conda + name: nightly_binary_macos_wheel_cp27mu + python_version: "2.7" + unicode_abi: "1" + - binary_macos_wheel: + name: nightly_binary_macos_wheel_cp35 + python_version: "3.5" + - binary_macos_wheel: + name: nightly_binary_macos_wheel_cp36 + python_version: "3.6" + - binary_macos_wheel: + name: nightly_binary_macos_wheel_cp37 + python_version: "3.7" +# - binary_macos_conda: +# name: nightly_binary_macos_conda # release: # jobs: diff --git a/build_tools/packaging/README.md b/packaging/README.md similarity index 100% rename from build_tools/packaging/README.md rename to packaging/README.md diff --git a/build_tools/packaging/build_from_source.sh b/packaging/build_from_source.sh similarity index 100% rename from build_tools/packaging/build_from_source.sh rename to packaging/build_from_source.sh diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh new file mode 100755 index 0000000000..13f4a47170 --- /dev/null +++ b/packaging/build_wheel.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +setup_python +setup_build_version 0.4.0 +setup_macos +"$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies +pip_install numpy future +pip_install_torch +python setup.py clean +IS_WHEEL=1 python setup.py bdist_wheel diff --git a/build_tools/packaging/conda/build_audio.sh b/packaging/conda/build_audio.sh similarity index 100% rename from build_tools/packaging/conda/build_audio.sh rename to packaging/conda/build_audio.sh diff --git a/build_tools/packaging/conda/get-latest.py b/packaging/conda/get-latest.py similarity index 100% rename from build_tools/packaging/conda/get-latest.py rename to packaging/conda/get-latest.py diff --git a/build_tools/packaging/conda/torchaudio/build.sh b/packaging/conda/torchaudio/build.sh similarity index 100% rename from build_tools/packaging/conda/torchaudio/build.sh rename to packaging/conda/torchaudio/build.sh diff --git a/build_tools/packaging/conda/torchaudio/meta.yaml b/packaging/conda/torchaudio/meta.yaml similarity index 100% rename from build_tools/packaging/conda/torchaudio/meta.yaml rename to packaging/conda/torchaudio/meta.yaml diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash new file mode 100644 index 0000000000..8ef80a650f --- /dev/null +++ b/packaging/pkg_helpers.bash @@ -0,0 +1,64 @@ +# A set of useful bash functions for common functionality we need to do in +# many build scripts + +# Respecting PYTHON_VERSION and UNICODE_ABI, add (or install) the correct +# version of Python to perform a build. Relevant to wheel builds. +setup_python() { + if [[ "$(uname)" == Darwin ]]; then + conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" + conda activate "env$PYTHON_VERSION" + else + case "$PYTHON_VERSION" in + 2.7) + if [[ -n "$UNICODE_ABI" ]]; then + python_abi=cp27-cp27mu + else + python_abi=cp27-cp27m + fi + ;; + 3.5) python_abi=cp35-cp35m ;; + 3.6) python_abi=cp36-cp36m ;; + 3.7) python_abi=cp37-cp37m ;; + esac + export PATH="/opt/python/$python_abi/bin:$PATH" + fi +} + +# Fill BUILD_VERSION if it doesn't exist already with a nightly string +# Usage: setup_build_version 0.2 +setup_build_version() { + if [[ -z "$BUILD_VERSION" ]]; then + export BUILD_VERSION="$1.dev$(date "+%Y%m%d")" + fi +} + +# Set some useful variables for OS X, if applicable +setup_macos() { + if [[ "$(uname)" == Darwin ]]; then + export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ + fi +} + +# Install with pip a bit more robustly than the default +pip_install() { + pip install --progress-bar off "$@" +} + +# Install torch with pip, respecting PYTORCH_VERSION, and record the installed +# version into PYTORCH_VERSION, if applicable +pip_install_torch() { + if [[ -z "$PYTORCH_VERSION" ]]; then + # Install latest prerelease CPU version of torch, per our nightlies. + pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we + # strip off the local package qualifier. + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" + else + # NB: We include the nightly channel to, since sometimes we stage + # prereleases in it. Those releases should get moved to stable + # when they're ready + pip_install "torch==$PYTORCH_VERSION" \ + -f https://download.pytorch.org/whl/torch_stable.html \ + -f https://download.pytorch.org/whl/nightly/torch_nightly.html + fi +} diff --git a/build_tools/packaging/setup_workdir b/packaging/setup_workdir similarity index 100% rename from build_tools/packaging/setup_workdir rename to packaging/setup_workdir diff --git a/build_tools/packaging/wheel/build_wheel.sh b/packaging/wheel/build_wheel.sh similarity index 100% rename from build_tools/packaging/wheel/build_wheel.sh rename to packaging/wheel/build_wheel.sh diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/packaging/wheel/linux_manywheel.sh similarity index 100% rename from build_tools/packaging/wheel/linux_manywheel.sh rename to packaging/wheel/linux_manywheel.sh diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/packaging/wheel/osx_wheel.sh similarity index 100% rename from build_tools/packaging/wheel/osx_wheel.sh rename to packaging/wheel/osx_wheel.sh diff --git a/setup.py b/setup.py index 0b8d0940ff..9ed91f2468 100644 --- a/setup.py +++ b/setup.py @@ -63,12 +63,8 @@ def check_env_flag(name, default=''): except Exception: pass -if os.getenv('TORCHAUDIO_BUILD_VERSION'): - assert os.getenv('TORCHAUDIO_BUILD_NUMBER') is not None - build_number = int(os.getenv('TORCHAUDIO_BUILD_NUMBER')) - version = os.getenv('TORCHAUDIO_BUILD_VERSION') - if build_number > 1: - version += '.post' + str(build_number) +if os.getenv('BUILD_VERSION'): + version = os.getenv('BUILD_VERSION') elif sha != 'Unknown': version += '+' + sha[:7] print('-- Building version ' + version) @@ -78,15 +74,14 @@ def check_env_flag(name, default=''): f.write("__version__ = '{}'\n".format(version)) f.write("git_version = {}\n".format(repr(sha))) -pytorch_package_name = os.getenv('TORCHAUDIO_PYTORCH_DEPENDENCY_NAME', 'torch') -pytorch_package_version = os.getenv('TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION') +pytorch_package_version = os.getenv('PYTORCH_VERSION') -pytorch_package_dep = pytorch_package_name +pytorch_package_dep = 'torch' if pytorch_package_version is not None: pytorch_package_dep += "==" + pytorch_package_version setup( - name=os.getenv('TORCHAUDIO_PACKAGE_NAME', "torchaudio"), + name="torchaudio", version=version, description="An audio package for PyTorch", url="https://github.com/pytorch/audio", From 72c8ebd2209e1dab0bc08b9a18b11d2c94dd8ffa Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 18:11:09 -0700 Subject: [PATCH 16/27] bugfix Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 43cbc0b7aa..c4258e8721 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,7 +62,7 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate - packaging/wheel/osx_wheel.sh + packaging/build_wheel.sh - store_artifacts: path: /Users/distiller/torchaudio_wheels From d7ba81235984070ebf7d86d11c39ea80495158d9 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 18:43:02 -0700 Subject: [PATCH 17/27] Properly activate conda Signed-off-by: Edward Z. Yang --- packaging/pkg_helpers.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 8ef80a650f..0ea42db289 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -5,6 +5,7 @@ # version of Python to perform a build. Relevant to wheel builds. setup_python() { if [[ "$(uname)" == Darwin ]]; then + eval "$(conda shell.bash hook)" conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" conda activate "env$PYTHON_VERSION" else From 5bb4b16d4cd137267161851066db2ae4126b0156 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 18:59:21 -0700 Subject: [PATCH 18/27] conda stuff Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 39 ++++++++++++++++++---- packaging/build_conda.sh | 13 ++++++++ packaging/build_wheel.sh | 2 +- packaging/pkg_helpers.bash | 32 ++++++++++++++---- packaging/{conda => }/torchaudio/build.sh | 0 packaging/{conda => }/torchaudio/meta.yaml | 0 6 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 packaging/build_conda.sh rename packaging/{conda => }/torchaudio/build.sh (100%) rename packaging/{conda => }/torchaudio/meta.yaml (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4258e8721..19ced9da65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout - - run: packaging/conda/build_audio.sh + - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 @@ -78,7 +78,7 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build - build_tools/packaging/conda/build_audio.sh + packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 @@ -89,6 +89,7 @@ release_parameters: &release_parameters workflows: build: jobs: + # Linux wheels - binary_linux_wheel: name: nightly_binary_linux_wheel_cp27m python_version: "2.7" @@ -105,8 +106,22 @@ workflows: - binary_linux_wheel: name: nightly_binary_linux_wheel_cp37 python_version: "3.7" -# - binary_linux_conda: -# name: nightly_binary_linux_conda + + # Linux conda + - binary_linux_conda: + name: nightly_binary_linux_conda_py27 + python_version: "2.7" + - binary_linux_conda: + name: nightly_binary_linux_conda_py35 + python_version: "3.5" + - binary_linux_conda: + name: nightly_binary_linux_conda_py36 + python_version: "3.6" + - binary_linux_conda: + name: nightly_binary_linux_conda_py37 + python_version: "3.7" + + # Macos wheel - binary_macos_wheel: name: nightly_binary_macos_wheel_cp27m python_version: "2.7" @@ -123,8 +138,20 @@ workflows: - binary_macos_wheel: name: nightly_binary_macos_wheel_cp37 python_version: "3.7" -# - binary_macos_conda: -# name: nightly_binary_macos_conda + + # Macos conda + - binary_macos_conda: + name: nightly_binary_macos_conda_py27 + python_version: "2.7" + - binary_macos_conda: + name: nightly_binary_macos_conda_py35 + python_version: "3.5" + - binary_macos_conda: + name: nightly_binary_macos_conda_py36 + python_version: "3.6" + - binary_macos_conda: + name: nightly_binary_macos_conda_py37 + python_version: "3.7" # release: # jobs: diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh new file mode 100644 index 0000000000..9f272e9b42 --- /dev/null +++ b/packaging/build_conda.sh @@ -0,0 +1,13 @@ +#!/bin/bbash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +setup_build_version 0.4.0 +setup_macos +export SOURCE_ROOT_DIR="$PWD" +setup_conda_pytorch_version +setup conda_cudatoolkit_constraint +conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio + diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 13f4a47170..8871629c83 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -9,6 +9,6 @@ setup_build_version 0.4.0 setup_macos "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies pip_install numpy future -pip_install_torch +setup_pip_pytorch_version python setup.py clean IS_WHEEL=1 python setup.py bdist_wheel diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 0ea42db289..135cbbfcb2 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -40,14 +40,19 @@ setup_macos() { fi } +# Function to retry functions that sometimes timeout or have flaky failures +retry () { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + # Install with pip a bit more robustly than the default pip_install() { - pip install --progress-bar off "$@" + retry pip install --progress-bar off "$@" } # Install torch with pip, respecting PYTORCH_VERSION, and record the installed # version into PYTORCH_VERSION, if applicable -pip_install_torch() { +setup_pip_pytorch_version() { if [[ -z "$PYTORCH_VERSION" ]]; then # Install latest prerelease CPU version of torch, per our nightlies. pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html @@ -55,11 +60,24 @@ pip_install_torch() { # strip off the local package qualifier. export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" else - # NB: We include the nightly channel to, since sometimes we stage - # prereleases in it. Those releases should get moved to stable - # when they're ready + # TODO: Maybe add staging too pip_install "torch==$PYTORCH_VERSION" \ - -f https://download.pytorch.org/whl/torch_stable.html \ - -f https://download.pytorch.org/whl/nightly/torch_nightly.html + -f https://download.pytorch.org/whl/torch_stable.html fi } + +# Fill PYTORCH_VERSION with the latest conda nightly version, and +# CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions +setup_conda_pytorch_version() { + if [[ -z "$PYTORCH_VERSION" ]]; then + export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export PYTORCH_VERSION=$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version'])") + else + export CONDA_CHANNEL_FLAGS="-c pytorch" + fi +} + +# Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT +setup_conda_cudatoolkit_constraint() { + export CONDA_CUDATOOLKIT_CONSTRAINT="" +} diff --git a/packaging/conda/torchaudio/build.sh b/packaging/torchaudio/build.sh similarity index 100% rename from packaging/conda/torchaudio/build.sh rename to packaging/torchaudio/build.sh diff --git a/packaging/conda/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml similarity index 100% rename from packaging/conda/torchaudio/meta.yaml rename to packaging/torchaudio/meta.yaml From 848792e6da8059d23228179c34461fa7e4c07143 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 19:28:27 -0700 Subject: [PATCH 19/27] Conda online Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 22 +++------------- packaging/build_conda.sh | 5 ++-- packaging/pkg_helpers.bash | 48 +++++++++++++++++++++++++++++++--- packaging/torchaudio/build.sh | 2 +- packaging/torchaudio/meta.yaml | 12 ++++----- 5 files changed, 56 insertions(+), 33 deletions(-) mode change 100644 => 100755 packaging/build_conda.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 19ced9da65..9f22df36e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,7 @@ version: 2.1 binary_common: &binary_common parameters: + # Edit these defaults to do a release` build_version: description: "version number of release binary; by default, build a nightly" type: string @@ -15,6 +16,7 @@ binary_common: &binary_common description: "PyTorch version to build against; by default, use a nightly" type: string default: "" + # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)" type: string @@ -27,6 +29,7 @@ binary_common: &binary_common BUILD_VERSION: << parameters.build_version >> PYTORCH_VERSION: << parameters.pytorch_version >> UNICODE_ABI: << parameters.unicode_abi >> + CUDA_VERSION: cpu jobs: binary_linux_wheel: @@ -82,10 +85,6 @@ jobs: - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 -release_parameters: &release_parameters - version: "0.3.0" - pytorch_version: "1.2.0" - workflows: build: jobs: @@ -152,18 +151,3 @@ workflows: - binary_macos_conda: name: nightly_binary_macos_conda_py37 python_version: "3.7" - -# release: -# jobs: -# - binary_linux_wheel: -# <<: *release_parameters -# name: release_binary_linux_wheel -# - binary_linux_conda: -# <<: *release_parameters -# name: release_binary_linux_conda -# - binary_macos_wheel: -# <<: *release_parameters -# name: release_binary_macos_wheel -# - binary_macos_conda: -# <<: *release_parameters -# name: release_binary_macos_conda diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh old mode 100644 new mode 100755 index 9f272e9b42..46382eb422 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -1,4 +1,4 @@ -#!/bin/bbash +#!/bin/bash set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -7,7 +7,6 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" setup_build_version 0.4.0 setup_macos export SOURCE_ROOT_DIR="$PWD" -setup_conda_pytorch_version -setup conda_cudatoolkit_constraint +setup_conda_pytorch_constraint conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 135cbbfcb2..47b7c3939a 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -20,12 +20,36 @@ setup_python() { 3.5) python_abi=cp35-cp35m ;; 3.6) python_abi=cp36-cp36m ;; 3.7) python_abi=cp37-cp37m ;; + *) + echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" + exit 1 + ;; esac export PATH="/opt/python/$python_abi/bin:$PATH" fi } -# Fill BUILD_VERSION if it doesn't exist already with a nightly string +# Fill CUDA_SUFFIX with CUDA_VERSION, but don't fill it for the default +# CUDA version (that's a blank suffix) +setup_cuda_suffix() { + case "$CUDA_VERSION" in + 10.0) + export CUDA_SUFFIX="" ;; # default! + 9.2) + export CUDA_SUFFIX="+cu92" ;; + cpu) + export CUDA_SUFFIX="+cpu" ;; + *) + echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" + esac +} + +# If a package is cpu-only, we never provide a cuda suffix +setup_cpuonly_cuda_suffix() { + export CUDA_SUFFIX="" +} + +# Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string # Usage: setup_build_version 0.2 setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then @@ -68,16 +92,32 @@ setup_pip_pytorch_version() { # Fill PYTORCH_VERSION with the latest conda nightly version, and # CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions -setup_conda_pytorch_version() { +# +# You MUST have populated CUDA_SUFFIX before hand. +# +# TODO: This is currently hard-coded for CPU-only case +setup_conda_pytorch_constraint() { if [[ -z "$PYTORCH_VERSION" ]]; then export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" - export PYTORCH_VERSION=$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version'])") + export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else export CONDA_CHANNEL_FLAGS="-c pytorch" fi + if [[ "$CUDA_VERSION" == cpu ]]; then + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION+cpu" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" + else + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" + fi } # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT setup_conda_cudatoolkit_constraint() { - export CONDA_CUDATOOLKIT_CONSTRAINT="" + if [[ "$CUDA_VERSION" == cpu ]]; then + export CONDA_CUDATOOLKIT_CONSTRAINT="" + else + echo + # TODO + fi } diff --git a/packaging/torchaudio/build.sh b/packaging/torchaudio/build.sh index 138826b69a..dff89cf31b 100644 --- a/packaging/torchaudio/build.sh +++ b/packaging/torchaudio/build.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -ex -build_tools/packaging/build_from_source.sh "$(pwd)" +packaging/build_from_source.sh "$(pwd)" IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt diff --git a/packaging/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml index 735019e0db..0b89969fa2 100644 --- a/packaging/torchaudio/meta.yaml +++ b/packaging/torchaudio/meta.yaml @@ -1,9 +1,9 @@ package: name: torchaudio - version: "{{ environ.get('TORCHAUDIO_BUILD_VERSION') }}" + version: "{{ environ.get('BUILD_VERSION') }}" source: - path: "{{ environ.get('TORCHAUDIO_GITHUB_ROOT_DIR') }}" + path: "{{ environ.get('SOURCE_ROOT_DIR') }}" requirements: build: @@ -13,16 +13,16 @@ requirements: host: - python - setuptools -{{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} + - cpuonly + {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} run: - python - typing -{{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} + {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} build: - number: {{ environ.get('TORCHAUDIO_BUILD_NUMBER') }} - string: py{{py}}_{{environ.get('TORCHAUDIO_BUILD_NUMBER')}} + string: py{{py}} test: imports: From c8f5d09c501380d4e587a6d0efcd9a089f3612bb Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 19:50:37 -0700 Subject: [PATCH 20/27] cpu suffix bugfix on osx Signed-off-by: Edward Z. Yang --- packaging/pkg_helpers.bash | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 47b7c3939a..5256b168b8 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -32,21 +32,31 @@ setup_python() { # Fill CUDA_SUFFIX with CUDA_VERSION, but don't fill it for the default # CUDA version (that's a blank suffix) setup_cuda_suffix() { - case "$CUDA_VERSION" in - 10.0) - export CUDA_SUFFIX="" ;; # default! - 9.2) - export CUDA_SUFFIX="+cu92" ;; - cpu) - export CUDA_SUFFIX="+cpu" ;; - *) - echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" - esac + if [[ "$(uname)" == Darwin ]]; then + if [[ "$CUDA_VERSION" != "cpu" ]]; then + echo "CUDA_VERSION on OS X must be cpu" + exit 1 + fi + export CPU_SUFFIX="" + else + case "$CUDA_VERSION" in + 10.0) + export CUDA_SUFFIX="" ;; # default! + 9.2) + export CUDA_SUFFIX="+cu92" ;; + cpu) + export CUDA_SUFFIX="+cpu" ;; + *) + echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" + esac + export CPU_SUFFIX="+cpu" + fi } # If a package is cpu-only, we never provide a cuda suffix setup_cpuonly_cuda_suffix() { export CUDA_SUFFIX="" + export CPU_SUFFIX="" } # Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string @@ -104,7 +114,7 @@ setup_conda_pytorch_constraint() { export CONDA_CHANNEL_FLAGS="-c pytorch" fi if [[ "$CUDA_VERSION" == cpu ]]; then - export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION+cpu" + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${CPU_SUFFIX}" export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" else export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" From 10b3240c5dbf22d33487e51bcdaaec90f33683db Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 19:59:23 -0700 Subject: [PATCH 21/27] Update README Signed-off-by: Edward Z. Yang --- packaging/README.md | 21 +++++--------- packaging/conda/build_audio.sh | 42 ---------------------------- packaging/conda/get-latest.py | 8 ------ packaging/wheel/build_wheel.sh | 44 ------------------------------ packaging/wheel/linux_manywheel.sh | 21 -------------- packaging/wheel/osx_wheel.sh | 34 ----------------------- 6 files changed, 7 insertions(+), 163 deletions(-) delete mode 100755 packaging/conda/build_audio.sh delete mode 100644 packaging/conda/get-latest.py delete mode 100755 packaging/wheel/build_wheel.sh delete mode 100755 packaging/wheel/linux_manywheel.sh delete mode 100755 packaging/wheel/osx_wheel.sh diff --git a/packaging/README.md b/packaging/README.md index a0525d84e2..f3ee62e62c 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -5,11 +5,9 @@ ### Linux ```bash -# NB: We don't actually need CUDA; you don't have to run this on nvidia-docker -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash -pushd remote/conda -conda config --add channels pytorch -./build_audio.sh +docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash +cd remote +PYTHON_VERSION=3.7 packaging/build_conda.sh ``` To install bz2, @@ -31,11 +29,7 @@ anaconda upload -u pytorch /opt/conda/conda-bld/linux-64/torchaudio*.bz2 ```bash # create a fresh anaconda environment / install and activate it -cd packaging/conda -conda install -y conda-build anaconda-client -conda config --add channels pytorch -conda config --add channels conda-forge -./build_audio.sh +PYTHON_VERSION=3.7 packaging/build_conda.sh ``` To install bz2, @@ -63,8 +57,8 @@ anaconda upload -u pytorch /Users/jamarshon/anaconda3/conda-bld/osx-64/torchaudi ```bash nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash -cd remote/wheel -./linux_manywheel.sh cpu +cd remote +PYTHON_VERSION=3.7 packaging/build_wheel.sh ``` To install wheels, @@ -83,8 +77,7 @@ cd ../cpu ### OSX ```bash -pushd wheel -./osx_wheel.sh +PYTHON_VERSION=3.7 packaging/build_wheel.sh ``` To install wheels, diff --git a/packaging/conda/build_audio.sh b/packaging/conda/build_audio.sh deleted file mode 100755 index 9afdd14475..0000000000 --- a/packaging/conda/build_audio.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -if [[ -x "/remote/anaconda_token" ]]; then - . /remote/anaconda_token || true -fi - -set -ex - - # Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -if [[ -z "$TORCHAUDIO_BUILD_VERSION" ]]; then - export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" -fi -if [[ -z "$TORCHAUDIO_BUILD_NUMBER" ]]; then - export TORCHAUDIO_BUILD_NUMBER=1 -fi - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -. "$script_dir/../setup_workdir" -export TORCHAUDIO_GITHUB_ROOT_DIR="$WORKDIR" - -cd "$script_dir" - -ANACONDA_USER=pytorch-nightly -conda config --set anaconda_upload no -conda config --set channel_priority strict - -LATEST_PYTORCH_NIGHTLY_VERSION=$(conda search --json 'pytorch[channel=pytorch-nightly]' | python "$script_dir/get-latest.py") -export CONDA_PYTORCH_CONSTRAINT=" - pytorch ==${LATEST_PYTORCH_NIGHTLY_VERSION}" -export CONDA_CUDATOOLKIT_CONSTRAINT="" -export CUDA_VERSION="None" -if [[ "$OSTYPE" == "darwin"* ]]; then - export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ -fi - -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchaudio diff --git a/packaging/conda/get-latest.py b/packaging/conda/get-latest.py deleted file mode 100644 index d2418d9409..0000000000 --- a/packaging/conda/get-latest.py +++ /dev/null @@ -1,8 +0,0 @@ -# Functionally equivalent to: -# jq -r '.["pytorch"][-1].version' | sed 's/+.*$//' - -import sys -import json -import re - -print(re.sub(r'\+.*$', '', json.load(sys.stdin)["pytorch"][-1]["version"])) diff --git a/packaging/wheel/build_wheel.sh b/packaging/wheel/build_wheel.sh deleted file mode 100755 index b52fb7503d..0000000000 --- a/packaging/wheel/build_wheel.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -set -ex - -# Assume we are in a valid Python environment that we want to build the wheel in - -if [[ -z "$OUT_DIR" ]]; then - export OUT_DIR="/tmp/torchaudio-build" -fi -if [[ -z "$TORCHAUDIO_BUILD_VERSION" ]]; then - export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" -fi -if [[ -z "$TORCHAUDIO_BUILD_NUMBER" ]]; then - export TORCHAUDIO_BUILD_NUMBER="1" -fi -if [[ "$(uname)" == Darwin ]]; then - export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ -fi - -if [[ -z "$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" ]]; then - is_nightly=1 # to unset later - pip install --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we - # strip off the local package qualifier. Also, we choose to build against - # the CPU build, because it takes less time to download. - export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" -else - is_nightly= - # NB: We include the nightly channel to, since sometimes we stage - # prereleases in it. Those releases should get moved to stable - # when they're ready - pip install --progress-bar off "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" \ - -f https://download.pytorch.org/whl/torch_stable.html \ - -f https://download.pytorch.org/whl/nightly/torch_nightly.html -fi -echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" - -# NB: do not actually install requirements.txt; that is only needed for -# testing -pip install --progress-bar off numpy future -IS_WHEEL=1 python setup.py clean -IS_WHEEL=1 python setup.py bdist_wheel -mkdir -p $OUT_DIR -cp dist/*.whl $OUT_DIR/ diff --git a/packaging/wheel/linux_manywheel.sh b/packaging/wheel/linux_manywheel.sh deleted file mode 100755 index f18ac1099d..0000000000 --- a/packaging/wheel/linux_manywheel.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -ex -export OUT_DIR="/remote/cpu" - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -cd /opt/python -DESIRED_PYTHON=(*/) -for desired_py in "${DESIRED_PYTHON[@]}"; do - python_installations+=("/opt/python/$desired_py") -done - -. "$script_dir/../setup_workdir" - -OLD_PATH=$PATH -for PYDIR in "${python_installations[@]}"; do - export PATH="$PYDIR/bin:$OLD_PATH" - pip install --upgrade pip - "$script_dir/build_wheel.sh" -done diff --git a/packaging/wheel/osx_wheel.sh b/packaging/wheel/osx_wheel.sh deleted file mode 100755 index 0dd0335051..0000000000 --- a/packaging/wheel/osx_wheel.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -ex - -if [[ ":$PATH:" == *"conda"* ]]; then - echo "existing anaconda install in PATH, remove it and run script" - exit 1 -fi -# download and activate anaconda -rm -rf ~/minconda_wheel_env_tmp -curl -o Miniconda3-latest-MacOSX-x86_64.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ - chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \ - ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \ - rm Miniconda3-latest-MacOSX-x86_64.sh - -. ~/minconda_wheel_env_tmp/bin/activate - -export OUT_DIR=~/torchaudio_wheels - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -. "$script_dir/../setup_workdir" - -desired_pythons=( "2.7" "3.5" "3.6" "3.7" ) -# for each python -for desired_python in "${desired_pythons[@]}" -do - # create and activate python env - env_name="env$desired_python" - conda create -yn $env_name python="$desired_python" - conda activate $env_name - - "$script_dir/build_wheel.sh" -done From 8dcab8f2afcb0b4c3452019767c4d9dcc0f53173 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 20:04:42 -0700 Subject: [PATCH 22/27] Remove dead file Signed-off-by: Edward Z. Yang --- packaging/setup_workdir | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 packaging/setup_workdir diff --git a/packaging/setup_workdir b/packaging/setup_workdir deleted file mode 100644 index c995ca9513..0000000000 --- a/packaging/setup_workdir +++ /dev/null @@ -1,24 +0,0 @@ -# Checkout or reuse existing work directory based on TARGET_COMMIT, -# build our static dependencies, and cd into the workdir -# -# Requires script_dir to be set, to a directory one BELOW this -# script (satisfied by conda/ and wheel/) Yeah this is hella -# janky - -if [[ "$TARGET_COMMIT" == HEAD ]]; then - # Assume that this script was called from a valid checkout - WORKDIR="$script_dir/../../.." -else - WORKDIR="/tmp/audio" - cd /tmp - rm -rf audio - git clone https://github.com/pytorch/audio - cd audio - git checkout "$TARGET_COMMIT" - git submodule update --init --recursive -fi - -mkdir "$WORKDIR/third_party" -"$script_dir/../build_from_source.sh" "$WORKDIR" - -cd "$WORKDIR" From 212445fe3a5d6d581336ce7232bb70e131beffda Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 20:14:04 -0700 Subject: [PATCH 23/27] Nuke old copy of env if it already exists Signed-off-by: Edward Z. Yang --- packaging/pkg_helpers.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 5256b168b8..0265d5bd60 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -6,6 +6,7 @@ setup_python() { if [[ "$(uname)" == Darwin ]]; then eval "$(conda shell.bash hook)" + conda env remove -n "env$PYTHON_VERSION" || true conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" conda activate "env$PYTHON_VERSION" else From 6fa39e9c984fdbef0b2fd8032e424172d162e395 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 20:53:09 -0700 Subject: [PATCH 24/27] a little jinja Signed-off-by: Edward Z. Yang --- .circleci/{config.yml => config.yml.in} | 88 ++++++++----------------- .circleci/regenerate.py | 11 ++++ .gitignore | 1 + 3 files changed, 38 insertions(+), 62 deletions(-) rename .circleci/{config.yml => config.yml.in} (56%) create mode 100644 .circleci/regenerate.py diff --git a/.circleci/config.yml b/.circleci/config.yml.in similarity index 56% rename from .circleci/config.yml rename to .circleci/config.yml.in index 9f22df36e6..d7a8ed3eb4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml.in @@ -32,6 +32,17 @@ binary_common: &binary_common CUDA_VERSION: cpu jobs: + ensure_consistency: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 + python .circleci/regenerate.py + git diff || (echo "Run .circleci/regenerate.py to update config"; exit 1) + binary_linux_wheel: <<: *binary_common docker: @@ -88,66 +99,19 @@ jobs: workflows: build: jobs: - # Linux wheels - - binary_linux_wheel: - name: nightly_binary_linux_wheel_cp27m - python_version: "2.7" - - binary_linux_wheel: - name: nightly_binary_linux_wheel_cp27mu - python_version: "2.7" + - ensure_consistency +{%- for btype in ["wheel", "conda"] -%} +{%- for os in ["linux", "macos"] -%} +{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] %} + - binary_{{os}}_{{btype}}: + name: binary_{{os}}_{{btype}}_py{{python_version}} + python_version: "{{python_version}}" +{%- if btype == "wheel" and python_version == "2.7" %} + - binary_{{os}}_{{btype}}: + name: binary_{{os}}_{{btype}}_py{{python_version}}_unicode + python_version: "{{python_version}}" unicode_abi: "1" - - binary_linux_wheel: - name: nightly_binary_linux_wheel_cp35 - python_version: "3.5" - - binary_linux_wheel: - name: nightly_binary_linux_wheel_cp36 - python_version: "3.6" - - binary_linux_wheel: - name: nightly_binary_linux_wheel_cp37 - python_version: "3.7" - - # Linux conda - - binary_linux_conda: - name: nightly_binary_linux_conda_py27 - python_version: "2.7" - - binary_linux_conda: - name: nightly_binary_linux_conda_py35 - python_version: "3.5" - - binary_linux_conda: - name: nightly_binary_linux_conda_py36 - python_version: "3.6" - - binary_linux_conda: - name: nightly_binary_linux_conda_py37 - python_version: "3.7" - - # Macos wheel - - binary_macos_wheel: - name: nightly_binary_macos_wheel_cp27m - python_version: "2.7" - - binary_macos_wheel: - name: nightly_binary_macos_wheel_cp27mu - python_version: "2.7" - unicode_abi: "1" - - binary_macos_wheel: - name: nightly_binary_macos_wheel_cp35 - python_version: "3.5" - - binary_macos_wheel: - name: nightly_binary_macos_wheel_cp36 - python_version: "3.6" - - binary_macos_wheel: - name: nightly_binary_macos_wheel_cp37 - python_version: "3.7" - - # Macos conda - - binary_macos_conda: - name: nightly_binary_macos_conda_py27 - python_version: "2.7" - - binary_macos_conda: - name: nightly_binary_macos_conda_py35 - python_version: "3.5" - - binary_macos_conda: - name: nightly_binary_macos_conda_py36 - python_version: "3.6" - - binary_macos_conda: - name: nightly_binary_macos_conda_py37 - python_version: "3.7" +{%- endif -%} +{%- endfor -%} +{%- endfor -%} +{%- endfor -%} diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py new file mode 100644 index 0000000000..8001ac0e64 --- /dev/null +++ b/.circleci/regenerate.py @@ -0,0 +1,11 @@ +import jinja2 +import os.path + +d = os.path.dirname(__file__) +env = jinja2.Environment( + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=False, +) +with open(os.path.join(d, 'config.yml'), 'w') as f: + f.write(env.get_template('config.yml.in').render()) diff --git a/.gitignore b/.gitignore index 98b0764cdc..131f2213c9 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,4 @@ ENV/ test/assets/sinewave.wav torchaudio/version.py gen.yml +.circleci/config.yml From 63078cdaf629cd2ad36e6f78cb298ea6742da918 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 20:55:59 -0700 Subject: [PATCH 25/27] Rename Signed-off-by: Edward Z. Yang --- .circleci/config.yml.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index d7a8ed3eb4..954de3588d 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -32,7 +32,7 @@ binary_common: &binary_common CUDA_VERSION: cpu jobs: - ensure_consistency: + circleci_consistency: docker: - image: circleci/python:3.7 steps: @@ -41,7 +41,7 @@ jobs: command: | pip install --user --progress-bar off jinja2 python .circleci/regenerate.py - git diff || (echo "Run .circleci/regenerate.py to update config"; exit 1) + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) binary_linux_wheel: <<: *binary_common @@ -99,7 +99,7 @@ jobs: workflows: build: jobs: - - ensure_consistency + - circleci_consistency {%- for btype in ["wheel", "conda"] -%} {%- for os in ["linux", "macos"] -%} {%- for python_version in ["2.7", "3.5", "3.6", "3.7"] %} From 8c771d26f9aa3e74bc30b744b590bf0d3d2b3723 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 21:00:03 -0700 Subject: [PATCH 26/27] Fix correctly Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 160 ++++++++++++++++++++++++++++++++++++++++ .circleci/config.yml.in | 30 ++++---- .circleci/regenerate.py | 10 ++- .gitignore | 1 - 4 files changed, 182 insertions(+), 19 deletions(-) create mode 100644 .circleci/config.yml mode change 100644 => 100755 .circleci/regenerate.py diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..ce88220a3c --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,160 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel +# - Replace binary_linux_wheel with the name of the job you want to test + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release` + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CUDA_VERSION: cpu + +commands: + setup_conda: + description: "Install conda in the environment" + steps: + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "soumith/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_linux_conda: + <<: *binary_common + docker: + - image: "soumith/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - setup_conda + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - setup_conda + - run: conda install -yq conda-build + - run: packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + +workflows: + build: + jobs: + - circleci_consistency + - binary_linux_wheel: + name: binary_linux_wheel_py2.7 + python_version: "2.7" + - binary_linux_wheel: + name: binary_linux_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_linux_wheel: + name: binary_linux_wheel_py3.5 + python_version: "3.5" + - binary_linux_wheel: + name: binary_linux_wheel_py3.6 + python_version: "3.6" + - binary_linux_wheel: + name: binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_macos_wheel: + name: binary_macos_wheel_py2.7 + python_version: "2.7" + - binary_macos_wheel: + name: binary_macos_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_macos_wheel: + name: binary_macos_wheel_py3.5 + python_version: "3.5" + - binary_macos_wheel: + name: binary_macos_wheel_py3.6 + python_version: "3.6" + - binary_macos_wheel: + name: binary_macos_wheel_py3.7 + python_version: "3.7" + - binary_linux_conda: + name: binary_linux_conda_py2.7 + python_version: "2.7" + - binary_linux_conda: + name: binary_linux_conda_py3.5 + python_version: "3.5" + - binary_linux_conda: + name: binary_linux_conda_py3.6 + python_version: "3.6" + - binary_linux_conda: + name: binary_linux_conda_py3.7 + python_version: "3.7" + - binary_macos_conda: + name: binary_macos_conda_py2.7 + python_version: "2.7" + - binary_macos_conda: + name: binary_macos_conda_py3.5 + python_version: "3.5" + - binary_macos_conda: + name: binary_macos_conda_py3.6 + python_version: "3.6" + - binary_macos_conda: + name: binary_macos_conda_py3.7 + python_version: "3.7" \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 954de3588d..7443d22315 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -31,6 +31,16 @@ binary_common: &binary_common UNICODE_ABI: << parameters.unicode_abi >> CUDA_VERSION: cpu +commands: + setup_conda: + description: "Install conda in the environment" + steps: + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + jobs: circleci_consistency: docker: @@ -71,14 +81,10 @@ jobs: xcode: "9.0" steps: - checkout - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh + - setup_conda + - run: packaging/build_wheel.sh - store_artifacts: - path: /Users/distiller/torchaudio_wheels + path: dist binary_macos_conda: <<: *binary_common @@ -86,13 +92,9 @@ jobs: xcode: "9.0" steps: - checkout - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build - packaging/build_conda.sh + - setup_conda + - run: conda install -yq conda-build + - run: packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py old mode 100644 new mode 100755 index 8001ac0e64..dcc3fb2c23 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -1,11 +1,13 @@ +#!/usr/bin/env python + import jinja2 import os.path d = os.path.dirname(__file__) env = jinja2.Environment( - loader=jinja2.FileSystemLoader(d), - lstrip_blocks=True, - autoescape=False, + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=False, ) with open(os.path.join(d, 'config.yml'), 'w') as f: - f.write(env.get_template('config.yml.in').render()) + f.write(env.get_template('config.yml.in').render()) diff --git a/.gitignore b/.gitignore index 131f2213c9..98b0764cdc 100644 --- a/.gitignore +++ b/.gitignore @@ -110,4 +110,3 @@ ENV/ test/assets/sinewave.wav torchaudio/version.py gen.yml -.circleci/config.yml From e47cadde3e1392503a802ccf26f9d2f3be2b09c5 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 6 Aug 2019 21:03:43 -0700 Subject: [PATCH 27/27] fixup Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 31 ++++++++++++++++--------------- .circleci/config.yml.in | 31 ++++++++++++++++--------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce88220a3c..222195c24c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,16 +31,6 @@ binary_common: &binary_common UNICODE_ABI: << parameters.unicode_abi >> CUDA_VERSION: cpu -commands: - setup_conda: - description: "Install conda in the environment" - steps: - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - jobs: circleci_consistency: docker: @@ -81,8 +71,15 @@ jobs: xcode: "9.0" steps: - checkout - - setup_conda - - run: packaging/build_wheel.sh + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh - store_artifacts: path: dist @@ -92,9 +89,13 @@ jobs: xcode: "9.0" steps: - checkout - - setup_conda - - run: conda install -yq conda-build - - run: packaging/build_conda.sh + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 7443d22315..47bf1ce630 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -31,16 +31,6 @@ binary_common: &binary_common UNICODE_ABI: << parameters.unicode_abi >> CUDA_VERSION: cpu -commands: - setup_conda: - description: "Install conda in the environment" - steps: - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - jobs: circleci_consistency: docker: @@ -81,8 +71,15 @@ jobs: xcode: "9.0" steps: - checkout - - setup_conda - - run: packaging/build_wheel.sh + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh - store_artifacts: path: dist @@ -92,9 +89,13 @@ jobs: xcode: "9.0" steps: - checkout - - setup_conda - - run: conda install -yq conda-build - - run: packaging/build_conda.sh + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64