From 45d1da5c053c987862b8057a6aab646a10ed286c Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 00:18:45 -0400 Subject: [PATCH 1/6] Rewrite our packaging from scratch (#217) * New entry points are packaging/build_wheel.sh and packaging/build_conda.sh. The only mandatory environment variable you have to set is PYTHON_VERSION * CircleCI configuration uses 2.1-style parametrized builds to let you toggle python version, etc. as you do builds. We create a separate job per build configuration for maximum parallelism * build_tools/packaging got moved to packaging, to be in-line with directory structure in torchvision * The build_conda.sh and build_wheel.sh delegate most of the heavy lifting to pkg_helpers.bash, which defines a number of bash functions for performing common operations. The intent is that I'll copy-paste this file between other domain API projects. * TORCHAUDIO_ prefix removed from envvars, so that I can more easily share packaging scripts between projects * BUILD_VERSION is completely gone; just change the version number if you need to rebuild * No more logic for cloning and checking out a fresh copy of torchaudio --- .circleci/config.yml | 161 ++++++++++++++++++ .circleci/config.yml.in | 120 +++++++++++++ .circleci/regenerate.py | 13 ++ build_tools/packaging/conda/build_audio.sh | 48 ------ .../packaging/wheel/linux_manywheel.sh | 46 ----- build_tools/packaging/wheel/osx_wheel.sh | 48 ------ .../packaging => packaging}/README.md | 22 +-- packaging/build_conda.sh | 12 ++ .../wheel => packaging}/build_from_source.sh | 24 ++- packaging/build_wheel.sh | 14 ++ packaging/pkg_helpers.bash | 134 +++++++++++++++ .../conda => packaging}/torchaudio/build.sh | 3 +- .../conda => packaging}/torchaudio/meta.yaml | 12 +- setup.py | 16 +- 14 files changed, 492 insertions(+), 181 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .circleci/config.yml.in create mode 100755 .circleci/regenerate.py delete mode 100755 build_tools/packaging/conda/build_audio.sh delete mode 100755 build_tools/packaging/wheel/linux_manywheel.sh delete mode 100755 build_tools/packaging/wheel/osx_wheel.sh rename {build_tools/packaging => packaging}/README.md (77%) create mode 100755 packaging/build_conda.sh rename {build_tools/packaging/wheel => packaging}/build_from_source.sh (74%) create mode 100755 packaging/build_wheel.sh create mode 100644 packaging/pkg_helpers.bash rename {build_tools/packaging/conda => packaging}/torchaudio/build.sh (64%) rename {build_tools/packaging/conda => packaging}/torchaudio/meta.yaml (69%) diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..222195c24c --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,161 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel +# - Replace binary_linux_wheel with the name of the job you want to test + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release` + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CUDA_VERSION: cpu + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "soumith/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_linux_conda: + <<: *binary_common + docker: + - image: "soumith/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + +workflows: + build: + jobs: + - circleci_consistency + - binary_linux_wheel: + name: binary_linux_wheel_py2.7 + python_version: "2.7" + - binary_linux_wheel: + name: binary_linux_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_linux_wheel: + name: binary_linux_wheel_py3.5 + python_version: "3.5" + - binary_linux_wheel: + name: binary_linux_wheel_py3.6 + python_version: "3.6" + - binary_linux_wheel: + name: binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_macos_wheel: + name: binary_macos_wheel_py2.7 + python_version: "2.7" + - binary_macos_wheel: + name: binary_macos_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_macos_wheel: + name: binary_macos_wheel_py3.5 + python_version: "3.5" + - binary_macos_wheel: + name: binary_macos_wheel_py3.6 + python_version: "3.6" + - binary_macos_wheel: + name: binary_macos_wheel_py3.7 + python_version: "3.7" + - binary_linux_conda: + name: binary_linux_conda_py2.7 + python_version: "2.7" + - binary_linux_conda: + name: binary_linux_conda_py3.5 + python_version: "3.5" + - binary_linux_conda: + name: binary_linux_conda_py3.6 + python_version: "3.6" + - binary_linux_conda: + name: binary_linux_conda_py3.7 + python_version: "3.7" + - binary_macos_conda: + name: binary_macos_conda_py2.7 + python_version: "2.7" + - binary_macos_conda: + name: binary_macos_conda_py3.5 + python_version: "3.5" + - binary_macos_conda: + name: binary_macos_conda_py3.6 + python_version: "3.6" + - binary_macos_conda: + name: binary_macos_conda_py3.7 + python_version: "3.7" \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in new file mode 100644 index 0000000000..47bf1ce630 --- /dev/null +++ b/.circleci/config.yml.in @@ -0,0 +1,120 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel +# - Replace binary_linux_wheel with the name of the job you want to test + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release` + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CUDA_VERSION: cpu + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "soumith/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_linux_conda: + <<: *binary_common + docker: + - image: "soumith/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + +workflows: + build: + jobs: + - circleci_consistency +{%- for btype in ["wheel", "conda"] -%} +{%- for os in ["linux", "macos"] -%} +{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] %} + - binary_{{os}}_{{btype}}: + name: binary_{{os}}_{{btype}}_py{{python_version}} + python_version: "{{python_version}}" +{%- if btype == "wheel" and python_version == "2.7" %} + - binary_{{os}}_{{btype}}: + name: binary_{{os}}_{{btype}}_py{{python_version}}_unicode + python_version: "{{python_version}}" + unicode_abi: "1" +{%- endif -%} +{%- endfor -%} +{%- endfor -%} +{%- endfor -%} diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py new file mode 100755 index 0000000000..dcc3fb2c23 --- /dev/null +++ b/.circleci/regenerate.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import jinja2 +import os.path + +d = os.path.dirname(__file__) +env = jinja2.Environment( + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=False, +) +with open(os.path.join(d, 'config.yml'), 'w') as f: + f.write(env.get_template('config.yml.in').render()) diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh deleted file mode 100755 index 98518200d7..0000000000 --- a/build_tools/packaging/conda/build_audio.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -if [[ -x "/remote/anaconda_token" ]]; then - . /remote/anaconda_token || true -fi - -set -ex - - # Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER=1 - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -audio_rootdir="$(pwd)/torchaudio-src" - -if [[ "$BRANCH" == "" ]]; then - BRANCH=master -fi - -if [[ ! -d "$audio_rootdir" ]]; then - rm -rf "$audio_rootdir" - git clone "https://github.com/pytorch/audio" "$audio_rootdir" -b "$BRANCH" -fi - -export TORCHAUDIO_GITHUB_ROOT_DIR="$audio_rootdir" - -cd "$SOURCE_DIR" - -ANACONDA_USER=pytorch -conda config --set anaconda_upload no - -export CONDA_CUDATOOLKIT_CONSTRAINT="" -export CUDA_VERSION="None" -if [[ "$OSTYPE" == "darwin"* ]]; then - export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ -fi - -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchaudio -# Currently disabled, see https://github.com/pytorch/pytorch/issues/23738 -# time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchaudio - - set +e diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh deleted file mode 100755 index 17be4c280c..0000000000 --- a/build_tools/packaging/wheel/linux_manywheel.sh +++ /dev/null @@ -1,46 +0,0 @@ -if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters. Pass cuda version" - echo "CUDA version should be cu90, cu100 or cpu" - exit 1 -fi -export CUVER="$1" # cu90 cu100 cpu - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER="1" -export OUT_DIR="/remote/$CUVER" - -cd /opt/python -DESIRED_PYTHON=(*/) -for desired_py in "${DESIRED_PYTHON[@]}"; do - python_installations+=("/opt/python/$desired_py") -done - -OLD_PATH=$PATH -cd /tmp -rm -rf audio -git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION} -mkdir audio/third_party - -export PREFIX="/tmp/audio" -. /remote/wheel/build_from_source.sh - -cd /tmp/audio - -for PYDIR in "${python_installations[@]}"; do - export PATH=$PYDIR/bin:$OLD_PATH - pip install --upgrade pip - - # For true hermetic builds, you ought to be constructing the docker - # from scratch each time. But this makes things marginally safer if - # you aren't doing this. - pip uninstall -y torch || true - pip uninstall -y torch_nightly || true - - # NB: do not actually install requirements.txt; that is only needed for - # testing - pip install torch numpy future - IS_WHEEL=1 python setup.py clean - IS_WHEEL=1 python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh deleted file mode 100755 index 9e23057861..0000000000 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ /dev/null @@ -1,48 +0,0 @@ -if [[ ":$PATH:" == *"conda"* ]]; then - echo "existing anaconda install in PATH, remove it and run script" - exit 1 -fi -# download and activate anaconda -rm -rf ~/minconda_wheel_env_tmp -wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ - chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \ - ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \ - rm Miniconda3-latest-MacOSX-x86_64.sh - -. ~/minconda_wheel_env_tmp/bin/activate - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER="1" -export OUT_DIR=~/torchaudio_wheels - -export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ - -# TODO remove when pytorch is good https://github.com/pytorch/pytorch/issues/20030 -brew install libomp -CURR_PATH=$(pwd) - -cd /tmp -rm -rf audio -git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION} -mkdir audio/third_party - -export PREFIX="/tmp/audio" -. $CURR_PATH/build_from_source.sh - -cd /tmp/audio - -desired_pythons=( "2.7" "3.5" "3.6" "3.7" ) -# for each python -for desired_python in "${desired_pythons[@]}" -do - # create and activate python env - env_name="env$desired_python" - conda create -yn $env_name python="$desired_python" - conda activate $env_name - - pip install torch numpy future - IS_WHEEL=1 python setup.py clean - IS_WHEEL=1 python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done diff --git a/build_tools/packaging/README.md b/packaging/README.md similarity index 77% rename from build_tools/packaging/README.md rename to packaging/README.md index 6d2efc6634..f3ee62e62c 100644 --- a/build_tools/packaging/README.md +++ b/packaging/README.md @@ -5,12 +5,9 @@ ### Linux ```bash -# NB: We don't actually need CUDA; you don't have to run this on nvidia-docker -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash -pushd remote/conda -conda config --add channels pytorch -conda config --add channels conda-forge -./build_audio.sh +docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash +cd remote +PYTHON_VERSION=3.7 packaging/build_conda.sh ``` To install bz2, @@ -32,11 +29,7 @@ anaconda upload -u pytorch /opt/conda/conda-bld/linux-64/torchaudio*.bz2 ```bash # create a fresh anaconda environment / install and activate it -cd packaging/conda -conda install -y conda-build anaconda-client -conda config --add channels pytorch -conda config --add channels conda-forge -./build_audio.sh +PYTHON_VERSION=3.7 packaging/build_conda.sh ``` To install bz2, @@ -64,8 +57,8 @@ anaconda upload -u pytorch /Users/jamarshon/anaconda3/conda-bld/osx-64/torchaudi ```bash nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash -cd remote/wheel -./linux_manywheel.sh cpu +cd remote +PYTHON_VERSION=3.7 packaging/build_wheel.sh ``` To install wheels, @@ -84,8 +77,7 @@ cd ../cpu ### OSX ```bash -pushd wheel -./osx_wheel.sh +PYTHON_VERSION=3.7 packaging/build_wheel.sh ``` To install wheels, diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh new file mode 100755 index 0000000000..46382eb422 --- /dev/null +++ b/packaging/build_conda.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +setup_build_version 0.4.0 +setup_macos +export SOURCE_ROOT_DIR="$PWD" +setup_conda_pytorch_constraint +conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio + diff --git a/build_tools/packaging/wheel/build_from_source.sh b/packaging/build_from_source.sh similarity index 74% rename from build_tools/packaging/wheel/build_from_source.sh rename to packaging/build_from_source.sh index eacd890852..3e16da5ede 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -1,11 +1,19 @@ -rm -rf source_code -mkdir source_code -pushd source_code +#!/bin/bash -wget -q -O sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -wget -q -O lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -wget -q -O flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -wget -q -O libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +set -ex + +# Arguments: PREFIX, specifying where to install dependencies into + +PREFIX="$1" + +rm -rf /tmp/torchaudio-deps +mkdir /tmp/torchaudio-deps +pushd /tmp/torchaudio-deps + +curl -L -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -L -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -L -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -L -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 @@ -43,7 +51,7 @@ pushd sox-14.4.2 ./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/sox" \ LDFLAGS="-L$PREFIX/third_party/lame/lib -L$PREFIX/third_party/flac/lib -L$PREFIX/third_party/mad/lib" \ CPPFLAGS="-I$PREFIX/third_party/lame/include -I$PREFIX/third_party/flac/include -I$PREFIX/third_party/mad/include" \ - --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking + --with-lame --with-flac --with-mad --without-coreaudio --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh new file mode 100755 index 0000000000..8871629c83 --- /dev/null +++ b/packaging/build_wheel.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +setup_python +setup_build_version 0.4.0 +setup_macos +"$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies +pip_install numpy future +setup_pip_pytorch_version +python setup.py clean +IS_WHEEL=1 python setup.py bdist_wheel diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash new file mode 100644 index 0000000000..0265d5bd60 --- /dev/null +++ b/packaging/pkg_helpers.bash @@ -0,0 +1,134 @@ +# A set of useful bash functions for common functionality we need to do in +# many build scripts + +# Respecting PYTHON_VERSION and UNICODE_ABI, add (or install) the correct +# version of Python to perform a build. Relevant to wheel builds. +setup_python() { + if [[ "$(uname)" == Darwin ]]; then + eval "$(conda shell.bash hook)" + conda env remove -n "env$PYTHON_VERSION" || true + conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" + conda activate "env$PYTHON_VERSION" + else + case "$PYTHON_VERSION" in + 2.7) + if [[ -n "$UNICODE_ABI" ]]; then + python_abi=cp27-cp27mu + else + python_abi=cp27-cp27m + fi + ;; + 3.5) python_abi=cp35-cp35m ;; + 3.6) python_abi=cp36-cp36m ;; + 3.7) python_abi=cp37-cp37m ;; + *) + echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" + exit 1 + ;; + esac + export PATH="/opt/python/$python_abi/bin:$PATH" + fi +} + +# Fill CUDA_SUFFIX with CUDA_VERSION, but don't fill it for the default +# CUDA version (that's a blank suffix) +setup_cuda_suffix() { + if [[ "$(uname)" == Darwin ]]; then + if [[ "$CUDA_VERSION" != "cpu" ]]; then + echo "CUDA_VERSION on OS X must be cpu" + exit 1 + fi + export CPU_SUFFIX="" + else + case "$CUDA_VERSION" in + 10.0) + export CUDA_SUFFIX="" ;; # default! + 9.2) + export CUDA_SUFFIX="+cu92" ;; + cpu) + export CUDA_SUFFIX="+cpu" ;; + *) + echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" + esac + export CPU_SUFFIX="+cpu" + fi +} + +# If a package is cpu-only, we never provide a cuda suffix +setup_cpuonly_cuda_suffix() { + export CUDA_SUFFIX="" + export CPU_SUFFIX="" +} + +# Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string +# Usage: setup_build_version 0.2 +setup_build_version() { + if [[ -z "$BUILD_VERSION" ]]; then + export BUILD_VERSION="$1.dev$(date "+%Y%m%d")" + fi +} + +# Set some useful variables for OS X, if applicable +setup_macos() { + if [[ "$(uname)" == Darwin ]]; then + export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ + fi +} + +# Function to retry functions that sometimes timeout or have flaky failures +retry () { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +# Install with pip a bit more robustly than the default +pip_install() { + retry pip install --progress-bar off "$@" +} + +# Install torch with pip, respecting PYTORCH_VERSION, and record the installed +# version into PYTORCH_VERSION, if applicable +setup_pip_pytorch_version() { + if [[ -z "$PYTORCH_VERSION" ]]; then + # Install latest prerelease CPU version of torch, per our nightlies. + pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we + # strip off the local package qualifier. + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" + else + # TODO: Maybe add staging too + pip_install "torch==$PYTORCH_VERSION" \ + -f https://download.pytorch.org/whl/torch_stable.html + fi +} + +# Fill PYTORCH_VERSION with the latest conda nightly version, and +# CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions +# +# You MUST have populated CUDA_SUFFIX before hand. +# +# TODO: This is currently hard-coded for CPU-only case +setup_conda_pytorch_constraint() { + if [[ -z "$PYTORCH_VERSION" ]]; then + export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" + else + export CONDA_CHANNEL_FLAGS="-c pytorch" + fi + if [[ "$CUDA_VERSION" == cpu ]]; then + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${CPU_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" + else + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" + fi +} + +# Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT +setup_conda_cudatoolkit_constraint() { + if [[ "$CUDA_VERSION" == cpu ]]; then + export CONDA_CUDATOOLKIT_CONSTRAINT="" + else + echo + # TODO + fi +} diff --git a/build_tools/packaging/conda/torchaudio/build.sh b/packaging/torchaudio/build.sh similarity index 64% rename from build_tools/packaging/conda/torchaudio/build.sh rename to packaging/torchaudio/build.sh index 7e5917060f..dff89cf31b 100644 --- a/build_tools/packaging/conda/torchaudio/build.sh +++ b/packaging/torchaudio/build.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -ex -PREFIX="$(pwd)" -. build_tools/packaging/wheel/build_from_source.sh +packaging/build_from_source.sh "$(pwd)" IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt diff --git a/build_tools/packaging/conda/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml similarity index 69% rename from build_tools/packaging/conda/torchaudio/meta.yaml rename to packaging/torchaudio/meta.yaml index fdf41e52a9..0b89969fa2 100644 --- a/build_tools/packaging/conda/torchaudio/meta.yaml +++ b/packaging/torchaudio/meta.yaml @@ -1,9 +1,9 @@ package: name: torchaudio - version: "{{ environ.get('TORCHAUDIO_BUILD_VERSION') }}" + version: "{{ environ.get('BUILD_VERSION') }}" source: - path: "{{ environ.get('TORCHAUDIO_GITHUB_ROOT_DIR') }}" + path: "{{ environ.get('SOURCE_ROOT_DIR') }}" requirements: build: @@ -13,16 +13,16 @@ requirements: host: - python - setuptools - - pytorch >=1.2.0 + - cpuonly + {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} run: - python - typing - - pytorch >=1.2.0 + {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} build: - number: {{ environ.get('TORCHAUDIO_BUILD_NUMBER') }} - string: py{{py}}_{{environ.get('TORCHAUDIO_BUILD_NUMBER')}} + string: py{{py}} test: imports: diff --git a/setup.py b/setup.py index eea67ae8ec..2d2ea43860 100644 --- a/setup.py +++ b/setup.py @@ -63,12 +63,8 @@ def check_env_flag(name, default=''): except Exception: pass -if os.getenv('TORCHAUDIO_BUILD_VERSION'): - assert os.getenv('TORCHAUDIO_BUILD_NUMBER') is not None - build_number = int(os.getenv('TORCHAUDIO_BUILD_NUMBER')) - version = os.getenv('TORCHAUDIO_BUILD_VERSION') - if build_number > 1: - version += '.post' + str(build_number) +if os.getenv('BUILD_VERSION'): + version = os.getenv('BUILD_VERSION') elif sha != 'Unknown': version += '+' + sha[:7] print('-- Building version ' + version) @@ -78,11 +74,15 @@ def check_env_flag(name, default=''): f.write("__version__ = '{}'\n".format(version)) f.write("git_version = {}\n".format(repr(sha))) -pytorch_package_name = os.getenv('TORCHAUDIO_PYTORCH_DEPENDENCY_NAME', 'torch') +pytorch_package_version = os.getenv('PYTORCH_VERSION') + +pytorch_package_dep = 'torch' +if pytorch_package_version is not None: + pytorch_package_dep += "==" + pytorch_package_version setup( name="torchaudio", - version="0.2", + version=version, description="An audio package for PyTorch", url="https://github.com/pytorch/audio", author="Soumith Chintala, David Pollack, Sean Naren, Peter Goldsborough", From a0cdfbf4b61f0edb37e5ba2cf1d2aec0b89ace52 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 10:06:44 -0400 Subject: [PATCH 2/6] Setup cuda_suffix (#218) Signed-off-by: Edward Z. Yang --- packaging/build_conda.sh | 1 + packaging/build_wheel.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 46382eb422..b785eeab62 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -5,6 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" setup_build_version 0.4.0 +setup_cuda_suffix setup_macos export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 8871629c83..ef3a0ba885 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -5,6 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" setup_python +setup_cuda_suffix setup_build_version 0.4.0 setup_macos "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies From 6ba50a022baf07ccfe57c1276e2be295397b4621 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 8 Aug 2019 08:38:00 -0400 Subject: [PATCH 3/6] Add uploading support (#220) Also bugfixes and updates to the packaging scripts. --- .circleci/config.yml | 225 ++++++++++++++++++++++++++++++++++++- .circleci/config.yml.in | 123 +++++++++++++++++--- packaging/build_conda.sh | 6 +- packaging/build_wheel.sh | 7 +- packaging/pkg_helpers.bash | 218 +++++++++++++++++++++++------------ 5 files changed, 479 insertions(+), 100 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 222195c24c..c3b443d844 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,12 +2,13 @@ version: 2.1 # How to test the Linux jobs: # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel -# - Replace binary_linux_wheel with the name of the job you want to test +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 +# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. +# Job names are 'name:' key. binary_common: &binary_common parameters: - # Edit these defaults to do a release` + # Edit these defaults to do a release build_version: description: "version number of release binary; by default, build a nightly" type: string @@ -29,7 +30,7 @@ binary_common: &binary_common BUILD_VERSION: << parameters.build_version >> PYTORCH_VERSION: << parameters.pytorch_version >> UNICODE_ABI: << parameters.unicode_abi >> - CUDA_VERSION: cpu + CU_VERSION: cpu jobs: circleci_consistency: @@ -53,6 +54,10 @@ jobs: - run: packaging/build_wheel.sh - store_artifacts: path: dist + - persist_to_workspace: + root: dist + paths: + - "*" binary_linux_conda: <<: *binary_common @@ -64,6 +69,10 @@ jobs: - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" binary_macos_wheel: <<: *binary_common @@ -82,6 +91,10 @@ jobs: packaging/build_wheel.sh - store_artifacts: path: dist + - persist_to_workspace: + root: dist + paths: + - "*" binary_macos_conda: <<: *binary_common @@ -98,11 +111,55 @@ jobs: packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" + + # Requires org-member context + binary_conda_upload: + docker: + - image: continuumio/miniconda + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + # Prevent credential from leaking + conda install -yq anaconda-client + set +x + anaconda login \ + --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ + --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" + set -x + anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + docker: + - image: circleci/python:3.7 + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run: + command: | + pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read + done workflows: build: jobs: - circleci_consistency + - binary_linux_wheel: name: binary_linux_wheel_py2.7 python_version: "2.7" @@ -158,4 +215,162 @@ workflows: python_version: "3.6" - binary_macos_conda: name: binary_macos_conda_py3.7 - python_version: "3.7" \ No newline at end of file + python_version: "3.7" + + nightly: + triggers: + - schedule: + cron: "0 9 * * *" + filters: + branches: + only: + - master + jobs: + - circleci_consistency + + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py2.7 + python_version: "2.7" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py2.7_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py2.7 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py2.7_unicode_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py2.7_unicode + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.5 + python_version: "3.5" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.5_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.5 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.6 + python_version: "3.6" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.6_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.6 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.7_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py2.7 + python_version: "2.7" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py2.7_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py2.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py2.7_unicode_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py2.7_unicode + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.5 + python_version: "3.5" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.5_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.5 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.6 + python_version: "3.6" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.6_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.6 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.7_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.7 + - binary_linux_conda: + name: nightly_binary_linux_conda_py2.7 + python_version: "2.7" + - binary_conda_upload: + name: nightly_binary_linux_conda_py2.7_upload + context: org-member + requires: + - nightly_binary_linux_conda_py2.7 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.5 + python_version: "3.5" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.5_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.5 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.6 + python_version: "3.6" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.6_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.6 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.7 + python_version: "3.7" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.7_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.7 + - binary_macos_conda: + name: nightly_binary_macos_conda_py2.7 + python_version: "2.7" + - binary_conda_upload: + name: nightly_binary_macos_conda_py2.7_upload + context: org-member + requires: + - nightly_binary_macos_conda_py2.7 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.5 + python_version: "3.5" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.5_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.5 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.6 + python_version: "3.6" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.6_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.6 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.7 + python_version: "3.7" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.7_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.7 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 47bf1ce630..771e17443b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -2,12 +2,13 @@ version: 2.1 # How to test the Linux jobs: # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel -# - Replace binary_linux_wheel with the name of the job you want to test +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 +# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. +# Job names are 'name:' key. binary_common: &binary_common parameters: - # Edit these defaults to do a release` + # Edit these defaults to do a release build_version: description: "version number of release binary; by default, build a nightly" type: string @@ -29,7 +30,7 @@ binary_common: &binary_common BUILD_VERSION: << parameters.build_version >> PYTORCH_VERSION: << parameters.pytorch_version >> UNICODE_ABI: << parameters.unicode_abi >> - CUDA_VERSION: cpu + CU_VERSION: cpu jobs: circleci_consistency: @@ -53,6 +54,10 @@ jobs: - run: packaging/build_wheel.sh - store_artifacts: path: dist + - persist_to_workspace: + root: dist + paths: + - "*" binary_linux_conda: <<: *binary_common @@ -64,6 +69,10 @@ jobs: - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" binary_macos_wheel: <<: *binary_common @@ -82,6 +91,10 @@ jobs: packaging/build_wheel.sh - store_artifacts: path: dist + - persist_to_workspace: + root: dist + paths: + - "*" binary_macos_conda: <<: *binary_common @@ -98,23 +111,97 @@ jobs: packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" -workflows: - build: - jobs: - - circleci_consistency -{%- for btype in ["wheel", "conda"] -%} -{%- for os in ["linux", "macos"] -%} -{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] %} - - binary_{{os}}_{{btype}}: - name: binary_{{os}}_{{btype}}_py{{python_version}} - python_version: "{{python_version}}" -{%- if btype == "wheel" and python_version == "2.7" %} + # Requires org-member context + binary_conda_upload: + docker: + - image: continuumio/miniconda + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + # Prevent credential from leaking + conda install -yq anaconda-client + set +x + anaconda login \ + --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ + --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" + set -x + anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + docker: + - image: circleci/python:3.7 + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run: + command: | + pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read + done + + +{%- macro workflow(btype, os, python_version, unicode, prefix='', upload=False) %} - binary_{{os}}_{{btype}}: - name: binary_{{os}}_{{btype}}_py{{python_version}}_unicode + name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }} python_version: "{{python_version}}" +{%- if unicode %} unicode_abi: "1" -{%- endif -%} +{%- endif %} + +{%- if upload %} + - binary_{{btype}}_upload: + name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_upload + context: org-member + requires: + - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }} +{%- endif %} +{%- endmacro %} + +{%- macro workflows(prefix='', upload=False) %} +{%- for btype in ["wheel", "conda"] -%} +{%- for os in ["linux", "macos"] -%} +{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] -%} +{%- for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]) -%} + {{ workflow(btype, os, python_version, unicode, prefix=prefix, upload=upload) }} +{%- endfor -%} {%- endfor -%} {%- endfor -%} -{%- endfor -%} +{%- endfor %} +{%- endmacro %} + +workflows: + build: +{%- if True %} + jobs: + - circleci_consistency + {{ workflows() }} + + nightly: + triggers: + - schedule: + cron: "0 9 * * *" + filters: + branches: + only: + - master +{%- endif %} + jobs: + - circleci_consistency + {{ workflows(prefix="nightly_", upload=True) }} + diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index b785eeab62..be00e74f3c 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -4,10 +4,8 @@ set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" -setup_build_version 0.4.0 -setup_cuda_suffix -setup_macos +export NO_CUDA_PACKAGE=1 +setup_env 0.4.0 export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio - diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index ef3a0ba885..60818f336b 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -4,10 +4,9 @@ set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" -setup_python -setup_cuda_suffix -setup_build_version 0.4.0 -setup_macos +export NO_CUDA_PACKAGE=1 +setup_env 0.4.0 +setup_wheel_python "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies pip_install numpy future setup_pip_pytorch_version diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 0265d5bd60..f5f473a758 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -1,70 +1,92 @@ # A set of useful bash functions for common functionality we need to do in # many build scripts -# Respecting PYTHON_VERSION and UNICODE_ABI, add (or install) the correct -# version of Python to perform a build. Relevant to wheel builds. -setup_python() { - if [[ "$(uname)" == Darwin ]]; then - eval "$(conda shell.bash hook)" - conda env remove -n "env$PYTHON_VERSION" || true - conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" - conda activate "env$PYTHON_VERSION" - else - case "$PYTHON_VERSION" in - 2.7) - if [[ -n "$UNICODE_ABI" ]]; then - python_abi=cp27-cp27mu - else - python_abi=cp27-cp27m - fi - ;; - 3.5) python_abi=cp35-cp35m ;; - 3.6) python_abi=cp36-cp36m ;; - 3.7) python_abi=cp37-cp37m ;; - *) - echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" - exit 1 - ;; - esac - export PATH="/opt/python/$python_abi/bin:$PATH" - fi -} -# Fill CUDA_SUFFIX with CUDA_VERSION, but don't fill it for the default -# CUDA version (that's a blank suffix) -setup_cuda_suffix() { - if [[ "$(uname)" == Darwin ]]; then - if [[ "$CUDA_VERSION" != "cpu" ]]; then - echo "CUDA_VERSION on OS X must be cpu" +# Setup CUDA environment variables, based on CU_VERSION +# +# Inputs: +# CU_VERSION (cpu, cu92, cu100) +# NO_CUDA_PACKAGE (bool) +# +# Outputs: +# VERSION_SUFFIX (e.g., "") +# PYTORCH_VERSION_SUFFIX (e.g., +cpu) +# WHEEL_DIR (e.g., cu100/) +# CUDA_HOME (e.g., /usr/local/cuda-9.2, respected by torch.utils.cpp_extension) +# FORCE_CUDA (respected by torchvision setup.py) +# NVCC_FLAGS (respected by torchvision setup.py) +# +# Precondition: CUDA versions are installed in their conventional locations in +# /usr/local/cuda-* +# +# NOTE: Why VERSION_SUFFIX versus PYTORCH_VERSION_SUFFIX? If you're building +# a package with CUDA on a platform we support CUDA on, VERSION_SUFFIX == +# PYTORCH_VERSION_SUFFIX and everyone is happy. However, if you are building a +# package with only CPU bits (e.g., torchaudio), then VERSION_SUFFIX is always +# empty, but PYTORCH_VERSION_SUFFIX is +cpu (because that's how you get a CPU +# version of a Python package. But that doesn't apply if you're on OS X, +# since the default CU_VERSION on OS X is cpu. +setup_cuda() { + if [[ "$(uname)" == Darwin ]] || [[ -n "$NO_CUDA_PACKAGE" ]]; then + if [[ "$CU_VERSION" != "cpu" ]]; then + echo "CU_VERSION on OS X / package with no CUDA must be cpu" exit 1 fi - export CPU_SUFFIX="" + if [[ "$(uname)" == Darwin ]]; then + export PYTORCH_VERSION_SUFFIX="" + else + export PYTORCH_VERSION_SUFFIX="+cpu" + fi + export VERSION_SUFFIX="" + # NB: When there is no CUDA package available, we put these + # packages in the top-level directory, so they are eligible + # for selection even if you are otherwise trying to install + # a cu100 stack. This differs from when there ARE CUDA packages + # available; then we don't want the cpu package; we want + # to give you as much goodies as possible. + export WHEEL_DIR="" else - case "$CUDA_VERSION" in - 10.0) - export CUDA_SUFFIX="" ;; # default! - 9.2) - export CUDA_SUFFIX="+cu92" ;; + case "$CU_VERSION" in + cu100) + export PYTORCH_VERSION_SUFFIX="" + export CUDA_HOME=/usr/local/cuda-10.0/ + export FORCE_CUDA=1 + # Hard-coding gencode flags is temporary situation until + # https://github.com/pytorch/pytorch/pull/23408 lands + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" + ;; + cu92) + export CUDA_HOME=/usr/local/cuda-9.2/ + export PYTORCH_VERSION_SUFFIX="+cu92" + export FORCE_CUDA=1 + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50" + ;; cpu) - export CUDA_SUFFIX="+cpu" ;; + export PYTORCH_VERSION_SUFFIX="+cpu" + ;; *) - echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" + echo "Unrecognized CU_VERSION=$CU_VERSION" esac - export CPU_SUFFIX="+cpu" + export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX" + export WHEEL_DIR="$CU_VERSION/" fi } -# If a package is cpu-only, we never provide a cuda suffix -setup_cpuonly_cuda_suffix() { - export CUDA_SUFFIX="" - export CPU_SUFFIX="" -} - -# Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string -# Usage: setup_build_version 0.2 +# Populate build version if necessary, and add version suffix +# +# Inputs: +# BUILD_VERSION (e.g., 0.2.0 or empty) +# +# Outputs: +# BUILD_VERSION (e.g., 0.2.0.dev20190807+cpu) +# +# Fill BUILD_VERSION if it doesn't exist already with a nightly string +# Usage: setup_build_version 0.2.0 setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then - export BUILD_VERSION="$1.dev$(date "+%Y%m%d")" + export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + else + export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi } @@ -75,11 +97,55 @@ setup_macos() { fi } +# Top-level entry point for things every package will need to do +# +# Usage: setup_env 0.2.0 +setup_env() { + setup_cuda + setup_build_version "$1" + setup_macos +} + # Function to retry functions that sometimes timeout or have flaky failures retry () { $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) } +# Inputs: +# PYTHON_VERSION (2.7, 3.5, 3.6, 3.7) +# UNICODE_ABI (bool) +# +# Outputs: +# PATH modified to put correct Python version in PATH +# +# Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image +setup_wheel_python() { + if [[ "$(uname)" == Darwin ]]; then + eval "$(conda shell.bash hook)" + conda env remove -n "env$PYTHON_VERSION" || true + conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" + conda activate "env$PYTHON_VERSION" + else + case "$PYTHON_VERSION" in + 2.7) + if [[ -n "$UNICODE_ABI" ]]; then + python_abi=cp27-cp27mu + else + python_abi=cp27-cp27m + fi + ;; + 3.5) python_abi=cp35-cp35m ;; + 3.6) python_abi=cp36-cp36m ;; + 3.7) python_abi=cp37-cp37m ;; + *) + echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" + exit 1 + ;; + esac + export PATH="/opt/python/$python_abi/bin:$PATH" + fi +} + # Install with pip a bit more robustly than the default pip_install() { retry pip install --progress-bar off "$@" @@ -89,15 +155,20 @@ pip_install() { # version into PYTORCH_VERSION, if applicable setup_pip_pytorch_version() { if [[ -z "$PYTORCH_VERSION" ]]; then - # Install latest prerelease CPU version of torch, per our nightlies. - pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we - # strip off the local package qualifier. - export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" + # Install latest prerelease version of torch, per our nightlies, consistent + # with the requested cuda version + pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${WHEEL_DIR}torch_nightly.html" + if [[ "$CUDA_VERSION" == "cpu" ]]; then + # CUDA and CPU are ABI compatible on the CPU-only parts, so strip + # in this case + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" + else + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')" + fi else - # TODO: Maybe add staging too - pip_install "torch==$PYTORCH_VERSION" \ - -f https://download.pytorch.org/whl/torch_stable.html + pip_install "torch==$PYTORCH_VERSION$CUDA_SUFFIX" \ + -f https://download.pytorch.org/whl/torch_stable.html \ + -f https://download.pytorch.org/whl/nightly/torch_nightly.html fi } @@ -105,30 +176,39 @@ setup_pip_pytorch_version() { # CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions # # You MUST have populated CUDA_SUFFIX before hand. -# -# TODO: This is currently hard-coded for CPU-only case setup_conda_pytorch_constraint() { if [[ -z "$PYTORCH_VERSION" ]]; then export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else - export CONDA_CHANNEL_FLAGS="-c pytorch" + export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" fi if [[ "$CUDA_VERSION" == cpu ]]; then - export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${CPU_SUFFIX}" + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" else - export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" - export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}" fi } # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT setup_conda_cudatoolkit_constraint() { - if [[ "$CUDA_VERSION" == cpu ]]; then + export CONDA_CPUONLY_FEATURE="" + if [[ "$(uname)" == Darwin ]]; then export CONDA_CUDATOOLKIT_CONSTRAINT="" else - echo - # TODO + case "$CUDA_VERSION" in + 10.0) + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]" + ;; + 9.2) + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]" + ;; + cpu) + export CONDA_CUDATOOLKIT_CONSTRAINT="" + export CONDA_CPUONLY_FEATURE="- cpuonly" + ;; + esac fi } From 1e184695a3158548134f77f939336c44bcb48bd9 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 8 Aug 2019 05:48:14 -0700 Subject: [PATCH 4/6] Finish up and set parameters Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 72 ++--------------------------------------- .circleci/config.yml.in | 6 ++-- setup.py | 4 +-- 3 files changed, 7 insertions(+), 75 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3b443d844..61b20fec28 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,11 +12,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "" + default: "0.3.0" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "" + default: "1.2.0.post4" # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)" @@ -160,74 +160,6 @@ workflows: jobs: - circleci_consistency - - binary_linux_wheel: - name: binary_linux_wheel_py2.7 - python_version: "2.7" - - binary_linux_wheel: - name: binary_linux_wheel_py2.7_unicode - python_version: "2.7" - unicode_abi: "1" - - binary_linux_wheel: - name: binary_linux_wheel_py3.5 - python_version: "3.5" - - binary_linux_wheel: - name: binary_linux_wheel_py3.6 - python_version: "3.6" - - binary_linux_wheel: - name: binary_linux_wheel_py3.7 - python_version: "3.7" - - binary_macos_wheel: - name: binary_macos_wheel_py2.7 - python_version: "2.7" - - binary_macos_wheel: - name: binary_macos_wheel_py2.7_unicode - python_version: "2.7" - unicode_abi: "1" - - binary_macos_wheel: - name: binary_macos_wheel_py3.5 - python_version: "3.5" - - binary_macos_wheel: - name: binary_macos_wheel_py3.6 - python_version: "3.6" - - binary_macos_wheel: - name: binary_macos_wheel_py3.7 - python_version: "3.7" - - binary_linux_conda: - name: binary_linux_conda_py2.7 - python_version: "2.7" - - binary_linux_conda: - name: binary_linux_conda_py3.5 - python_version: "3.5" - - binary_linux_conda: - name: binary_linux_conda_py3.6 - python_version: "3.6" - - binary_linux_conda: - name: binary_linux_conda_py3.7 - python_version: "3.7" - - binary_macos_conda: - name: binary_macos_conda_py2.7 - python_version: "2.7" - - binary_macos_conda: - name: binary_macos_conda_py3.5 - python_version: "3.5" - - binary_macos_conda: - name: binary_macos_conda_py3.6 - python_version: "3.6" - - binary_macos_conda: - name: binary_macos_conda_py3.7 - python_version: "3.7" - - nightly: - triggers: - - schedule: - cron: "0 9 * * *" - filters: - branches: - only: - - master - jobs: - - circleci_consistency - - binary_linux_wheel: name: nightly_binary_linux_wheel_py2.7 python_version: "2.7" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 771e17443b..9b5b51acfc 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -12,11 +12,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "" + default: "0.3.0" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "" + default: "1.2.0.post4" # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)" @@ -187,7 +187,7 @@ jobs: workflows: build: -{%- if True %} +{%- if False %} jobs: - circleci_consistency {{ workflows() }} diff --git a/setup.py b/setup.py index 2d2ea43860..08eb3fb792 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def check_env_flag(name, default=''): # Creating the version file cwd = os.path.dirname(os.path.abspath(__file__)) -version = '0.3.0a0' +version = '0.3.0' sha = 'Unknown' try: @@ -115,5 +115,5 @@ def check_env_flag(name, default=''): extra_link_args=ela), ], cmdclass={'build_ext': BuildExtension}, - install_requires=[pytorch_package_name] + install_requires=[pytorch_package_dep] ) From 8cc350a880ffe296fdddb29fabb1bc21031644d9 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 8 Aug 2019 07:00:16 -0700 Subject: [PATCH 5/6] Updates from vision Signed-off-by: Edward Z. Yang --- packaging/pkg_helpers.bash | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index f5f473a758..461cc980b9 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -183,7 +183,7 @@ setup_conda_pytorch_constraint() { else export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" fi - if [[ "$CUDA_VERSION" == cpu ]]; then + if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" else @@ -198,17 +198,21 @@ setup_conda_cudatoolkit_constraint() { if [[ "$(uname)" == Darwin ]]; then export CONDA_CUDATOOLKIT_CONSTRAINT="" else - case "$CUDA_VERSION" in - 10.0) + case "$CU_VERSION" in + cu100) export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]" ;; - 9.2) + cu92) export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]" ;; cpu) export CONDA_CUDATOOLKIT_CONSTRAINT="" export CONDA_CPUONLY_FEATURE="- cpuonly" ;; + *) + echo "Unrecognized CU_VERSION=$CU_VERSION" + exit 1 + ;; esac fi } From bf88aef3f388332d735704e7ecad528a76e5bc42 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 8 Aug 2019 08:37:46 -0700 Subject: [PATCH 6/6] Go time Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 61b20fec28..1d8cca2d9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ binary_common: &binary_common pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "1.2.0.post4" + default: "1.2.0" # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 9b5b51acfc..76468db6ba 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -16,7 +16,7 @@ binary_common: &binary_common pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "1.2.0.post4" + default: "1.2.0" # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)"