diff --git a/.circleci/config.yml b/.circleci/config.yml index b2b0dd45c3..a4adf356cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,12 +32,8 @@ commands: - run: name: adding UPLOAD_CHANNEL to BASH_ENV command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + # Hardcoded for release branch + echo "export UPLOAD_CHANNEL=test" >> ${BASH_ENV} install_build_tools_macos: description: "installs tools required to build torchaudio" steps: @@ -45,6 +41,16 @@ commands: name: Install cmake and pkg-config command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget # Disable brew auto update which is very slow + load_conda_channel_flags: + description: "Determines whether we need extra conda channels" + steps: + - run: + name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV + command: | + CONDA_CHANNEL_FLAGS="" + if [[ "${PYTHON_VERSION}" = *3.9* ]]; then + echo "export CONDA_CHANNEL_FLAGS=-c=conda-forge" >> ${BASH_ENV} + fi binary_common: &binary_common parameters: @@ -52,11 +58,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "" + default: "0.7.2" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "" + default: "1.7.1" # Don't edit these python_version: description: "Python version to build against (e.g., 3.8)" @@ -117,7 +123,7 @@ jobs: binary_linux_wheel: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout @@ -138,6 +144,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - load_conda_channel_flags - attach_workspace: at: third_party - run: packaging/build_conda.sh @@ -155,6 +162,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -180,6 +188,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -202,6 +211,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -221,6 +231,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -278,6 +289,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -296,12 +308,14 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchaudio*.whl) -f https://download.pytorch.org/whl/nightly/torch_nightly.html -f https://download.pytorch.org/whl/test/torch_test.html + pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | @@ -315,6 +329,8 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -323,7 +339,7 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-test -c pytorch-nightly pytorch + conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch conda install -v -y $(ls ~/workspace/torchaudio*.tar.bz2) - run: name: smoke test @@ -339,6 +355,8 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -347,7 +365,7 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchaudio*.whl) -f https://download.pytorch.org/whl/nightly/torch_nightly.html -f https://download.pytorch.org/whl/test/torch_test.html + pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | @@ -391,6 +409,8 @@ jobs: - checkout - attach_workspace: at: third_party + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -433,6 +453,8 @@ jobs: - checkout - attach_workspace: at: third_party + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -470,6 +492,8 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -506,6 +530,8 @@ jobs: CUDA_VERSION: "10.1" steps: - checkout + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -542,27 +568,13 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - - generate_cache_key - - restore_cache: - - keys: - - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} - + - designate_upload_channel - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} - - paths: - - conda - - env - - third_party/build - - third_party/install - - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh @@ -582,6 +594,8 @@ jobs: resource_class: medium steps: - checkout + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -612,6 +626,7 @@ jobs: steps: - checkout - generate_cache_key + - load_conda_channel_flags - restore_cache: keys: @@ -662,6 +677,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_linux_wheel: + name: binary_linux_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_macos_wheel: name: binary_macos_wheel_py3.6 python_version: '3.6' @@ -677,6 +697,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_macos_wheel: + name: binary_macos_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_windows_wheel: name: binary_windows_wheel_py3.6 python_version: '3.6' @@ -686,6 +711,9 @@ workflows: - binary_windows_wheel: name: binary_windows_wheel_py3.8 python_version: '3.8' + - binary_windows_wheel: + name: binary_windows_wheel_py3.9 + python_version: '3.9' - binary_linux_conda: name: binary_linux_conda_py3.6 python_version: '3.6' @@ -701,6 +729,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_linux_conda: + name: binary_linux_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_macos_conda: name: binary_macos_conda_py3.6 python_version: '3.6' @@ -716,6 +749,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_macos_conda: + name: binary_macos_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_windows_conda: name: binary_windows_conda_py3.6 python_version: '3.6' @@ -725,6 +763,9 @@ workflows: - binary_windows_conda: name: binary_windows_conda_py3.8 python_version: '3.8' + - binary_windows_conda: + name: binary_windows_conda_py3.9 + python_version: '3.9' unittest: jobs: - torchscript_bc_test: @@ -750,6 +791,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_linux_cpu: + name: unittest_linux_cpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - unittest_linux_gpu: filters: branches: @@ -786,6 +832,18 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_linux_gpu: + filters: + branches: + only: + - master + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: unittest_linux_gpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - unittest_windows_cpu: name: unittest_windows_cpu_py3.6 python_version: '3.6' @@ -795,6 +853,9 @@ workflows: - unittest_windows_cpu: name: unittest_windows_cpu_py3.8 python_version: '3.8' + - unittest_windows_cpu: + name: unittest_windows_cpu_py3.9 + python_version: '3.9' - unittest_windows_gpu: filters: branches: @@ -825,6 +886,16 @@ workflows: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_windows_gpu_py3.8 python_version: '3.8' + - unittest_windows_gpu: + filters: + branches: + only: + - master + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: unittest_windows_gpu_py3.9 + python_version: '3.9' - unittest_macos_cpu: name: unittest_macos_cpu_py3.6 python_version: '3.6' @@ -840,6 +911,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_macos_cpu: + name: unittest_macos_cpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix nightly: jobs: - circleci_consistency: @@ -953,6 +1029,39 @@ workflows: python_version: '3.8' requires: - nightly_binary_linux_wheel_py3.8_upload + - binary_linux_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9_upload + requires: + - nightly_binary_linux_wheel_py3.9 + - smoke_test_linux_pip: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9_smoke_test_pip + python_version: '3.9' + requires: + - nightly_binary_linux_wheel_py3.9_upload - binary_macos_wheel: filters: branches: @@ -1019,6 +1128,28 @@ workflows: name: nightly_binary_macos_wheel_py3.8_upload requires: - nightly_binary_macos_wheel_py3.8 + - binary_macos_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_wheel_py3.9_upload + requires: + - nightly_binary_macos_wheel_py3.9 - binary_windows_wheel: filters: branches: @@ -1112,6 +1243,37 @@ workflows: python_version: '3.8' requires: - nightly_binary_windows_wheel_py3.8_upload + - binary_windows_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9 + python_version: '3.9' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9_upload + requires: + - nightly_binary_windows_wheel_py3.9 + - smoke_test_windows_pip: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9_smoke_test_pip + python_version: '3.9' + requires: + - nightly_binary_windows_wheel_py3.9_upload - binary_linux_conda: filters: branches: @@ -1211,6 +1373,39 @@ workflows: python_version: '3.8' requires: - nightly_binary_linux_conda_py3.8_upload + - binary_linux_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9_upload + requires: + - nightly_binary_linux_conda_py3.9 + - smoke_test_linux_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9_smoke_test_conda + python_version: '3.9' + requires: + - nightly_binary_linux_conda_py3.9_upload - binary_macos_conda: filters: branches: @@ -1277,6 +1472,28 @@ workflows: name: nightly_binary_macos_conda_py3.8_upload requires: - nightly_binary_macos_conda_py3.8 + - binary_macos_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_conda_py3.9_upload + requires: + - nightly_binary_macos_conda_py3.9 - binary_windows_conda: filters: branches: @@ -1370,6 +1587,37 @@ workflows: python_version: '3.8' requires: - nightly_binary_windows_conda_py3.8_upload + - binary_windows_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9 + python_version: '3.9' + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9_upload + requires: + - nightly_binary_windows_conda_py3.9 + - smoke_test_windows_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9_smoke_test_conda + python_version: '3.9' + requires: + - nightly_binary_windows_conda_py3.9_upload docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index b1f40c24c0..7b00a7fbcd 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -32,12 +32,8 @@ commands: - run: name: adding UPLOAD_CHANNEL to BASH_ENV command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + # Hardcoded for release branch + echo "export UPLOAD_CHANNEL=test" >> ${BASH_ENV} install_build_tools_macos: description: "installs tools required to build torchaudio" steps: @@ -45,6 +41,16 @@ commands: name: Install cmake and pkg-config command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget # Disable brew auto update which is very slow + load_conda_channel_flags: + description: "Determines whether we need extra conda channels" + steps: + - run: + name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV + command: | + CONDA_CHANNEL_FLAGS="" + if [[ "${PYTHON_VERSION}" = *3.9* ]]; then + echo "export CONDA_CHANNEL_FLAGS=-c=conda-forge" >> ${BASH_ENV} + fi binary_common: &binary_common parameters: @@ -52,11 +58,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "" + default: "0.7.2" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "" + default: "1.7.1" # Don't edit these python_version: description: "Python version to build against (e.g., 3.8)" @@ -117,7 +123,7 @@ jobs: binary_linux_wheel: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout @@ -138,6 +144,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - load_conda_channel_flags - attach_workspace: at: third_party - run: packaging/build_conda.sh @@ -155,6 +162,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -180,6 +188,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -202,6 +211,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -221,6 +231,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -278,6 +289,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -296,12 +308,14 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchaudio*.whl) -f https://download.pytorch.org/whl/nightly/torch_nightly.html -f https://download.pytorch.org/whl/test/torch_test.html + pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | @@ -315,6 +329,8 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -323,7 +339,7 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-test -c pytorch-nightly pytorch + conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch conda install -v -y $(ls ~/workspace/torchaudio*.tar.bz2) - run: name: smoke test @@ -339,6 +355,8 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -347,7 +365,7 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchaudio*.whl) -f https://download.pytorch.org/whl/nightly/torch_nightly.html -f https://download.pytorch.org/whl/test/torch_test.html + pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | @@ -391,6 +409,8 @@ jobs: - checkout - attach_workspace: at: third_party + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -433,6 +453,8 @@ jobs: - checkout - attach_workspace: at: third_party + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -470,6 +492,8 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -506,6 +530,8 @@ jobs: CUDA_VERSION: "10.1" steps: - checkout + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -542,27 +568,13 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - - generate_cache_key - - restore_cache: - {% raw %} - keys: - - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} - {% endraw %} + - designate_upload_channel - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - {% raw %} - key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} - {% endraw %} - paths: - - conda - - env - - third_party/build - - third_party/install - - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh @@ -582,6 +594,8 @@ jobs: resource_class: medium steps: - checkout + - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -612,6 +626,7 @@ jobs: steps: - checkout - generate_cache_key + - load_conda_channel_flags - restore_cache: {% raw %} keys: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 2c9fe7ee4f..85b01e6b72 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -19,7 +19,7 @@ import os.path -PYTHON_VERSIONS = ["3.6", "3.7", "3.8"] +PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6): diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml deleted file mode 100644 index 4315a2205b..0000000000 --- a/.circleci/unittest/linux/scripts/environment.yml +++ /dev/null @@ -1,17 +0,0 @@ -# a comment for these trying times -channels: - - conda-forge - - defaults -dependencies: - - flake8 - - numpy >= 1.11 - - pytest - - pytest-cov - - codecov - - librosa>=0.8.0 - - llvmlite==0.31 # See https://github.com/pytorch/audio/pull/766 - - pip - - pip: - - kaldi-io - - scipy - - parameterized diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 8346e6c825..75314fe4a7 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -11,13 +11,34 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env if [ -z "${CUDA_VERSION:-}" ] ; then - cudatoolkit="cpuonly" + case "$(uname -s)" in + Darwin*) cudatoolkit="";; + *) cudatoolkit="cpuonly" + esac else version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" +conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} + +printf "* Installing dependencies for test\n" +CONDA_PKGS="librosa>=0.8.0" +# TODO: Remove this after packages become available +# Currently there's no librosa package available for Python 3.9, so lets just skip the dependency for now +if [[ $(python --version) = *3.9* ]]; then + CONDA_PKGS="pysoundfile" +fi +conda install -y -c conda-forge pytest pytest-cov codecov scipy parameterized ${CONDA_PKGS} +pip install kaldi-io + +printf "* Building codecs\n" +mkdir -p third_party/build +( + cd third_party/build + cmake .. + cmake --build . +) printf "* Installing torchaudio\n" BUILD_SOX=1 python setup.py install diff --git a/.circleci/unittest/linux/scripts/run_style_checks.sh b/.circleci/unittest/linux/scripts/run_style_checks.sh index 70b83e19f5..8eff8a2b52 100755 --- a/.circleci/unittest/linux/scripts/run_style_checks.sh +++ b/.circleci/unittest/linux/scripts/run_style_checks.sh @@ -5,6 +5,8 @@ set -u eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env +pip install clang-format flake8 + # We want to run all the style checks even if one of them fail. exit_status=0 diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 290f0f8f97..09c10bb3b1 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -7,7 +7,6 @@ set -e -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" conda_dir="${root_dir}/conda" env_dir="${root_dir}/env" @@ -32,19 +31,3 @@ if [ ! -d "${env_dir}" ]; then printf "* Creating a test environment\n" conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" fi -conda activate "${env_dir}" - -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -conda env update --file "${this_dir}/environment.yml" --prune -if [ "${os}" == Linux ] ; then - pip install clang-format -fi - -# 4. Buld codecs -mkdir -p third_party/build -( - cd third_party/build - cmake .. - cmake --build . -) diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 4a0b71d653..4fe8134d04 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -7,12 +7,11 @@ dependencies: - pytest - pytest-cov - codecov + - scipy >= 1.4.1 - pip - pip: - - scipy == 1.4.1 - kaldi-io - PySoundFile - - librosa >= 0.8.0 - future - parameterized - dataclasses diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index 2b9c88b4af..7b3b6b1ed9 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -17,7 +17,13 @@ else cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" +conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" + +# TODO: Remove this after packages become available +# Currently there's no librosa package available for Python 3.9, so lets just skip the dependency for now +if [[ $(python --version) != *3.9* ]]; then + pip install 'librosa>=0.8.0' +fi printf "* Installing torchaudio\n" python setup.py install diff --git a/docs/requirements.txt b/docs/requirements.txt index c591f08491..99a7811de6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx +sphinx==2.4.4 -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme sphinxcontrib.katex matplotlib diff --git a/docs/source/conf.py b/docs/source/conf.py index 9d8faaa61b..b91994c3d0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -82,10 +82,10 @@ # # The short X.Y version. # TODO: change to [:2] at v1.0 -version = 'master ' +version = '0.7.1' # The full version, including alpha/beta/rc tags. # TODO: verify this works as expected -release = 'master' +release = '0.7.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/datasets.rst b/docs/source/datasets.rst index 2ceb93a440..2fbb2c7929 100644 --- a/docs/source/datasets.rst +++ b/docs/source/datasets.rst @@ -2,7 +2,7 @@ torchaudio.datasets ==================== All datasets are subclasses of :class:`torch.utils.data.Dataset` -i.e, they have ``__getitem__`` and ``__len__`` methods implemented. +and have ``__getitem__`` and ``__len__`` methods implemented. Hence, they can all be passed to a :class:`torch.utils.data.DataLoader` which can load multiple samples parallelly using ``torch.multiprocessing`` workers. For example: :: diff --git a/docs/source/functional.rst b/docs/source/functional.rst index 89e992bb89..ebed60de6b 100644 --- a/docs/source/functional.rst +++ b/docs/source/functional.rst @@ -8,145 +8,176 @@ torchaudio.functional Functions to perform common audio operations. -:hidden:`spectrogram` -~~~~~~~~~~~~~~~~~~~~~ +:hidden:`Filtering` +~~~~~~~~~~~~~~~~~~~ -.. autofunction:: spectrogram -:hidden:`amplitude_to_DB` -~~~~~~~~~~~~~~~~~~~~~~~~~ +allpass_biquad +-------------- -.. autofunction:: amplitude_to_DB +.. autofunction:: allpass_biquad -:hidden:`create_fb_matrix` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +band_biquad +----------- -.. autofunction:: create_fb_matrix +.. autofunction:: band_biquad -:hidden:`create_dct` -~~~~~~~~~~~~~~~~~~~~ +bandpass_biquad +--------------- -.. autofunction:: create_dct +.. autofunction:: bandpass_biquad -:hidden:`mu_law_encoding` -~~~~~~~~~~~~~~~~~~~~~~~~~ +bandreject_biquad +----------------- -.. autofunction:: mu_law_encoding +.. autofunction:: bandreject_biquad -:hidden:`mu_law_decoding` -~~~~~~~~~~~~~~~~~~~~~~~~~ +bass_biquad +----------- -.. autofunction:: mu_law_decoding +.. autofunction:: bass_biquad -:hidden:`complex_norm` -~~~~~~~~~~~~~~~~~~~~~~ +biquad +------ -.. autofunction:: complex_norm +.. autofunction:: biquad -:hidden:`angle` -~~~~~~~~~~~~~~~ +contrast +-------- -.. autofunction:: angle +.. autofunction:: contrast -:hidden:`magphase` -~~~~~~~~~~~~~~~~~~ +dcshift +------- -.. autofunction:: magphase +.. autofunction:: dcshift -:hidden:`phase_vocoder` -~~~~~~~~~~~~~~~~~~~~~~~ +deemph_biquad +------------- -.. autofunction:: phase_vocoder +.. autofunction:: deemph_biquad -:hidden:`lfilter` -~~~~~~~~~~~~~~~~~ -.. autofunction:: lfilter +dither +------ -:hidden:`biquad` -~~~~~~~~~~~~~~~~ +.. autofunction:: dither -.. autofunction:: biquad +equalizer_biquad +---------------- -:hidden:`lowpass_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: equalizer_biquad -.. autofunction:: lowpass_biquad +flanger +------- -:hidden:`highpass_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: flanger + +gain +---- + +.. autofunction:: gain + +highpass_biquad +--------------- .. autofunction:: highpass_biquad -:hidden:`allpass_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~ +lfilter +------- -.. autofunction:: allpass_biquad +.. autofunction:: lfilter -:hidden:`equalizer_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +lowpass_biquad +-------------- -.. autofunction:: equalizer_biquad +.. autofunction:: lowpass_biquad -:hidden:`bandpass_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +overdrive +--------- -.. autofunction:: bandpass_biquad +.. autofunction:: overdrive -:hidden:`bandreject_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +phaser +------ -.. autofunction:: bandreject_biquad +.. autofunction:: phaser -:hidden:`band_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +riaa_biquad +----------- -.. autofunction:: band_biquad +.. autofunction:: riaa_biquad -:hidden:`treble_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +treble_biquad +------------- .. autofunction:: treble_biquad -:hidden:`bass_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autofunction:: bass_biquad +vad +--- -:hidden:`deemph_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: vad -.. autofunction:: deemph_biquad +:hidden:`spectrogram` +~~~~~~~~~~~~~~~~~~~~~ -:hidden:`riaa_biquad` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: spectrogram -.. autofunction:: riaa_biquad +:hidden:`griffinlim` +~~~~~~~~~~~~~~~~~~~~~~~~~ -:hidden:`contrast` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: griffinlim -.. autofunction:: contrast +:hidden:`amplitude_to_DB` +~~~~~~~~~~~~~~~~~~~~~~~~~ -:hidden:`dcshift` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: amplitude_to_DB -.. autofunction:: dcshift +:hidden:`DB_to_amplitude` +~~~~~~~~~~~~~~~~~~~~~~~~~ -:hidden:`overdrive` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: DB_to_amplitude -.. autofunction:: overdrive +:hidden:`create_fb_matrix` +~~~~~~~~~~~~~~~~~~~~~~~~~~ -:hidden:`phaser` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: create_fb_matrix -.. autofunction:: phaser +:hidden:`create_dct` +~~~~~~~~~~~~~~~~~~~~ -:hidden:`flanger` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. autofunction:: create_dct -.. autofunction:: flanger +:hidden:`mu_law_encoding` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autofunction:: mu_law_encoding + +:hidden:`mu_law_decoding` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autofunction:: mu_law_decoding + +:hidden:`complex_norm` +~~~~~~~~~~~~~~~~~~~~~~ + +.. autofunction:: complex_norm + +:hidden:`angle` +~~~~~~~~~~~~~~~ + +.. autofunction:: angle + +:hidden:`magphase` +~~~~~~~~~~~~~~~~~~ + +.. autofunction:: magphase + +:hidden:`phase_vocoder` +~~~~~~~~~~~~~~~~~~~~~~~ + +.. autofunction:: phase_vocoder :hidden:`mask_along_axis` ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -172,8 +203,3 @@ Functions to perform common audio operations. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. autofunction:: sliding_window_cmn - -:hidden:`vad` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: vad diff --git a/docs/source/index.rst b/docs/source/index.rst index 8146e545c3..596a636b1b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,27 @@ torchaudio ========== +This library is part of the `PyTorch +`_ project. PyTorch is an open source +machine learning framework. + +Features described in this documentation are classified by release status: + + *Stable:* These features will be maintained long-term and there should generally + be no major performance limitations or gaps in documentation. + We also expect to maintain backwards compatibility (although + breaking changes can happen and notice will be given one release ahead + of time). + + *Beta:* Features are tagged as Beta because the API may change based on + user feedback, because the performance needs to improve, or because + coverage across operators is not yet complete. For Beta features, we are + committing to seeing the feature through to the Stable classification. + We are not, however, committing to backwards compatibility. + + *Prototype:* These features are typically not available as part of + binary distributions like PyPI or Conda, except sometimes behind run-time + flags, and are at an early stage for feedback and testing. + The :mod:`torchaudio` package consists of I/O, popular datasets and common audio transformations. @@ -17,3 +39,16 @@ The :mod:`torchaudio` package consists of I/O, popular datasets and common audio compliance.kaldi kaldi_io utils + + +.. toctree:: + :maxdepth: 1 + :caption: PyTorch Libraries + + PyTorch + torchaudio + torchtext + torchvision + TorchElastic + TorchServe + PyTorch on XLA Devices diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 57b23b93d3..24887f5806 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -140,6 +140,7 @@ setup_wheel_python() { 3.6) python_abi=cp36-cp36m ;; 3.7) python_abi=cp37-cp37m ;; 3.8) python_abi=cp38-cp38 ;; + 3.9) python_abi=cp39-cp39 ;; *) echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" exit 1 @@ -181,11 +182,12 @@ setup_pip_pytorch_version() { # # You MUST have populated PYTORCH_VERSION_SUFFIX before hand. setup_conda_pytorch_constraint() { + CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS}" if [[ -z "$PYTORCH_VERSION" ]]; then - export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-test -c pytorch-nightly" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch -c pytorch-test -c pytorch-nightly" fi if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" diff --git a/test/torchaudio_unittest/sox_effect/dataset_test.py b/test/torchaudio_unittest/sox_effect/dataset_test.py index e98095d0dd..7e89960186 100644 --- a/test/torchaudio_unittest/sox_effect/dataset_test.py +++ b/test/torchaudio_unittest/sox_effect/dataset_test.py @@ -1,4 +1,8 @@ +import sys +import platform +from unittest import skipIf from typing import List, Tuple +from concurrent.futures import ProcessPoolExecutor import numpy as np import torch @@ -68,6 +72,13 @@ def init_random_seed(worker_id): @skipIfNoExtension +@skipIf( + platform.system() == 'Darwin' and + sys.version_info.major == 3 and + sys.version_info.minor in [6, 7], + 'This test is known to get stuck for macOS with Python < 3.8. ' + 'See https://github.com/pytorch/pytorch/issues/46409' +) class TestSoxEffectsDataset(TempDirMixin, PytorchTestCase): """Test `apply_effects_file` in multi-process dataloader setting""" @@ -112,3 +123,36 @@ def test_apply_effects_tensor(self): ) for batch in loader: assert batch.shape == (32, 2, 2 * sample_rate) + + +def speed(path): + wav, sample_rate = torchaudio.backend.sox_io_backend.load(path) + effects = [ + ['speed', '1.03756523535464655'], + ['rate', f'{sample_rate}'], + ] + return torchaudio.sox_effects.apply_effects_tensor(wav, sample_rate, effects)[0] + + +@skipIfNoExtension +class TestProcessPoolExecutor(TempDirMixin, PytorchTestCase): + backend = "sox_io" + + def setUp(self): + sample_rate = 16000 + self.flist = [] + for i in range(10): + path = self.get_temp_path(f'{i}.wav') + data = get_whitenoise(n_channels=1, sample_rate=sample_rate, duration=1, dtype='float') + save_wav(path, data, sample_rate) + self.flist.append(path) + + def test_executor(self): + """Test that apply_effects_tensor with speed + rate does not crush + + https://github.com/pytorch/audio/issues/1021 + """ + executor = ProcessPoolExecutor(1) + futures = [executor.submit(speed, path) for path in self.flist] + for future in futures: + future.result() diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index c4be4cb7b0..da068be8f3 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -76,5 +76,7 @@ ExternalProject_Add(libsox DOWNLOAD_DIR ${ARCHIVE_DIR} URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2 URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus + # OpenMP is by default compiled against GNU OpenMP, which conflicts with the version of OpenMP that PyTorch uses. + # See https://github.com/pytorch/audio/pull/1026 + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus --disable-openmp ) diff --git a/torchaudio/__init__.py b/torchaudio/__init__.py index 389d91f635..0d6b495eb1 100644 --- a/torchaudio/__init__.py +++ b/torchaudio/__init__.py @@ -37,7 +37,8 @@ @_mod_utils.deprecated( "Please remove the function call to initialize_sox. " - "Resource initialization is now automatically handled.") + "Resource initialization is now automatically handled.", + "0.8.0") def initialize_sox(): """Initialize sox effects. @@ -50,7 +51,8 @@ def initialize_sox(): "Please remove the function call to torchaudio.shutdown_sox. " "Resource clean up is now automatically handled. " "In the unlikely event that you need to manually shutdown sox, " - "please use torchaudio.sox_effects.shutdown_sox_effects.") + "please use torchaudio.sox_effects.shutdown_sox_effects.", + "0.8.0") def shutdown_sox(): """Shutdown sox effects. diff --git a/torchaudio/datasets/commonvoice.py b/torchaudio/datasets/commonvoice.py index 825cf3e9ab..5b92d9bf21 100644 --- a/torchaudio/datasets/commonvoice.py +++ b/torchaudio/datasets/commonvoice.py @@ -179,6 +179,15 @@ def __init__(self, "romansh sursilvan": "rm-sursilv" } + if download: + raise RuntimeError( + "Common Voice dataset requires user agreement on the usage term, " + "and torchaudio no longer provides the download feature. " + "Please download the dataset manually and extract it in the root directory, " + "then provide the target language to `url` argument.") + if url not in languages: + raise ValueError(f"`url` must be one of available languages: {languages.keys()}") + if url in languages: ext_archive = ".tar.gz" language = languages[url] diff --git a/torchaudio/sox_effects/sox_effects.py b/torchaudio/sox_effects/sox_effects.py index 9262786541..6cd77b0ff4 100644 --- a/torchaudio/sox_effects/sox_effects.py +++ b/torchaudio/sox_effects/sox_effects.py @@ -1,3 +1,4 @@ +import warnings from typing import Any, Callable, List, Optional, Tuple, Union import torch @@ -260,7 +261,7 @@ def apply_effects_file( @_mod_utils.requires_module('torchaudio._torchaudio') -@_mod_utils.deprecated('Please migrate to `apply_effects_file` or `apply_effects_tensor`.') +@_mod_utils.deprecated('Please migrate to `apply_effects_file` or `apply_effects_tensor`.', "0.8.0") def SoxEffect(): r"""Create an object for passing sox effect information between python and c++ @@ -275,7 +276,6 @@ def SoxEffect(): return _torchaudio.SoxEffect() -@_mod_utils.deprecated('Please migrate to `apply_effects_file` or `apply_effects_tensor`.') class SoxEffectsChain(object): r"""SoX effects chain class. @@ -339,6 +339,11 @@ def __init__(self, out_siginfo: Any = None, out_encinfo: Any = None, filetype: str = "raw") -> None: + warnings.warn( + 'torchaudio.sox_effects.SoxEffectsChain has been deprecated and ' + 'will be removed from 0.8.0 release. ' + 'Please migrate to `apply_effects_file` or `apply_effects_tensor`.' + ) self.input_file: Optional[str] = None self.chain: List[str] = [] self.MAX_EFFECT_OPTS = 20