From 3fa0de9988db3ab0b4d18e68832edcfa687d0202 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 14 Oct 2020 21:24:10 +0000 Subject: [PATCH 1/5] Disable CI cache --- .circleci/config.yml | 16 ---------------- .circleci/config.yml.in | 16 ---------------- 2 files changed, 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef0060200c..fefff2523a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -552,25 +552,9 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel - - generate_cache_key - - restore_cache: - - keys: - - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} - - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} - - paths: - - conda - - env - - third_party/build - - third_party/install - - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index bb073c6625..6aa6d9738f 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -552,25 +552,9 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel - - generate_cache_key - - restore_cache: - {% raw %} - keys: - - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} - {% endraw %} - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - {% raw %} - key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} - {% endraw %} - paths: - - conda - - env - - third_party/build - - third_party/install - - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh From 7be8470328483d02df28d36215031db63aa97d4d Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Fri, 9 Oct 2020 21:07:35 +0000 Subject: [PATCH 2/5] Remove cpuonly from macOS --- .circleci/unittest/linux/scripts/install.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index f037d1dbad..bbbca64d59 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -11,13 +11,16 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env if [ -z "${CUDA_VERSION:-}" ] ; then - cudatoolkit="cpuonly" + case "$(uname -s)" in + Darwin*) cudatoolkit="";; + *) cudatoolkit="cpuonly" + esac else version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} printf "* Installing torchaudio\n" BUILD_SOX=1 python setup.py install From 8ec0f78e3214cc811e3d60a5cf0281d2e7008c7d Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 14 Oct 2020 22:01:28 +0000 Subject: [PATCH 3/5] Print python version before installing --- .circleci/unittest/linux/scripts/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index bbbca64d59..5c66f5929d 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -9,6 +9,7 @@ set -e eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env +python --version if [ -z "${CUDA_VERSION:-}" ] ; then case "$(uname -s)" in From cb7b5973bb099bea819134ba6d11d0c161a46396 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 14 Oct 2020 22:55:27 +0000 Subject: [PATCH 4/5] Install PyTorch/torchaudio first --- .circleci/unittest/linux/scripts/install.sh | 13 ++++++++++++- .../unittest/linux/scripts/run_style_checks.sh | 2 ++ .circleci/unittest/linux/scripts/setup_env.sh | 17 ----------------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 5c66f5929d..bde82ff9ee 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -9,7 +9,6 @@ set -e eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env -python --version if [ -z "${CUDA_VERSION:-}" ] ; then case "$(uname -s)" in @@ -23,5 +22,17 @@ fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} +printf "* Installing dependencies for test\n" +conda install -y -c conda-forge pytest pytest-cov codecov 'librosa>=0.8.0' scipy parameterized +pip install kaldi-io + +printf "* Building codecs\n" +mkdir -p third_party/build +( + cd third_party/build + cmake .. + cmake --build . +) + printf "* Installing torchaudio\n" BUILD_SOX=1 python setup.py install diff --git a/.circleci/unittest/linux/scripts/run_style_checks.sh b/.circleci/unittest/linux/scripts/run_style_checks.sh index 70b83e19f5..8eff8a2b52 100755 --- a/.circleci/unittest/linux/scripts/run_style_checks.sh +++ b/.circleci/unittest/linux/scripts/run_style_checks.sh @@ -5,6 +5,8 @@ set -u eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env +pip install clang-format flake8 + # We want to run all the style checks even if one of them fail. exit_status=0 diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 290f0f8f97..09c10bb3b1 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -7,7 +7,6 @@ set -e -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" conda_dir="${root_dir}/conda" env_dir="${root_dir}/env" @@ -32,19 +31,3 @@ if [ ! -d "${env_dir}" ]; then printf "* Creating a test environment\n" conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" fi -conda activate "${env_dir}" - -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -conda env update --file "${this_dir}/environment.yml" --prune -if [ "${os}" == Linux ] ; then - pip install clang-format -fi - -# 4. Buld codecs -mkdir -p third_party/build -( - cd third_party/build - cmake .. - cmake --build . -) From 04f3c05ca7e3f75b7685ea043d5cc90a13a0e454 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Fri, 16 Oct 2020 20:29:40 +0000 Subject: [PATCH 5/5] Remove env file --- .../unittest/linux/scripts/environment.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .circleci/unittest/linux/scripts/environment.yml diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml deleted file mode 100644 index 4315a2205b..0000000000 --- a/.circleci/unittest/linux/scripts/environment.yml +++ /dev/null @@ -1,17 +0,0 @@ -# a comment for these trying times -channels: - - conda-forge - - defaults -dependencies: - - flake8 - - numpy >= 1.11 - - pytest - - pytest-cov - - codecov - - librosa>=0.8.0 - - llvmlite==0.31 # See https://github.com/pytorch/audio/pull/766 - - pip - - pip: - - kaldi-io - - scipy - - parameterized