From f6e87264d16128d92609e983fa9e6d2838f8d0e3 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Fri, 6 Nov 2020 19:55:59 +0000 Subject: [PATCH 1/2] Run GPU test for all PRs --- .circleci/config.yml | 48 ----------------------------------------- .circleci/regenerate.py | 3 --- 2 files changed, 51 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e2f964f723..e68387b5b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -760,40 +760,16 @@ workflows: requires: - download_third_parties_nix - unittest_linux_gpu: - filters: - branches: - only: - - master - - nightly - - release/* - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_linux_gpu_py3.6 python_version: '3.6' requires: - download_third_parties_nix - unittest_linux_gpu: - filters: - branches: - only: - - master - - nightly - - release/* - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_linux_gpu_py3.7 python_version: '3.7' requires: - download_third_parties_nix - unittest_linux_gpu: - filters: - branches: - only: - - master - - nightly - - release/* - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_linux_gpu_py3.8 python_version: '3.8' requires: @@ -808,36 +784,12 @@ workflows: name: unittest_windows_cpu_py3.8 python_version: '3.8' - unittest_windows_gpu: - filters: - branches: - only: - - master - - nightly - - release/* - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_windows_gpu_py3.6 python_version: '3.6' - unittest_windows_gpu: - filters: - branches: - only: - - master - - nightly - - release/* - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_windows_gpu_py3.7 python_version: '3.7' - unittest_windows_gpu: - filters: - branches: - only: - - master - - nightly - - release/* - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_windows_gpu_py3.8 python_version: '3.8' - unittest_macos_cpu: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 2f4d4088d9..7527dc8c69 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -147,9 +147,6 @@ def unittest_workflows(indentation=6): "python_version": python_version, } - if device_type == 'gpu': - job['filters'] = gen_filter_branch_tree('master', 'nightly', 'release/*') - if os_type != "windows": job['requires'] = ['download_third_parties_nix'] From ffa67f3d9ac5ac689cf6757b079349a56c42020b Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Fri, 6 Nov 2020 20:41:49 +0000 Subject: [PATCH 2/2] Fix linux GPU test setup --- .circleci/config.yml | 9 +++++++-- .circleci/config.yml.in | 9 +++++++-- .circleci/unittest/linux/scripts/setup_env.sh | 16 +++++++++++----- .circleci/unittest/windows/scripts/setup_env.sh | 14 +++++++++----- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e68387b5b2..b27052977f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,7 +61,7 @@ binary_common: &binary_common python_version: description: "Python version to build against (e.g., 3.8)" type: string - environment: + environment: &environment PYTHON_VERSION: << parameters.python_version >> BUILD_VERSION: << parameters.build_version >> PYTORCH_VERSION: << parameters.pytorch_version >> @@ -432,6 +432,7 @@ jobs: image: ubuntu-1604-cuda-10.1:201909-23 resource_class: gpu.small environment: + <<: *environment image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1" steps: - checkout @@ -444,9 +445,12 @@ jobs: keys: - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} + - run: + name: Pull Docker image + command: docker pull --quiet "${image_name}" - run: name: Setup - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh + command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - save_cache: key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} @@ -509,6 +513,7 @@ jobs: executor: name: windows-gpu environment: + <<: *environment CUDA_VERSION: "10.1" steps: - checkout diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index b9fccbaf30..296be53463 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -61,7 +61,7 @@ binary_common: &binary_common python_version: description: "Python version to build against (e.g., 3.8)" type: string - environment: + environment: &environment PYTHON_VERSION: << parameters.python_version >> BUILD_VERSION: << parameters.build_version >> PYTORCH_VERSION: << parameters.pytorch_version >> @@ -432,6 +432,7 @@ jobs: image: ubuntu-1604-cuda-10.1:201909-23 resource_class: gpu.small environment: + <<: *environment image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1" steps: - checkout @@ -444,9 +445,12 @@ jobs: keys: - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} {% endraw %} + - run: + name: Pull Docker image + command: docker pull --quiet "${image_name}" - run: name: Setup - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh + command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - save_cache: {% raw %} key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }} @@ -509,6 +513,7 @@ jobs: executor: name: windows-gpu environment: + <<: *environment CUDA_VERSION: "10.1" steps: - checkout diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 6e203dea27..26292a00d5 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -21,20 +21,26 @@ esac # 1. Install conda at ./conda if [ ! -d "${conda_dir}" ]; then printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" + wget --quiet -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" bash ./miniconda.sh -b -f -p "${conda_dir}" + eval "$("${conda_dir}/bin/conda" shell.bash hook)" + conda update --quiet -y conda + printf "* Updating the base Python version to %s\n" "${PYTHON_VERSION}" + conda install --quiet -y python="${PYTHON_VERSION}" +else + eval "$("${conda_dir}/bin/conda" shell.bash hook)" fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" + # 2. Create test environment at ./env if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" + printf "* Creating a test environment with PYTHON_VERSION=%s\n" "${PYTHON_VERSION}\n" + conda create --prefix "${env_dir}" -y python="${PYTHON_VERSION}" fi conda activate "${env_dir}" # 3. Install minimal build tools -pip install cmake ninja +pip --quiet install cmake ninja # 4. Buld codecs mkdir -p third_party/build diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh index 71b556d2bd..929a98cb8f 100644 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -19,17 +19,21 @@ if [ ! -d "${conda_dir}" ]; then printf "* Installing conda\n" export tmp_conda="$(echo $conda_dir | tr '/' '\\')" export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe" - curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O + curl --silent --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O "$this_dir/install_conda.bat" unset tmp_conda unset miniconda_exe + eval "$("${conda_dir}/Scripts/conda.exe" 'shell.bash' 'hook')" + conda update --quiet -y conda + printf "* Updating the base Python version to %s\n" "${PYTHON_VERSION}" + conda install --quiet -y python="$PYTHON_VERSION" +else + eval "$("${conda_dir}/Scripts/conda.exe" 'shell.bash' 'hook')" fi -eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')" - # 2. Create test environment at ./env if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" + printf "* Creating a test environment with PYTHON_VERSION=%s\n" "${PYTHON_VERSION}" + conda create --prefix "${env_dir}" -y python="${PYTHON_VERSION}" fi conda activate "${env_dir}"