From bf91515639c1307169a942190c96c3df48c06cb9 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 09:29:22 +0100 Subject: [PATCH 01/20] [DIRTY] cleanup CI config --- .circleci/config.yml | 160 +++++++++++++++++++++++----------------- .pre-commit-config.yaml | 5 ++ 2 files changed, 99 insertions(+), 66 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a5a06f10ef..41aaa136667 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,6 +99,52 @@ commands: - brew_install: formulae: libtool + pip_install: + parameters: + name: + default: pip install + type: string + package: + type: string + options: + default: --user --progress-bar=off + type: string + steps: + - run: + name: <> <> + command: pip install <> <> + + checkout_and_install: + steps: + - checkout + - run: + name: Install torch + command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + - checkout + - run: + name: Install torchvision + command: pip install --user --progress-bar=off --no-build-isolation . + + install_prototype_dependencies: + steps: + - run: + name: Install prototype dependencies + command: pip install --user --progress-bar=off iopath git+https://github.com/pytorch/data + + run_tests: + parameters: + file_or_dir: + type: string + steps: + - pip_install: + name: Install test utilities + package: pytest pytest-mock + - run: + name: Run tests + command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> + - store_test_results: + path: test-results + binary_common: &binary_common parameters: # Edit these defaults to do a release @@ -177,111 +223,93 @@ jobs: python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - python_lint: + lint_python_and_config: docker: - image: circleci/python:3.7 steps: - checkout - run: + name: Install lint utilities command: | pip install --user --progress-bar off pre-commit pre-commit install-hooks - - run: pre-commit run --all-files + - run: + name: Lint Python code and config files + command: pre-commit run --all-files - run: name: Required lint modifications when: on_fail command: git --no-pager diff - python_type_check: + lint_c: docker: - image: circleci/python:3.7 steps: - - checkout - run: + name: Install additional system libraries command: | - sudo apt-get update -y - sudo apt install -y libturbojpeg-dev - pip install --user --progress-bar off mypy - pip install --user --progress-bar off types-requests - pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip install --user --progress-bar off git+https://github.com/pytorch/data.git - pip install --user --progress-bar off --no-build-isolation --editable . - mypy --config-file mypy.ini - - docstring_parameters_sync: - docker: - - image: circleci/python:3.7 - steps: + - sudo apt update -y + - sudo apt install -y libtinfo5 - checkout - run: + name: Install lint utilities command: | - pip install --user pydocstyle - pydocstyle + curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format + chmod +x clang-format + sudo mv clang-format /opt/clang-format + - run: + name: Lint C code + command: ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format + - run: + name: Required lint modifications + when: on_fail + command: git --no-pager diff - clang_format: + type_check_python: docker: - image: circleci/python:3.7 steps: - - checkout - run: + name: Install additional system libraries command: | - sudo apt-get update -y - sudo apt install -y libtinfo5 - curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format - chmod +x clang-format - sudo mv clang-format /opt/clang-format - ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format + - sudo apt update -y + - sudo apt install -y libturbojpeg-dev + - checkout_and_install + - install_prototype_dependencies + - run: + name: Install Python type check utilities + command: pip install --user --progress-bar=off mypy + - run: + name: Check Python types statically + command: mypy --config-file mypy.ini - torchhub_test: + test_torchhub: docker: - image: circleci/python:3.7 steps: - - checkout - - run: - command: | - pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # need to install torchvision dependencies due to transitive imports - pip install --user --progress-bar off --no-build-isolation . - pip install pytest - python test/test_hub.py + - checkout_and_install + - run_tests: + - file_or_dir: test/test_hub.py - torch_onnx_test: + test_onnx: docker: - image: circleci/python:3.7 steps: - - checkout + - checkout_and_install - run: - command: | - pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # need to install torchvision dependencies due to transitive imports - pip install --user --progress-bar off --no-build-isolation . - pip install --user onnx - pip install --user onnxruntime - pip install --user pytest - python test/test_onnx.py - - prototype_test: + name: Install ONNX + command: pip install --user --progress-bar=off onnx onnx-runtime + - run_tests: + - file_or_dir: test/test_onnx.py + + test_prototype: docker: - image: circleci/python:3.7 steps: - - run: - name: Install torch - command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - run: - name: Install prototype dependencies - command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git - - checkout - - run: - name: Install torchvision - command: pip install --user --progress-bar off --no-build-isolation . - - run: - name: Install test requirements - command: pip install --user --progress-bar=off pytest pytest-mock scipy iopath - - run: - name: Run tests - command: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py - - store_test_results: - path: test-results + - checkout_and_install + - install_prototype_dependencies + - run_tests: + - file_or_dir: test/test_prototype_*.py binary_linux_wheel: <<: *binary_common diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 920916030be..38bae737044 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,3 +28,8 @@ repos: hooks: - id: flake8 args: [--config=setup.cfg] + + - repo: https://github.com/PyCQA/pydocstyle/ + rev: 6.1.1 + hooks: + - id: pydocstyle From 0f2408101f862084ef6f4885db16627e3ed0b617 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 09:38:57 +0100 Subject: [PATCH 02/20] remove pip install command --- .circleci/config.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 41aaa136667..53c917859f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,20 +99,20 @@ commands: - brew_install: formulae: libtool - pip_install: - parameters: - name: - default: pip install - type: string - package: - type: string - options: - default: --user --progress-bar=off - type: string - steps: - - run: - name: <> <> - command: pip install <> <> +# pip_install: +# parameters: +# name: +# default: pip install +# type: string +# package: +# type: string +# options: +# default: --user --progress-bar=off +# type: string +# steps: +# - run: +# name: <> <> +# command: pip install <> <> checkout_and_install: steps: @@ -136,9 +136,9 @@ commands: file_or_dir: type: string steps: - - pip_install: + - run: name: Install test utilities - package: pytest pytest-mock + command: pip install --user --progress-bar=off pytest pytest-mock - run: name: Run tests command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> From ce555bc494936408e5851c71e60bc5b31dcc1d77 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 09:46:11 +0100 Subject: [PATCH 03/20] fix syntax --- .circleci/config.yml | 47 ++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53c917859f9..e6b80f6cbbf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,21 +99,6 @@ commands: - brew_install: formulae: libtool -# pip_install: -# parameters: -# name: -# default: pip install -# type: string -# package: -# type: string -# options: -# default: --user --progress-bar=off -# type: string -# steps: -# - run: -# name: <> <> -# command: pip install <> <> - checkout_and_install: steps: - checkout @@ -289,7 +274,7 @@ jobs: steps: - checkout_and_install - run_tests: - - file_or_dir: test/test_hub.py + file_or_dir: test/test_hub.py test_onnx: docker: @@ -300,7 +285,7 @@ jobs: name: Install ONNX command: pip install --user --progress-bar=off onnx onnx-runtime - run_tests: - - file_or_dir: test/test_onnx.py + file_or_dir: test/test_onnx.py test_prototype: docker: @@ -309,7 +294,7 @@ jobs: - checkout_and_install - install_prototype_dependencies - run_tests: - - file_or_dir: test/test_prototype_*.py + file_or_dir: test/test_prototype_*.py binary_linux_wheel: <<: *binary_common @@ -1531,13 +1516,12 @@ workflows: python_version: '3.7' requires: - build_docs - - python_lint - - python_type_check - - docstring_parameters_sync - - clang_format - - torchhub_test - - torch_onnx_test - - prototype_test + - lint_python_and_config + - lint_c + - type_check_python + - test_torchhub + - test_onnx + - test_prototype - binary_ios_build: build_environment: binary-libtorchvision_ops-ios-12.0.0-x86_64 ios_arch: x86_64 @@ -1692,13 +1676,12 @@ workflows: nightly: jobs: - circleci_consistency - - python_lint - - python_type_check - - docstring_parameters_sync - - clang_format - - torchhub_test - - torch_onnx_test - - prototype_test + - lint_python_and_config + - lint_c + - type_check_python + - test_torchhub + - test_onnx + - test_prototype - binary_ios_build: build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-x86_64 filters: From 25d9986dbbbe11b239400e83a68afc5c11e94404 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 09:54:19 +0100 Subject: [PATCH 04/20] fix jobs --- .circleci/config.yml | 14 ++++++++------ .pre-commit-config.yaml | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e6b80f6cbbf..995960258b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,7 +101,6 @@ commands: checkout_and_install: steps: - - checkout - run: name: Install torch command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html @@ -233,8 +232,8 @@ jobs: - run: name: Install additional system libraries command: | - - sudo apt update -y - - sudo apt install -y libtinfo5 + sudo apt update -y + sudo apt install -y libtinfo5 - checkout - run: name: Install lint utilities @@ -257,8 +256,8 @@ jobs: - run: name: Install additional system libraries command: | - - sudo apt update -y - - sudo apt install -y libturbojpeg-dev + sudo apt update -y + sudo apt install -y libturbojpeg-dev - checkout_and_install - install_prototype_dependencies - run: @@ -283,7 +282,7 @@ jobs: - checkout_and_install - run: name: Install ONNX - command: pip install --user --progress-bar=off onnx onnx-runtime + command: pip install --user --progress-bar=off onnx onnxruntime - run_tests: file_or_dir: test/test_onnx.py @@ -293,6 +292,9 @@ jobs: steps: - checkout_and_install - install_prototype_dependencies + - run: + name: Install optional dependencies + run: pip install --user --progress-bar=off scipy - run_tests: file_or_dir: test/test_prototype_*.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38bae737044..7f66a0de672 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: flake8 args: [--config=setup.cfg] - - repo: https://github.com/PyCQA/pydocstyle/ + - repo: https://github.com/PyCQA/pydocstyle rev: 6.1.1 hooks: - id: pydocstyle From 7bf794eb572d0948c65b8250ea1a3f894e7114af Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 09:58:48 +0100 Subject: [PATCH 05/20] fix syntax --- .circleci/config.yml | 2 +- .circleci/regenerate.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 995960258b6..1b21c87acc4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -294,7 +294,7 @@ jobs: - install_prototype_dependencies - run: name: Install optional dependencies - run: pip install --user --progress-bar=off scipy + command: pip install --user --progress-bar=off scipy - run_tests: file_or_dir: test/test_prototype_*.py diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 2413e4adbac..e8098f54a3f 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -347,5 +347,7 @@ def android_workflows(indentation=6, nightly=False): cmake_workflows=cmake_workflows, ios_workflows=ios_workflows, android_workflows=android_workflows, + pip_install="pip install --user --progress-bar=off", + apt_install="sudo apt update -y && apt install -y", ) ) From 2c6d236b6f4d9943bfb518f26a67c03b3bfe6593 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 10:11:31 +0100 Subject: [PATCH 06/20] add support for editable install --- .circleci/config.yml | 31 +++++++++++++++++++------------ .circleci/regenerate.py | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b21c87acc4..75569eeced1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,14 +100,23 @@ commands: formulae: libtool checkout_and_install: + parameters: + editable: + type: boolean + default: true steps: - run: name: Install torch - command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + command: > + pip install --user --progress-bar=off + --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - checkout - run: name: Install torchvision - command: pip install --user --progress-bar=off --no-build-isolation . + command: > + pip install --user --progress-bar=off + <<# parameters.editable >> --editable <> + --no-build-isolation . install_prototype_dependencies: steps: @@ -258,7 +267,8 @@ jobs: command: | sudo apt update -y sudo apt install -y libturbojpeg-dev - - checkout_and_install + - checkout_and_install: + editable: true - install_prototype_dependencies - run: name: Install Python type check utilities @@ -267,7 +277,7 @@ jobs: name: Check Python types statically command: mypy --config-file mypy.ini - test_torchhub: + unittest_torchhub: docker: - image: circleci/python:3.7 steps: @@ -275,7 +285,7 @@ jobs: - run_tests: file_or_dir: test/test_hub.py - test_onnx: + unittest_onnx: docker: - image: circleci/python:3.7 steps: @@ -286,7 +296,7 @@ jobs: - run_tests: file_or_dir: test/test_onnx.py - test_prototype: + unittest_prototype: docker: - image: circleci/python:3.7 steps: @@ -1521,9 +1531,6 @@ workflows: - lint_python_and_config - lint_c - type_check_python - - test_torchhub - - test_onnx - - test_prototype - binary_ios_build: build_environment: binary-libtorchvision_ops-ios-12.0.0-x86_64 ios_arch: x86_64 @@ -1540,6 +1547,9 @@ workflows: unittest: jobs: + - unittest_torchhub + - unittest_onnx + - unittest_prototype - unittest_linux_cpu: cu_version: cpu name: unittest_linux_cpu_py3.6 @@ -1681,9 +1691,6 @@ workflows: - lint_python_and_config - lint_c - type_check_python - - test_torchhub - - test_onnx - - test_prototype - binary_ios_build: build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-x86_64 filters: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index e8098f54a3f..698e7a905a3 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -348,6 +348,6 @@ def android_workflows(indentation=6, nightly=False): ios_workflows=ios_workflows, android_workflows=android_workflows, pip_install="pip install --user --progress-bar=off", - apt_install="sudo apt update -y && apt install -y", + apt_install="sudo apt update -qy && apt install -qy", ) ) From 0f3906435cc52652f90c6ac88f20e1873ccd3ccc Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 10:26:50 +0100 Subject: [PATCH 07/20] sync --- .circleci/config.yml | 34 +++---- .circleci/config.yml.in | 193 +++++++++++++++++++++------------------- 2 files changed, 117 insertions(+), 110 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 75569eeced1..a1c43644f7e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -212,7 +212,7 @@ jobs: - checkout - run: command: | - pip install --user --progress-bar off jinja2 pyyaml + pip install --user --progress-bar=off jinja2 pyyaml python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) @@ -224,7 +224,7 @@ jobs: - run: name: Install lint utilities command: | - pip install --user --progress-bar off pre-commit + pip install --user --progress-bar=off pre-commit pre-commit install-hooks - run: name: Lint Python code and config files @@ -240,9 +240,7 @@ jobs: steps: - run: name: Install additional system libraries - command: | - sudo apt update -y - sudo apt install -y libtinfo5 + command: sudo apt update -qy && apt install -qy libtinfo5 - checkout - run: name: Install lint utilities @@ -264,9 +262,7 @@ jobs: steps: - run: name: Install additional system libraries - command: | - sudo apt update -y - sudo apt install -y libturbojpeg-dev + command: sudo apt update -qy && apt install -qy libturbojpeg-dev - checkout_and_install: editable: true - install_prototype_dependencies @@ -544,7 +540,7 @@ jobs: - checkout - run: command: | - pip install --user awscli + pip install --user --progress-bar=off awscli export PATH="$HOME/.local/bin:$PATH" # Prevent credential from leaking set +x @@ -585,7 +581,7 @@ jobs: command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + pip install --user --progress-bar=off $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -654,7 +650,7 @@ jobs: eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + pip install --user --progress-bar=off $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -980,7 +976,7 @@ jobs: eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env pushd docs - pip install -r requirements.txt + pip install --user --progress-bar=off -r requirements.txt make html popd - persist_to_workspace: @@ -1021,9 +1017,14 @@ jobs: workflows: - build: + lint: jobs: - circleci_consistency + - lint_python_and_config + - lint_c + - type_check_python + build: + jobs: - binary_linux_wheel: conda_docker_image: pytorch/conda-builder:cpu cu_version: cpu @@ -1528,9 +1529,6 @@ workflows: python_version: '3.7' requires: - build_docs - - lint_python_and_config - - lint_c - - type_check_python - binary_ios_build: build_environment: binary-libtorchvision_ops-ios-12.0.0-x86_64 ios_arch: x86_64 @@ -1687,10 +1685,6 @@ workflows: nightly: jobs: - - circleci_consistency - - lint_python_and_config - - lint_c - - type_check_python - binary_ios_build: build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-x86_64 filters: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index d0cbba9e255..45ac66eb723 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -99,6 +99,45 @@ commands: - brew_install: formulae: libtool + checkout_and_install: + parameters: + editable: + type: boolean + default: true + steps: + - run: + name: Install torch + command: > + {{ pip_install }} + --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + - checkout + - run: + name: Install torchvision + command: > + {{ pip_install }} + <<# parameters.editable >> --editable <> + --no-build-isolation . + + install_prototype_dependencies: + steps: + - run: + name: Install prototype dependencies + command: {{ pip_install }} iopath git+https://github.com/pytorch/data + + run_tests: + parameters: + file_or_dir: + type: string + steps: + - run: + name: Install test utilities + command: {{ pip_install }} pytest pytest-mock + - run: + name: Run tests + command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> + - store_test_results: + path: test-results + binary_common: &binary_common parameters: # Edit these defaults to do a release @@ -173,115 +212,97 @@ jobs: - checkout - run: command: | - pip install --user --progress-bar off jinja2 pyyaml + {{ pip_install }} jinja2 pyyaml python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - python_lint: + lint_python_and_config: docker: - image: circleci/python:3.7 steps: - checkout - run: + name: Install lint utilities command: | - pip install --user --progress-bar off pre-commit + {{ pip_install }} pre-commit pre-commit install-hooks - - run: pre-commit run --all-files + - run: + name: Lint Python code and config files + command: pre-commit run --all-files - run: name: Required lint modifications when: on_fail command: git --no-pager diff - python_type_check: + lint_c: docker: - image: circleci/python:3.7 steps: - - checkout - run: - command: | - sudo apt-get update -y - sudo apt install -y libturbojpeg-dev - pip install --user --progress-bar off mypy - pip install --user --progress-bar off types-requests - pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip install --user --progress-bar off git+https://github.com/pytorch/data.git - pip install --user --progress-bar off --no-build-isolation --editable . - mypy --config-file mypy.ini - - docstring_parameters_sync: - docker: - - image: circleci/python:3.7 - steps: + name: Install additional system libraries + command: {{ apt_install }} libtinfo5 - checkout - run: + name: Install lint utilities command: | - pip install --user pydocstyle - pydocstyle + curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format + chmod +x clang-format + sudo mv clang-format /opt/clang-format + - run: + name: Lint C code + command: ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format + - run: + name: Required lint modifications + when: on_fail + command: git --no-pager diff - clang_format: + type_check_python: docker: - image: circleci/python:3.7 steps: - - checkout - run: - command: | - sudo apt-get update -y - sudo apt install -y libtinfo5 - curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format - chmod +x clang-format - sudo mv clang-format /opt/clang-format - ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format + name: Install additional system libraries + command: {{ apt_install }} libturbojpeg-dev + - checkout_and_install: + editable: true + - install_prototype_dependencies + - run: + name: Install Python type check utilities + command: {{ pip_install }} mypy + - run: + name: Check Python types statically + command: mypy --config-file mypy.ini - torchhub_test: + unittest_torchhub: docker: - image: circleci/python:3.7 steps: - - checkout - - run: - command: | - pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # need to install torchvision dependencies due to transitive imports - pip install --user --progress-bar off --no-build-isolation . - pip install pytest - python test/test_hub.py + - checkout_and_install + - run_tests: + file_or_dir: test/test_hub.py - torch_onnx_test: + unittest_onnx: docker: - image: circleci/python:3.7 steps: - - checkout + - checkout_and_install - run: - command: | - pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - # need to install torchvision dependencies due to transitive imports - pip install --user --progress-bar off --no-build-isolation . - pip install --user onnx - pip install --user onnxruntime - pip install --user pytest - python test/test_onnx.py + name: Install ONNX + command: {{ pip_install }} onnx onnxruntime + - run_tests: + file_or_dir: test/test_onnx.py - prototype_test: + unittest_prototype: docker: - image: circleci/python:3.7 steps: + - checkout_and_install + - install_prototype_dependencies - run: - name: Install torch - command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - run: - name: Install prototype dependencies - command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git - - checkout - - run: - name: Install torchvision - command: pip install --user --progress-bar off --no-build-isolation . - - run: - name: Install test requirements - command: pip install --user --progress-bar=off pytest pytest-mock scipy iopath - - run: - name: Run tests - command: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py - - store_test_results: - path: test-results + name: Install optional dependencies + command: {{ pip_install }} scipy + - run_tests: + file_or_dir: test/test_prototype_*.py binary_linux_wheel: <<: *binary_common @@ -519,7 +540,7 @@ jobs: - checkout - run: command: | - pip install --user awscli + {{ pip_install }} awscli export PATH="$HOME/.local/bin:$PATH" # Prevent credential from leaking set +x @@ -560,7 +581,7 @@ jobs: command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + {{ pip_install }} $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -629,7 +650,7 @@ jobs: eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + {{ pip_install }} $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -955,7 +976,7 @@ jobs: eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env pushd docs - pip install -r requirements.txt + {{ pip_install }} -r requirements.txt make html popd - persist_to_workspace: @@ -996,23 +1017,24 @@ jobs: workflows: - build: -{%- if True %} + lint: jobs: - circleci_consistency + - lint_python_and_config + - lint_c + - type_check_python + + build: + jobs: {{ build_workflows(windows_latest_only=True) }} - - python_lint - - python_type_check - - docstring_parameters_sync - - clang_format - - torchhub_test - - torch_onnx_test - - prototype_test {{ ios_workflows() }} {{ android_workflows() }} unittest: jobs: + - unittest_torchhub + - unittest_onnx + - unittest_prototype {{ unittest_workflows() }} cmake: @@ -1020,16 +1042,7 @@ workflows: {{ cmake_workflows() }} nightly: -{%- endif %} jobs: - - circleci_consistency - - python_lint - - python_type_check - - docstring_parameters_sync - - clang_format - - torchhub_test - - torch_onnx_test - - prototype_test {{ ios_workflows(nightly=True) }} {{ android_workflows(nightly=True) }} {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }} From d175715aa42b0c242f934a2df9eeb2e15c51eab2 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 10:29:54 +0100 Subject: [PATCH 08/20] fix sudo apt install --- .circleci/regenerate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 698e7a905a3..b408a7eefec 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -348,6 +348,6 @@ def android_workflows(indentation=6, nightly=False): ios_workflows=ios_workflows, android_workflows=android_workflows, pip_install="pip install --user --progress-bar=off", - apt_install="sudo apt update -qy && apt install -qy", + apt_install="sudo apt update -qy && sudo apt install -qy", ) ) From e9f780a29c0b0bc3b277af9d55b56ac26e1f7771 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 10:31:52 +0100 Subject: [PATCH 09/20] fix editable install --- .circleci/config.yml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 45ac66eb723..0f96406a24a 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -115,8 +115,8 @@ commands: name: Install torchvision command: > {{ pip_install }} - <<# parameters.editable >> --editable <> - --no-build-isolation . + --no-build-isolation + <<# parameters.editable >> --editable <> . install_prototype_dependencies: steps: From e8ac13c974fdc45786b816d533ce03ba1f6f27dd Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 10:32:38 +0100 Subject: [PATCH 10/20] sync --- .circleci/config.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1c43644f7e..b98849d8890 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -115,8 +115,8 @@ commands: name: Install torchvision command: > pip install --user --progress-bar=off - <<# parameters.editable >> --editable <> - --no-build-isolation . + --no-build-isolation + <<# parameters.editable >> --editable <> . install_prototype_dependencies: steps: @@ -240,7 +240,7 @@ jobs: steps: - run: name: Install additional system libraries - command: sudo apt update -qy && apt install -qy libtinfo5 + command: sudo apt update -qy && sudo apt install -qy libtinfo5 - checkout - run: name: Install lint utilities @@ -262,7 +262,7 @@ jobs: steps: - run: name: Install additional system libraries - command: sudo apt update -qy && apt install -qy libturbojpeg-dev + command: sudo apt update -qy && sudo apt install -qy libturbojpeg-dev - checkout_and_install: editable: true - install_prototype_dependencies @@ -1023,6 +1023,7 @@ workflows: - lint_python_and_config - lint_c - type_check_python + build: jobs: - binary_linux_wheel: From be6d7b3a5209ed700b78ce56a1f24d699d9294e4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 10:52:54 +0100 Subject: [PATCH 11/20] try pip without user install --- .circleci/config.yml | 26 +++++++++++++------------- .circleci/regenerate.py | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b98849d8890..db8b7b21942 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,13 +108,13 @@ commands: - run: name: Install torch command: > - pip install --user --progress-bar=off + pip install --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - checkout - run: name: Install torchvision command: > - pip install --user --progress-bar=off + pip install --progress-bar=off --no-build-isolation <<# parameters.editable >> --editable <> . @@ -122,7 +122,7 @@ commands: steps: - run: name: Install prototype dependencies - command: pip install --user --progress-bar=off iopath git+https://github.com/pytorch/data + command: pip install --progress-bar=off iopath git+https://github.com/pytorch/data run_tests: parameters: @@ -131,7 +131,7 @@ commands: steps: - run: name: Install test utilities - command: pip install --user --progress-bar=off pytest pytest-mock + command: pip install --progress-bar=off pytest pytest-mock - run: name: Run tests command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> @@ -212,7 +212,7 @@ jobs: - checkout - run: command: | - pip install --user --progress-bar=off jinja2 pyyaml + pip install --progress-bar=off jinja2 pyyaml python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) @@ -224,7 +224,7 @@ jobs: - run: name: Install lint utilities command: | - pip install --user --progress-bar=off pre-commit + pip install --progress-bar=off pre-commit pre-commit install-hooks - run: name: Lint Python code and config files @@ -268,7 +268,7 @@ jobs: - install_prototype_dependencies - run: name: Install Python type check utilities - command: pip install --user --progress-bar=off mypy + command: pip install --progress-bar=off mypy - run: name: Check Python types statically command: mypy --config-file mypy.ini @@ -288,7 +288,7 @@ jobs: - checkout_and_install - run: name: Install ONNX - command: pip install --user --progress-bar=off onnx onnxruntime + command: pip install --progress-bar=off onnx onnxruntime - run_tests: file_or_dir: test/test_onnx.py @@ -300,7 +300,7 @@ jobs: - install_prototype_dependencies - run: name: Install optional dependencies - command: pip install --user --progress-bar=off scipy + command: pip install --progress-bar=off scipy - run_tests: file_or_dir: test/test_prototype_*.py @@ -540,7 +540,7 @@ jobs: - checkout - run: command: | - pip install --user --progress-bar=off awscli + pip install --progress-bar=off awscli export PATH="$HOME/.local/bin:$PATH" # Prevent credential from leaking set +x @@ -581,7 +581,7 @@ jobs: command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install --user --progress-bar=off $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + pip install --progress-bar=off $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -650,7 +650,7 @@ jobs: eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - pip install --user --progress-bar=off $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + pip install --progress-bar=off $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -976,7 +976,7 @@ jobs: eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env pushd docs - pip install --user --progress-bar=off -r requirements.txt + pip install --progress-bar=off -r requirements.txt make html popd - persist_to_workspace: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index b408a7eefec..9915e749577 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -347,7 +347,7 @@ def android_workflows(indentation=6, nightly=False): cmake_workflows=cmake_workflows, ios_workflows=ios_workflows, android_workflows=android_workflows, - pip_install="pip install --user --progress-bar=off", + pip_install="pip install --progress-bar=off", apt_install="sudo apt update -qy && sudo apt install -qy", ) ) From f861cc3b90695418855076067f9ea3dccff23dc8 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 14:19:00 +0100 Subject: [PATCH 12/20] [DIRTY] address review comments --- .circleci/config.yml | 81 ++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index db8b7b21942..c442df62f14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,19 +20,6 @@ executors: shell: bash.exe commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi designate_upload_channel: description: "inserts the correct upload channel into ${BASH_ENV}" steps: @@ -99,17 +86,37 @@ commands: - brew_install: formulae: libtool - checkout_and_install: + pip_install: parameters: - editable: + args: + type: string + default: args + user: type: boolean default: true + steps: - run: - name: Install torch + name: pip install << parameters.args >> command: > - pip install --progress-bar=off - --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip install + <<# parameters.user >> --user <> + --progress-bar=off + << parameters.args >> + + install_torchvision: + parameters: + editable: + type: boolean + default: true + steps: + - pip_install: + args: --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html +# - run: +# name: Install torch +# command: > +# pip install --progress-bar=off +# --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - checkout - run: name: Install torchvision @@ -124,7 +131,7 @@ commands: name: Install prototype dependencies command: pip install --progress-bar=off iopath git+https://github.com/pytorch/data - run_tests: + run_tests_selective: parameters: file_or_dir: type: string @@ -263,7 +270,7 @@ jobs: - run: name: Install additional system libraries command: sudo apt update -qy && sudo apt install -qy libturbojpeg-dev - - checkout_and_install: + - install_torchvision: editable: true - install_prototype_dependencies - run: @@ -277,31 +284,31 @@ jobs: docker: - image: circleci/python:3.7 steps: - - checkout_and_install - - run_tests: + - install_torchvision + - run_tests_selective: file_or_dir: test/test_hub.py unittest_onnx: docker: - image: circleci/python:3.7 steps: - - checkout_and_install + - install_torchvision - run: name: Install ONNX command: pip install --progress-bar=off onnx onnxruntime - - run_tests: + - run_tests_selective: file_or_dir: test/test_onnx.py unittest_prototype: docker: - image: circleci/python:3.7 steps: - - checkout_and_install + - install_torchvision - install_prototype_dependencies - run: name: Install optional dependencies command: pip install --progress-bar=off scipy - - run_tests: + - run_tests_selective: file_or_dir: test/test_prototype_*.py binary_linux_wheel: @@ -310,7 +317,7 @@ jobs: - image: << parameters.wheel_docker_image >> resource_class: 2xlarge+ steps: - - checkout_merge + - checkout - designate_upload_channel - run: packaging/build_wheel.sh - store_artifacts: @@ -326,7 +333,7 @@ jobs: - image: "<< parameters.conda_docker_image >>" resource_class: 2xlarge+ steps: - - checkout_merge + - checkout - designate_upload_channel - run: packaging/build_conda.sh - store_artifacts: @@ -342,7 +349,7 @@ jobs: <<: *binary_common executor: windows-cpu steps: - - checkout_merge + - checkout - designate_upload_channel - run: name: Build conda packages @@ -373,7 +380,7 @@ jobs: <<: *binary_common executor: windows-cpu steps: - - checkout_merge + - checkout - designate_upload_channel - run: name: Build wheel packages @@ -396,7 +403,7 @@ jobs: macos: xcode: "12.0" steps: - - checkout_merge + - checkout - designate_upload_channel - run: # Cannot easily deduplicate this as source'ing activate @@ -492,7 +499,7 @@ jobs: macos: xcode: "12.0" steps: - - checkout_merge + - checkout - designate_upload_channel - run: command: | @@ -881,7 +888,7 @@ jobs: - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - - checkout_merge + - checkout - designate_upload_channel - run: name: Setup conda @@ -899,7 +906,7 @@ jobs: UNICODE_ABI: << parameters.unicode_abi >> CU_VERSION: << parameters.cu_version >> steps: - - checkout_merge + - checkout - designate_upload_channel - run: name: Setup conda @@ -913,7 +920,7 @@ jobs: macos: xcode: "12.0" steps: - - checkout_merge + - checkout - designate_upload_channel - run: command: | @@ -928,7 +935,7 @@ jobs: executor: name: windows-cpu steps: - - checkout_merge + - checkout - designate_upload_channel - run: command: | @@ -941,7 +948,7 @@ jobs: executor: name: windows-gpu steps: - - checkout_merge + - checkout - designate_upload_channel - run: command: | From d60d443cacf07aa966ed39b10921ebbba12382ce Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 16:44:32 +0100 Subject: [PATCH 13/20] [DIRTY] try use dynamic name --- .circleci/config.yml | 43 +++++++++++++++--------- .circleci/config.yml.in | 73 +++++++++++++++++++++++++++++++---------- .circleci/regenerate.py | 2 -- 3 files changed, 82 insertions(+), 36 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c442df62f14..206694dda25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,18 +86,37 @@ commands: - brew_install: formulae: libtool + apt_install: + parameters: + args: + type: string + default: args + update: + type: boolean + default: true + steps: + - run: + name: apt install << parameters.args >> + command: | + <<# parameters.update >> sudo apt update -qy <> + sudo apt install << parameters.args >> + pip_install: parameters: args: type: string default: args + descr: + type: string + default: "" user: type: boolean default: true - steps: - run: - name: pip install << parameters.args >> + name: > + <<# parameters.descr >> pip install << parameters.args >> <> + <<^ parameters.descr >> << parameters.descr >> <> command: > pip install <<# parameters.user >> --user <> @@ -112,25 +131,17 @@ commands: steps: - pip_install: args: --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html -# - run: -# name: Install torch -# command: > -# pip install --progress-bar=off -# --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + descr: Install PyTorch from nightly releases - checkout - - run: - name: Install torchvision - command: > - pip install --progress-bar=off - --no-build-isolation - <<# parameters.editable >> --editable <> . + - pip_install: + args: --no-build-isolation <<# parameters.editable >> --editable <> . install_prototype_dependencies: steps: - - run: - name: Install prototype dependencies - command: pip install --progress-bar=off iopath git+https://github.com/pytorch/data + - pip_install: + args: iopath git+https://github.com/pytorch/data + # Most of our tests will be run by default. This command can be used if only a selection of tests need to be run run_tests_selective: parameters: file_or_dir: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 0f96406a24a..0079c3fa47b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -99,39 +99,70 @@ commands: - brew_install: formulae: libtool - checkout_and_install: + apt_install: parameters: - editable: + args: + type: string + default: args + update: type: boolean default: true steps: - run: - name: Install torch - command: > - {{ pip_install }} - --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - checkout + name: apt install << parameters.args >> + command: | + <<# parameters.update >> sudo apt update -qy <> + sudo apt install << parameters.args >> + + pip_install: + parameters: + args: + type: string + default: args + descr: + type: string + default: "" + user: + type: boolean + default: true + steps: - run: - name: Install torchvision + name: > + <<# parameters.descr >> pip install << parameters.args >> <> + <<^ parameters.descr >> << parameters.descr >> <> command: > - {{ pip_install }} - --no-build-isolation - <<# parameters.editable >> --editable <> . + pip install + <<# parameters.user >> --user <> + --progress-bar=off + << parameters.args >> + + install_torchvision: + parameters: + editable: + type: boolean + default: true + steps: + - pip_install: + args: --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + descr: Install PyTorch from nightly releases + - checkout + - pip_install: + args: --no-build-isolation <<# parameters.editable >> --editable <> . install_prototype_dependencies: steps: - - run: - name: Install prototype dependencies - command: {{ pip_install }} iopath git+https://github.com/pytorch/data + - pip_install: + args: iopath git+https://github.com/pytorch/data - run_tests: + # Most of our tests will be run by default. This command can be used if only a selection of tests need to be run + run_tests_selective: parameters: file_or_dir: type: string steps: - run: name: Install test utilities - command: {{ pip_install }} pytest pytest-mock + command: pip install --progress-bar=off pytest pytest-mock - run: name: Run tests command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> @@ -210,11 +241,17 @@ jobs: - image: circleci/python:3.7 steps: - checkout + - pip_install: + args: jinja2 pyyaml - run: + name: Re-generate CircleCI config command: | - {{ pip_install }} jinja2 pyyaml python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + git diff --exit-code + - run: + name: Required action + when: on_fail + command: echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config" lint_python_and_config: docker: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 9915e749577..2413e4adbac 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -347,7 +347,5 @@ def android_workflows(indentation=6, nightly=False): cmake_workflows=cmake_workflows, ios_workflows=ios_workflows, android_workflows=android_workflows, - pip_install="pip install --progress-bar=off", - apt_install="sudo apt update -qy && sudo apt install -qy", ) ) From 8fd1f452bb663c1f7207bd0c9c20a35a2eed78c7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 16:49:09 +0100 Subject: [PATCH 14/20] switch logic --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 206694dda25..c9333edf0e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -115,8 +115,8 @@ commands: steps: - run: name: > - <<# parameters.descr >> pip install << parameters.args >> <> - <<^ parameters.descr >> << parameters.descr >> <> + <<^ parameters.descr >> pip install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> command: > pip install <<# parameters.user >> --user <> From 6ee023f0a90bc335f6ec67bd4f286895a8e13d7a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 17:00:49 +0100 Subject: [PATCH 15/20] address remaining comments --- .circleci/config.yml | 104 +++++++++++++++++++++++++--------------- .circleci/config.yml.in | 79 ++++++++++++++++-------------- 2 files changed, 109 insertions(+), 74 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9333edf0e6..48ba7c8db9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,6 +20,19 @@ executors: shell: bash.exe commands: + checkout_merge: + description: "checkout merge branch" + steps: + - checkout +# - run: +# name: Checkout merge branch +# command: | +# set -ex +# BRANCH=$(git rev-parse --abbrev-ref HEAD) +# if [[ "$BRANCH" != "main" ]]; then +# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} +# git checkout "merged/$CIRCLE_BRANCH" +# fi designate_upload_channel: description: "inserts the correct upload channel into ${BASH_ENV}" steps: @@ -90,13 +103,17 @@ commands: parameters: args: type: string - default: args + descr: + type: string + default: "" update: type: boolean default: true steps: - run: - name: apt install << parameters.args >> + name: > + <<^ parameters.descr >> apt install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> command: | <<# parameters.update >> sudo apt update -qy <> sudo apt install << parameters.args >> @@ -105,7 +122,6 @@ commands: parameters: args: type: string - default: args descr: type: string default: "" @@ -222,28 +238,37 @@ smoke_test_common: &smoke_test_common docker: - image: torchvision/smoke_test:latest +# Trigger error + jobs: circleci_consistency: docker: - image: circleci/python:3.7 steps: - checkout + - pip_install: + args: jinja2 pyyaml - run: + name: Re-generate CircleCI config command: | - pip install --progress-bar=off jinja2 pyyaml python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + git diff --exit-code + - run: + name: Required action + when: on_fail + command: echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config" lint_python_and_config: docker: - image: circleci/python:3.7 steps: - checkout + - pip_install: + args: pre-commit + descr: Install lint utilities - run: - name: Install lint utilities - command: | - pip install --progress-bar=off pre-commit - pre-commit install-hooks + name: Install pre-commit hooks + command: pre-commit install-hooks - run: name: Lint Python code and config files command: pre-commit run --all-files @@ -256,9 +281,9 @@ jobs: docker: - image: circleci/python:3.7 steps: - - run: - name: Install additional system libraries - command: sudo apt update -qy && sudo apt install -qy libtinfo5 + - apt_install: + args: libtinfo5 + descr: Install additional system libraries - checkout - run: name: Install lint utilities @@ -278,15 +303,15 @@ jobs: docker: - image: circleci/python:3.7 steps: - - run: - name: Install additional system libraries - command: sudo apt update -qy && sudo apt install -qy libturbojpeg-dev + - apt_install: + args: libturbojpeg-dev + descr: Install additional system libraries - install_torchvision: editable: true - install_prototype_dependencies - - run: - name: Install Python type check utilities - command: pip install --progress-bar=off mypy + - pip_install: + args: mypy + descr: Install Python type check utilities - run: name: Check Python types statically command: mypy --config-file mypy.ini @@ -304,9 +329,9 @@ jobs: - image: circleci/python:3.7 steps: - install_torchvision - - run: - name: Install ONNX - command: pip install --progress-bar=off onnx onnxruntime + - pip_install: + args: onnx onnxruntime + descr: Install ONNX - run_tests_selective: file_or_dir: test/test_onnx.py @@ -316,9 +341,9 @@ jobs: steps: - install_torchvision - install_prototype_dependencies - - run: - name: Install optional dependencies - command: pip install --progress-bar=off scipy + - pip_install: + args: scipy + descr: Install optional dependencies - run_tests_selective: file_or_dir: test/test_prototype_*.py @@ -328,7 +353,7 @@ jobs: - image: << parameters.wheel_docker_image >> resource_class: 2xlarge+ steps: - - checkout + - checkout_merge - designate_upload_channel - run: packaging/build_wheel.sh - store_artifacts: @@ -344,7 +369,7 @@ jobs: - image: "<< parameters.conda_docker_image >>" resource_class: 2xlarge+ steps: - - checkout + - checkout_merge - designate_upload_channel - run: packaging/build_conda.sh - store_artifacts: @@ -360,7 +385,7 @@ jobs: <<: *binary_common executor: windows-cpu steps: - - checkout + - checkout_merge - designate_upload_channel - run: name: Build conda packages @@ -391,7 +416,7 @@ jobs: <<: *binary_common executor: windows-cpu steps: - - checkout + - checkout_merge - designate_upload_channel - run: name: Build wheel packages @@ -414,7 +439,7 @@ jobs: macos: xcode: "12.0" steps: - - checkout + - checkout_merge - designate_upload_channel - run: # Cannot easily deduplicate this as source'ing activate @@ -510,7 +535,7 @@ jobs: macos: xcode: "12.0" steps: - - checkout + - checkout_merge - designate_upload_channel - run: command: | @@ -556,9 +581,10 @@ jobs: at: ~/workspace - designate_upload_channel - checkout + - pip_install: + args: awscli - run: command: | - pip install --progress-bar=off awscli export PATH="$HOME/.local/bin:$PATH" # Prevent credential from leaking set +x @@ -599,7 +625,8 @@ jobs: command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install --progress-bar=off $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + - pip_install: + args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -668,7 +695,8 @@ jobs: eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - pip install --progress-bar=off $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + - pip_install: + args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -899,7 +927,7 @@ jobs: - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - - checkout + - checkout_merge - designate_upload_channel - run: name: Setup conda @@ -917,7 +945,7 @@ jobs: UNICODE_ABI: << parameters.unicode_abi >> CU_VERSION: << parameters.cu_version >> steps: - - checkout + - checkout_merge - designate_upload_channel - run: name: Setup conda @@ -931,7 +959,7 @@ jobs: macos: xcode: "12.0" steps: - - checkout + - checkout_merge - designate_upload_channel - run: command: | @@ -946,7 +974,7 @@ jobs: executor: name: windows-cpu steps: - - checkout + - checkout_merge - designate_upload_channel - run: command: | @@ -959,7 +987,7 @@ jobs: executor: name: windows-gpu steps: - - checkout + - checkout_merge - designate_upload_channel - run: command: | diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 0079c3fa47b..0750615bd39 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -99,17 +99,21 @@ commands: - brew_install: formulae: libtool - apt_install: + apt_install: parameters: args: type: string - default: args + descr: + type: string + default: "" update: type: boolean default: true steps: - run: - name: apt install << parameters.args >> + name: > + <<^ parameters.descr >> apt install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> command: | <<# parameters.update >> sudo apt update -qy <> sudo apt install << parameters.args >> @@ -118,7 +122,6 @@ commands: parameters: args: type: string - default: args descr: type: string default: "" @@ -128,8 +131,8 @@ commands: steps: - run: name: > - <<# parameters.descr >> pip install << parameters.args >> <> - <<^ parameters.descr >> << parameters.descr >> <> + <<^ parameters.descr >> pip install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> command: > pip install <<# parameters.user >> --user <> @@ -258,11 +261,12 @@ jobs: - image: circleci/python:3.7 steps: - checkout + - pip_install: + args: pre-commit + descr: Install lint utilities - run: - name: Install lint utilities - command: | - {{ pip_install }} pre-commit - pre-commit install-hooks + name: Install pre-commit hooks + command: pre-commit install-hooks - run: name: Lint Python code and config files command: pre-commit run --all-files @@ -275,9 +279,9 @@ jobs: docker: - image: circleci/python:3.7 steps: - - run: - name: Install additional system libraries - command: {{ apt_install }} libtinfo5 + - apt_install: + args: libtinfo5 + descr: Install additional system libraries - checkout - run: name: Install lint utilities @@ -297,15 +301,15 @@ jobs: docker: - image: circleci/python:3.7 steps: - - run: - name: Install additional system libraries - command: {{ apt_install }} libturbojpeg-dev - - checkout_and_install: + - apt_install: + args: libturbojpeg-dev + descr: Install additional system libraries + - install_torchvision: editable: true - install_prototype_dependencies - - run: - name: Install Python type check utilities - command: {{ pip_install }} mypy + - pip_install: + args: mypy + descr: Install Python type check utilities - run: name: Check Python types statically command: mypy --config-file mypy.ini @@ -314,31 +318,31 @@ jobs: docker: - image: circleci/python:3.7 steps: - - checkout_and_install - - run_tests: + - install_torchvision + - run_tests_selective: file_or_dir: test/test_hub.py unittest_onnx: docker: - image: circleci/python:3.7 steps: - - checkout_and_install - - run: - name: Install ONNX - command: {{ pip_install }} onnx onnxruntime - - run_tests: + - install_torchvision + - pip_install: + args: onnx onnxruntime + descr: Install ONNX + - run_tests_selective: file_or_dir: test/test_onnx.py unittest_prototype: docker: - image: circleci/python:3.7 steps: - - checkout_and_install + - install_torchvision - install_prototype_dependencies - - run: - name: Install optional dependencies - command: {{ pip_install }} scipy - - run_tests: + - pip_install: + args: scipy + descr: Install optional dependencies + - run_tests_selective: file_or_dir: test/test_prototype_*.py binary_linux_wheel: @@ -575,9 +579,10 @@ jobs: at: ~/workspace - designate_upload_channel - checkout + - pip_install: + args: awscli - run: command: | - {{ pip_install }} awscli export PATH="$HOME/.local/bin:$PATH" # Prevent credential from leaking set +x @@ -618,7 +623,8 @@ jobs: command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - {{ pip_install }} $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + - pip_install: + args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -687,7 +693,8 @@ jobs: eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - {{ pip_install }} $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + - pip_install: + args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - run: name: smoke test command: | @@ -1013,7 +1020,7 @@ jobs: eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env pushd docs - {{ pip_install }} -r requirements.txt + pip install --progress-bar=off -r requirements.txt make html popd - persist_to_workspace: From 9564ccd85869837e99ef0949061f2c1f3805429c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 17:03:11 +0100 Subject: [PATCH 16/20] cleanup --- .circleci/config.yml | 10 ++-------- .circleci/config.yml.in | 8 ++------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 48ba7c8db9d..03a4886883a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -238,8 +238,6 @@ smoke_test_common: &smoke_test_common docker: - image: torchvision/smoke_test:latest -# Trigger error - jobs: circleci_consistency: docker: @@ -249,14 +247,10 @@ jobs: - pip_install: args: jinja2 pyyaml - run: - name: Re-generate CircleCI config + name: Check CircleCI config consistency command: | python .circleci/regenerate.py - git diff --exit-code - - run: - name: Required action - when: on_fail - command: echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config" + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) lint_python_and_config: docker: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 0750615bd39..fcdf2afe3c2 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -247,14 +247,10 @@ jobs: - pip_install: args: jinja2 pyyaml - run: - name: Re-generate CircleCI config + name: Check CircleCI config consistency command: | python .circleci/regenerate.py - git diff --exit-code - - run: - name: Required action - when: on_fail - command: echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config" + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) lint_python_and_config: docker: From 2b8b236ae29cf8ccfd0888fe7bb1e8d007997cfd Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 Nov 2021 17:10:36 +0100 Subject: [PATCH 17/20] more cleanup --- .circleci/config.yml | 2 ++ .circleci/config.yml.in | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03a4886883a..108f8a20ea0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -151,11 +151,13 @@ commands: - checkout - pip_install: args: --no-build-isolation <<# parameters.editable >> --editable <> . + descr: Install torchvision <<# parameters.editable >> in editable mode <> install_prototype_dependencies: steps: - pip_install: args: iopath git+https://github.com/pytorch/data + descr: Install prototype dependencies # Most of our tests will be run by default. This command can be used if only a selection of tests need to be run run_tests_selective: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index fcdf2afe3c2..ed7237696a4 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -151,11 +151,13 @@ commands: - checkout - pip_install: args: --no-build-isolation <<# parameters.editable >> --editable <> . + descr: Install torchvision <<# parameters.editable >> in editable mode <> install_prototype_dependencies: steps: - pip_install: args: iopath git+https://github.com/pytorch/data + descr: Install prototype dependencies # Most of our tests will be run by default. This command can be used if only a selection of tests need to be run run_tests_selective: From 85d628a4b6c75a4daa3136b61c8213ae8c63a4df Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 29 Nov 2021 11:08:24 +0100 Subject: [PATCH 18/20] fix enabling prototype tests --- .circleci/config.yml | 5 +++-- .circleci/config.yml.in | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 22c714bd9ca..4adac4d6b88 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -351,10 +351,11 @@ jobs: - pip_install: args: scipy descr: Install optional dependencies + - run: + name: Enable prototype tests + command: echo 'export PYTORCH_TEST_WITH_PROTOTYPE=1' >> $BASH_ENV - run_tests_selective: file_or_dir: test/test_prototype_*.py - environment: - PYTORCH_TEST_WITH_PROTOTYPE: 1 binary_linux_wheel: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 127e26f1092..39d5ae7951f 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -351,10 +351,11 @@ jobs: - pip_install: args: scipy descr: Install optional dependencies + - run: + name: Enable prototype tests + command: echo 'export PYTORCH_TEST_WITH_PROTOTYPE=1' >> $BASH_ENV - run_tests_selective: file_or_dir: test/test_prototype_*.py - environment: - PYTORCH_TEST_WITH_PROTOTYPE: 1 binary_linux_wheel: <<: *binary_common From 82c271f4ae7d3d43b4f393c1406fbff0a4f3323e Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 30 Nov 2021 08:18:11 +0100 Subject: [PATCH 19/20] Update .circleci/config.yml.in Co-authored-by: Nicolas Hug --- .circleci/config.yml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index ed7237696a4..c0951780049 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -159,7 +159,7 @@ commands: args: iopath git+https://github.com/pytorch/data descr: Install prototype dependencies - # Most of our tests will be run by default. This command can be used if only a selection of tests need to be run + # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. This command can be used if only a selection of tests need to be run, for ad-hoc files. run_tests_selective: parameters: file_or_dir: From 8513fab55d386ad7521434b6ddf9712007457de7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 30 Nov 2021 08:20:40 +0100 Subject: [PATCH 20/20] linebreak --- .circleci/config.yml | 3 ++- .circleci/config.yml.in | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c98642e50fb..90ff1ffe079 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -158,7 +158,8 @@ commands: args: iopath git+https://github.com/pytorch/data descr: Install prototype dependencies - # Most of our tests will be run by default. This command can be used if only a selection of tests need to be run + # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. + # This command can be used if only a selection of tests need to be run, for ad-hoc files. run_tests_selective: parameters: file_or_dir: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index be8a37079ef..cadd8efccb2 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -158,7 +158,8 @@ commands: args: iopath git+https://github.com/pytorch/data descr: Install prototype dependencies - # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. This command can be used if only a selection of tests need to be run, for ad-hoc files. + # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. + # This command can be used if only a selection of tests need to be run, for ad-hoc files. run_tests_selective: parameters: file_or_dir: