From b1604bb892d1e7f0b875dadba2767215c67aa677 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Wed, 4 Sep 2019 10:59:36 +0200 Subject: [PATCH 01/35] [WIP] Add CircleCI for CI --- .travis.yml | 76 -------------------------------------- packaging/pkg_helpers.bash | 3 +- 2 files changed, 2 insertions(+), 77 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fc0aa7240a4..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,76 +0,0 @@ -language: python - -dist: xenial -matrix: - include: - - env: FORMAT_CHECK - language: cpp - addons: - apt: - sources: - - llvm-toolchain-xenial-7 - packages: - - clang-7 - - clang-format-7 - before_install: skip - install: skip - script: ./travis-scripts/run-clang-format/run-clang-format.py -r torchvision/csrc - - env: LINT_CHECK - python: "2.7" - install: pip install flake8 - script: flake8 - after_success: [] - - python: "2.7" - env: IMAGE_BACKEND=Pillow-SIMD - - python: "2.7" - - python: "3.6" - env: IMAGE_BACKEND=Pillow-SIMD - - python: "3.6" - -before_install: - - sudo apt-get update - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - # Useful for debugging any issues with conda - - conda info -a - - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytorch scipy -c pytorch-nightly - - source activate test-environment - - | - if [[ "$IMAGE_BACKEND" == "Pillow-SIMD" ]]; then - pip uninstall -y pillow && CC="cc -march=native" pip install --force-reinstall pillow-simd - fi - - pip install future - - pip install pytest pytest-cov codecov - - pip install mock - - conda install av -c conda-forge - - -install: - # Using pip instead of setup.py ensures we install a non-compressed version of the package - # (as opposed to an egg), which is necessary to collect coverage. - # We still get the benefit of testing an installed version over the - # test version to iron out installation file-inclusion bugs but can - # also collect coverage. - - pip install . - # Move to home dir, otherwise we'll end up with the path to the - # package in $PWD rather than the installed v - - | - cd $HOME - export TV_INSTALL_PATH="$(python -c 'import os; import torchvision; print(os.path.dirname(os.path.abspath(torchvision.__file__)))')" - echo "$TV_INSTALL_PATH" - cd - - -script: - - pytest --cov-config .coveragerc --cov torchvision --cov $TV_INSTALL_PATH test - -after_success: - # Necessary to run coverage combine to rewrite paths from - # /travis/env/path/site-packages/torchvision to actual path - - coverage combine .coverage - - coverage report - - codecov diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index bd9cbedcdef..51c98b7dd7f 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -45,7 +45,8 @@ setup_cuda() { export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX" # If the suffix is non-empty, we will use a wheel subdirectory if [[ -n "$PYTORCH_VERSION_SUFFIX" ]]; then - export WHEEL_DIR="$PYTORCH_VERSION_SUFFIX/" + # export WHEEL_DIR="$PYTORCH_VERSION_SUFFIX/" + export WHEEL_DIR="$CU_VERSION/" fi fi fi From 9b80188a1a0343100dfc896347878834307a077f Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Wed, 4 Sep 2019 11:02:17 +0200 Subject: [PATCH 02/35] Make jobs only run on master --- .circleci/config.yml | 288 ++++++++++++++++++++++++++++++++++++++++ .circleci/config.yml.in | 4 + 2 files changed, 292 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index a5be618c982..4df468321d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -175,159 +175,303 @@ workflows: name: binary_linux_wheel_py2.7_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_wheel: name: binary_linux_wheel_py2.7u_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master unicode_abi: "1" - binary_linux_wheel: name: binary_linux_wheel_py2.7_cu92 python_version: "2.7" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py2.7u_cu92 python_version: "2.7" cu_version: "cu92" + filters: + branches: + only: + - master unicode_abi: "1" wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py2.7_cu100 python_version: "2.7" cu_version: "cu100" + filters: + branches: + only: + - master - binary_linux_wheel: name: binary_linux_wheel_py2.7u_cu100 python_version: "2.7" cu_version: "cu100" + filters: + branches: + only: + - master unicode_abi: "1" - binary_linux_wheel: name: binary_linux_wheel_py3.5_cpu python_version: "3.5" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_wheel: name: binary_linux_wheel_py3.5_cu92 python_version: "3.5" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py3.5_cu100 python_version: "3.5" cu_version: "cu100" + filters: + branches: + only: + - master - binary_linux_wheel: name: binary_linux_wheel_py3.6_cpu python_version: "3.6" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_wheel: name: binary_linux_wheel_py3.6_cu92 python_version: "3.6" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py3.6_cu100 python_version: "3.6" cu_version: "cu100" + filters: + branches: + only: + - master - binary_linux_wheel: name: binary_linux_wheel_py3.7_cpu python_version: "3.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_wheel: name: binary_linux_wheel_py3.7_cu92 python_version: "3.7" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py3.7_cu100 python_version: "3.7" cu_version: "cu100" + filters: + branches: + only: + - master - binary_macos_wheel: name: binary_macos_wheel_py2.7_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_macos_wheel: name: binary_macos_wheel_py2.7u_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master unicode_abi: "1" - binary_macos_wheel: name: binary_macos_wheel_py3.5_cpu python_version: "3.5" cu_version: "cpu" + filters: + branches: + only: + - master - binary_macos_wheel: name: binary_macos_wheel_py3.6_cpu python_version: "3.6" cu_version: "cpu" + filters: + branches: + only: + - master - binary_macos_wheel: name: binary_macos_wheel_py3.7_cpu python_version: "3.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_conda: name: binary_linux_conda_py2.7_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_conda: name: binary_linux_conda_py2.7_cu92 python_version: "2.7" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_conda: name: binary_linux_conda_py2.7_cu100 python_version: "2.7" cu_version: "cu100" + filters: + branches: + only: + - master - binary_linux_conda: name: binary_linux_conda_py3.5_cpu python_version: "3.5" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_conda: name: binary_linux_conda_py3.5_cu92 python_version: "3.5" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_conda: name: binary_linux_conda_py3.5_cu100 python_version: "3.5" cu_version: "cu100" + filters: + branches: + only: + - master - binary_linux_conda: name: binary_linux_conda_py3.6_cpu python_version: "3.6" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_conda: name: binary_linux_conda_py3.6_cu92 python_version: "3.6" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_conda: name: binary_linux_conda_py3.6_cu100 python_version: "3.6" cu_version: "cu100" + filters: + branches: + only: + - master - binary_linux_conda: name: binary_linux_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_linux_conda: name: binary_linux_conda_py3.7_cu92 python_version: "3.7" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_conda: name: binary_linux_conda_py3.7_cu100 python_version: "3.7" cu_version: "cu100" + filters: + branches: + only: + - master - binary_macos_conda: name: binary_macos_conda_py2.7_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_macos_conda: name: binary_macos_conda_py3.5_cpu python_version: "3.5" cu_version: "cpu" + filters: + branches: + only: + - master - binary_macos_conda: name: binary_macos_conda_py3.6_cpu python_version: "3.6" cu_version: "cpu" + filters: + branches: + only: + - master - binary_macos_conda: name: binary_macos_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" + filters: + branches: + only: + - master nightly: triggers: @@ -344,6 +488,10 @@ workflows: name: nightly_binary_linux_wheel_py2.7_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7_cpu_upload context: org-member @@ -354,6 +502,10 @@ workflows: name: nightly_binary_linux_wheel_py2.7u_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master unicode_abi: "1" - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7u_cpu_upload @@ -365,6 +517,10 @@ workflows: name: nightly_binary_linux_wheel_py2.7_cu92 python_version: "2.7" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7_cu92_upload @@ -376,6 +532,10 @@ workflows: name: nightly_binary_linux_wheel_py2.7u_cu92 python_version: "2.7" cu_version: "cu92" + filters: + branches: + only: + - master unicode_abi: "1" wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: @@ -388,6 +548,10 @@ workflows: name: nightly_binary_linux_wheel_py2.7_cu100 python_version: "2.7" cu_version: "cu100" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7_cu100_upload context: org-member @@ -398,6 +562,10 @@ workflows: name: nightly_binary_linux_wheel_py2.7u_cu100 python_version: "2.7" cu_version: "cu100" + filters: + branches: + only: + - master unicode_abi: "1" - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7u_cu100_upload @@ -409,6 +577,10 @@ workflows: name: nightly_binary_linux_wheel_py3.5_cpu python_version: "3.5" cu_version: "cpu" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.5_cpu_upload context: org-member @@ -419,6 +591,10 @@ workflows: name: nightly_binary_linux_wheel_py3.5_cu92 python_version: "3.5" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.5_cu92_upload @@ -430,6 +606,10 @@ workflows: name: nightly_binary_linux_wheel_py3.5_cu100 python_version: "3.5" cu_version: "cu100" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.5_cu100_upload context: org-member @@ -440,6 +620,10 @@ workflows: name: nightly_binary_linux_wheel_py3.6_cpu python_version: "3.6" cu_version: "cpu" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.6_cpu_upload context: org-member @@ -450,6 +634,10 @@ workflows: name: nightly_binary_linux_wheel_py3.6_cu92 python_version: "3.6" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.6_cu92_upload @@ -461,6 +649,10 @@ workflows: name: nightly_binary_linux_wheel_py3.6_cu100 python_version: "3.6" cu_version: "cu100" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.6_cu100_upload context: org-member @@ -471,6 +663,10 @@ workflows: name: nightly_binary_linux_wheel_py3.7_cpu python_version: "3.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.7_cpu_upload context: org-member @@ -481,6 +677,10 @@ workflows: name: nightly_binary_linux_wheel_py3.7_cu92 python_version: "3.7" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.7_cu92_upload @@ -492,6 +692,10 @@ workflows: name: nightly_binary_linux_wheel_py3.7_cu100 python_version: "3.7" cu_version: "cu100" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.7_cu100_upload context: org-member @@ -502,6 +706,10 @@ workflows: name: nightly_binary_macos_wheel_py2.7_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_macos_wheel_py2.7_cpu_upload context: org-member @@ -512,6 +720,10 @@ workflows: name: nightly_binary_macos_wheel_py2.7u_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master unicode_abi: "1" - binary_wheel_upload: name: nightly_binary_macos_wheel_py2.7u_cpu_upload @@ -523,6 +735,10 @@ workflows: name: nightly_binary_macos_wheel_py3.5_cpu python_version: "3.5" cu_version: "cpu" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_macos_wheel_py3.5_cpu_upload context: org-member @@ -533,6 +749,10 @@ workflows: name: nightly_binary_macos_wheel_py3.6_cpu python_version: "3.6" cu_version: "cpu" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_macos_wheel_py3.6_cpu_upload context: org-member @@ -543,6 +763,10 @@ workflows: name: nightly_binary_macos_wheel_py3.7_cpu python_version: "3.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_wheel_upload: name: nightly_binary_macos_wheel_py3.7_cpu_upload context: org-member @@ -553,6 +777,10 @@ workflows: name: nightly_binary_linux_conda_py2.7_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_linux_conda_py2.7_cpu_upload context: org-member @@ -562,6 +790,10 @@ workflows: name: nightly_binary_linux_conda_py2.7_cu92 python_version: "2.7" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_conda_upload: name: nightly_binary_linux_conda_py2.7_cu92_upload @@ -572,6 +804,10 @@ workflows: name: nightly_binary_linux_conda_py2.7_cu100 python_version: "2.7" cu_version: "cu100" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_linux_conda_py2.7_cu100_upload context: org-member @@ -581,6 +817,10 @@ workflows: name: nightly_binary_linux_conda_py3.5_cpu python_version: "3.5" cu_version: "cpu" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.5_cpu_upload context: org-member @@ -590,6 +830,10 @@ workflows: name: nightly_binary_linux_conda_py3.5_cu92 python_version: "3.5" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_conda_upload: name: nightly_binary_linux_conda_py3.5_cu92_upload @@ -600,6 +844,10 @@ workflows: name: nightly_binary_linux_conda_py3.5_cu100 python_version: "3.5" cu_version: "cu100" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.5_cu100_upload context: org-member @@ -609,6 +857,10 @@ workflows: name: nightly_binary_linux_conda_py3.6_cpu python_version: "3.6" cu_version: "cpu" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.6_cpu_upload context: org-member @@ -618,6 +870,10 @@ workflows: name: nightly_binary_linux_conda_py3.6_cu92 python_version: "3.6" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_conda_upload: name: nightly_binary_linux_conda_py3.6_cu92_upload @@ -628,6 +884,10 @@ workflows: name: nightly_binary_linux_conda_py3.6_cu100 python_version: "3.6" cu_version: "cu100" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.6_cu100_upload context: org-member @@ -637,6 +897,10 @@ workflows: name: nightly_binary_linux_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.7_cpu_upload context: org-member @@ -646,6 +910,10 @@ workflows: name: nightly_binary_linux_conda_py3.7_cu92 python_version: "3.7" cu_version: "cu92" + filters: + branches: + only: + - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_conda_upload: name: nightly_binary_linux_conda_py3.7_cu92_upload @@ -656,6 +924,10 @@ workflows: name: nightly_binary_linux_conda_py3.7_cu100 python_version: "3.7" cu_version: "cu100" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.7_cu100_upload context: org-member @@ -665,6 +937,10 @@ workflows: name: nightly_binary_macos_conda_py2.7_cpu python_version: "2.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_macos_conda_py2.7_cpu_upload context: org-member @@ -674,6 +950,10 @@ workflows: name: nightly_binary_macos_conda_py3.5_cpu python_version: "3.5" cu_version: "cpu" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_macos_conda_py3.5_cpu_upload context: org-member @@ -683,6 +963,10 @@ workflows: name: nightly_binary_macos_conda_py3.6_cpu python_version: "3.6" cu_version: "cpu" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_macos_conda_py3.6_cpu_upload context: org-member @@ -692,6 +976,10 @@ workflows: name: nightly_binary_macos_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" + filters: + branches: + only: + - master - binary_conda_upload: name: nightly_binary_macos_conda_py3.7_cpu_upload context: org-member diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 91046fcd7b8..834cb7546fd 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -171,6 +171,10 @@ jobs: name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}} python_version: "{{python_version}}" cu_version: "{{cu_version}}" + filters: + branches: + only: + - master {%- if unicode %} unicode_abi: "1" {%- endif %} From 61745678e916b067f108a4b8f32903899de4c7fb Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Wed, 4 Sep 2019 11:16:46 +0200 Subject: [PATCH 03/35] Add initial CI --- .circleci/config.yml | 4 ++++ .circleci/config.yml.in | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4df468321d7..1a9665d0270 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -472,6 +472,10 @@ workflows: branches: only: - master + - binary_linux_conda: + name: torchvision_linux_py3.6_cpu + python_version: "3.6" + cu_version: "cpu" nightly: triggers: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 834cb7546fd..43ec2921ae8 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -218,6 +218,10 @@ workflows: jobs: - circleci_consistency {{ workflows() }} + - binary_linux_conda: + name: torchvision_linux_py3.6_cpu + python_version: "3.6" + cu_version: "cpu" nightly: triggers: From f13c67c2d74402c93926b1fdf7d52512339a7180 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Wed, 4 Sep 2019 11:20:30 +0200 Subject: [PATCH 04/35] [wip] testing if works --- .circleci/config.yml | 6 +++--- .circleci/config.yml.in | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a9665d0270..dea62dc60d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -473,9 +473,9 @@ workflows: only: - master - binary_linux_conda: - name: torchvision_linux_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + name: torchvision_linux_py3.6_cpu + python_version: "3.6" + cu_version: "cpu" nightly: triggers: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 43ec2921ae8..e4db8609e67 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -219,9 +219,9 @@ workflows: - circleci_consistency {{ workflows() }} - binary_linux_conda: - name: torchvision_linux_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + name: torchvision_linux_py3.6_cpu + python_version: "3.6" + cu_version: "cpu" nightly: triggers: From 956893fc7d64bbdaa485d86f6a2281716dbe5040 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Wed, 4 Sep 2019 13:30:33 +0200 Subject: [PATCH 05/35] Trying some basic GPU tests --- .circleci/config.yml | 30 ++++++++++++++++++++++++++++++ .circleci/config.yml.in | 30 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dea62dc60d5..ef5c88c9b94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,6 +81,32 @@ jobs: paths: - "*" + binary_linux_conda_cuda: + <<: *binary_common + machine: + image: ubuntu-1604:201903-01 + resource_class: gpu.medium + steps: + - checkout + - run: + name: Build + command: | + set -e + docker --version + # from https://github.com/NVIDIA/nvidia-docker#ubuntu-16041804-debian-jessiestretchbuster + # Add the package repositories + distribution=$(. /etc/os-release;echo $ID$VERSION_ID) + curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + + sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit + sudo systemctl restart docker + + docker --version + # start + docker pull soumith/conda-cuda >/dev/null + + binary_macos_wheel: <<: *binary_common macos: @@ -476,6 +502,10 @@ workflows: name: torchvision_linux_py3.6_cpu python_version: "3.6" cu_version: "cpu" + - binary_linux_conda_cuda: + name: torchvision_linux_py3.6_cu100 + python_version: "3.6" + cu_version: "cu100" nightly: triggers: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index e4db8609e67..9d4ba97244e 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -81,6 +81,32 @@ jobs: paths: - "*" + binary_linux_conda_cuda: + <<: *binary_common + machine: + image: ubuntu-1604:201903-01 + resource_class: gpu.medium + steps: + - checkout + - run: + name: Build + command: | + set -e + docker --version + # from https://github.com/NVIDIA/nvidia-docker#ubuntu-16041804-debian-jessiestretchbuster + # Add the package repositories + distribution=$(. /etc/os-release;echo $ID$VERSION_ID) + curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + + sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit + sudo systemctl restart docker + + docker --version + # start + docker pull soumith/conda-cuda >/dev/null + + binary_macos_wheel: <<: *binary_common macos: @@ -222,6 +248,10 @@ workflows: name: torchvision_linux_py3.6_cpu python_version: "3.6" cu_version: "cpu" + - binary_linux_conda_cuda: + name: torchvision_linux_py3.6_cu100 + python_version: "3.6" + cu_version: "cu100" nightly: triggers: From 203a913676dd29aac7e33d5febecec0bb3628c35 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Wed, 4 Sep 2019 17:07:25 +0200 Subject: [PATCH 06/35] [WIP] maybe it will work? --- .circleci/config.yml | 75 +++++++++++++++++++++++++++++++++++++---- .circleci/config.yml.in | 75 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 136 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef5c88c9b94..e6cfbf1d4ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,8 +92,29 @@ jobs: name: Build command: | set -e - docker --version - # from https://github.com/NVIDIA/nvidia-docker#ubuntu-16041804-debian-jessiestretchbuster + + curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add - + curl -L https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add - + + sudo apt-get update + + sudo apt-get install \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common + + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + + sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io + # Add the package repositories distribution=$(. /etc/os-release;echo $ID$VERSION_ID) curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - @@ -102,9 +123,49 @@ jobs: sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit sudo systemctl restart docker - docker --version - # start - docker pull soumith/conda-cuda >/dev/null + + DRIVER_FN="NVIDIA-Linux-x86_64-410.104.run" + wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN" + sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false) + nvidia-smi + + # export DOCKER_IMAGE=nvidia/cuda:10.0-base + export DOCKER_IMAGE=soumith/conda-cuda + docker pull $DOCKER_IMAGE >/dev/null + + export id=$(docker run --gpus all -t -d ${DOCKER_IMAGE}) + + docker cp /home/circleci/project/. $id:/workspace + + cat >/home/circleci/install_and_test.sh </dev/null + + DRIVER_FN="NVIDIA-Linux-x86_64-410.104.run" + wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN" + sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false) + nvidia-smi + + # export DOCKER_IMAGE=nvidia/cuda:10.0-base + export DOCKER_IMAGE=soumith/conda-cuda + docker pull $DOCKER_IMAGE >/dev/null + + export id=$(docker run --gpus all -t -d ${DOCKER_IMAGE}) + + docker cp /home/circleci/project/. $id:/workspace + + cat >/home/circleci/install_and_test.sh < Date: Wed, 4 Sep 2019 17:12:21 +0200 Subject: [PATCH 07/35] [WIP] One more try --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e6cfbf1d4ba..b1edadbd6e8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,7 +137,7 @@ jobs: docker cp /home/circleci/project/. $id:/workspace - cat >/home/circleci/install_and_test.sh </home/circleci/install_and_test.sh \</home/circleci/install_and_test.sh </home/circleci/install_and_test.sh \< Date: Thu, 5 Sep 2019 11:30:38 +0200 Subject: [PATCH 08/35] Pin versions --- .circleci/config.yml | 7 +++++-- .circleci/config.yml.in | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1edadbd6e8..844bdfd12ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,14 +113,16 @@ jobs: stable" sudo apt-get update - sudo apt-get install docker-ce docker-ce-cli containerd.io + export DOCKER_VERSION="5:19.03.2~3-0~ubuntu-xenial" + sudo apt-get install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io # Add the package repositories distribution=$(. /etc/os-release;echo $ID$VERSION_ID) curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit + export NVIDIA_CONTAINER_VERSION="1.0.3-1" + sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION} sudo systemctl restart docker @@ -131,6 +133,7 @@ jobs: # export DOCKER_IMAGE=nvidia/cuda:10.0-base export DOCKER_IMAGE=soumith/conda-cuda + echo Pulling docker image $DOCKER_IMAGE docker pull $DOCKER_IMAGE >/dev/null export id=$(docker run --gpus all -t -d ${DOCKER_IMAGE}) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index e03588502cf..10c70ffa222 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -113,14 +113,16 @@ jobs: stable" sudo apt-get update - sudo apt-get install docker-ce docker-ce-cli containerd.io + export DOCKER_VERSION="5:19.03.2~3-0~ubuntu-xenial" + sudo apt-get install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io # Add the package repositories distribution=$(. /etc/os-release;echo $ID$VERSION_ID) curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit + export NVIDIA_CONTAINER_VERSION="1.0.3-1" + sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION} sudo systemctl restart docker @@ -131,6 +133,7 @@ jobs: # export DOCKER_IMAGE=nvidia/cuda:10.0-base export DOCKER_IMAGE=soumith/conda-cuda + echo Pulling docker image $DOCKER_IMAGE docker pull $DOCKER_IMAGE >/dev/null export id=$(docker run --gpus all -t -d ${DOCKER_IMAGE}) From 3a0dfe3b0cbb1e3e6be65757f2c76d78e6832b91 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Thu, 5 Sep 2019 15:31:13 +0200 Subject: [PATCH 09/35] Simplify and reuse --- .circleci/config.yml | 47 ++++++++++++----------------------------- .circleci/config.yml.in | 47 ++++++++++++----------------------------- 2 files changed, 26 insertions(+), 68 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 844bdfd12ec..e00563dc4a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ jobs: steps: - checkout - run: - name: Build + name: Setup environment command: | set -e @@ -125,51 +125,29 @@ jobs: sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION} sudo systemctl restart docker - DRIVER_FN="NVIDIA-Linux-x86_64-410.104.run" wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN" sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false) nvidia-smi - # export DOCKER_IMAGE=nvidia/cuda:10.0-base + - run: + name: Pull docker image + command: | + set -e export DOCKER_IMAGE=soumith/conda-cuda echo Pulling docker image $DOCKER_IMAGE docker pull $DOCKER_IMAGE >/dev/null - export id=$(docker run --gpus all -t -d ${DOCKER_IMAGE}) - - docker cp /home/circleci/project/. $id:/workspace - - cat >/home/circleci/install_and_test.sh \</dev/null - export id=$(docker run --gpus all -t -d ${DOCKER_IMAGE}) - - docker cp /home/circleci/project/. $id:/workspace - - cat >/home/circleci/install_and_test.sh \< Date: Thu, 5 Sep 2019 15:49:43 +0200 Subject: [PATCH 10/35] Fix --- .circleci/config.yml | 1 + .circleci/config.yml.in | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e00563dc4a0..7115e3f94fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -145,6 +145,7 @@ jobs: cd ${HOME}/project/ + export DOCKER_IMAGE=soumith/conda-cuda export VARS_TO_PASS="-e PYTHON_VERSION -e BUILD_VERSION -e PYTORCH_VERSION -e UNICODE_ABI -e CU_VERSION" docker run --gpus all --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 29a8123a8c9..c77142300f8 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -145,6 +145,7 @@ jobs: cd ${HOME}/project/ + export DOCKER_IMAGE=soumith/conda-cuda export VARS_TO_PASS="-e PYTHON_VERSION -e BUILD_VERSION -e PYTORCH_VERSION -e UNICODE_ABI -e CU_VERSION" docker run --gpus all --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh From fdf96854659a75a7d4461b786b1a6d555f2309e9 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Thu, 5 Sep 2019 16:46:18 +0200 Subject: [PATCH 11/35] [WIP] testing windows --- .circleci/config.yml | 12 ++++++++++++ .circleci/config.yml.in | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7115e3f94fa..1f6c02094f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,9 @@ version: 2.1 # - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. # Job names are 'name:' key. +orbs: + win: circleci/windows@1.0.0 + binary_common: &binary_common parameters: # Edit these defaults to do a release` @@ -150,6 +153,15 @@ jobs: docker run --gpus all --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh + binary_win_conda: + <<: *binary_common + executor: + name: win/vs2019 + shell: bash.exe + steps: + - checkout + + binary_macos_wheel: <<: *binary_common macos: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index c77142300f8..c9cdae38096 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -6,6 +6,9 @@ version: 2.1 # - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. # Job names are 'name:' key. +orbs: + win: circleci/windows@1.0.0 + binary_common: &binary_common parameters: # Edit these defaults to do a release` @@ -150,6 +153,15 @@ jobs: docker run --gpus all --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh + binary_win_conda: + <<: *binary_common + executor: + name: win/vs2019 + shell: bash.exe + steps: + - checkout + + binary_macos_wheel: <<: *binary_common macos: From 16cff573d3791a80999d2a9d888f8a6054d2cf02 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Thu, 5 Sep 2019 16:49:03 +0200 Subject: [PATCH 12/35] [WIP] testing windows --- .circleci/config.yml | 4 ++++ .circleci/config.yml.in | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f6c02094f2..533a7db9b85 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -562,6 +562,10 @@ workflows: python_version: "3.7" # cu_version: "cu100" cu_version: "cpu" # FIXME temporary + - binary_win_conda: + name: torchvision_win_py3.6_cpu + python_version: "3.6" + cu_version: "cpu" nightly: triggers: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index c9cdae38096..ce6d45ecd1c 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -308,6 +308,10 @@ workflows: python_version: "3.7" # cu_version: "cu100" cu_version: "cpu" # FIXME temporary + - binary_win_conda: + name: torchvision_win_py3.6_cpu + python_version: "3.6" + cu_version: "cpu" nightly: triggers: From e6d580c0ce8295b4bfc4fe3148bd6d65aac2f664 Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 09:55:31 +0800 Subject: [PATCH 13/35] Try windows --- .circleci/config.yml.in | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index ce6d45ecd1c..f411d4fd0b8 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -160,7 +160,13 @@ jobs: shell: bash.exe steps: - checkout - + - run: + command: | + choco install miniconda3 + # For some reason, refreshenv doesn't work + $env:Path += ";C:\tools\miniconda3\Scripts" + conda install -yq conda-build + bash packaging/build_conda.sh binary_macos_wheel: <<: *binary_common From 200ece22d1883385caf23310e1b371241fe043a0 Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 09:57:52 +0800 Subject: [PATCH 14/35] Try Windows --- .circleci/config.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 533a7db9b85..959ed9465e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,7 +160,13 @@ jobs: shell: bash.exe steps: - checkout - + - run: + command: | + choco install miniconda3 + # For some reason, refreshenv doesn't work + $env:Path += ";C:\tools\miniconda3\Scripts" + conda install -yq conda-build + bash packaging/build_conda.sh binary_macos_wheel: <<: *binary_common @@ -1078,4 +1084,4 @@ workflows: name: nightly_binary_macos_conda_py3.7_cpu_upload context: org-member requires: - - nightly_binary_macos_conda_py3.7_cpu \ No newline at end of file + - nightly_binary_macos_conda_py3.7_cpu From a7728b729cd823b8a3f7680e9caf1f06e969f343 Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 10:06:08 +0800 Subject: [PATCH 15/35] Try windows --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 959ed9465e9..60af9c7b912 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,10 +163,10 @@ jobs: - run: command: | choco install miniconda3 - # For some reason, refreshenv doesn't work - $env:Path += ";C:\tools\miniconda3\Scripts" + refreshenv conda install -yq conda-build bash packaging/build_conda.sh + shell: cmd.exe binary_macos_wheel: <<: *binary_common From 5b8e0197362a5d8caa91b543c020eeee00d9b02f Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 10:11:44 +0800 Subject: [PATCH 16/35] Try windows --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60af9c7b912..e067470b50a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,7 +163,7 @@ jobs: - run: command: | choco install miniconda3 - refreshenv + set "PATH=%PATH%;C:\tools\miniconda3\Scripts" conda install -yq conda-build bash packaging/build_conda.sh shell: cmd.exe From 98493b586cc8cfa48c1e79892deae141bd43c767 Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 10:15:18 +0800 Subject: [PATCH 17/35] Try windows --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e067470b50a..8f257494cac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,7 +166,7 @@ jobs: set "PATH=%PATH%;C:\tools\miniconda3\Scripts" conda install -yq conda-build bash packaging/build_conda.sh - shell: cmd.exe + shell: powershell.exe binary_macos_wheel: <<: *binary_common From 0f25a0449bc42fbcd16c431745ffba08b54303b6 Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 10:16:27 +0800 Subject: [PATCH 18/35] Try windows --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f257494cac..aca4907889d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,7 +163,7 @@ jobs: - run: command: | choco install miniconda3 - set "PATH=%PATH%;C:\tools\miniconda3\Scripts" + $env:PATH += ";C:\tools\miniconda3\Scripts" conda install -yq conda-build bash packaging/build_conda.sh shell: powershell.exe From 5c9b57780c14c771e94569662f9ffe77ba3cc729 Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 10:32:40 +0800 Subject: [PATCH 19/35] Try windows --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aca4907889d..dd8ce9d79af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -164,7 +164,7 @@ jobs: command: | choco install miniconda3 $env:PATH += ";C:\tools\miniconda3\Scripts" - conda install -yq conda-build + conda install -yq conda-build python=3.7 bash packaging/build_conda.sh shell: powershell.exe From 5ac46c73f092c559b8232f17ff40fe65adb348df Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 10:38:49 +0800 Subject: [PATCH 20/35] Try windows --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd8ce9d79af..2ec2d4bf68e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,8 +163,8 @@ jobs: - run: command: | choco install miniconda3 - $env:PATH += ";C:\tools\miniconda3\Scripts" - conda install -yq conda-build python=3.7 + $env:PATH = "C:\tools\miniconda3;C:\tools\miniconda3\Library\usr\bin;C:\tools\miniconda3\Scripts;C:\tools\miniconda3\bin" + $env:PATH + conda install -yq conda-build bash packaging/build_conda.sh shell: powershell.exe From b68da997cf47ede15923243555d40de54ce4706b Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 16:22:25 +0800 Subject: [PATCH 21/35] Windows speedup --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ec2d4bf68e..c950b69643f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,6 +163,8 @@ jobs: - run: command: | choco install miniconda3 + # Use multiple cores during build + $env:CL = "/MP" $env:PATH = "C:\tools\miniconda3;C:\tools\miniconda3\Library\usr\bin;C:\tools\miniconda3\Scripts;C:\tools\miniconda3\bin" + $env:PATH conda install -yq conda-build bash packaging/build_conda.sh From 420b6e7fb046fe0b9d3f90dd2733a862ad1aed9b Mon Sep 17 00:00:00 2001 From: peterjc123 Date: Fri, 6 Sep 2019 16:31:48 +0800 Subject: [PATCH 22/35] Windows multicores --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c950b69643f..7fa347e648b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -164,7 +164,7 @@ jobs: command: | choco install miniconda3 # Use multiple cores during build - $env:CL = "/MP" + $env:CL = " /MP" $env:PATH = "C:\tools\miniconda3;C:\tools\miniconda3\Library\usr\bin;C:\tools\miniconda3\Scripts;C:\tools\miniconda3\bin" + $env:PATH conda install -yq conda-build bash packaging/build_conda.sh From 71c4d0f128dbf8a0cbba0b3f7674f99614abcc77 Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 6 Sep 2019 16:59:10 +0800 Subject: [PATCH 23/35] Add parallel flags for Windows --- .circleci/config.yml | 2 -- setup.py | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7fa347e648b..2ec2d4bf68e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,8 +163,6 @@ jobs: - run: command: | choco install miniconda3 - # Use multiple cores during build - $env:CL = " /MP" $env:PATH = "C:\tools\miniconda3;C:\tools\miniconda3\Library\usr\bin;C:\tools\miniconda3\Scripts;C:\tools\miniconda3\bin" + $env:PATH conda install -yq conda-build bash packaging/build_conda.sh diff --git a/setup.py b/setup.py index b96fbd43ebc..d6dac6c0deb 100644 --- a/setup.py +++ b/setup.py @@ -116,6 +116,9 @@ def get_extensions(): if sys.platform == 'win32': define_macros += [('torchvision_EXPORTS', None)] + extra_compile_args.setdefault('cxx', []) + extra_compile_args['cxx'].append('/MP') + sources = [os.path.join(extensions_dir, s) for s in sources] include_dirs = [extensions_dir] From 99806777ab308a9ba6c91bc4a69f91d7c8a33212 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Fri, 6 Sep 2019 11:38:39 +0200 Subject: [PATCH 24/35] Skip some tests on Windows --- test/test_datasets.py | 2 ++ test/test_datasets_utils.py | 3 +++ test/test_datasets_video_utils.py | 3 +++ test/test_io.py | 1 + test/test_utils.py | 3 +++ 5 files changed, 12 insertions(+) diff --git a/test/test_datasets.py b/test/test_datasets.py index d8f17d1acd4..f4ef4721370 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -1,3 +1,4 @@ +import sys import os import unittest import mock @@ -149,6 +150,7 @@ def test_cifar100(self, mock_ext_check, mock_int_check): img, target = dataset[0] self.assertEqual(dataset.class_to_idx[dataset.classes[0]], target) + @unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') def test_cityscapes(self): with cityscapes_root() as root: diff --git a/test/test_datasets_utils.py b/test/test_datasets_utils.py index 43fdbe8239b..376fe018e92 100644 --- a/test/test_datasets_utils.py +++ b/test/test_datasets_utils.py @@ -72,6 +72,7 @@ def test_download_url_dont_exist(self): with self.assertRaises(URLError): utils.download_url(url, temp_dir) + @unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') def test_extract_zip(self): with get_tmp_dir() as temp_dir: with tempfile.NamedTemporaryFile(suffix='.zip') as f: @@ -83,6 +84,7 @@ def test_extract_zip(self): data = nf.read() self.assertEqual(data, 'this is the content') + @unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') def test_extract_tar(self): for ext, mode in zip(['.tar', '.tar.gz'], ['w', 'w:gz']): with get_tmp_dir() as temp_dir: @@ -98,6 +100,7 @@ def test_extract_tar(self): data = nf.read() self.assertEqual(data, 'this is the content') + @unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') def test_extract_gzip(self): with get_tmp_dir() as temp_dir: with tempfile.NamedTemporaryFile(suffix='.gz') as f: diff --git a/test/test_datasets_video_utils.py b/test/test_datasets_video_utils.py index d47d469ea31..a9cb7ab50ef 100644 --- a/test/test_datasets_video_utils.py +++ b/test/test_datasets_video_utils.py @@ -1,4 +1,5 @@ import contextlib +import sys import os import torch import unittest @@ -58,6 +59,7 @@ def test_unfold(self): self.assertTrue(r.equal(expected)) @unittest.skipIf(not io.video._av_available(), "this test requires av") + @unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') def test_video_clips(self): with get_list_of_videos(num_videos=3) as video_list: video_clips = VideoClips(video_list, 5, 5) @@ -112,6 +114,7 @@ def test_video_sampler_unequal(self): self.assertTrue(count.equal(torch.tensor([3, 3]))) @unittest.skipIf(not io.video._av_available(), "this test requires av") + @unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') def test_video_clips_custom_fps(self): with get_list_of_videos(num_videos=3, sizes=[12, 12, 12], fps=[3, 4, 6]) as video_list: num_frames = 4 diff --git a/test/test_io.py b/test/test_io.py index 8b75cdea1c1..96c33a4be68 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -55,6 +55,7 @@ def temp_video(num_frames, height, width, fps, lossless=False, video_codec=None, @unittest.skipIf(av is None, "PyAV unavailable") +@unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') class Tester(unittest.TestCase): # compression adds artifacts, thus we add a tolerance of # 6 in 0-255 range diff --git a/test/test_utils.py b/test/test_utils.py index 4c39520a692..9c833f3432f 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,4 +1,5 @@ import os +import sys import tempfile import torch import torchvision.utils as utils @@ -37,12 +38,14 @@ def test_normalize_in_make_grid(self): assert torch.equal(norm_max, rounded_grid_max), 'Normalized max is not equal to 1' assert torch.equal(norm_min, rounded_grid_min), 'Normalized min is not equal to 0' + @unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') def test_save_image(self): with tempfile.NamedTemporaryFile(suffix='.png') as f: t = torch.rand(2, 3, 64, 64) utils.save_image(t, f.name) assert os.path.exists(f.name), 'The image is not present after save' + @unittest.skipIf('win' in sys.platform, 'temporarily disabled on Windows') def test_save_image_single_pixel(self): with tempfile.NamedTemporaryFile(suffix='.png') as f: t = torch.rand(1, 3, 1, 1) From 7a309c3154f86cca213ac54d8bd1980175beb264 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Fri, 6 Sep 2019 11:40:31 +0200 Subject: [PATCH 25/35] Sync config.yml and config.yml.in --- .circleci/config.yml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index f411d4fd0b8..7663c5db7a5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -163,10 +163,10 @@ jobs: - run: command: | choco install miniconda3 - # For some reason, refreshenv doesn't work - $env:Path += ";C:\tools\miniconda3\Scripts" + $env:PATH = "C:\tools\miniconda3;C:\tools\miniconda3\Library\usr\bin;C:\tools\miniconda3\Scripts;C:\tools\miniconda3\bin" + $env:PATH conda install -yq conda-build bash packaging/build_conda.sh + shell: powershell.exe binary_macos_wheel: <<: *binary_common From 2f0685e3cc8d40d7bf6f206a34373b86985463f5 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Fri, 6 Sep 2019 11:41:51 +0200 Subject: [PATCH 26/35] Regenerate --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ec2d4bf68e..4986a6f7727 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1084,4 +1084,4 @@ workflows: name: nightly_binary_macos_conda_py3.7_cpu_upload context: org-member requires: - - nightly_binary_macos_conda_py3.7_cpu + - nightly_binary_macos_conda_py3.7_cpu \ No newline at end of file From a94edd1e404e62a2fbc57c8b25b5590872508f24 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Fri, 6 Sep 2019 13:06:13 +0200 Subject: [PATCH 27/35] Run all tests --- .circleci/config.yml | 288 ------------------------------------- .circleci/config.yml.in | 4 - packaging/pkg_helpers.bash | 1 - 3 files changed, 293 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4986a6f7727..9abb609549e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -262,303 +262,159 @@ workflows: name: binary_linux_wheel_py2.7_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_wheel: name: binary_linux_wheel_py2.7u_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master unicode_abi: "1" - binary_linux_wheel: name: binary_linux_wheel_py2.7_cu92 python_version: "2.7" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py2.7u_cu92 python_version: "2.7" cu_version: "cu92" - filters: - branches: - only: - - master unicode_abi: "1" wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py2.7_cu100 python_version: "2.7" cu_version: "cu100" - filters: - branches: - only: - - master - binary_linux_wheel: name: binary_linux_wheel_py2.7u_cu100 python_version: "2.7" cu_version: "cu100" - filters: - branches: - only: - - master unicode_abi: "1" - binary_linux_wheel: name: binary_linux_wheel_py3.5_cpu python_version: "3.5" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_wheel: name: binary_linux_wheel_py3.5_cu92 python_version: "3.5" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py3.5_cu100 python_version: "3.5" cu_version: "cu100" - filters: - branches: - only: - - master - binary_linux_wheel: name: binary_linux_wheel_py3.6_cpu python_version: "3.6" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_wheel: name: binary_linux_wheel_py3.6_cu92 python_version: "3.6" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py3.6_cu100 python_version: "3.6" cu_version: "cu100" - filters: - branches: - only: - - master - binary_linux_wheel: name: binary_linux_wheel_py3.7_cpu python_version: "3.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_wheel: name: binary_linux_wheel_py3.7_cu92 python_version: "3.7" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_wheel: name: binary_linux_wheel_py3.7_cu100 python_version: "3.7" cu_version: "cu100" - filters: - branches: - only: - - master - binary_macos_wheel: name: binary_macos_wheel_py2.7_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_macos_wheel: name: binary_macos_wheel_py2.7u_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master unicode_abi: "1" - binary_macos_wheel: name: binary_macos_wheel_py3.5_cpu python_version: "3.5" cu_version: "cpu" - filters: - branches: - only: - - master - binary_macos_wheel: name: binary_macos_wheel_py3.6_cpu python_version: "3.6" cu_version: "cpu" - filters: - branches: - only: - - master - binary_macos_wheel: name: binary_macos_wheel_py3.7_cpu python_version: "3.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_conda: name: binary_linux_conda_py2.7_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_conda: name: binary_linux_conda_py2.7_cu92 python_version: "2.7" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_conda: name: binary_linux_conda_py2.7_cu100 python_version: "2.7" cu_version: "cu100" - filters: - branches: - only: - - master - binary_linux_conda: name: binary_linux_conda_py3.5_cpu python_version: "3.5" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_conda: name: binary_linux_conda_py3.5_cu92 python_version: "3.5" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_conda: name: binary_linux_conda_py3.5_cu100 python_version: "3.5" cu_version: "cu100" - filters: - branches: - only: - - master - binary_linux_conda: name: binary_linux_conda_py3.6_cpu python_version: "3.6" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_conda: name: binary_linux_conda_py3.6_cu92 python_version: "3.6" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_conda: name: binary_linux_conda_py3.6_cu100 python_version: "3.6" cu_version: "cu100" - filters: - branches: - only: - - master - binary_linux_conda: name: binary_linux_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_conda: name: binary_linux_conda_py3.7_cu92 python_version: "3.7" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_linux_conda: name: binary_linux_conda_py3.7_cu100 python_version: "3.7" cu_version: "cu100" - filters: - branches: - only: - - master - binary_macos_conda: name: binary_macos_conda_py2.7_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_macos_conda: name: binary_macos_conda_py3.5_cpu python_version: "3.5" cu_version: "cpu" - filters: - branches: - only: - - master - binary_macos_conda: name: binary_macos_conda_py3.6_cpu python_version: "3.6" cu_version: "cpu" - filters: - branches: - only: - - master - binary_macos_conda: name: binary_macos_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_linux_conda: name: torchvision_linux_py3.6_cpu python_version: "3.6" @@ -588,10 +444,6 @@ workflows: name: nightly_binary_linux_wheel_py2.7_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7_cpu_upload context: org-member @@ -602,10 +454,6 @@ workflows: name: nightly_binary_linux_wheel_py2.7u_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master unicode_abi: "1" - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7u_cpu_upload @@ -617,10 +465,6 @@ workflows: name: nightly_binary_linux_wheel_py2.7_cu92 python_version: "2.7" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7_cu92_upload @@ -632,10 +476,6 @@ workflows: name: nightly_binary_linux_wheel_py2.7u_cu92 python_version: "2.7" cu_version: "cu92" - filters: - branches: - only: - - master unicode_abi: "1" wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: @@ -648,10 +488,6 @@ workflows: name: nightly_binary_linux_wheel_py2.7_cu100 python_version: "2.7" cu_version: "cu100" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7_cu100_upload context: org-member @@ -662,10 +498,6 @@ workflows: name: nightly_binary_linux_wheel_py2.7u_cu100 python_version: "2.7" cu_version: "cu100" - filters: - branches: - only: - - master unicode_abi: "1" - binary_wheel_upload: name: nightly_binary_linux_wheel_py2.7u_cu100_upload @@ -677,10 +509,6 @@ workflows: name: nightly_binary_linux_wheel_py3.5_cpu python_version: "3.5" cu_version: "cpu" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.5_cpu_upload context: org-member @@ -691,10 +519,6 @@ workflows: name: nightly_binary_linux_wheel_py3.5_cu92 python_version: "3.5" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.5_cu92_upload @@ -706,10 +530,6 @@ workflows: name: nightly_binary_linux_wheel_py3.5_cu100 python_version: "3.5" cu_version: "cu100" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.5_cu100_upload context: org-member @@ -720,10 +540,6 @@ workflows: name: nightly_binary_linux_wheel_py3.6_cpu python_version: "3.6" cu_version: "cpu" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.6_cpu_upload context: org-member @@ -734,10 +550,6 @@ workflows: name: nightly_binary_linux_wheel_py3.6_cu92 python_version: "3.6" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.6_cu92_upload @@ -749,10 +561,6 @@ workflows: name: nightly_binary_linux_wheel_py3.6_cu100 python_version: "3.6" cu_version: "cu100" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.6_cu100_upload context: org-member @@ -763,10 +571,6 @@ workflows: name: nightly_binary_linux_wheel_py3.7_cpu python_version: "3.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.7_cpu_upload context: org-member @@ -777,10 +581,6 @@ workflows: name: nightly_binary_linux_wheel_py3.7_cu92 python_version: "3.7" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.7_cu92_upload @@ -792,10 +592,6 @@ workflows: name: nightly_binary_linux_wheel_py3.7_cu100 python_version: "3.7" cu_version: "cu100" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_linux_wheel_py3.7_cu100_upload context: org-member @@ -806,10 +602,6 @@ workflows: name: nightly_binary_macos_wheel_py2.7_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_macos_wheel_py2.7_cpu_upload context: org-member @@ -820,10 +612,6 @@ workflows: name: nightly_binary_macos_wheel_py2.7u_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master unicode_abi: "1" - binary_wheel_upload: name: nightly_binary_macos_wheel_py2.7u_cpu_upload @@ -835,10 +623,6 @@ workflows: name: nightly_binary_macos_wheel_py3.5_cpu python_version: "3.5" cu_version: "cpu" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_macos_wheel_py3.5_cpu_upload context: org-member @@ -849,10 +633,6 @@ workflows: name: nightly_binary_macos_wheel_py3.6_cpu python_version: "3.6" cu_version: "cpu" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_macos_wheel_py3.6_cpu_upload context: org-member @@ -863,10 +643,6 @@ workflows: name: nightly_binary_macos_wheel_py3.7_cpu python_version: "3.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_wheel_upload: name: nightly_binary_macos_wheel_py3.7_cpu_upload context: org-member @@ -877,10 +653,6 @@ workflows: name: nightly_binary_linux_conda_py2.7_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_linux_conda_py2.7_cpu_upload context: org-member @@ -890,10 +662,6 @@ workflows: name: nightly_binary_linux_conda_py2.7_cu92 python_version: "2.7" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_conda_upload: name: nightly_binary_linux_conda_py2.7_cu92_upload @@ -904,10 +672,6 @@ workflows: name: nightly_binary_linux_conda_py2.7_cu100 python_version: "2.7" cu_version: "cu100" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_linux_conda_py2.7_cu100_upload context: org-member @@ -917,10 +681,6 @@ workflows: name: nightly_binary_linux_conda_py3.5_cpu python_version: "3.5" cu_version: "cpu" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.5_cpu_upload context: org-member @@ -930,10 +690,6 @@ workflows: name: nightly_binary_linux_conda_py3.5_cu92 python_version: "3.5" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_conda_upload: name: nightly_binary_linux_conda_py3.5_cu92_upload @@ -944,10 +700,6 @@ workflows: name: nightly_binary_linux_conda_py3.5_cu100 python_version: "3.5" cu_version: "cu100" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.5_cu100_upload context: org-member @@ -957,10 +709,6 @@ workflows: name: nightly_binary_linux_conda_py3.6_cpu python_version: "3.6" cu_version: "cpu" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.6_cpu_upload context: org-member @@ -970,10 +718,6 @@ workflows: name: nightly_binary_linux_conda_py3.6_cu92 python_version: "3.6" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_conda_upload: name: nightly_binary_linux_conda_py3.6_cu92_upload @@ -984,10 +728,6 @@ workflows: name: nightly_binary_linux_conda_py3.6_cu100 python_version: "3.6" cu_version: "cu100" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.6_cu100_upload context: org-member @@ -997,10 +737,6 @@ workflows: name: nightly_binary_linux_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.7_cpu_upload context: org-member @@ -1010,10 +746,6 @@ workflows: name: nightly_binary_linux_conda_py3.7_cu92 python_version: "3.7" cu_version: "cu92" - filters: - branches: - only: - - master wheel_docker_image: "soumith/manylinux-cuda92" - binary_conda_upload: name: nightly_binary_linux_conda_py3.7_cu92_upload @@ -1024,10 +756,6 @@ workflows: name: nightly_binary_linux_conda_py3.7_cu100 python_version: "3.7" cu_version: "cu100" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_linux_conda_py3.7_cu100_upload context: org-member @@ -1037,10 +765,6 @@ workflows: name: nightly_binary_macos_conda_py2.7_cpu python_version: "2.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_macos_conda_py2.7_cpu_upload context: org-member @@ -1050,10 +774,6 @@ workflows: name: nightly_binary_macos_conda_py3.5_cpu python_version: "3.5" cu_version: "cpu" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_macos_conda_py3.5_cpu_upload context: org-member @@ -1063,10 +783,6 @@ workflows: name: nightly_binary_macos_conda_py3.6_cpu python_version: "3.6" cu_version: "cpu" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_macos_conda_py3.6_cpu_upload context: org-member @@ -1076,10 +792,6 @@ workflows: name: nightly_binary_macos_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" - filters: - branches: - only: - - master - binary_conda_upload: name: nightly_binary_macos_conda_py3.7_cpu_upload context: org-member diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 7663c5db7a5..a40e8519ad6 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -258,10 +258,6 @@ jobs: name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}} python_version: "{{python_version}}" cu_version: "{{cu_version}}" - filters: - branches: - only: - - master {%- if unicode %} unicode_abi: "1" {%- endif %} diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 51c98b7dd7f..049c83fb078 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -45,7 +45,6 @@ setup_cuda() { export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX" # If the suffix is non-empty, we will use a wheel subdirectory if [[ -n "$PYTORCH_VERSION_SUFFIX" ]]; then - # export WHEEL_DIR="$PYTORCH_VERSION_SUFFIX/" export WHEEL_DIR="$CU_VERSION/" fi fi From cb7a33d45d0449b4b0b3b20f8db5c3c3bf2f16ea Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 6 Sep 2019 19:27:52 +0800 Subject: [PATCH 28/35] Limit python and cuda version for finding pytorch --- packaging/pkg_helpers.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 049c83fb078..b258cdb4a6b 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -181,7 +181,7 @@ setup_pip_pytorch_version() { setup_conda_pytorch_constraint() { if [[ -z "$PYTORCH_VERSION" ]]; then export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" - export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" + export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import os, sys, json, re; print(re.sub(r'\\+.*$', '', [x['version'] for x in json.load(sys.stdin)['pytorch'] if os.environ['CU_VERSION'] in x['fn'] and 'py' + os.environ['PYTHON_VERSION'] in x['fn']][-1]))")" else export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" fi From d6ba43fc5a93d3f40d9efd40d7f52c0a3dadc22e Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 6 Sep 2019 20:16:20 +0800 Subject: [PATCH 29/35] Skip darwin for previous check --- packaging/pkg_helpers.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index b258cdb4a6b..f1a9e00c61b 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -181,7 +181,7 @@ setup_pip_pytorch_version() { setup_conda_pytorch_constraint() { if [[ -z "$PYTORCH_VERSION" ]]; then export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" - export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import os, sys, json, re; print(re.sub(r'\\+.*$', '', [x['version'] for x in json.load(sys.stdin)['pytorch'] if os.environ['CU_VERSION'] in x['fn'] and 'py' + os.environ['PYTHON_VERSION'] in x['fn']][-1]))")" + export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import os, sys, json, re; print(re.sub(r'\\+.*$', '', [x['version'] for x in json.load(sys.stdin)['pytorch'] if (x['platform'] == 'darwin' or os.environ['CU_VERSION'] in x['fn']) and 'py' + os.environ['PYTHON_VERSION'] in x['fn']][-1]))")" else export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" fi From 24ebf1ea1bdfbdd4d6176662c71d9dfe564cd365 Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 6 Sep 2019 20:33:58 +0800 Subject: [PATCH 30/35] Add description --- packaging/pkg_helpers.bash | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index f1a9e00c61b..e5b5c82cad8 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -181,7 +181,16 @@ setup_pip_pytorch_version() { setup_conda_pytorch_constraint() { if [[ -z "$PYTORCH_VERSION" ]]; then export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" - export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import os, sys, json, re; print(re.sub(r'\\+.*$', '', [x['version'] for x in json.load(sys.stdin)['pytorch'] if (x['platform'] == 'darwin' or os.environ['CU_VERSION'] in x['fn']) and 'py' + os.environ['PYTHON_VERSION'] in x['fn']][-1]))")" + export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \ + python -c "import os, sys, json, re; print(re.sub(r'\\+.*$', '', \ + [x['version'] for x in json.load(sys.stdin)['pytorch'] \ + if (x['platform'] == 'darwin' or os.environ['CU_VERSION'] in x['fn']) \ + and 'py' + os.environ['PYTHON_VERSION'] in x['fn']][-1]))")" + if [[ -z "$PYTORCH_VERSION" ]]; then + echo "PyTorch version auto detection failed" + echo "No package found for CU_VERSION=$CU_VERSION and PYTHON_VERSION=$PYTHON_VERSION" + exit 1 + fi else export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" fi From ac53f1a76bb53aea68e28b9372a93232fed3f1b9 Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 6 Sep 2019 21:01:05 +0800 Subject: [PATCH 31/35] Fix logic --- packaging/pkg_helpers.bash | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index e5b5c82cad8..936eeceeaab 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -182,9 +182,11 @@ setup_conda_pytorch_constraint() { if [[ -z "$PYTORCH_VERSION" ]]; then export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \ - python -c "import os, sys, json, re; print(re.sub(r'\\+.*$', '', \ + python -c "import os, sys, json, re; cuver = os.environ.get('CU_VERSION'); \ + cuver = (cuver[:-1] + '.' + cuver[-1]).replace('cu', 'cuda') if cuver != 'cpu' else cuver; \ + print(re.sub(r'\\+.*$', '', \ [x['version'] for x in json.load(sys.stdin)['pytorch'] \ - if (x['platform'] == 'darwin' or os.environ['CU_VERSION'] in x['fn']) \ + if (x['platform'] == 'darwin' or cuver in x['fn']) \ and 'py' + os.environ['PYTHON_VERSION'] in x['fn']][-1]))")" if [[ -z "$PYTORCH_VERSION" ]]; then echo "PyTorch version auto detection failed" From 0e58a6903838b83cd80e591601fb927da9d2ae46 Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 6 Sep 2019 21:03:15 +0800 Subject: [PATCH 32/35] Remove space --- packaging/pkg_helpers.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 936eeceeaab..4e7d55e4be6 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -183,7 +183,7 @@ setup_conda_pytorch_constraint() { export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \ python -c "import os, sys, json, re; cuver = os.environ.get('CU_VERSION'); \ - cuver = (cuver[:-1] + '.' + cuver[-1]).replace('cu', 'cuda') if cuver != 'cpu' else cuver; \ + cuver = (cuver[:-1] + '.' + cuver[-1]).replace('cu', 'cuda') if cuver != 'cpu' else cuver; \ print(re.sub(r'\\+.*$', '', \ [x['version'] for x in json.load(sys.stdin)['pytorch'] \ if (x['platform'] == 'darwin' or cuver in x['fn']) \ From f3c23c6c5ca40e03c9c0d2874fa7399e02ae0628 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Mon, 9 Sep 2019 10:40:13 +0200 Subject: [PATCH 33/35] Add CUDA test back --- .circleci/config.yml | 3 +-- .circleci/config.yml.in | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9abb609549e..bcf6fe40b3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -422,8 +422,7 @@ workflows: - binary_linux_conda_cuda: name: torchvision_linux_py3.7_cu100 python_version: "3.7" - # cu_version: "cu100" - cu_version: "cpu" # FIXME temporary + cu_version: "cu100" - binary_win_conda: name: torchvision_win_py3.6_cpu python_version: "3.6" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index a40e8519ad6..bf28efa12e0 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -308,8 +308,7 @@ workflows: - binary_linux_conda_cuda: name: torchvision_linux_py3.7_cu100 python_version: "3.7" - # cu_version: "cu100" - cu_version: "cpu" # FIXME temporary + cu_version: "cu100" - binary_win_conda: name: torchvision_win_py3.6_cpu python_version: "3.6" From 274b84c28fd1a33ccbbe169ae4c0e9164d0e2637 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Mon, 9 Sep 2019 11:06:26 +0200 Subject: [PATCH 34/35] Add back .travis.yml for now and remove duplicate test --- .circleci/config.yml | 4 --- .circleci/config.yml.in | 4 --- .travis.yml | 75 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 .travis.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index bcf6fe40b3d..abd9c8ccf74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -415,10 +415,6 @@ workflows: name: binary_macos_conda_py3.7_cpu python_version: "3.7" cu_version: "cpu" - - binary_linux_conda: - name: torchvision_linux_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" - binary_linux_conda_cuda: name: torchvision_linux_py3.7_cu100 python_version: "3.7" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index bf28efa12e0..4ff3849db53 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -301,10 +301,6 @@ workflows: jobs: - circleci_consistency {{ workflows() }} - - binary_linux_conda: - name: torchvision_linux_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" - binary_linux_conda_cuda: name: torchvision_linux_py3.7_cu100 python_version: "3.7" diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000..27bd89cbd73 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,75 @@ +language: python + +dist: xenial +matrix: + include: + - env: FORMAT_CHECK + language: cpp + addons: + apt: + sources: + - llvm-toolchain-xenial-7 + packages: + - clang-7 + - clang-format-7 + before_install: skip + install: skip + script: ./travis-scripts/run-clang-format/run-clang-format.py -r torchvision/csrc + - env: LINT_CHECK + python: "2.7" + install: pip install flake8 + script: flake8 + after_success: [] + - python: "2.7" + env: IMAGE_BACKEND=Pillow-SIMD + - python: "2.7" + - python: "3.6" + env: IMAGE_BACKEND=Pillow-SIMD + - python: "3.6" + +before_install: + - sudo apt-get update + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + # Useful for debugging any issues with conda + - conda info -a + + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytorch scipy -c pytorch-nightly + - source activate test-environment + - | + if [[ "$IMAGE_BACKEND" == "Pillow-SIMD" ]]; then + pip uninstall -y pillow && CC="cc -march=native" pip install --force-reinstall pillow-simd + fi + - pip install future + - pip install pytest pytest-cov codecov + - pip install mock + - conda install av -c conda-forge + + +install: + # Using pip instead of setup.py ensures we install a non-compressed version of the package + # (as opposed to an egg), which is necessary to collect coverage. + # We still get the benefit of testing an installed version over the + # test version to iron out installation file-inclusion bugs but can + # also collect coverage. + - pip install . + # Move to home dir, otherwise we'll end up with the path to the + # package in $PWD rather than the installed v + - | + cd $HOME + export TV_INSTALL_PATH="$(python -c 'import os; import torchvision; print(os.path.dirname(os.path.abspath(torchvision.__file__)))')" + echo "$TV_INSTALL_PATH" + cd - +script: + - pytest --cov-config .coveragerc --cov torchvision --cov $TV_INSTALL_PATH test + +after_success: + # Necessary to run coverage combine to rewrite paths from + # /travis/env/path/site-packages/torchvision to actual path + - coverage combine .coverage + - coverage report + - codecov From 2755fbc7ef3fadc5983dcedfb587692ee03011ed Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Mon, 9 Sep 2019 11:07:45 +0200 Subject: [PATCH 35/35] Add newline --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 27bd89cbd73..fc0aa7240a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ install: export TV_INSTALL_PATH="$(python -c 'import os; import torchvision; print(os.path.dirname(os.path.abspath(torchvision.__file__)))')" echo "$TV_INSTALL_PATH" cd - + script: - pytest --cov-config .coveragerc --cov torchvision --cov $TV_INSTALL_PATH test