From 3e3a495f4469643a920043fa724d52d6c633fc1d Mon Sep 17 00:00:00 2001 From: Karl Ostmo Date: Mon, 9 Sep 2019 23:46:28 -0700 Subject: [PATCH] use native python code generation logic use flake8 with Python 3 on .circleci directory --- .circleci/.gitignore | 1 + .circleci/config.yml | 529 ++++++++++++++++++++-------------------- .circleci/config.yml.in | 43 +--- .circleci/regenerate.py | 96 +++++++- .travis.yml | 7 +- 5 files changed, 359 insertions(+), 317 deletions(-) create mode 100644 .circleci/.gitignore diff --git a/.circleci/.gitignore b/.circleci/.gitignore new file mode 100644 index 00000000000..485dee64bcf --- /dev/null +++ b/.circleci/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/.circleci/config.yml b/.circleci/config.yml index abd9c8ccf74..5c9301ec4a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ jobs: - checkout - run: command: | - pip install --user --progress-bar off jinja2 + pip install --user --progress-bar off jinja2 pyyaml python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) @@ -253,168 +253,168 @@ jobs: aws s3 cp "$pkg" "s3://pytorch/whl/nightly/<< parameters.subfolder >>" --acl public-read done + workflows: build: jobs: - circleci_consistency - - binary_linux_wheel: + cu_version: cpu name: binary_linux_wheel_py2.7_cpu - python_version: "2.7" - cu_version: "cpu" + python_version: '2.7' - binary_linux_wheel: + cu_version: cpu name: binary_linux_wheel_py2.7u_cpu - python_version: "2.7" - cu_version: "cpu" - unicode_abi: "1" + python_version: '2.7' + unicode_abi: '1' - binary_linux_wheel: + cu_version: cu92 name: binary_linux_wheel_py2.7_cu92 - python_version: "2.7" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '2.7' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_wheel: + cu_version: cu92 name: binary_linux_wheel_py2.7u_cu92 - python_version: "2.7" - cu_version: "cu92" - unicode_abi: "1" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '2.7' + unicode_abi: '1' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_wheel: + cu_version: cu100 name: binary_linux_wheel_py2.7_cu100 - python_version: "2.7" - cu_version: "cu100" + python_version: '2.7' - binary_linux_wheel: + cu_version: cu100 name: binary_linux_wheel_py2.7u_cu100 - python_version: "2.7" - cu_version: "cu100" - unicode_abi: "1" + python_version: '2.7' + unicode_abi: '1' - binary_linux_wheel: + cu_version: cpu name: binary_linux_wheel_py3.5_cpu - python_version: "3.5" - cu_version: "cpu" + python_version: '3.5' - binary_linux_wheel: + cu_version: cu92 name: binary_linux_wheel_py3.5_cu92 - python_version: "3.5" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.5' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_wheel: + cu_version: cu100 name: binary_linux_wheel_py3.5_cu100 - python_version: "3.5" - cu_version: "cu100" + python_version: '3.5' - binary_linux_wheel: + cu_version: cpu name: binary_linux_wheel_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + python_version: '3.6' - binary_linux_wheel: + cu_version: cu92 name: binary_linux_wheel_py3.6_cu92 - python_version: "3.6" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.6' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_wheel: + cu_version: cu100 name: binary_linux_wheel_py3.6_cu100 - python_version: "3.6" - cu_version: "cu100" + python_version: '3.6' - binary_linux_wheel: + cu_version: cpu name: binary_linux_wheel_py3.7_cpu - python_version: "3.7" - cu_version: "cpu" + python_version: '3.7' - binary_linux_wheel: + cu_version: cu92 name: binary_linux_wheel_py3.7_cu92 - python_version: "3.7" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.7' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_wheel: + cu_version: cu100 name: binary_linux_wheel_py3.7_cu100 - python_version: "3.7" - cu_version: "cu100" + python_version: '3.7' - binary_macos_wheel: + cu_version: cpu name: binary_macos_wheel_py2.7_cpu - python_version: "2.7" - cu_version: "cpu" + python_version: '2.7' - binary_macos_wheel: + cu_version: cpu name: binary_macos_wheel_py2.7u_cpu - python_version: "2.7" - cu_version: "cpu" - unicode_abi: "1" + python_version: '2.7' + unicode_abi: '1' - binary_macos_wheel: + cu_version: cpu name: binary_macos_wheel_py3.5_cpu - python_version: "3.5" - cu_version: "cpu" + python_version: '3.5' - binary_macos_wheel: + cu_version: cpu name: binary_macos_wheel_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + python_version: '3.6' - binary_macos_wheel: + cu_version: cpu name: binary_macos_wheel_py3.7_cpu - python_version: "3.7" - cu_version: "cpu" + python_version: '3.7' - binary_linux_conda: + cu_version: cpu name: binary_linux_conda_py2.7_cpu - python_version: "2.7" - cu_version: "cpu" + python_version: '2.7' - binary_linux_conda: + cu_version: cu92 name: binary_linux_conda_py2.7_cu92 - python_version: "2.7" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '2.7' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_conda: + cu_version: cu100 name: binary_linux_conda_py2.7_cu100 - python_version: "2.7" - cu_version: "cu100" + python_version: '2.7' - binary_linux_conda: + cu_version: cpu name: binary_linux_conda_py3.5_cpu - python_version: "3.5" - cu_version: "cpu" + python_version: '3.5' - binary_linux_conda: + cu_version: cu92 name: binary_linux_conda_py3.5_cu92 - python_version: "3.5" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.5' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_conda: + cu_version: cu100 name: binary_linux_conda_py3.5_cu100 - python_version: "3.5" - cu_version: "cu100" + python_version: '3.5' - binary_linux_conda: + cu_version: cpu name: binary_linux_conda_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + python_version: '3.6' - binary_linux_conda: + cu_version: cu92 name: binary_linux_conda_py3.6_cu92 - python_version: "3.6" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.6' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_conda: + cu_version: cu100 name: binary_linux_conda_py3.6_cu100 - python_version: "3.6" - cu_version: "cu100" + python_version: '3.6' - binary_linux_conda: + cu_version: cpu name: binary_linux_conda_py3.7_cpu - python_version: "3.7" - cu_version: "cpu" + python_version: '3.7' - binary_linux_conda: + cu_version: cu92 name: binary_linux_conda_py3.7_cu92 - python_version: "3.7" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.7' + wheel_docker_image: soumith/manylinux-cuda92 - binary_linux_conda: + cu_version: cu100 name: binary_linux_conda_py3.7_cu100 - python_version: "3.7" - cu_version: "cu100" + python_version: '3.7' - binary_macos_conda: + cu_version: cpu name: binary_macos_conda_py2.7_cpu - python_version: "2.7" - cu_version: "cpu" + python_version: '2.7' - binary_macos_conda: + cu_version: cpu name: binary_macos_conda_py3.5_cpu - python_version: "3.5" - cu_version: "cpu" + python_version: '3.5' - binary_macos_conda: + cu_version: cpu name: binary_macos_conda_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + python_version: '3.6' - binary_macos_conda: + cu_version: cpu name: binary_macos_conda_py3.7_cpu - python_version: "3.7" - cu_version: "cpu" + python_version: '3.7' - binary_linux_conda_cuda: name: torchvision_linux_py3.7_cu100 python_version: "3.7" @@ -434,361 +434,360 @@ workflows: - master jobs: - circleci_consistency - - binary_linux_wheel: + cu_version: cpu name: nightly_binary_linux_wheel_py2.7_cpu - python_version: "2.7" - cu_version: "cpu" + python_version: '2.7' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py2.7_cpu_upload context: org-member + name: nightly_binary_linux_wheel_py2.7_cpu_upload requires: - - nightly_binary_linux_wheel_py2.7_cpu - subfolder: "cpu/" + - nightly_binary_linux_wheel_py2.7_cpu + subfolder: cpu/ - binary_linux_wheel: + cu_version: cpu name: nightly_binary_linux_wheel_py2.7u_cpu - python_version: "2.7" - cu_version: "cpu" - unicode_abi: "1" + python_version: '2.7' + unicode_abi: '1' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py2.7u_cpu_upload context: org-member + name: nightly_binary_linux_wheel_py2.7u_cpu_upload requires: - - nightly_binary_linux_wheel_py2.7u_cpu - subfolder: "cpu/" + - nightly_binary_linux_wheel_py2.7u_cpu + subfolder: cpu/ - binary_linux_wheel: + cu_version: cu92 name: nightly_binary_linux_wheel_py2.7_cu92 - python_version: "2.7" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '2.7' + wheel_docker_image: soumith/manylinux-cuda92 - binary_wheel_upload: - name: nightly_binary_linux_wheel_py2.7_cu92_upload context: org-member + name: nightly_binary_linux_wheel_py2.7_cu92_upload requires: - - nightly_binary_linux_wheel_py2.7_cu92 - subfolder: "cu92/" + - nightly_binary_linux_wheel_py2.7_cu92 + subfolder: cu92/ - binary_linux_wheel: + cu_version: cu92 name: nightly_binary_linux_wheel_py2.7u_cu92 - python_version: "2.7" - cu_version: "cu92" - unicode_abi: "1" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '2.7' + unicode_abi: '1' + wheel_docker_image: soumith/manylinux-cuda92 - binary_wheel_upload: - name: nightly_binary_linux_wheel_py2.7u_cu92_upload context: org-member + name: nightly_binary_linux_wheel_py2.7u_cu92_upload requires: - - nightly_binary_linux_wheel_py2.7u_cu92 - subfolder: "cu92/" + - nightly_binary_linux_wheel_py2.7u_cu92 + subfolder: cu92/ - binary_linux_wheel: + cu_version: cu100 name: nightly_binary_linux_wheel_py2.7_cu100 - python_version: "2.7" - cu_version: "cu100" + python_version: '2.7' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py2.7_cu100_upload context: org-member + name: nightly_binary_linux_wheel_py2.7_cu100_upload requires: - - nightly_binary_linux_wheel_py2.7_cu100 - subfolder: "cu100/" + - nightly_binary_linux_wheel_py2.7_cu100 + subfolder: cu100/ - binary_linux_wheel: + cu_version: cu100 name: nightly_binary_linux_wheel_py2.7u_cu100 - python_version: "2.7" - cu_version: "cu100" - unicode_abi: "1" + python_version: '2.7' + unicode_abi: '1' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py2.7u_cu100_upload context: org-member + name: nightly_binary_linux_wheel_py2.7u_cu100_upload requires: - - nightly_binary_linux_wheel_py2.7u_cu100 - subfolder: "cu100/" + - nightly_binary_linux_wheel_py2.7u_cu100 + subfolder: cu100/ - binary_linux_wheel: + cu_version: cpu name: nightly_binary_linux_wheel_py3.5_cpu - python_version: "3.5" - cu_version: "cpu" + python_version: '3.5' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.5_cpu_upload context: org-member + name: nightly_binary_linux_wheel_py3.5_cpu_upload requires: - - nightly_binary_linux_wheel_py3.5_cpu - subfolder: "cpu/" + - nightly_binary_linux_wheel_py3.5_cpu + subfolder: cpu/ - binary_linux_wheel: + cu_version: cu92 name: nightly_binary_linux_wheel_py3.5_cu92 - python_version: "3.5" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.5' + wheel_docker_image: soumith/manylinux-cuda92 - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.5_cu92_upload context: org-member + name: nightly_binary_linux_wheel_py3.5_cu92_upload requires: - - nightly_binary_linux_wheel_py3.5_cu92 - subfolder: "cu92/" + - nightly_binary_linux_wheel_py3.5_cu92 + subfolder: cu92/ - binary_linux_wheel: + cu_version: cu100 name: nightly_binary_linux_wheel_py3.5_cu100 - python_version: "3.5" - cu_version: "cu100" + python_version: '3.5' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.5_cu100_upload context: org-member + name: nightly_binary_linux_wheel_py3.5_cu100_upload requires: - - nightly_binary_linux_wheel_py3.5_cu100 - subfolder: "cu100/" + - nightly_binary_linux_wheel_py3.5_cu100 + subfolder: cu100/ - binary_linux_wheel: + cu_version: cpu name: nightly_binary_linux_wheel_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + python_version: '3.6' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.6_cpu_upload context: org-member + name: nightly_binary_linux_wheel_py3.6_cpu_upload requires: - - nightly_binary_linux_wheel_py3.6_cpu - subfolder: "cpu/" + - nightly_binary_linux_wheel_py3.6_cpu + subfolder: cpu/ - binary_linux_wheel: + cu_version: cu92 name: nightly_binary_linux_wheel_py3.6_cu92 - python_version: "3.6" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.6' + wheel_docker_image: soumith/manylinux-cuda92 - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.6_cu92_upload context: org-member + name: nightly_binary_linux_wheel_py3.6_cu92_upload requires: - - nightly_binary_linux_wheel_py3.6_cu92 - subfolder: "cu92/" + - nightly_binary_linux_wheel_py3.6_cu92 + subfolder: cu92/ - binary_linux_wheel: + cu_version: cu100 name: nightly_binary_linux_wheel_py3.6_cu100 - python_version: "3.6" - cu_version: "cu100" + python_version: '3.6' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.6_cu100_upload context: org-member + name: nightly_binary_linux_wheel_py3.6_cu100_upload requires: - - nightly_binary_linux_wheel_py3.6_cu100 - subfolder: "cu100/" + - nightly_binary_linux_wheel_py3.6_cu100 + subfolder: cu100/ - binary_linux_wheel: + cu_version: cpu name: nightly_binary_linux_wheel_py3.7_cpu - python_version: "3.7" - cu_version: "cpu" + python_version: '3.7' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.7_cpu_upload context: org-member + name: nightly_binary_linux_wheel_py3.7_cpu_upload requires: - - nightly_binary_linux_wheel_py3.7_cpu - subfolder: "cpu/" + - nightly_binary_linux_wheel_py3.7_cpu + subfolder: cpu/ - binary_linux_wheel: + cu_version: cu92 name: nightly_binary_linux_wheel_py3.7_cu92 - python_version: "3.7" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.7' + wheel_docker_image: soumith/manylinux-cuda92 - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.7_cu92_upload context: org-member + name: nightly_binary_linux_wheel_py3.7_cu92_upload requires: - - nightly_binary_linux_wheel_py3.7_cu92 - subfolder: "cu92/" + - nightly_binary_linux_wheel_py3.7_cu92 + subfolder: cu92/ - binary_linux_wheel: + cu_version: cu100 name: nightly_binary_linux_wheel_py3.7_cu100 - python_version: "3.7" - cu_version: "cu100" + python_version: '3.7' - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.7_cu100_upload context: org-member + name: nightly_binary_linux_wheel_py3.7_cu100_upload requires: - - nightly_binary_linux_wheel_py3.7_cu100 - subfolder: "cu100/" + - nightly_binary_linux_wheel_py3.7_cu100 + subfolder: cu100/ - binary_macos_wheel: + cu_version: cpu name: nightly_binary_macos_wheel_py2.7_cpu - python_version: "2.7" - cu_version: "cpu" + python_version: '2.7' - binary_wheel_upload: - name: nightly_binary_macos_wheel_py2.7_cpu_upload context: org-member + name: nightly_binary_macos_wheel_py2.7_cpu_upload requires: - - nightly_binary_macos_wheel_py2.7_cpu - subfolder: "" + - nightly_binary_macos_wheel_py2.7_cpu + subfolder: '' - binary_macos_wheel: + cu_version: cpu name: nightly_binary_macos_wheel_py2.7u_cpu - python_version: "2.7" - cu_version: "cpu" - unicode_abi: "1" + python_version: '2.7' + unicode_abi: '1' - binary_wheel_upload: - name: nightly_binary_macos_wheel_py2.7u_cpu_upload context: org-member + name: nightly_binary_macos_wheel_py2.7u_cpu_upload requires: - - nightly_binary_macos_wheel_py2.7u_cpu - subfolder: "" + - nightly_binary_macos_wheel_py2.7u_cpu + subfolder: '' - binary_macos_wheel: + cu_version: cpu name: nightly_binary_macos_wheel_py3.5_cpu - python_version: "3.5" - cu_version: "cpu" + python_version: '3.5' - binary_wheel_upload: - name: nightly_binary_macos_wheel_py3.5_cpu_upload context: org-member + name: nightly_binary_macos_wheel_py3.5_cpu_upload requires: - - nightly_binary_macos_wheel_py3.5_cpu - subfolder: "" + - nightly_binary_macos_wheel_py3.5_cpu + subfolder: '' - binary_macos_wheel: + cu_version: cpu name: nightly_binary_macos_wheel_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + python_version: '3.6' - binary_wheel_upload: - name: nightly_binary_macos_wheel_py3.6_cpu_upload context: org-member + name: nightly_binary_macos_wheel_py3.6_cpu_upload requires: - - nightly_binary_macos_wheel_py3.6_cpu - subfolder: "" + - nightly_binary_macos_wheel_py3.6_cpu + subfolder: '' - binary_macos_wheel: + cu_version: cpu name: nightly_binary_macos_wheel_py3.7_cpu - python_version: "3.7" - cu_version: "cpu" + python_version: '3.7' - binary_wheel_upload: - name: nightly_binary_macos_wheel_py3.7_cpu_upload context: org-member + name: nightly_binary_macos_wheel_py3.7_cpu_upload requires: - - nightly_binary_macos_wheel_py3.7_cpu - subfolder: "" + - nightly_binary_macos_wheel_py3.7_cpu + subfolder: '' - binary_linux_conda: + cu_version: cpu name: nightly_binary_linux_conda_py2.7_cpu - python_version: "2.7" - cu_version: "cpu" + python_version: '2.7' - binary_conda_upload: - name: nightly_binary_linux_conda_py2.7_cpu_upload context: org-member + name: nightly_binary_linux_conda_py2.7_cpu_upload requires: - - nightly_binary_linux_conda_py2.7_cpu + - nightly_binary_linux_conda_py2.7_cpu - binary_linux_conda: + cu_version: cu92 name: nightly_binary_linux_conda_py2.7_cu92 - python_version: "2.7" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '2.7' + wheel_docker_image: soumith/manylinux-cuda92 - binary_conda_upload: - name: nightly_binary_linux_conda_py2.7_cu92_upload context: org-member + name: nightly_binary_linux_conda_py2.7_cu92_upload requires: - - nightly_binary_linux_conda_py2.7_cu92 + - nightly_binary_linux_conda_py2.7_cu92 - binary_linux_conda: + cu_version: cu100 name: nightly_binary_linux_conda_py2.7_cu100 - python_version: "2.7" - cu_version: "cu100" + python_version: '2.7' - binary_conda_upload: - name: nightly_binary_linux_conda_py2.7_cu100_upload context: org-member + name: nightly_binary_linux_conda_py2.7_cu100_upload requires: - - nightly_binary_linux_conda_py2.7_cu100 + - nightly_binary_linux_conda_py2.7_cu100 - binary_linux_conda: + cu_version: cpu name: nightly_binary_linux_conda_py3.5_cpu - python_version: "3.5" - cu_version: "cpu" + python_version: '3.5' - binary_conda_upload: - name: nightly_binary_linux_conda_py3.5_cpu_upload context: org-member + name: nightly_binary_linux_conda_py3.5_cpu_upload requires: - - nightly_binary_linux_conda_py3.5_cpu + - nightly_binary_linux_conda_py3.5_cpu - binary_linux_conda: + cu_version: cu92 name: nightly_binary_linux_conda_py3.5_cu92 - python_version: "3.5" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.5' + wheel_docker_image: soumith/manylinux-cuda92 - binary_conda_upload: - name: nightly_binary_linux_conda_py3.5_cu92_upload context: org-member + name: nightly_binary_linux_conda_py3.5_cu92_upload requires: - - nightly_binary_linux_conda_py3.5_cu92 + - nightly_binary_linux_conda_py3.5_cu92 - binary_linux_conda: + cu_version: cu100 name: nightly_binary_linux_conda_py3.5_cu100 - python_version: "3.5" - cu_version: "cu100" + python_version: '3.5' - binary_conda_upload: - name: nightly_binary_linux_conda_py3.5_cu100_upload context: org-member + name: nightly_binary_linux_conda_py3.5_cu100_upload requires: - - nightly_binary_linux_conda_py3.5_cu100 + - nightly_binary_linux_conda_py3.5_cu100 - binary_linux_conda: + cu_version: cpu name: nightly_binary_linux_conda_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + python_version: '3.6' - binary_conda_upload: - name: nightly_binary_linux_conda_py3.6_cpu_upload context: org-member + name: nightly_binary_linux_conda_py3.6_cpu_upload requires: - - nightly_binary_linux_conda_py3.6_cpu + - nightly_binary_linux_conda_py3.6_cpu - binary_linux_conda: + cu_version: cu92 name: nightly_binary_linux_conda_py3.6_cu92 - python_version: "3.6" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.6' + wheel_docker_image: soumith/manylinux-cuda92 - binary_conda_upload: - name: nightly_binary_linux_conda_py3.6_cu92_upload context: org-member + name: nightly_binary_linux_conda_py3.6_cu92_upload requires: - - nightly_binary_linux_conda_py3.6_cu92 + - nightly_binary_linux_conda_py3.6_cu92 - binary_linux_conda: + cu_version: cu100 name: nightly_binary_linux_conda_py3.6_cu100 - python_version: "3.6" - cu_version: "cu100" + python_version: '3.6' - binary_conda_upload: - name: nightly_binary_linux_conda_py3.6_cu100_upload context: org-member + name: nightly_binary_linux_conda_py3.6_cu100_upload requires: - - nightly_binary_linux_conda_py3.6_cu100 + - nightly_binary_linux_conda_py3.6_cu100 - binary_linux_conda: + cu_version: cpu name: nightly_binary_linux_conda_py3.7_cpu - python_version: "3.7" - cu_version: "cpu" + python_version: '3.7' - binary_conda_upload: - name: nightly_binary_linux_conda_py3.7_cpu_upload context: org-member + name: nightly_binary_linux_conda_py3.7_cpu_upload requires: - - nightly_binary_linux_conda_py3.7_cpu + - nightly_binary_linux_conda_py3.7_cpu - binary_linux_conda: + cu_version: cu92 name: nightly_binary_linux_conda_py3.7_cu92 - python_version: "3.7" - cu_version: "cu92" - wheel_docker_image: "soumith/manylinux-cuda92" + python_version: '3.7' + wheel_docker_image: soumith/manylinux-cuda92 - binary_conda_upload: - name: nightly_binary_linux_conda_py3.7_cu92_upload context: org-member + name: nightly_binary_linux_conda_py3.7_cu92_upload requires: - - nightly_binary_linux_conda_py3.7_cu92 + - nightly_binary_linux_conda_py3.7_cu92 - binary_linux_conda: + cu_version: cu100 name: nightly_binary_linux_conda_py3.7_cu100 - python_version: "3.7" - cu_version: "cu100" + python_version: '3.7' - binary_conda_upload: - name: nightly_binary_linux_conda_py3.7_cu100_upload context: org-member + name: nightly_binary_linux_conda_py3.7_cu100_upload requires: - - nightly_binary_linux_conda_py3.7_cu100 + - nightly_binary_linux_conda_py3.7_cu100 - binary_macos_conda: + cu_version: cpu name: nightly_binary_macos_conda_py2.7_cpu - python_version: "2.7" - cu_version: "cpu" + python_version: '2.7' - binary_conda_upload: - name: nightly_binary_macos_conda_py2.7_cpu_upload context: org-member + name: nightly_binary_macos_conda_py2.7_cpu_upload requires: - - nightly_binary_macos_conda_py2.7_cpu + - nightly_binary_macos_conda_py2.7_cpu - binary_macos_conda: + cu_version: cpu name: nightly_binary_macos_conda_py3.5_cpu - python_version: "3.5" - cu_version: "cpu" + python_version: '3.5' - binary_conda_upload: - name: nightly_binary_macos_conda_py3.5_cpu_upload context: org-member + name: nightly_binary_macos_conda_py3.5_cpu_upload requires: - - nightly_binary_macos_conda_py3.5_cpu + - nightly_binary_macos_conda_py3.5_cpu - binary_macos_conda: + cu_version: cpu name: nightly_binary_macos_conda_py3.6_cpu - python_version: "3.6" - cu_version: "cpu" + python_version: '3.6' - binary_conda_upload: - name: nightly_binary_macos_conda_py3.6_cpu_upload context: org-member + name: nightly_binary_macos_conda_py3.6_cpu_upload requires: - - nightly_binary_macos_conda_py3.6_cpu + - nightly_binary_macos_conda_py3.6_cpu - binary_macos_conda: + cu_version: cpu name: nightly_binary_macos_conda_py3.7_cpu - python_version: "3.7" - cu_version: "cpu" + python_version: '3.7' - binary_conda_upload: - name: nightly_binary_macos_conda_py3.7_cpu_upload context: org-member + name: nightly_binary_macos_conda_py3.7_cpu_upload requires: - - nightly_binary_macos_conda_py3.7_cpu \ No newline at end of file + - nightly_binary_macos_conda_py3.7_cpu \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 4ff3849db53..70d5a613508 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -50,7 +50,7 @@ jobs: - checkout - run: command: | - pip install --user --progress-bar off jinja2 + pip install --user --progress-bar off jinja2 pyyaml python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) @@ -253,47 +253,6 @@ jobs: aws s3 cp "$pkg" "s3://pytorch/whl/nightly/<< parameters.subfolder >>" --acl public-read done -{%- macro workflow(btype, os, python_version, cu_version, unicode, prefix='', upload=False) %} - - binary_{{os}}_{{btype}}: - name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}} - python_version: "{{python_version}}" - cu_version: "{{cu_version}}" -{%- if unicode %} - unicode_abi: "1" -{%- endif %} -{%- if cu_version == "cu92" %} - wheel_docker_image: "soumith/manylinux-cuda92" -{%- endif %} - -{%- if upload %} - - binary_{{btype}}_upload: - name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}_upload - context: org-member - requires: - - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}} -{%- if btype == 'wheel' %} -{%- if os == 'macos' %} - subfolder: "" -{%- else %} - subfolder: "{{cu_version}}/" -{%- endif %} -{%- endif %} -{%- endif %} -{%- endmacro %} - -{%- macro workflows(prefix='', upload=False) %} -{%- for btype in ["wheel", "conda"] -%} -{%- for os in ["linux", "macos"] -%} -{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] -%} -{%- for cu_version in (["cpu", "cu92", "cu100"] if os == "linux" else ["cpu"]) -%} -{%- for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]) -%} - {{ workflow(btype, os, python_version, cu_version, unicode, prefix=prefix, upload=upload) }} -{%- endfor -%} -{%- endfor -%} -{%- endfor -%} -{%- endfor -%} -{%- endfor %} -{%- endmacro %} workflows: build: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index dcc3fb2c23b..7e6fd747399 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -1,13 +1,91 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 + +""" +This script should use a very simple, functional programming style. +Avoid Jinja macros in favor of native Python functions. + +Don't go overboard on code generation; use Python only to generate +content that can't be easily declared statically using CircleCI's YAML API. + +Data declarations (e.g. the nested loops for defining the configuration matrix) +should be at the top of the file for easy updating. + +See this comment for design rationale: +https://github.com/pytorch/vision/pull/1321#issuecomment-531033978 +""" import jinja2 +import yaml import os.path -d = os.path.dirname(__file__) -env = jinja2.Environment( - loader=jinja2.FileSystemLoader(d), - lstrip_blocks=True, - autoescape=False, -) -with open(os.path.join(d, 'config.yml'), 'w') as f: - f.write(env.get_template('config.yml.in').render()) + +def workflows(prefix='', upload=False, indentation=6): + w = [] + for btype in ["wheel", "conda"]: + for os_type in ["linux", "macos"]: + for python_version in ["2.7", "3.5", "3.6", "3.7"]: + for cu_version in (["cpu", "cu92", "cu100"] if os_type == "linux" else ["cpu"]): + for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]): + w += workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix, upload) + + return indent(indentation, w) + + +def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False): + + w = [] + unicode_suffix = "u" if unicode else "" + base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}" + + w.append(generate_base_workflow(base_workflow_name, python_version, cu_version, unicode, os_type, btype)) + + if upload: + w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version)) + + return w + + +def generate_base_workflow(base_workflow_name, python_version, cu_version, unicode, os_type, btype): + + d = { + "name": base_workflow_name, + "python_version": python_version, + "cu_version": cu_version, + } + + if unicode: + d["unicode_abi"] = '1' + + if cu_version == "cu92": + d["wheel_docker_image"] = "soumith/manylinux-cuda92" + + return {f"binary_{os_type}_{btype}": d} + + +def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version): + d = { + "name": f"{base_workflow_name}_upload", + "context": "org-member", + "requires": [base_workflow_name], + } + + if btype == 'wheel': + d["subfolder"] = "" if os_type == 'macos' else cu_version + "/" + + return {f"binary_{btype}_upload": d} + + +def indent(indentation, data_list): + return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines()) + + +if __name__ == "__main__": + d = os.path.dirname(__file__) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=False, + ) + + with open(os.path.join(d, 'config.yml'), 'w') as f: + f.write(env.get_template('config.yml.in').render(workflows=workflows)) diff --git a/.travis.yml b/.travis.yml index fc0aa7240a4..0ac76d86f95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,12 @@ matrix: - env: LINT_CHECK python: "2.7" install: pip install flake8 - script: flake8 + script: flake8 --exclude .circleci + after_success: [] + - env: LINT_CHECK + python: "3.6" + install: pip install flake8 + script: flake8 .circleci after_success: [] - python: "2.7" env: IMAGE_BACKEND=Pillow-SIMD