From 29f1d7d890db9abd67023e0b531c7bc9aa2fec27 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Tue, 29 Nov 2022 15:51:23 +0000 Subject: [PATCH 01/27] Only run linux-3.9 unittest and show top 5000 durations --- .circleci/config.yml | 109 ------------------- .circleci/unittest/linux/scripts/run_test.sh | 2 +- 2 files changed, 1 insertion(+), 110 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d4cecee1069..183eda88268 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1488,35 +1488,10 @@ workflows: - unittest_torchhub - unittest_onnx - unittest_extended - - unittest_linux_cpu: - cu_version: cpu - name: unittest_linux_cpu_py3.7 - python_version: '3.7' - - unittest_linux_cpu: - cu_version: cpu - name: unittest_linux_cpu_py3.8 - python_version: '3.8' - unittest_linux_cpu: cu_version: cpu name: unittest_linux_cpu_py3.9 python_version: '3.9' - - unittest_linux_cpu: - cu_version: cpu - name: unittest_linux_cpu_py3.10 - python_version: '3.10' - - unittest_linux_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_linux_gpu_py3.7 - python_version: '3.7' - - unittest_linux_gpu: - cu_version: cu116 - name: unittest_linux_gpu_py3.8 - python_version: '3.8' - unittest_linux_gpu: cu_version: cu116 filters: @@ -1526,78 +1501,6 @@ workflows: - nightly name: unittest_linux_gpu_py3.9 python_version: '3.9' - - unittest_linux_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_linux_gpu_py3.10 - python_version: '3.10' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.7 - python_version: '3.7' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.8 - python_version: '3.8' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.9 - python_version: '3.9' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.10 - python_version: '3.10' - - unittest_windows_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_windows_gpu_py3.7 - python_version: '3.7' - - unittest_windows_gpu: - cu_version: cu116 - name: unittest_windows_gpu_py3.8 - python_version: '3.8' - - unittest_windows_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_windows_gpu_py3.9 - python_version: '3.9' - - unittest_windows_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_windows_gpu_py3.10 - python_version: '3.10' - - unittest_macos_cpu: - cu_version: cpu - name: unittest_macos_cpu_py3.7 - python_version: '3.7' - - unittest_macos_cpu: - cu_version: cpu - name: unittest_macos_cpu_py3.8 - python_version: '3.8' - - unittest_macos_cpu: - cu_version: cpu - name: unittest_macos_cpu_py3.9 - python_version: '3.9' - - unittest_macos_cpu: - cu_version: cpu - name: unittest_macos_cpu_py3.10 - python_version: '3.10' cmake: jobs: @@ -1610,18 +1513,6 @@ workflows: name: cmake_linux_gpu python_version: '3.8' wheel_docker_image: pytorch/manylinux-cuda116 - - cmake_windows_cpu: - cu_version: cpu - name: cmake_windows_cpu - python_version: '3.8' - - cmake_windows_gpu: - cu_version: cu116 - name: cmake_windows_gpu - python_version: '3.8' - - cmake_macos_cpu: - cu_version: cpu - name: cmake_macos_cpu - python_version: '3.8' nightly: jobs: diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index 8f6b8cb8485..1ff1c38814d 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -6,4 +6,4 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -pytest --junitxml=test-results/junit.xml -v --durations 20 +pytest --junitxml=test-results/junit.xml -v --durations 5000 From f44a6088747a5e51d0e101b5b3248a8ef0fbf0a9 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Tue, 29 Nov 2022 16:03:05 +0000 Subject: [PATCH 02/27] Only run on linux and python 3.9 by editing regenerate.py to make sure it is in sync --- .circleci/config.yml | 1103 ++------------------------------------- .circleci/regenerate.py | 6 +- 2 files changed, 58 insertions(+), 1051 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 183eda88268..2db2fc2edaf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1097,124 +1097,14 @@ workflows: build: jobs: - - binary_linux_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: /.*/ - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_linux_wheel_py3.7_cpu - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cpu - - binary_linux_wheel: - conda_docker_image: pytorch/conda-builder:cuda116 - cu_version: cu116 - name: binary_linux_wheel_py3.7_cu116 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cuda116 - - binary_linux_wheel: - conda_docker_image: pytorch/conda-builder:cuda117 - cu_version: cu117 - name: binary_linux_wheel_py3.7_cu117 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cuda117 - - binary_linux_wheel: - cu_version: rocm5.1.1 - name: binary_linux_wheel_py3.7_rocm5.1.1 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-rocm:5.1.1 - - binary_linux_wheel: - cu_version: rocm5.2 - name: binary_linux_wheel_py3.7_rocm5.2 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-rocm:5.2 - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_wheel_py3.7_cpu - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cpu - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_wheel_py3.8_cpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cpu - binary_macos_wheel: conda_docker_image: pytorch/conda-builder:cpu cu_version: cpu name: binary_macos_wheel_py3.9_cpu python_version: '3.9' wheel_docker_image: pytorch/manylinux-cpu - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_wheel_py3.10_cpu - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cpu - - binary_win_wheel: - cu_version: cpu - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_wheel_py3.7_cpu - python_version: '3.7' - - binary_win_wheel: - cu_version: cu116 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_wheel_py3.7_cu116 - python_version: '3.7' - - binary_win_wheel: - cu_version: cu117 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_wheel_py3.7_cu117 - python_version: '3.7' - - binary_win_wheel: - cu_version: cpu - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_wheel_py3.8_cpu - python_version: '3.8' - - binary_win_wheel: - cu_version: cu116 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_wheel_py3.8_cu116 - python_version: '3.8' - - binary_win_wheel: - cu_version: cu117 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_wheel_py3.8_cu117 - python_version: '3.8' - binary_win_wheel: cu_version: cpu - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: binary_win_wheel_py3.9_cpu python_version: '3.9' - binary_win_wheel: @@ -1228,66 +1118,8 @@ workflows: python_version: '3.9' - binary_win_wheel: cu_version: cu117 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: binary_win_wheel_py3.9_cu117 python_version: '3.9' - - binary_win_wheel: - cu_version: cpu - name: binary_win_wheel_py3.10_cpu - python_version: '3.10' - - binary_win_wheel: - cu_version: cu116 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_wheel_py3.10_cu116 - python_version: '3.10' - - binary_win_wheel: - cu_version: cu117 - name: binary_win_wheel_py3.10_cu117 - python_version: '3.10' - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_linux_conda_py3.7_cpu - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cpu - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda116 - cu_version: cu116 - name: binary_linux_conda_py3.7_cu116 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cuda116 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda117 - cu_version: cu117 - name: binary_linux_conda_py3.7_cu117 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cuda117 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_linux_conda_py3.8_cpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cpu - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda116 - cu_version: cu116 - name: binary_linux_conda_py3.8_cu116 - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda116 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda117 - cu_version: cu117 - name: binary_linux_conda_py3.8_cu117 - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda117 - binary_linux_conda: conda_docker_image: pytorch/conda-builder:cpu cu_version: cpu @@ -1306,109 +1138,14 @@ workflows: name: binary_linux_conda_py3.9_cu117 python_version: '3.9' wheel_docker_image: pytorch/manylinux-cuda117 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_linux_conda_py3.10_cpu - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cpu - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda116 - cu_version: cu116 - name: binary_linux_conda_py3.10_cu116 - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cuda116 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda117 - cu_version: cu117 - name: binary_linux_conda_py3.10_cu117 - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cuda117 - - binary_macos_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_conda_py3.7_cpu - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cpu - - binary_macos_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_conda_py3.8_cpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cpu - binary_macos_conda: conda_docker_image: pytorch/conda-builder:cpu cu_version: cpu name: binary_macos_conda_py3.9_cpu python_version: '3.9' wheel_docker_image: pytorch/manylinux-cpu - - binary_macos_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_conda_py3.10_cpu - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cpu - - binary_win_conda: - cu_version: cpu - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_conda_py3.7_cpu - python_version: '3.7' - - binary_win_conda: - cu_version: cu116 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_conda_py3.7_cu116 - python_version: '3.7' - - binary_win_conda: - cu_version: cu117 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_conda_py3.7_cu117 - python_version: '3.7' - - binary_win_conda: - cu_version: cpu - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_conda_py3.8_cpu - python_version: '3.8' - - binary_win_conda: - cu_version: cu116 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_conda_py3.8_cu116 - python_version: '3.8' - - binary_win_conda: - cu_version: cu117 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_conda_py3.8_cu117 - python_version: '3.8' - binary_win_conda: cu_version: cpu - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: binary_win_conda_py3.9_cpu python_version: '3.9' - binary_win_conda: @@ -1422,30 +1159,8 @@ workflows: python_version: '3.9' - binary_win_conda: cu_version: cu117 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: binary_win_conda_py3.9_cu117 python_version: '3.9' - - binary_win_conda: - cu_version: cpu - name: binary_win_conda_py3.10_cpu - python_version: '3.10' - - binary_win_conda: - cu_version: cu116 - filters: - branches: - only: main - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_win_conda_py3.10_cu116 - python_version: '3.10' - - binary_win_conda: - cu_version: cu117 - name: binary_win_conda_py3.10_cu117 - python_version: '3.10' - build_docs: filters: branches: @@ -1513,6 +1228,18 @@ workflows: name: cmake_linux_gpu python_version: '3.8' wheel_docker_image: pytorch/manylinux-cuda116 + - cmake_windows_cpu: + cu_version: cpu + name: cmake_windows_cpu + python_version: '3.8' + - cmake_windows_gpu: + cu_version: cu116 + name: cmake_windows_gpu + python_version: '3.8' + - cmake_macos_cpu: + cu_version: cpu + name: cmake_macos_cpu + python_version: '3.8' nightly: jobs: @@ -1552,7 +1279,7 @@ workflows: only: - nightly name: nightly_binary_libtorchvision_ops_android_upload - - binary_linux_wheel: + - binary_macos_wheel: conda_docker_image: pytorch/conda-builder:cpu cu_version: cpu filters: @@ -1560,568 +1287,80 @@ workflows: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.7_cpu - python_version: '3.7' + name: nightly_binary_macos_wheel_py3.9_cpu + python_version: '3.9' wheel_docker_image: pytorch/manylinux-cpu - - binary_linux_wheel: - conda_docker_image: pytorch/conda-builder:cuda116 - cu_version: cu116 + - binary_wheel_upload: + context: org-member filters: branches: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.7_cu116 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cuda116 - - binary_linux_wheel: - conda_docker_image: pytorch/conda-builder:cuda117 - cu_version: cu117 + name: nightly_binary_macos_wheel_py3.9_cpu_upload + requires: + - nightly_binary_macos_wheel_py3.9_cpu + subfolder: '' + - binary_win_wheel: + cu_version: cpu filters: branches: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.7_cu117 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cuda117 - - binary_linux_wheel: - cu_version: rocm5.1.1 + name: nightly_binary_win_wheel_py3.9_cpu + python_version: '3.9' + - binary_wheel_upload: + context: org-member filters: branches: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.7_rocm5.1.1 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-rocm:5.1.1 - - binary_linux_wheel: - cu_version: rocm5.2 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.7_rocm5.2 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-rocm:5.2 - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_wheel_py3.7_cpu - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cpu - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_wheel_py3.7_cpu_upload - requires: - - nightly_binary_macos_wheel_py3.7_cpu - subfolder: '' - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_wheel_py3.8_cpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cpu - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_wheel_py3.8_cpu_upload - requires: - - nightly_binary_macos_wheel_py3.8_cpu - subfolder: '' - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_wheel_py3.9_cpu - python_version: '3.9' - wheel_docker_image: pytorch/manylinux-cpu - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_wheel_py3.9_cpu_upload - requires: - - nightly_binary_macos_wheel_py3.9_cpu - subfolder: '' - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_wheel_py3.10_cpu - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cpu - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_wheel_py3.10_cpu_upload - requires: - - nightly_binary_macos_wheel_py3.10_cpu - subfolder: '' - - binary_win_wheel: - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.7_cpu - python_version: '3.7' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.7_cpu_upload - requires: - - nightly_binary_win_wheel_py3.7_cpu - subfolder: cpu/ - - binary_win_wheel: - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.7_cu116 - python_version: '3.7' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.7_cu116_upload - requires: - - nightly_binary_win_wheel_py3.7_cu116 - subfolder: cu116/ - - binary_win_wheel: - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.7_cu117 - python_version: '3.7' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.7_cu117_upload - requires: - - nightly_binary_win_wheel_py3.7_cu117 - subfolder: cu117/ - - binary_win_wheel: - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.8_cpu - python_version: '3.8' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.8_cpu_upload - requires: - - nightly_binary_win_wheel_py3.8_cpu - subfolder: cpu/ - - binary_win_wheel: - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.8_cu116 - python_version: '3.8' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.8_cu116_upload - requires: - - nightly_binary_win_wheel_py3.8_cu116 - subfolder: cu116/ - - binary_win_wheel: - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.8_cu117 - python_version: '3.8' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.8_cu117_upload - requires: - - nightly_binary_win_wheel_py3.8_cu117 - subfolder: cu117/ - - binary_win_wheel: - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.9_cpu - python_version: '3.9' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.9_cpu_upload - requires: - - nightly_binary_win_wheel_py3.9_cpu - subfolder: cpu/ - - binary_win_wheel: - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.9_cu116 - python_version: '3.9' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.9_cu116_upload - requires: - - nightly_binary_win_wheel_py3.9_cu116 - subfolder: cu116/ - - binary_win_wheel: - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.9_cu117 - python_version: '3.9' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.9_cu117_upload - requires: - - nightly_binary_win_wheel_py3.9_cu117 - subfolder: cu117/ - - binary_win_wheel: - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.10_cpu - python_version: '3.10' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.10_cpu_upload - requires: - - nightly_binary_win_wheel_py3.10_cpu - subfolder: cpu/ - - binary_win_wheel: - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.10_cu116 - python_version: '3.10' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.10_cu116_upload - requires: - - nightly_binary_win_wheel_py3.10_cu116 - subfolder: cu116/ - - binary_win_wheel: - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.10_cu117 - python_version: '3.10' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_wheel_py3.10_cu117_upload - requires: - - nightly_binary_win_wheel_py3.10_cu117 - subfolder: cu117/ - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.7_cpu - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cpu - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.7_cpu_upload - requires: - - nightly_binary_linux_conda_py3.7_cpu - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda116 - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.7_cu116 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cuda116 - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.7_cu116_upload - requires: - - nightly_binary_linux_conda_py3.7_cu116 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda117 - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.7_cu117 - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cuda117 - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.7_cu117_upload - requires: - - nightly_binary_linux_conda_py3.7_cu117 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.8_cpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cpu - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.8_cpu_upload - requires: - - nightly_binary_linux_conda_py3.8_cpu - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda116 - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.8_cu116 - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda116 - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.8_cu116_upload - requires: - - nightly_binary_linux_conda_py3.8_cu116 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda117 - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.8_cu117 - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda117 - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.8_cu117_upload - requires: - - nightly_binary_linux_conda_py3.8_cu117 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.9_cpu - python_version: '3.9' - wheel_docker_image: pytorch/manylinux-cpu - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.9_cpu_upload + name: nightly_binary_win_wheel_py3.9_cpu_upload requires: - - nightly_binary_linux_conda_py3.9_cpu - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda116 + - nightly_binary_win_wheel_py3.9_cpu + subfolder: cpu/ + - binary_win_wheel: cu_version: cu116 filters: branches: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.9_cu116 + name: nightly_binary_win_wheel_py3.9_cu116 python_version: '3.9' - wheel_docker_image: pytorch/manylinux-cuda116 - - binary_conda_upload: + - binary_wheel_upload: context: org-member filters: branches: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.9_cu116_upload + name: nightly_binary_win_wheel_py3.9_cu116_upload requires: - - nightly_binary_linux_conda_py3.9_cu116 - - binary_linux_conda: - conda_docker_image: pytorch/conda-builder:cuda117 + - nightly_binary_win_wheel_py3.9_cu116 + subfolder: cu116/ + - binary_win_wheel: cu_version: cu117 filters: branches: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.9_cu117 + name: nightly_binary_win_wheel_py3.9_cu117 python_version: '3.9' - wheel_docker_image: pytorch/manylinux-cuda117 - - binary_conda_upload: + - binary_wheel_upload: context: org-member filters: branches: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.9_cu117_upload + name: nightly_binary_win_wheel_py3.9_cu117_upload requires: - - nightly_binary_linux_conda_py3.9_cu117 + - nightly_binary_win_wheel_py3.9_cu117 + subfolder: cu117/ - binary_linux_conda: conda_docker_image: pytorch/conda-builder:cpu cu_version: cpu @@ -2130,8 +1369,8 @@ workflows: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.10_cpu - python_version: '3.10' + name: nightly_binary_linux_conda_py3.9_cpu + python_version: '3.9' wheel_docker_image: pytorch/manylinux-cpu - binary_conda_upload: context: org-member @@ -2140,9 +1379,9 @@ workflows: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.10_cpu_upload + name: nightly_binary_linux_conda_py3.9_cpu_upload requires: - - nightly_binary_linux_conda_py3.10_cpu + - nightly_binary_linux_conda_py3.9_cpu - binary_linux_conda: conda_docker_image: pytorch/conda-builder:cuda116 cu_version: cu116 @@ -2151,8 +1390,8 @@ workflows: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.10_cu116 - python_version: '3.10' + name: nightly_binary_linux_conda_py3.9_cu116 + python_version: '3.9' wheel_docker_image: pytorch/manylinux-cuda116 - binary_conda_upload: context: org-member @@ -2161,9 +1400,9 @@ workflows: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.10_cu116_upload + name: nightly_binary_linux_conda_py3.9_cu116_upload requires: - - nightly_binary_linux_conda_py3.10_cu116 + - nightly_binary_linux_conda_py3.9_cu116 - binary_linux_conda: conda_docker_image: pytorch/conda-builder:cuda117 cu_version: cu117 @@ -2172,8 +1411,8 @@ workflows: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.10_cu117 - python_version: '3.10' + name: nightly_binary_linux_conda_py3.9_cu117 + python_version: '3.9' wheel_docker_image: pytorch/manylinux-cuda117 - binary_conda_upload: context: org-member @@ -2182,51 +1421,9 @@ workflows: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_conda_py3.10_cu117_upload - requires: - - nightly_binary_linux_conda_py3.10_cu117 - - binary_macos_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_conda_py3.7_cpu - python_version: '3.7' - wheel_docker_image: pytorch/manylinux-cpu - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_conda_py3.7_cpu_upload - requires: - - nightly_binary_macos_conda_py3.7_cpu - - binary_macos_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_conda_py3.8_cpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cpu - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_conda_py3.8_cpu_upload + name: nightly_binary_linux_conda_py3.9_cu117_upload requires: - - nightly_binary_macos_conda_py3.8_cpu + - nightly_binary_linux_conda_py3.9_cu117 - binary_macos_conda: conda_docker_image: pytorch/conda-builder:cpu cu_version: cpu @@ -2248,141 +1445,6 @@ workflows: name: nightly_binary_macos_conda_py3.9_cpu_upload requires: - nightly_binary_macos_conda_py3.9_cpu - - binary_macos_conda: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_conda_py3.10_cpu - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cpu - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_macos_conda_py3.10_cpu_upload - requires: - - nightly_binary_macos_conda_py3.10_cpu - - binary_win_conda: - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.7_cpu - python_version: '3.7' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.7_cpu_upload - requires: - - nightly_binary_win_conda_py3.7_cpu - - binary_win_conda: - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.7_cu116 - python_version: '3.7' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.7_cu116_upload - requires: - - nightly_binary_win_conda_py3.7_cu116 - - binary_win_conda: - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.7_cu117 - python_version: '3.7' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.7_cu117_upload - requires: - - nightly_binary_win_conda_py3.7_cu117 - - binary_win_conda: - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.8_cpu - python_version: '3.8' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.8_cpu_upload - requires: - - nightly_binary_win_conda_py3.8_cpu - - binary_win_conda: - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.8_cu116 - python_version: '3.8' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.8_cu116_upload - requires: - - nightly_binary_win_conda_py3.8_cu116 - - binary_win_conda: - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.8_cu117 - python_version: '3.8' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.8_cu117_upload - requires: - - nightly_binary_win_conda_py3.8_cu117 - binary_win_conda: cu_version: cpu filters: @@ -2440,63 +1502,6 @@ workflows: name: nightly_binary_win_conda_py3.9_cu117_upload requires: - nightly_binary_win_conda_py3.9_cu117 - - binary_win_conda: - cu_version: cpu - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.10_cpu - python_version: '3.10' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.10_cpu_upload - requires: - - nightly_binary_win_conda_py3.10_cpu - - binary_win_conda: - cu_version: cu116 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.10_cu116 - python_version: '3.10' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.10_cu116_upload - requires: - - nightly_binary_win_conda_py3.10_cu116 - - binary_win_conda: - cu_version: cu117 - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.10_cu117 - python_version: '3.10' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_win_conda_py3.10_cu117_upload - requires: - - nightly_binary_win_conda_py3.10_cu117 docker_build: triggers: - schedule: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 3a3ab14a4e1..6ca75bb22c8 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -21,7 +21,8 @@ from jinja2 import select_autoescape -PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +# PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +PYTHON_VERSIONS = ["3.9"] RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" @@ -246,7 +247,8 @@ def indent(indentation, data_list): def unittest_workflows(indentation=6): jobs = [] - for os_type in ["linux", "windows", "macos"]: + # for os_type in ["linux", "windows", "macos"]: + for os_type in ["linux"]: for device_type in ["cpu", "gpu"]: if os_type == "macos" and device_type == "gpu": continue From 118e73160a516c78afa4732dc794bf3090ef1a51 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Tue, 29 Nov 2022 17:28:11 +0000 Subject: [PATCH 03/27] Remove build workflow --- .circleci/config.yml | 432 ---------------------------------------- .circleci/config.yml.in | 79 -------- 2 files changed, 511 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2db2fc2edaf..c6062ab454b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1019,74 +1019,6 @@ jobs: packaging/windows/internal/cuda_install.bat packaging/build_cmake.sh - build_docs: - <<: *binary_common - docker: - - image: cimg/python:3.7 - resource_class: 2xlarge+ - steps: - - attach_workspace: - at: ~/workspace - - checkout - - download_model_weights - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - designate_upload_channel - - run: - name: Install torchvision - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Build docs - command: | - set -ex - # turn v1.12.0rc3 into 1.12.0 - tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9.]*\).*/\1/') - VERSION=${tag:-main} - eval "$(./conda/bin/conda shell.bash hook)" - conda activate ./env - pushd docs - pip install --progress-bar=off -r requirements.txt - make html - popd - - persist_to_workspace: - root: ./ - paths: - - "*" - - store_artifacts: - path: ./docs/build/html - destination: docs - - upload_docs: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda100" - resource_class: 2xlarge+ - steps: - - attach_workspace: - at: ~/workspace - - run: - name: Generate netrc - command: | - # set credentials for https pushing - # requires the org-member context - cat > ~/.netrc \< ~/.netrc \< Date: Tue, 29 Nov 2022 18:55:52 +0000 Subject: [PATCH 04/27] Change github workflow to show slowest 5000 durations --- .github/workflows/test-linux-cpu.yml | 2 +- .github/workflows/test-linux-gpu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 234ad97f4d6..77ea2eb5ca8 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -54,4 +54,4 @@ jobs: # Run Tests python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 + python3 -m pytest --junitxml=test-results/junit.xml -v --durations 5000 diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml index a4d938f23ed..6e2e8026ef6 100644 --- a/.github/workflows/test-linux-gpu.yml +++ b/.github/workflows/test-linux-gpu.yml @@ -58,4 +58,4 @@ jobs: # Run Tests python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 + python3 -m pytest --junitxml=test-results/junit.xml -v --durations 5000 From 900d47899fa77d37b2ad4f4e34725d2457177142 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Tue, 29 Nov 2022 19:31:53 +0000 Subject: [PATCH 05/27] Fix indent in test-linux-gpu --- .github/workflows/test-linux-gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml index 6e2e8026ef6..1b548a3adcf 100644 --- a/.github/workflows/test-linux-gpu.yml +++ b/.github/workflows/test-linux-gpu.yml @@ -58,4 +58,4 @@ jobs: # Run Tests python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 5000 + python3 -m pytest --junitxml=test-results/junit.xml -v --durations 5000 From 19f2d2979fe16730a497bbbd28fc6c1db58c3f18 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Tue, 29 Nov 2022 22:01:55 +0000 Subject: [PATCH 06/27] Use python38 instead of 39 to spawn gpu process --- .circleci/config.yml | 13 ++++--------- .circleci/regenerate.py | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c6062ab454b..d60ca8baeb6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1034,17 +1034,12 @@ workflows: - unittest_extended - unittest_linux_cpu: cu_version: cpu - name: unittest_linux_cpu_py3.9 - python_version: '3.9' + name: unittest_linux_cpu_py3.8 + python_version: '3.8' - unittest_linux_gpu: cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_linux_gpu_py3.9 - python_version: '3.9' + name: unittest_linux_gpu_py3.8 + python_version: '3.8' cmake: jobs: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 6ca75bb22c8..4b9faa4abc1 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -22,7 +22,7 @@ # PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] -PYTHON_VERSIONS = ["3.9"] +PYTHON_VERSIONS = ["3.8"] RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" From bafa20df3051547c2156dfd0ad09a1c62350828a Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 09:40:21 +0000 Subject: [PATCH 07/27] Reduce image size in TestFXFeature to 112 --- test/test_backbone_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_backbone_utils.py b/test/test_backbone_utils.py index 4fba3c3d098..b494dcb8adb 100644 --- a/test/test_backbone_utils.py +++ b/test/test_backbone_utils.py @@ -90,7 +90,7 @@ def forward(self, x): class TestFxFeatureExtraction: - inp = torch.rand(1, 3, 224, 224, dtype=torch.float32, device="cpu") + inp = torch.rand(1, 3, 112, 112, dtype=torch.float32, device="cpu") model_defaults = {"num_classes": 1} leaf_modules = [] From 66a7aef40b20705775bc08b0d37afd70ef5d2314 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 13:39:17 +0000 Subject: [PATCH 08/27] Revert back test FXFeature to use image size of 224 since it doesn't improve speed too much --- test/test_backbone_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_backbone_utils.py b/test/test_backbone_utils.py index b494dcb8adb..4fba3c3d098 100644 --- a/test/test_backbone_utils.py +++ b/test/test_backbone_utils.py @@ -90,7 +90,7 @@ def forward(self, x): class TestFxFeatureExtraction: - inp = torch.rand(1, 3, 112, 112, dtype=torch.float32, device="cpu") + inp = torch.rand(1, 3, 224, 224, dtype=torch.float32, device="cpu") model_defaults = {"num_classes": 1} leaf_modules = [] From a6d3628c9b89a4e1783c4194d80e6d771996d6f4 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 14:32:48 +0000 Subject: [PATCH 09/27] Ignore test_models.py on linux run --- .circleci/unittest/linux/scripts/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index 1ff1c38814d..4cb2374babc 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -6,4 +6,4 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -pytest --junitxml=test-results/junit.xml -v --durations 5000 +pytest --ignore=test/test_models.py --junitxml=test-results/junit.xml -v --durations 5000 From 6f0d5fbc7cd3a7fa9611f0285e2082c4649dd6bf Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 15:47:58 +0000 Subject: [PATCH 10/27] Separate models and backbone, reduce image size for other unittest --- .circleci/config.yml | 16 +++++++++++++++- .circleci/config.yml.in | 16 +++++++++++++++- .circleci/unittest/linux/scripts/run_test.sh | 2 +- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d60ca8baeb6..2633155ccdf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -359,6 +359,20 @@ jobs: - run_tests_selective: file_or_dir: test/test_extended_*.py + unittest_models: + docker: + - image: cimg/python:3.7 + resource_class: 2xlarge+ + steps: + - checkout + - download_model_weights + - install_torchvision + - run: + name: Enable extended tests + command: echo 'export PYTORCH_TEST_WITH_EXTENDED=1' >> $BASH_ENV + - run_tests_selective: + file_or_dir: "test/test_models.py test/test_backbone_utils.py" + binary_linux_wheel: <<: *binary_common docker: @@ -726,7 +740,7 @@ jobs: <<: *binary_common docker: - image: "pytorch/manylinux-cpu" - resource_class: 2xlarge+ + resource_class: xlarge steps: - checkout - designate_upload_channel diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index b6915b3582c..8f0541a582b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -359,6 +359,20 @@ jobs: - run_tests_selective: file_or_dir: test/test_extended_*.py + unittest_models: + docker: + - image: cimg/python:3.7 + resource_class: 2xlarge+ + steps: + - checkout + - download_model_weights + - install_torchvision + - run: + name: Enable extended tests + command: echo 'export PYTORCH_TEST_WITH_EXTENDED=1' >> $BASH_ENV + - run_tests_selective: + file_or_dir: "test/test_models.py test/test_backbone_utils.py" + binary_linux_wheel: <<: *binary_common docker: @@ -726,7 +740,7 @@ jobs: <<: *binary_common docker: - image: "pytorch/manylinux-cpu" - resource_class: 2xlarge+ + resource_class: xlarge steps: - checkout - designate_upload_channel diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index 4cb2374babc..ee21a7ed87e 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -6,4 +6,4 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -pytest --ignore=test/test_models.py --junitxml=test-results/junit.xml -v --durations 5000 +pytest --ignore=test/test_models.py --ignore=test/test_backbone_utils.py --junitxml=test-results/junit.xml -v --durations 5000 From ccc2fb1312a1ed44c14c4f0a57a781c0ad45157b Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 16:06:52 +0000 Subject: [PATCH 11/27] Add unittest_models in job --- .circleci/config.yml | 1 + .circleci/config.yml.in | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2633155ccdf..84dadb0abb8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1046,6 +1046,7 @@ workflows: - unittest_torchhub - unittest_onnx - unittest_extended + - unittest_models - unittest_linux_cpu: cu_version: cpu name: unittest_linux_cpu_py3.8 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 8f0541a582b..9c1ddc37723 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -1046,6 +1046,7 @@ workflows: - unittest_torchhub - unittest_onnx - unittest_extended + - unittest_models {{ unittest_workflows() }} cmake: From 732a228d69620b24852e8a97e866d6dc3004fe0d Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 18:00:29 +0000 Subject: [PATCH 12/27] Use pytest mark --- .circleci/config.yml | 37 +++++++++++++++++--- .circleci/config.yml.in | 8 ++--- .circleci/regenerate.py | 15 +++++++- .circleci/unittest/linux/scripts/run_test.sh | 4 ++- pytest.ini | 3 ++ test/test_backbone_utils.py | 3 ++ test/test_models.py | 3 ++ 7 files changed, 63 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84dadb0abb8..b080e97649a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -740,7 +740,7 @@ jobs: <<: *binary_common docker: - image: "pytorch/manylinux-cpu" - resource_class: xlarge + resource_class: << parameters.resource_class >> steps: - checkout - designate_upload_channel @@ -768,7 +768,7 @@ jobs: command: .circleci/unittest/linux/scripts/install.sh - run: name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh + command: .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post process command: .circleci/unittest/linux/scripts/post_process.sh @@ -779,7 +779,7 @@ jobs: <<: *binary_common machine: image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.medium + resource_class: << parameters.resource_class >> environment: image_name: "pytorch/manylinux-cuda116" CU_VERSION: << parameters.cu_version >> @@ -819,7 +819,7 @@ jobs: command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests - command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh + command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" ".circleci/unittest/linux/scripts/run_test.sh \"<< parameters.pytest_additional_args >>\"" - run: name: Post Process command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh @@ -1051,10 +1051,39 @@ workflows: cu_version: cpu name: unittest_linux_cpu_py3.8 python_version: '3.8' + resource_class: xlarge + - unittest_linux_cpu: + cu_version: cpu + name: unittest_linux_cpu_py3.8 + pytest_additional_args: -m slow + python_version: '3.8' + resource_class: 2xlarge+ + - unittest_linux_cpu: + cu_version: cpu + name: unittest_linux_cpu_py3.9 + python_version: '3.9' + resource_class: xlarge - unittest_linux_gpu: cu_version: cu116 name: unittest_linux_gpu_py3.8 python_version: '3.8' + resource_class: gpu.nvidia.medium + - unittest_linux_gpu: + cu_version: cu116 + name: unittest_linux_gpu_py3.8 + pytest_additional_args: -m slow + python_version: '3.8' + resource_class: gpu.nvidia.medium + - unittest_linux_gpu: + cu_version: cu116 + filters: + branches: + only: + - main + - nightly + name: unittest_linux_gpu_py3.9 + python_version: '3.9' + resource_class: gpu.nvidia.medium cmake: jobs: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 9c1ddc37723..07389662bab 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -740,7 +740,7 @@ jobs: <<: *binary_common docker: - image: "pytorch/manylinux-cpu" - resource_class: xlarge + resource_class: << parameters.resource_class >> steps: - checkout - designate_upload_channel @@ -768,7 +768,7 @@ jobs: command: .circleci/unittest/linux/scripts/install.sh - run: name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh + command: .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post process command: .circleci/unittest/linux/scripts/post_process.sh @@ -779,7 +779,7 @@ jobs: <<: *binary_common machine: image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.medium + resource_class: << parameters.resource_class >> environment: image_name: "pytorch/manylinux-cuda116" CU_VERSION: << parameters.cu_version >> @@ -819,7 +819,7 @@ jobs: command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests - command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh + command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" ".circleci/unittest/linux/scripts/run_test.sh \"<< parameters.pytest_additional_args >>\"" - run: name: Post Process command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 4b9faa4abc1..8596d1dacc2 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -16,13 +16,14 @@ import os.path +import copy import jinja2 import yaml from jinja2 import select_autoescape # PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] -PYTHON_VERSIONS = ["3.8"] +PYTHON_VERSIONS = ["3.8", "3.9"] RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" @@ -262,11 +263,23 @@ def unittest_workflows(indentation=6): if python_version != "3.8": job["filters"] = gen_filter_branch_tree("main", "nightly") job["cu_version"] = "cu116" + if os_type == "linux": + job["resource_class"] = "gpu.nvidia.medium" else: job["cu_version"] = "cpu" + if os_type == "linux": + job["resource_class"] = "xlarge" jobs.append({f"unittest_{os_type}_{device_type}": job}) + # Add slow_only job for python 3.8: + if python_version == "3.8": + slow_only_job = copy.deepcopy(job) + slow_only_job["pytest_additional_args"] = "-m slow" + if os_type == "linux" and device_type == "cpu": + slow_only_job["resource_class"] = "2xlarge+" + jobs.append({f"unittest_{os_type}_{device_type}": slow_only_job}) + return indent(indentation, jobs) diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index ee21a7ed87e..abac92b6662 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -2,8 +2,10 @@ set -e +PYTEST_ADDITIONAL_ARGS=$1 + eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -pytest --ignore=test/test_models.py --ignore=test/test_backbone_utils.py --junitxml=test-results/junit.xml -v --durations 5000 +pytest --junitxml=test-results/junit.xml -v --durations 5000 ${PYTEST_ADDITIONAL_ARGS} diff --git a/pytest.ini b/pytest.ini index a2f59ecec46..fc541a55fa3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,6 +8,9 @@ addopts = -Wd --ignore=test/test_datasets_download.py --ignore-glob=test/test_prototype_*.py + -m "not slow" testpaths = test xfail_strict = True +markers = + slow diff --git a/test/test_backbone_utils.py b/test/test_backbone_utils.py index 4fba3c3d098..deb75aca174 100644 --- a/test/test_backbone_utils.py +++ b/test/test_backbone_utils.py @@ -11,6 +11,9 @@ from torchvision.models.feature_extraction import create_feature_extractor, get_graph_node_names +pytestmark = pytest.mark.slow + + @pytest.mark.parametrize("backbone_name", ("resnet18", "resnet50")) def test_resnet_fpn_backbone(backbone_name): x = torch.rand(1, 3, 300, 300, dtype=torch.float32, device="cpu") diff --git a/test/test_models.py b/test/test_models.py index 083096ae2ee..f0caebc43b1 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -21,6 +21,9 @@ from torchvision.models import get_model_builder, list_models +pytestmark = pytest.mark.slow + + ACCEPT = os.getenv("EXPECTTEST_ACCEPT", "0") == "1" SKIP_BIG_MODEL = os.getenv("SKIP_BIG_MODEL", "1") == "1" From 44d74acb7b867ee14b97b13c7acfcdfb382787a1 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 18:34:14 +0000 Subject: [PATCH 13/27] Register the param resource_class and pytest_additional_args --- .circleci/config.yml | 6 ++++++ .circleci/config.yml.in | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b080e97649a..47cba31ae6d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -207,6 +207,12 @@ binary_common: &binary_common description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" type: string default: "cpu" + pytest_additional_args: + description: "Additional arguments to be given to pytest call" + default: "" + resource_class: + description: "The resource class (machine type) that used to run the test" + default: "xlarge" unicode_abi: description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" type: string diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 07389662bab..5ee4d4bf99e 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -207,6 +207,12 @@ binary_common: &binary_common description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" type: string default: "cpu" + pytest_additional_args: + description: "Additional arguments to be given to pytest call" + default: "" + resource_class: + description: "The resource class (machine type) that used to run the test" + default: "xlarge" unicode_abi: description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" type: string From b47b7bcb79b6ae8934b8c95c0884b5cf8df3ce21 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 18:42:56 +0000 Subject: [PATCH 14/27] Fix config.yml --- .circleci/config.yml | 22 ++++++++++++---------- .circleci/config.yml.in | 10 ++++++---- .circleci/regenerate.py | 6 +++--- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47cba31ae6d..55ce5f9a537 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -209,9 +209,11 @@ binary_common: &binary_common default: "cpu" pytest_additional_args: description: "Additional arguments to be given to pytest call" + type: string default: "" - resource_class: - description: "The resource class (machine type) that used to run the test" + machine_type: + description: "The machine type (or resource class) that used to run the test" + type: string default: "xlarge" unicode_abi: description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" @@ -746,7 +748,7 @@ jobs: <<: *binary_common docker: - image: "pytorch/manylinux-cpu" - resource_class: << parameters.resource_class >> + resource_class: << parameters.machine_type >> steps: - checkout - designate_upload_channel @@ -785,7 +787,7 @@ jobs: <<: *binary_common machine: image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: << parameters.resource_class >> + resource_class: << parameters.machine_type >> environment: image_name: "pytorch/manylinux-cuda116" CU_VERSION: << parameters.cu_version >> @@ -1055,31 +1057,31 @@ workflows: - unittest_models - unittest_linux_cpu: cu_version: cpu + machine_type: xlarge name: unittest_linux_cpu_py3.8 python_version: '3.8' - resource_class: xlarge - unittest_linux_cpu: cu_version: cpu + machine_type: 2xlarge+ name: unittest_linux_cpu_py3.8 pytest_additional_args: -m slow python_version: '3.8' - resource_class: 2xlarge+ - unittest_linux_cpu: cu_version: cpu + machine_type: xlarge name: unittest_linux_cpu_py3.9 python_version: '3.9' - resource_class: xlarge - unittest_linux_gpu: cu_version: cu116 + machine_type: gpu.nvidia.medium name: unittest_linux_gpu_py3.8 python_version: '3.8' - resource_class: gpu.nvidia.medium - unittest_linux_gpu: cu_version: cu116 + machine_type: gpu.nvidia.medium name: unittest_linux_gpu_py3.8 pytest_additional_args: -m slow python_version: '3.8' - resource_class: gpu.nvidia.medium - unittest_linux_gpu: cu_version: cu116 filters: @@ -1087,9 +1089,9 @@ workflows: only: - main - nightly + machine_type: gpu.nvidia.medium name: unittest_linux_gpu_py3.9 python_version: '3.9' - resource_class: gpu.nvidia.medium cmake: jobs: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 5ee4d4bf99e..fc21116c3b0 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -209,9 +209,11 @@ binary_common: &binary_common default: "cpu" pytest_additional_args: description: "Additional arguments to be given to pytest call" + type: string default: "" - resource_class: - description: "The resource class (machine type) that used to run the test" + machine_type: + description: "The machine type (or resource class) that used to run the test" + type: string default: "xlarge" unicode_abi: description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" @@ -746,7 +748,7 @@ jobs: <<: *binary_common docker: - image: "pytorch/manylinux-cpu" - resource_class: << parameters.resource_class >> + resource_class: << parameters.machine_type >> steps: - checkout - designate_upload_channel @@ -785,7 +787,7 @@ jobs: <<: *binary_common machine: image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: << parameters.resource_class >> + resource_class: << parameters.machine_type >> environment: image_name: "pytorch/manylinux-cuda116" CU_VERSION: << parameters.cu_version >> diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 8596d1dacc2..cdaecabebca 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -264,11 +264,11 @@ def unittest_workflows(indentation=6): job["filters"] = gen_filter_branch_tree("main", "nightly") job["cu_version"] = "cu116" if os_type == "linux": - job["resource_class"] = "gpu.nvidia.medium" + job["machine_type"] = "gpu.nvidia.medium" else: job["cu_version"] = "cpu" if os_type == "linux": - job["resource_class"] = "xlarge" + job["machine_type"] = "xlarge" jobs.append({f"unittest_{os_type}_{device_type}": job}) @@ -277,7 +277,7 @@ def unittest_workflows(indentation=6): slow_only_job = copy.deepcopy(job) slow_only_job["pytest_additional_args"] = "-m slow" if os_type == "linux" and device_type == "cpu": - slow_only_job["resource_class"] = "2xlarge+" + slow_only_job["machine_type"] = "2xlarge+" jobs.append({f"unittest_{os_type}_{device_type}": slow_only_job}) return indent(indentation, jobs) From a1d44abcf34c9aaa3585091de86d4c7666290910 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 18:46:53 +0000 Subject: [PATCH 15/27] ufmt format --- .circleci/regenerate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index cdaecabebca..968a9d86004 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -14,9 +14,9 @@ https://github.com/pytorch/vision/pull/1321#issuecomment-531033978 """ +import copy import os.path -import copy import jinja2 import yaml from jinja2 import select_autoescape From 5af1b9c96d134a65bf4030e1be6632c7141acce3 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 18:53:14 +0000 Subject: [PATCH 16/27] Change name for slow only --- .circleci/config.yml | 4 ++-- .circleci/regenerate.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55ce5f9a537..b14bb6541bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1063,7 +1063,7 @@ workflows: - unittest_linux_cpu: cu_version: cpu machine_type: 2xlarge+ - name: unittest_linux_cpu_py3.8 + name: unittest_linux_cpu_py3.8_slow_only pytest_additional_args: -m slow python_version: '3.8' - unittest_linux_cpu: @@ -1079,7 +1079,7 @@ workflows: - unittest_linux_gpu: cu_version: cu116 machine_type: gpu.nvidia.medium - name: unittest_linux_gpu_py3.8 + name: unittest_linux_gpu_py3.8_slow_only pytest_additional_args: -m slow python_version: '3.8' - unittest_linux_gpu: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 968a9d86004..da2278a6c7a 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -276,6 +276,7 @@ def unittest_workflows(indentation=6): if python_version == "3.8": slow_only_job = copy.deepcopy(job) slow_only_job["pytest_additional_args"] = "-m slow" + slow_only_job["name"] = f"{slow_only_job['name']}_slow_only" if os_type == "linux" and device_type == "cpu": slow_only_job["machine_type"] = "2xlarge+" jobs.append({f"unittest_{os_type}_{device_type}": slow_only_job}) From da3c4107bb2298a261a630f492c9d8b5637955ca Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 21:40:55 +0000 Subject: [PATCH 17/27] Try to fix docker run param for linux_gpu --- .circleci/config.yml.in | 2 +- .circleci/unittest/linux/scripts/run_test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index fc21116c3b0..ef1dc91bf12 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -827,7 +827,7 @@ jobs: command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests - command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" ".circleci/unittest/linux/scripts/run_test.sh \"<< parameters.pytest_additional_args >>\"" + command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post Process command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index abac92b6662..c30c90b3a0f 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -8,4 +8,4 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -pytest --junitxml=test-results/junit.xml -v --durations 5000 ${PYTEST_ADDITIONAL_ARGS} +pytest --junitxml=test-results/junit.xml -v --durations 20 ${PYTEST_ADDITIONAL_ARGS} From 2ae7589404ed852f90181de8395573925457af24 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 30 Nov 2022 22:09:18 +0000 Subject: [PATCH 18/27] regenerate config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b14bb6541bc..b942f787197 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -827,7 +827,7 @@ jobs: command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests - command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" ".circleci/unittest/linux/scripts/run_test.sh \"<< parameters.pytest_additional_args >>\"" + command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post Process command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh From a6bfa5a9b4a88dd46afd0db211945bd1d5ddc790 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Thu, 1 Dec 2022 00:25:17 +0000 Subject: [PATCH 19/27] Test with macos and windows --- .circleci/config.yml | 60 +++++++++++++++---- .circleci/config.yml.in | 10 ++-- .circleci/regenerate.py | 19 ++++-- .../unittest/windows/scripts/run_test.sh | 4 +- 4 files changed, 70 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b942f787197..3d9d16a3b5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -214,7 +214,7 @@ binary_common: &binary_common machine_type: description: "The machine type (or resource class) that used to run the test" type: string - default: "xlarge" + default: "large" unicode_abi: description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" type: string @@ -865,7 +865,7 @@ jobs: command: .circleci/unittest/windows/scripts/install.sh - run: name: Run tests - command: .circleci/unittest/windows/scripts/run_test.sh + command: .circleci/unittest/windows/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post process command: .circleci/unittest/windows/scripts/post_process.sh @@ -912,7 +912,7 @@ jobs: command: .circleci/unittest/windows/scripts/install.sh - run: name: Run tests - command: .circleci/unittest/windows/scripts/run_test.sh + command: .circleci/unittest/windows/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post process command: .circleci/unittest/windows/scripts/post_process.sh @@ -923,7 +923,7 @@ jobs: <<: *binary_common macos: xcode: "14.0" - resource_class: large + resource_class: << parameters.machine_type >> steps: - checkout - designate_upload_channel @@ -955,7 +955,7 @@ jobs: command: .circleci/unittest/linux/scripts/install.sh - run: name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh + command: .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post process command: .circleci/unittest/linux/scripts/post_process.sh @@ -1057,7 +1057,7 @@ workflows: - unittest_models - unittest_linux_cpu: cu_version: cpu - machine_type: xlarge + machine_type: large name: unittest_linux_cpu_py3.8 python_version: '3.8' - unittest_linux_cpu: @@ -1068,29 +1068,67 @@ workflows: python_version: '3.8' - unittest_linux_cpu: cu_version: cpu - machine_type: xlarge + machine_type: large name: unittest_linux_cpu_py3.9 python_version: '3.9' - unittest_linux_gpu: cu_version: cu116 machine_type: gpu.nvidia.medium name: unittest_linux_gpu_py3.8 + pytest_additional_args: -m "slow or not slow" python_version: '3.8' - unittest_linux_gpu: cu_version: cu116 + filters: + branches: + only: + - main + - nightly machine_type: gpu.nvidia.medium - name: unittest_linux_gpu_py3.8_slow_only + name: unittest_linux_gpu_py3.9 + python_version: '3.9' + - unittest_windows_cpu: + cu_version: cpu + name: unittest_windows_cpu_py3.8 + python_version: '3.8' + - unittest_windows_cpu: + cu_version: cpu + name: unittest_windows_cpu_py3.8_slow_only pytest_additional_args: -m slow python_version: '3.8' - - unittest_linux_gpu: + - unittest_windows_cpu: + cu_version: cpu + name: unittest_windows_cpu_py3.9 + python_version: '3.9' + - unittest_windows_gpu: + cu_version: cu116 + name: unittest_windows_gpu_py3.8 + pytest_additional_args: -m "slow or not slow" + python_version: '3.8' + - unittest_windows_gpu: cu_version: cu116 filters: branches: only: - main - nightly - machine_type: gpu.nvidia.medium - name: unittest_linux_gpu_py3.9 + name: unittest_windows_gpu_py3.9 + python_version: '3.9' + - unittest_macos_cpu: + cu_version: cpu + machine_type: medium + name: unittest_macos_cpu_py3.8 + python_version: '3.8' + - unittest_macos_cpu: + cu_version: cpu + machine_type: large + name: unittest_macos_cpu_py3.8_slow_only + pytest_additional_args: -m slow + python_version: '3.8' + - unittest_macos_cpu: + cu_version: cpu + machine_type: medium + name: unittest_macos_cpu_py3.9 python_version: '3.9' cmake: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index ef1dc91bf12..b2e332e1705 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -214,7 +214,7 @@ binary_common: &binary_common machine_type: description: "The machine type (or resource class) that used to run the test" type: string - default: "xlarge" + default: "large" unicode_abi: description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" type: string @@ -865,7 +865,7 @@ jobs: command: .circleci/unittest/windows/scripts/install.sh - run: name: Run tests - command: .circleci/unittest/windows/scripts/run_test.sh + command: .circleci/unittest/windows/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post process command: .circleci/unittest/windows/scripts/post_process.sh @@ -912,7 +912,7 @@ jobs: command: .circleci/unittest/windows/scripts/install.sh - run: name: Run tests - command: .circleci/unittest/windows/scripts/run_test.sh + command: .circleci/unittest/windows/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post process command: .circleci/unittest/windows/scripts/post_process.sh @@ -923,7 +923,7 @@ jobs: <<: *binary_common macos: xcode: "14.0" - resource_class: large + resource_class: << parameters.machine_type >> steps: - checkout - designate_upload_channel @@ -955,7 +955,7 @@ jobs: command: .circleci/unittest/linux/scripts/install.sh - run: name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh + command: .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>" - run: name: Post process command: .circleci/unittest/linux/scripts/post_process.sh diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index da2278a6c7a..7471fb4845e 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -248,8 +248,8 @@ def indent(indentation, data_list): def unittest_workflows(indentation=6): jobs = [] - # for os_type in ["linux", "windows", "macos"]: - for os_type in ["linux"]: + # for os_type in ["linux"]: + for os_type in ["linux", "windows", "macos"]: for device_type in ["cpu", "gpu"]: if os_type == "macos" and device_type == "gpu": continue @@ -262,23 +262,30 @@ def unittest_workflows(indentation=6): if device_type == "gpu": if python_version != "3.8": job["filters"] = gen_filter_branch_tree("main", "nightly") + elif python_version == "3.8": + job["pytest_additional_args"] = '-m "slow or not slow"' job["cu_version"] = "cu116" if os_type == "linux": job["machine_type"] = "gpu.nvidia.medium" else: job["cu_version"] = "cpu" if os_type == "linux": - job["machine_type"] = "xlarge" + job["machine_type"] = "large" + elif os_type == "macos": + job["machine_type"] = "medium" jobs.append({f"unittest_{os_type}_{device_type}": job}) # Add slow_only job for python 3.8: - if python_version == "3.8": + if python_version == "3.8" and device_type == "cpu": slow_only_job = copy.deepcopy(job) slow_only_job["pytest_additional_args"] = "-m slow" slow_only_job["name"] = f"{slow_only_job['name']}_slow_only" - if os_type == "linux" and device_type == "cpu": - slow_only_job["machine_type"] = "2xlarge+" + if device_type == "cpu": + if os_type == "linux": + slow_only_job["machine_type"] = "2xlarge+" + elif os_type == "macos": + slow_only_job["machine_type"] = "large" jobs.append({f"unittest_{os_type}_{device_type}": slow_only_job}) return indent(indentation, jobs) diff --git a/.circleci/unittest/windows/scripts/run_test.sh b/.circleci/unittest/windows/scripts/run_test.sh index 802ad37f511..f202a6b3fe6 100644 --- a/.circleci/unittest/windows/scripts/run_test.sh +++ b/.circleci/unittest/windows/scripts/run_test.sh @@ -2,6 +2,8 @@ set -e +PYTEST_ADDITIONAL_ARGS=$1 + eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" conda activate ./env @@ -9,4 +11,4 @@ this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source "$this_dir/set_cuda_envs.sh" python -m torch.utils.collect_env -pytest --junitxml=test-results/junit.xml -v --durations 20 +pytest --junitxml=test-results/junit.xml -v --durations 20 ${PYTEST_ADDITIONAL_ARGS} From 571692019a030949bad33e9c145c8e168b2e9d22 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Thu, 1 Dec 2022 00:42:41 +0000 Subject: [PATCH 20/27] Remove unittest_models and use xlarge for linux because it will failed during build using large --- .circleci/config.yml | 19 ++----------------- .circleci/config.yml.in | 15 --------------- .circleci/regenerate.py | 2 +- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d9d16a3b5f..332aa101288 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -367,20 +367,6 @@ jobs: - run_tests_selective: file_or_dir: test/test_extended_*.py - unittest_models: - docker: - - image: cimg/python:3.7 - resource_class: 2xlarge+ - steps: - - checkout - - download_model_weights - - install_torchvision - - run: - name: Enable extended tests - command: echo 'export PYTORCH_TEST_WITH_EXTENDED=1' >> $BASH_ENV - - run_tests_selective: - file_or_dir: "test/test_models.py test/test_backbone_utils.py" - binary_linux_wheel: <<: *binary_common docker: @@ -1054,10 +1040,9 @@ workflows: - unittest_torchhub - unittest_onnx - unittest_extended - - unittest_models - unittest_linux_cpu: cu_version: cpu - machine_type: large + machine_type: xlarge name: unittest_linux_cpu_py3.8 python_version: '3.8' - unittest_linux_cpu: @@ -1068,7 +1053,7 @@ workflows: python_version: '3.8' - unittest_linux_cpu: cu_version: cpu - machine_type: large + machine_type: xlarge name: unittest_linux_cpu_py3.9 python_version: '3.9' - unittest_linux_gpu: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index b2e332e1705..97a0cf26515 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -367,20 +367,6 @@ jobs: - run_tests_selective: file_or_dir: test/test_extended_*.py - unittest_models: - docker: - - image: cimg/python:3.7 - resource_class: 2xlarge+ - steps: - - checkout - - download_model_weights - - install_torchvision - - run: - name: Enable extended tests - command: echo 'export PYTORCH_TEST_WITH_EXTENDED=1' >> $BASH_ENV - - run_tests_selective: - file_or_dir: "test/test_models.py test/test_backbone_utils.py" - binary_linux_wheel: <<: *binary_common docker: @@ -1054,7 +1040,6 @@ workflows: - unittest_torchhub - unittest_onnx - unittest_extended - - unittest_models {{ unittest_workflows() }} cmake: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 7471fb4845e..133b124db23 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -270,7 +270,7 @@ def unittest_workflows(indentation=6): else: job["cu_version"] = "cpu" if os_type == "linux": - job["machine_type"] = "large" + job["machine_type"] = "xlarge" elif os_type == "macos": job["machine_type"] = "medium" From 77b8a6d7203f80075ce304ea3e472ce5ffd940d0 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Thu, 1 Dec 2022 01:56:11 +0000 Subject: [PATCH 21/27] Try using large for macos normal test --- .circleci/config.yml | 4 ++-- .circleci/regenerate.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 332aa101288..e8b18f4e47b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1101,7 +1101,7 @@ workflows: python_version: '3.9' - unittest_macos_cpu: cu_version: cpu - machine_type: medium + machine_type: large name: unittest_macos_cpu_py3.8 python_version: '3.8' - unittest_macos_cpu: @@ -1112,7 +1112,7 @@ workflows: python_version: '3.8' - unittest_macos_cpu: cu_version: cpu - machine_type: medium + machine_type: large name: unittest_macos_cpu_py3.9 python_version: '3.9' diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 133b124db23..96c48ba972c 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -272,7 +272,7 @@ def unittest_workflows(indentation=6): if os_type == "linux": job["machine_type"] = "xlarge" elif os_type == "macos": - job["machine_type"] = "medium" + job["machine_type"] = "large" jobs.append({f"unittest_{os_type}_{device_type}": job}) From 0fae9f487d320b27b55b3a08689a60a810d58d24 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Thu, 1 Dec 2022 09:18:39 +0000 Subject: [PATCH 22/27] Run with original machine type for baseline --- .circleci/config.yml | 4 ++-- .circleci/regenerate.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8b18f4e47b..46157037d20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1042,7 +1042,7 @@ workflows: - unittest_extended - unittest_linux_cpu: cu_version: cpu - machine_type: xlarge + machine_type: 2xlarge+ name: unittest_linux_cpu_py3.8 python_version: '3.8' - unittest_linux_cpu: @@ -1053,7 +1053,7 @@ workflows: python_version: '3.8' - unittest_linux_cpu: cu_version: cpu - machine_type: xlarge + machine_type: 2xlarge+ name: unittest_linux_cpu_py3.9 python_version: '3.9' - unittest_linux_gpu: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 96c48ba972c..8e3859cf6a8 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -270,7 +270,7 @@ def unittest_workflows(indentation=6): else: job["cu_version"] = "cpu" if os_type == "linux": - job["machine_type"] = "xlarge" + job["machine_type"] = "2xlarge+" elif os_type == "macos": job["machine_type"] = "large" From 6e9d9000f2d9e90f0eded66f3b5e7b6b1bbcfff1 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Thu, 1 Dec 2022 10:09:07 +0000 Subject: [PATCH 23/27] Trying out with all python version with reduced machine type --- .circleci/config.yml | 74 ++++++++++++++++++++++++++++++++++++++--- .circleci/regenerate.py | 8 ++--- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 46157037d20..bd1ab8053b5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1042,7 +1042,12 @@ workflows: - unittest_extended - unittest_linux_cpu: cu_version: cpu - machine_type: 2xlarge+ + machine_type: xlarge + name: unittest_linux_cpu_py3.7 + python_version: '3.7' + - unittest_linux_cpu: + cu_version: cpu + machine_type: xlarge name: unittest_linux_cpu_py3.8 python_version: '3.8' - unittest_linux_cpu: @@ -1053,9 +1058,24 @@ workflows: python_version: '3.8' - unittest_linux_cpu: cu_version: cpu - machine_type: 2xlarge+ + machine_type: xlarge name: unittest_linux_cpu_py3.9 python_version: '3.9' + - unittest_linux_cpu: + cu_version: cpu + machine_type: xlarge + name: unittest_linux_cpu_py3.10 + python_version: '3.10' + - unittest_linux_gpu: + cu_version: cu116 + filters: + branches: + only: + - main + - nightly + machine_type: gpu.nvidia.medium + name: unittest_linux_gpu_py3.7 + python_version: '3.7' - unittest_linux_gpu: cu_version: cu116 machine_type: gpu.nvidia.medium @@ -1072,6 +1092,20 @@ workflows: machine_type: gpu.nvidia.medium name: unittest_linux_gpu_py3.9 python_version: '3.9' + - unittest_linux_gpu: + cu_version: cu116 + filters: + branches: + only: + - main + - nightly + machine_type: gpu.nvidia.medium + name: unittest_linux_gpu_py3.10 + python_version: '3.10' + - unittest_windows_cpu: + cu_version: cpu + name: unittest_windows_cpu_py3.7 + python_version: '3.7' - unittest_windows_cpu: cu_version: cpu name: unittest_windows_cpu_py3.8 @@ -1085,6 +1119,19 @@ workflows: cu_version: cpu name: unittest_windows_cpu_py3.9 python_version: '3.9' + - unittest_windows_cpu: + cu_version: cpu + name: unittest_windows_cpu_py3.10 + python_version: '3.10' + - unittest_windows_gpu: + cu_version: cu116 + filters: + branches: + only: + - main + - nightly + name: unittest_windows_gpu_py3.7 + python_version: '3.7' - unittest_windows_gpu: cu_version: cu116 name: unittest_windows_gpu_py3.8 @@ -1099,9 +1146,23 @@ workflows: - nightly name: unittest_windows_gpu_py3.9 python_version: '3.9' + - unittest_windows_gpu: + cu_version: cu116 + filters: + branches: + only: + - main + - nightly + name: unittest_windows_gpu_py3.10 + python_version: '3.10' - unittest_macos_cpu: cu_version: cpu - machine_type: large + machine_type: medium + name: unittest_macos_cpu_py3.7 + python_version: '3.7' + - unittest_macos_cpu: + cu_version: cpu + machine_type: medium name: unittest_macos_cpu_py3.8 python_version: '3.8' - unittest_macos_cpu: @@ -1112,9 +1173,14 @@ workflows: python_version: '3.8' - unittest_macos_cpu: cu_version: cpu - machine_type: large + machine_type: medium name: unittest_macos_cpu_py3.9 python_version: '3.9' + - unittest_macos_cpu: + cu_version: cpu + machine_type: medium + name: unittest_macos_cpu_py3.10 + python_version: '3.10' cmake: jobs: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 8e3859cf6a8..fde37acc7cb 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -22,8 +22,8 @@ from jinja2 import select_autoescape -# PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] -PYTHON_VERSIONS = ["3.8", "3.9"] +PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +# PYTHON_VERSIONS = ["3.8", "3.9"] RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" @@ -270,9 +270,9 @@ def unittest_workflows(indentation=6): else: job["cu_version"] = "cpu" if os_type == "linux": - job["machine_type"] = "2xlarge+" + job["machine_type"] = "xlarge" elif os_type == "macos": - job["machine_type"] = "large" + job["machine_type"] = "medium" jobs.append({f"unittest_{os_type}_{device_type}": job}) From 8efe0d116ccaa51df8e94b2f4013ad1711bc7d31 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Thu, 1 Dec 2022 11:13:21 +0000 Subject: [PATCH 24/27] Optimize some test --- test/test_models.py | 23 +++++++++++++++-------- test/test_transforms.py | 17 +++++++++++------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index f0caebc43b1..81e37ac400a 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -512,7 +512,8 @@ def test_resnet_dilation(dilate_layer_2, dilate_layer_3, dilate_layer_4): model = _make_sliced_model(model, stop_layer="layer4") model.eval() x = torch.rand(1, 3, 224, 224) - out = model(x) + with torch.no_grad(), freeze_rng_state(): + out = model(x) f = 2 ** sum((dilate_layer_2, dilate_layer_3, dilate_layer_4)) assert out.shape == (1, 2048, 7 * f, 7 * f) @@ -521,7 +522,8 @@ def test_mobilenet_v2_residual_setting(): model = models.mobilenet_v2(inverted_residual_setting=[[1, 16, 1, 1], [6, 24, 2, 2]]) model.eval() x = torch.rand(1, 3, 224, 224) - out = model(x) + with torch.no_grad(), freeze_rng_state(): + out = model(x) assert out.shape[-1] == 1000 @@ -560,7 +562,8 @@ def test_fasterrcnn_double(): input_shape = (3, 300, 300) x = torch.rand(input_shape, dtype=torch.float64) model_input = [x] - out = model(model_input) + with torch.no_grad(), freeze_rng_state(): + out = model(model_input) assert model_input[0] is x assert len(out) == 1 assert "boxes" in out[0] @@ -599,7 +602,8 @@ def checkOut(out): input_shape = (3, 300, 300) x = torch.rand(input_shape, device="cuda") model_input = [x] - out = model(model_input) + with torch.no_grad(), freeze_rng_state(): + out = model(model_input) assert model_input[0] is x checkOut(out) @@ -688,14 +692,15 @@ def test_classification_model(model_fn, dev): model = model_fn(**kwargs) model.eval().to(device=dev) x = _get_image(input_shape=input_shape, real_image=real_image, device=dev) - out = model(x) + with torch.no_grad(), freeze_rng_state(): + out = model(x) _assert_expected(out.cpu(), model_name, prec=1e-3) assert out.shape[-1] == num_classes _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(model_name, None), eager_out=out) _check_fx_compatible(model, x, eager_out=out) if dev == "cuda": - with torch.cuda.amp.autocast(): + with torch.cuda.amp.autocast(), torch.no_grad(), freeze_rng_state(): out = model(x) # See autocast_flaky_numerics comment at top of file. if model_name not in autocast_flaky_numerics: @@ -919,7 +924,8 @@ def test_video_model(model_fn, dev): model.eval().to(device=dev) # RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests x = torch.rand(input_shape).to(device=dev) - out = model(x) + with torch.no_grad(), freeze_rng_state(): + out = model(x) _assert_expected(out.cpu(), model_name, prec=1e-5) assert out.shape[-1] == num_classes _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(model_name, None), eager_out=out) @@ -960,7 +966,8 @@ def test_quantized_classification_model(model_fn): model = model_fn(**kwargs) model.eval() x = torch.rand(input_shape) - out = model(x) + with torch.no_grad(), freeze_rng_state(): + out = model(x) if model_name not in quantized_flaky_models: _assert_expected(out.cpu(), model_name + "_quantized", prec=2e-2) diff --git a/test/test_transforms.py b/test/test_transforms.py index e0f8d4a5927..5db1f6b2933 100644 --- a/test/test_transforms.py +++ b/test/test_transforms.py @@ -1607,12 +1607,17 @@ def test_trivialaugmentwide(fill, num_magnitude_bins, grayscale): transform.__repr__() -@pytest.mark.parametrize("fill", [None, 85, (128, 128, 128)]) -@pytest.mark.parametrize("severity", [1, 10]) -@pytest.mark.parametrize("mixture_width", [1, 2]) -@pytest.mark.parametrize("chain_depth", [-1, 2]) -@pytest.mark.parametrize("all_ops", [True, False]) -@pytest.mark.parametrize("grayscale", [True, False]) +@pytest.mark.parametrize( + "fill,severity,mixture_width,chain_depth,all_ops,grayscale", + [ + (None, 1, 1, -1, True, True), + (85, 10, 2, 2, False, False), + ((128, 128, 128), 1, 2, -1, False, True), + (None, 10, 1, 2, True, False), + (85, 1, 1, -1, False, False), + ((128, 128, 128), 10, 2, -1, True, False), + ], +) def test_augmix(fill, severity, mixture_width, chain_depth, all_ops, grayscale): random.seed(42) img = Image.open(GRACE_HOPPER) From 2cbf94234007b669aab4196a638b256dca3b0683 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Thu, 1 Dec 2022 11:14:01 +0000 Subject: [PATCH 25/27] Limit the python version for testing test optimation --- .circleci/config.yml | 66 ----------------------------------------- .circleci/regenerate.py | 4 +-- 2 files changed, 2 insertions(+), 68 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd1ab8053b5..332aa101288 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1040,11 +1040,6 @@ workflows: - unittest_torchhub - unittest_onnx - unittest_extended - - unittest_linux_cpu: - cu_version: cpu - machine_type: xlarge - name: unittest_linux_cpu_py3.7 - python_version: '3.7' - unittest_linux_cpu: cu_version: cpu machine_type: xlarge @@ -1061,21 +1056,6 @@ workflows: machine_type: xlarge name: unittest_linux_cpu_py3.9 python_version: '3.9' - - unittest_linux_cpu: - cu_version: cpu - machine_type: xlarge - name: unittest_linux_cpu_py3.10 - python_version: '3.10' - - unittest_linux_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - machine_type: gpu.nvidia.medium - name: unittest_linux_gpu_py3.7 - python_version: '3.7' - unittest_linux_gpu: cu_version: cu116 machine_type: gpu.nvidia.medium @@ -1092,20 +1072,6 @@ workflows: machine_type: gpu.nvidia.medium name: unittest_linux_gpu_py3.9 python_version: '3.9' - - unittest_linux_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - machine_type: gpu.nvidia.medium - name: unittest_linux_gpu_py3.10 - python_version: '3.10' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.7 - python_version: '3.7' - unittest_windows_cpu: cu_version: cpu name: unittest_windows_cpu_py3.8 @@ -1119,19 +1085,6 @@ workflows: cu_version: cpu name: unittest_windows_cpu_py3.9 python_version: '3.9' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.10 - python_version: '3.10' - - unittest_windows_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_windows_gpu_py3.7 - python_version: '3.7' - unittest_windows_gpu: cu_version: cu116 name: unittest_windows_gpu_py3.8 @@ -1146,20 +1099,6 @@ workflows: - nightly name: unittest_windows_gpu_py3.9 python_version: '3.9' - - unittest_windows_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_windows_gpu_py3.10 - python_version: '3.10' - - unittest_macos_cpu: - cu_version: cpu - machine_type: medium - name: unittest_macos_cpu_py3.7 - python_version: '3.7' - unittest_macos_cpu: cu_version: cpu machine_type: medium @@ -1176,11 +1115,6 @@ workflows: machine_type: medium name: unittest_macos_cpu_py3.9 python_version: '3.9' - - unittest_macos_cpu: - cu_version: cpu - machine_type: medium - name: unittest_macos_cpu_py3.10 - python_version: '3.10' cmake: jobs: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index fde37acc7cb..133b124db23 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -22,8 +22,8 @@ from jinja2 import select_autoescape -PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] -# PYTHON_VERSIONS = ["3.8", "3.9"] +# PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +PYTHON_VERSIONS = ["3.8", "3.9"] RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" From 9d2db9ade1c4890671d403ff3c0e0838c3223854 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Mon, 5 Dec 2022 17:23:23 +0000 Subject: [PATCH 26/27] Remove unnecessary test and test GHA linux on model with smaller machine type --- .circleci/config.yml | 58 ------- .circleci/regenerate.py | 6 +- .github/workflows/build-conda-linux.yml | 47 ----- .github/workflows/build-conda-m1.yml | 48 ------ .github/workflows/build-conda-macos.yml | 48 ------ .github/workflows/build-m1-binaries.yml | 160 ------------------ .github/workflows/build-wheels-linux.yml | 44 ----- .github/workflows/build-wheels-m1.yml | 47 ----- .github/workflows/build-wheels-macos.yml | 47 ----- ...linux-cpu.yml => test-linux-cpu-model.yml} | 6 +- .github/workflows/test-linux-gpu.yml | 61 ------- .github/workflows/test-m1.yml | 50 ------ 12 files changed, 6 insertions(+), 616 deletions(-) delete mode 100644 .github/workflows/build-conda-linux.yml delete mode 100644 .github/workflows/build-conda-m1.yml delete mode 100644 .github/workflows/build-conda-macos.yml delete mode 100644 .github/workflows/build-m1-binaries.yml delete mode 100644 .github/workflows/build-wheels-linux.yml delete mode 100644 .github/workflows/build-wheels-m1.yml delete mode 100644 .github/workflows/build-wheels-macos.yml rename .github/workflows/{test-linux-cpu.yml => test-linux-cpu-model.yml} (93%) delete mode 100644 .github/workflows/test-linux-gpu.yml delete mode 100644 .github/workflows/test-m1.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 332aa101288..f1b8d534120 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1051,70 +1051,12 @@ workflows: name: unittest_linux_cpu_py3.8_slow_only pytest_additional_args: -m slow python_version: '3.8' - - unittest_linux_cpu: - cu_version: cpu - machine_type: xlarge - name: unittest_linux_cpu_py3.9 - python_version: '3.9' - unittest_linux_gpu: cu_version: cu116 machine_type: gpu.nvidia.medium name: unittest_linux_gpu_py3.8 pytest_additional_args: -m "slow or not slow" python_version: '3.8' - - unittest_linux_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - machine_type: gpu.nvidia.medium - name: unittest_linux_gpu_py3.9 - python_version: '3.9' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.8 - python_version: '3.8' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.8_slow_only - pytest_additional_args: -m slow - python_version: '3.8' - - unittest_windows_cpu: - cu_version: cpu - name: unittest_windows_cpu_py3.9 - python_version: '3.9' - - unittest_windows_gpu: - cu_version: cu116 - name: unittest_windows_gpu_py3.8 - pytest_additional_args: -m "slow or not slow" - python_version: '3.8' - - unittest_windows_gpu: - cu_version: cu116 - filters: - branches: - only: - - main - - nightly - name: unittest_windows_gpu_py3.9 - python_version: '3.9' - - unittest_macos_cpu: - cu_version: cpu - machine_type: medium - name: unittest_macos_cpu_py3.8 - python_version: '3.8' - - unittest_macos_cpu: - cu_version: cpu - machine_type: large - name: unittest_macos_cpu_py3.8_slow_only - pytest_additional_args: -m slow - python_version: '3.8' - - unittest_macos_cpu: - cu_version: cpu - machine_type: medium - name: unittest_macos_cpu_py3.9 - python_version: '3.9' cmake: jobs: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 133b124db23..d8f10f8a673 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -23,7 +23,7 @@ # PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] -PYTHON_VERSIONS = ["3.8", "3.9"] +PYTHON_VERSIONS = ["3.8"] RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" @@ -248,8 +248,8 @@ def indent(indentation, data_list): def unittest_workflows(indentation=6): jobs = [] - # for os_type in ["linux"]: - for os_type in ["linux", "windows", "macos"]: + # for os_type in ["linux", "windows", "macos"]: + for os_type in ["linux"]: for device_type in ["cpu", "gpu"]: if os_type == "macos" and device_type == "gpu": continue diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml deleted file mode 100644 index 85d6f08aaf2..00000000000 --- a/.github/workflows/build-conda-linux.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Build Linux Conda - -on: - pull_request: - push: - branches: - - nightly - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml deleted file mode 100644 index dd4559f78a5..00000000000 --- a/.github/workflows/build-conda-m1.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Build M1 Conda - -on: - pull_request: - push: - branches: - - nightly - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-m1-12 - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml deleted file mode 100644 index 11871ac2174..00000000000 --- a/.github/workflows/build-conda-macos.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Build Macos Conda - -on: - pull_request: - push: - branches: - - nightly - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-12 - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml deleted file mode 100644 index 81b15172a7e..00000000000 --- a/.github/workflows/build-m1-binaries.yml +++ /dev/null @@ -1,160 +0,0 @@ -name: Build on M1 -on: - pull_request: - paths: - - .github/workflows/build-m1-binaries.yml - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: -env: - CHANNEL: "nightly" -jobs: - build_wheels: - name: "Build TorchVision M1 wheels" - runs-on: macos-m1-12 - strategy: - matrix: - py_vers: [ "3.8", "3.9", "3.10" ] - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set CHANNEL (only for tagged pushes) - if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} - run: | - # reference ends with an RC suffix - if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then - echo "CHANNEL=test" >> "$GITHUB_ENV" - fi - - name: Set Release CHANNEL (for release) - if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} - run: | - echo "CHANNEL=test" >> "$GITHUB_ENV" - - name: Setup miniconda - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - - name: Build TorchVision M1 wheel - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - # Needed for JPEG library detection as setup.py detects conda presence by running `shutil.which('conda')` - set -ex - . packaging/pkg_helpers.bash - # if we are uploading to test channell, our version consist only of the base: 0.x.x - no date string or suffix added - if [[ $CHANNEL == "test" ]]; then - setup_base_build_version - else - setup_build_version - fi - - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} - conda run -p ${ENV_NAME} python3 -mpip install delocate - conda run -p ${ENV_NAME} python3 setup.py bdist_wheel - export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" - conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/*.whl - conda env remove -p ${ENV_NAME} - - name: Test wheel - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-test-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - set -ex - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} - conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl - # Test torch is importable, by changing cwd and running import commands - conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" - conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchvision;print('Is torchvision useable?', all(x is not None for x in [torch.ops.image.decode_png, torch.ops.torchvision.roi_align]))" - conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print(torchvision.io.read_image('${PWD}/gallery/assets/dog1.jpg').shape)" - conda env remove -p ${ENV_NAME} - - name: Upload wheel to GitHub - uses: actions/upload-artifact@v3 - with: - name: torchvision-py${{ matrix.py_vers }}-macos11-m1 - path: dist/ - - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} - shell: arch -arch arm64 bash {0} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} - run: | - for pkg in dist/*; do - aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read - done - build_conda: - name: "Build TorchVision M1 conda packages" - runs-on: macos-m1-12 - strategy: - matrix: - py_vers: [ "3.8", "3.9", "3.10" ] - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set CHANNEL (only for tagged pushes) - if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} - run: | - # reference ends with an RC suffix - if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then - echo "CHANNEL=test" >> "$GITHUB_ENV" - fi - - name: Set CHANNEL Release (for release) - if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} - run: | - echo "CHANNEL=test" >> "$GITHUB_ENV" - - name: Setup miniconda - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - - name: Install conda-build and purge previous artifacts - shell: arch -arch arm64 bash {0} - run: | - conda install -yq conda-build - conda build purge-all - - name: Build TorchVision M1 conda package - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PYTHON_VERSION: ${{ matrix.py_vers }} - CU_VERSION: cpu - run: | - set -ex - . packaging/pkg_helpers.bash - - if [[ $CHANNEL == "test" ]]; then - setup_base_build_version - else - setup_build_version - fi - - setup_conda_pytorch_constraint - export SOURCE_ROOT_DIR=$(pwd) - conda build \ - -c defaults \ - $CONDA_CHANNEL_FLAGS \ - --no-anaconda-upload \ - --python "$PYTHON_VERSION" \ - --output-folder=dist/ \ - packaging/torchvision - - name: Upload package to GitHub - uses: actions/upload-artifact@v3 - with: - name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda - path: dist/ - - name: Upload package to conda - if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} - shell: arch -arch arm64 bash {0} - env: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - run: | - conda install -yq anaconda-client - set -x - export ANACONDA_PATH=$(conda info --base)/bin - $ANACONDA_PATH/anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload dist/osx-arm64/*.tar.bz2 -u "pytorch-${CHANNEL}" --label main --no-progress --force diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml deleted file mode 100644 index d2000cae995..00000000000 --- a/.github/workflows/build-wheels-linux.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Build Linux Wheels - -on: - pull_request: - push: - branches: - - nightly - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml deleted file mode 100644 index c73fe698f90..00000000000 --- a/.github/workflows/build-wheels-m1.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Build M1 Wheels - -on: - pull_request: - push: - branches: - - nightly - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-m1-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml deleted file mode 100644 index 1d61622c7b2..00000000000 --- a/.github/workflows/build-wheels-macos.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Build Macos Wheels - -on: - pull_request: - push: - branches: - - nightly - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu-model.yml similarity index 93% rename from .github/workflows/test-linux-cpu.yml rename to .github/workflows/test-linux-cpu-model.yml index 77ea2eb5ca8..25cf5681d4b 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu-model.yml @@ -16,11 +16,11 @@ jobs: tests: strategy: matrix: - python_version: ["3.7", "3.8", "3.9", "3.10"] + python_version: ["3.8"] fail-fast: false uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: - runner: linux.12xlarge + runner: linux.4xlarge repository: pytorch/vision script: | # Mark Build Directory Safe @@ -54,4 +54,4 @@ jobs: # Run Tests python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 5000 + python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 -m "slow" diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml deleted file mode 100644 index 1b548a3adcf..00000000000 --- a/.github/workflows/test-linux-gpu.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Unit-tests on Linux GPU - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -env: - CHANNEL: "nightly" - -jobs: - tests: - strategy: - matrix: - python_version: ["3.8"] - cuda_arch_version: ["11.6"] - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: linux.g5.4xlarge.nvidia.gpu - repository: pytorch/vision - gpu-arch-type: cuda - gpu-arch-version: ${{ matrix.cuda_arch_version }} - timeout: 120 - script: | - # Mark Build Directory Safe - git config --global --add safe.directory /__w/vision/vision - - # Set up Environment Variables - export PYTHON_VERSION="${{ matrix.python_version }}" - export VERSION="${{ matrix.cuda_arch_version }}" - export CUDATOOLKIT="pytorch-cuda=${VERSION}" - - # Set CHANNEL - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export CHANNEL=test - else - export CHANNEL=nightly - fi - - # Create Conda Env - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy - conda activate /work/ci_env - - # Install PyTorch, Torchvision, and testing libraries - set -ex - conda install \ - --yes \ - -c "pytorch-${CHANNEL}" \ - -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ - "${CUDATOOLKIT}" - python3 setup.py develop - python3 -m pip install pytest pytest-mock 'av<10' - - # Run Tests - python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 5000 diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml deleted file mode 100644 index c03fa9f76e4..00000000000 --- a/.github/workflows/test-m1.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Unit-tests on M1 -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: -env: - CHANNEL: "nightly" -jobs: - tests: - name: "Unit-tests on M1" - runs-on: macos-m1-12 - strategy: - matrix: - py_vers: [ "3.8"] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set Release CHANNEL (for release) - if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} - run: | - echo "CHANNEL=test" >> "$GITHUB_ENV" - - name: Install TorchVision - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - # Needed for JPEG library detection as setup.py detects conda presence by running `shutil.which('conda')` - export PATH=~/miniconda3/bin:$PATH - set -ex - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy - conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} - conda run -p ${ENV_NAME} python3 setup.py develop - conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock 'av<10' - - name: Run tests - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - set -ex - conda run -p ${ENV_NAME} --no-capture-output python3 -u -mpytest -v --tb=long --durations 20 - conda env remove -p ${ENV_NAME} From 616e20a6c30cc806384ad75d774b9b5189669bb3 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Mon, 5 Dec 2022 17:30:53 +0000 Subject: [PATCH 27/27] Bring back most test on GHA since otherwise it break --- .github/workflows/build-conda-linux.yml | 47 ++++++ .github/workflows/build-conda-m1.yml | 48 +++++++ .github/workflows/build-conda-macos.yml | 48 +++++++ .github/workflows/build-m1-binaries.yml | 160 +++++++++++++++++++++ .github/workflows/build-wheels-linux.yml | 44 ++++++ .github/workflows/build-wheels-m1.yml | 47 ++++++ .github/workflows/build-wheels-macos.yml | 47 ++++++ .github/workflows/test-linux-cpu-model.yml | 2 +- .github/workflows/test-linux-cpu.yml | 57 ++++++++ .github/workflows/test-m1.yml | 50 +++++++ 10 files changed, 549 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-conda-linux.yml create mode 100644 .github/workflows/build-conda-m1.yml create mode 100644 .github/workflows/build-conda-macos.yml create mode 100644 .github/workflows/build-m1-binaries.yml create mode 100644 .github/workflows/build-wheels-linux.yml create mode 100644 .github/workflows/build-wheels-m1.yml create mode 100644 .github/workflows/build-wheels-macos.yml create mode 100644 .github/workflows/test-linux-cpu.yml create mode 100644 .github/workflows/test-m1.yml diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml new file mode 100644 index 00000000000..85d6f08aaf2 --- /dev/null +++ b/.github/workflows/build-conda-linux.yml @@ -0,0 +1,47 @@ +name: Build Linux Conda + +on: + pull_request: + push: + branches: + - nightly + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + # Using "development" as trigger event so these binaries are not uploaded + # to official channels yet + trigger-event: development + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml new file mode 100644 index 00000000000..dd4559f78a5 --- /dev/null +++ b/.github/workflows/build-conda-m1.yml @@ -0,0 +1,48 @@ +name: Build M1 Conda + +on: + pull_request: + push: + branches: + - nightly + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + runner-type: macos-m1-12 + # Using "development" as trigger event so these binaries are not uploaded + # to official channels yet + trigger-event: development + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml new file mode 100644 index 00000000000..11871ac2174 --- /dev/null +++ b/.github/workflows/build-conda-macos.yml @@ -0,0 +1,48 @@ +name: Build Macos Conda + +on: + pull_request: + push: + branches: + - nightly + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: macos + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + runner-type: macos-12 + # Using "development" as trigger event so these binaries are not uploaded + # to official channels yet + trigger-event: development + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml new file mode 100644 index 00000000000..81b15172a7e --- /dev/null +++ b/.github/workflows/build-m1-binaries.yml @@ -0,0 +1,160 @@ +name: Build on M1 +on: + pull_request: + paths: + - .github/workflows/build-m1-binaries.yml + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: +env: + CHANNEL: "nightly" +jobs: + build_wheels: + name: "Build TorchVision M1 wheels" + runs-on: macos-m1-12 + strategy: + matrix: + py_vers: [ "3.8", "3.9", "3.10" ] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set CHANNEL (only for tagged pushes) + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + run: | + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi + - name: Set Release CHANNEL (for release) + if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" + - name: Setup miniconda + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + - name: Build TorchVision M1 wheel + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + # Needed for JPEG library detection as setup.py detects conda presence by running `shutil.which('conda')` + set -ex + . packaging/pkg_helpers.bash + # if we are uploading to test channell, our version consist only of the base: 0.x.x - no date string or suffix added + if [[ $CHANNEL == "test" ]]; then + setup_base_build_version + else + setup_build_version + fi + + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg wheel pkg-config + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} + conda run -p ${ENV_NAME} python3 -mpip install delocate + conda run -p ${ENV_NAME} python3 setup.py bdist_wheel + export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" + conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/*.whl + conda env remove -p ${ENV_NAME} + - name: Test wheel + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} + conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl + # Test torch is importable, by changing cwd and running import commands + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchvision;print('Is torchvision useable?', all(x is not None for x in [torch.ops.image.decode_png, torch.ops.torchvision.roi_align]))" + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print(torchvision.io.read_image('${PWD}/gallery/assets/dog1.jpg').shape)" + conda env remove -p ${ENV_NAME} + - name: Upload wheel to GitHub + uses: actions/upload-artifact@v3 + with: + name: torchvision-py${{ matrix.py_vers }}-macos11-m1 + path: dist/ + - name: Upload wheel to S3 + if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + shell: arch -arch arm64 bash {0} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} + run: | + for pkg in dist/*; do + aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read + done + build_conda: + name: "Build TorchVision M1 conda packages" + runs-on: macos-m1-12 + strategy: + matrix: + py_vers: [ "3.8", "3.9", "3.10" ] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set CHANNEL (only for tagged pushes) + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + run: | + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi + - name: Set CHANNEL Release (for release) + if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" + - name: Setup miniconda + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + - name: Install conda-build and purge previous artifacts + shell: arch -arch arm64 bash {0} + run: | + conda install -yq conda-build + conda build purge-all + - name: Build TorchVision M1 conda package + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PYTHON_VERSION: ${{ matrix.py_vers }} + CU_VERSION: cpu + run: | + set -ex + . packaging/pkg_helpers.bash + + if [[ $CHANNEL == "test" ]]; then + setup_base_build_version + else + setup_build_version + fi + + setup_conda_pytorch_constraint + export SOURCE_ROOT_DIR=$(pwd) + conda build \ + -c defaults \ + $CONDA_CHANNEL_FLAGS \ + --no-anaconda-upload \ + --python "$PYTHON_VERSION" \ + --output-folder=dist/ \ + packaging/torchvision + - name: Upload package to GitHub + uses: actions/upload-artifact@v3 + with: + name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda + path: dist/ + - name: Upload package to conda + if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + shell: arch -arch arm64 bash {0} + env: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} + run: | + conda install -yq anaconda-client + set -x + export ANACONDA_PATH=$(conda info --base)/bin + $ANACONDA_PATH/anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload dist/osx-arm64/*.tar.bz2 -u "pytorch-${CHANNEL}" --label main --no-progress --force diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml new file mode 100644 index 00000000000..d2000cae995 --- /dev/null +++ b/.github/workflows/build-wheels-linux.yml @@ -0,0 +1,44 @@ +name: Build Linux Wheels + +on: + pull_request: + push: + branches: + - nightly + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml new file mode 100644 index 00000000000..c73fe698f90 --- /dev/null +++ b/.github/workflows/build-wheels-m1.yml @@ -0,0 +1,47 @@ +name: Build M1 Wheels + +on: + pull_request: + push: + branches: + - nightly + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + runner-type: macos-m1-12 + smoke-test-script: ${{ matrix.smoke-test-script }} + # Using "development" as trigger event so these binaries are not uploaded + # to official channels yet + trigger-event: development + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml new file mode 100644 index 00000000000..1d61622c7b2 --- /dev/null +++ b/.github/workflows/build-wheels-macos.yml @@ -0,0 +1,47 @@ +name: Build Macos Wheels + +on: + pull_request: + push: + branches: + - nightly + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + runner-type: macos-12 + smoke-test-script: ${{ matrix.smoke-test-script }} + # Using "development" as trigger event so these binaries are not uploaded + # to official channels yet + trigger-event: development + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/test-linux-cpu-model.yml b/.github/workflows/test-linux-cpu-model.yml index 25cf5681d4b..f4f93898dfa 100644 --- a/.github/workflows/test-linux-cpu-model.yml +++ b/.github/workflows/test-linux-cpu-model.yml @@ -1,4 +1,4 @@ -name: Unit-tests on Linux CPU +name: Unit-tests on Linux CPU - Model Test on: pull_request: diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml new file mode 100644 index 00000000000..5b61dd35f4b --- /dev/null +++ b/.github/workflows/test-linux-cpu.yml @@ -0,0 +1,57 @@ +name: Unit-tests on Linux CPU + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +env: + CHANNEL: "nightly" + +jobs: + tests: + strategy: + matrix: + python_version: ["3.8"] + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + runner: linux.xlarge + repository: pytorch/vision + script: | + # Mark Build Directory Safe + git config --global --add safe.directory /__w/vision/vision + + # Set up Environment Variables + export PYTHON_VERSION="${{ matrix.python_version }}" + export VERSION="cpu" + export CUDATOOLKIT="cpuonly" + + # Set CHANNEL + if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + export CHANNEL=test + else + export CHANNEL=nightly + fi + + # Create Conda Env + conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy + conda activate /work/ci_env + + # Install PyTorch, Torchvision, and testing libraries + set -ex + conda install \ + --yes \ + -c "pytorch-${CHANNEL}" \ + -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ + "${CUDATOOLKIT}" + python3 setup.py develop + python3 -m pip install pytest pytest-mock 'av<10' + + # Run Tests + python3 -m torch.utils.collect_env + python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml new file mode 100644 index 00000000000..c03fa9f76e4 --- /dev/null +++ b/.github/workflows/test-m1.yml @@ -0,0 +1,50 @@ +name: Unit-tests on M1 +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: +env: + CHANNEL: "nightly" +jobs: + tests: + name: "Unit-tests on M1" + runs-on: macos-m1-12 + strategy: + matrix: + py_vers: [ "3.8"] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set Release CHANNEL (for release) + if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" + - name: Install TorchVision + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + # Needed for JPEG library detection as setup.py detects conda presence by running `shutil.which('conda')` + export PATH=~/miniconda3/bin:$PATH + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy + conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} + conda run -p ${ENV_NAME} python3 setup.py develop + conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock 'av<10' + - name: Run tests + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda run -p ${ENV_NAME} --no-capture-output python3 -u -mpytest -v --tb=long --durations 20 + conda env remove -p ${ENV_NAME}