[nomerge] Experiment on tests

.circleci/config.yml.in

@@ -207,6 +207,14 @@ binary_common: &binary_common
       description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
       type: string
       default: "cpu"
+    pytest_additional_args:
+      description: "Additional arguments to be given to pytest call"
+      type: string
+      default: ""
+    machine_type:
+      description: "The machine type (or resource class) that used to run the test"
+      type: string
+      default: "large"
     unicode_abi:
       description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
       type: string
@@ -726,7 +734,7 @@ jobs:
     <<: *binary_common
     docker:
       - image: "pytorch/manylinux-cpu"
-    resource_class: 2xlarge+
+    resource_class: << parameters.machine_type >>
     steps:
       - checkout
       - designate_upload_channel
@@ -754,7 +762,7 @@ jobs:
           command: .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
-          command: .circleci/unittest/linux/scripts/run_test.sh
+          command: .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>"
       - run:
           name: Post process
           command: .circleci/unittest/linux/scripts/post_process.sh
@@ -765,7 +773,7 @@ jobs:
     <<: *binary_common
     machine:
       image: ubuntu-2004-cuda-11.4:202110-01
-    resource_class: gpu.nvidia.medium
+    resource_class: << parameters.machine_type >>
     environment:
       image_name: "pytorch/manylinux-cuda116"
       CU_VERSION: << parameters.cu_version >>
@@ -805,7 +813,7 @@ jobs:
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
-          command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
+          command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>"
       - run:
           name: Post Process
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
@@ -843,7 +851,7 @@ jobs:
           command: .circleci/unittest/windows/scripts/install.sh
       - run:
           name: Run tests
-          command: .circleci/unittest/windows/scripts/run_test.sh
+          command: .circleci/unittest/windows/scripts/run_test.sh "<< parameters.pytest_additional_args >>"
       - run:
           name: Post process
           command: .circleci/unittest/windows/scripts/post_process.sh
@@ -890,7 +898,7 @@ jobs:
           command: .circleci/unittest/windows/scripts/install.sh
       - run:
           name: Run tests
-          command: .circleci/unittest/windows/scripts/run_test.sh
+          command: .circleci/unittest/windows/scripts/run_test.sh "<< parameters.pytest_additional_args >>"
       - run:
           name: Post process
           command: .circleci/unittest/windows/scripts/post_process.sh
@@ -901,7 +909,7 @@ jobs:
     <<: *binary_common
     macos:
       xcode: "14.0"
-    resource_class: large
+    resource_class: << parameters.machine_type >>
     steps:
       - checkout
       - designate_upload_channel
@@ -933,7 +941,7 @@ jobs:
           command: .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
-          command: .circleci/unittest/linux/scripts/run_test.sh
+          command: .circleci/unittest/linux/scripts/run_test.sh "<< parameters.pytest_additional_args >>"
       - run:
           name: Post process
           command: .circleci/unittest/linux/scripts/post_process.sh
@@ -1019,74 +1027,6 @@ jobs:
             packaging/windows/internal/cuda_install.bat
             packaging/build_cmake.sh
 
-  build_docs:
-    <<: *binary_common
-    docker:
-      - image: cimg/python:3.7
-    resource_class: 2xlarge+
-    steps:
-      - attach_workspace:
-          at: ~/workspace
-      - checkout
-      - download_model_weights
-      - run:
-          name: Setup
-          command: .circleci/unittest/linux/scripts/setup_env.sh
-      - designate_upload_channel
-      - run:
-          name: Install torchvision
-          command: .circleci/unittest/linux/scripts/install.sh
-      - run:
-          name: Build docs
-          command: |
-            set -ex
-            # turn v1.12.0rc3 into 1.12.0
-            tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9.]*\).*/\1/')
-            VERSION=${tag:-main}
-            eval "$(./conda/bin/conda shell.bash hook)"
-            conda activate ./env
-            pushd docs
-            pip install --progress-bar=off -r requirements.txt
-            make html
-            popd
-      - persist_to_workspace:
-          root: ./
-          paths:
-            - "*"
-      - store_artifacts:
-          path: ./docs/build/html
-          destination: docs
-
-  upload_docs:
-    <<: *binary_common
-    docker:
-      - image: "pytorch/manylinux-cuda100"
-    resource_class: 2xlarge+
-    steps:
-      - attach_workspace:
-          at: ~/workspace
-      - run:
-          name: Generate netrc
-          command: |
-            # set credentials for https pushing
-            # requires the org-member context
-            cat > ~/.netrc \<<DONE
-              machine github.com
-              login pytorchbot
-              password ${GITHUB_PYTORCHBOT_TOKEN}
-            DONE
-      - run:
-          name: Upload docs
-          command: |
-            # Don't use "checkout" step since it uses ssh, which cannot git push
-            # https://circleci.com/docs/2.0/configuration-reference/#checkout
-            set -ex
-            # Change v1.12.1rc1 into 1.12 (only major.minor)
-            tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9]*\.[0-9]*\).*/\1/')
-            target=${tag:-main}
-            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target
-
-
 workflows:
   lint:
     jobs:
@@ -1095,12 +1035,6 @@ workflows:
       - lint_c
       - type_check_python
 
-  build:
-    jobs:
-      {{ build_workflows(windows_latest_only=True) }}
-      {{ ios_workflows() }}
-      {{ android_workflows() }}
-
   unittest:
     jobs:
       - unittest_torchhub
@@ -1112,11 +1046,6 @@ workflows:
     jobs:
       {{ cmake_workflows() }}
 
-  nightly:
-    jobs:
-      {{ ios_workflows(nightly=True) }}
-      {{ android_workflows(nightly=True) }}
-      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
   docker_build:
     triggers:
       - schedule:

.circleci/regenerate.py

@@ -14,14 +14,16 @@
 https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
 """
 
+import copy
 import os.path
 
 import jinja2
 import yaml
 from jinja2 import select_autoescape
 
 
-PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
+# PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
+PYTHON_VERSIONS = ["3.8"]
 
 RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
 
@@ -246,7 +248,8 @@ def indent(indentation, data_list):
 
 def unittest_workflows(indentation=6):
     jobs = []
-    for os_type in ["linux", "windows", "macos"]:
+    # for os_type in ["linux", "windows", "macos"]:
+    for os_type in ["linux"]:
         for device_type in ["cpu", "gpu"]:
             if os_type == "macos" and device_type == "gpu":
                 continue
@@ -259,12 +262,32 @@ def unittest_workflows(indentation=6):
                 if device_type == "gpu":
                     if python_version != "3.8":
                         job["filters"] = gen_filter_branch_tree("main", "nightly")
+                    elif python_version == "3.8":
+                        job["pytest_additional_args"] = '-m "slow or not slow"'
                     job["cu_version"] = "cu116"
+                    if os_type == "linux":
+                        job["machine_type"] = "gpu.nvidia.medium"
                 else:
                     job["cu_version"] = "cpu"
+                    if os_type == "linux":
+                        job["machine_type"] = "xlarge"
+                    elif os_type == "macos":
+                        job["machine_type"] = "medium"
 
                 jobs.append({f"unittest_{os_type}_{device_type}": job})
 
+                # Add slow_only job for python 3.8:
+                if python_version == "3.8" and device_type == "cpu":
+                    slow_only_job = copy.deepcopy(job)
+                    slow_only_job["pytest_additional_args"] = "-m slow"
+                    slow_only_job["name"] = f"{slow_only_job['name']}_slow_only"
+                    if device_type == "cpu":
+                        if os_type == "linux":
+                            slow_only_job["machine_type"] = "2xlarge+"
+                        elif os_type == "macos":
+                            slow_only_job["machine_type"] = "large"
+                    jobs.append({f"unittest_{os_type}_{device_type}": slow_only_job})
+
     return indent(indentation, jobs)
 
 

.circleci/unittest/linux/scripts/run_test.sh

@@ -2,8 +2,10 @@
 
 set -e
 
+PYTEST_ADDITIONAL_ARGS=$1
+
 eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
 python -m torch.utils.collect_env
-pytest --junitxml=test-results/junit.xml -v --durations 20
+pytest --junitxml=test-results/junit.xml -v --durations 20 ${PYTEST_ADDITIONAL_ARGS}

.circleci/unittest/windows/scripts/run_test.sh

@@ -2,11 +2,13 @@
 
 set -e
 
+PYTEST_ADDITIONAL_ARGS=$1
+
 eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')"
 conda activate ./env
 
 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$this_dir/set_cuda_envs.sh"
 
 python -m torch.utils.collect_env
-pytest --junitxml=test-results/junit.xml -v --durations 20
+pytest --junitxml=test-results/junit.xml -v --durations 20 ${PYTEST_ADDITIONAL_ARGS}

.github/workflows/test-linux-gpu.yml → .github/workflows/test-linux-cpu-model.yml

@@ -1,4 +1,4 @@
-name: Unit-tests on Linux GPU
+name: Unit-tests on Linux CPU - Model Test
 
 on:
   pull_request:
@@ -17,23 +17,19 @@ jobs:
     strategy:
       matrix:
         python_version: ["3.8"]
-        cuda_arch_version: ["11.6"]
       fail-fast: false
     uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
     with:
-      runner: linux.g5.4xlarge.nvidia.gpu
+      runner: linux.4xlarge
       repository: pytorch/vision
-      gpu-arch-type: cuda
-      gpu-arch-version: ${{ matrix.cuda_arch_version }}
-      timeout: 120
       script: |
         # Mark Build Directory Safe
         git config --global --add safe.directory /__w/vision/vision
 
         # Set up Environment Variables
         export PYTHON_VERSION="${{ matrix.python_version }}"
-        export VERSION="${{ matrix.cuda_arch_version }}"
-        export CUDATOOLKIT="pytorch-cuda=${VERSION}"
+        export VERSION="cpu"
+        export CUDATOOLKIT="cpuonly"
 
         # Set CHANNEL
         if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
@@ -58,4 +54,4 @@ jobs:
 
         # Run Tests
         python3 -m torch.utils.collect_env
-        python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20
+        python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 -m "slow"

.github/workflows/test-linux-cpu.yml

@@ -16,11 +16,11 @@ jobs:
   tests:
     strategy:
       matrix:
-        python_version: ["3.7", "3.8", "3.9", "3.10"]
+        python_version: ["3.8"]
       fail-fast: false
     uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
     with:
-      runner: linux.12xlarge
+      runner: linux.xlarge
       repository: pytorch/vision
       script: |
         # Mark Build Directory Safe

pytest.ini

@@ -8,6 +8,9 @@ addopts =
     -Wd
     --ignore=test/test_datasets_download.py
     --ignore-glob=test/test_prototype_*.py
+    -m "not slow"
 testpaths =
     test
 xfail_strict = True
+markers = 
+    slow

test/test_backbone_utils.py

@@ -11,6 +11,9 @@
 from torchvision.models.feature_extraction import create_feature_extractor, get_graph_node_names
 
 
+pytestmark = pytest.mark.slow
+
+
 @pytest.mark.parametrize("backbone_name", ("resnet18", "resnet50"))
 def test_resnet_fpn_backbone(backbone_name):
     x = torch.rand(1, 3, 300, 300, dtype=torch.float32, device="cpu")