Split up CI unit tests into two distinct shards

.travis.yml

@@ -939,32 +939,57 @@ matrix:
     - <<: *cargo_audit
 
     - <<: *py27_linux_test_config
-      name: "Unit tests (Py2.7 PEX)"
+      name: "Unit tests - V2 test runner (Py2.7 PEX)"
       stage: *test
       env:
         - *py27_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py27
+        - CACHE_NAME=unit_tests.v2.py27
+      script:
+        - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2 --python-version 2.7
+
+    - <<: *py36_linux_test_config
+      name: "Unit tests - V2 test runner (Py3.6 PEX)"
+      env:
+        - *py36_linux_test_config_env
+        - CACHE_NAME=unit_tests.v2.py36
+      script:
+        - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2
+
+    - <<: *py37_linux_test_config
+      name: "Unit tests - V2 test runner (Py3.7 PEX)"
+      env:
+        - *py37_linux_test_config_env
+        - CACHE_NAME=unit_tests.v2.py37
+      script:
+       - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2 --python-version 3.7
+
+    - <<: *py27_linux_test_config
+      name: "Unit tests - V1 test runner (Py2.7 PEX)"
+      stage: *test
+      env:
+        - *py27_linux_test_config_env
+        - CACHE_NAME=unit_tests.v1.py27
       script:
         - ./build-support/bin/ci.py --plugin-tests --python-version 2.7
-        - travis_wait 60 ./build-support/bin/ci.py --python-tests --python-version 2.7
+        - ./build-support/bin/ci.py --python-tests-v1 --python-version 2.7
 
     - <<: *py36_linux_test_config
-      name: "Unit tests (Py3.6 PEX)"
+      name: "Unit tests - V1 test runner (Py3.6 PEX)"
       env:
         - *py36_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py36
+        - CACHE_NAME=unit_tests.v1.py36
       script:
         - ./build-support/bin/ci.py --plugin-tests
-        - travis_wait 60 ./build-support/bin/ci.py --python-tests
+        - ./build-support/bin/ci.py --python-tests-v1
 
     - <<: *py37_linux_test_config
-      name: "Unit tests (Py3.7 PEX)"
+      name: "Unit tests - V1 test runner (Py3.7 PEX)"
       env:
         - *py37_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py37
+        - CACHE_NAME=unit_tests.v1.py37
       script:
        - ./build-support/bin/ci.py --plugin-tests --python-version 3.7
-       - travis_wait 60 ./build-support/bin/ci.py --python-tests --python-version 3.7
+       - ./build-support/bin/ci.py --python-tests-v1 --python-version 3.7
 
     - <<: *py27_linux_build_wheels_ucs2
     - <<: *py27_linux_build_wheels_ucs4

build-support/bin/ci.py

@@ -35,8 +35,10 @@ def main() -> None:
     run_clippy()
   if args.cargo_audit:
     run_cargo_audit()
-  if args.python_tests:
-    run_python_tests()
+  if args.python_tests_v1:
+    run_python_tests_v1()
+  if args.python_tests_v2:
+    run_python_tests_v2()
   if args.rust_tests:
     run_rust_tests()
   if args.jvm_tests:
@@ -92,7 +94,15 @@ def create_parser() -> argparse.ArgumentParser:
   parser.add_argument(
     "--cargo-audit", action="store_true", help="Run Cargo audit of Rust dependencies."
   )
-  parser.add_argument("--python-tests", action="store_true", help="Run Python unit tests.")
+  # TODO(#7772): Simplify below to always use V2 and drop the blacklist.
+  parser.add_argument(
+    "--python-tests-v1", action="store_true",
+    help="Run Python unit tests with V1 test runner over the blacklist and contrib tests."
+  )
+  parser.add_argument(
+    "--python-tests-v2", action="store_true",
+    help="Run Python unit tests with V2 test runner."
+  )
   parser.add_argument("--rust-tests", action="store_true", help="Run Rust tests.")
   parser.add_argument("--jvm-tests", action="store_true", help="Run JVM tests.")
   parser.add_argument(
@@ -283,35 +293,10 @@ def run_cargo_audit() -> None:
       die("Cargo audit failure")
 
 
-def run_python_tests() -> None:
-  # TODO(#7772): Simplify below to always use V2 and drop the blacklist.
+def run_python_tests_v1() -> None:
   known_v2_failures_file = "build-support/unit_test_v2_blacklist.txt"
-  with open(known_v2_failures_file, "r") as f:
-    blacklisted_targets = {line.strip() for line in f.readlines()}
-
-  with travis_section("CoreTests", "Running Python unit tests"):
+  with travis_section("PythonTestsV1", "Running Python unit tests with V1 test runner"):
     check_pants_pex_exists()
-    try:
-      all_targets = subprocess.run([
-        "./pants.pex",
-        "--tag=-integration",
-        "--filter-type=python_tests",
-        "filter",
-        "src/python::",
-        "tests/python::",
-      ], stdout=subprocess.PIPE, encoding="utf-8", check=True).stdout.strip().split("\n")
-      v2_targets = set(all_targets) - blacklisted_targets
-      subprocess.run([
-        "./pants.pex",
-        "--no-v1",
-        "--v2",
-        "test.pytest"
-      ] + sorted(v2_targets) + PYTEST_PASSTHRU_ARGS, check=True)
-    except subprocess.CalledProcessError:
-      die("Core Python test failure (V2 test runner)")
-    else:
-      green("V2 unit tests passed.")
-
     try:
       subprocess.run([
         "./pants.pex",
@@ -338,6 +323,34 @@ def run_python_tests() -> None:
       green("Contrib unit tests passed.")
 
 
+def run_python_tests_v2() -> None:
+  known_v2_failures_file = "build-support/unit_test_v2_blacklist.txt"
+  with open(known_v2_failures_file, "r") as f:
+    blacklisted_targets = {line.strip() for line in f.readlines()}
+  with travis_section("PythonTestsV1", "Running Python unit tests with V2 test runner"):
+    check_pants_pex_exists()
+    try:
+      all_targets = subprocess.run([
+        "./pants.pex",
+        "--tag=-integration",
+        "--filter-type=python_tests",
+        "filter",
+        "src/python::",
+        "tests/python::",
+      ], stdout=subprocess.PIPE, encoding="utf-8", check=True).stdout.strip().split("\n")
+      v2_targets = set(all_targets) - blacklisted_targets
+      subprocess.run([
+        "./pants.pex",
+        "--no-v1",
+        "--v2",
+        "test.pytest"
+      ] + sorted(v2_targets) + PYTEST_PASSTHRU_ARGS, check=True)
+    except subprocess.CalledProcessError:
+      die("Python unit tests failure (V2 test runner)")
+    else:
+      green("V2 unit tests passed.")
+
+
 def run_rust_tests() -> None:
   command = [
     "build-support/bin/native/cargo",

build-support/travis/travis.yml.mustache

@@ -882,32 +882,57 @@ matrix:
     - <<: *cargo_audit
 
     - <<: *py27_linux_test_config
-      name: "Unit tests (Py2.7 PEX)"
+      name: "Unit tests - V2 test runner (Py2.7 PEX)"
       stage: *test
       env:
         - *py27_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py27
+        - CACHE_NAME=unit_tests.v2.py27
+      script:
+        - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2 --python-version 2.7
+
+    - <<: *py36_linux_test_config
+      name: "Unit tests - V2 test runner (Py3.6 PEX)"
+      env:
+        - *py36_linux_test_config_env
+        - CACHE_NAME=unit_tests.v2.py36
+      script:
+        - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2
+
+    - <<: *py37_linux_test_config
+      name: "Unit tests - V2 test runner (Py3.7 PEX)"
+      env:
+        - *py37_linux_test_config_env
+        - CACHE_NAME=unit_tests.v2.py37
+      script:
+       - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2 --python-version 3.7
+
+    - <<: *py27_linux_test_config
+      name: "Unit tests - V1 test runner (Py2.7 PEX)"
+      stage: *test
+      env:
+        - *py27_linux_test_config_env
+        - CACHE_NAME=unit_tests.v1.py27
       script:
         - ./build-support/bin/ci.py --plugin-tests --python-version 2.7
-        - travis_wait 60 ./build-support/bin/ci.py --python-tests --python-version 2.7
+        - ./build-support/bin/ci.py --python-tests-v1 --python-version 2.7
 
     - <<: *py36_linux_test_config
-      name: "Unit tests (Py3.6 PEX)"
+      name: "Unit tests - V1 test runner (Py3.6 PEX)"
       env:
         - *py36_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py36
+        - CACHE_NAME=unit_tests.v1.py36
       script:
         - ./build-support/bin/ci.py --plugin-tests
-        - travis_wait 60 ./build-support/bin/ci.py --python-tests
+        - ./build-support/bin/ci.py --python-tests-v1
 
     - <<: *py37_linux_test_config
-      name: "Unit tests (Py3.7 PEX)"
+      name: "Unit tests - V1 test runner (Py3.7 PEX)"
       env:
         - *py37_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py37
+        - CACHE_NAME=unit_tests.v1.py37
       script:
        - ./build-support/bin/ci.py --plugin-tests --python-version 3.7
-       - travis_wait 60 ./build-support/bin/ci.py --python-tests --python-version 3.7
+       - ./build-support/bin/ci.py --python-tests-v1 --python-version 3.7
 
     - <<: *py27_linux_build_wheels_ucs2
     - <<: *py27_linux_build_wheels_ucs4