Port more integration tests to V1 chroot and some to V2 (#8365)

This brings ITs to:

* 10% V1 no chroot
* 8.5% V1 chroot
* 81.5% V2 remote

Notably, this moves over several tests that were previously marked V1 no chroot but really could have been V1 chroot. While this does not actually remote any more tests, it helps to keep track of what work remains; it is much easier to port a V1 chroot test to V2 than a no-chroot target. This also prevents any regressions.

build-support/ci_lists/integration_chroot_blacklist.txt

@@ -1,24 +1,13 @@
 tests/python/pants_test/backend/jvm/subsystems:jar_dependency_management_integration
 tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:cache_compile_integration
-tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:java_compile_integration
-tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc:rsc_compile_integration
-tests/python/pants_test/backend/jvm/tasks/jvm_compile/zinc:zinc_compile_integration
-tests/python/pants_test/backend/jvm/tasks/jvm_compile/zinc:zinc_compile_integration_with_zjars
 tests/python/pants_test/backend/jvm/tasks:checkstyle_integration
 tests/python/pants_test/backend/jvm/tasks:ivy_resolve_integration
 tests/python/pants_test/backend/jvm/tasks:jar_publish_integration
-tests/python/pants_test/backend/jvm/tasks:junit_run_integration
-tests/python/pants_test/backend/jvm/tasks:junit_tests_concurrency_integration
-tests/python/pants_test/backend/jvm/tasks:jvm_run_integration
 tests/python/pants_test/backend/python/tasks/native:integration
 tests/python/pants_test/backend/python/tasks:build_local_python_distributions_integration
 tests/python/pants_test/backend/python/tasks:pytest_run_integration
 tests/python/pants_test/backend/python/tasks:python_binary_integration
 tests/python/pants_test/backend/python/tasks:setup_py_integration
 tests/python/pants_test/backend/python:integration
-tests/python/pants_test/invalidation:strict_deps_invalidation_integration
-tests/python/pants_test/logging:test_native_engine_logging
 tests/python/pants_test/pantsd:pantsd_integration
 tests/python/pants_test/projects:testprojects_integration
-tests/python/pants_test/targets:jvm_app_integration
-tests/python/pants_test/tasks:bootstrap_jvm_tools_integration

build-support/ci_lists/integration_v2_blacklist.txt

@@ -1,2 +1,12 @@
+tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:java_compile_integration
+tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:zinc_compile_integration
+tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc:rsc_compile_integration
+tests/python/pants_test/backend/jvm/tasks/jvm_compile/zinc:zinc_compile_integration
+tests/python/pants_test/backend/jvm/tasks/jvm_compile/zinc:zinc_compile_integration_with_zjars
+tests/python/pants_test/backend/jvm/tasks:junit_run_integration
+tests/python/pants_test/backend/jvm/tasks:junit_tests_concurrency_integration
+tests/python/pants_test/backend/jvm/tasks:junit_tests_integration
+tests/python/pants_test/backend/jvm/tasks:jvm_run_integration
 tests/python/pants_test/base:exception_sink_integration
 tests/python/pants_test/reporting:reporting_integration
+tests/python/pants_test/tasks:bootstrap_jvm_tools_integration

testprojects/src/python/BUILD

@@ -9,6 +9,11 @@ files(
   ],
 )
 
+files(
+  name = 'bad_requirements_directory',
+  sources = rglobs('bad_requirements/*'),
+)
+
 files(
   name = 'build_file_imports_function_directory',
   sources = rglobs('build_file_imports_function/*'),
@@ -29,6 +34,11 @@ files(
   sources = rglobs('interpreter_selection/*'),
 )
 
+files(
+  name = 'nested_runs_directory',
+  sources = rglobs('nested_runs/*'),
+)
+
 files(
   name = 'no_build_file_directory',
   sources = rglobs('no_build_file/*'),

tests/python/pants_test/backend/jvm/tasks/BUILD

@@ -319,7 +319,7 @@ python_tests(
     'tests/python/pants_test:int-test',
     'testprojects/src/java/org/pantsbuild/testproject:unicode_directory',
   ],
-  timeout = 120,
+  timeout = 240,
   tags = {'integration'},
 )
 

tests/python/pants_test/backend/python/tasks/native/BUILD

@@ -44,6 +44,7 @@ python_tests(
     'tests/python/pants_test:int-test',
     'tests/python/pants_test/backend/python/tasks/util',
     'tests/python/pants_test/engine:scheduler_test_base',
+    'testprojects/src/python:python_distribution_directory',
   ],
   tags={'platform_specific_behavior', 'integration'},
   timeout=2400,

tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py

@@ -5,9 +5,8 @@
 import os
 import re
 import subprocess
-import sys
 from functools import wraps
-from unittest import skip, skipIf
+from unittest import skip
 from zipfile import ZipFile
 
 from pants.backend.native.config.environment import Platform
@@ -32,10 +31,6 @@ def wrapper(*args, **kwargs):
   return wrapper
 
 
-_IS_OSX = Platform.current == Platform.darwin
-_PYTHON_MAJOR_MINOR = sys.version_info[:2]
-
-
 class CTypesIntegrationTest(PantsRunIntegrationTest):
 
   @classmethod
@@ -59,8 +54,6 @@ def use_pantsd_env_var(cls):
     'testprojects/src/python/python_distribution/ctypes_with_extra_compiler_flags:bin'
   )
 
-  @skipIf(_IS_OSX and _PYTHON_MAJOR_MINOR == (2, 7),
-          reason='Broken as described in https://github.com/pantsbuild/pants/issues/7763.')
   @_toolchain_variants
   def test_ctypes_binary_creation(self, toolchain_variant):
     """Create a python_binary() with all native toolchain variants, and test the result."""

tests/python/pants_test/logging/test_native_engine_logging.py

@@ -19,12 +19,12 @@ def use_pantsd_env_var(cls):
   def test_native_logging(self):
     expected_msg = "\[DEBUG\] engine::scheduler: Launching \d+ root"
     pants_run = self.run_pants([
-      "-linfo", "list", "src/scala::"
+      "-linfo", "list", "3rdparty::"
     ])
     self.assertNotRegex(pants_run.stderr_data, expected_msg)
 
     pants_run = self.run_pants([
-      "-ldebug", "list", "src/scala::"
+      "-ldebug", "list", "3rdparty::"
     ])
     self.assertRegex(pants_run.stderr_data, expected_msg)
 

tests/python/pants_test/pantsd/BUILD

@@ -81,6 +81,11 @@ python_tests(
     'examples/src/java/org/pantsbuild/example:hello_directory',
     'examples/src/python/example:hello_directory',
     'examples/src/scala/org/pantsbuild/example:hello_directory',
+    'testprojects/src/java/org/pantsbuild/testproject:bundle_directory',
+    'testprojects/src/python:bad_requirements_directory',
+    'testprojects/src/python:coordinated_runs_directory',
+    'testprojects/src/python:nested_runs_directory',
+    'testprojects/src/python:print_env_directory',
   ],
   tags = {'integration'},
   timeout = 900

tests/python/pants_test/pantsd/test_pantsd_integration.py

@@ -400,7 +400,9 @@ def test_pantsd_memory_usage(self):
     number_of_runs = 10
     max_memory_increase_fraction = 0.40 # TODO https://github.com/pantsbuild/pants/issues/7647
     with self.pantsd_successful_run_context() as (pantsd_run, checker, workdir, config):
-      cmd = ['filter', 'testprojects::']
+      # NB: This doesn't actually run against all testprojects, only those that are in the chroot,
+      # i.e. explicitly declared in this test file's BUILD.
+      cmd = ['list', 'testprojects::']
       self.assert_success(pantsd_run(cmd))
       initial_memory_usage = checker.current_memory_usage()
       for _ in range(number_of_runs):

tests/python/pants_test/targets/BUILD

@@ -43,6 +43,7 @@ python_tests(
   sources = ['test_jvm_app_integration.py'],
   dependencies = [
     'tests/python/pants_test:int-test',
+    'testprojects/src/java/org/pantsbuild/testproject:bundle_directory',
   ],
   tags = {'integration'},
   timeout = 180,