diff --git a/build-support/docker/remote_execution/Dockerfile b/build-support/docker/remote_execution/Dockerfile index 7da6b763dce..11469a9ee58 100644 --- a/build-support/docker/remote_execution/Dockerfile +++ b/build-support/docker/remote_execution/Dockerfile @@ -44,3 +44,5 @@ RUN ${PYENV_BIN} install ${PYTHON_37_VERSION} ENV PATH "${PYENV_ROOT}/versions/${PYTHON_27_VERSION}/bin:${PATH}" ENV PATH "${PYENV_ROOT}/versions/${PYTHON_36_VERSION}/bin:${PATH}" ENV PATH "${PYENV_ROOT}/versions/${PYTHON_37_VERSION}/bin:${PATH}" + +RUN touch /PANTS_GCP_REMOTE && chmod 755 /PANTS_GCP_REMOTE diff --git a/build-support/ivy/ivysettings.xml b/build-support/ivy/ivysettings.xml index 05daf1e17fa..9a1fecf7016 100644 --- a/build-support/ivy/ivysettings.xml +++ b/build-support/ivy/ivysettings.xml @@ -26,6 +26,13 @@ Licensed under the Apache License, Version 2.0 (see LICENSE). This can lead to jars downloading without their transitive deps which leads to confusing failures later when classpaths are constructed and used. We setup the maven central resolver to require successful pom downloads here. --> + + + + diff --git a/build-support/ivy/remote.ivysettings.xml b/build-support/ivy/remote.ivysettings.xml new file mode 100644 index 00000000000..2876c04b7f0 --- /dev/null +++ b/build-support/ivy/remote.ivysettings.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pants.ini b/pants.ini index 8d37324a080..f7268f17f31 100644 --- a/pants.ini +++ b/pants.ini @@ -451,6 +451,15 @@ enable_libc_search: True [sourcefile-validation] config: @build-support/regexes/config.yaml +[coursier] +repos: [ + # Google RBE maven central mirror that avoids throttling by maven central. + 'https://maven-central.storage-download.googleapis.com/repos/central/data', + 'https://repo1.maven.org/maven2', + # Custom repo for Kythe jars, as Google doesn't currently publish them anywhere. + 'https://raw.githubusercontent.com/toolchainlabs/binhost/master/' + ] + [cache.resolve.coursier] # Only use local artifact for coursier because the cache content contains abs path # and is not portable. That said, if remote cache is enabled, this would not break diff --git a/pants.remote.ini b/pants.remote.ini index 6559dc9952b..5f3b722bf31 100644 --- a/pants.remote.ini +++ b/pants.remote.ini @@ -20,7 +20,7 @@ remote_instance_name: projects/pants-remoting-beta/instances/default_instance remote_execution_extra_platform_properties: [ # This allows network requests, e.g. to resolve dependencies with Pex. "dockerNetwork=standard", - "container-image=docker://gcr.io/pants-remoting-beta/rbe-remote-execution@sha256:5d818cd71c9180d977e16ca7a20e90ced14211621b69fe1d6c3fc4c42c537a14", + "container-image=docker://gcr.io/pants-remoting-beta/rbe-remote-execution@sha256:dbc9005efdcf30f9e22b450c2ee0aef50077ff98dee84a9922e63f905aff926e", ] # This should correspond to the number of workers running in Google RBE. See @@ -32,6 +32,11 @@ process_execution_speculation_strategy: none # p95 of RBE appears to be ~ 2 seconds, but we need to factor in local queue time which can be much longer, but no metrics yet. process_execution_speculation_delay: 15 +# DRY up maven central mirror setup in variables used below. +maven_central_mirror_root_url: https://maven-central.storage-download.googleapis.com/repos/central/data +maven_central_mirror_ivy_bootstrap_jar_url: %(maven_central_mirror_root_url)s/org/apache/ivy/ivy/2.4.0/ivy-2.4.0.jar +maven_central_mirror_ivy_settings: %(pants_supportdir)s/ivy/remote.ivysettings.xml + [python-setup] # TODO(#7735): This config is not ideal, that we must specify the PATH for both local and remote # platforms. This should be replaced by a proper mechanism to differentiate between the two. @@ -47,3 +52,31 @@ interpreter_search_paths: [ [python-native-code] ld_flags: [] cpp_flags: [] + +# Note that the logic below is repeated in jvm_task_test_base.py. See there for details. +# If you set remote-specific ivy/coursier options here, set them there as well. + +[ivy] +bootstrap_jar_url: %(maven_central_mirror_ivy_bootstrap_jar_url)s +bootstrap_ivy_settings: %(maven_central_mirror_ivy_settings)s +ivy_settings: %(maven_central_mirror_ivy_settings)s + +[ivy.outdated] +bootstrap_jar_url: %(maven_central_mirror_ivy_bootstrap_jar_url)s +bootstrap_ivy_settings: %(maven_central_mirror_ivy_settings)s +ivy_settings: %(maven_central_mirror_ivy_settings)s + +[ivy.outdated.ivy] +bootstrap_jar_url: %(maven_central_mirror_ivy_bootstrap_jar_url)s +bootstrap_ivy_settings: %(maven_central_mirror_ivy_settings)s +ivy_settings: %(maven_central_mirror_ivy_settings)s + +[coursier] +repos: [ + # Google RBE maven central mirror that avoids DDOSing maven central. + '%(maven_central_mirror_root_url)s', + + # Custom repo for Kythe jars, as Google doesn't currently publish them anywhere. + 'https://raw.githubusercontent.com/toolchainlabs/binhost/master/' + ] + diff --git a/src/python/pants/testutil/jvm/BUILD b/src/python/pants/testutil/jvm/BUILD index e14001b2602..28fa13e0c96 100644 --- a/src/python/pants/testutil/jvm/BUILD +++ b/src/python/pants/testutil/jvm/BUILD @@ -42,6 +42,7 @@ python_library( name='jvm_task_test_base', sources=['jvm_task_test_base.py'], dependencies=[ + 'build-support/ivy', 'src/python/pants/backend/jvm/tasks:classpath_products', 'src/python/pants/testutil/subsystem', 'src/python/pants/testutil:task_test_base', diff --git a/src/python/pants/testutil/jvm/jvm_task_test_base.py b/src/python/pants/testutil/jvm/jvm_task_test_base.py index fa4f3b55a64..a4f504a457c 100644 --- a/src/python/pants/testutil/jvm/jvm_task_test_base.py +++ b/src/python/pants/testutil/jvm/jvm_task_test_base.py @@ -22,6 +22,21 @@ def setUp(self): super().setUp() init_subsystem(JvmResolveSubsystem) self.set_options_for_scope('resolver', resolver='ivy') + if os.path.isfile('/PANTS_GCP_REMOTE'): + # Use the GCP maven mirrors, so we don't get throttled for DOSing maven central. + # Note that this recapitulates the logic in pants.remote.ini. Unfortunately we + # can't access the real options inside tests, and plumbing these through everywhere + # they're needed is prohibitive. So we repeat the logic here. + maven_central_mirror_root_url = 'https://maven-central.storage-download.googleapis.com/repos/central/data' + maven_central_mirror_ivy_bootstrap_jar_url = f'{maven_central_mirror_root_url}/org/apache/ivy/ivy/2.4.0/ivy-2.4.0.jar' + maven_central_mirror_ivy_settings = 'build-support/ivy/remote.ivysettings.xml' + + self.set_options_for_scope('coursier', repos=[maven_central_mirror_root_url]) + for scope in ['ivy', 'ivy.outdated', 'ivy.outdated.ivy']: + self.set_options_for_scope(scope, + bootstrap_jar_url=maven_central_mirror_ivy_bootstrap_jar_url, + bootstrap_ivy_settings=maven_central_mirror_ivy_settings, + ivy_settings=maven_central_mirror_ivy_settings) def populate_runtime_classpath(self, context, classpath=None): """