From a9e1869f05fc39b628e7edeec6cc2bb4d1576d56 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 17 Nov 2018 23:16:39 -0800 Subject: [PATCH 01/13] using explicit arg works --- .../backend/jvm/tasks/bootstrap_jvm_tools.py | 22 +++++++++++++++++-- .../jvm/tasks/coursier/coursier_subsystem.py | 2 +- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py index 77009a4acb9..57a37b0ac1b 100644 --- a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py +++ b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py @@ -16,8 +16,11 @@ from future.utils import PY3 from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolMixin +from pants.backend.jvm.subsystems.resolve_subsystem import JvmResolveSubsystem from pants.backend.jvm.subsystems.shader import Shader from pants.backend.jvm.targets.jar_library import JarLibrary +from pants.backend.jvm.tasks.classpath_products import ClasspathProducts +from pants.backend.jvm.tasks.coursier_resolve import CoursierMixin from pants.backend.jvm.tasks.ivy_task_mixin import IvyResolveFingerprintStrategy, IvyTaskMixin from pants.backend.jvm.tasks.jar_task import JarTask from pants.base.exceptions import TaskError @@ -70,7 +73,7 @@ def __eq__(self, other): return type(self) == type(other) and self._tuple() == other._tuple() -class BootstrapJvmTools(IvyTaskMixin, JarTask): +class BootstrapJvmTools(IvyTaskMixin, CoursierMixin, JarTask): class ToolUnderspecified(Exception): pass @@ -237,7 +240,22 @@ def _check_underspecified_tools(self, jvm_tool, targets): def _bootstrap_classpath(self, jvm_tool, targets): self._check_underspecified_tools(jvm_tool, targets) workunit_name = 'bootstrap-{}'.format(jvm_tool.key) - return self.ivy_classpath(targets, silent=True, workunit_name=workunit_name) + if JvmResolveSubsystem.global_instance().get_options().resolver == 'ivy': + ivy_classpath = self.ivy_classpath(targets, silent=True, workunit_name=workunit_name) + return ivy_classpath + else: + classpath_holder = ClasspathProducts(self.get_options().pants_workdir) + CoursierMixin.resolve(self, targets, classpath_holder, + sources=False, + javadoc=False) + coursier_classpath = [cp_entry for _, cp_entry in classpath_holder.get_for_targets(targets)] + return coursier_classpath + + # ivynames = list(sorted(os.path.basename(path) for path in ivy_classpath)) + # coursiernames = list(sorted(os.path.basename(path) for path in coursier_classpath)) + # if ivynames != coursiernames: + # x = 5 + # return coursier_classpath @memoized_property def shader(self): diff --git a/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py b/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py index ace4dd37a30..efa8eee5313 100644 --- a/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py +++ b/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py @@ -49,7 +49,7 @@ def register_options(cls, register): ], help='Additional options to pass to coursier fetch. See `coursier fetch --help`') register('--artifact-types', type=list, fingerprint=True, - default=['jar', 'bundle', 'test-jar', 'maven-plugin', 'src', 'doc', 'aar'], + default=['jar', 'bundle', 'test-jar', 'maven-plugin', 'src', 'doc'], help='Specify the type of artifacts to fetch. See `packaging` at https://maven.apache.org/pom.html#Maven_Coordinates, ' 'except `src` and `doc` being coursier specific terms for sources and javadoc.') register('--bootstrap-jar-url', fingerprint=True, From 38d562994adf86bd96754ac9ab0be01f592d460b Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 17 Nov 2018 23:35:49 -0800 Subject: [PATCH 02/13] add execution override --- .../pants/backend/jvm/tasks/bootstrap_jvm_tools.py | 11 +++++------ src/python/pants/backend/jvm/tasks/nailgun_task.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py index 57a37b0ac1b..e5ca303e68f 100644 --- a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py +++ b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py @@ -237,6 +237,11 @@ def _check_underspecified_tools(self, jvm_tool, targets): default option was provided to use for bootstrap. """.format(jvm_tool.key))) + # Force non-nailgun execution because nailgun will try to bootstrap itself via coursier, + # and then coursier will try it run via nailgun, which creates a chicken-egg problem. + def force_non_nailgun_execution(self): + return True + def _bootstrap_classpath(self, jvm_tool, targets): self._check_underspecified_tools(jvm_tool, targets) workunit_name = 'bootstrap-{}'.format(jvm_tool.key) @@ -251,12 +256,6 @@ def _bootstrap_classpath(self, jvm_tool, targets): coursier_classpath = [cp_entry for _, cp_entry in classpath_holder.get_for_targets(targets)] return coursier_classpath - # ivynames = list(sorted(os.path.basename(path) for path in ivy_classpath)) - # coursiernames = list(sorted(os.path.basename(path) for path in coursier_classpath)) - # if ivynames != coursiernames: - # x = 5 - # return coursier_classpath - @memoized_property def shader(self): return Shader.Factory.create(self.context) diff --git a/src/python/pants/backend/jvm/tasks/nailgun_task.py b/src/python/pants/backend/jvm/tasks/nailgun_task.py index ce2229a0c73..bd0e2a0c62d 100644 --- a/src/python/pants/backend/jvm/tasks/nailgun_task.py +++ b/src/python/pants/backend/jvm/tasks/nailgun_task.py @@ -58,6 +58,15 @@ def __init__(self, *args, **kwargs): self._executor_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, *id_tuple) + def force_non_nailgun_execution(self): + """ + Provide an option for subclass tasks to overwrite the execution strategy. + E.g. pants.backend.jvm.tasks.bootstrap_jvm_tools.BootstrapJvmTools + + :return: True to force non-nailgun. + """ + return False + @memoized_property def execution_strategy(self): return self.get_options().execution_strategy @@ -68,7 +77,9 @@ def create_java_executor(self, dist=None): Call only in execute() or later. TODO: Enforce this. """ dist = dist or self.dist - if self.execution_strategy == self.NAILGUN: + if self.force_non_nailgun_execution(): + return SubprocessExecutor(dist) + elif self.execution_strategy == self.NAILGUN: classpath = os.pathsep.join(self.tool_classpath('nailgun-server')) return NailgunExecutor(self._identity, self._executor_workdir, From c2b0565ad8c332b266f102728330dd4dc9a652db Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 17 Nov 2018 23:39:57 -0800 Subject: [PATCH 03/13] fmt --- src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py index e5ca303e68f..f73846ce93d 100644 --- a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py +++ b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py @@ -250,9 +250,7 @@ def _bootstrap_classpath(self, jvm_tool, targets): return ivy_classpath else: classpath_holder = ClasspathProducts(self.get_options().pants_workdir) - CoursierMixin.resolve(self, targets, classpath_holder, - sources=False, - javadoc=False) + CoursierMixin.resolve(self, targets, classpath_holder, sources=False, javadoc=False) coursier_classpath = [cp_entry for _, cp_entry in classpath_holder.get_for_targets(targets)] return coursier_classpath From 19261448d3c55d85f232be4cc59d91c8bfdb5b8a Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 17 Nov 2018 23:41:25 -0800 Subject: [PATCH 04/13] typo --- src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py index f73846ce93d..087365eabff 100644 --- a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py +++ b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py @@ -238,7 +238,7 @@ def _check_underspecified_tools(self, jvm_tool, targets): """.format(jvm_tool.key))) # Force non-nailgun execution because nailgun will try to bootstrap itself via coursier, - # and then coursier will try it run via nailgun, which creates a chicken-egg problem. + # and then coursier will try to run via nailgun, which creates a chicken-egg problem. def force_non_nailgun_execution(self): return True From 6b2d7293bf57b228041a246024e0cbec5f111b6d Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sun, 18 Nov 2018 02:00:50 -0800 Subject: [PATCH 05/13] add integration tests --- .../backend/jvm/tasks/test_bootstrap_jvm_tools_integration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/python/pants_test/backend/jvm/tasks/test_bootstrap_jvm_tools_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_bootstrap_jvm_tools_integration.py index 76231b672ca..8c1be6c6147 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_bootstrap_jvm_tools_integration.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_bootstrap_jvm_tools_integration.py @@ -4,11 +4,12 @@ from __future__ import absolute_import, division, print_function, unicode_literals -from pants_test.pants_run_integration_test import PantsRunIntegrationTest +from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_resolver class BootstrapJvmToolsIntegrationTest(PantsRunIntegrationTest): + @ensure_resolver def test_bootstrap_jarjar_succeeds_normally(self): # NB(gmalmquist): The choice of jarjar here is arbitrary; any jvm-tool that is integral to pants # would suffice (eg, nailgun or jar-tool). @@ -16,6 +17,7 @@ def test_bootstrap_jarjar_succeeds_normally(self): pants_run = self.run_pants(['bootstrap', '3rdparty:junit']) self.assert_success(pants_run) + @ensure_resolver def test_bootstrap_jarjar_failure(self): self.assert_success(self.run_pants(['clean-all'])) pants_run = self.run_pants(['bootstrap', '--shader-jarjar="fake-target"', '3rdparty:junit']) From 840947a21b83379ea9a714a3539c9da97353d137 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sun, 18 Nov 2018 17:47:46 -0800 Subject: [PATCH 06/13] init at base --- tests/python/pants_test/jvm/jvm_task_test_base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/python/pants_test/jvm/jvm_task_test_base.py b/tests/python/pants_test/jvm/jvm_task_test_base.py index 4186a33f295..5481ecc31ad 100644 --- a/tests/python/pants_test/jvm/jvm_task_test_base.py +++ b/tests/python/pants_test/jvm/jvm_task_test_base.py @@ -6,8 +6,10 @@ import os +from pants.backend.jvm.subsystems.resolve_subsystem import JvmResolveSubsystem from pants.backend.jvm.tasks.classpath_products import ClasspathProducts from pants.util.dirutil import safe_file_dump, safe_mkdir, safe_mkdtemp +from pants_test.subsystem.subsystem_util import init_subsystem from pants_test.task_test_base import TaskTestBase @@ -16,6 +18,14 @@ class JvmTaskTestBase(TaskTestBase): :API: public """ + def setUp(self): + """ + :API: public + """ + super(JvmTaskTestBase, self).setUp() + init_subsystem(JvmResolveSubsystem) + self.set_options_for_scope('resolver', resolver='ivy') + def populate_runtime_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'runtime_classpath' products data mapping From bf981f1fdfa5167486ede9358a8504f9a8643f0f Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sun, 18 Nov 2018 18:56:02 -0800 Subject: [PATCH 07/13] one more ensure --- .../pants_test/tasks/test_bootstrap_jvm_tools_integration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/pants_test/tasks/test_bootstrap_jvm_tools_integration.py b/tests/python/pants_test/tasks/test_bootstrap_jvm_tools_integration.py index 991b3300344..570242b4927 100644 --- a/tests/python/pants_test/tasks/test_bootstrap_jvm_tools_integration.py +++ b/tests/python/pants_test/tasks/test_bootstrap_jvm_tools_integration.py @@ -5,11 +5,12 @@ from __future__ import absolute_import, division, print_function, unicode_literals from pants.util.contextutil import temporary_dir -from pants_test.pants_run_integration_test import PantsRunIntegrationTest +from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_resolver class BootstrapJvmToolsIntegrationTest(PantsRunIntegrationTest): + @ensure_resolver def test_zinc_tool_reuse_between_scala_and_java(self): with temporary_dir() as artifact_cache: bootstrap_args = [ From ecfe884393c650ac2e7953f82e80bd6115da1000 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 1 Dec 2018 16:38:33 -0800 Subject: [PATCH 08/13] plumb executor down coursier resolve --- .../backend/jvm/tasks/bootstrap_jvm_tools.py | 2 +- .../backend/jvm/tasks/coursier_resolve.py | 50 +++++++++++-------- .../pants/backend/jvm/tasks/nailgun_task.py | 5 +- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py index 087365eabff..2755289e293 100644 --- a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py +++ b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py @@ -250,7 +250,7 @@ def _bootstrap_classpath(self, jvm_tool, targets): return ivy_classpath else: classpath_holder = ClasspathProducts(self.get_options().pants_workdir) - CoursierMixin.resolve(self, targets, classpath_holder, sources=False, javadoc=False) + CoursierMixin.resolve(self, targets, classpath_holder, sources=False, javadoc=False, executor=None) coursier_classpath = [cp_entry for _, cp_entry in classpath_holder.get_for_targets(targets)] return coursier_classpath diff --git a/src/python/pants/backend/jvm/tasks/coursier_resolve.py b/src/python/pants/backend/jvm/tasks/coursier_resolve.py index dc4198f12dd..43a83ccaceb 100644 --- a/src/python/pants/backend/jvm/tasks/coursier_resolve.py +++ b/src/python/pants/backend/jvm/tasks/coursier_resolve.py @@ -28,6 +28,9 @@ from pants.base.fingerprint_strategy import FingerprintStrategy from pants.base.workunit import WorkUnitLabel from pants.invalidation.cache_manager import VersionedTargetSet +from pants.java import util +from pants.java.distribution.distribution import DistributionLocator +from pants.java.executor import Executor, SubprocessExecutor from pants.java.jar.jar_dependency_utils import M2Coordinate, ResolvedJar from pants.util.contextutil import temporary_file from pants.util.dirutil import safe_mkdir @@ -38,7 +41,7 @@ class CoursierResultNotFound(Exception): pass -class CoursierMixin(NailgunTask, JvmResolverBase): +class CoursierMixin(JvmResolverBase): """ Experimental 3rdparty resolver using coursier. @@ -98,7 +101,7 @@ def _compute_jars_to_resolve_and_pin(raw_jars, artifact_set, manager): return jar_list, untouched_pinned_artifact - def resolve(self, targets, compile_classpath, sources, javadoc): + def resolve(self, targets, compile_classpath, sources, javadoc, executor): """ This is the core function for coursier resolve. @@ -119,12 +122,18 @@ def resolve(self, targets, compile_classpath, sources, javadoc): :param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed. :param sources: if True, fetch sources for 3rdparty :param javadoc: if True, fetch javadoc for 3rdparty + :param executor: An instance of `pants.java.executor.Executor`. If None, a subprocess executor will be assigned. :return: n/a """ manager = JarDependencyManagement.global_instance() jar_targets = manager.targets_by_artifact_set(targets) + executor = executor or SubprocessExecutor(DistributionLocator.cached()) + if not isinstance(executor, Executor): + raise ValueError('The executor argument must be an Executor instance, given {} of type {}'.format( + executor, type(executor))) + for artifact_set, target_subset in jar_targets.items(): # TODO(wisechengyi): this is the only place we are using IvyUtil method, which isn't specific to ivy really. raw_jar_deps, global_excludes = IvyUtils.calculate_classpath(target_subset) @@ -165,7 +174,7 @@ def resolve(self, targets, compile_classpath, sources, javadoc): manager) results = self._get_result_from_coursier(jars_to_resolve, global_excludes, pinned_coords, - coursier_cache_dir, sources, javadoc) + coursier_cache_dir, sources, javadoc, executor) for conf, result_list in results.items(): for result in result_list: @@ -198,7 +207,7 @@ def _prepare_workdir(self): return pants_jar_base_dir def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coords, - coursier_cache_path, sources, javadoc): + coursier_cache_path, sources, javadoc, executor): """ Calling coursier and return the result per invocation. @@ -209,6 +218,7 @@ def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coo :param global_excludes: List of `M2Coordinate`s to exclude globally :param pinned_coords: List of `M2Coordinate`s that need to be pinned. :param coursier_cache_path: path to where coursier cache is stored. + :param executor: An instance of `pants.java.executor.Executor` :return: The aggregation of results by conf from coursier. Each coursier call could return the following: @@ -264,18 +274,18 @@ def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coo results_by_conf = self._get_default_conf_results(common_args, coursier_jar, global_excludes, jars_to_resolve, coursier_work_temp_dir, - pinned_coords) + pinned_coords, executor) if sources or javadoc: non_default_conf_results = self._get_non_default_conf_results(common_args, coursier_jar, global_excludes, jars_to_resolve, coursier_work_temp_dir, - pinned_coords, sources, javadoc) + pinned_coords, sources, javadoc, executor) results_by_conf.update(non_default_conf_results) return results_by_conf def _get_default_conf_results(self, common_args, coursier_jar, global_excludes, jars_to_resolve, coursier_work_temp_dir, - pinned_coords): + pinned_coords, executor): # Variable to store coursier result each run. results = defaultdict(list) @@ -289,13 +299,13 @@ def _get_default_conf_results(self, common_args, coursier_jar, global_excludes, coursier_work_temp_dir, output_fn) - results['default'].append(self._call_coursier(cmd_args, coursier_jar, output_fn)) + results['default'].append(self._call_coursier(cmd_args, coursier_jar, output_fn, executor)) return results def _get_non_default_conf_results(self, common_args, coursier_jar, global_excludes, jars_to_resolve, coursier_work_temp_dir, pinned_coords, - sources, javadoc): + sources, javadoc, executor): # To prevent improper api usage during development. User should not see this anyway. if not sources and not javadoc: raise TaskError("sources or javadoc has to be True.") @@ -331,21 +341,19 @@ def _get_non_default_conf_results(self, common_args, coursier_jar, global_exclud cmd_args.extend(special_args) # sources and/or javadoc share the same conf - results['src_doc'] = [self._call_coursier(cmd_args, coursier_jar, output_fn)] + results['src_doc'] = [self._call_coursier(cmd_args, coursier_jar, output_fn, executor)] return results - def _call_coursier(self, cmd_args, coursier_jar, output_fn): - - labels = [WorkUnitLabel.COMPILER] if self.get_options().report else [WorkUnitLabel.TOOL] + def _call_coursier(self, cmd_args, coursier_jar, output_fn, executor): - return_code = self.runjava( + runner = executor.runner( classpath=[coursier_jar], main='coursier.cli.Coursier', - args=cmd_args, jvm_options=self.get_options().jvm_options, - workunit_name='coursier', - workunit_labels=labels, - ) + args=cmd_args) + + labels = [WorkUnitLabel.COMPILER] if self.get_options().report else [WorkUnitLabel.TOOL] + return_code = util.execute_runner(runner, self.context.new_workunit, 'coursier', labels) if return_code: raise TaskError('The coursier process exited non-zero: {0}'.format(return_code)) @@ -653,7 +661,7 @@ def _get_path_to_jar(cls, coursier_cache_path, pants_jar_path_base, jar_path): return os.path.join(pants_jar_path_base, 'relative', os.path.relpath(jar_path, coursier_cache_path)) -class CoursierResolve(CoursierMixin): +class CoursierResolve(CoursierMixin, NailgunTask): @classmethod def subsystem_dependencies(cls): @@ -692,8 +700,8 @@ def execute(self): classpath_products = self.context.products.get_data('compile_classpath', init_func=ClasspathProducts.init_func( self.get_options().pants_workdir)) - - self.resolve(self.context.targets(), classpath_products, sources=False, javadoc=False) + executor = self.create_java_executor() + self.resolve(self.context.targets(), classpath_products, sources=False, javadoc=False, executor=executor) def check_artifact_cache_for(self, invalidation_check): # Coursier resolution is an output dependent on the entire target set, and is not divisible diff --git a/src/python/pants/backend/jvm/tasks/nailgun_task.py b/src/python/pants/backend/jvm/tasks/nailgun_task.py index bd0e2a0c62d..b1f35d7c507 100644 --- a/src/python/pants/backend/jvm/tasks/nailgun_task.py +++ b/src/python/pants/backend/jvm/tasks/nailgun_task.py @@ -77,9 +77,7 @@ def create_java_executor(self, dist=None): Call only in execute() or later. TODO: Enforce this. """ dist = dist or self.dist - if self.force_non_nailgun_execution(): - return SubprocessExecutor(dist) - elif self.execution_strategy == self.NAILGUN: + if self.execution_strategy == self.NAILGUN: classpath = os.pathsep.join(self.tool_classpath('nailgun-server')) return NailgunExecutor(self._identity, self._executor_workdir, @@ -100,6 +98,7 @@ def runjava(self, classpath, main, jvm_options=None, args=None, workunit_name=No :API: public """ + print('hi') executor = self.create_java_executor(dist=dist) # Creating synthetic jar to work around system arg length limit is not necessary From f71162e5d5ddc1f94bba51413a87ebc21671c2cb Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 1 Dec 2018 16:39:43 -0800 Subject: [PATCH 09/13] remove force_non_nailgun_execution --- .../pants/backend/jvm/tasks/bootstrap_jvm_tools.py | 5 ----- src/python/pants/backend/jvm/tasks/nailgun_task.py | 9 --------- 2 files changed, 14 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py index 2755289e293..759d9244f40 100644 --- a/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py +++ b/src/python/pants/backend/jvm/tasks/bootstrap_jvm_tools.py @@ -237,11 +237,6 @@ def _check_underspecified_tools(self, jvm_tool, targets): default option was provided to use for bootstrap. """.format(jvm_tool.key))) - # Force non-nailgun execution because nailgun will try to bootstrap itself via coursier, - # and then coursier will try to run via nailgun, which creates a chicken-egg problem. - def force_non_nailgun_execution(self): - return True - def _bootstrap_classpath(self, jvm_tool, targets): self._check_underspecified_tools(jvm_tool, targets) workunit_name = 'bootstrap-{}'.format(jvm_tool.key) diff --git a/src/python/pants/backend/jvm/tasks/nailgun_task.py b/src/python/pants/backend/jvm/tasks/nailgun_task.py index b1f35d7c507..aee04d97ab9 100644 --- a/src/python/pants/backend/jvm/tasks/nailgun_task.py +++ b/src/python/pants/backend/jvm/tasks/nailgun_task.py @@ -58,15 +58,6 @@ def __init__(self, *args, **kwargs): self._executor_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, *id_tuple) - def force_non_nailgun_execution(self): - """ - Provide an option for subclass tasks to overwrite the execution strategy. - E.g. pants.backend.jvm.tasks.bootstrap_jvm_tools.BootstrapJvmTools - - :return: True to force non-nailgun. - """ - return False - @memoized_property def execution_strategy(self): return self.get_options().execution_strategy From 58d4a9493d7a9f8bef418522dddfa3bc3355c566 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 1 Dec 2018 16:41:27 -0800 Subject: [PATCH 10/13] remvoe print --- src/python/pants/backend/jvm/tasks/nailgun_task.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python/pants/backend/jvm/tasks/nailgun_task.py b/src/python/pants/backend/jvm/tasks/nailgun_task.py index aee04d97ab9..ce2229a0c73 100644 --- a/src/python/pants/backend/jvm/tasks/nailgun_task.py +++ b/src/python/pants/backend/jvm/tasks/nailgun_task.py @@ -89,7 +89,6 @@ def runjava(self, classpath, main, jvm_options=None, args=None, workunit_name=No :API: public """ - print('hi') executor = self.create_java_executor(dist=dist) # Creating synthetic jar to work around system arg length limit is not necessary From 4b5651cb36b2d8e420f509302433f53c34128e91 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 1 Dec 2018 17:36:24 -0800 Subject: [PATCH 11/13] fix tests --- .../pants_test/backend/jvm/tasks/test_coursier_resolve.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/python/pants_test/backend/jvm/tasks/test_coursier_resolve.py b/tests/python/pants_test/backend/jvm/tasks/test_coursier_resolve.py index f77ad232989..23b82958bdb 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_coursier_resolve.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_coursier_resolve.py @@ -23,6 +23,7 @@ from pants.backend.jvm.tasks.coursier_resolve import (CoursierResolve, CoursierResolveFingerprintStrategy) from pants.base.exceptions import TaskError +from pants.java import util from pants.java.jar.exclude import Exclude from pants.java.jar.jar_dependency import JarDependency from pants.task.task import Task @@ -100,7 +101,7 @@ def test_resolve_specific_with_sources_javadocs(self): compile_classpath = context.products.get_data('compile_classpath', init_func=ClasspathProducts.init_func(workdir) ) - task.resolve([jar_lib, scala_lib], compile_classpath, sources=True, javadoc=True) + task.resolve([jar_lib, scala_lib], compile_classpath, sources=True, javadoc=True, executor=None) # Both javadoc and sources jars are added to the classpath product self.assertEqual(['default', 'src_doc', 'src_doc'], @@ -251,7 +252,7 @@ def test_when_invalid_hardlink_and_coursier_cache_should_trigger_resolve(self): # Remove coursier's cache safe_rmtree(couriser_cache_dir) - task.runjava = MagicMock() + util.execute_runner = MagicMock() # Ignore any error because runjava may fail due to undefined behavior try: @@ -259,7 +260,7 @@ def test_when_invalid_hardlink_and_coursier_cache_should_trigger_resolve(self): except TaskError: pass - task.runjava.assert_called() + util.execute_runner.assert_called() def test_resolve_jarless_pom(self): jar = JarDependency('org.apache.commons', 'commons-weaver-privilizer-parent', '1.3') From 59a944c693da4d44a6def460a95ceb03618a06f2 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 1 Dec 2018 20:09:54 -0800 Subject: [PATCH 12/13] fix more tests --- src/python/pants/backend/project_info/tasks/export.py | 5 ++++- tests/python/pants_test/jvm/jvm_task_test_base.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/project_info/tasks/export.py b/src/python/pants/backend/project_info/tasks/export.py index aa30fc47ca2..0422d3d2ecc 100644 --- a/src/python/pants/backend/project_info/tasks/export.py +++ b/src/python/pants/backend/project_info/tasks/export.py @@ -113,6 +113,8 @@ def register_options(cls, register): help='Causes sources to be output.') register('--formatted', type=bool, implicit_value=False, help='Causes output to be a single line of JSON.') + register('--jvm-options', type=list, metavar='