Skip to content

Commit

Permalink
Fix DistributionLocator dependency declarations.
Browse files Browse the repository at this point in the history
This fixes all usages of `DistributionLocator` to declare a subsystem
dependency and removes all unused dependency declarations save for
those in the `Ivy` family where the tangles run deep.

Discovered working #7872.
  • Loading branch information
jsirois committed Jun 8, 2019
1 parent 97f50b7 commit 96c5279
Show file tree
Hide file tree
Showing 7 changed files with 31 additions and 17 deletions.
Expand Up @@ -10,7 +10,6 @@

from pants.base.build_environment import get_buildroot, get_pants_cachedir
from pants.base.workunit import WorkUnit, WorkUnitLabel
from pants.java.distribution.distribution import DistributionLocator
from pants.net.http.fetcher import Fetcher
from pants.subsystem.subsystem import Subsystem
from pants.util.dirutil import safe_concurrent_creation
Expand Down Expand Up @@ -62,10 +61,6 @@ def register_options(cls, register):
register('--bootstrap-fetch-timeout-secs', type=int, advanced=True, default=10,
help='Timeout the fetch if the connection is idle for longer than this value.')

@classmethod
def subsystem_dependencies(cls):
return super(CoursierSubsystem, cls).subsystem_dependencies() + (DistributionLocator,)

def bootstrap_coursier(self, workunit_factory):

opts = self.get_options()
Expand Down
12 changes: 8 additions & 4 deletions src/python/pants/backend/jvm/tasks/coursier_resolve.py
Expand Up @@ -57,7 +57,11 @@ def implementation_version(cls):

@classmethod
def subsystem_dependencies(cls):
return super(CoursierMixin, cls).subsystem_dependencies() + (JarDependencyManagement, CoursierSubsystem)
return super(CoursierMixin, cls).subsystem_dependencies() + (
CoursierSubsystem,
DistributionLocator,
JarDependencyManagement
)

@classmethod
def register_options(cls, register):
Expand Down Expand Up @@ -382,7 +386,7 @@ def _construct_cmd_args(jars, common_args, global_excludes,
if j.get_url():
jar_url = j.get_url()
module += ',url={}'.format(parse.quote_plus(jar_url))

if j.intransitive:
cmd_args.append('--intransitive')

Expand Down Expand Up @@ -646,7 +650,7 @@ def to_m2_coord(cls, coord_str):
def _get_path_to_jar(cls, coursier_cache_path, pants_jar_path_base, jar_path):
"""
Create the path to the jar that will live in .pants.d
:param coursier_cache_path: coursier cache location
:param pants_jar_path_base: location under pants workdir to store the hardlink to the coursier cache
:param jar_path: path of the jar
Expand Down Expand Up @@ -686,7 +690,7 @@ def register_options(cls, register):
@classmethod
def implementation_version(cls):
return super(CoursierResolve, cls).implementation_version() + [('CoursierResolve', 2)]

def execute(self):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
Expand Down
3 changes: 1 addition & 2 deletions src/python/pants/backend/jvm/tasks/junit_run.py
Expand Up @@ -28,7 +28,6 @@
from pants.base.workunit import WorkUnitLabel
from pants.build_graph.target import Target
from pants.build_graph.target_scopes import Scopes
from pants.java.distribution.distribution import DistributionLocator
from pants.java.executor import SubprocessExecutor
from pants.java.junit.junit_xml_parser import RegistryOfTests, Test, parse_failed_targets
from pants.process.lock import OwnerPrintingInterProcessFileLock
Expand Down Expand Up @@ -107,7 +106,7 @@ def register_options(cls, register):

@classmethod
def subsystem_dependencies(cls):
return super(JUnitRun, cls).subsystem_dependencies() + (CodeCoverage, DistributionLocator, JUnit)
return super(JUnitRun, cls).subsystem_dependencies() + (CodeCoverage, JUnit)

@classmethod
def prepare(cls, options, round_manager):
Expand Down
Expand Up @@ -365,6 +365,7 @@ def __init__(self, *args, **kwargs):
@memoized_property
def _missing_deps_finder(self):
dep_analyzer = JvmDependencyAnalyzer(get_buildroot(),
self._get_jvm_distribution(),
self.context.products.get_data('runtime_classpath'))
return MissingDependencyFinder(dep_analyzer, CompileErrorExtractor(
self.get_options().class_not_found_error_patterns))
Expand Down
6 changes: 3 additions & 3 deletions src/python/pants/backend/jvm/tasks/jvm_dependency_analyzer.py
Expand Up @@ -16,7 +16,6 @@
from pants.build_graph.aliased_target import AliasTarget
from pants.build_graph.build_graph import sort_targets
from pants.build_graph.target import Target
from pants.java.distribution.distribution import DistributionLocator
from pants.util.memo import memoized_method, memoized_property


Expand All @@ -27,8 +26,9 @@ class JvmDependencyAnalyzer(object):
determining which targets correspond to the actual source dependencies of any given target.
"""

def __init__(self, buildroot, runtime_classpath):
def __init__(self, buildroot, distribution, runtime_classpath):
self.buildroot = buildroot
self.distribution = distribution
self.runtime_classpath = runtime_classpath

@memoized_method
Expand Down Expand Up @@ -122,7 +122,7 @@ def bootstrap_jar_classfiles(self):

def _find_all_bootstrap_jars(self):
def get_path(key):
return DistributionLocator.cached().system_properties.get(key, '').split(':')
return self.distribution.system_properties.get(key, '').split(':')

def find_jars_in_dirs(dirs):
ret = []
Expand Down
12 changes: 10 additions & 2 deletions src/python/pants/backend/jvm/tasks/jvm_dependency_check.py
Expand Up @@ -53,7 +53,10 @@ def register_options(cls, register):

@classmethod
def subsystem_dependencies(cls):
return super(JvmDependencyCheck, cls).subsystem_dependencies() + (DependencyContext,)
return super(JvmDependencyCheck, cls).subsystem_dependencies() + (
DependencyContext,
DistributionLocator
)

@staticmethod
def _skip(options):
Expand Down Expand Up @@ -85,9 +88,14 @@ def munge_flag(flag):
def cache_target_dirs(self):
return True

@memoized_property
def _distribution(self):
return DistributionLocator.cached()

@memoized_property
def _analyzer(self):
return JvmDependencyAnalyzer(get_buildroot(),
self._distribution,
self.context.products.get_data('runtime_classpath'))

def execute(self):
Expand Down Expand Up @@ -180,7 +188,7 @@ def must_be_explicit_dep(dep):
# We don't require explicit deps on the java runtime, so we shouldn't consider that
# a missing dep.
return (dep not in analyzer.bootstrap_jar_classfiles
and not dep.startswith(DistributionLocator.cached().real_home))
and not dep.startswith(self._distribution.real_home))

def target_or_java_dep_in_targets(target, targets):
# We want to check if the target is in the targets collection
Expand Down
9 changes: 8 additions & 1 deletion src/python/pants/backend/jvm/tasks/jvm_dependency_usage.py
Expand Up @@ -19,6 +19,7 @@
from pants.build_graph.resources import Resources
from pants.build_graph.target import Target
from pants.build_graph.target_scopes import Scopes
from pants.java.distribution.distribution import DistributionLocator
from pants.task.task import Task
from pants.util.fileutil import create_size_estimators

Expand Down Expand Up @@ -66,6 +67,10 @@ def register_options(cls, register):
'result can differ from direct execution because cached information '
'doesn\'t depend on 3rdparty libraries versions.')

@classmethod
def subsystem_dependencies(cls):
return super(JvmDependencyUsage, cls).subsystem_dependencies() + (DistributionLocator,)

@classmethod
def prepare(cls, options, round_manager):
super(JvmDependencyUsage, cls).prepare(options, round_manager)
Expand Down Expand Up @@ -159,7 +164,9 @@ def calculating_node_creator(self, classes_by_source, runtime_classpath, product
`classes_by_source`, `runtime_classpath`, `product_deps_by_src` parameters and
stores the result to the build cache.
"""
analyzer = JvmDependencyAnalyzer(get_buildroot(), runtime_classpath)
analyzer = JvmDependencyAnalyzer(get_buildroot(),
DistributionLocator.cached(),
runtime_classpath)
targets = self.context.targets()
targets_by_file = analyzer.targets_by_file(targets)
transitive_deps_by_target = analyzer.compute_transitive_deps_by_target(targets)
Expand Down

0 comments on commit 96c5279

Please sign in to comment.