From 8745c058c984bc79a7b7ffbbdf7342ef8a4472ad Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sun, 5 May 2024 21:18:58 -0500 Subject: [PATCH] Deep exclude working, but unit tests need re-work. --- pex/bin/pex.py | 5 +- pex/cli/commands/lock.py | 6 +- pex/dependency_manager.py | 68 +----- pex/environment.py | 20 +- pex/{asserts.py => exceptions.py} | 37 +-- pex/exclude_configuration.py | 8 +- pex/fetcher.py | 4 +- pex/resolve/configured_resolve.py | 5 + pex/resolve/lock_resolver.py | 3 + pex/resolve/locked_resolve.py | 29 ++- pex/resolve/lockfile/subset.py | 3 + pex/resolve/pex_repository_resolver.py | 6 +- pex/resolver.py | 305 +++++++++++++++++++++---- tests/test_dependency_manager.py | 9 +- 14 files changed, 365 insertions(+), 143 deletions(-) rename pex/{asserts.py => exceptions.py} (72%) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 104d6d1ea..f58fb27ba 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -27,6 +27,7 @@ from pex.dependency_manager import DependencyManager from pex.docs.command import serve_html_docs from pex.enum import Enum +from pex.exclude_configuration import ExcludeConfiguration from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraints from pex.layout import Layout, ensure_installed @@ -897,6 +898,7 @@ def build_pex( ) excluded.extend(requirements_pex_info.excluded) + exclude_configuration = ExcludeConfiguration.create(excluded) with TRACER.timed( "Resolving distributions for requirements: {}".format( " ".join( @@ -922,13 +924,14 @@ def build_pex( if options.pre_install_wheels else InstallableType.WHEEL_FILE ), + exclude_configuration=exclude_configuration, ) ) except Unsatisfiable as e: die(str(e)) with TRACER.timed("Configuring PEX dependencies"): - dependency_manager.configure(pex_builder, excluded=excluded) + dependency_manager.configure(pex_builder, exclude_configuration=exclude_configuration) if options.entry_point: pex_builder.set_entry_point(options.entry_point) diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index bacbc53dc..64a1f53e0 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -12,7 +12,6 @@ from pex import pex_warnings from pex.argparse import HandleBoolAction -from pex.asserts import production_assert from pex.cli.command import BuildTimeCommand from pex.commands.command import JsonMixin, OutputMixin from pex.common import is_exe, pluralize, safe_delete, safe_open @@ -25,6 +24,7 @@ RequirementParseError, ) from pex.enum import Enum +from pex.exceptions import production_assert from pex.interpreter import PythonInterpreter from pex.pep_376 import InstalledWheel, Record from pex.pep_427 import InstallableType @@ -1103,7 +1103,7 @@ def _process_lock_update( updates = [] # type: List[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]] warnings = [] # type: List[str] for resolve_update in lock_update.resolves: - platform = resolve_update.updated_resolve.platform_tag or "universal" + platform = resolve_update.updated_resolve.target_platform if not resolve_update.updates: print( "No updates for lock generated by {platform}.".format(platform=platform), @@ -1468,7 +1468,7 @@ def _sync(self): "Would lock {count} {project} for platform {platform}:".format( count=len(locked_resolve.locked_requirements), project=pluralize(locked_resolve.locked_requirements, "project"), - platform=locked_resolve.platform_tag or "universal", + platform=locked_resolve.target_platform, ), file=output, ) diff --git a/pex/dependency_manager.py b/pex/dependency_manager.py index b1262b489..0c221d9c0 100644 --- a/pex/dependency_manager.py +++ b/pex/dependency_manager.py @@ -3,24 +3,17 @@ from __future__ import absolute_import -from collections import defaultdict - -from pex import pex_warnings from pex.dist_metadata import Requirement from pex.environment import PEXEnvironment from pex.exclude_configuration import ExcludeConfiguration from pex.fingerprinted_distribution import FingerprintedDistribution from pex.orderedset import OrderedSet -from pex.pep_503 import ProjectName from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pex.resolve.resolvers import ResolveResult -from pex.tracer import TRACER from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import DefaultDict, Iterable, Iterator - import attr # vendor:skip else: from pex.third_party import attr @@ -58,67 +51,14 @@ def add_from_resolved(self, resolved): def configure( self, pex_builder, # type: PEXBuilder - excluded=(), # type: Iterable[str] + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> None - exclude_configuration = ExcludeConfiguration.create(excluded) exclude_configuration.configure(pex_builder.info) - - dists_by_project_name = defaultdict( - OrderedSet - ) # type: DefaultDict[ProjectName, OrderedSet[FingerprintedDistribution]] - for dist in self._distributions: - dists_by_project_name[dist.distribution.metadata.project_name].add(dist) - - root_requirements_by_project_name = defaultdict( - OrderedSet - ) # type: DefaultDict[ProjectName, OrderedSet[Requirement]] - for root_req in self._requirements: - root_requirements_by_project_name[root_req.project_name].add(root_req) - - def iter_non_excluded_distributions(requirements): - # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution] - for req in requirements: - candidate_dists = dists_by_project_name[req.project_name] - for candidate_dist in tuple(candidate_dists): - if candidate_dist.distribution not in req: - continue - candidate_dists.discard(candidate_dist) - - excluded_by = exclude_configuration.excluded_by(candidate_dist.distribution) - if excluded_by: - excludes = " and ".join(map(str, excluded_by)) - TRACER.log( - "Skipping adding {candidate}: excluded by {excludes}".format( - candidate=candidate_dist.distribution, excludes=excludes - ) - ) - for root_req in root_requirements_by_project_name[ - candidate_dist.distribution.metadata.project_name - ]: - if candidate_dist.distribution in root_req: - pex_warnings.warn( - "The distribution {dist} was required by the input requirement " - "{root_req} but excluded by configured excludes: " - "{excludes}".format( - dist=candidate_dist.distribution, - root_req=root_req, - excludes=excludes, - ) - ) - continue - - yield candidate_dist - for dep in iter_non_excluded_distributions( - candidate_dist.distribution.requires() - ): - yield dep - - for fingerprinted_dist in iter_non_excluded_distributions(self._requirements): + for requirement in self._requirements: + pex_builder.add_requirement(requirement) + for fingerprinted_dist in self._distributions: pex_builder.add_distribution( dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint ) - - for requirement in self._requirements: - pex_builder.add_requirement(requirement) diff --git a/pex/environment.py b/pex/environment.py index b7e22b696..47992ee0a 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -459,8 +459,12 @@ def _resolve_requirement( ): yield not_found - def _root_requirements_iter(self, reqs): - # type: (Iterable[Requirement]) -> Iterator[QualifiedRequirementOrNotFound] + def _root_requirements_iter( + self, + reqs, # type: Iterable[Requirement] + exclude_configuration, # type: ExcludeConfiguration + ): + # type: (...) -> Iterator[QualifiedRequirementOrNotFound] # We want to pick one requirement for each key (required project) to then resolve # recursively. @@ -477,6 +481,16 @@ def _root_requirements_iter(self, reqs): OrderedDict() ) # type: OrderedDict[ProjectName, List[_QualifiedRequirement]] for req in reqs: + excluded_by = exclude_configuration.excluded_by(req) + if excluded_by: + TRACER.log( + "Skipping resolving {requirement}: excluded by {excludes}".format( + requirement=req, + excludes=" and ".join(map(str, excluded_by)), + ) + ) + continue + required = self._evaluate_marker(req) if not required: continue @@ -591,7 +605,7 @@ def record_unresolved(dist_not_found): resolved_dists_by_key = ( OrderedDict() ) # type: OrderedDict[_RequirementKey, FingerprintedDistribution] - for qualified_req_or_not_found in self._root_requirements_iter(reqs): + for qualified_req_or_not_found in self._root_requirements_iter(reqs, exclude_configuration): if isinstance(qualified_req_or_not_found, _DistributionNotFound): record_unresolved(qualified_req_or_not_found) continue diff --git a/pex/asserts.py b/pex/exceptions.py similarity index 72% rename from pex/asserts.py rename to pex/exceptions.py index f7e2c58da..308142e75 100644 --- a/pex/asserts.py +++ b/pex/exceptions.py @@ -13,11 +13,11 @@ _ASSERT_DETAILS = ( dedent( """\ - Pex {version} - platform: {platform} - python: {python_version} - argv: {argv} - """ + Pex {version} + platform: {platform} + python: {python_version} + argv: {argv} + """ ) .format( version=__version__, platform=platform.platform(), python_version=sys.version, argv=sys.argv @@ -27,11 +27,11 @@ _ASSERT_ADVICE = dedent( """\ - The error reported above resulted from an unexpected programming error which - you should never encounter. - + The error reported above resulted from an unexpected error which you should + never encounter. + Firstly, please accept our apology! - + If you could file an issue with the error and details above, we'd be grateful. You can do that at https://github.com/pex-tool/pex/issues/new and redact or amend any details that expose sensitive information. @@ -39,11 +39,8 @@ ).strip() -def production_assert(condition, msg=""): - # type: (...) -> None - - if condition: - return +def reportable_unexpected_error_msg(msg=""): + # type: (str) -> str message = [msg, "---", _ASSERT_DETAILS] pex = os.environ.get("PEX") @@ -64,4 +61,14 @@ def production_assert(condition, msg=""): message.append("---") message.append(_ASSERT_ADVICE) - raise AssertionError("\n".join(message)) + return "\n".join(message) + + +def production_assert( + condition, # type: bool + msg="", # type: str +): + # type: (...) -> None + + if not condition: + raise AssertionError(reportable_unexpected_error_msg(msg=msg)) diff --git a/pex/exclude_configuration.py b/pex/exclude_configuration.py index 466ae9a8f..0a19dceac 100644 --- a/pex/exclude_configuration.py +++ b/pex/exclude_configuration.py @@ -22,7 +22,7 @@ def create(cls, excluded): # type: (Iterable[str]) -> ExcludeConfiguration return cls(excluded=tuple(Requirement.parse(req) for req in excluded)) - _excluded = attr.ib(factory=tuple) # type: Tuple[Requirement, ...] + _excluded = attr.ib(default=()) # type: Tuple[Requirement, ...] def configure(self, pex_info): # type: (PexInfo) -> None @@ -30,7 +30,11 @@ def configure(self, pex_info): pex_info.add_excluded(excluded) def excluded_by(self, item): - # type: (Union[Distribution, Requirement]) -> Iterable[Requirement] + # type: (Union[Distribution, Requirement]) -> Tuple[Requirement, ...] if isinstance(item, Distribution): return tuple(req for req in self._excluded if item in req) return tuple(req for req in self._excluded if item.project_name == req.project_name) + + def __bool__(self): + # type: () -> bool + return bool(self._excluded) diff --git a/pex/fetcher.py b/pex/fetcher.py index bbf842519..f6dddaadd 100644 --- a/pex/fetcher.py +++ b/pex/fetcher.py @@ -12,7 +12,6 @@ from contextlib import closing, contextmanager from ssl import SSLContext -from pex import asserts from pex.auth import PasswordDatabase, PasswordEntry from pex.compatibility import ( FileHandler, @@ -26,6 +25,7 @@ build_opener, in_main_thread, ) +from pex.exceptions import production_assert from pex.network_configuration import NetworkConfiguration from pex.typing import TYPE_CHECKING, cast from pex.version import __version__ @@ -102,7 +102,7 @@ def create_ssl_context(self): # [^5]: https://github.com/openssl/openssl/blob/c3cc0f1386b0544383a61244a4beeb762b67498f/ssl/ssl_init.c#L86-L116 # [^6]: https://github.com/indygreg/python-build-standalone/releases/tag/20240107 # [^7]: https://gitlab.com/redhat-crypto/fedora-crypto-policies/-/merge_requests/110/diffs#269a48e71ac25ad1d07ff00db2390834c8ba7596_11_16 - asserts.production_assert( + production_assert( in_main_thread(), msg=( "An SSLContext must be initialized from the main thread. An attempt was made to " diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py index c31c12fba..a90f7ff30 100644 --- a/pex/resolve/configured_resolve.py +++ b/pex/resolve/configured_resolve.py @@ -3,6 +3,7 @@ from __future__ import absolute_import +from pex.exclude_configuration import ExcludeConfiguration from pex.pep_427 import InstallableType from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.lock_resolver import resolve_from_lock @@ -30,6 +31,7 @@ def resolve( compile_pyc=False, # type: bool ignore_errors=False, # type: bool result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> ResolveResult if isinstance(resolver_configuration, LockRepositoryConfiguration): @@ -58,6 +60,7 @@ def resolve( pip_version=lock.pip_version, use_pip_config=pip_configuration.use_pip_config, result_type=result_type, + exclude_configuration=exclude_configuration, ) ) elif isinstance(resolver_configuration, PexRepositoryConfiguration): @@ -76,6 +79,7 @@ def resolve( transitive=resolver_configuration.transitive, ignore_errors=ignore_errors, result_type=result_type, + exclude_configuration=exclude_configuration, ) else: with TRACER.timed("Resolving requirements."): @@ -100,4 +104,5 @@ def resolve( resolver=ConfiguredResolver(pip_configuration=resolver_configuration), use_pip_config=resolver_configuration.use_pip_config, result_type=result_type, + exclude_configuration=exclude_configuration, ) diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index ad1cdd438..081c10a4c 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -14,6 +14,7 @@ from pex.auth import PasswordDatabase, PasswordEntry from pex.common import pluralize from pex.compatibility import cpu_count +from pex.exclude_configuration import ExcludeConfiguration from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.pep_427 import InstallableType @@ -247,6 +248,7 @@ def resolve_from_lock( pip_version=None, # type: Optional[PipVersionValue] use_pip_config=False, # type: bool result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> Union[ResolveResult, Error] @@ -262,6 +264,7 @@ def resolve_from_lock( network_configuration=network_configuration, build_configuration=build_configuration, transitive=transitive, + exclude_configuration=exclude_configuration, ) ) downloadable_artifacts_and_targets = OrderedSet( diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index ad346569c..55f40a56f 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -11,6 +11,7 @@ from pex.common import pluralize from pex.dist_metadata import DistMetadata, Requirement from pex.enum import Enum +from pex.exclude_configuration import ExcludeConfiguration from pex.orderedset import OrderedSet from pex.pep_425 import CompatibilityTags, TagRank from pex.pep_503 import ProjectName @@ -27,6 +28,7 @@ from pex.result import Error from pex.sorted_tuple import SortedTuple from pex.targets import Target +from pex.tracer import TRACER from pex.typing import TYPE_CHECKING if TYPE_CHECKING: @@ -588,6 +590,11 @@ def resolve_fingerprint(partial_artifact): locked_requirements = attr.ib() # type: SortedTuple[LockedRequirement] platform_tag = attr.ib(order=str, default=None) # type: Optional[tags.Tag] + @property + def target_platform(self): + # type: () -> str + return str(self.platform_tag) if self.platform_tag else "universal" + def resolve( self, target, # type: Target @@ -597,6 +604,7 @@ def resolve( transitive=True, # type: bool build_configuration=BuildConfiguration(), # type: BuildConfiguration include_all_matches=False, # type: bool + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> Union[Resolved, Error] @@ -610,11 +618,22 @@ def resolve( def request_resolve(requests): # type: (Iterable[_ResolveRequest]) -> None - to_be_resolved.extend( - request - for request in requests - if target.requirement_applies(request.requirement, extras=request.extras) - ) + for request in requests: + excluded_by = exclude_configuration.excluded_by(request.requirement) + if excluded_by: + TRACER.log( + "Locked requirement {requirement} from {platform} lock excluded by " + "{exclude} {excluded_by}.".format( + requirement=request.requirement, + exclude=pluralize(excluded_by, "exclude"), + excluded_by=" and ".join(map(str, excluded_by)), + platform=self.target_platform, + ) + ) + continue + if not target.requirement_applies(request.requirement, extras=request.extras): + continue + to_be_resolved.append(request) resolved = {} # type: Dict[ProjectName, Set[str]] request_resolve(_ResolveRequest.root(requirement) for requirement in requirements) diff --git a/pex/resolve/lockfile/subset.py b/pex/resolve/lockfile/subset.py index fbda77fee..01692ed93 100644 --- a/pex/resolve/lockfile/subset.py +++ b/pex/resolve/lockfile/subset.py @@ -7,6 +7,7 @@ from collections import OrderedDict from pex.common import pluralize +from pex.exclude_configuration import ExcludeConfiguration from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.requirements import LocalProjectRequirement, parse_requirement_strings @@ -49,6 +50,7 @@ def subset( build_configuration=BuildConfiguration(), # type: BuildConfiguration transitive=True, # type: bool include_all_matches=False, # type: bool + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> Union[SubsetResult, Error] @@ -89,6 +91,7 @@ def subset( build_configuration=build_configuration, transitive=transitive, include_all_matches=include_all_matches, + exclude_configuration=exclude_configuration, # TODO(John Sirois): Plumb `--ignore-errors` to support desired but technically # invalid `pip-legacy-resolver` locks: # https://github.com/pex-tool/pex/issues/1652 diff --git a/pex/resolve/pex_repository_resolver.py b/pex/resolve/pex_repository_resolver.py index 205e49fdc..bce213c7b 100644 --- a/pex/resolve/pex_repository_resolver.py +++ b/pex/resolve/pex_repository_resolver.py @@ -9,6 +9,7 @@ from pex import environment from pex.dist_metadata import Requirement from pex.environment import PEXEnvironment +from pex.exclude_configuration import ExcludeConfiguration from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.pep_427 import InstallableType @@ -34,6 +35,7 @@ def resolve_from_pex( transitive=True, # type: bool ignore_errors=False, # type: bool result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> ResolveResult @@ -75,7 +77,9 @@ def resolve_from_pex( for target in targets.unique_targets(): pex_env = PEXEnvironment.mount(pex, target=target) try: - fingerprinted_distributions = pex_env.resolve_dists(all_reqs, result_type=result_type) + fingerprinted_distributions = pex_env.resolve_dists( + all_reqs, result_type=result_type, exclude_configuration=exclude_configuration + ) except environment.ResolveError as e: raise Unsatisfiable(str(e)) diff --git a/pex/resolver.py b/pex/resolver.py index 6f1964103..45d512495 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -9,18 +9,28 @@ import hashlib import itertools import os +import tarfile import zipfile from abc import abstractmethod from collections import OrderedDict, defaultdict -from pex import targets +from pex import pex_warnings, targets from pex.atomic_directory import AtomicDirectory, atomic_directory from pex.auth import PasswordEntry -from pex.common import safe_mkdir, safe_mkdtemp +from pex.build_system.pep_517 import spawn_prepare_metadata +from pex.common import open_zip, pluralize, safe_mkdir, safe_mkdtemp from pex.compatibility import url_unquote, urlparse -from pex.dist_metadata import DistMetadata, Distribution, ProjectNameAndVersion, Requirement +from pex.dist_metadata import ( + DistMetadata, + Distribution, + MetadataType, + ProjectNameAndVersion, + Requirement, +) +from pex.exceptions import production_assert, reportable_unexpected_error_msg +from pex.exclude_configuration import ExcludeConfiguration from pex.fingerprinted_distribution import FingerprintedDistribution -from pex.jobs import Raise, SpawnedJob, execute_parallel, iter_map_parallel +from pex.jobs import Raise, Retain, SpawnedJob, execute_parallel, iter_map_parallel from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.pep_376 import InstalledWheel @@ -32,7 +42,13 @@ from pex.pip.installation import get_pip from pex.pip.tool import PackageIndexConfiguration from pex.pip.version import PipVersionValue -from pex.requirements import LocalProjectRequirement +from pex.requirements import ( + LocalProjectRequirement, + PyPIRequirement, + URLRequirement, + VCSRequirement, + parse_requirement_from_project_name_and_specifier, +) from pex.resolve.downloads import get_downloads_dir from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion @@ -45,13 +61,14 @@ ) from pex.targets import Target, Targets from pex.tracer import TRACER -from pex.typing import TYPE_CHECKING +from pex.typing import TYPE_CHECKING, cast from pex.util import CacheHelper from pex.variables import ENV if TYPE_CHECKING: from typing import ( DefaultDict, + Dict, Iterable, Iterator, List, @@ -89,13 +106,8 @@ class DownloadRequest(object): preserve_log = attr.ib(default=False) # type: bool pip_version = attr.ib(default=None) # type: Optional[PipVersionValue] resolver = attr.ib(default=None) # type: Optional[Resolver] - - def iter_local_projects(self): - # type: () -> Iterator[BuildRequest] - for requirement in self.direct_requirements: - if isinstance(requirement, LocalProjectRequirement): - for target in self.targets: - yield BuildRequest.create(target=target, source_path=requirement.path) + max_parallel_jobs = attr.ib(default=None) # type: Optional[int] + exclude_configuration = attr.ib(default=ExcludeConfiguration()) # type: ExcludeConfiguration def download_distributions(self, dest=None, max_parallel_jobs=None): # type: (...) -> List[DownloadResult] @@ -113,7 +125,7 @@ def download_distributions(self, dest=None, max_parallel_jobs=None): inputs=self.targets, spawn_func=spawn_download, error_handler=Raise[Target, DownloadResult](Unsatisfiable), - max_jobs=max_parallel_jobs, + max_jobs=self.max_parallel_jobs, ) ) @@ -130,7 +142,15 @@ def _spawn_download( else None ) - download_result = DownloadResult(target, download_dir) + download_result = DownloadResult( + target=target, + resolver=self.resolver, + pip_version=self.pip_version, + download_dir=download_dir, + direct_requirements=self.direct_requirements, + max_parallel_jobs=self.max_parallel_jobs, + exclude_configuration=self.exclude_configuration, + ) download_job = get_pip( interpreter=target.get_interpreter(), version=self.pip_version, @@ -152,7 +172,7 @@ def _spawn_download( return SpawnedJob.wait(job=download_job, result=download_result) -@attr.s(frozen=True) +@attr.s class DownloadResult(object): @staticmethod def _is_wheel(path): @@ -161,23 +181,219 @@ def _is_wheel(path): target = attr.ib() # type: Target download_dir = attr.ib() # type: str + direct_requirements = attr.ib() # type: Iterable[ParsedRequirement] + resolver = attr.ib(default=None) # type: Optional[Resolver] + pip_version = attr.ib(default=None) # type: Optional[PipVersionValue] + max_parallel_jobs = attr.ib(default=None) # type: Optional[int] + exclude_configuration = attr.ib(default=ExcludeConfiguration()) # type: ExcludeConfiguration + __cache = attr.ib( + default=None + ) # type: Optional[Tuple[Tuple[ParsedRequirement, ...], Tuple[str, ...]]] + + def _spawn_generate_metadata(self, distribution_path): + # type: (str) -> SpawnedJob[DistMetadata] + if distribution_path.endswith(".whl"): + dist_metadata = DistMetadata.load(distribution_path, MetadataType.DIST_INFO) + return SpawnedJob.completed(dist_metadata) + + production_assert( + self.resolver is not None, + "Expected DownloadResult to be configured with a resolver to process excludes for " + "sdists but no resolver was configured.", + ) - def _iter_distribution_paths(self): - # type: () -> Iterator[str] - if not os.path.exists(self.download_dir): - return - for distribution in os.listdir(self.download_dir): - yield os.path.join(self.download_dir, distribution) + resolver = cast(Resolver, self.resolver) + if os.path.isdir(distribution_path): + project_directory = distribution_path + else: + distribution_name = os.path.basename(distribution_path) + extract_directory = safe_mkdtemp(prefix=distribution_name) + if distribution_path.endswith(".zip"): + with open_zip(distribution_path) as zf: + zf.extractall(extract_directory) + else: + with tarfile.open(distribution_path) as tf: + tf.extractall(extract_directory) + listing = glob.glob(os.path.join(extract_directory, "*")) + count = len(listing) + production_assert( + count == 1, + "Expected {dist} to contain one top-level directory, but found {count} top-level" + " entries: {entries}".format( + dist=distribution_name, count=count, entries=", ".join(listing) + ), + ) + project_directory = listing[0] + + return spawn_prepare_metadata( + project_directory, self.target, resolver, pip_version=self.pip_version + ) + + @property + def _cache(self): + # type: () -> Tuple[Tuple[ParsedRequirement, ...], Tuple[str, ...]] + if self.__cache is None: + unexcluded_direct_requirements = [] # type: List[ParsedRequirement] + distribution_paths = [] # type: List[str] + if not self.exclude_configuration: + unexcluded_direct_requirements.extend(self.direct_requirements) + distribution_paths.extend( + parsed_requirement.path + for parsed_requirement in self.direct_requirements + if isinstance(parsed_requirement, LocalProjectRequirement) + ) + if os.path.exists(self.download_dir): + distribution_paths.extend( + os.path.join(self.download_dir, distribution) + for distribution in os.listdir(self.download_dir) + ) + else: + downloads = glob.glob(os.path.join(self.download_dir, "*")) + + requirements = OrderedDict() # type: OrderedDict[ParsedRequirement, Requirement] + local_project_requirements = {} # type: Dict[str, LocalProjectRequirement] + for parsed_requirement in self.direct_requirements: + if isinstance( + parsed_requirement, + (PyPIRequirement, URLRequirement, VCSRequirement), + ): + requirements[parsed_requirement] = parsed_requirement.requirement + else: + downloads.append(parsed_requirement.path) + local_project_requirements[parsed_requirement.path] = parsed_requirement + + project_to_distribution = {} # type: Dict[ProjectName, Distribution] + errors = [] # type: List[Tuple[str, str]] + for distribution_path, dist_metadata_or_error in zip( + downloads, + execute_parallel( + downloads, + spawn_func=self._spawn_generate_metadata, # type: ignore[arg-type] + error_handler=Retain[str](), # type: ignore[arg-type] + max_jobs=self.max_parallel_jobs, + ), + ): + if isinstance(dist_metadata_or_error, DistMetadata): + dist = Distribution( + location=distribution_path, metadata=dist_metadata_or_error + ) + project_to_distribution[dist.metadata.project_name] = dist + local_project_requirement = local_project_requirements.pop( + distribution_path, None + ) + if local_project_requirement: + requirement = parse_requirement_from_project_name_and_specifier( + project_name=dist_metadata_or_error.project_name.raw, + extras=local_project_requirement.extras, + marker=local_project_requirement.marker, + ) + requirements[local_project_requirement] = requirement + else: + errors.append((distribution_path, str(dist_metadata_or_error))) + + if errors: + raise Untranslatable( + reportable_unexpected_error_msg( + "Failed to extract dependency metadata for the following " + "{distributions}:\n{errors}".format( + distributions=pluralize(errors, "distribution"), + errors="\n".join( + "{dist}: {err}".format(dist=os.path.basename(path), err=err) + for path, err in errors + ), + ) + ) + ) + + for parsed_requirement, requirement in requirements.items(): + if not self.target.requirement_applies(requirement): + continue + excluded_by = self.exclude_configuration.excluded_by(requirement) + if excluded_by: + dist = project_to_distribution[requirement.project_name] + pex_warnings.warn( + "The distribution {dist} was required by the input requirement " + "{root_req} but excluded by configured excludes: " + "{excludes}".format( + dist=dist, + root_req="{path} ({req})".format( + path=parsed_requirement.path, req=requirement + ) + if isinstance(parsed_requirement, LocalProjectRequirement) + else requirement, + excludes=" and ".join(map(str, excluded_by)), + ) + ) + continue + unexcluded_direct_requirements.append(parsed_requirement) + distribution_paths.extend( + self._iter_unexcluded_distribution_paths( + project_to_distribution, requirements.values(), set() + ) + ) + self.__cache = tuple(unexcluded_direct_requirements), tuple(distribution_paths) + return self.__cache + + def _iter_unexcluded_distribution_paths( + self, + project_to_distribution, # type: Mapping[ProjectName, Distribution] + requirements, # type: Iterable[Requirement] + seen, # type: Set[ProjectName] + extras=(), # type: Iterable[str] + ): + # type: (...) -> Iterator[str] + for requirement in requirements: + if not self.target.requirement_applies(requirement, extras=extras): + continue + project_name = requirement.project_name + if project_name in seen: + continue + seen.add(project_name) + dist = project_to_distribution[project_name] + if self._excluded(dist): + continue + yield dist.location + for dependency_distribution_path in self._iter_unexcluded_distribution_paths( + project_to_distribution, dist.requires(), seen=seen, extras=requirement.extras + ): + yield dependency_distribution_path + + def _excluded(self, dist): + # type: (Distribution) -> bool + if self.exclude_configuration: + excluded_by = self.exclude_configuration.excluded_by(dist) + if excluded_by: + TRACER.log( + "Downloaded distribution {dist} excluded by {exclude} {excluded_by} for " + "{target}.".format( + dist=dist, + exclude=pluralize(excluded_by, "exclude"), + excluded_by=" and ".join(map(str, excluded_by)), + target=self.target.render_description(), + ) + ) + return True + return False + + @property + def unexcluded_direct_requirements(self): + unexcluded_direct_requirements, _ = self._cache + return unexcluded_direct_requirements + + @property + def _distribution_paths(self): + _, distribution_paths = self._cache + return distribution_paths def build_requests(self): # type: () -> Iterator[BuildRequest] - for distribution_path in self._iter_distribution_paths(): + for distribution_path in self._distribution_paths: if not self._is_wheel(distribution_path): yield BuildRequest.create(target=self.target, source_path=distribution_path) def install_requests(self): # type: () -> Iterator[InstallRequest] - for distribution_path in self._iter_distribution_paths(): + for distribution_path in self._distribution_paths: if self._is_wheel(distribution_path): yield InstallRequest.create(target=self.target, wheel_path=distribution_path) @@ -1032,6 +1248,7 @@ def resolve( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> ResolveResult """Resolves all distributions needed to meet requirements for multiple distribution targets. @@ -1115,7 +1332,7 @@ def resolve( password_entries=password_entries, use_pip_config=use_pip_config, ) - build_requests, download_results = _download_internal( + download_results = _download_internal( targets=targets, direct_requirements=direct_requirements, requirements=requirements, @@ -1129,17 +1346,21 @@ def resolve( preserve_log=preserve_log, pip_version=pip_version, resolver=resolver, + exclude_configuration=exclude_configuration, ) + unexcluded_direct_requirements = OrderedSet() # type: OrderedSet[ParsedRequirement] + build_requests = [] # type: List[BuildRequest] install_requests = [] # type: List[InstallRequest] for download_result in download_results: + unexcluded_direct_requirements.update(download_result.unexcluded_direct_requirements) build_requests.extend(download_result.build_requests()) install_requests.extend(download_result.install_requests()) build_and_install_request = BuildAndInstallRequest( build_requests=build_requests, install_requests=install_requests, - direct_requirements=direct_requirements, + direct_requirements=unexcluded_direct_requirements, package_index_configuration=package_index_configuration, compile=compile, build_configuration=build_configuration, @@ -1177,8 +1398,9 @@ def _download_internal( preserve_log=False, # type: bool pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): - # type: (...) -> Tuple[List[BuildRequest], List[DownloadResult]] + # type: (...) -> List[DownloadResult] unique_targets = targets.unique_targets() download_request = DownloadRequest( @@ -1195,13 +1417,10 @@ def _download_internal( preserve_log=preserve_log, pip_version=pip_version, resolver=resolver, + max_parallel_jobs=max_parallel_jobs, + exclude_configuration=exclude_configuration, ) - - local_projects = list(download_request.iter_local_projects()) - download_results = download_request.download_distributions( - dest=dest, max_parallel_jobs=max_parallel_jobs - ) - return local_projects, download_results + return download_request.download_distributions(dest=dest, max_parallel_jobs=max_parallel_jobs) @attr.s(frozen=True) @@ -1255,6 +1474,7 @@ def download( pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> Downloaded """Downloads all distributions needed to meet requirements for multiple distribution targets. @@ -1300,7 +1520,7 @@ def download( password_entries=password_entries, use_pip_config=use_pip_config, ) - build_requests, download_results = _download_internal( + download_results = _download_internal( targets=targets, direct_requirements=direct_requirements, requirements=requirements, @@ -1316,24 +1536,19 @@ def download( preserve_log=preserve_log, pip_version=pip_version, resolver=resolver, + exclude_configuration=exclude_configuration, ) local_distributions = [] - - def add_build_requests(requests): - # type: (Iterable[BuildRequest]) -> None - for request in requests: + for download_result in download_results: + for build_request in download_result.build_requests(): local_distributions.append( LocalDistribution( - target=request.target, - path=request.source_path, - fingerprint=request.fingerprint, + target=build_request.target, + path=build_request.source_path, + fingerprint=build_request.fingerprint, ) ) - - add_build_requests(build_requests) - for download_result in download_results: - add_build_requests(download_result.build_requests()) for install_request in download_result.install_requests(): local_distributions.append( LocalDistribution( diff --git a/tests/test_dependency_manager.py b/tests/test_dependency_manager.py index 2195eb3d8..5fca935fd 100644 --- a/tests/test_dependency_manager.py +++ b/tests/test_dependency_manager.py @@ -9,6 +9,7 @@ from pex.dependency_manager import DependencyManager from pex.dist_metadata import Distribution, Requirement +from pex.exclude_configuration import ExcludeConfiguration from pex.fingerprinted_distribution import FingerprintedDistribution from pex.orderedset import OrderedSet from pex.pep_503 import ProjectName @@ -109,7 +110,9 @@ def test_exclude_root_reqs(dist_graph): pex_builder = PEXBuilder(pex_info=pex_info) with warnings.catch_warnings(record=True) as events: - dependency_manager.configure(pex_builder, excluded=["a", "b"]) + dependency_manager.configure( + pex_builder, exclude_configuration=ExcludeConfiguration.create(["a", "b"]) + ) assert 2 == len(events) warning = events[0] @@ -142,7 +145,9 @@ def test_exclude_complex(dist_graph): pex_info = PexInfo.default() pex_builder = PEXBuilder(pex_info=pex_info) - dependency_manager.configure(pex_builder, excluded=["c"]) + dependency_manager.configure( + pex_builder, exclude_configuration=ExcludeConfiguration.create(["c"]) + ) pex_builder.freeze() assert ["a", "b"] == list(pex_info.requirements)