Skip to content

Commit

Permalink
Deep exclude working, but unit tests need re-work.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsirois
jsirois committed May 6, 2024
1 parent 41ad944 commit 8745c05
Show file tree
Hide file tree
Showing 14 changed files with 365 additions and 143 deletions.
5 changes: 4 additions & 1 deletion pex/bin/pex.py
View file
Expand Up @@ -27,6 +27,7 @@
from pex.dependency_manager import DependencyManager
from pex.docs.command import serve_html_docs
from pex.enum import Enum
from pex.exclude_configuration import ExcludeConfiguration
from pex.inherit_path import InheritPath
from pex.interpreter_constraints import InterpreterConstraints
from pex.layout import Layout, ensure_installed
Expand Down Expand Up @@ -897,6 +898,7 @@ def build_pex(
)
excluded.extend(requirements_pex_info.excluded)

exclude_configuration = ExcludeConfiguration.create(excluded)
with TRACER.timed(
"Resolving distributions for requirements: {}".format(
" ".join(
Expand All @@ -922,13 +924,14 @@ def build_pex(
if options.pre_install_wheels
else InstallableType.WHEEL_FILE
),
exclude_configuration=exclude_configuration,
)
)
except Unsatisfiable as e:
die(str(e))

with TRACER.timed("Configuring PEX dependencies"):
dependency_manager.configure(pex_builder, excluded=excluded)
dependency_manager.configure(pex_builder, exclude_configuration=exclude_configuration)

if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
Expand Down
6 changes: 3 additions & 3 deletions pex/cli/commands/lock.py
View file
Expand Up @@ -12,7 +12,6 @@

from pex import pex_warnings
from pex.argparse import HandleBoolAction
from pex.asserts import production_assert
from pex.cli.command import BuildTimeCommand
from pex.commands.command import JsonMixin, OutputMixin
from pex.common import is_exe, pluralize, safe_delete, safe_open
Expand All @@ -25,6 +24,7 @@
RequirementParseError,
)
from pex.enum import Enum
from pex.exceptions import production_assert
from pex.interpreter import PythonInterpreter
from pex.pep_376 import InstalledWheel, Record
from pex.pep_427 import InstallableType
Expand Down Expand Up @@ -1103,7 +1103,7 @@ def _process_lock_update(
updates = [] # type: List[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]
warnings = [] # type: List[str]
for resolve_update in lock_update.resolves:
platform = resolve_update.updated_resolve.platform_tag or "universal"
platform = resolve_update.updated_resolve.target_platform
if not resolve_update.updates:
print(
"No updates for lock generated by {platform}.".format(platform=platform),
Expand Down Expand Up @@ -1468,7 +1468,7 @@ def _sync(self):
"Would lock {count} {project} for platform {platform}:".format(
count=len(locked_resolve.locked_requirements),
project=pluralize(locked_resolve.locked_requirements, "project"),
platform=locked_resolve.platform_tag or "universal",
platform=locked_resolve.target_platform,
),
file=output,
)
Expand Down
68 changes: 4 additions & 64 deletions pex/dependency_manager.py
View file
Expand Up @@ -3,24 +3,17 @@

from __future__ import absolute_import

from collections import defaultdict

from pex import pex_warnings
from pex.dist_metadata import Requirement
from pex.environment import PEXEnvironment
from pex.exclude_configuration import ExcludeConfiguration
from pex.fingerprinted_distribution import FingerprintedDistribution
from pex.orderedset import OrderedSet
from pex.pep_503 import ProjectName
from pex.pex_builder import PEXBuilder
from pex.pex_info import PexInfo
from pex.resolve.resolvers import ResolveResult
from pex.tracer import TRACER
from pex.typing import TYPE_CHECKING

if TYPE_CHECKING:
from typing import DefaultDict, Iterable, Iterator

import attr # vendor:skip
else:
from pex.third_party import attr
Expand Down Expand Up @@ -58,67 +51,14 @@ def add_from_resolved(self, resolved):
def configure(
self,
pex_builder, # type: PEXBuilder
excluded=(), # type: Iterable[str]
exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration
):
# type: (...) -> None

exclude_configuration = ExcludeConfiguration.create(excluded)
exclude_configuration.configure(pex_builder.info)

dists_by_project_name = defaultdict(
OrderedSet
) # type: DefaultDict[ProjectName, OrderedSet[FingerprintedDistribution]]
for dist in self._distributions:
dists_by_project_name[dist.distribution.metadata.project_name].add(dist)

root_requirements_by_project_name = defaultdict(
OrderedSet
) # type: DefaultDict[ProjectName, OrderedSet[Requirement]]
for root_req in self._requirements:
root_requirements_by_project_name[root_req.project_name].add(root_req)

def iter_non_excluded_distributions(requirements):
# type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution]
for req in requirements:
candidate_dists = dists_by_project_name[req.project_name]
for candidate_dist in tuple(candidate_dists):
if candidate_dist.distribution not in req:
continue
candidate_dists.discard(candidate_dist)

excluded_by = exclude_configuration.excluded_by(candidate_dist.distribution)
if excluded_by:
excludes = " and ".join(map(str, excluded_by))
TRACER.log(
"Skipping adding {candidate}: excluded by {excludes}".format(
candidate=candidate_dist.distribution, excludes=excludes
)
)
for root_req in root_requirements_by_project_name[
candidate_dist.distribution.metadata.project_name
]:
if candidate_dist.distribution in root_req:
pex_warnings.warn(
"The distribution {dist} was required by the input requirement "
"{root_req} but excluded by configured excludes: "
"{excludes}".format(
dist=candidate_dist.distribution,
root_req=root_req,
excludes=excludes,
)
)
continue

yield candidate_dist
for dep in iter_non_excluded_distributions(
candidate_dist.distribution.requires()
):
yield dep

for fingerprinted_dist in iter_non_excluded_distributions(self._requirements):
for requirement in self._requirements:
pex_builder.add_requirement(requirement)
for fingerprinted_dist in self._distributions:
pex_builder.add_distribution(
dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint
)

for requirement in self._requirements:
pex_builder.add_requirement(requirement)
20 changes: 17 additions & 3 deletions pex/environment.py
View file
Expand Up @@ -459,8 +459,12 @@ def _resolve_requirement(
):
yield not_found

def _root_requirements_iter(self, reqs):
# type: (Iterable[Requirement]) -> Iterator[QualifiedRequirementOrNotFound]
def _root_requirements_iter(
self,
reqs, # type: Iterable[Requirement]
exclude_configuration, # type: ExcludeConfiguration
):
# type: (...) -> Iterator[QualifiedRequirementOrNotFound]

# We want to pick one requirement for each key (required project) to then resolve
# recursively.
Expand All @@ -477,6 +481,16 @@ def _root_requirements_iter(self, reqs):
OrderedDict()
) # type: OrderedDict[ProjectName, List[_QualifiedRequirement]]
for req in reqs:
excluded_by = exclude_configuration.excluded_by(req)
if excluded_by:
TRACER.log(
"Skipping resolving {requirement}: excluded by {excludes}".format(
requirement=req,
excludes=" and ".join(map(str, excluded_by)),
)
)
continue

required = self._evaluate_marker(req)
if not required:
continue
Expand Down Expand Up @@ -591,7 +605,7 @@ def record_unresolved(dist_not_found):
resolved_dists_by_key = (
OrderedDict()
) # type: OrderedDict[_RequirementKey, FingerprintedDistribution]
for qualified_req_or_not_found in self._root_requirements_iter(reqs):
for qualified_req_or_not_found in self._root_requirements_iter(reqs, exclude_configuration):
if isinstance(qualified_req_or_not_found, _DistributionNotFound):
record_unresolved(qualified_req_or_not_found)
continue
Expand Down
37 changes: 22 additions & 15 deletions pex/asserts.py → pex/exceptions.py
View file
Expand Up @@ -13,11 +13,11 @@
_ASSERT_DETAILS = (
dedent(
"""\
Pex {version}
platform: {platform}
python: {python_version}
argv: {argv}
"""
Pex {version}
platform: {platform}
python: {python_version}
argv: {argv}
"""
)
.format(
version=__version__, platform=platform.platform(), python_version=sys.version, argv=sys.argv
Expand All @@ -27,23 +27,20 @@

_ASSERT_ADVICE = dedent(
"""\
The error reported above resulted from an unexpected programming error which
you should never encounter.
The error reported above resulted from an unexpected error which you should
never encounter.
Firstly, please accept our apology!
If you could file an issue with the error and details above, we'd be
grateful. You can do that at https://github.com/pex-tool/pex/issues/new and
redact or amend any details that expose sensitive information.
"""
).strip()


def production_assert(condition, msg=""):
# type: (...) -> None

if condition:
return
def reportable_unexpected_error_msg(msg=""):
# type: (str) -> str

message = [msg, "---", _ASSERT_DETAILS]
pex = os.environ.get("PEX")
Expand All @@ -64,4 +61,14 @@ def production_assert(condition, msg=""):
message.append("---")
message.append(_ASSERT_ADVICE)

raise AssertionError("\n".join(message))
return "\n".join(message)


def production_assert(
condition, # type: bool
msg="", # type: str
):
# type: (...) -> None

if not condition:
raise AssertionError(reportable_unexpected_error_msg(msg=msg))
8 changes: 6 additions & 2 deletions pex/exclude_configuration.py
View file
Expand Up @@ -22,15 +22,19 @@ def create(cls, excluded):
# type: (Iterable[str]) -> ExcludeConfiguration
return cls(excluded=tuple(Requirement.parse(req) for req in excluded))

_excluded = attr.ib(factory=tuple) # type: Tuple[Requirement, ...]
_excluded = attr.ib(default=()) # type: Tuple[Requirement, ...]

def configure(self, pex_info):
# type: (PexInfo) -> None
for excluded in self._excluded:
pex_info.add_excluded(excluded)

def excluded_by(self, item):
# type: (Union[Distribution, Requirement]) -> Iterable[Requirement]
# type: (Union[Distribution, Requirement]) -> Tuple[Requirement, ...]
if isinstance(item, Distribution):
return tuple(req for req in self._excluded if item in req)
return tuple(req for req in self._excluded if item.project_name == req.project_name)

def __bool__(self):
# type: () -> bool
return bool(self._excluded)
4 changes: 2 additions & 2 deletions pex/fetcher.py
View file
Expand Up @@ -12,7 +12,6 @@
from contextlib import closing, contextmanager
from ssl import SSLContext

from pex import asserts
from pex.auth import PasswordDatabase, PasswordEntry
from pex.compatibility import (
FileHandler,
Expand All @@ -26,6 +25,7 @@
build_opener,
in_main_thread,
)
from pex.exceptions import production_assert
from pex.network_configuration import NetworkConfiguration
from pex.typing import TYPE_CHECKING, cast
from pex.version import __version__
Expand Down Expand Up @@ -102,7 +102,7 @@ def create_ssl_context(self):
# [^5]: https://github.com/openssl/openssl/blob/c3cc0f1386b0544383a61244a4beeb762b67498f/ssl/ssl_init.c#L86-L116
# [^6]: https://github.com/indygreg/python-build-standalone/releases/tag/20240107
# [^7]: https://gitlab.com/redhat-crypto/fedora-crypto-policies/-/merge_requests/110/diffs#269a48e71ac25ad1d07ff00db2390834c8ba7596_11_16
asserts.production_assert(
production_assert(
in_main_thread(),
msg=(
"An SSLContext must be initialized from the main thread. An attempt was made to "
Expand Down
5 changes: 5 additions & 0 deletions pex/resolve/configured_resolve.py
View file
Expand Up @@ -3,6 +3,7 @@

from __future__ import absolute_import

from pex.exclude_configuration import ExcludeConfiguration
from pex.pep_427 import InstallableType
from pex.resolve.configured_resolver import ConfiguredResolver
from pex.resolve.lock_resolver import resolve_from_lock
Expand Down Expand Up @@ -30,6 +31,7 @@ def resolve(
compile_pyc=False, # type: bool
ignore_errors=False, # type: bool
result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value
exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration
):
# type: (...) -> ResolveResult
if isinstance(resolver_configuration, LockRepositoryConfiguration):
Expand Down Expand Up @@ -58,6 +60,7 @@ def resolve(
pip_version=lock.pip_version,
use_pip_config=pip_configuration.use_pip_config,
result_type=result_type,
exclude_configuration=exclude_configuration,
)
)
elif isinstance(resolver_configuration, PexRepositoryConfiguration):
Expand All @@ -76,6 +79,7 @@ def resolve(
transitive=resolver_configuration.transitive,
ignore_errors=ignore_errors,
result_type=result_type,
exclude_configuration=exclude_configuration,
)
else:
with TRACER.timed("Resolving requirements."):
Expand All @@ -100,4 +104,5 @@ def resolve(
resolver=ConfiguredResolver(pip_configuration=resolver_configuration),
use_pip_config=resolver_configuration.use_pip_config,
result_type=result_type,
exclude_configuration=exclude_configuration,
)
3 changes: 3 additions & 0 deletions pex/resolve/lock_resolver.py
View file
Expand Up @@ -14,6 +14,7 @@
from pex.auth import PasswordDatabase, PasswordEntry
from pex.common import pluralize
from pex.compatibility import cpu_count
from pex.exclude_configuration import ExcludeConfiguration
from pex.network_configuration import NetworkConfiguration
from pex.orderedset import OrderedSet
from pex.pep_427 import InstallableType
Expand Down Expand Up @@ -247,6 +248,7 @@ def resolve_from_lock(
pip_version=None, # type: Optional[PipVersionValue]
use_pip_config=False, # type: bool
result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value
exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration
):
# type: (...) -> Union[ResolveResult, Error]

Expand All @@ -262,6 +264,7 @@ def resolve_from_lock(
network_configuration=network_configuration,
build_configuration=build_configuration,
transitive=transitive,
exclude_configuration=exclude_configuration,
)
)
downloadable_artifacts_and_targets = OrderedSet(
Expand Down

0 comments on commit 8745c05

Please sign in to comment.