From 9ba4171d042afe5d234facc6222abc860f3eae0f Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 7 Mar 2023 14:07:31 -0800 Subject: [PATCH] Fix lock subsetting for local projects. Previously local project indexing was broken for subset resolves and the operation would crash. Fixes #2083 --- pex/resolve/lock_resolver.py | 5 + pex/resolve/lockfile/model.py | 20 ++-- pex/resolver.py | 7 ++ .../cli/commands/test_issue_2057.py | 101 +++++++++++++++--- 4 files changed, 115 insertions(+), 18 deletions(-) diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index b1fb8abd1..2ed22af26 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -426,5 +426,10 @@ def resolve_from_lock( # `LockedResolve.resolve` above and need not waste time (~O(100ms)) doing this again. ignore_errors=True, max_parallel_jobs=max_parallel_jobs, + local_project_directory_to_sdist={ + downloadable_artifact.artifact.directory: downloaded_artifact.path + for downloadable_artifact, downloaded_artifact in downloaded_artifacts.items() + if isinstance(downloadable_artifact.artifact, LocalProjectArtifact) + }, ) return Installed(installed_distributions=tuple(installed_distributions)) diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py index 591fc5c4c..b78b886c3 100644 --- a/pex/resolve/lockfile/model.py +++ b/pex/resolve/lockfile/model.py @@ -10,6 +10,7 @@ from pex.pip.version import PipVersionValue from pex.requirements import LocalProjectRequirement from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem +from pex.resolve.resolved_requirement import Pin from pex.resolve.resolver_configuration import ResolverVersion from pex.sorted_tuple import SortedTuple from pex.typing import TYPE_CHECKING @@ -49,13 +50,17 @@ def create( ): # type: (...) -> Lockfile - pin_by_local_project_directory = { - locked_requirement.artifact.directory: locked_requirement.pin - for locked_resolve in locked_resolves - for locked_requirement in locked_resolve.locked_requirements - if isinstance(locked_requirement.artifact, LocalProjectArtifact) - } + pin_by_local_project_directory = {} # type: Dict[str, Pin] requirement_by_local_project_directory = {} # type: Dict[str, Requirement] + for locked_resolve in locked_resolves: + for locked_requirement in locked_resolve.locked_requirements: + if isinstance(locked_requirement.artifact, LocalProjectArtifact): + local_directory = locked_requirement.artifact.directory + local_pin = locked_requirement.pin + pin_by_local_project_directory[local_directory] = local_pin + requirement_by_local_project_directory[ + local_directory + ] = local_pin.as_requirement() def extract_requirement(req): # type: (Union[Requirement, ParsedRequirement]) -> Requirement @@ -74,6 +79,9 @@ def extract_requirement(req): marker="; {marker}".format(marker=req.marker) if req.marker else "", ) ) + # N.B.: We've already mapped all available local projects above, but the user may + # have supplied the local project requirement with more specific constraints ( + # extras and / or marker restrictions) and we need to honor those; so we over-write. requirement_by_local_project_directory[local_project_directory] = requirement return requirement return req.requirement diff --git a/pex/resolver.py b/pex/resolver.py index 608376301..e7f7e5d53 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -741,6 +741,7 @@ def install_distributions( self, ignore_errors=False, # type: bool max_parallel_jobs=None, # type: Optional[int] + local_project_directory_to_sdist=None, # type: Optional[Mapping[str, str]] ): # type: (...) -> Iterable[InstalledDistribution] if not any((self._build_requests, self._install_requests)): @@ -783,6 +784,12 @@ def iter_direct_requirements(): continue install_reqs = build_results.get(requirement.path) + if not install_reqs and local_project_directory_to_sdist: + local_project_directory = local_project_directory_to_sdist.get( + requirement.path + ) + if local_project_directory: + install_reqs = build_results.get(local_project_directory) if not install_reqs: raise AssertionError( "Failed to compute a project name for {requirement}. No corresponding " diff --git a/tests/integration/cli/commands/test_issue_2057.py b/tests/integration/cli/commands/test_issue_2057.py index a2b0e523c..f0540f3a9 100644 --- a/tests/integration/cli/commands/test_issue_2057.py +++ b/tests/integration/cli/commands/test_issue_2057.py @@ -4,10 +4,12 @@ import os.path import shutil import subprocess +import sys import tempfile from textwrap import dedent import colors +import pytest from pex.cli.testing import run_pex3 from pex.resolve.lockfile import json_codec @@ -15,15 +17,31 @@ from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Any + from typing import Any, Callable, List -def test_pex_archive_direct_reference(tmpdir): - # type: (Any) -> None +@pytest.mark.parametrize( + ["archive_pex_requirement"], + [ + pytest.param( + "https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip#egg=cowsay", + id="Create Pex [Pip Proprietary]", + ), + pytest.param( + "cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip", + id="Create Pex [PEP-508]", + ), + ], +) +def test_pex_archive_direct_reference( + tmpdir, # type: Any + archive_pex_requirement, # type: str +): + # type: (...) -> None result = run_pex_command( args=[ - "cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip", + archive_pex_requirement, "-c", "cowsay", "--", @@ -34,8 +52,39 @@ def test_pex_archive_direct_reference(tmpdir): assert "Moo!" in result.output -def test_lock_create_archive_direct_reference(tmpdir): - # type: (Any) -> None +@pytest.mark.parametrize( + ["archive_lock_requirement"], + [ + pytest.param( + "https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip#egg=cowsay", + id="Create Lock [Pip Proprietary]", + ), + pytest.param( + "cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip", + id="Create Lock [PEP-508]", + ), + ], +) +@pytest.mark.parametrize( + ["archive_pex_requirements"], + [ + pytest.param( + ["https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip#egg=cowsay"], + id="Subset [Pip Proprietary]", + ), + pytest.param( + ["cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip"], + id="Subset [PEP-508]", + ), + pytest.param([], id="Full"), + ], +) +def test_lock_create_archive_direct_reference( + tmpdir, # type: Any + archive_lock_requirement, # type: str + archive_pex_requirements, # type: List[str] +): + # type: (...) -> None pex_root = os.path.join(str(tmpdir), "pex_root") lock = os.path.join(str(tmpdir), "lock.json") @@ -44,7 +93,7 @@ def test_lock_create_archive_direct_reference(tmpdir): "create", "--pex-root", pex_root, - "cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip", + archive_lock_requirement, "--indent", "2", "-o", @@ -54,7 +103,8 @@ def test_lock_create_archive_direct_reference(tmpdir): def assert_create_and_run_pex_from_lock(): # type: () -> None result = run_pex_command( - args=[ + args=archive_pex_requirements + + [ "--pex-root", pex_root, "--runtime-pex-root", @@ -75,8 +125,31 @@ def assert_create_and_run_pex_from_lock(): assert_create_and_run_pex_from_lock() -def test_lock_create_local_project_direct_reference(tmpdir): - # type: (Any) -> None +@pytest.mark.parametrize( + ["create_local_project_lock_requirement"], + [ + pytest.param(lambda clone_dir: clone_dir, id="Create Lock [Pip Proprietary]"), + pytest.param( + lambda clone_dir: "ansicolors @ file://{}".format(clone_dir), id="Create Lock [PEP-508]" + ), + ], +) +@pytest.mark.parametrize( + ["create_local_project_pex_requirements"], + [ + pytest.param(lambda clone_dir: [clone_dir], id="Subset [Pip Proprietary]"), + pytest.param( + lambda clone_dir: ["ansicolors @ file://{}".format(clone_dir)], id="Subset [PEP-508]" + ), + pytest.param(lambda clone_dir: [], id="Full"), + ], +) +def test_lock_create_local_project_direct_reference( + tmpdir, # type: Any + create_local_project_lock_requirement, # type: Callable[[str], str] + create_local_project_pex_requirements, # type: Callable[[str], List[str]] +): + # type: (...) -> None clone_dir = os.path.join(str(tmpdir), "ansicolors") subprocess.check_call(args=["git", "init", clone_dir]) @@ -95,6 +168,9 @@ def test_lock_create_local_project_direct_reference(tmpdir): ) subprocess.check_call(args=["git", "reset", "--hard", ansicolors_1_1_8_sha], cwd=clone_dir) + lock_requirement = create_local_project_lock_requirement(clone_dir) + pex_requirements = create_local_project_pex_requirements(clone_dir) + pex_root = os.path.join(str(tmpdir), "pex_root") lock = os.path.join(str(tmpdir), "lock.json") run_pex3( @@ -102,7 +178,7 @@ def test_lock_create_local_project_direct_reference(tmpdir): "create", "--pex-root", pex_root, - "ansicolors @ file://{}".format(clone_dir), + lock_requirement, "--indent", "2", "-o", @@ -112,7 +188,8 @@ def test_lock_create_local_project_direct_reference(tmpdir): def assert_create_and_run_pex_from_lock(): # type: () -> None result = run_pex_command( - args=[ + args=pex_requirements + + [ "--pex-root", pex_root, "--runtime-pex-root",