Fix lock update constraint handling.

Previously contraints were taken directly from requirement strings which
led to invalid constraints for requirements with extras or a direct
reference URL. Introduce a distinct `Constraint` type that conforms to
Pip's constraint expectations and fix up the lock model and lock updates
to use `Constraints` to ensure Pip's expectations are met.

pex/cli/commands/lock.py

@@ -17,7 +17,13 @@
 from pex.commands.command import JsonMixin, OutputMixin
 from pex.common import is_exe, pluralize, safe_delete, safe_open
 from pex.compatibility import commonpath, shlex_quote
-from pex.dist_metadata import Distribution, MetadataType, Requirement, RequirementParseError
+from pex.dist_metadata import (
+    Constraint,
+    Distribution,
+    MetadataType,
+    Requirement,
+    RequirementParseError,
+)
 from pex.enum import Enum
 from pex.interpreter import PythonInterpreter
 from pex.pep_376 import InstalledWheel, Record
@@ -1158,7 +1164,7 @@ def _process_lock_update(
                         # grab the latest version in the range already constrained by an existing
                         # requirement or constraint.
                         if update_req and str(update_req) != update_req.name:
-                            constraints_by_project_name[project_name] = update_req
+                            constraints_by_project_name[project_name] = update_req.as_constraint()
                     else:
                         print(
                             "  {lead_in} {project_name} {updated_version}".format(
@@ -1222,8 +1228,8 @@ def _process_lock_update(
                 )
 
         def process_req_edit(
-            original,  # type: Optional[Requirement]
-            final,  # type: Optional[Requirement]
+            original,  # type: Optional[Constraint]
+            final,  # type: Optional[Constraint]
         ):
             # type: (...) -> None
             if not original:
@@ -1254,11 +1260,11 @@ def process_req_edit(
 
         def process_req_edits(
             requirement_type,  # type: str
-            original,  # type: Mapping[ProjectName, Requirement]
-            final,  # type: Mapping[ProjectName, Requirement]
+            original,  # type: Mapping[ProjectName, Constraint]
+            final,  # type: Mapping[ProjectName, Constraint]
         ):
-            # type: (...) -> Tuple[Tuple[Optional[Requirement], Optional[Requirement]], ...]
-            edits = []  # type: List[Tuple[Optional[Requirement], Optional[Requirement]]]
+            # type: (...) -> Tuple[Tuple[Optional[Constraint], Optional[Constraint]], ...]
+            edits = []  # type: List[Tuple[Optional[Constraint], Optional[Constraint]]]
             for name, original_req in original.items():
                 final_req = final.get(name)
                 if final_req != original_req:

pex/dist_metadata.py

@@ -626,29 +626,23 @@ class RequirementParseError(Exception):
 
 
 @attr.s(frozen=True)
-class Requirement(object):
+class Constraint(object):
     @classmethod
-    def parse(cls, requirement):
-        # type: (Text) -> Requirement
+    def parse(cls, constraint):
+        # type: (Text) -> Constraint
         try:
-            return cls.from_packaging_requirement(PackagingRequirement(requirement))
+            return cls.from_packaging_requirement(PackagingRequirement(constraint))
         except InvalidRequirement as e:
             raise RequirementParseError(str(e))
 
     @classmethod
     def from_packaging_requirement(cls, requirement):
-        # type: (PackagingRequirement) -> Requirement
+        # type: (PackagingRequirement) -> Constraint
         return cls(
-            name=requirement.name,
-            url=requirement.url,
-            extras=frozenset(requirement.extras),
-            specifier=requirement.specifier,
-            marker=requirement.marker,
+            name=requirement.name, specifier=requirement.specifier, marker=requirement.marker
         )
 
     name = attr.ib(eq=False)  # type: str
-    url = attr.ib(default=None)  # type: Optional[str]
-    extras = attr.ib(default=frozenset())  # type: FrozenSet[str]
     specifier = attr.ib(factory=SpecifierSet)  # type: SpecifierSet
     marker = attr.ib(default=None, eq=str)  # type: Optional[Marker]
 
@@ -657,17 +651,12 @@ def from_packaging_requirement(cls, requirement):
     _legacy_version = attr.ib(init=False, repr=False)  # type: Optional[str]
 
     def __attrs_post_init__(self):
+        # type: () -> None
         object.__setattr__(self, "project_name", ProjectName(self.name))
 
         parts = [self.name]
-        if self.extras:
-            parts.append("[{extras}]".format(extras=",".join(sorted(self.extras))))
         if self.specifier:
             parts.append(str(self.specifier))
-        if self.url:
-            parts.append("@ {url}".format(url=self.url))
-            if self.marker:
-                parts.append(" ")
         if self.marker:
             parts.append("; {marker}".format(marker=self.marker))
         object.__setattr__(self, "_str", "".join(parts))
@@ -744,6 +733,52 @@ def __str__(self):
         return self._str
 
 
+@attr.s(frozen=True)
+class Requirement(Constraint):
+    @classmethod
+    def parse(cls, requirement):
+        # type: (Text) -> Requirement
+        try:
+            return cls.from_packaging_requirement(PackagingRequirement(requirement))
+        except InvalidRequirement as e:
+            raise RequirementParseError(str(e))
+
+    @classmethod
+    def from_packaging_requirement(cls, requirement):
+        # type: (PackagingRequirement) -> Requirement
+        return cls(
+            name=requirement.name,
+            url=requirement.url,
+            extras=frozenset(requirement.extras),
+            specifier=requirement.specifier,
+            marker=requirement.marker,
+        )
+
+    url = attr.ib(default=None)  # type: Optional[str]
+    extras = attr.ib(default=frozenset())  # type: FrozenSet[str]
+
+    def __attrs_post_init__(self):
+        # type: () -> None
+        super(Requirement, self).__attrs_post_init__()
+
+        parts = [self.name]
+        if self.extras:
+            parts.append("[{extras}]".format(extras=",".join(sorted(self.extras))))
+        if self.specifier:
+            parts.append(str(self.specifier))
+        if self.url:
+            parts.append("@ {url}".format(url=self.url))
+            if self.marker:
+                parts.append(" ")
+        if self.marker:
+            parts.append("; {marker}".format(marker=self.marker))
+        object.__setattr__(self, "_str", "".join(parts))
+
+    def as_constraint(self):
+        # type: () -> Constraint
+        return Constraint(name=self.name, specifier=self.specifier, marker=self.marker)
+
+
 # N.B.: DistributionMetadata can have an expensive hash when a distribution has many requirements;
 # so we cache the hash. See: https://github.com/pex-tool/pex/issues/1928
 @attr.s(frozen=True, cache_hash=True)

pex/resolve/lockfile/create.py

@@ -355,7 +355,7 @@ def create(
     network_configuration = pip_configuration.network_configuration
     parsed_requirements = tuple(requirement_configuration.parse_requirements(network_configuration))
     constraints = tuple(
-        parsed_constraint.requirement
+        parsed_constraint.requirement.as_constraint()
         for parsed_constraint in requirement_configuration.parse_constraints(network_configuration)
     )
 

pex/resolve/lockfile/json_codec.py

@@ -225,7 +225,9 @@ def parse_version_specifier(
     ]
 
     constraints = [
-        parse_requirement(constraint, path=".constraints[{index}]".format(index=index))
+        parse_requirement(
+            constraint, path=".constraints[{index}]".format(index=index)
+        ).as_constraint()
         for index, constraint in enumerate(get("constraints", list))
     ]
 

pex/resolve/lockfile/model.py

@@ -5,7 +5,7 @@
 
 import os
 
-from pex.dist_metadata import Requirement
+from pex.dist_metadata import Constraint, Requirement
 from pex.orderedset import OrderedSet
 from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersion, PipVersionValue
@@ -36,7 +36,7 @@ def create(
         requires_python,  # type: Iterable[str]
         target_systems,  # type: Iterable[TargetSystem.Value]
         requirements,  # type: Iterable[Union[Requirement, ParsedRequirement]]
-        constraints,  # type: Iterable[Requirement]
+        constraints,  # type: Iterable[Constraint]
         allow_prereleases,  # type: bool
         build_configuration,  # type: BuildConfiguration
         transitive,  # type: bool
@@ -116,7 +116,7 @@ def extract_requirement(req):
     pip_version = attr.ib()  # type: PipVersionValue
     resolver_version = attr.ib()  # type: ResolverVersion.Value
     requirements = attr.ib()  # type: SortedTuple[Requirement]
-    constraints = attr.ib()  # type: SortedTuple[Requirement]
+    constraints = attr.ib()  # type: SortedTuple[Constraint]
     allow_prereleases = attr.ib()  # type: bool
     allow_wheels = attr.ib()  # type: bool
     only_wheels = attr.ib()  # type: SortedTuple[ProjectName]

pex/resolve/lockfile/updater.py

@@ -10,7 +10,7 @@
 from contextlib import contextmanager
 
 from pex.common import pluralize
-from pex.dist_metadata import Requirement
+from pex.dist_metadata import Constraint, Requirement
 from pex.network_configuration import NetworkConfiguration
 from pex.orderedset import OrderedSet
 from pex.pep_440 import Version
@@ -250,11 +250,11 @@ def derived(
 
         original_constraints_by_project_name = OrderedDict(
             (constraint.project_name, constraint) for constraint in lock_file.constraints
-        )  # type: OrderedDict[ProjectName, Requirement]
+        )  # type: OrderedDict[ProjectName, Constraint]
 
         update_constraints_by_project_name = (
             OrderedDict()
-        )  # type: OrderedDict[ProjectName, Requirement]
+        )  # type: OrderedDict[ProjectName, Constraint]
         for requirement in replace_requirements:
             project_name = requirement.project_name
             original_constraint = original_constraints_by_project_name.get(project_name)
@@ -264,7 +264,7 @@ def derived(
                         original=original_constraint, override=requirement
                     )
                 )
-            update_constraints_by_project_name[project_name] = requirement
+            update_constraints_by_project_name[project_name] = requirement.as_constraint()
 
         original_requirements = OrderedDict(
             (requirement.project_name, requirement) for requirement in lock_file.requirements
@@ -302,11 +302,11 @@ def specified(
 
         original_constraints_by_project_name = OrderedDict(
             (constraint.project_name, constraint) for constraint in lock_file.constraints
-        )  # type: OrderedDict[ProjectName, Requirement]
+        )  # type: OrderedDict[ProjectName, Constraint]
 
         update_constraints_by_project_name = (
             OrderedDict()
-        )  # type: OrderedDict[ProjectName, Requirement]
+        )  # type: OrderedDict[ProjectName, Constraint]
         for change in itertools.chain(updates, replacements):
             project_name = change.project_name
             original_constraint = original_constraints_by_project_name.get(project_name)
@@ -316,7 +316,7 @@ def specified(
                         original=original_constraint, override=change
                     )
                 )
-            update_constraints_by_project_name[project_name] = change
+            update_constraints_by_project_name[project_name] = change.as_constraint()
 
         return cls(
             requirement_configuration=RequirementConfiguration(
@@ -332,7 +332,7 @@ def specified(
 
     requirement_configuration = attr.ib()  # type: RequirementConfiguration
     original_requirements = attr.ib()  # type: Iterable[Requirement]
-    update_constraints_by_project_name = attr.ib()  # type: Mapping[ProjectName, Requirement]
+    update_constraints_by_project_name = attr.ib()  # type: Mapping[ProjectName, Constraint]
     deletes = attr.ib()  # type: Container[ProjectName]
     lock_configuration = attr.ib()  # type: LockConfiguration
     pip_configuration = attr.ib()  # type: PipConfiguration

tests/integration/cli/commands/test_lock.py

@@ -12,7 +12,7 @@
 import pytest
 
 from pex.common import safe_open
-from pex.dist_metadata import Requirement
+from pex.dist_metadata import Constraint, Requirement
 from pex.interpreter import PythonInterpreter
 from pex.interpreter_constraints import InterpreterConstraint
 from pex.pep_440 import Version
@@ -739,7 +739,7 @@ def test_update_targeted_upgrade(lock_file_path):
     )
 
     lock_file = json_codec.load(lock_file_path)
-    assert SortedTuple([Requirement.parse("urllib3<1.26.7")]) == lock_file.constraints
+    assert SortedTuple([Constraint.parse("urllib3<1.26.7")]) == lock_file.constraints
     assert 1 == len(lock_file.locked_resolves)
     locked_resolve = lock_file.locked_resolves[0]
     assert 5 == len(locked_resolve.locked_requirements)

tests/integration/cli/commands/test_lock_sync.py

@@ -21,7 +21,13 @@
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersion
-from pex.resolve.locked_resolve import Artifact, FileArtifact, LockedResolve, LockStyle
+from pex.resolve.locked_resolve import (
+    Artifact,
+    FileArtifact,
+    LockedRequirement,
+    LockedResolve,
+    LockStyle,
+)
 from pex.resolve.lockfile import json_codec
 from pex.resolve.lockfile.model import Lockfile
 from pex.resolve.path_mappings import PathMapping, PathMappings
@@ -32,6 +38,7 @@
 from pex.venv.virtualenv import Virtualenv
 from testing import (
     IS_PYPY,
+    IS_X86_64,
     PY39,
     PY310,
     PY_VER,
@@ -44,7 +51,7 @@
 from testing.find_links import FindLinksRepo
 
 if TYPE_CHECKING:
-    from typing import Any, Iterable, List, Mapping, Optional, Union
+    from typing import Any, Dict, Iterable, List, Mapping, Optional, Union
 
     import attr  # vendor:skip
 else:
@@ -1273,8 +1280,11 @@ def assert_p537_lock(
 
 
 skip_unless_p537_compatible = pytest.mark.skipif(
-    PY_VER < (3, 6) or PY_VER >= (3, 13) or IS_PYPY,
-    reason="The p537 1.0.7 release only supports CPython >=3.6,<3.13",
+    PY_VER < (3, 6) or PY_VER >= (3, 13) or IS_PYPY or not IS_X86_64,
+    reason=(
+        "The p537 1.0.7 release only supports CPython >=3.6,<3.13 and only has published wheels "
+        "for Linux and Mac x86_64",
+    ),
 )
 
 
@@ -1421,3 +1431,33 @@ def test_sync_universal_to_universal(tmpdir):
     assert p537_py310.pin == p537_py311.pin
     assert len(p537_py311.artifacts_by_tag) == 4
     assert p537_py310_sdist == p537_py311.artifacts_by_tag[None]
+
+
+@pytest.mark.skipif(PY_VER < (3, 6), reason="The shiv 1.0.5 release requires Python >=3.6.")
+def test_sync_extras(tmpdir):
+    # type: (Any) -> None
+
+    # Verify requirements with extras work with the lock update constraint system.
+
+    lock = os.path.join(str(tmpdir), "lock.json")
+
+    def collect_locked_requirements():
+        # type: () -> Dict[ProjectName, LockedRequirement]
+        locked_resolves = json_codec.load(lock).locked_resolves
+        assert len(locked_resolves) == 1
+        return {
+            locked_requirement.pin.project_name: locked_requirement
+            for locked_requirement in locked_resolves[0].locked_requirements
+        }
+
+    run_pex3("lock", "sync", "shiv==1.0.5", "--indent", "2", "--lock", lock).assert_success()
+    original_locked_requirements = collect_locked_requirements()
+    assert ProjectName("sphinx-click") not in original_locked_requirements
+
+    run_pex3("lock", "sync", "shiv[rtd]==1.0.5", "--indent", "2", "--lock", lock).assert_success()
+    updated_locked_requirements = collect_locked_requirements()
+
+    for project_name, locked_requirement in original_locked_requirements.items():
+        assert locked_requirement == updated_locked_requirements.pop(project_name)
+
+    assert ProjectName("sphinx-click") in updated_locked_requirements

tests/resolve/lockfile/test_json_codec.py

@@ -10,7 +10,7 @@
 import pytest
 
 from pex.compatibility import PY2
-from pex.dist_metadata import Requirement
+from pex.dist_metadata import Constraint, Requirement
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersion
@@ -50,7 +50,7 @@ def test_roundtrip(tmpdir):
             Requirement.parse("ansicolors"),
             Requirement.parse("requests>=2; sys_platform == 'darwin'"),
         ),
-        constraints=(Requirement.parse("ansicolors==1.1.8"),),
+        constraints=(Constraint.parse("ansicolors==1.1.8"),),
         allow_prereleases=True,
         build_configuration=BuildConfiguration(),
         transitive=False,