Allow --platform resolves for current interpreter. (#1364)

Previously, if a `--platform` resolve was being executed with an
interpreter that matched the platform, a regular full-featured
interpreter resolve was performed. This prevented, for example,
resolving using a CPython 3.8 interpreter on a manylinux2014 capable
host for deployment to a host that is only manylinux2010 capable. The
`--resolve-local-platforms` option exists to force this sort of
fallback to a full-featured interpreter resolve when that is desired.

Fixes #1355

pex/distribution_target.py

@@ -18,6 +18,12 @@
 class DistributionTarget(object):
     """Represents the target of a python distribution."""
 
+    class AmbiguousTargetError(ValueError):
+        pass
+
+    class ManylinuxOutOfContextError(ValueError):
+        pass
+
     @classmethod
     def current(cls):
         # type: () -> DistributionTarget
@@ -45,25 +51,51 @@ def __init__(
     ):
         # type: (...) -> None
         if interpreter and platform:
-            raise ValueError(
+            raise self.AmbiguousTargetError(
                 "A {class_name} can represent an interpreter or a platform but not both at the "
                 "same time. Given interpreter {interpreter} and platform {platform}.".format(
                     class_name=self.__class__.__name__, interpreter=interpreter, platform=platform
                 )
             )
+        if not interpreter and not platform:
+            interpreter = PythonInterpreter.get()
         if manylinux and not platform:
-            raise ValueError(
+            raise self.ManylinuxOutOfContextError(
                 "A value for manylinux only makes sense for platform distribution targets. Given "
                 "manylinux={!r} but no platform.".format(manylinux)
             )
         self._interpreter = interpreter
         self._platform = platform
         self._manylinux = manylinux
 
+    @property
+    def is_platform(self):
+        # type: () -> bool
+        """Is the distribution target a platform specification.
+
+        N.B.: This value will always be the opposite of `is_interpreter` since a distribution target
+        can only encapsulate either a platform specification or a local interpreter.
+        """
+        return self._platform is not None
+
+    @property
+    def is_interpreter(self):
+        # type: () -> bool
+        """Is the distribution target a local interpreter.
+
+        N.B.: This value will always be the opposite of `is_platform` since a distribution target
+        can only encapsulate either a platform specification or a local interpreter.
+        """
+        return self._interpreter is not None
+
     @property
     def is_foreign(self):
         # type: () -> bool
-        if self._platform is None:
+        """Does the distribution target represent a foreign platform.
+
+        A foreign platform is one not matching the current interpreter.
+        """
+        if self.is_interpreter:
             return False
         return self._platform not in self.get_interpreter().supported_platforms
 
@@ -73,7 +105,7 @@ def get_interpreter(self):
 
     def get_python_version_str(self):
         # type: () -> Optional[str]
-        if self._platform is not None:
+        if self.is_platform:
             return None
         return self.get_interpreter().identity.version_str
 
@@ -106,8 +138,9 @@ def requirement_applies(
         if requirement.marker is None:
             return True
 
-        if self._platform is not None:
-            # We can have no opinion for foreign platforms.
+        if not self.is_interpreter:
+            # We can have no opinion without an interpreter to answer questions about enviornment
+            # markers.
             return None
 
         if not extras:
@@ -126,15 +159,15 @@ def requirement_applies(
     def id(self):
         # type: () -> str
         """A unique id for this distribution target suitable as a path name component."""
-        if self._platform is None:
+        if self.is_interpreter:
             interpreter = self.get_interpreter()
             return interpreter.binary.replace(os.sep, ".").lstrip(".")
         else:
             return str(self._platform)
 
     def __repr__(self):
         # type: () -> str
-        if self._platform is None:
+        if self.is_interpreter:
             return "{}(interpreter={!r})".format(self.__class__.__name__, self.get_interpreter())
         else:
             return "{}(platform={!r})".format(self.__class__.__name__, self._platform)

pex/interpreter.py

@@ -509,6 +509,7 @@ def latest_release_of_min_compatible_version(interps):
 
     @classmethod
     def get(cls):
+        # type: () -> PythonInterpreter
         return cls.from_binary(sys.executable)
 
     @staticmethod

pex/pex_bootstrapper.py

@@ -26,6 +26,7 @@
         List,
         NoReturn,
         Optional,
+        Set,
         Tuple,
         Union,
     )
@@ -73,7 +74,7 @@ def iter_compatible_interpreters(
 
     def _iter_interpreters():
         # type: () -> Iterator[InterpreterOrError]
-        seen = set()
+        seen = set()  # type: Set[InterpreterOrError]
 
         normalized_paths = (
             OrderedSet(PythonInterpreter.canonicalize_path(p) for p in path.split(os.pathsep))

pex/pip.py

@@ -414,14 +414,14 @@ def spawn_download_distributions(
                     "Cannot both ignore wheels (use_wheel=False) and refrain from building "
                     "distributions (build=False)."
                 )
-            elif target.is_foreign:
+            elif target.is_platform:
                 raise ValueError(
-                    "Cannot ignore wheels (use_wheel=False) when resolving for a foreign "
-                    "platform: {}".format(platform)
+                    "Cannot ignore wheels (use_wheel=False) when resolving for a platform: "
+                    "{}".format(platform)
                 )
 
         download_cmd = ["download", "--dest", download_dir]
-        if target.is_foreign:
+        if target.is_platform:
             # We're either resolving for a different host / platform or a different interpreter for
             # the current platform that we have no access to; so we need to let pip know and not
             # otherwise pickup platform info from the interpreter we execute pip with.
@@ -435,7 +435,7 @@ def spawn_download_distributions(
                 )
             )
 
-        if target.is_foreign or not build:
+        if target.is_platform or not build:
             download_cmd.extend(["--only-binary", ":all:"])
 
         if not use_wheel:

tests/test_distribution_target.py

@@ -0,0 +1,58 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import pytest
+
+from pex.distribution_target import DistributionTarget
+from pex.interpreter import PythonInterpreter
+
+
+@pytest.fixture
+def current_interpreter():
+    # type: () -> PythonInterpreter
+    return PythonInterpreter.get()
+
+
+def test_interpreter_platform_mutex(current_interpreter):
+    # type: (PythonInterpreter) -> None
+
+    def assert_is_platform(target):
+        # type: (DistributionTarget) -> None
+        assert target.is_platform
+        assert not target.is_interpreter
+
+    def assert_is_interpreter(target):
+        # type: (DistributionTarget) -> None
+        assert target.is_interpreter
+        assert not target.is_platform
+
+    assert_is_interpreter(DistributionTarget.current())
+    assert_is_interpreter(DistributionTarget())
+    assert_is_interpreter(DistributionTarget.for_interpreter(current_interpreter))
+    assert_is_platform(DistributionTarget.for_platform(current_interpreter.platform))
+
+    with pytest.raises(DistributionTarget.AmbiguousTargetError):
+        DistributionTarget(interpreter=current_interpreter, platform=current_interpreter.platform)
+
+
+def test_manylinux(current_interpreter):
+    # type: (PythonInterpreter) -> None
+
+    current_platform = current_interpreter.platform
+
+    target = DistributionTarget.for_platform(current_platform, manylinux="foo")
+    assert (current_platform, "foo") == target.get_platform()
+
+    target = DistributionTarget(platform=current_platform, manylinux="bar")
+    assert (current_platform, "bar") == target.get_platform()
+
+    with pytest.raises(DistributionTarget.ManylinuxOutOfContextError):
+        DistributionTarget(manylinux="baz")
+
+    with pytest.raises(DistributionTarget.ManylinuxOutOfContextError):
+        DistributionTarget(interpreter=PythonInterpreter.get(), manylinux="baz")
+
+    target = DistributionTarget.for_interpreter(current_interpreter)
+    assert (current_platform, None) == target.get_platform()

tests/test_pip.py

@@ -3,28 +3,56 @@
 
 from __future__ import absolute_import
 
+import glob
 import os
 import warnings
 
+import pytest
+
+from pex.common import safe_rmtree
+from pex.distribution_target import DistributionTarget
 from pex.interpreter import PythonInterpreter
-from pex.pip import Pip
+from pex.jobs import Job
+from pex.pip import PackageIndexConfiguration, Pip
 from pex.typing import TYPE_CHECKING
 from pex.variables import ENV
 
 if TYPE_CHECKING:
-    from typing import Any
+    from typing import Any, Callable, Iterator, Optional
+
+    CreatePip = Callable[[Optional[PythonInterpreter]], Pip]
+
 
+@pytest.fixture
+def current_interpreter():
+    # type: () -> PythonInterpreter
+    return PythonInterpreter.get()
 
-def test_no_duplicate_constraints_pex_warnings(tmpdir):
-    # type: (Any) -> None
+
+@pytest.fixture
+def create_pip(tmpdir):
+    # type: (Any) -> Iterator[CreatePip]
     pex_root = os.path.join(str(tmpdir), "pex_root")
     pip_root = os.path.join(str(tmpdir), "pip_root")
-    interpreter = PythonInterpreter.get()
-    platform = interpreter.platform
 
-    with ENV.patch(PEX_ROOT=pex_root), warnings.catch_warnings(record=True) as events:
-        pip = Pip.create(path=pip_root, interpreter=interpreter)
+    with ENV.patch(PEX_ROOT=pex_root):
+
+        def create_pip(interpreter):
+            # type: (Optional[PythonInterpreter]) -> Pip
+            return Pip.create(path=pip_root, interpreter=interpreter)
 
+        yield create_pip
+
+
+def test_no_duplicate_constraints_pex_warnings(
+    create_pip,  # type: CreatePip
+    current_interpreter,  # type: PythonInterpreter
+):
+    # type: (...) -> None
+    with warnings.catch_warnings(record=True) as events:
+        pip = create_pip(current_interpreter)
+
+    platform = current_interpreter.platform
     pip.spawn_debug(
         platform=platform.platform, impl=platform.impl, version=platform.version, abi=platform.abi
     ).wait()
@@ -33,3 +61,64 @@ def test_no_duplicate_constraints_pex_warnings(tmpdir):
         "Expected no duplicate constraints warnings to be emitted when creating a Pip venv but "
         "found\n{}".format("\n".join(map(str, events)))
     )
+
+
+def test_download_platform_issues_1355(
+    create_pip,  # type: CreatePip
+    current_interpreter,  # type: PythonInterpreter
+    tmpdir,  # type: Any
+):
+    # type: (...) -> None
+    pip = create_pip(current_interpreter)
+    download_dir = os.path.join(str(tmpdir), "downloads")
+
+    def download_ansicolors(
+        target=None,  # type: Optional[DistributionTarget]
+        package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
+    ):
+        # type: (...) -> Job
+        safe_rmtree(download_dir)
+        return pip.spawn_download_distributions(
+            download_dir=download_dir,
+            requirements=["ansicolors==1.0.2"],
+            transitive=False,
+            target=target,
+            package_index_configuration=package_index_configuration,
+        )
+
+    def assert_ansicolors_downloaded(target=None):
+        download_ansicolors(target=target).wait()
+        assert ["ansicolors-1.0.2.tar.gz"] == os.listdir(download_dir)
+
+    # The only ansicolors 1.0.2 dist on PyPI is an sdist and we should be able to download one of
+    # those with the current interpreter since we have an interpreter in hand to build a wheel from
+    # it with later.
+    assert_ansicolors_downloaded()
+    assert_ansicolors_downloaded(target=DistributionTarget.current())
+    assert_ansicolors_downloaded(target=DistributionTarget.for_interpreter(current_interpreter))
+
+    wheel_dir = os.path.join(str(tmpdir), "wheels")
+    pip.spawn_build_wheels(
+        distributions=glob.glob(os.path.join(download_dir, "*.tar.gz")),
+        wheel_dir=wheel_dir,
+        interpreter=current_interpreter,
+    ).wait()
+    built_wheels = glob.glob(os.path.join(wheel_dir, "*.whl"))
+    assert len(built_wheels) == 1
+
+    ansicolors_wheel = built_wheels[0]
+    local_wheel_repo = PackageIndexConfiguration.create(find_links=[wheel_dir])
+    current_platform = DistributionTarget.for_platform(current_interpreter.platform)
+
+    # We should fail to find a wheel for ansicolors 1.0.2 and thus fail to download for a target
+    # Platform, even if that target platform happens to match the current interpreter we're
+    # executing Pip with.
+    with pytest.raises(Job.Error):
+        download_ansicolors(target=current_platform).wait()
+
+    # If we point the target Platform to a find-links repo with the wheel just-built though, the
+    # download should proceed without error.
+    download_ansicolors(
+        target=current_platform, package_index_configuration=local_wheel_repo
+    ).wait()
+    assert [os.path.basename(ansicolors_wheel)] == os.listdir(download_dir)