use explicit Pex options for package "binary-ness" (#20598)

Previously to pass along only/no-binary options Pants would create a
"requirements" file and use Pip's ability to put cli args in said file
[1]. This worked fine to generate the artifacts in a lockfile from
scratch, but meant that the lockfile itself did not contain sufficient
information to carry its own configuration forward. That is if one did
`pex3 lock update` on a Pants created lockfile that originally required
everything to use wheels, that specification could not be preserved in
the updated lockfile. Since v2.1.161 [2] Pex has provided
`--only-{wheel,build}` options to support this directly.

[1]
https://pip.pypa.io/en/stable/reference/requirements-file-format/#global-options

[2] https://github.com/pex-tool/pex/releases/tag/v2.1.161

ref #15704

src/python/pants/backend/python/goals/lockfile.py

@@ -79,18 +79,6 @@ async def _setup_pip_args_and_constraints_file(resolve_name: str) -> _PipArgsAnd
     args = list(resolve_config.pex_args())
     digests = []
 
-    if resolve_config.no_binary or resolve_config.only_binary:
-        pip_args_file = "__pip_args.txt"
-        args.extend(["-r", pip_args_file])
-        pip_args_file_content = "\n".join(
-            [f"--no-binary {pkg}" for pkg in resolve_config.no_binary]
-            + [f"--only-binary {pkg}" for pkg in resolve_config.only_binary]
-        )
-        pip_args_digest = await Get(
-            Digest, CreateDigest([FileContent(pip_args_file, pip_args_file_content.encode())])
-        )
-        digests.append(pip_args_digest)
-
     if resolve_config.constraints_file:
         args.append(f"--constraints={resolve_config.constraints_file.path}")
         digests.append(resolve_config.constraints_file.digest)

src/python/pants/backend/python/util_rules/pex_cli.py

@@ -40,7 +40,7 @@ class PexCli(TemplatedExternalTool):
 
     default_version = "v2.2.1"
     default_url_template = "https://github.com/pex-tool/pex/releases/download/{version}/pex"
-    version_constraints = ">=2.1.148,<3.0"
+    version_constraints = ">=2.1.161,<3.0"
 
     @classproperty
     def default_known_versions(cls):

src/python/pants/backend/python/util_rules/pex_requirements.py

@@ -349,8 +349,7 @@ class ResolvePexConfig:
     def pex_args(self) -> Iterator[str]:
         """Arguments for Pex for indexes/--find-links, manylinux, and path mappings.
 
-        Does not include arguments for constraints files, --only-binary, and --no-binary, which must
-        be set up independently.
+        Does not include arguments for constraints files, which must be set up independently.
         """
         # NB: In setting `--no-pypi`, we rely on the default value of `[python-repos].indexes`
         # including PyPI, which will override `--no-pypi` and result in using PyPI in the default
@@ -367,6 +366,30 @@ def pex_args(self) -> Iterator[str]:
         else:
             yield "--no-manylinux"
 
+        # Pex logically plumbs through equivalent settings, but uses a
+        # separate flag instead of the Pip magic :all:/:none: syntax.  To
+        # support the exitings Pants config settings we need to go from
+        # :all:/:none: --> Pex options, which Pex will translate back into Pip
+        # options.  Note that Pex's --wheel (for example) means "allow
+        # wheels", not "require wheels".
+        if self.only_binary and ":all:" in self.only_binary:
+            yield "--wheel"
+            yield "--no-build"
+        elif self.only_binary and ":none:" in self.only_binary:
+            yield "--no-wheel"
+            yield "--build"
+        elif self.only_binary:
+            yield from (f"--only-binary={pkg}" for pkg in self.only_binary)
+
+        if self.no_binary and ":all:" in self.no_binary:
+            yield "--no-wheel"
+            yield "--build"
+        elif self.no_binary and ":none:" in self.no_binary:
+            yield "--wheel"
+            yield "--no-build"
+        elif self.no_binary:
+            yield from (f"--only-build={pkg}" for pkg in self.no_binary)
+
         yield from (f"--path-mapping={v}" for v in self.path_mappings)
 
 

src/python/pants/backend/python/util_rules/pex_requirements_test.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import itertools
 import json
 import textwrap
 
@@ -347,3 +348,78 @@ def test_pex_lockfile_requirement_count() -> None:
         )
         == 3
     )
+
+
+class TestResolvePexConfigPexArgs:
+    def pairwise(self, iterable):
+        # Drop once on 3.10
+        # https://docs.python.org/3/library/itertools.html#itertools.pairwise
+        a, b = itertools.tee(iterable)
+        next(b, None)
+        return zip(a, b)
+
+    def simple_config_args(self, manylinux=None, only_binary=None, no_binary=None):
+        return tuple(
+            ResolvePexConfig(
+                indexes=[],
+                find_links=[],
+                manylinux=manylinux,
+                constraints_file=None,
+                no_binary=FrozenOrderedSet(no_binary) if no_binary else FrozenOrderedSet(),
+                only_binary=FrozenOrderedSet(only_binary) if only_binary else FrozenOrderedSet(),
+                path_mappings=[],
+            ).pex_args()
+        )
+
+    def test_minimal(self):
+        args = self.simple_config_args()
+        assert len(args) == 2
+
+    def test_manylinux(self):
+        assert "--no-manylinux" in self.simple_config_args()
+
+        many = "manylinux2014_ppc64le"
+        args = self.simple_config_args(manylinux=many)
+        assert len(args) == 3
+        assert ("--manylinux", many) in self.pairwise(args)
+
+    def test_only_binary(self):
+        assert "--only-binary=foo" in self.simple_config_args(only_binary=["foo"])
+        assert ("--only-binary=foo", "--only-binary=bar") in self.pairwise(
+            self.simple_config_args(only_binary=["foo", "bar"])
+        )
+
+    def test_only_binary_all(self):
+        args = self.simple_config_args(only_binary=[":all:"])
+        assert "--wheel" in args
+        assert "--no-build" in args
+        assert "--only-binary" not in " ".join(args)
+
+        args = self.simple_config_args(only_binary=["foo", ":all:"])
+        assert "--wheel" in args
+        assert "--no-build" in args
+        assert "--only-binary" not in " ".join(args)
+
+    def test_only_binary_none(self):
+        assert "--wheel" not in self.simple_config_args(only_binary=[":none:"])
+        assert "--only-binary" not in " ".join(self.simple_config_args(only_binary=[":none:"]))
+        assert "--build" in self.simple_config_args(only_binary=[":none:"])
+
+    def test_no_binary(self):
+        assert "--only-build=foo" in self.simple_config_args(no_binary=["foo"])
+        assert ("--only-build=foo", "--only-build=bar") in self.pairwise(
+            self.simple_config_args(no_binary=["foo", "bar"])
+        )
+
+    def test_no_binary_all(self):
+        args = self.simple_config_args(no_binary=[":all:"])
+        assert "--build" in args
+        assert "--no-wheel" in args
+        assert "--no-binary" not in args
+
+    def test_no_binary_none(self):
+        assert "--wheel" in self.simple_config_args(no_binary=[":none:"])
+        assert "--only-build" not in " ".join(self.simple_config_args(no_binary=[":none:"]))
+
+        assert "--wheel" in self.simple_config_args(no_binary=["foo", ":none:"])
+        assert "--only-build" not in " ".join(self.simple_config_args(no_binary=["foo", ":none:"]))