Add --scie option to produce native PEX exes. (#2466)

You can now specify `--scie {eager,lazy}` when building a PEX file and
one or more additional native executable PEX scies will be produced
along side the PEX file. These PEX scies will contain a portable CPython
interpreter from [Python Standalone Builds][PBS] in the `--scie eager`
case and will instead fetch a portable CPython interpreter just in time
on first boot on a given machine if needed in the `--scie lazy` case.

Although Pex will pick the target platforms and target portable CPython
interpreter version automatically, if more control is desired over which
platforms are targeted and which Python version is used, then
`--scie-platform`, `--scie-pbs-release`, and `--scie-python-version` can
be specified.

Closes #636
Closes #1007
Closes #2096

[PBS]: https://github.com/indygreg/python-build-standalone

.github/workflows/ci.yml

@@ -11,6 +11,9 @@ env:
   # importing builtins like `fcntl` as outlined in https://github.com/pex-tool/pex/issues/1391.
   _PEX_TEST_PYENV_VERSIONS: "2.7 3.7 3.10"
   _PEX_PEXPECT_TIMEOUT: 10
+  # We have integration tests that exercise `--scie` support and these can trigger downloads from
+  # GitHub Releases that needed elevated rate limit quota, which this gives.
+  SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }}
 concurrency:
   group: CI-${{ github.ref }}
   # Queue on all branches and tags, but only cancel overlapping PR burns.

CHANGES.md

@@ -1,5 +1,22 @@
 # Release Notes
 
+## 2.11.0
+
+This release adds support for creating native PEX executables that
+contain their own hermetic CPython interpreter courtesy of
+[Python Standalone Builds][PBS] and the [Science project][scie].
+
+You can now specify `--scie {eager,lazy}` when building a PEX file and
+one or more native executable PEX scies will be produced (one for each
+platform the PEX supports). These PEX scies are single file
+executables that look and behave like traditional PEXes, but unlike
+PEXes they can run on a machine with no Python interpreter available.
+
+[PBS]: https://github.com/indygreg/python-build-standalone
+[scie]: https://github.com/a-scie
+
+* Add `--scie` option to produce native PEX exes. (#2466)
+
 ## 2.10.1
 
 This release fixes a long-standing bug in Pex parsing of editable

dtox.sh

@@ -117,11 +117,21 @@ if [[ -n "${TERM:-}" ]]; then
   )
 fi
 
+if [[ -f "${HOME}/.netrc" ]]; then
+  DOCKER_ARGS+=(
+    --volume "${HOME}/.netrc:${CONTAINER_HOME}/.netrc"
+  )
+fi
+
+if [[ -d "${HOME}/.ssh" ]]; then
+  DOCKER_ARGS+=(
+    --volume "${HOME}/.ssh:${CONTAINER_HOME}/.ssh"
+  )
+fi
+
 exec docker run \
   --rm \
   --volume pex-tmp:/tmp \
-  --volume "${HOME}/.netrc:${CONTAINER_HOME}/.netrc" \
-  --volume "${HOME}/.ssh:${CONTAINER_HOME}/.ssh" \
   --volume "pex-root:${CONTAINER_HOME}/.pex" \
   --volume pex-caches:/development/pex_dev \
   --volume "${ROOT}:/development/pex" \

pex/bin/pex.py

@@ -16,7 +16,7 @@
 from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentError, ArgumentParser
 from textwrap import TextWrapper
 
-from pex import dependency_configuration, pex_warnings
+from pex import dependency_configuration, pex_warnings, scie
 from pex.argparse import HandleBoolAction
 from pex.commands.command import (
     GlobalConfigurationError,
@@ -29,6 +29,7 @@
 from pex.dist_metadata import Requirement
 from pex.docs.command import serve_html_docs
 from pex.enum import Enum
+from pex.fetcher import URLFetcher
 from pex.inherit_path import InheritPath
 from pex.interpreter_constraints import InterpreterConstraint, InterpreterConstraints
 from pex.layout import Layout, ensure_installed
@@ -56,6 +57,7 @@
 from pex.resolve.resolver_options import create_pip_configuration
 from pex.resolve.resolvers import Unsatisfiable, sorted_requirements
 from pex.result import Error, ResultError, catch, try_
+from pex.scie import ScieConfiguration
 from pex.targets import Targets
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, cast
@@ -314,6 +316,8 @@ def configure_clp_pex_options(parser):
         ),
     )
 
+    scie.register_options(group)
+
     group.add_argument(
         "--always-write-cache",
         dest="always_write_cache",
@@ -1233,6 +1237,27 @@ def do_main(
     cmdline,  # type: List[str]
     env,  # type: Dict[str, str]
 ):
+    scie_options = scie.extract_options(options)
+    if scie_options and not options.pex_name:
+        raise ValueError(
+            "You must specify `-o`/`--output-file` to use `{scie_options}`.".format(
+                scie_options=scie.render_options(scie_options)
+            )
+        )
+    scie_configuration = None  # type: Optional[ScieConfiguration]
+    if scie_options:
+        scie_configuration = scie_options.create_configuration(targets=targets)
+        if not scie_configuration:
+            raise ValueError(
+                "You selected `{scie_options}`, but none of the selected targets have "
+                "compatible interpreters that can be embedded to form a scie:\n{targets}".format(
+                    scie_options=scie.render_options(scie_options),
+                    targets="\n".join(
+                        target.render_description() for target in targets.unique_targets()
+                    ),
+                )
+            )
+
     with TRACER.timed("Building pex"):
         pex_builder = build_pex(
             requirement_configuration=requirement_configuration,
@@ -1276,6 +1301,24 @@ def do_main(
                 verbose=options.seed == Seed.VERBOSE,
             )
             print(seed_info)
+        if scie_configuration:
+            url_fetcher = URLFetcher(
+                network_configuration=resolver_configuration.network_configuration,
+                password_entries=resolver_configuration.repos_configuration.password_entries,
+                handle_file_urls=True,
+            )
+            with TRACER.timed("Building scie(s)"):
+                for scie_info in scie.build(
+                    configuration=scie_configuration, pex_file=pex_file, url_fetcher=url_fetcher
+                ):
+                    log(
+                        "Saved PEX scie for CPython {version} on {platform} to {scie}".format(
+                            version=scie_info.target.version_str,
+                            platform=scie_info.platform,
+                            scie=os.path.relpath(scie_info.file),
+                        ),
+                        V=options.verbosity,
+                    )
     else:
         if not _compatible_with_current_platform(interpreter, targets.platforms):
             log("WARNING: attempting to run PEX with incompatible platforms!", V=1)

pex/pex_bootstrapper.py

@@ -314,7 +314,7 @@ def gather_constraints():
                         path=(
                             os.pathsep.join(ENV.PEX_PYTHON_PATH)
                             if ENV.PEX_PYTHON_PATH
-                            else os.getenv("PATH")
+                            else os.getenv("PATH", "(The PATH is empty!)")
                         )
                     )
                 )

pex/platforms.py

@@ -29,7 +29,7 @@
 
 def _normalize_platform(platform):
     # type: (str) -> str
-    return platform.replace("-", "_").replace(".", "_")
+    return platform.lower().replace("-", "_").replace(".", "_")
 
 
 @attr.s(frozen=True)

pex/resolve/resolver_configuration.py

@@ -199,9 +199,24 @@ class PexRepositoryConfiguration(object):
     network_configuration = attr.ib(default=NetworkConfiguration())  # type: NetworkConfiguration
     transitive = attr.ib(default=True)  # type: bool
 
+    @property
+    def repos_configuration(self):
+        # type: () -> ReposConfiguration
+        return ReposConfiguration()
+
 
 @attr.s(frozen=True)
 class LockRepositoryConfiguration(object):
     parse_lock = attr.ib()  # type: Callable[[], Union[Lockfile, Error]]
     lock_file_path = attr.ib()  # type: str
     pip_configuration = attr.ib()  # type: PipConfiguration
+
+    @property
+    def repos_configuration(self):
+        # type: () -> ReposConfiguration
+        return self.pip_configuration.repos_configuration
+
+    @property
+    def network_configuration(self):
+        # type: () -> NetworkConfiguration
+        return self.pip_configuration.network_configuration