Add --scie option to produce native PEX exes.

pex/bin/pex.py

@@ -16,7 +16,7 @@
 from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentError, ArgumentParser
 from textwrap import TextWrapper
 
-from pex import dependency_configuration, pex_warnings
+from pex import dependency_configuration, pex_warnings, scie
 from pex.argparse import HandleBoolAction
 from pex.commands.command import (
     GlobalConfigurationError,
@@ -29,6 +29,7 @@
 from pex.dist_metadata import Requirement
 from pex.docs.command import serve_html_docs
 from pex.enum import Enum
+from pex.fetcher import URLFetcher
 from pex.inherit_path import InheritPath
 from pex.interpreter_constraints import InterpreterConstraint, InterpreterConstraints
 from pex.layout import Layout, ensure_installed
@@ -56,6 +57,7 @@
 from pex.resolve.resolver_options import create_pip_configuration
 from pex.resolve.resolvers import Unsatisfiable, sorted_requirements
 from pex.result import Error, ResultError, catch, try_
+from pex.scie import ScieConfiguration
 from pex.targets import Targets
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, cast
@@ -314,6 +316,8 @@ def configure_clp_pex_options(parser):
         ),
     )
 
+    scie.register_options(group)
+
     group.add_argument(
         "--always-write-cache",
         dest="always_write_cache",
@@ -1233,6 +1237,27 @@ def do_main(
     cmdline,  # type: List[str]
     env,  # type: Dict[str, str]
 ):
+    scie_options = scie.extract_options(options)
+    if scie_options and not options.pex_name:
+        raise ValueError(
+            "You must specify `-o`/`--output-file` to use `{scie_options}`.".format(
+                scie_options=scie.render_options(scie_options)
+            )
+        )
+    scie_configuration = None  # type: Optional[ScieConfiguration]
+    if scie_options:
+        scie_configuration = scie_options.create_configuration(targets=targets)
+        if not scie_configuration:
+            raise ValueError(
+                "You selected `{scie_options}`, but none of the selected targets have "
+                "compatible interpreters that can be embedded to form a scie:\n{targets}".format(
+                    scie_options=scie.render_options(scie_options),
+                    targets="\n".join(
+                        target.render_description() for target in targets.unique_targets()
+                    ),
+                )
+            )
+
     with TRACER.timed("Building pex"):
         pex_builder = build_pex(
             requirement_configuration=requirement_configuration,
@@ -1276,6 +1301,23 @@ def do_main(
                 verbose=options.seed == Seed.VERBOSE,
             )
             print(seed_info)
+        if scie_configuration:
+            url_fetcher = URLFetcher(
+                network_configuration=resolver_configuration.network_configuration,
+                password_entries=resolver_configuration.repos_configuration.password_entries,
+            )
+            with TRACER.timed("Building scie(s)"):
+                for par_info in scie.build(
+                    configuration=scie_configuration, pex_file=pex_file, url_fetcher=url_fetcher
+                ):
+                    log(
+                        "Saved PEX scie for CPython {version} on {platform} to {scie}".format(
+                            version=par_info.target.version_str,
+                            platform=par_info.platform,
+                            scie=os.path.relpath(par_info.file),
+                        ),
+                        V=options.verbosity,
+                    )
     else:
         if not _compatible_with_current_platform(interpreter, targets.platforms):
             log("WARNING: attempting to run PEX with incompatible platforms!", V=1)

pex/platforms.py

@@ -29,7 +29,7 @@
 
 def _normalize_platform(platform):
     # type: (str) -> str
-    return platform.replace("-", "_").replace(".", "_")
+    return platform.lower().replace("-", "_").replace(".", "_")
 
 
 @attr.s(frozen=True)

pex/resolve/resolver_configuration.py

@@ -199,9 +199,24 @@ class PexRepositoryConfiguration(object):
     network_configuration = attr.ib(default=NetworkConfiguration())  # type: NetworkConfiguration
     transitive = attr.ib(default=True)  # type: bool
 
+    @property
+    def repos_configuration(self):
+        # type: () -> ReposConfiguration
+        return ReposConfiguration()
+
 
 @attr.s(frozen=True)
 class LockRepositoryConfiguration(object):
     parse_lock = attr.ib()  # type: Callable[[], Union[Lockfile, Error]]
     lock_file_path = attr.ib()  # type: str
     pip_configuration = attr.ib()  # type: PipConfiguration
+
+    @property
+    def repos_configuration(self):
+        # type: () -> ReposConfiguration
+        return self.pip_configuration.repos_configuration
+
+    @property
+    def network_configuration(self):
+        # type: () -> NetworkConfiguration
+        return self.pip_configuration.network_configuration

pex/scie/__init__.py

@@ -0,0 +1,175 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+from argparse import Namespace, _ActionsContainer
+
+from pex.fetcher import URLFetcher
+from pex.orderedset import OrderedSet
+from pex.pep_440 import Version
+from pex.scie import science
+from pex.scie.model import (
+    ScieConfiguration,
+    ScieInfo,
+    ScieOptions,
+    SciePlatform,
+    ScieStyle,
+    ScieTarget,
+)
+from pex.typing import TYPE_CHECKING, cast
+from pex.variables import ENV, Variables
+
+if TYPE_CHECKING:
+    from typing import Iterator, Optional, Tuple, Union
+
+
+__all__ = (
+    "ScieConfiguration",
+    "ScieInfo",
+    "SciePlatform",
+    "ScieStyle",
+    "ScieTarget",
+    "build",
+)
+
+
+def register_options(parser):
+    # type: (_ActionsContainer) -> None
+
+    parser.add_argument(
+        "--scie",
+        dest="scie_style",
+        default=None,
+        type=ScieStyle.for_value,
+        choices=ScieStyle.values(),
+        help=(
+            "Create one or more native executable scies from your PEX that include a portable "
+            "CPython interpreter along with your PEX making for a truly hermetic PEX that can run "
+            "on machines with no Python installed at all. If your PEX has multiple targets, "
+            "whether `--platform`s, `--complete-platform`s or local interpreters in any "
+            "combination, then one PEX scie will be made for each platform, selecting the latest "
+            "compatible portable CPython interpreter. Note that only CPython>=3.8 is supported. If "
+            "you'd like to explicitly control the target platforms or the exact portable CPython "
+            "selected, see `--scie-platform`, `--scie-pbs-release` and `--scie-python-version`. "
+            "Specifying `--scie {lazy}` will fetch the portable CPython interpreter just in time "
+            "on first boot of the PEX scie on a given machine if needed. The URL(s) to fetch the "
+            "portable CPython interpreter from can be customized by exporting the "
+            "PEX_BOOTSTRAP_URLS environment variable pointing to a json file with the format: "
+            '`{{"ptex": {{<file name 1>: <url>, ...}}}}` where the file names should match those '
+            "found via `SCIE=inspect <the PEX scie> | jq .ptex` with appropriate replacement URLs. "
+            "Specifying `--scie {eager}` will embed the portable CPython interpreter in your PEX "
+            "scie making for a larger file, but requiring no internet access to boot. If you have "
+            "customization needs not addressed by the Pex `--scie*` options, consider using "
+            "`science` to build your scies (which is what Pex uses behind the scenes); see: "
+            "https://science.scie.app.".format(lazy=ScieStyle.LAZY, eager=ScieStyle.EAGER)
+        ),
+    )
+    parser.add_argument(
+        "--scie-platform",
+        dest="scie_platforms",
+        default=[],
+        action="append",
+        type=SciePlatform.for_value,
+        choices=SciePlatform.values(),
+        help=(
+            "The platform to produce the native PEX scie executable for. Can be specified multiple "
+            "times."
+        ),
+    )
+    parser.add_argument(
+        "--scie-pbs-release",
+        dest="scie_pbs_release",
+        default=None,
+        type=str,
+        help=(
+            "The Python Standalone Builds release to use. Currently releases are dates of the form "
+            "YYYYMMDD, e.g.: '20240713'. See their GitHub releases page at "
+            "https://github.com/indygreg/python-build-standalone/releases to discover available "
+            "releases. If left unspecified the latest release is used. N.B.: The latest lookup is "
+            "cached for 5 days. To force a fresh lookup you can remove the cache at "
+            "<USER CACHE DIR>/science/downloads."
+        ),
+    )
+    parser.add_argument(
+        "--scie-python-version",
+        dest="scie_python_version",
+        default=None,
+        type=Version,
+        help=(
+            "The portable CPython version to select. Can be either in `<major>.<minor>` form; "
+            "e.g.: '3.11', or else fully specified as `<major>.<minor>.<patch>`; e.g.: '3.11.3'. "
+            "If you don't specify this option, Pex will do its best to guess appropriate portable "
+            "CPython versions. N.B.: Python Standalone Builds does not provide all patch versions; "
+            "so you should check their releases at "
+            "https://github.com/indygreg/python-build-standalone/releases if you wish to pin down "
+            "to the patch level."
+        ),
+    )
+
+
+def render_options(options):
+    # type: (ScieOptions) -> str
+
+    args = ["--scie", str(options.style)]
+    for platform in options.platforms:
+        args.append("--scie-platform")
+        args.append(str(platform))
+    if options.pbs_release:
+        args.append("--scie-pbs-release")
+        args.append(options.pbs_release)
+    if options.python_version:
+        args.append("--scie-python-version")
+        args.append(".".join(map(str, options.python_version)))
+    return " ".join(args)
+
+
+def extract_options(options):
+    # type: (Namespace) -> Optional[ScieOptions]
+
+    if not options.scie_style:
+        return None
+
+    python_version = None  # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]]
+    if options.scie_python_version:
+        if (
+            not options.scie_python_version.parsed_version.release
+            or len(options.scie_python_version.parsed_version.release) < 2
+        ):
+            raise ValueError(
+                "Invalid Python version: '{python_version}'.\n"
+                "Must be in the form `<major>.<minor>` or `<major>.<minor>.<release>`".format(
+                    python_version=options.scie_python_version
+                )
+            )
+        python_version = cast(
+            "Union[Tuple[int, int], Tuple[int, int, int]]",
+            options.scie_python_version.parsed_version.release,
+        )
+        if python_version < (3, 8):
+            raise ValueError(
+                "Invalid Python version: '{python_version}'.\n"
+                "Scies are built using Python Standalone Builds which only supports Python >=3.8.\n"
+                "To find supported Python versions, you can browse the releases here:\n"
+                "  https://github.com/indygreg/python-build-standalone/releases".format(
+                    python_version=options.scie_python_version
+                )
+            )
+
+    return ScieOptions(
+        style=options.scie_style,
+        platforms=tuple(OrderedSet(options.scie_platforms)),
+        pbs_release=options.scie_pbs_release,
+        python_version=python_version,
+    )
+
+
+def build(
+    configuration,  # type: ScieConfiguration
+    pex_file,  # type: str
+    url_fetcher=None,  # type: Optional[URLFetcher]
+    env=ENV,  # type: Variables
+):
+    # type: (...) -> Iterator[ScieInfo]
+
+    return science.build(configuration, pex_file, url_fetcher=url_fetcher, env=env)

pex/scie/configure-binding.py

@@ -0,0 +1,30 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import print_function
+
+import os
+import sys
+
+
+def write_bindings(
+    env_file,  # type: str
+    installed_pex_dir,  # type: str
+):
+    # type: (...) -> None
+    with open(env_file, "a") as fp:
+        print("PYTHON=" + sys.executable, file=fp)
+        print("PEX=" + os.path.realpath(os.path.join(installed_pex_dir, "__main__.py")), file=fp)
+
+
+if __name__ == "__main__":
+    write_bindings(
+        env_file=os.environ["SCIE_BINDING_ENV"],
+        installed_pex_dir=(
+            # The zipapp case:
+            os.environ["_PEX_SCIE_INSTALLED_PEX_DIR"]
+            # The --venv case:
+            or os.environ.get("VIRTUAL_ENV", os.path.dirname(os.path.dirname(sys.executable)))
+        ),
+    )
+    sys.exit(0)