From 96cf9a26fbdab762fb0ee498ad443829b92a5b7d Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 15 Jul 2024 00:19:38 -0700 Subject: [PATCH 01/12] Add `--scie` option to produce native PEX exes. You can now specify `--scie {eager,lazy}` when building a PEX file and one or more additional native executable PEX scies will be produced along side the PEX file. These PEX scies will contain a portable CPython interpreter from [Python Standalone Builds][PBS] in the `--scie eager` case and will instead fetch a portable CPython interpreter just in time on first boot on a given machine if needed in the `--scie lazy` case. Although Pex will pick the target platforms and target portable CPython interpreter version automatically, if more control is desired over which platforms are targeted and which Python version is used, then `--scie-platform`, `--scie-pbs-release`, and `--scie-python-version` can be specified. Closes #636 Closes #1007 Closes #2096 [PBS]: https://github.com/indygreg/python-build-standalone --- pex/bin/pex.py | 44 +++- pex/platforms.py | 2 +- pex/resolve/resolver_configuration.py | 15 ++ pex/scie/__init__.py | 175 ++++++++++++++ pex/scie/configure-binding.py | 30 +++ pex/scie/model.py | 250 ++++++++++++++++++++ pex/scie/science.py | 315 ++++++++++++++++++++++++++ pex/targets.py | 17 +- pex/variables.py | 7 +- 9 files changed, 844 insertions(+), 11 deletions(-) create mode 100644 pex/scie/__init__.py create mode 100644 pex/scie/configure-binding.py create mode 100644 pex/scie/model.py create mode 100644 pex/scie/science.py diff --git a/pex/bin/pex.py b/pex/bin/pex.py index c60aa1627..6171fa917 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -16,7 +16,7 @@ from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentError, ArgumentParser from textwrap import TextWrapper -from pex import dependency_configuration, pex_warnings +from pex import dependency_configuration, pex_warnings, scie from pex.argparse import HandleBoolAction from pex.commands.command import ( GlobalConfigurationError, @@ -29,6 +29,7 @@ from pex.dist_metadata import Requirement from pex.docs.command import serve_html_docs from pex.enum import Enum +from pex.fetcher import URLFetcher from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraint, InterpreterConstraints from pex.layout import Layout, ensure_installed @@ -56,6 +57,7 @@ from pex.resolve.resolver_options import create_pip_configuration from pex.resolve.resolvers import Unsatisfiable, sorted_requirements from pex.result import Error, ResultError, catch, try_ +from pex.scie import ScieConfiguration from pex.targets import Targets from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast @@ -314,6 +316,8 @@ def configure_clp_pex_options(parser): ), ) + scie.register_options(group) + group.add_argument( "--always-write-cache", dest="always_write_cache", @@ -1233,6 +1237,27 @@ def do_main( cmdline, # type: List[str] env, # type: Dict[str, str] ): + scie_options = scie.extract_options(options) + if scie_options and not options.pex_name: + raise ValueError( + "You must specify `-o`/`--output-file` to use `{scie_options}`.".format( + scie_options=scie.render_options(scie_options) + ) + ) + scie_configuration = None # type: Optional[ScieConfiguration] + if scie_options: + scie_configuration = scie_options.create_configuration(targets=targets) + if not scie_configuration: + raise ValueError( + "You selected `{scie_options}`, but none of the selected targets have " + "compatible interpreters that can be embedded to form a scie:\n{targets}".format( + scie_options=scie.render_options(scie_options), + targets="\n".join( + target.render_description() for target in targets.unique_targets() + ), + ) + ) + with TRACER.timed("Building pex"): pex_builder = build_pex( requirement_configuration=requirement_configuration, @@ -1276,6 +1301,23 @@ def do_main( verbose=options.seed == Seed.VERBOSE, ) print(seed_info) + if scie_configuration: + url_fetcher = URLFetcher( + network_configuration=resolver_configuration.network_configuration, + password_entries=resolver_configuration.repos_configuration.password_entries, + ) + with TRACER.timed("Building scie(s)"): + for par_info in scie.build( + configuration=scie_configuration, pex_file=pex_file, url_fetcher=url_fetcher + ): + log( + "Saved PEX scie for CPython {version} on {platform} to {scie}".format( + version=par_info.target.version_str, + platform=par_info.platform, + scie=os.path.relpath(par_info.file), + ), + V=options.verbosity, + ) else: if not _compatible_with_current_platform(interpreter, targets.platforms): log("WARNING: attempting to run PEX with incompatible platforms!", V=1) diff --git a/pex/platforms.py b/pex/platforms.py index f43950e87..a1693347d 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -29,7 +29,7 @@ def _normalize_platform(platform): # type: (str) -> str - return platform.replace("-", "_").replace(".", "_") + return platform.lower().replace("-", "_").replace(".", "_") @attr.s(frozen=True) diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index 83badd4fb..61f788fcd 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -199,9 +199,24 @@ class PexRepositoryConfiguration(object): network_configuration = attr.ib(default=NetworkConfiguration()) # type: NetworkConfiguration transitive = attr.ib(default=True) # type: bool + @property + def repos_configuration(self): + # type: () -> ReposConfiguration + return ReposConfiguration() + @attr.s(frozen=True) class LockRepositoryConfiguration(object): parse_lock = attr.ib() # type: Callable[[], Union[Lockfile, Error]] lock_file_path = attr.ib() # type: str pip_configuration = attr.ib() # type: PipConfiguration + + @property + def repos_configuration(self): + # type: () -> ReposConfiguration + return self.pip_configuration.repos_configuration + + @property + def network_configuration(self): + # type: () -> NetworkConfiguration + return self.pip_configuration.network_configuration diff --git a/pex/scie/__init__.py b/pex/scie/__init__.py new file mode 100644 index 000000000..8330d6747 --- /dev/null +++ b/pex/scie/__init__.py @@ -0,0 +1,175 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from argparse import Namespace, _ActionsContainer + +from pex.fetcher import URLFetcher +from pex.orderedset import OrderedSet +from pex.pep_440 import Version +from pex.scie import science +from pex.scie.model import ( + ScieConfiguration, + ScieInfo, + ScieOptions, + SciePlatform, + ScieStyle, + ScieTarget, +) +from pex.typing import TYPE_CHECKING, cast +from pex.variables import ENV, Variables + +if TYPE_CHECKING: + from typing import Iterator, Optional, Tuple, Union + + +__all__ = ( + "ScieConfiguration", + "ScieInfo", + "SciePlatform", + "ScieStyle", + "ScieTarget", + "build", +) + + +def register_options(parser): + # type: (_ActionsContainer) -> None + + parser.add_argument( + "--scie", + dest="scie_style", + default=None, + type=ScieStyle.for_value, + choices=ScieStyle.values(), + help=( + "Create one or more native executable scies from your PEX that include a portable " + "CPython interpreter along with your PEX making for a truly hermetic PEX that can run " + "on machines with no Python installed at all. If your PEX has multiple targets, " + "whether `--platform`s, `--complete-platform`s or local interpreters in any " + "combination, then one PEX scie will be made for each platform, selecting the latest " + "compatible portable CPython interpreter. Note that only CPython>=3.8 is supported. If " + "you'd like to explicitly control the target platforms or the exact portable CPython " + "selected, see `--scie-platform`, `--scie-pbs-release` and `--scie-python-version`. " + "Specifying `--scie {lazy}` will fetch the portable CPython interpreter just in time " + "on first boot of the PEX scie on a given machine if needed. The URL(s) to fetch the " + "portable CPython interpreter from can be customized by exporting the " + "PEX_BOOTSTRAP_URLS environment variable pointing to a json file with the format: " + '`{{"ptex": {{: , ...}}}}` where the file names should match those ' + "found via `SCIE=inspect | jq .ptex` with appropriate replacement URLs. " + "Specifying `--scie {eager}` will embed the portable CPython interpreter in your PEX " + "scie making for a larger file, but requiring no internet access to boot. If you have " + "customization needs not addressed by the Pex `--scie*` options, consider using " + "`science` to build your scies (which is what Pex uses behind the scenes); see: " + "https://science.scie.app.".format(lazy=ScieStyle.LAZY, eager=ScieStyle.EAGER) + ), + ) + parser.add_argument( + "--scie-platform", + dest="scie_platforms", + default=[], + action="append", + type=SciePlatform.for_value, + choices=SciePlatform.values(), + help=( + "The platform to produce the native PEX scie executable for. Can be specified multiple " + "times." + ), + ) + parser.add_argument( + "--scie-pbs-release", + dest="scie_pbs_release", + default=None, + type=str, + help=( + "The Python Standalone Builds release to use. Currently releases are dates of the form " + "YYYYMMDD, e.g.: '20240713'. See their GitHub releases page at " + "https://github.com/indygreg/python-build-standalone/releases to discover available " + "releases. If left unspecified the latest release is used. N.B.: The latest lookup is " + "cached for 5 days. To force a fresh lookup you can remove the cache at " + "/science/downloads." + ), + ) + parser.add_argument( + "--scie-python-version", + dest="scie_python_version", + default=None, + type=Version, + help=( + "The portable CPython version to select. Can be either in `.` form; " + "e.g.: '3.11', or else fully specified as `..`; e.g.: '3.11.3'. " + "If you don't specify this option, Pex will do its best to guess appropriate portable " + "CPython versions. N.B.: Python Standalone Builds does not provide all patch versions; " + "so you should check their releases at " + "https://github.com/indygreg/python-build-standalone/releases if you wish to pin down " + "to the patch level." + ), + ) + + +def render_options(options): + # type: (ScieOptions) -> str + + args = ["--scie", str(options.style)] + for platform in options.platforms: + args.append("--scie-platform") + args.append(str(platform)) + if options.pbs_release: + args.append("--scie-pbs-release") + args.append(options.pbs_release) + if options.python_version: + args.append("--scie-python-version") + args.append(".".join(map(str, options.python_version))) + return " ".join(args) + + +def extract_options(options): + # type: (Namespace) -> Optional[ScieOptions] + + if not options.scie_style: + return None + + python_version = None # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + if options.scie_python_version: + if ( + not options.scie_python_version.parsed_version.release + or len(options.scie_python_version.parsed_version.release) < 2 + ): + raise ValueError( + "Invalid Python version: '{python_version}'.\n" + "Must be in the form `.` or `..`".format( + python_version=options.scie_python_version + ) + ) + python_version = cast( + "Union[Tuple[int, int], Tuple[int, int, int]]", + options.scie_python_version.parsed_version.release, + ) + if python_version < (3, 8): + raise ValueError( + "Invalid Python version: '{python_version}'.\n" + "Scies are built using Python Standalone Builds which only supports Python >=3.8.\n" + "To find supported Python versions, you can browse the releases here:\n" + " https://github.com/indygreg/python-build-standalone/releases".format( + python_version=options.scie_python_version + ) + ) + + return ScieOptions( + style=options.scie_style, + platforms=tuple(OrderedSet(options.scie_platforms)), + pbs_release=options.scie_pbs_release, + python_version=python_version, + ) + + +def build( + configuration, # type: ScieConfiguration + pex_file, # type: str + url_fetcher=None, # type: Optional[URLFetcher] + env=ENV, # type: Variables +): + # type: (...) -> Iterator[ScieInfo] + + return science.build(configuration, pex_file, url_fetcher=url_fetcher, env=env) diff --git a/pex/scie/configure-binding.py b/pex/scie/configure-binding.py new file mode 100644 index 000000000..8f3b1c049 --- /dev/null +++ b/pex/scie/configure-binding.py @@ -0,0 +1,30 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import print_function + +import os +import sys + + +def write_bindings( + env_file, # type: str + installed_pex_dir, # type: str +): + # type: (...) -> None + with open(env_file, "a") as fp: + print("PYTHON=" + sys.executable, file=fp) + print("PEX=" + os.path.realpath(os.path.join(installed_pex_dir, "__main__.py")), file=fp) + + +if __name__ == "__main__": + write_bindings( + env_file=os.environ["SCIE_BINDING_ENV"], + installed_pex_dir=( + # The zipapp case: + os.environ["_PEX_SCIE_INSTALLED_PEX_DIR"] + # The --venv case: + or os.environ.get("VIRTUAL_ENV", os.path.dirname(os.path.dirname(sys.executable))) + ), + ) + sys.exit(0) diff --git a/pex/scie/model.py b/pex/scie/model.py new file mode 100644 index 000000000..4c5f68a0d --- /dev/null +++ b/pex/scie/model.py @@ -0,0 +1,250 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import itertools +import os +import platform +from collections import defaultdict + +from pex.enum import Enum +from pex.platforms import Platform +from pex.targets import Targets +from pex.third_party.packaging import tags # noqa +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import DefaultDict, Iterable, Optional, Set, Tuple, Union + + import attr # vendor:skip +else: + from pex.third_party import attr + + +class ScieStyle(Enum["ScieStyle.Value"]): + class Value(Enum.Value): + pass + + LAZY = Value("lazy") + EAGER = Value("eager") + + +class SciePlatform(Enum["SciePlatform.Value"]): + class Value(Enum.Value): + @property + def extension(self): + # type: () -> str + return ( + ".exe" + if self in (SciePlatform.WINDOWS_AARCH64, SciePlatform.WINDOWS_X86_64) + else "" + ) + + def binary_name(self, binary_name): + # type: (str) -> str + return "{binary_name}{extension}".format( + binary_name=binary_name, extension=self.extension + ) + + def qualified_binary_name(self, binary_name): + # type: (str) -> str + return "{binary_name}-{platform}{extension}".format( + binary_name=binary_name, platform=self, extension=self.extension + ) + + def qualified_file_name(self, file_name): + # type: (str) -> str + stem, ext = os.path.splitext(file_name) + return "{stem}-{platform}{ext}".format(stem=stem, platform=self, ext=ext) + + LINUX_AARCH64 = Value("linux-aarch64") + LINUX_X86_64 = Value("linux-x86_64") + MACOS_AARCH64 = Value("macos-aarch64") + MACOS_X86_64 = Value("macos-x86_64") + WINDOWS_AARCH64 = Value("windows-x86_64") + WINDOWS_X86_64 = Value("windows-aarch64") + + @classmethod + def parse(cls, value): + # type: (str) -> SciePlatform.Value + return cls.current() if "current" == value else cls.for_value(value) + + @classmethod + def current(cls): + # type: () -> SciePlatform.Value + system = platform.system().lower() + machine = platform.machine().lower() + if "linux" == system: + if machine in ("aarch64", "arm64"): + return cls.LINUX_AARCH64 + elif machine in ("amd64", "x86_64"): + return cls.LINUX_X86_64 + elif "darwin" == system: + if machine in ("aarch64", "arm64"): + return cls.MACOS_AARCH64 + elif machine in ("amd64", "x86_64"): + return cls.MACOS_X86_64 + elif "windows" == system: + if machine in ("aarch64", "arm64"): + return cls.WINDOWS_AARCH64 + elif machine in ("amd64", "x86_64"): + return cls.WINDOWS_X86_64 + raise ValueError( + "The current operating system / machine pair is not supported!: " + "{system} / {machine}".format(system=system, machine=machine) + ) + + +@attr.s(frozen=True) +class ScieTarget(object): + platform = attr.ib() # type: SciePlatform.Value + python_version = attr.ib() # type: Union[Tuple[int, int], Tuple[int, int, int]] + pbs_release = attr.ib(default=None) # type: Optional[str] + + @property + def version_str(self): + # type: () -> str + return ".".join(map(str, self.python_version)) + + +@attr.s(frozen=True) +class ScieInfo(object): + style = attr.ib() # type: ScieStyle.Value + target = attr.ib() # type: ScieTarget + file = attr.ib() # type: str + + @property + def platform(self): + # type: () -> SciePlatform.Value + return self.target.platform + + @property + def python_version(self): + # type: () -> Union[Tuple[int, int], Tuple[int, int, int]] + return self.target.python_version + + +@attr.s(frozen=True) +class ScieOptions(object): + style = attr.ib(default=ScieStyle.LAZY) # type: ScieStyle.Value + platforms = attr.ib(default=()) # type: Tuple[SciePlatform.Value, ...] + pbs_release = attr.ib(default=None) # type: Optional[str] + python_version = attr.ib( + default=None + ) # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + + def create_configuration(self, targets): + # type: (Targets) -> ScieConfiguration + return ScieConfiguration.from_targets(self, targets) + + +@attr.s(frozen=True) +class ScieConfiguration(object): + @classmethod + def from_tags( + cls, + options, # type: ScieOptions + tags, # type: Iterable[tags.Tag] + ): + # type: (...) -> ScieConfiguration + return cls._from_platforms( + options=options, platforms=tuple(Platform.from_tag(tag) for tag in tags) + ) + + @classmethod + def from_targets( + cls, + options, # type: ScieOptions + targets, # type: Targets + ): + # type: (...) -> ScieConfiguration + return cls._from_platforms( + options=options, + platforms=tuple(target.platform for target in targets.unique_targets()), + ) + + @classmethod + def _from_platforms( + cls, + options, # type: ScieOptions + platforms, # type: Iterable[Platform] + ): + # type: (...) -> ScieConfiguration + + python_version = options.python_version + python_versions_by_platform = defaultdict( + set + ) # type: DefaultDict[SciePlatform.Value, Set[Union[Tuple[int, int], Tuple[int, int, int]]]] + for plat in platforms: + if python_version: + plat_python_version = python_version + elif len(plat.version_info) < 2: + continue + else: + plat_python_version = cast( + "Union[Tuple[int, int], Tuple[int, int, int]]", plat.version_info + ) + + # We use Python Build Standalone to create scies, and we know it does not support + # CPython<3.8. + if plat_python_version < (3, 8): + continue + + # We use Python Build Standalone to create scies, and we know it only provides CPython + # interpreters. + if plat.impl not in ("py", "cp"): + continue + + platform_str = plat.platform + is_aarch64 = "arm64" in platform_str or "aarch64" in platform_str + is_x86_64 = "amd64" in platform_str or "x86_64" in platform_str + if not is_aarch64 ^ is_x86_64: + continue + + if "linux" in platform_str: + scie_platform = ( + SciePlatform.LINUX_AARCH64 if is_aarch64 else SciePlatform.LINUX_X86_64 + ) + elif "mac" in platform_str: + scie_platform = ( + SciePlatform.MACOS_AARCH64 if is_aarch64 else SciePlatform.MACOS_X86_64 + ) + elif "win" in platform_str: + scie_platform = ( + SciePlatform.WINDOWS_AARCH64 if is_aarch64 else SciePlatform.WINDOWS_X86_64 + ) + else: + continue + + python_versions_by_platform[scie_platform].add(plat_python_version) + + for explicit_platform in options.platforms: + if explicit_platform not in python_versions_by_platform: + if options.python_version: + python_versions_by_platform[explicit_platform] = {options.python_version} + else: + python_versions_by_platform[explicit_platform] = set( + itertools.chain.from_iterable(python_versions_by_platform.values()) + ) + if options.platforms: + for configured_platform in tuple(python_versions_by_platform): + if configured_platform not in options.platforms: + python_versions_by_platform.pop(configured_platform, None) + + scie_targets = tuple( + ScieTarget( + platform=scie_platform, + pbs_release=options.pbs_release, + python_version=max(python_versions), + ) + for scie_platform, python_versions in sorted(python_versions_by_platform.items()) + ) + return cls(options=options, targets=tuple(scie_targets)) + + options = attr.ib() # type: ScieOptions + targets = attr.ib() # type: Tuple[ScieTarget, ...] + + def __len__(self): + # type: () -> int + return len(self.targets) diff --git a/pex/scie/science.py b/pex/scie/science.py new file mode 100644 index 000000000..26ad9018d --- /dev/null +++ b/pex/scie/science.py @@ -0,0 +1,315 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import re +import shutil +import subprocess +from collections import OrderedDict +from subprocess import CalledProcessError + +from pex.atomic_directory import atomic_directory +from pex.common import is_exe, pluralize, safe_mkdtemp, safe_open +from pex.compatibility import shlex_quote +from pex.exceptions import production_assert +from pex.fetcher import URLFetcher +from pex.hashing import Sha256 +from pex.pep_440 import Version +from pex.pex_info import PexInfo +from pex.scie.model import ScieConfiguration, ScieInfo, SciePlatform, ScieStyle, ScieTarget +from pex.third_party.packaging.version import InvalidVersion +from pex.typing import TYPE_CHECKING, cast +from pex.util import CacheHelper +from pex.variables import ENV, Variables, unzip_dir_relpath + +if TYPE_CHECKING: + from typing import Any, Dict, Iterator, Optional, cast + + import attr # vendor:skip + import toml # vendor:skip +else: + from pex.third_party import attr, toml + + +@attr.s(frozen=True) +class Manifest(object): + target = attr.ib() # type: ScieTarget + path = attr.ib() # type: str + + def binary_name(self, binary_name): + # type: (str) -> str + return self.target.platform.binary_name(binary_name) + + def qualified_binary_name(self, binary_name): + # type: (str) -> str + return self.target.platform.qualified_binary_name(binary_name) + + +MIN_SCIENCE_VERSION = Version("0.3.0") +PTEX_VERSION = "1.1.1" +SCIE_JUMP_VERSION = "1.1.1" + + +def create_manifests( + configuration, # type: ScieConfiguration + name, # type: str + pex_info, # type: PexInfo +): + # type: (...) -> Iterator[Manifest] + + pex_root = "{scie.bindings}/pex_root" + if pex_info.venv: + # We let the configure-binding calculate the venv dir at runtime since it depends on the + # interpreter executing the venv PEX. + installed_pex_dir = "" + else: + production_assert(pex_info.pex_hash is not None) + pex_hash = cast(str, pex_info.pex_hash) + installed_pex_dir = os.path.join(pex_root, unzip_dir_relpath(pex_hash)) + + env_replace = { + "PEX_ROOT": pex_root, + } + env = { + "remove_re": {"PEX_.*"}, + "replace": env_replace, + } + + lift = { + "name": name, + "ptex": { + "id": "ptex", + "version": PTEX_VERSION, + "argv1": "{scie.env.PEX_BOOTSTRAP_URLS={scie.lift}}", + }, + "scie_jump": {"version": SCIE_JUMP_VERSION}, + "files": [{"name": "configure-binding.py"}, {"name": "pex"}], + "commands": [ + { + "env": env, + "exe": "{scie.bindings.configure:PYTHON}", + "args": ["{scie.bindings.configure:PEX}"], + } + ], + "bindings": [ + { + "env": dict( + env, + replace=dict( + env_replace, + PEX_INTERPRETER="1", + _PEX_SCIE_INSTALLED_PEX_DIR=installed_pex_dir, + # We can get a warning about too-long script shebangs, but this is not + # relevant since we above run the PEX via python and not via shebang. + PEX_EMIT_WARNINGS="0", + ), + ), + "name": "configure", + "exe": "#{cpython:python}", + "args": ["{pex}", "{configure-binding.py}"], + } + ], + } # type: Dict[str, Any] + + for target in configuration.targets: + manifest_path = os.path.join( + safe_mkdtemp(), + target.platform.qualified_file_name("{name}-lift.toml".format(name=name)), + ) + with safe_open(manifest_path, "w") as fp: + toml.dump( + { + "lift": dict( + lift, + platforms=[target.platform.value], + interpreters=[ + { + "id": "cpython", + "provider": "PythonBuildStandalone", + "release": target.pbs_release, + "version": target.version_str, + "lazy": configuration.options.style is ScieStyle.LAZY, + } + ], + ) + }, + fp, + ) + yield Manifest(target=target, path=manifest_path) + + +def _science_dir( + env, # type: Variables + *components # type: str +): + # type: (...) -> str + return os.path.join(env.PEX_ROOT, "scies", "science", str(MIN_SCIENCE_VERSION), *components) + + +def _qualified_science_binary_name(): + # type: () -> str + return SciePlatform.current().qualified_binary_name("science") + + +def _science_binary_names(): + # type: () -> Iterator[str] + yield "science" + yield _qualified_science_binary_name() + + +def _path_science(): + # type: () -> Optional[str] + for path_element in os.environ.get("PATH", os.defpath).split(os.pathsep): + for binary in ( + os.path.join(path_element, binary_name) for binary_name in _science_binary_names() + ): + if not is_exe(binary): + continue + try: + if ( + Version(subprocess.check_output(args=[binary, "--version"]).decode("utf-8")) + < MIN_SCIENCE_VERSION + ): + continue + except (CalledProcessError, InvalidVersion): + continue + return binary + return None + + +def _science_binary_url(suffix=""): + # type: (str) -> str + return "https://github.com/a-scie/science/releases/download/v{version}/{binary}{suffix}".format( + version=MIN_SCIENCE_VERSION, + binary=_qualified_science_binary_name(), + suffix=suffix, + ) + + +def _ensure_science( + url_fetcher=None, # type: Optional[URLFetcher] + env=ENV, # type: Variables +): + # type: (...) -> str + + target_dir = _science_dir(env, "bin") + with atomic_directory(target_dir=target_dir) as atomic_dir: + if not atomic_dir.is_finalized(): + target_science = os.path.join(atomic_dir.work_dir, "science") + path_science = _path_science() + if path_science: + shutil.copy(path_science, target_science) + else: + fetcher = url_fetcher or URLFetcher() + science_binary_url = _science_binary_url() + with open(target_science, "wb") as write_fp, fetcher.get_body_stream( + science_binary_url + ) as read_fp: + shutil.copyfileobj(read_fp, write_fp) + + science_sha256_url = _science_binary_url(".sha256") + with fetcher.get_body_stream(science_sha256_url) as fp: + expected_sha256, _, _ = fp.read().decode("utf-8").partition(" ") + actual_sha256 = CacheHelper.hash(target_science, hasher=Sha256) + if expected_sha256 != actual_sha256: + raise ValueError( + "The science binary downloaded from {science_binary_url} does not match " + "the expected SHA-256 fingerprint recorded in {science_sha256_url}.\n" + "Expected {expected_sha256} but found {actual_sha256}.".format( + science_binary_url=science_binary_url, + science_sha256_url=science_sha256_url, + expected_sha256=expected_sha256, + actual_sha256=actual_sha256, + ) + ) + return os.path.join(target_dir, "science") + + +class ScienceError(Exception): + """Indicates an error executing science.""" + + +def build( + configuration, # type: ScieConfiguration + pex_file, # type: str + url_fetcher=None, # type: Optional[URLFetcher] + env=ENV, # type: Variables +): + # type: (...) -> Iterator[ScieInfo] + + science = _ensure_science(url_fetcher=url_fetcher, env=env) + name = re.sub(r"\.pex$", "", os.path.basename(pex_file), flags=re.IGNORECASE) + pex_info = PexInfo.from_pex(pex_file) + use_platform_suffix = len(configuration.targets) > 1 + errors = OrderedDict() # type: OrderedDict[Manifest, str] + for manifest in create_manifests(configuration, name, pex_info): + args = [science, "--cache-dir", _science_dir(env, "cache")] + if env.PEX_VERBOSE: + args.append("-{verbosity}".format(verbosity="v" * env.PEX_VERBOSE)) + dest_dir = os.path.dirname(os.path.abspath(pex_file)) + args.extend( + [ + "lift", + "--file", + "pex={pex_file}".format(pex_file=pex_file), + "--file", + "configure-binding.py={configure_binding}".format( + configure_binding=os.path.join( + os.path.dirname(__file__), "configure-binding.py" + ) + ), + "build", + "--dest-dir", + dest_dir, + ] + ) + if use_platform_suffix: + args.append("--use-platform-suffix") + args.append(manifest.path) + with open(os.devnull, "wb") as devnull: + process = subprocess.Popen(args=args, stdout=devnull, stderr=subprocess.PIPE) + _, stderr = process.communicate() + if process.returncode != 0: + saved_manifest = os.path.relpath( + os.path.join(dest_dir, os.path.basename(manifest.path)) + ) + shutil.copy(manifest.path, saved_manifest) + errors[manifest] = ( + "Command `{command}` failed with exit code {exit_code} (saved lift manifest to " + "{saved_manifest} for inspection):\n{stderr}" + ).format( + command=" ".join(shlex_quote(arg) for arg in args[:-1] + [saved_manifest]), + exit_code=process.returncode, + saved_manifest=saved_manifest, + stderr=stderr.decode("utf-8").strip(), + ) + else: + yield ScieInfo( + style=configuration.options.style, + target=manifest.target, + file=os.path.join( + dest_dir, + manifest.qualified_binary_name(name) + if use_platform_suffix + else manifest.binary_name(name), + ), + ) + if errors: + + raise ScienceError( + "Failed to build {count} {scies}:\n\n{errors}".format( + count=len(errors), + scies=pluralize(errors, "scie"), + errors="\n\n".join( + "{index}. For CPython {version} on {platform}: {err}".format( + index=index, + platform=manifest.target.platform, + version=manifest.target.version_str, + err=err, + ) + for index, (manifest, err) in enumerate(errors.items(), start=1) + ), + ) + ) diff --git a/pex/targets.py b/pex/targets.py index 511e05904..37c02f229 100644 --- a/pex/targets.py +++ b/pex/targets.py @@ -44,13 +44,14 @@ def binary_name(self, version_components=2): @property def python_version(self): - # type: () -> Optional[Tuple[int, int]] + # type: () -> Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + python_full_version = self.marker_environment.python_full_version + if python_full_version: + return cast("Tuple[int, int, int]", tuple(map(int, python_full_version.split(".")))[:3]) python_version = self.marker_environment.python_version - return ( - cast("Tuple[int, int]", tuple(map(int, python_version.split(".")))[:2]) - if python_version - else None - ) + if python_version: + return cast("Tuple[int, int]", tuple(map(int, python_version.split(".")))[:2]) + return None @property def supported_tags(self): @@ -181,8 +182,8 @@ def binary_name(self, version_components=2): @property def python_version(self): - # type: () -> Tuple[int, int] - return self.interpreter.identity.version[:2] + # type: () -> Tuple[int, int, int] + return self.interpreter.identity.version[:3] @property def is_foreign(self): diff --git a/pex/variables.py b/pex/variables.py index 41ff3b287..f952eede0 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -789,6 +789,11 @@ def _expand_pex_root(pex_root): return os.path.expanduser(Variables.PEX_ROOT.value_or(ENV, fallback=fallback)) +def unzip_dir_relpath(pex_hash): + # type: (str) -> str + return os.path.join("unzipped_pexes", pex_hash) + + def unzip_dir( pex_root, # type: str pex_hash, # type: str @@ -796,7 +801,7 @@ def unzip_dir( ): # type: (...) -> str pex_root = _expand_pex_root(pex_root) if expand_pex_root else pex_root - return os.path.join(pex_root, "unzipped_pexes", pex_hash) + return os.path.join(pex_root, unzip_dir_relpath(pex_hash)) def venv_dir( From fa18f602b68b98272b78f11be6709bb7c13c030d Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 15 Jul 2024 19:08:36 -0700 Subject: [PATCH 02/12] Add integration tests for basic `--scie` support. This stresses the full matrix of basic cases (no `--scie-*` options). --- .github/workflows/ci.yml | 3 + pex/scie/model.py | 5 +- pex/scie/science.py | 7 +- pex/venv/installer.py | 2 + tests/integration/scie/__init__.py | 2 + tests/integration/scie/test_pex_scie.py | 89 +++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 tests/integration/scie/__init__.py create mode 100644 tests/integration/scie/test_pex_scie.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cf23e4c..5c849752a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,9 @@ env: # importing builtins like `fcntl` as outlined in https://github.com/pex-tool/pex/issues/1391. _PEX_TEST_PYENV_VERSIONS: "2.7 3.7 3.10" _PEX_PEXPECT_TIMEOUT: 10 + # We have integration tests that exercise `--scie` support and these can trigger downloads from + # GitHub Releases that needed elevated rate limit quota, which this gives. + SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }} concurrency: group: CI-${{ github.ref }} # Queue on all branches and tags, but only cancel overlapping PR burns. diff --git a/pex/scie/model.py b/pex/scie/model.py index 4c5f68a0d..3f5bce21e 100644 --- a/pex/scie/model.py +++ b/pex/scie/model.py @@ -182,9 +182,12 @@ def _from_platforms( elif len(plat.version_info) < 2: continue else: + # Here were guessing an available PBS CPython version. Since a triple is unlikely to + # hit, we just use major / minor. If the user wants control they can specify + # options.python_version via `--scie-python-version`. plat_python_version = cast( "Union[Tuple[int, int], Tuple[int, int, int]]", plat.version_info - ) + )[:2] # We use Python Build Standalone to create scies, and we know it does not support # CPython<3.8. diff --git a/pex/scie/science.py b/pex/scie/science.py index 26ad9018d..f3c345763 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -16,6 +16,7 @@ from pex.exceptions import production_assert from pex.fetcher import URLFetcher from pex.hashing import Sha256 +from pex.layout import Layout from pex.pep_440 import Version from pex.pex_info import PexInfo from pex.scie.model import ScieConfiguration, ScieInfo, SciePlatform, ScieStyle, ScieTarget @@ -56,6 +57,7 @@ def create_manifests( configuration, # type: ScieConfiguration name, # type: str pex_info, # type: PexInfo + layout, # type: Layout.Value ): # type: (...) -> Iterator[Manifest] @@ -64,6 +66,8 @@ def create_manifests( # We let the configure-binding calculate the venv dir at runtime since it depends on the # interpreter executing the venv PEX. installed_pex_dir = "" + elif layout is Layout.LOOSE: + installed_pex_dir = "{pex}" else: production_assert(pex_info.pex_hash is not None) pex_hash = cast(str, pex_info.pex_hash) @@ -242,9 +246,10 @@ def build( science = _ensure_science(url_fetcher=url_fetcher, env=env) name = re.sub(r"\.pex$", "", os.path.basename(pex_file), flags=re.IGNORECASE) pex_info = PexInfo.from_pex(pex_file) + layout = Layout.identify(pex_file) use_platform_suffix = len(configuration.targets) > 1 errors = OrderedDict() # type: OrderedDict[Manifest, str] - for manifest in create_manifests(configuration, name, pex_info): + for manifest in create_manifests(configuration, name, pex_info, layout): args = [science, "--cache-dir", _science_dir(env, "cache")] if env.PEX_VERBOSE: args.append("-{verbosity}".format(verbosity="v" * env.PEX_VERBOSE)) diff --git a/pex/venv/installer.py b/pex/venv/installer.py index c0c6be2d1..b6b759c8d 100644 --- a/pex/venv/installer.py +++ b/pex/venv/installer.py @@ -703,6 +703,8 @@ def sys_executable_paths(): "_PEX_DEP_CONFIG_FILE", # This is used as an experiment knob for atomic_directory locking. "_PEX_FILE_LOCK_STYLE", + # This is used in the scie binding command for ZIPAPP PEXes. + "_PEX_SCIE_INSTALLED_PEX_DIR", ) ] if ignored_pex_env_vars: diff --git a/tests/integration/scie/__init__.py b/tests/integration/scie/__init__.py new file mode 100644 index 000000000..87fb2ed9a --- /dev/null +++ b/tests/integration/scie/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py new file mode 100644 index 000000000..2db61e212 --- /dev/null +++ b/tests/integration/scie/test_pex_scie.py @@ -0,0 +1,89 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import re +import subprocess + +import pytest + +from pex.layout import Layout +from pex.scie import ScieStyle +from pex.targets import LocalInterpreter +from pex.typing import TYPE_CHECKING +from testing import IS_PYPY, PY_VER, make_env, run_pex_command + +if TYPE_CHECKING: + from typing import Any, List + + +@pytest.mark.parametrize( + "scie_style", [pytest.param(style, id=str(style)) for style in ScieStyle.values()] +) +@pytest.mark.parametrize( + "layout", [pytest.param(layout, id=str(layout)) for layout in Layout.values()] +) +@pytest.mark.parametrize( + "execution_mode_args", + [ + pytest.param([], id="ZIPAPP"), + pytest.param(["--venv"], id="VENV"), + pytest.param(["--sh-boot"], id="ZIPAPP-sh-boot"), + pytest.param(["--venv", "--sh-boot"], id="VENV-sh-boot"), + ], +) +def test_basic( + tmpdir, # type: Any + scie_style, # type: ScieStyle.Value + layout, # type: Layout.Value + execution_mode_args, # type: List[str] +): + # type: (...) -> None + + pex = os.path.join(str(tmpdir), "cowsay.pex") + result = run_pex_command( + args=[ + "cowsay==5.0", + "-c", + "cowsay", + "-o", + pex, + "--scie", + str(scie_style), + "--layout", + str(layout), + ] + + execution_mode_args + ) + if PY_VER < (3, 8) or IS_PYPY: + result.assert_failure( + expected_error_re=r".*^{message}$".format( + message=re.escape( + "You selected `--scie {style}`, but none of the selected targets have " + "compatible interpreters that can be embedded to form a scie:\n" + "{target}".format( + style=scie_style, target=LocalInterpreter.create().render_description() + ) + ) + ), + re_flags=re.DOTALL | re.MULTILINE, + ) + return + if PY_VER >= (3, 13): + result.assert_failure( + expected_error_re=( + r".*" + r"^Failed to build 1 scie:$" + r".*XXX" + r"^Provider: No released assets found for release [0-9]{{8}} Python {version} " + r"of flavor install_only\.$".format(version=".".join(map(str, PY_VER))) + ), + re_flags=re.DOTALL | re.MULTILINE, + ) + return + result.assert_success() + + scie = os.path.join(str(tmpdir), "cowsay") + assert b"| PAR! |" in subprocess.check_output(args=[scie, "PAR!"], env=make_env(PATH=None)) From 66aeb9f5ea67cc047dff71a71654707497e9c5f9 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 15 Jul 2024 19:13:16 -0700 Subject: [PATCH 03/12] Fix stray debug XXX. --- tests/integration/scie/test_pex_scie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py index 2db61e212..f7612d6b0 100644 --- a/tests/integration/scie/test_pex_scie.py +++ b/tests/integration/scie/test_pex_scie.py @@ -76,7 +76,7 @@ def test_basic( expected_error_re=( r".*" r"^Failed to build 1 scie:$" - r".*XXX" + r".*" r"^Provider: No released assets found for release [0-9]{{8}} Python {version} " r"of flavor install_only\.$".format(version=".".join(map(str, PY_VER))) ), From 82c629dffaa2aeb2e9a0f9210fcc7a46aeb1c43f Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 15 Jul 2024 19:51:01 -0700 Subject: [PATCH 04/12] Punch a hole for SCIENCE_AUTH_API_GITHUB_COM_BEARER through tox. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index d1d129e5b..4664152f6 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,8 @@ passenv = SSH_AUTH_SOCK # Needed for pexpect tests. TERM + # Needed to prevent hitting rate limits on GitHub Releases APIs in `--scie` integration tests. + SCIENCE_AUTH_API_GITHUB_COM_BEARER setenv = pip20: _PEX_PIP_VERSION=20.3.4-patched pip22_2: _PEX_PIP_VERSION=22.2.2 From cbb96fde1072b7f14e1ad67852d617442bc3bf7c Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 10:02:44 -0700 Subject: [PATCH 05/12] Various fixes and improvements from SJ's feedback. --- pex/bin/pex.py | 1 + pex/scie/__init__.py | 29 +++++++++- pex/scie/model.py | 1 + pex/scie/science.py | 125 ++++++++++++++++++++++++++++++------------- 4 files changed, 117 insertions(+), 39 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 6171fa917..e6a4239d0 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -1305,6 +1305,7 @@ def do_main( url_fetcher = URLFetcher( network_configuration=resolver_configuration.network_configuration, password_entries=resolver_configuration.repos_configuration.password_entries, + handle_file_urls=True, ) with TRACER.timed("Building scie(s)"): for par_info in scie.build( diff --git a/pex/scie/__init__.py b/pex/scie/__init__.py index 8330d6747..a4efaf187 100644 --- a/pex/scie/__init__.py +++ b/pex/scie/__init__.py @@ -3,8 +3,10 @@ from __future__ import absolute_import +import os.path from argparse import Namespace, _ActionsContainer +from pex.compatibility import urlparse from pex.fetcher import URLFetcher from pex.orderedset import OrderedSet from pex.pep_440 import Version @@ -17,6 +19,7 @@ ScieStyle, ScieTarget, ) +from pex.scie.science import SCIENCE_RELEASES_URL, SCIENCE_REQUIREMENT from pex.typing import TYPE_CHECKING, cast from pex.variables import ENV, Variables @@ -39,6 +42,7 @@ def register_options(parser): parser.add_argument( "--scie", + "--par", dest="scie_style", default=None, type=ScieStyle.for_value, @@ -106,6 +110,19 @@ def register_options(parser): "to the patch level." ), ) + parser.add_argument( + "--scie-science-binary", + dest="scie_science_binary", + default=None, + type=str, + help=( + "The file path of a `science` binary or a URL to use to fetch the `science` binary " + "when there is no `science` on the PATH with a version matching {science_requirement}. " + "Pex uses the official `science` releases at {science_releases_url} by default.".format( + science_requirement=SCIENCE_REQUIREMENT, science_releases_url=SCIENCE_RELEASES_URL + ) + ), + ) def render_options(options): @@ -121,6 +138,9 @@ def render_options(options): if options.python_version: args.append("--scie-python-version") args.append(".".join(map(str, options.python_version))) + if options.science_binary_url: + args.append("--scie-science-binary") + args.append(options.science_binary_url) return " ".join(args) @@ -138,7 +158,7 @@ def extract_options(options): ): raise ValueError( "Invalid Python version: '{python_version}'.\n" - "Must be in the form `.` or `..`".format( + "Must be in the form `.` or `..`".format( python_version=options.scie_python_version ) ) @@ -156,11 +176,18 @@ def extract_options(options): ) ) + science_binary_url = options.scie_science_binary + if science_binary_url: + url_info = urlparse.urlparse(options.scie_science_binary) + if not url_info.scheme and url_info.path and os.path.isfile(url_info.path): + science_binary_url = "file://{path}".format(path=os.path.abspath(url_info.path)) + return ScieOptions( style=options.scie_style, platforms=tuple(OrderedSet(options.scie_platforms)), pbs_release=options.scie_pbs_release, python_version=python_version, + science_binary_url=science_binary_url, ) diff --git a/pex/scie/model.py b/pex/scie/model.py index 3f5bce21e..5f1737718 100644 --- a/pex/scie/model.py +++ b/pex/scie/model.py @@ -133,6 +133,7 @@ class ScieOptions(object): python_version = attr.ib( default=None ) # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + science_binary_url = attr.ib(default=None) # type: Optional[str] def create_configuration(self, targets): # type: (Targets) -> ScieConfiguration diff --git a/pex/scie/science.py b/pex/scie/science.py index f3c345763..f357c9026 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -11,7 +11,7 @@ from subprocess import CalledProcessError from pex.atomic_directory import atomic_directory -from pex.common import is_exe, pluralize, safe_mkdtemp, safe_open +from pex.common import chmod_plus_x, is_exe, pluralize, safe_mkdtemp, safe_open from pex.compatibility import shlex_quote from pex.exceptions import production_assert from pex.fetcher import URLFetcher @@ -19,14 +19,17 @@ from pex.layout import Layout from pex.pep_440 import Version from pex.pex_info import PexInfo +from pex.result import Error, try_ from pex.scie.model import ScieConfiguration, ScieInfo, SciePlatform, ScieStyle, ScieTarget +from pex.third_party.packaging.specifiers import SpecifierSet from pex.third_party.packaging.version import InvalidVersion +from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast from pex.util import CacheHelper from pex.variables import ENV, Variables, unzip_dir_relpath if TYPE_CHECKING: - from typing import Any, Dict, Iterator, Optional, cast + from typing import Any, Dict, Iterator, Optional, Union, cast import attr # vendor:skip import toml # vendor:skip @@ -48,7 +51,21 @@ def qualified_binary_name(self, binary_name): return self.target.platform.qualified_binary_name(binary_name) +SCIENCE_RELEASES_URL = "https://github.com/a-scie/lift/releases" MIN_SCIENCE_VERSION = Version("0.3.0") +SCIENCE_REQUIREMENT = SpecifierSet("~={min_version}".format(min_version=MIN_SCIENCE_VERSION)) + + +def _science_binary_url(suffix=""): + # type: (str) -> str + return "{science_releases_url}/download/v{version}/{binary}{suffix}".format( + science_releases_url=SCIENCE_RELEASES_URL, + version=MIN_SCIENCE_VERSION.raw, + binary=_qualified_science_fat_binary_name(), + suffix=suffix, + ) + + PTEX_VERSION = "1.1.1" SCIE_JUMP_VERSION = "1.1.1" @@ -149,18 +166,47 @@ def _science_dir( *components # type: str ): # type: (...) -> str - return os.path.join(env.PEX_ROOT, "scies", "science", str(MIN_SCIENCE_VERSION), *components) + return os.path.join(env.PEX_ROOT, "scies", "science", MIN_SCIENCE_VERSION.raw, *components) -def _qualified_science_binary_name(): +def _qualified_science_fat_binary_name(): # type: () -> str - return SciePlatform.current().qualified_binary_name("science") + return SciePlatform.current().qualified_binary_name("science-fat") def _science_binary_names(): # type: () -> Iterator[str] yield "science" - yield _qualified_science_binary_name() + yield _qualified_science_fat_binary_name() + yield SciePlatform.current().qualified_binary_name("science") + + +def _is_compatible_science_binary( + binary, # type: str + source=None, # type: Optional[str] +): + # type: (...) -> Union[Version, Error] + try: + version = Version( + subprocess.check_output(args=[binary, "--version"]).decode("utf-8").strip() + ) + except (CalledProcessError, InvalidVersion) as e: + return Error( + "Failed to determine --version of science binary at {source}: {err}".format( + source=source or binary, err=e + ) + ) + else: + if version.raw in SCIENCE_REQUIREMENT: + return version + return Error( + "The science binary at {source} is version {version} which does not match Pex's " + "science requirement of {science_requirement}.".format( + source=source or binary, + version=version.raw, + science_requirement=SCIENCE_REQUIREMENT, + ) + ) def _path_science(): @@ -171,29 +217,15 @@ def _path_science(): ): if not is_exe(binary): continue - try: - if ( - Version(subprocess.check_output(args=[binary, "--version"]).decode("utf-8")) - < MIN_SCIENCE_VERSION - ): - continue - except (CalledProcessError, InvalidVersion): + if isinstance(_is_compatible_science_binary(binary), Error): continue return binary return None -def _science_binary_url(suffix=""): - # type: (str) -> str - return "https://github.com/a-scie/science/releases/download/v{version}/{binary}{suffix}".format( - version=MIN_SCIENCE_VERSION, - binary=_qualified_science_binary_name(), - suffix=suffix, - ) - - def _ensure_science( url_fetcher=None, # type: Optional[URLFetcher] + science_binary_url=None, # type: Optional[str] env=ENV, # type: Variables ): # type: (...) -> str @@ -207,27 +239,40 @@ def _ensure_science( shutil.copy(path_science, target_science) else: fetcher = url_fetcher or URLFetcher() - science_binary_url = _science_binary_url() with open(target_science, "wb") as write_fp, fetcher.get_body_stream( - science_binary_url + science_binary_url or _science_binary_url() ) as read_fp: shutil.copyfileobj(read_fp, write_fp) + chmod_plus_x(target_science) - science_sha256_url = _science_binary_url(".sha256") - with fetcher.get_body_stream(science_sha256_url) as fp: - expected_sha256, _, _ = fp.read().decode("utf-8").partition(" ") - actual_sha256 = CacheHelper.hash(target_science, hasher=Sha256) - if expected_sha256 != actual_sha256: - raise ValueError( - "The science binary downloaded from {science_binary_url} does not match " - "the expected SHA-256 fingerprint recorded in {science_sha256_url}.\n" - "Expected {expected_sha256} but found {actual_sha256}.".format( - science_binary_url=science_binary_url, - science_sha256_url=science_sha256_url, - expected_sha256=expected_sha256, - actual_sha256=actual_sha256, + if science_binary_url: + custom_science_binary_version = try_( + _is_compatible_science_binary(target_science, source=science_binary_url) + ) + TRACER.log( + "Using custom science binary from {source} with version {version}.".format( + source=science_binary_url, version=custom_science_binary_version.raw ) ) + else: + # Since we used the canonical GitHub Releases URL, we know a checksum file is + # available we can use to verify. + science_sha256_url = _science_binary_url(".sha256") + with fetcher.get_body_stream(science_sha256_url) as fp: + expected_sha256, _, _ = fp.read().decode("utf-8").partition(" ") + actual_sha256 = CacheHelper.hash(target_science, hasher=Sha256) + if expected_sha256 != actual_sha256: + raise ValueError( + "The science binary downloaded from {science_binary_url} does not " + "match the expected SHA-256 fingerprint recorded in " + "{science_sha256_url}.\n" + "Expected {expected_sha256} but found {actual_sha256}.".format( + science_binary_url=science_binary_url, + science_sha256_url=science_sha256_url, + expected_sha256=expected_sha256, + actual_sha256=actual_sha256, + ) + ) return os.path.join(target_dir, "science") @@ -243,7 +288,11 @@ def build( ): # type: (...) -> Iterator[ScieInfo] - science = _ensure_science(url_fetcher=url_fetcher, env=env) + science = _ensure_science( + url_fetcher=url_fetcher, + science_binary_url=configuration.options.science_binary_url, + env=env, + ) name = re.sub(r"\.pex$", "", os.path.basename(pex_file), flags=re.IGNORECASE) pex_info = PexInfo.from_pex(pex_file) layout = Layout.identify(pex_file) From 701001c6cbb1823fd25fdb43fa476e6b8d463cdc Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 11:19:31 -0700 Subject: [PATCH 06/12] Only mount ~/.netrc and ~/.ssh when present. --- dtox.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dtox.sh b/dtox.sh index 8e258634c..c3d306943 100755 --- a/dtox.sh +++ b/dtox.sh @@ -117,11 +117,21 @@ if [[ -n "${TERM:-}" ]]; then ) fi +if [[ -f "${HOME}/.netrc" ]]; then + DOCKER_ARGS+=( + --volume "${HOME}/.netrc:${CONTAINER_HOME}/.netrc" + ) +fi + +if [[ -d "${HOME}/.ssh" ]]; then + DOCKER_ARGS+=( + --volume "${HOME}/.ssh:${CONTAINER_HOME}/.ssh" + ) +fi + exec docker run \ --rm \ --volume pex-tmp:/tmp \ - --volume "${HOME}/.netrc:${CONTAINER_HOME}/.netrc" \ - --volume "${HOME}/.ssh:${CONTAINER_HOME}/.ssh" \ --volume "pex-root:${CONTAINER_HOME}/.pex" \ --volume pex-caches:/development/pex_dev \ --volume "${ROOT}:/development/pex" \ From 61f55a458a23623130ee462f6ab11046e351bf4a Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 13:24:45 -0700 Subject: [PATCH 07/12] Add a test for multiple platforms. Exercise platform / interpreter auto detection as well as explicit restriction via `--scie-platform`. Also open up control via `PEX_*` env vars: there was no need to mask thse for proper scie operation and the end result is a scie that works exactly like the PEX it was built from. --- pex/scie/science.py | 29 +++--- tests/integration/scie/test_pex_scie.py | 123 +++++++++++++++++++++++- 2 files changed, 134 insertions(+), 18 deletions(-) diff --git a/pex/scie/science.py b/pex/scie/science.py index f357c9026..61f9f7ac5 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -90,13 +90,9 @@ def create_manifests( pex_hash = cast(str, pex_info.pex_hash) installed_pex_dir = os.path.join(pex_root, unzip_dir_relpath(pex_hash)) - env_replace = { + env_default = { "PEX_ROOT": pex_root, } - env = { - "remove_re": {"PEX_.*"}, - "replace": env_replace, - } lift = { "name": name, @@ -109,24 +105,24 @@ def create_manifests( "files": [{"name": "configure-binding.py"}, {"name": "pex"}], "commands": [ { - "env": env, + "env": {"default": env_default}, "exe": "{scie.bindings.configure:PYTHON}", "args": ["{scie.bindings.configure:PEX}"], } ], "bindings": [ { - "env": dict( - env, - replace=dict( - env_replace, - PEX_INTERPRETER="1", - _PEX_SCIE_INSTALLED_PEX_DIR=installed_pex_dir, + "env": { + "default": env_default, + "remove_re": ["PEX_.*"], + "replace": { + "PEX_INTERPRETER": "1", + "_PEX_SCIE_INSTALLED_PEX_DIR": installed_pex_dir, # We can get a warning about too-long script shebangs, but this is not # relevant since we above run the PEX via python and not via shebang. - PEX_EMIT_WARNINGS="0", - ), - ), + "PEX_EMIT_WARNINGS": "0", + }, + }, "name": "configure", "exe": "#{cpython:python}", "args": ["{pex}", "{configure-binding.py}"], @@ -297,6 +293,7 @@ def build( pex_info = PexInfo.from_pex(pex_file) layout = Layout.identify(pex_file) use_platform_suffix = len(configuration.targets) > 1 + errors = OrderedDict() # type: OrderedDict[Manifest, str] for manifest in create_manifests(configuration, name, pex_info, layout): args = [science, "--cache-dir", _science_dir(env, "cache")] @@ -350,8 +347,8 @@ def build( else manifest.binary_name(name), ), ) - if errors: + if errors: raise ScienceError( "Failed to build {count} {scies}:\n\n{errors}".format( count=len(errors), diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py index f7612d6b0..82db0b466 100644 --- a/tests/integration/scie/test_pex_scie.py +++ b/tests/integration/scie/test_pex_scie.py @@ -9,14 +9,16 @@ import pytest +from pex.common import is_exe from pex.layout import Layout -from pex.scie import ScieStyle +from pex.orderedset import OrderedSet +from pex.scie import SciePlatform, ScieStyle from pex.targets import LocalInterpreter from pex.typing import TYPE_CHECKING from testing import IS_PYPY, PY_VER, make_env, run_pex_command if TYPE_CHECKING: - from typing import Any, List + from typing import Any, Iterable, List @pytest.mark.parametrize( @@ -87,3 +89,120 @@ def test_basic( scie = os.path.join(str(tmpdir), "cowsay") assert b"| PAR! |" in subprocess.check_output(args=[scie, "PAR!"], env=make_env(PATH=None)) + + +def test_multiple_platforms(tmpdir): + # type: (Any) -> None + + def create_scies( + output_dir, # type: str + extra_args=(), # type: Iterable[str] + ): + pex = os.path.join(output_dir, "cowsay.pex") + run_pex_command( + args=[ + "cowsay==5.0", + "-c", + "cowsay", + "-o", + pex, + "--scie", + "lazy", + "--platform", + "linux-aarch64-cp-39-cp39", + "--platform", + "linux-x86_64-cp-310-cp310", + "--platform", + "macosx-10.9-arm64-cp-311-cp311", + "--platform", + "macosx-10.9-x86_64-cp-312-cp312", + ] + + list(extra_args) + ).assert_success() + + python_version_by_platform = { + SciePlatform.LINUX_AARCH64: "3.9", + SciePlatform.LINUX_X86_64: "3.10", + SciePlatform.MACOS_AARCH64: "3.11", + SciePlatform.MACOS_X86_64: "3.12", + } + assert SciePlatform.current() in python_version_by_platform + + def assert_platforms( + output_dir, # type: str + expected_platforms, # type: Iterable[SciePlatform.Value] + ): + # type: (...) -> None + + all_output_files = set( + path + for path in os.listdir(output_dir) + if os.path.isfile(os.path.join(output_dir, path)) + ) + for platform in OrderedSet(expected_platforms): + python_version = python_version_by_platform[platform] + binary = platform.qualified_binary_name("cowsay") + assert binary in all_output_files + all_output_files.remove(binary) + scie = os.path.join(output_dir, binary) + assert is_exe(scie), "Expected --scie build to produce a {binary} binary.".format( + binary=binary + ) + if platform is SciePlatform.current(): + assert b"| PEX-scie wabbit! |" in subprocess.check_output( + args=[scie, "PEX-scie wabbit!"], env=make_env(PATH=None) + ) + assert ( + python_version + == subprocess.check_output( + args=[ + scie, + "-c", + "import sys; print('.'.join(map(str, sys.version_info[:2])))", + ], + env=make_env(PEX_INTERPRETER=1), + ) + .decode("utf-8") + .strip() + ) + assert {"cowsay.pex"} == all_output_files, ( + "Expected one output scie for each platform plus the original cowsay.pex. All expected " + "scies were found, but the remaining files are: {remaining_files}".format( + remaining_files=all_output_files + ) + ) + + all_platforms_output_dir = os.path.join(str(tmpdir), "all-platforms") + create_scies(output_dir=all_platforms_output_dir) + assert_platforms( + output_dir=all_platforms_output_dir, + expected_platforms=( + SciePlatform.LINUX_AARCH64, + SciePlatform.LINUX_X86_64, + SciePlatform.MACOS_AARCH64, + SciePlatform.MACOS_X86_64, + ), + ) + + # Now restrict the PEX's implied natural platform set of 4 down to 2 or 3 using + # `--scie-platform`. + restricted_platforms_output_dir = os.path.join(str(tmpdir), "restricted-platforms") + create_scies( + output_dir=restricted_platforms_output_dir, + extra_args=[ + "--scie-platform", + str(SciePlatform.current()), + "--scie-platform", + str(SciePlatform.LINUX_AARCH64), + "--scie-platform", + str(SciePlatform.LINUX_X86_64), + ], + ) + assert_platforms( + output_dir=restricted_platforms_output_dir, + expected_platforms=( + SciePlatform.current(), + SciePlatform.LINUX_AARCH64, + SciePlatform.LINUX_X86_64, + ), + ) From 0cdd1b2bbdf2c4f90b037669137c3f2193b14864 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 16:42:03 -0700 Subject: [PATCH 08/12] Add remaining tests. --- pex/scie/__init__.py | 3 + tests/integration/scie/test_pex_scie.py | 163 ++++++++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/pex/scie/__init__.py b/pex/scie/__init__.py index a4efaf187..14a96dd9a 100644 --- a/pex/scie/__init__.py +++ b/pex/scie/__init__.py @@ -34,6 +34,9 @@ "ScieStyle", "ScieTarget", "build", + "extract_options", + "register_options", + "render_options", ) diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py index 82db0b466..b671d74da 100644 --- a/tests/integration/scie/test_pex_scie.py +++ b/tests/integration/scie/test_pex_scie.py @@ -3,9 +3,13 @@ from __future__ import absolute_import +import glob +import json import os.path import re import subprocess +import sys +from typing import Optional import pytest @@ -206,3 +210,162 @@ def assert_platforms( SciePlatform.LINUX_X86_64, ), ) + + +PRINT_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))" + + +skip_if_pypy = pytest.mark.skipif(IS_PYPY, reason="PyPy targeted PEXes do not support --scie.") + + +@skip_if_pypy +def test_specified_interpreter(tmpdir): + # type: (Any) -> None + + pex = os.path.join(str(tmpdir), "empty.pex") + run_pex_command( + args=[ + "-o", + pex, + "--scie", + "lazy", + # We pick a specific version that is not in the latest release but is known to provide + # distributions for all platforms Pex tests run on. + "--scie-pbs-release", + "20221002", + "--scie-python-version", + "3.10.7", + ], + ).assert_success() + + assert ( + ".".join(map(str, sys.version_info[:3])) + == subprocess.check_output(args=[pex, "-c", PRINT_VERSION_SCRIPT]).decode("utf-8").strip() + ) + + scie = os.path.join(str(tmpdir), "empty") + assert b"3.10.7\n" == subprocess.check_output(args=[scie, "-c", PRINT_VERSION_SCRIPT]) + + +@skip_if_pypy +def test_specified_science_binary(tmpdir): + # type: (Any) -> None + + pex_root = os.path.join(str(tmpdir), "pex_root") + scie = os.path.join(str(tmpdir), "cowsay") + run_pex_command( + args=[ + "--pex-root", + pex_root, + "cowsay==6.0", + "-c", + "cowsay", + "--scie", + "lazy", + "--scie-python-version", + "3.12", + "-o", + scie, + "--scie-science-binary", + # N.B.: This custom version is both lower than the latest available version (0.4.2 + # at the time of writing) and higher than the minimum supported version of 0.3.0; so + # we can prove we downloaded the custom version via this URL by checking the version + # below since our next floor bump will be from 0.3.0 to at least 0.4.3. + "https://github.com/a-scie/lift/releases/download/v0.4.0/{binary}".format( + binary=SciePlatform.current().qualified_binary_name("science") + ), + ], + env=make_env(PATH=None), + ).assert_success() + + assert b"| Alternative SCIENCE Facts! |" in subprocess.check_output( + args=[scie, "-t", "Alternative SCIENCE Facts!"] + ) + + science_binaries = glob.glob(os.path.join(pex_root, "scies", "science", "*", "bin", "science")) + assert 1 == len(science_binaries) + science = science_binaries[0] + assert "0.4.0" == subprocess.check_output(args=[science, "--version"]).decode("utf-8").strip() + + +@skip_if_pypy +def test_custom_lazy_urls(tmpdir): + # type: (Any) -> None + + scie = os.path.join(str(tmpdir), "empty") + run_pex_command( + args=[ + "-o", + scie, + "--scie", + "lazy", + "--scie-pbs-release", + "20221002", + "--scie-python-version", + "3.10.7", + ], + ).assert_success() + + assert b"3.10.7\n" == subprocess.check_output(args=[scie, "-c", PRINT_VERSION_SCRIPT]) + + pex_bootstrap_urls = os.path.join(str(tmpdir), "pex_bootstrap_urls.json") + + def make_20221002_3_10_7_file(platform): + # type: (str) -> str + return "cpython-3.10.7+20221002-{platform}-install_only.tar.gz".format(platform=platform) + + def make_20240415_3_10_14_url(platform): + # type: (str) -> str + return ( + "https://github.com/indygreg/python-build-standalone/releases/download/20240415/" + "cpython-3.10.14+20240415-{platform}-install_only.tar.gz".format(platform=platform) + ) + + with open(pex_bootstrap_urls, "w") as fp: + json.dump( + { + "ptex": { + make_20221002_3_10_7_file(platform): make_20240415_3_10_14_url(platform) + for platform in ( + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ) + } + }, + fp, + ) + + process = subprocess.Popen( + args=[scie, "-c", PRINT_VERSION_SCRIPT], + env=make_env( + PEX_BOOTSTRAP_URLS=pex_bootstrap_urls, SCIE_BASE=os.path.join(str(tmpdir), "nce") + ), + stderr=subprocess.PIPE, + ) + _, stderr = process.communicate() + assert 0 != process.returncode, ( + "Expected PEX_BOOTSTRAP_URLS to be used and the resulting fetched interpreter distribution " + "to fail its digest check." + ) + + expected_platform = None # type: Optional[str] + if SciePlatform.current() is SciePlatform.LINUX_AARCH64: + expected_platform = "aarch64-unknown-linux-gnu" + elif SciePlatform.current() is SciePlatform.LINUX_X86_64: + expected_platform = "x86_64-unknown-linux-gnu" + elif SciePlatform.current() is SciePlatform.MACOS_X86_64: + expected_platform = "aarch64-apple-darwin" + elif SciePlatform.current() is SciePlatform.MACOS_X86_64: + expected_platform = "x86_64-apple-darwin" + assert expected_platform is not None + + assert re.match( + r"^.*Population of work directory failed: The tar\.gz destination .*{expected_file_name} " + r"of size \d+ had unexpected hash: [a-f0-9]{{64}}$.*".format( + expected_file_name=re.escape(make_20221002_3_10_7_file(expected_platform)) + ), + stderr.decode("utf-8"), + flags=re.DOTALL | re.MULTILINE, + ), stderr.decode("utf-8") From df152017cc10c9135958da9b2f51fb14ede42dc0 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 14:37:28 -0700 Subject: [PATCH 09/12] Perpare the 2.11.0 release with `--scie` support. --- CHANGES.md | 17 +++++++++++++++++ pex/version.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index fc44c3984..d35564b33 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,22 @@ # Release Notes +## 2.11.0 + +This release adds support for creating native PEX executables that +contain their own hermetic CPython interpreter courtesy of +[Python Standalone Builds][PBS] and the [Science project][scie]. + +You can now specify `--scie {eager,lazy}` when building a PEX file and +one or more native executable PEX scies will be produced (one for each +platform the PEX supports). These PEX scies are single file +executables that look and behave like traditional PEXes, but unlike +PEXes they can run on a machine with no Python interpreter available. + +[PBS]: https://github.com/indygreg/python-build-standalone +[scie]: https://github.com/a-scie + +* Add `--scie` option to produce native PEX exes. (#2466) + ## 2.10.1 This release fixes a long-standing bug in Pex parsing of editable diff --git a/pex/version.py b/pex/version.py index 9c06c95ad..be6b005a1 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.10.1" +__version__ = "2.11.0" From 0f3eba1b0e9903926de4272b3ac3c38416cf51cf Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 16:55:05 -0700 Subject: [PATCH 10/12] Fix copypasta. --- tests/integration/scie/test_pex_scie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py index b671d74da..929917ffb 100644 --- a/tests/integration/scie/test_pex_scie.py +++ b/tests/integration/scie/test_pex_scie.py @@ -355,7 +355,7 @@ def make_20240415_3_10_14_url(platform): expected_platform = "aarch64-unknown-linux-gnu" elif SciePlatform.current() is SciePlatform.LINUX_X86_64: expected_platform = "x86_64-unknown-linux-gnu" - elif SciePlatform.current() is SciePlatform.MACOS_X86_64: + elif SciePlatform.current() is SciePlatform.MACOS_AARCH64: expected_platform = "aarch64-apple-darwin" elif SciePlatform.current() is SciePlatform.MACOS_X86_64: expected_platform = "x86_64-apple-darwin" From d7515761b40a239ba9e05d526d4fc2644c166eaa Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 19:05:18 -0700 Subject: [PATCH 11/12] Various cleanups. --- pex/bin/pex.py | 8 ++-- pex/pex_bootstrapper.py | 2 +- pex/scie/__init__.py | 14 ++++-- pex/scie/model.py | 57 ++++++++++++++----------- pex/scie/science.py | 14 +++--- tests/integration/scie/test_pex_scie.py | 18 ++++---- 6 files changed, 61 insertions(+), 52 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index e6a4239d0..22925f4ad 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -1308,14 +1308,14 @@ def do_main( handle_file_urls=True, ) with TRACER.timed("Building scie(s)"): - for par_info in scie.build( + for scie_info in scie.build( configuration=scie_configuration, pex_file=pex_file, url_fetcher=url_fetcher ): log( "Saved PEX scie for CPython {version} on {platform} to {scie}".format( - version=par_info.target.version_str, - platform=par_info.platform, - scie=os.path.relpath(par_info.file), + version=scie_info.target.version_str, + platform=scie_info.platform, + scie=os.path.relpath(scie_info.file), ), V=options.verbosity, ) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index a097736fa..e3609efc9 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -314,7 +314,7 @@ def gather_constraints(): path=( os.pathsep.join(ENV.PEX_PYTHON_PATH) if ENV.PEX_PYTHON_PATH - else os.getenv("PATH") + else os.getenv("PATH", "(The PATH is empty!)") ) ) ) diff --git a/pex/scie/__init__.py b/pex/scie/__init__.py index 14a96dd9a..3e168c9d8 100644 --- a/pex/scie/__init__.py +++ b/pex/scie/__init__.py @@ -77,11 +77,19 @@ def register_options(parser): dest="scie_platforms", default=[], action="append", - type=SciePlatform.for_value, - choices=SciePlatform.values(), + type=SciePlatform.parse, + choices=[ + platform + for platform in SciePlatform.values() + if platform not in (SciePlatform.WINDOWS_AARCH64, SciePlatform.WINDOWS_X86_64) + ], help=( "The platform to produce the native PEX scie executable for. Can be specified multiple " - "times." + "times. You can use a value of 'current' to select the current platform. If left " + "unspecified, the platforms implied by the targets selected to build the PEX with are " + "used. Those targets are influenced by the current interpreter running Pex as well as " + "use of `--python`, `--interpreter-constraint`, `--platform` or `--complete-platform` " + "options." ), ) parser.add_argument( diff --git a/pex/scie/model.py b/pex/scie/model.py index 5f1737718..07e46c6a8 100644 --- a/pex/scie/model.py +++ b/pex/scie/model.py @@ -30,6 +30,35 @@ class Value(Enum.Value): EAGER = Value("eager") +class _CurrentPlatform(object): + def __get__(self, obj, objtype=None): + # type: (...) -> SciePlatform.Value + if not hasattr(self, "_current"): + system = platform.system().lower() + machine = platform.machine().lower() + if "linux" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.LINUX_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.LINUX_X86_64 + elif "darwin" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.MACOS_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.MACOS_X86_64 + elif "windows" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.WINDOWS_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.WINDOWS_X86_64 + if not hasattr(self, "_current"): + raise ValueError( + "The current operating system / machine pair is not supported!: " + "{system} / {machine}".format(system=system, machine=machine) + ) + return self._current + + class SciePlatform(Enum["SciePlatform.Value"]): class Value(Enum.Value): @property @@ -64,36 +93,12 @@ def qualified_file_name(self, file_name): MACOS_X86_64 = Value("macos-x86_64") WINDOWS_AARCH64 = Value("windows-x86_64") WINDOWS_X86_64 = Value("windows-aarch64") + CURRENT = _CurrentPlatform() @classmethod def parse(cls, value): # type: (str) -> SciePlatform.Value - return cls.current() if "current" == value else cls.for_value(value) - - @classmethod - def current(cls): - # type: () -> SciePlatform.Value - system = platform.system().lower() - machine = platform.machine().lower() - if "linux" == system: - if machine in ("aarch64", "arm64"): - return cls.LINUX_AARCH64 - elif machine in ("amd64", "x86_64"): - return cls.LINUX_X86_64 - elif "darwin" == system: - if machine in ("aarch64", "arm64"): - return cls.MACOS_AARCH64 - elif machine in ("amd64", "x86_64"): - return cls.MACOS_X86_64 - elif "windows" == system: - if machine in ("aarch64", "arm64"): - return cls.WINDOWS_AARCH64 - elif machine in ("amd64", "x86_64"): - return cls.WINDOWS_X86_64 - raise ValueError( - "The current operating system / machine pair is not supported!: " - "{system} / {machine}".format(system=system, machine=machine) - ) + return cls.CURRENT if "current" == value else cls.for_value(value) @attr.s(frozen=True) diff --git a/pex/scie/science.py b/pex/scie/science.py index 61f9f7ac5..a9677e397 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -61,7 +61,7 @@ def _science_binary_url(suffix=""): return "{science_releases_url}/download/v{version}/{binary}{suffix}".format( science_releases_url=SCIENCE_RELEASES_URL, version=MIN_SCIENCE_VERSION.raw, - binary=_qualified_science_fat_binary_name(), + binary=SciePlatform.CURRENT.qualified_binary_name("science-fat"), suffix=suffix, ) @@ -165,16 +165,12 @@ def _science_dir( return os.path.join(env.PEX_ROOT, "scies", "science", MIN_SCIENCE_VERSION.raw, *components) -def _qualified_science_fat_binary_name(): - # type: () -> str - return SciePlatform.current().qualified_binary_name("science-fat") - - def _science_binary_names(): # type: () -> Iterator[str] - yield "science" - yield _qualified_science_fat_binary_name() - yield SciePlatform.current().qualified_binary_name("science") + yield SciePlatform.CURRENT.binary_name("science-fat") + yield SciePlatform.CURRENT.qualified_binary_name("science-fat") + yield SciePlatform.CURRENT.binary_name("science") + yield SciePlatform.CURRENT.qualified_binary_name("science") def _is_compatible_science_binary( diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py index 929917ffb..34b95ec23 100644 --- a/tests/integration/scie/test_pex_scie.py +++ b/tests/integration/scie/test_pex_scie.py @@ -130,7 +130,7 @@ def create_scies( SciePlatform.MACOS_AARCH64: "3.11", SciePlatform.MACOS_X86_64: "3.12", } - assert SciePlatform.current() in python_version_by_platform + assert SciePlatform.CURRENT in python_version_by_platform def assert_platforms( output_dir, # type: str @@ -152,7 +152,7 @@ def assert_platforms( assert is_exe(scie), "Expected --scie build to produce a {binary} binary.".format( binary=binary ) - if platform is SciePlatform.current(): + if platform is SciePlatform.CURRENT: assert b"| PEX-scie wabbit! |" in subprocess.check_output( args=[scie, "PEX-scie wabbit!"], env=make_env(PATH=None) ) @@ -195,7 +195,7 @@ def assert_platforms( output_dir=restricted_platforms_output_dir, extra_args=[ "--scie-platform", - str(SciePlatform.current()), + "current", "--scie-platform", str(SciePlatform.LINUX_AARCH64), "--scie-platform", @@ -205,7 +205,7 @@ def assert_platforms( assert_platforms( output_dir=restricted_platforms_output_dir, expected_platforms=( - SciePlatform.current(), + SciePlatform.CURRENT, SciePlatform.LINUX_AARCH64, SciePlatform.LINUX_X86_64, ), @@ -272,7 +272,7 @@ def test_specified_science_binary(tmpdir): # we can prove we downloaded the custom version via this URL by checking the version # below since our next floor bump will be from 0.3.0 to at least 0.4.3. "https://github.com/a-scie/lift/releases/download/v0.4.0/{binary}".format( - binary=SciePlatform.current().qualified_binary_name("science") + binary=SciePlatform.CURRENT.qualified_binary_name("science") ), ], env=make_env(PATH=None), @@ -351,13 +351,13 @@ def make_20240415_3_10_14_url(platform): ) expected_platform = None # type: Optional[str] - if SciePlatform.current() is SciePlatform.LINUX_AARCH64: + if SciePlatform.CURRENT is SciePlatform.LINUX_AARCH64: expected_platform = "aarch64-unknown-linux-gnu" - elif SciePlatform.current() is SciePlatform.LINUX_X86_64: + elif SciePlatform.CURRENT is SciePlatform.LINUX_X86_64: expected_platform = "x86_64-unknown-linux-gnu" - elif SciePlatform.current() is SciePlatform.MACOS_AARCH64: + elif SciePlatform.CURRENT is SciePlatform.MACOS_AARCH64: expected_platform = "aarch64-apple-darwin" - elif SciePlatform.current() is SciePlatform.MACOS_X86_64: + elif SciePlatform.CURRENT is SciePlatform.MACOS_X86_64: expected_platform = "x86_64-apple-darwin" assert expected_platform is not None From 7a8f197c2ba9807044a9d677096b30776ec7af21 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 19:13:22 -0700 Subject: [PATCH 12/12] @zmanji speaks sense --- pex/scie/science.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pex/scie/science.py b/pex/scie/science.py index a9677e397..509358941 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -114,6 +114,7 @@ def create_manifests( { "env": { "default": env_default, + "remove_exact": ["PATH"], "remove_re": ["PEX_.*"], "replace": { "PEX_INTERPRETER": "1",