diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cf23e4c..5c849752a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,9 @@ env: # importing builtins like `fcntl` as outlined in https://github.com/pex-tool/pex/issues/1391. _PEX_TEST_PYENV_VERSIONS: "2.7 3.7 3.10" _PEX_PEXPECT_TIMEOUT: 10 + # We have integration tests that exercise `--scie` support and these can trigger downloads from + # GitHub Releases that needed elevated rate limit quota, which this gives. + SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }} concurrency: group: CI-${{ github.ref }} # Queue on all branches and tags, but only cancel overlapping PR burns. diff --git a/CHANGES.md b/CHANGES.md index fc44c3984..d35564b33 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,22 @@ # Release Notes +## 2.11.0 + +This release adds support for creating native PEX executables that +contain their own hermetic CPython interpreter courtesy of +[Python Standalone Builds][PBS] and the [Science project][scie]. + +You can now specify `--scie {eager,lazy}` when building a PEX file and +one or more native executable PEX scies will be produced (one for each +platform the PEX supports). These PEX scies are single file +executables that look and behave like traditional PEXes, but unlike +PEXes they can run on a machine with no Python interpreter available. + +[PBS]: https://github.com/indygreg/python-build-standalone +[scie]: https://github.com/a-scie + +* Add `--scie` option to produce native PEX exes. (#2466) + ## 2.10.1 This release fixes a long-standing bug in Pex parsing of editable diff --git a/dtox.sh b/dtox.sh index 8e258634c..c3d306943 100755 --- a/dtox.sh +++ b/dtox.sh @@ -117,11 +117,21 @@ if [[ -n "${TERM:-}" ]]; then ) fi +if [[ -f "${HOME}/.netrc" ]]; then + DOCKER_ARGS+=( + --volume "${HOME}/.netrc:${CONTAINER_HOME}/.netrc" + ) +fi + +if [[ -d "${HOME}/.ssh" ]]; then + DOCKER_ARGS+=( + --volume "${HOME}/.ssh:${CONTAINER_HOME}/.ssh" + ) +fi + exec docker run \ --rm \ --volume pex-tmp:/tmp \ - --volume "${HOME}/.netrc:${CONTAINER_HOME}/.netrc" \ - --volume "${HOME}/.ssh:${CONTAINER_HOME}/.ssh" \ --volume "pex-root:${CONTAINER_HOME}/.pex" \ --volume pex-caches:/development/pex_dev \ --volume "${ROOT}:/development/pex" \ diff --git a/pex/bin/pex.py b/pex/bin/pex.py index c60aa1627..22925f4ad 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -16,7 +16,7 @@ from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentError, ArgumentParser from textwrap import TextWrapper -from pex import dependency_configuration, pex_warnings +from pex import dependency_configuration, pex_warnings, scie from pex.argparse import HandleBoolAction from pex.commands.command import ( GlobalConfigurationError, @@ -29,6 +29,7 @@ from pex.dist_metadata import Requirement from pex.docs.command import serve_html_docs from pex.enum import Enum +from pex.fetcher import URLFetcher from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraint, InterpreterConstraints from pex.layout import Layout, ensure_installed @@ -56,6 +57,7 @@ from pex.resolve.resolver_options import create_pip_configuration from pex.resolve.resolvers import Unsatisfiable, sorted_requirements from pex.result import Error, ResultError, catch, try_ +from pex.scie import ScieConfiguration from pex.targets import Targets from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast @@ -314,6 +316,8 @@ def configure_clp_pex_options(parser): ), ) + scie.register_options(group) + group.add_argument( "--always-write-cache", dest="always_write_cache", @@ -1233,6 +1237,27 @@ def do_main( cmdline, # type: List[str] env, # type: Dict[str, str] ): + scie_options = scie.extract_options(options) + if scie_options and not options.pex_name: + raise ValueError( + "You must specify `-o`/`--output-file` to use `{scie_options}`.".format( + scie_options=scie.render_options(scie_options) + ) + ) + scie_configuration = None # type: Optional[ScieConfiguration] + if scie_options: + scie_configuration = scie_options.create_configuration(targets=targets) + if not scie_configuration: + raise ValueError( + "You selected `{scie_options}`, but none of the selected targets have " + "compatible interpreters that can be embedded to form a scie:\n{targets}".format( + scie_options=scie.render_options(scie_options), + targets="\n".join( + target.render_description() for target in targets.unique_targets() + ), + ) + ) + with TRACER.timed("Building pex"): pex_builder = build_pex( requirement_configuration=requirement_configuration, @@ -1276,6 +1301,24 @@ def do_main( verbose=options.seed == Seed.VERBOSE, ) print(seed_info) + if scie_configuration: + url_fetcher = URLFetcher( + network_configuration=resolver_configuration.network_configuration, + password_entries=resolver_configuration.repos_configuration.password_entries, + handle_file_urls=True, + ) + with TRACER.timed("Building scie(s)"): + for scie_info in scie.build( + configuration=scie_configuration, pex_file=pex_file, url_fetcher=url_fetcher + ): + log( + "Saved PEX scie for CPython {version} on {platform} to {scie}".format( + version=scie_info.target.version_str, + platform=scie_info.platform, + scie=os.path.relpath(scie_info.file), + ), + V=options.verbosity, + ) else: if not _compatible_with_current_platform(interpreter, targets.platforms): log("WARNING: attempting to run PEX with incompatible platforms!", V=1) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index a097736fa..e3609efc9 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -314,7 +314,7 @@ def gather_constraints(): path=( os.pathsep.join(ENV.PEX_PYTHON_PATH) if ENV.PEX_PYTHON_PATH - else os.getenv("PATH") + else os.getenv("PATH", "(The PATH is empty!)") ) ) ) diff --git a/pex/platforms.py b/pex/platforms.py index f43950e87..a1693347d 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -29,7 +29,7 @@ def _normalize_platform(platform): # type: (str) -> str - return platform.replace("-", "_").replace(".", "_") + return platform.lower().replace("-", "_").replace(".", "_") @attr.s(frozen=True) diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index 83badd4fb..61f788fcd 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -199,9 +199,24 @@ class PexRepositoryConfiguration(object): network_configuration = attr.ib(default=NetworkConfiguration()) # type: NetworkConfiguration transitive = attr.ib(default=True) # type: bool + @property + def repos_configuration(self): + # type: () -> ReposConfiguration + return ReposConfiguration() + @attr.s(frozen=True) class LockRepositoryConfiguration(object): parse_lock = attr.ib() # type: Callable[[], Union[Lockfile, Error]] lock_file_path = attr.ib() # type: str pip_configuration = attr.ib() # type: PipConfiguration + + @property + def repos_configuration(self): + # type: () -> ReposConfiguration + return self.pip_configuration.repos_configuration + + @property + def network_configuration(self): + # type: () -> NetworkConfiguration + return self.pip_configuration.network_configuration diff --git a/pex/scie/__init__.py b/pex/scie/__init__.py new file mode 100644 index 000000000..3e168c9d8 --- /dev/null +++ b/pex/scie/__init__.py @@ -0,0 +1,213 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +from argparse import Namespace, _ActionsContainer + +from pex.compatibility import urlparse +from pex.fetcher import URLFetcher +from pex.orderedset import OrderedSet +from pex.pep_440 import Version +from pex.scie import science +from pex.scie.model import ( + ScieConfiguration, + ScieInfo, + ScieOptions, + SciePlatform, + ScieStyle, + ScieTarget, +) +from pex.scie.science import SCIENCE_RELEASES_URL, SCIENCE_REQUIREMENT +from pex.typing import TYPE_CHECKING, cast +from pex.variables import ENV, Variables + +if TYPE_CHECKING: + from typing import Iterator, Optional, Tuple, Union + + +__all__ = ( + "ScieConfiguration", + "ScieInfo", + "SciePlatform", + "ScieStyle", + "ScieTarget", + "build", + "extract_options", + "register_options", + "render_options", +) + + +def register_options(parser): + # type: (_ActionsContainer) -> None + + parser.add_argument( + "--scie", + "--par", + dest="scie_style", + default=None, + type=ScieStyle.for_value, + choices=ScieStyle.values(), + help=( + "Create one or more native executable scies from your PEX that include a portable " + "CPython interpreter along with your PEX making for a truly hermetic PEX that can run " + "on machines with no Python installed at all. If your PEX has multiple targets, " + "whether `--platform`s, `--complete-platform`s or local interpreters in any " + "combination, then one PEX scie will be made for each platform, selecting the latest " + "compatible portable CPython interpreter. Note that only CPython>=3.8 is supported. If " + "you'd like to explicitly control the target platforms or the exact portable CPython " + "selected, see `--scie-platform`, `--scie-pbs-release` and `--scie-python-version`. " + "Specifying `--scie {lazy}` will fetch the portable CPython interpreter just in time " + "on first boot of the PEX scie on a given machine if needed. The URL(s) to fetch the " + "portable CPython interpreter from can be customized by exporting the " + "PEX_BOOTSTRAP_URLS environment variable pointing to a json file with the format: " + '`{{"ptex": {{: , ...}}}}` where the file names should match those ' + "found via `SCIE=inspect | jq .ptex` with appropriate replacement URLs. " + "Specifying `--scie {eager}` will embed the portable CPython interpreter in your PEX " + "scie making for a larger file, but requiring no internet access to boot. If you have " + "customization needs not addressed by the Pex `--scie*` options, consider using " + "`science` to build your scies (which is what Pex uses behind the scenes); see: " + "https://science.scie.app.".format(lazy=ScieStyle.LAZY, eager=ScieStyle.EAGER) + ), + ) + parser.add_argument( + "--scie-platform", + dest="scie_platforms", + default=[], + action="append", + type=SciePlatform.parse, + choices=[ + platform + for platform in SciePlatform.values() + if platform not in (SciePlatform.WINDOWS_AARCH64, SciePlatform.WINDOWS_X86_64) + ], + help=( + "The platform to produce the native PEX scie executable for. Can be specified multiple " + "times. You can use a value of 'current' to select the current platform. If left " + "unspecified, the platforms implied by the targets selected to build the PEX with are " + "used. Those targets are influenced by the current interpreter running Pex as well as " + "use of `--python`, `--interpreter-constraint`, `--platform` or `--complete-platform` " + "options." + ), + ) + parser.add_argument( + "--scie-pbs-release", + dest="scie_pbs_release", + default=None, + type=str, + help=( + "The Python Standalone Builds release to use. Currently releases are dates of the form " + "YYYYMMDD, e.g.: '20240713'. See their GitHub releases page at " + "https://github.com/indygreg/python-build-standalone/releases to discover available " + "releases. If left unspecified the latest release is used. N.B.: The latest lookup is " + "cached for 5 days. To force a fresh lookup you can remove the cache at " + "/science/downloads." + ), + ) + parser.add_argument( + "--scie-python-version", + dest="scie_python_version", + default=None, + type=Version, + help=( + "The portable CPython version to select. Can be either in `.` form; " + "e.g.: '3.11', or else fully specified as `..`; e.g.: '3.11.3'. " + "If you don't specify this option, Pex will do its best to guess appropriate portable " + "CPython versions. N.B.: Python Standalone Builds does not provide all patch versions; " + "so you should check their releases at " + "https://github.com/indygreg/python-build-standalone/releases if you wish to pin down " + "to the patch level." + ), + ) + parser.add_argument( + "--scie-science-binary", + dest="scie_science_binary", + default=None, + type=str, + help=( + "The file path of a `science` binary or a URL to use to fetch the `science` binary " + "when there is no `science` on the PATH with a version matching {science_requirement}. " + "Pex uses the official `science` releases at {science_releases_url} by default.".format( + science_requirement=SCIENCE_REQUIREMENT, science_releases_url=SCIENCE_RELEASES_URL + ) + ), + ) + + +def render_options(options): + # type: (ScieOptions) -> str + + args = ["--scie", str(options.style)] + for platform in options.platforms: + args.append("--scie-platform") + args.append(str(platform)) + if options.pbs_release: + args.append("--scie-pbs-release") + args.append(options.pbs_release) + if options.python_version: + args.append("--scie-python-version") + args.append(".".join(map(str, options.python_version))) + if options.science_binary_url: + args.append("--scie-science-binary") + args.append(options.science_binary_url) + return " ".join(args) + + +def extract_options(options): + # type: (Namespace) -> Optional[ScieOptions] + + if not options.scie_style: + return None + + python_version = None # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + if options.scie_python_version: + if ( + not options.scie_python_version.parsed_version.release + or len(options.scie_python_version.parsed_version.release) < 2 + ): + raise ValueError( + "Invalid Python version: '{python_version}'.\n" + "Must be in the form `.` or `..`".format( + python_version=options.scie_python_version + ) + ) + python_version = cast( + "Union[Tuple[int, int], Tuple[int, int, int]]", + options.scie_python_version.parsed_version.release, + ) + if python_version < (3, 8): + raise ValueError( + "Invalid Python version: '{python_version}'.\n" + "Scies are built using Python Standalone Builds which only supports Python >=3.8.\n" + "To find supported Python versions, you can browse the releases here:\n" + " https://github.com/indygreg/python-build-standalone/releases".format( + python_version=options.scie_python_version + ) + ) + + science_binary_url = options.scie_science_binary + if science_binary_url: + url_info = urlparse.urlparse(options.scie_science_binary) + if not url_info.scheme and url_info.path and os.path.isfile(url_info.path): + science_binary_url = "file://{path}".format(path=os.path.abspath(url_info.path)) + + return ScieOptions( + style=options.scie_style, + platforms=tuple(OrderedSet(options.scie_platforms)), + pbs_release=options.scie_pbs_release, + python_version=python_version, + science_binary_url=science_binary_url, + ) + + +def build( + configuration, # type: ScieConfiguration + pex_file, # type: str + url_fetcher=None, # type: Optional[URLFetcher] + env=ENV, # type: Variables +): + # type: (...) -> Iterator[ScieInfo] + + return science.build(configuration, pex_file, url_fetcher=url_fetcher, env=env) diff --git a/pex/scie/configure-binding.py b/pex/scie/configure-binding.py new file mode 100644 index 000000000..8f3b1c049 --- /dev/null +++ b/pex/scie/configure-binding.py @@ -0,0 +1,30 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import print_function + +import os +import sys + + +def write_bindings( + env_file, # type: str + installed_pex_dir, # type: str +): + # type: (...) -> None + with open(env_file, "a") as fp: + print("PYTHON=" + sys.executable, file=fp) + print("PEX=" + os.path.realpath(os.path.join(installed_pex_dir, "__main__.py")), file=fp) + + +if __name__ == "__main__": + write_bindings( + env_file=os.environ["SCIE_BINDING_ENV"], + installed_pex_dir=( + # The zipapp case: + os.environ["_PEX_SCIE_INSTALLED_PEX_DIR"] + # The --venv case: + or os.environ.get("VIRTUAL_ENV", os.path.dirname(os.path.dirname(sys.executable))) + ), + ) + sys.exit(0) diff --git a/pex/scie/model.py b/pex/scie/model.py new file mode 100644 index 000000000..07e46c6a8 --- /dev/null +++ b/pex/scie/model.py @@ -0,0 +1,259 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import itertools +import os +import platform +from collections import defaultdict + +from pex.enum import Enum +from pex.platforms import Platform +from pex.targets import Targets +from pex.third_party.packaging import tags # noqa +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import DefaultDict, Iterable, Optional, Set, Tuple, Union + + import attr # vendor:skip +else: + from pex.third_party import attr + + +class ScieStyle(Enum["ScieStyle.Value"]): + class Value(Enum.Value): + pass + + LAZY = Value("lazy") + EAGER = Value("eager") + + +class _CurrentPlatform(object): + def __get__(self, obj, objtype=None): + # type: (...) -> SciePlatform.Value + if not hasattr(self, "_current"): + system = platform.system().lower() + machine = platform.machine().lower() + if "linux" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.LINUX_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.LINUX_X86_64 + elif "darwin" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.MACOS_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.MACOS_X86_64 + elif "windows" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.WINDOWS_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.WINDOWS_X86_64 + if not hasattr(self, "_current"): + raise ValueError( + "The current operating system / machine pair is not supported!: " + "{system} / {machine}".format(system=system, machine=machine) + ) + return self._current + + +class SciePlatform(Enum["SciePlatform.Value"]): + class Value(Enum.Value): + @property + def extension(self): + # type: () -> str + return ( + ".exe" + if self in (SciePlatform.WINDOWS_AARCH64, SciePlatform.WINDOWS_X86_64) + else "" + ) + + def binary_name(self, binary_name): + # type: (str) -> str + return "{binary_name}{extension}".format( + binary_name=binary_name, extension=self.extension + ) + + def qualified_binary_name(self, binary_name): + # type: (str) -> str + return "{binary_name}-{platform}{extension}".format( + binary_name=binary_name, platform=self, extension=self.extension + ) + + def qualified_file_name(self, file_name): + # type: (str) -> str + stem, ext = os.path.splitext(file_name) + return "{stem}-{platform}{ext}".format(stem=stem, platform=self, ext=ext) + + LINUX_AARCH64 = Value("linux-aarch64") + LINUX_X86_64 = Value("linux-x86_64") + MACOS_AARCH64 = Value("macos-aarch64") + MACOS_X86_64 = Value("macos-x86_64") + WINDOWS_AARCH64 = Value("windows-x86_64") + WINDOWS_X86_64 = Value("windows-aarch64") + CURRENT = _CurrentPlatform() + + @classmethod + def parse(cls, value): + # type: (str) -> SciePlatform.Value + return cls.CURRENT if "current" == value else cls.for_value(value) + + +@attr.s(frozen=True) +class ScieTarget(object): + platform = attr.ib() # type: SciePlatform.Value + python_version = attr.ib() # type: Union[Tuple[int, int], Tuple[int, int, int]] + pbs_release = attr.ib(default=None) # type: Optional[str] + + @property + def version_str(self): + # type: () -> str + return ".".join(map(str, self.python_version)) + + +@attr.s(frozen=True) +class ScieInfo(object): + style = attr.ib() # type: ScieStyle.Value + target = attr.ib() # type: ScieTarget + file = attr.ib() # type: str + + @property + def platform(self): + # type: () -> SciePlatform.Value + return self.target.platform + + @property + def python_version(self): + # type: () -> Union[Tuple[int, int], Tuple[int, int, int]] + return self.target.python_version + + +@attr.s(frozen=True) +class ScieOptions(object): + style = attr.ib(default=ScieStyle.LAZY) # type: ScieStyle.Value + platforms = attr.ib(default=()) # type: Tuple[SciePlatform.Value, ...] + pbs_release = attr.ib(default=None) # type: Optional[str] + python_version = attr.ib( + default=None + ) # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + science_binary_url = attr.ib(default=None) # type: Optional[str] + + def create_configuration(self, targets): + # type: (Targets) -> ScieConfiguration + return ScieConfiguration.from_targets(self, targets) + + +@attr.s(frozen=True) +class ScieConfiguration(object): + @classmethod + def from_tags( + cls, + options, # type: ScieOptions + tags, # type: Iterable[tags.Tag] + ): + # type: (...) -> ScieConfiguration + return cls._from_platforms( + options=options, platforms=tuple(Platform.from_tag(tag) for tag in tags) + ) + + @classmethod + def from_targets( + cls, + options, # type: ScieOptions + targets, # type: Targets + ): + # type: (...) -> ScieConfiguration + return cls._from_platforms( + options=options, + platforms=tuple(target.platform for target in targets.unique_targets()), + ) + + @classmethod + def _from_platforms( + cls, + options, # type: ScieOptions + platforms, # type: Iterable[Platform] + ): + # type: (...) -> ScieConfiguration + + python_version = options.python_version + python_versions_by_platform = defaultdict( + set + ) # type: DefaultDict[SciePlatform.Value, Set[Union[Tuple[int, int], Tuple[int, int, int]]]] + for plat in platforms: + if python_version: + plat_python_version = python_version + elif len(plat.version_info) < 2: + continue + else: + # Here were guessing an available PBS CPython version. Since a triple is unlikely to + # hit, we just use major / minor. If the user wants control they can specify + # options.python_version via `--scie-python-version`. + plat_python_version = cast( + "Union[Tuple[int, int], Tuple[int, int, int]]", plat.version_info + )[:2] + + # We use Python Build Standalone to create scies, and we know it does not support + # CPython<3.8. + if plat_python_version < (3, 8): + continue + + # We use Python Build Standalone to create scies, and we know it only provides CPython + # interpreters. + if plat.impl not in ("py", "cp"): + continue + + platform_str = plat.platform + is_aarch64 = "arm64" in platform_str or "aarch64" in platform_str + is_x86_64 = "amd64" in platform_str or "x86_64" in platform_str + if not is_aarch64 ^ is_x86_64: + continue + + if "linux" in platform_str: + scie_platform = ( + SciePlatform.LINUX_AARCH64 if is_aarch64 else SciePlatform.LINUX_X86_64 + ) + elif "mac" in platform_str: + scie_platform = ( + SciePlatform.MACOS_AARCH64 if is_aarch64 else SciePlatform.MACOS_X86_64 + ) + elif "win" in platform_str: + scie_platform = ( + SciePlatform.WINDOWS_AARCH64 if is_aarch64 else SciePlatform.WINDOWS_X86_64 + ) + else: + continue + + python_versions_by_platform[scie_platform].add(plat_python_version) + + for explicit_platform in options.platforms: + if explicit_platform not in python_versions_by_platform: + if options.python_version: + python_versions_by_platform[explicit_platform] = {options.python_version} + else: + python_versions_by_platform[explicit_platform] = set( + itertools.chain.from_iterable(python_versions_by_platform.values()) + ) + if options.platforms: + for configured_platform in tuple(python_versions_by_platform): + if configured_platform not in options.platforms: + python_versions_by_platform.pop(configured_platform, None) + + scie_targets = tuple( + ScieTarget( + platform=scie_platform, + pbs_release=options.pbs_release, + python_version=max(python_versions), + ) + for scie_platform, python_versions in sorted(python_versions_by_platform.items()) + ) + return cls(options=options, targets=tuple(scie_targets)) + + options = attr.ib() # type: ScieOptions + targets = attr.ib() # type: Tuple[ScieTarget, ...] + + def __len__(self): + # type: () -> int + return len(self.targets) diff --git a/pex/scie/science.py b/pex/scie/science.py new file mode 100644 index 000000000..509358941 --- /dev/null +++ b/pex/scie/science.py @@ -0,0 +1,363 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import re +import shutil +import subprocess +from collections import OrderedDict +from subprocess import CalledProcessError + +from pex.atomic_directory import atomic_directory +from pex.common import chmod_plus_x, is_exe, pluralize, safe_mkdtemp, safe_open +from pex.compatibility import shlex_quote +from pex.exceptions import production_assert +from pex.fetcher import URLFetcher +from pex.hashing import Sha256 +from pex.layout import Layout +from pex.pep_440 import Version +from pex.pex_info import PexInfo +from pex.result import Error, try_ +from pex.scie.model import ScieConfiguration, ScieInfo, SciePlatform, ScieStyle, ScieTarget +from pex.third_party.packaging.specifiers import SpecifierSet +from pex.third_party.packaging.version import InvalidVersion +from pex.tracer import TRACER +from pex.typing import TYPE_CHECKING, cast +from pex.util import CacheHelper +from pex.variables import ENV, Variables, unzip_dir_relpath + +if TYPE_CHECKING: + from typing import Any, Dict, Iterator, Optional, Union, cast + + import attr # vendor:skip + import toml # vendor:skip +else: + from pex.third_party import attr, toml + + +@attr.s(frozen=True) +class Manifest(object): + target = attr.ib() # type: ScieTarget + path = attr.ib() # type: str + + def binary_name(self, binary_name): + # type: (str) -> str + return self.target.platform.binary_name(binary_name) + + def qualified_binary_name(self, binary_name): + # type: (str) -> str + return self.target.platform.qualified_binary_name(binary_name) + + +SCIENCE_RELEASES_URL = "https://github.com/a-scie/lift/releases" +MIN_SCIENCE_VERSION = Version("0.3.0") +SCIENCE_REQUIREMENT = SpecifierSet("~={min_version}".format(min_version=MIN_SCIENCE_VERSION)) + + +def _science_binary_url(suffix=""): + # type: (str) -> str + return "{science_releases_url}/download/v{version}/{binary}{suffix}".format( + science_releases_url=SCIENCE_RELEASES_URL, + version=MIN_SCIENCE_VERSION.raw, + binary=SciePlatform.CURRENT.qualified_binary_name("science-fat"), + suffix=suffix, + ) + + +PTEX_VERSION = "1.1.1" +SCIE_JUMP_VERSION = "1.1.1" + + +def create_manifests( + configuration, # type: ScieConfiguration + name, # type: str + pex_info, # type: PexInfo + layout, # type: Layout.Value +): + # type: (...) -> Iterator[Manifest] + + pex_root = "{scie.bindings}/pex_root" + if pex_info.venv: + # We let the configure-binding calculate the venv dir at runtime since it depends on the + # interpreter executing the venv PEX. + installed_pex_dir = "" + elif layout is Layout.LOOSE: + installed_pex_dir = "{pex}" + else: + production_assert(pex_info.pex_hash is not None) + pex_hash = cast(str, pex_info.pex_hash) + installed_pex_dir = os.path.join(pex_root, unzip_dir_relpath(pex_hash)) + + env_default = { + "PEX_ROOT": pex_root, + } + + lift = { + "name": name, + "ptex": { + "id": "ptex", + "version": PTEX_VERSION, + "argv1": "{scie.env.PEX_BOOTSTRAP_URLS={scie.lift}}", + }, + "scie_jump": {"version": SCIE_JUMP_VERSION}, + "files": [{"name": "configure-binding.py"}, {"name": "pex"}], + "commands": [ + { + "env": {"default": env_default}, + "exe": "{scie.bindings.configure:PYTHON}", + "args": ["{scie.bindings.configure:PEX}"], + } + ], + "bindings": [ + { + "env": { + "default": env_default, + "remove_exact": ["PATH"], + "remove_re": ["PEX_.*"], + "replace": { + "PEX_INTERPRETER": "1", + "_PEX_SCIE_INSTALLED_PEX_DIR": installed_pex_dir, + # We can get a warning about too-long script shebangs, but this is not + # relevant since we above run the PEX via python and not via shebang. + "PEX_EMIT_WARNINGS": "0", + }, + }, + "name": "configure", + "exe": "#{cpython:python}", + "args": ["{pex}", "{configure-binding.py}"], + } + ], + } # type: Dict[str, Any] + + for target in configuration.targets: + manifest_path = os.path.join( + safe_mkdtemp(), + target.platform.qualified_file_name("{name}-lift.toml".format(name=name)), + ) + with safe_open(manifest_path, "w") as fp: + toml.dump( + { + "lift": dict( + lift, + platforms=[target.platform.value], + interpreters=[ + { + "id": "cpython", + "provider": "PythonBuildStandalone", + "release": target.pbs_release, + "version": target.version_str, + "lazy": configuration.options.style is ScieStyle.LAZY, + } + ], + ) + }, + fp, + ) + yield Manifest(target=target, path=manifest_path) + + +def _science_dir( + env, # type: Variables + *components # type: str +): + # type: (...) -> str + return os.path.join(env.PEX_ROOT, "scies", "science", MIN_SCIENCE_VERSION.raw, *components) + + +def _science_binary_names(): + # type: () -> Iterator[str] + yield SciePlatform.CURRENT.binary_name("science-fat") + yield SciePlatform.CURRENT.qualified_binary_name("science-fat") + yield SciePlatform.CURRENT.binary_name("science") + yield SciePlatform.CURRENT.qualified_binary_name("science") + + +def _is_compatible_science_binary( + binary, # type: str + source=None, # type: Optional[str] +): + # type: (...) -> Union[Version, Error] + try: + version = Version( + subprocess.check_output(args=[binary, "--version"]).decode("utf-8").strip() + ) + except (CalledProcessError, InvalidVersion) as e: + return Error( + "Failed to determine --version of science binary at {source}: {err}".format( + source=source or binary, err=e + ) + ) + else: + if version.raw in SCIENCE_REQUIREMENT: + return version + return Error( + "The science binary at {source} is version {version} which does not match Pex's " + "science requirement of {science_requirement}.".format( + source=source or binary, + version=version.raw, + science_requirement=SCIENCE_REQUIREMENT, + ) + ) + + +def _path_science(): + # type: () -> Optional[str] + for path_element in os.environ.get("PATH", os.defpath).split(os.pathsep): + for binary in ( + os.path.join(path_element, binary_name) for binary_name in _science_binary_names() + ): + if not is_exe(binary): + continue + if isinstance(_is_compatible_science_binary(binary), Error): + continue + return binary + return None + + +def _ensure_science( + url_fetcher=None, # type: Optional[URLFetcher] + science_binary_url=None, # type: Optional[str] + env=ENV, # type: Variables +): + # type: (...) -> str + + target_dir = _science_dir(env, "bin") + with atomic_directory(target_dir=target_dir) as atomic_dir: + if not atomic_dir.is_finalized(): + target_science = os.path.join(atomic_dir.work_dir, "science") + path_science = _path_science() + if path_science: + shutil.copy(path_science, target_science) + else: + fetcher = url_fetcher or URLFetcher() + with open(target_science, "wb") as write_fp, fetcher.get_body_stream( + science_binary_url or _science_binary_url() + ) as read_fp: + shutil.copyfileobj(read_fp, write_fp) + chmod_plus_x(target_science) + + if science_binary_url: + custom_science_binary_version = try_( + _is_compatible_science_binary(target_science, source=science_binary_url) + ) + TRACER.log( + "Using custom science binary from {source} with version {version}.".format( + source=science_binary_url, version=custom_science_binary_version.raw + ) + ) + else: + # Since we used the canonical GitHub Releases URL, we know a checksum file is + # available we can use to verify. + science_sha256_url = _science_binary_url(".sha256") + with fetcher.get_body_stream(science_sha256_url) as fp: + expected_sha256, _, _ = fp.read().decode("utf-8").partition(" ") + actual_sha256 = CacheHelper.hash(target_science, hasher=Sha256) + if expected_sha256 != actual_sha256: + raise ValueError( + "The science binary downloaded from {science_binary_url} does not " + "match the expected SHA-256 fingerprint recorded in " + "{science_sha256_url}.\n" + "Expected {expected_sha256} but found {actual_sha256}.".format( + science_binary_url=science_binary_url, + science_sha256_url=science_sha256_url, + expected_sha256=expected_sha256, + actual_sha256=actual_sha256, + ) + ) + return os.path.join(target_dir, "science") + + +class ScienceError(Exception): + """Indicates an error executing science.""" + + +def build( + configuration, # type: ScieConfiguration + pex_file, # type: str + url_fetcher=None, # type: Optional[URLFetcher] + env=ENV, # type: Variables +): + # type: (...) -> Iterator[ScieInfo] + + science = _ensure_science( + url_fetcher=url_fetcher, + science_binary_url=configuration.options.science_binary_url, + env=env, + ) + name = re.sub(r"\.pex$", "", os.path.basename(pex_file), flags=re.IGNORECASE) + pex_info = PexInfo.from_pex(pex_file) + layout = Layout.identify(pex_file) + use_platform_suffix = len(configuration.targets) > 1 + + errors = OrderedDict() # type: OrderedDict[Manifest, str] + for manifest in create_manifests(configuration, name, pex_info, layout): + args = [science, "--cache-dir", _science_dir(env, "cache")] + if env.PEX_VERBOSE: + args.append("-{verbosity}".format(verbosity="v" * env.PEX_VERBOSE)) + dest_dir = os.path.dirname(os.path.abspath(pex_file)) + args.extend( + [ + "lift", + "--file", + "pex={pex_file}".format(pex_file=pex_file), + "--file", + "configure-binding.py={configure_binding}".format( + configure_binding=os.path.join( + os.path.dirname(__file__), "configure-binding.py" + ) + ), + "build", + "--dest-dir", + dest_dir, + ] + ) + if use_platform_suffix: + args.append("--use-platform-suffix") + args.append(manifest.path) + with open(os.devnull, "wb") as devnull: + process = subprocess.Popen(args=args, stdout=devnull, stderr=subprocess.PIPE) + _, stderr = process.communicate() + if process.returncode != 0: + saved_manifest = os.path.relpath( + os.path.join(dest_dir, os.path.basename(manifest.path)) + ) + shutil.copy(manifest.path, saved_manifest) + errors[manifest] = ( + "Command `{command}` failed with exit code {exit_code} (saved lift manifest to " + "{saved_manifest} for inspection):\n{stderr}" + ).format( + command=" ".join(shlex_quote(arg) for arg in args[:-1] + [saved_manifest]), + exit_code=process.returncode, + saved_manifest=saved_manifest, + stderr=stderr.decode("utf-8").strip(), + ) + else: + yield ScieInfo( + style=configuration.options.style, + target=manifest.target, + file=os.path.join( + dest_dir, + manifest.qualified_binary_name(name) + if use_platform_suffix + else manifest.binary_name(name), + ), + ) + + if errors: + raise ScienceError( + "Failed to build {count} {scies}:\n\n{errors}".format( + count=len(errors), + scies=pluralize(errors, "scie"), + errors="\n\n".join( + "{index}. For CPython {version} on {platform}: {err}".format( + index=index, + platform=manifest.target.platform, + version=manifest.target.version_str, + err=err, + ) + for index, (manifest, err) in enumerate(errors.items(), start=1) + ), + ) + ) diff --git a/pex/targets.py b/pex/targets.py index 511e05904..37c02f229 100644 --- a/pex/targets.py +++ b/pex/targets.py @@ -44,13 +44,14 @@ def binary_name(self, version_components=2): @property def python_version(self): - # type: () -> Optional[Tuple[int, int]] + # type: () -> Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + python_full_version = self.marker_environment.python_full_version + if python_full_version: + return cast("Tuple[int, int, int]", tuple(map(int, python_full_version.split(".")))[:3]) python_version = self.marker_environment.python_version - return ( - cast("Tuple[int, int]", tuple(map(int, python_version.split(".")))[:2]) - if python_version - else None - ) + if python_version: + return cast("Tuple[int, int]", tuple(map(int, python_version.split(".")))[:2]) + return None @property def supported_tags(self): @@ -181,8 +182,8 @@ def binary_name(self, version_components=2): @property def python_version(self): - # type: () -> Tuple[int, int] - return self.interpreter.identity.version[:2] + # type: () -> Tuple[int, int, int] + return self.interpreter.identity.version[:3] @property def is_foreign(self): diff --git a/pex/variables.py b/pex/variables.py index 41ff3b287..f952eede0 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -789,6 +789,11 @@ def _expand_pex_root(pex_root): return os.path.expanduser(Variables.PEX_ROOT.value_or(ENV, fallback=fallback)) +def unzip_dir_relpath(pex_hash): + # type: (str) -> str + return os.path.join("unzipped_pexes", pex_hash) + + def unzip_dir( pex_root, # type: str pex_hash, # type: str @@ -796,7 +801,7 @@ def unzip_dir( ): # type: (...) -> str pex_root = _expand_pex_root(pex_root) if expand_pex_root else pex_root - return os.path.join(pex_root, "unzipped_pexes", pex_hash) + return os.path.join(pex_root, unzip_dir_relpath(pex_hash)) def venv_dir( diff --git a/pex/venv/installer.py b/pex/venv/installer.py index c0c6be2d1..b6b759c8d 100644 --- a/pex/venv/installer.py +++ b/pex/venv/installer.py @@ -703,6 +703,8 @@ def sys_executable_paths(): "_PEX_DEP_CONFIG_FILE", # This is used as an experiment knob for atomic_directory locking. "_PEX_FILE_LOCK_STYLE", + # This is used in the scie binding command for ZIPAPP PEXes. + "_PEX_SCIE_INSTALLED_PEX_DIR", ) ] if ignored_pex_env_vars: diff --git a/pex/version.py b/pex/version.py index 9c06c95ad..be6b005a1 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.10.1" +__version__ = "2.11.0" diff --git a/tests/integration/scie/__init__.py b/tests/integration/scie/__init__.py new file mode 100644 index 000000000..87fb2ed9a --- /dev/null +++ b/tests/integration/scie/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py new file mode 100644 index 000000000..34b95ec23 --- /dev/null +++ b/tests/integration/scie/test_pex_scie.py @@ -0,0 +1,371 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import glob +import json +import os.path +import re +import subprocess +import sys +from typing import Optional + +import pytest + +from pex.common import is_exe +from pex.layout import Layout +from pex.orderedset import OrderedSet +from pex.scie import SciePlatform, ScieStyle +from pex.targets import LocalInterpreter +from pex.typing import TYPE_CHECKING +from testing import IS_PYPY, PY_VER, make_env, run_pex_command + +if TYPE_CHECKING: + from typing import Any, Iterable, List + + +@pytest.mark.parametrize( + "scie_style", [pytest.param(style, id=str(style)) for style in ScieStyle.values()] +) +@pytest.mark.parametrize( + "layout", [pytest.param(layout, id=str(layout)) for layout in Layout.values()] +) +@pytest.mark.parametrize( + "execution_mode_args", + [ + pytest.param([], id="ZIPAPP"), + pytest.param(["--venv"], id="VENV"), + pytest.param(["--sh-boot"], id="ZIPAPP-sh-boot"), + pytest.param(["--venv", "--sh-boot"], id="VENV-sh-boot"), + ], +) +def test_basic( + tmpdir, # type: Any + scie_style, # type: ScieStyle.Value + layout, # type: Layout.Value + execution_mode_args, # type: List[str] +): + # type: (...) -> None + + pex = os.path.join(str(tmpdir), "cowsay.pex") + result = run_pex_command( + args=[ + "cowsay==5.0", + "-c", + "cowsay", + "-o", + pex, + "--scie", + str(scie_style), + "--layout", + str(layout), + ] + + execution_mode_args + ) + if PY_VER < (3, 8) or IS_PYPY: + result.assert_failure( + expected_error_re=r".*^{message}$".format( + message=re.escape( + "You selected `--scie {style}`, but none of the selected targets have " + "compatible interpreters that can be embedded to form a scie:\n" + "{target}".format( + style=scie_style, target=LocalInterpreter.create().render_description() + ) + ) + ), + re_flags=re.DOTALL | re.MULTILINE, + ) + return + if PY_VER >= (3, 13): + result.assert_failure( + expected_error_re=( + r".*" + r"^Failed to build 1 scie:$" + r".*" + r"^Provider: No released assets found for release [0-9]{{8}} Python {version} " + r"of flavor install_only\.$".format(version=".".join(map(str, PY_VER))) + ), + re_flags=re.DOTALL | re.MULTILINE, + ) + return + result.assert_success() + + scie = os.path.join(str(tmpdir), "cowsay") + assert b"| PAR! |" in subprocess.check_output(args=[scie, "PAR!"], env=make_env(PATH=None)) + + +def test_multiple_platforms(tmpdir): + # type: (Any) -> None + + def create_scies( + output_dir, # type: str + extra_args=(), # type: Iterable[str] + ): + pex = os.path.join(output_dir, "cowsay.pex") + run_pex_command( + args=[ + "cowsay==5.0", + "-c", + "cowsay", + "-o", + pex, + "--scie", + "lazy", + "--platform", + "linux-aarch64-cp-39-cp39", + "--platform", + "linux-x86_64-cp-310-cp310", + "--platform", + "macosx-10.9-arm64-cp-311-cp311", + "--platform", + "macosx-10.9-x86_64-cp-312-cp312", + ] + + list(extra_args) + ).assert_success() + + python_version_by_platform = { + SciePlatform.LINUX_AARCH64: "3.9", + SciePlatform.LINUX_X86_64: "3.10", + SciePlatform.MACOS_AARCH64: "3.11", + SciePlatform.MACOS_X86_64: "3.12", + } + assert SciePlatform.CURRENT in python_version_by_platform + + def assert_platforms( + output_dir, # type: str + expected_platforms, # type: Iterable[SciePlatform.Value] + ): + # type: (...) -> None + + all_output_files = set( + path + for path in os.listdir(output_dir) + if os.path.isfile(os.path.join(output_dir, path)) + ) + for platform in OrderedSet(expected_platforms): + python_version = python_version_by_platform[platform] + binary = platform.qualified_binary_name("cowsay") + assert binary in all_output_files + all_output_files.remove(binary) + scie = os.path.join(output_dir, binary) + assert is_exe(scie), "Expected --scie build to produce a {binary} binary.".format( + binary=binary + ) + if platform is SciePlatform.CURRENT: + assert b"| PEX-scie wabbit! |" in subprocess.check_output( + args=[scie, "PEX-scie wabbit!"], env=make_env(PATH=None) + ) + assert ( + python_version + == subprocess.check_output( + args=[ + scie, + "-c", + "import sys; print('.'.join(map(str, sys.version_info[:2])))", + ], + env=make_env(PEX_INTERPRETER=1), + ) + .decode("utf-8") + .strip() + ) + assert {"cowsay.pex"} == all_output_files, ( + "Expected one output scie for each platform plus the original cowsay.pex. All expected " + "scies were found, but the remaining files are: {remaining_files}".format( + remaining_files=all_output_files + ) + ) + + all_platforms_output_dir = os.path.join(str(tmpdir), "all-platforms") + create_scies(output_dir=all_platforms_output_dir) + assert_platforms( + output_dir=all_platforms_output_dir, + expected_platforms=( + SciePlatform.LINUX_AARCH64, + SciePlatform.LINUX_X86_64, + SciePlatform.MACOS_AARCH64, + SciePlatform.MACOS_X86_64, + ), + ) + + # Now restrict the PEX's implied natural platform set of 4 down to 2 or 3 using + # `--scie-platform`. + restricted_platforms_output_dir = os.path.join(str(tmpdir), "restricted-platforms") + create_scies( + output_dir=restricted_platforms_output_dir, + extra_args=[ + "--scie-platform", + "current", + "--scie-platform", + str(SciePlatform.LINUX_AARCH64), + "--scie-platform", + str(SciePlatform.LINUX_X86_64), + ], + ) + assert_platforms( + output_dir=restricted_platforms_output_dir, + expected_platforms=( + SciePlatform.CURRENT, + SciePlatform.LINUX_AARCH64, + SciePlatform.LINUX_X86_64, + ), + ) + + +PRINT_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))" + + +skip_if_pypy = pytest.mark.skipif(IS_PYPY, reason="PyPy targeted PEXes do not support --scie.") + + +@skip_if_pypy +def test_specified_interpreter(tmpdir): + # type: (Any) -> None + + pex = os.path.join(str(tmpdir), "empty.pex") + run_pex_command( + args=[ + "-o", + pex, + "--scie", + "lazy", + # We pick a specific version that is not in the latest release but is known to provide + # distributions for all platforms Pex tests run on. + "--scie-pbs-release", + "20221002", + "--scie-python-version", + "3.10.7", + ], + ).assert_success() + + assert ( + ".".join(map(str, sys.version_info[:3])) + == subprocess.check_output(args=[pex, "-c", PRINT_VERSION_SCRIPT]).decode("utf-8").strip() + ) + + scie = os.path.join(str(tmpdir), "empty") + assert b"3.10.7\n" == subprocess.check_output(args=[scie, "-c", PRINT_VERSION_SCRIPT]) + + +@skip_if_pypy +def test_specified_science_binary(tmpdir): + # type: (Any) -> None + + pex_root = os.path.join(str(tmpdir), "pex_root") + scie = os.path.join(str(tmpdir), "cowsay") + run_pex_command( + args=[ + "--pex-root", + pex_root, + "cowsay==6.0", + "-c", + "cowsay", + "--scie", + "lazy", + "--scie-python-version", + "3.12", + "-o", + scie, + "--scie-science-binary", + # N.B.: This custom version is both lower than the latest available version (0.4.2 + # at the time of writing) and higher than the minimum supported version of 0.3.0; so + # we can prove we downloaded the custom version via this URL by checking the version + # below since our next floor bump will be from 0.3.0 to at least 0.4.3. + "https://github.com/a-scie/lift/releases/download/v0.4.0/{binary}".format( + binary=SciePlatform.CURRENT.qualified_binary_name("science") + ), + ], + env=make_env(PATH=None), + ).assert_success() + + assert b"| Alternative SCIENCE Facts! |" in subprocess.check_output( + args=[scie, "-t", "Alternative SCIENCE Facts!"] + ) + + science_binaries = glob.glob(os.path.join(pex_root, "scies", "science", "*", "bin", "science")) + assert 1 == len(science_binaries) + science = science_binaries[0] + assert "0.4.0" == subprocess.check_output(args=[science, "--version"]).decode("utf-8").strip() + + +@skip_if_pypy +def test_custom_lazy_urls(tmpdir): + # type: (Any) -> None + + scie = os.path.join(str(tmpdir), "empty") + run_pex_command( + args=[ + "-o", + scie, + "--scie", + "lazy", + "--scie-pbs-release", + "20221002", + "--scie-python-version", + "3.10.7", + ], + ).assert_success() + + assert b"3.10.7\n" == subprocess.check_output(args=[scie, "-c", PRINT_VERSION_SCRIPT]) + + pex_bootstrap_urls = os.path.join(str(tmpdir), "pex_bootstrap_urls.json") + + def make_20221002_3_10_7_file(platform): + # type: (str) -> str + return "cpython-3.10.7+20221002-{platform}-install_only.tar.gz".format(platform=platform) + + def make_20240415_3_10_14_url(platform): + # type: (str) -> str + return ( + "https://github.com/indygreg/python-build-standalone/releases/download/20240415/" + "cpython-3.10.14+20240415-{platform}-install_only.tar.gz".format(platform=platform) + ) + + with open(pex_bootstrap_urls, "w") as fp: + json.dump( + { + "ptex": { + make_20221002_3_10_7_file(platform): make_20240415_3_10_14_url(platform) + for platform in ( + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ) + } + }, + fp, + ) + + process = subprocess.Popen( + args=[scie, "-c", PRINT_VERSION_SCRIPT], + env=make_env( + PEX_BOOTSTRAP_URLS=pex_bootstrap_urls, SCIE_BASE=os.path.join(str(tmpdir), "nce") + ), + stderr=subprocess.PIPE, + ) + _, stderr = process.communicate() + assert 0 != process.returncode, ( + "Expected PEX_BOOTSTRAP_URLS to be used and the resulting fetched interpreter distribution " + "to fail its digest check." + ) + + expected_platform = None # type: Optional[str] + if SciePlatform.CURRENT is SciePlatform.LINUX_AARCH64: + expected_platform = "aarch64-unknown-linux-gnu" + elif SciePlatform.CURRENT is SciePlatform.LINUX_X86_64: + expected_platform = "x86_64-unknown-linux-gnu" + elif SciePlatform.CURRENT is SciePlatform.MACOS_AARCH64: + expected_platform = "aarch64-apple-darwin" + elif SciePlatform.CURRENT is SciePlatform.MACOS_X86_64: + expected_platform = "x86_64-apple-darwin" + assert expected_platform is not None + + assert re.match( + r"^.*Population of work directory failed: The tar\.gz destination .*{expected_file_name} " + r"of size \d+ had unexpected hash: [a-f0-9]{{64}}$.*".format( + expected_file_name=re.escape(make_20221002_3_10_7_file(expected_platform)) + ), + stderr.decode("utf-8"), + flags=re.DOTALL | re.MULTILINE, + ), stderr.decode("utf-8") diff --git a/tox.ini b/tox.ini index d1d129e5b..4664152f6 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,8 @@ passenv = SSH_AUTH_SOCK # Needed for pexpect tests. TERM + # Needed to prevent hitting rate limits on GitHub Releases APIs in `--scie` integration tests. + SCIENCE_AUTH_API_GITHUB_COM_BEARER setenv = pip20: _PEX_PIP_VERSION=20.3.4-patched pip22_2: _PEX_PIP_VERSION=22.2.2