From 3b95f4a7348a0c0bf979b2fc43e5e6263669ef9d Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 23 Jun 2023 11:54:40 +0200 Subject: [PATCH] Support graalpy --- README.md | 5 +- bin/update_pythons.py | 91 ++++++++++++++++++++- cibuildwheel/linux.py | 38 +++++++++ cibuildwheel/logger.py | 2 + cibuildwheel/macos.py | 17 ++++ cibuildwheel/resources/build-platforms.toml | 4 + test/test_abi_variants.py | 2 +- test/test_build_skip.py | 2 +- test/test_manylinuxXXXX_only.py | 7 ++ test/utils.py | 9 ++ 10 files changed, 172 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c45589290..bdf8b6a46 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,12 @@ What does it do? | PyPy 3.8 v7.3 | ✅ | ✅⁴ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | | PyPy 3.9 v7.3 | ✅ | ✅⁴ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | | PyPy 3.10 v7.3 | ✅ | ✅⁴ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | +| GraalPy 23.0 | ✅ | ✅⁴ | N/A | N/A | N/A | ✅¹ | N/A | ✅¹ | N/A | N/A | -¹ PyPy is only supported for manylinux wheels.
+¹ PyPy & GraalPy are only supported for manylinux wheels.
² Windows arm64 support is experimental.
³ Alpine 3.14 and very briefly 3.15's default python3 [was not able to load](https://github.com/pypa/cibuildwheel/issues/934) musllinux wheels. This has been fixed; please upgrade the python package if using Alpine from before the fix.
-⁴ Cross-compilation not supported with PyPy - to build these wheels you need to run cibuildwheel on an Apple Silicon machine.
+⁴ Cross-compilation not supported with PyPy & GraalPy - to build these wheels you need to run cibuildwheel on an Apple Silicon machine.
⁵ CPython 3.12 is available using the [CIBW_PRERELEASE_PYTHONS](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons) option.
- Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy diff --git a/bin/update_pythons.py b/bin/update_pythons.py index e57a71735..f03ebc0f0 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -5,6 +5,7 @@ import copy import difflib import logging +import re from collections.abc import Mapping, MutableMapping from pathlib import Path from typing import Any, Union @@ -51,7 +52,13 @@ class ConfigMacOS(TypedDict): url: str -AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS] +class ConfigLinux(TypedDict): + identifier: str + version: str + url: str + + +AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS, ConfigLinux] # The following set of "Versions" classes allow the initial call to the APIs to @@ -102,6 +109,76 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None: ) +class GraalPyVersions: + def __init__(self): + response = requests.get("https://api.github.com/repos/oracle/graalpython/releases") + response.raise_for_status() + + releases = response.json() + gp_version_re = re.compile(r"-(\d+\.\d+\.\d+)$") + cp_version_re = re.compile(r"a Python (\d+\.\d+(?:\.\d+)?) implementation") + for release in releases: + m = gp_version_re.search(release["tag_name"]) + if m: + release["graalpy_version"] = Version(m.group(1)) + m = cp_version_re.search(release["body"]) + if m: + release["python_version"] = Version(m.group(1)) + + self.releases = [ + r + for r in releases + if "graalpy_version" in r and "python_version" in r + ] + + def update_version(self, identifier: str, spec: Specifier) -> AnyConfig: + if "x86_64" in identifier: + arch = "x86_64" + elif "arm64" in identifier: + arch = "arm64" + elif "aarch64" in identifier: + arch = "aarch64" + else: + msg = f"{identifier} not supported yet on GraalPy" + raise RuntimeError(msg) + + releases = [r for r in self.releases if spec.contains(r["python_version"])] + releases = sorted(releases, key=lambda r: r["graalpy_version"]) # type: ignore[no-any-return] + + if not releases: + msg = f"GraalPy {arch} not found for {spec}!" + raise RuntimeError(msg) + + release = releases[-1] + version = release["python_version"] + gpversion = release["graalpy_version"] + + if "macosx" in identifier: + identifier = f"gp{gpversion.major}{gpversion.minor}-macosx_{arch}" + config = ConfigMacOS + platform = "macos" + elif "linux" in identifier: + identifier = f"gp{gpversion.major}{gpversion.minor}-manylinux_{arch}" + config = ConfigLinux + platform = "linux" + else: + msg = f"GraalPy supports on macOS and Linux so far!" + raise RuntimeError(msg) + + arch = "amd64" if arch == "x86_64" else "aarch64" + (url,) = ( + rf["browser_download_url"] + for rf in release["assets"] + if rf["name"].endswith(f"{platform}-{arch}.tar.gz") + ) + + return config( + identifier=identifier, + version=f"{version.major}.{version.minor}", + url=url, + ) + + class PyPyVersions: def __init__(self, arch_str: ArchStr): response = requests.get("https://downloads.python.org/pypy/versions.json") @@ -245,6 +322,8 @@ def __init__(self) -> None: self.macos_pypy = PyPyVersions("64") self.macos_pypy_arm64 = PyPyVersions("ARM64") + self.graalpy = GraalPyVersions() + def update_config(self, config: MutableMapping[str, str]) -> None: identifier = config["identifier"] version = Version(config["version"]) @@ -262,6 +341,8 @@ def update_config(self, config: MutableMapping[str, str]) -> None: config_update = self.macos_pypy.update_version_macos(spec) elif "macosx_arm64" in identifier: config_update = self.macos_pypy_arm64.update_version_macos(spec) + elif identifier.startswith("gp"): + config_update = self.graalpy.update_version(identifier, spec) elif "win32" in identifier: if identifier.startswith("cp"): config_update = self.windows_32.update_version_windows(spec) @@ -272,6 +353,11 @@ def update_config(self, config: MutableMapping[str, str]) -> None: config_update = self.windows_pypy_64.update_version_windows(spec) elif "win_arm64" in identifier and identifier.startswith("cp"): config_update = self.windows_arm64.update_version_windows(spec) + elif "linux" in identifier: + if identifier.startswith("gp"): + config_update = self.graalpy.update_version(identifier, spec) + else: + return assert config_update is not None, f"{identifier} not found!" config.update(**config_update) @@ -301,6 +387,9 @@ def update_pythons(force: bool, level: str) -> None: with toml_file_path.open("rb") as f: configs = tomllib.load(f) + for config in configs["linux"]["python_configurations"]: + all_versions.update_config(config) + for config in configs["windows"]["python_configurations"]: all_versions.update_config(config) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 44bafc55f..5b2c3c60e 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -8,6 +8,8 @@ from pathlib import Path, PurePath, PurePosixPath from typing import Tuple +from filelock import FileLock + from ._compat.typing import OrderedDict, assert_never from .architecture import Architecture from .logger import log @@ -15,10 +17,13 @@ from .options import Options from .typing import PathOrStr from .util import ( + CIBW_CACHE_PATH, AlreadyBuiltWheelError, BuildSelector, NonPlatformWheelError, build_frontend_or_default, + call, + download, find_compatible_wheel, get_build_verbosity_extra_flags, prepare_command, @@ -34,6 +39,7 @@ class PythonConfiguration: version: str identifier: str path_str: str + url: str = "" @property def path(self) -> PurePosixPath: @@ -113,6 +119,36 @@ def get_build_steps( yield from steps.values() +def install_python(container: OCIContainer, config: PythonConfiguration) -> bool: + url = config.url + if not url: + return False + archive = url.rsplit("/", 1)[-1] + parts = archive.rsplit(".", 2) + if parts[-1] == "zip": + extension = ".zip" + elif parts[-1] == "gz": + extension = ".tar.gz" + else: + extension = ".tar.bz2" + assert archive.endswith(extension) + installation_path = CIBW_CACHE_PATH / archive[: -len(extension)] + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + downloaded_archive = CIBW_CACHE_PATH / archive + download(url, downloaded_archive) + installation_path.parent.mkdir(parents=True, exist_ok=True) + call("tar", "-C", installation_path.parent, "-xzf", downloaded_archive) + downloaded_archive.unlink() + container.copy_into(installation_path, config.path) + try: + container.call(["test", "-x", config.path / "bin" / "python"]) + except subprocess.CalledProcessError: + return False + else: + return True + + def check_all_python_exist( *, platform_configs: Iterable[PythonConfiguration], container: OCIContainer ) -> None: @@ -123,6 +159,8 @@ def check_all_python_exist( try: container.call(["test", "-x", python_path]) except subprocess.CalledProcessError: + if install_python(container, config): + continue messages.append( f" '{python_path}' executable doesn't exist in image '{container.image}' to build '{config.identifier}'." ) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 9a546c973..d429e7ba8 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -208,6 +208,8 @@ def build_description_from_identifier(identifier: str) -> str: build_description += "CPython" elif python_interpreter == "pp": build_description += "PyPy" + elif python_interpreter == "gp": + build_description += "GraalPy" else: msg = f"unknown python {python_interpreter!r}" raise Exception(msg) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 1c103629d..92ff494cd 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -160,6 +160,21 @@ def install_pypy(tmp: Path, url: str) -> Path: return installation_path / "bin" / "pypy3" +def install_graalpy(tmp: Path, url: str) -> Path: + graalpy_archive = url.rsplit("/", 1)[-1] + extension = ".tar.gz" + assert graalpy_archive.endswith(extension) + installation_path = CIBW_CACHE_PATH / graalpy_archive[: -len(extension)] + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + downloaded_archive = tmp / graalpy_archive + download(url, downloaded_archive) + installation_path.parent.mkdir(parents=True, exist_ok=True) + call("tar", "-C", installation_path.parent, "-xzf", downloaded_archive) + downloaded_archive.unlink() + return installation_path / "bin" / "graalpy" + + def setup_python( tmp: Path, python_configuration: PythonConfiguration, @@ -174,6 +189,8 @@ def setup_python( base_python = install_cpython(tmp, python_configuration.version, python_configuration.url) elif implementation_id.startswith("pp"): base_python = install_pypy(tmp, python_configuration.url) + elif implementation_id.startswith("gp"): + base_python = install_graalpy(tmp, python_configuration.url) else: msg = "Unknown Python implementation" raise ValueError(msg) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 293db7790..0ab99456b 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -18,6 +18,7 @@ python_configurations = [ { identifier = "pp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, { identifier = "pp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, { identifier = "pp310-manylinux_x86_64", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" }, + { identifier = "gp230-manylinux_x86_64", version = "3.10", path_str = "/opt/python/gp230-graalpy230_310_native_x86_64_linux", url = "https://github.com/oracle/graalpython/releases/download/graal-23.0.0/graalpython-23.0.0-linux-amd64.tar.gz" }, { identifier = "cp36-manylinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -43,6 +44,7 @@ python_configurations = [ { identifier = "pp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, { identifier = "pp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, { identifier = "pp310-manylinux_aarch64", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" }, + { identifier = "gp230-manylinux_aarch64", version = "3.10", path_str = "/opt/python/gp230-graalpy230_310_native_aarch64_linux", url = "https://github.com/oracle/graalpython/releases/download/graal-23.0.0/graalpython-23.0.0-linux-aarch64.tar.gz" }, { identifier = "pp37-manylinux_i686", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, { identifier = "pp38-manylinux_i686", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, { identifier = "pp39-manylinux_i686", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, @@ -110,6 +112,8 @@ python_configurations = [ { identifier = "pp39-macosx_arm64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.12-macos_arm64.tar.bz2" }, { identifier = "pp310-macosx_x86_64", version = "3.10", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.12-macos_x86_64.tar.bz2" }, { identifier = "pp310-macosx_arm64", version = "3.10", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.12-macos_arm64.tar.bz2" }, + { identifier = "gp230-macosx_x86_64", version = "3.10", url = "https://github.com/oracle/graalpython/releases/download/graal-23.0.0/graalpython-23.0.0-macos-amd64.tar.gz" }, + { identifier = "gp230-macosx_arm64", version = "3.10", url = "https://github.com/oracle/graalpython/releases/download/graal-23.0.0/graalpython-23.0.0-macos-aarch64.tar.gz" }, ] [windows] diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index 131b1ee9f..70bf51ede 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -39,7 +39,7 @@ def test_abi3(tmp_path): actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ - "CIBW_SKIP": "pp* ", # PyPy does not have a Py_LIMITED_API equivalent + "CIBW_SKIP": "pp* gp*", # PyPy and GraalPy do not have a Py_LIMITED_API equivalent }, ) diff --git a/test/test_build_skip.py b/test/test_build_skip.py index e29d14f10..559cf31e8 100644 --- a/test/test_build_skip.py +++ b/test/test_build_skip.py @@ -7,7 +7,7 @@ project_with_skip_asserts = test_projects.new_c_project( setup_py_add=textwrap.dedent( r""" - # explode if run on PyPyor Python 3.7 (these should be skipped) + # explode if run on PyPy or Python 3.7 (these should be skipped) if sys.implementation.name != "cpython": raise Exception("Only CPython shall be built") if sys.version_info[0:2] == (3, 7): diff --git a/test/test_manylinuxXXXX_only.py b/test/test_manylinuxXXXX_only.py index 4d1fe87e8..e7a884201 100644 --- a/test/test_manylinuxXXXX_only.py +++ b/test/test_manylinuxXXXX_only.py @@ -92,6 +92,9 @@ def test(manylinux_image, tmp_path): if manylinux_image == "manylinux_2_28" and platform.machine() == "x86_64": # We don't have a manylinux_2_28 image for i686 add_env["CIBW_ARCHS"] = "x86_64" + if manylinux_image != "manylinux2014": + # GraalPy only works on manylinux2014 + add_env["CIBW_SKIP"] = add_env.get("CIBW_SKIP", "") + " gp*" actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) @@ -126,4 +129,8 @@ def test(manylinux_image, tmp_path): # We don't have a manylinux_2_28 image for i686 expected_wheels = [w for w in expected_wheels if "i686" not in w] + if manylinux_image != "manylinux2014": + # No GraalPy wheels on anything except manylinux2014 + expected_wheels = [w for w in expected_wheels if "graalpy" not in w] + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/utils.py b/test/utils.py index f4578a58a..0ebfe28bb 100644 --- a/test/utils.py +++ b/test/utils.py @@ -181,6 +181,14 @@ def expected_wheels( "pp310-pypy310_pp73", ] + # GraalPy encodes compilation platform and arch in the tag + if machine_arch == "x86_64" and platform == "linux": + python_abi_tags += ["graalpy230_310_native_x86_64_linux"] + elif machine_arch == "aarch64" and platform == "linux": + python_abi_tags += ["graalpy230_310_native_aarch64_linux"] + elif machine_arch == "AMD64" and platform == "macos": + python_abi_tags += ["graalpy230_310_native_x86_64_darwin"] + if platform == "macos" and machine_arch == "arm64": # arm64 macs are only supported by cp38+ python_abi_tags = [ @@ -192,6 +200,7 @@ def expected_wheels( "pp38-pypy38_pp73", "pp39-pypy39_pp73", "pp310-pypy310_pp73", + "graalpy230_310_native_aarch64_darwin", ] wheels = []