From d9b6e9bbf32110c38c6f2274079474ec495474bf Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 4 Oct 2024 16:43:33 +0200 Subject: [PATCH 1/5] gh-127629: Add ctypes to the Emscripten build --- Tools/wasm/emscripten/__main__.py | 31 ++++++++++++++++++++++++++-- Tools/wasm/emscripten/make_libffi.sh | 22 ++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100755 Tools/wasm/emscripten/make_libffi.sh diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index c998ed71309dad..62256329fa8134 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -159,6 +159,26 @@ def make_build_python(context, working_dir): print(f"🎉 {binary} {version}") +@subdir(HOST_DIR, clean_ok=True) +def make_emscripten_libffi(context, working_dir): + shutil.rmtree(HOST_DIR/"libffi", ignore_errors=True) + + call([ + "git", + "clone", + "https://github.com/libffi/libffi", + "--depth=1" + ], + quiet=context.quiet, + ) + call( + [EMSCRIPTEN_DIR / "make_libffi.sh"], + env=updated_env({"PREFIX": HOST_DIR}), + cwd=HOST_DIR/"libffi", + quiet=context.quiet, + ) + + @subdir(HOST_DIR, clean_ok=True) def configure_emscripten_python(context, working_dir): """Configure the emscripten/host build.""" @@ -183,12 +203,14 @@ def configure_emscripten_python(context, working_dir): sysconfig_data += "-pydebug" host_runner = context.host_runner - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + pkg_config_path_dir = (working_dir / "lib/pkgconfig/").resolve() + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner, "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir)} build_python = os.fsdecode(build_python_path()) configure = [ "emconfigure", os.path.relpath(CHECKOUT / "configure", working_dir), "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", + 'PKG_CONFIG=pkg-config', f"--host={HOST_TRIPLE}", f"--build={build_platform()}", f"--with-build-python={build_python}", @@ -264,6 +286,7 @@ def build_all(context): steps = [ configure_build_python, make_build_python, + make_emscripten_libffi, configure_emscripten_python, make_emscripten_python, ] @@ -292,6 +315,9 @@ def main(): configure_build = subcommands.add_parser( "configure-build-python", help="Run `configure` for the " "build Python" ) + make_libffi_cmd = subcommands.add_parser( + "make-libffi", help="Clone libffi repo, configure and build it for emscripten" + ) make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" ) @@ -303,7 +329,7 @@ def main(): clean = subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) - for subcommand in build, configure_build, make_build, configure_host, make_host: + for subcommand in build, configure_build, make_libffi_cmd, make_build, configure_host, make_host: subcommand.add_argument( "--quiet", action="store_true", @@ -336,6 +362,7 @@ def main(): context = parser.parse_args() dispatch = { + "make-libffi": make_emscripten_libffi, "configure-build-python": configure_build_python, "make-build-python": make_build_python, "configure-host": configure_emscripten_python, diff --git a/Tools/wasm/emscripten/make_libffi.sh b/Tools/wasm/emscripten/make_libffi.sh new file mode 100755 index 00000000000000..aa22a8a7f04819 --- /dev/null +++ b/Tools/wasm/emscripten/make_libffi.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set +e + +export CFLAGS="-O2 -fPIC -DWASM_BIGINT" +export CXXFLAGS="$CFLAGS" + +# Build paths +export CPATH="$PREFIX/include" +export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig" +export EM_PKG_CONFIG_PATH="$PKG_CONFIG_PATH" + +# Specific variables for cross-compilation +export CHOST="wasm32-unknown-linux" # wasm32-unknown-emscripten + +autoreconf -fi +emconfigure ./configure --host=$CHOST --prefix="$PREFIX" --enable-static --disable-shared --disable-dependency-tracking \ + --disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs + +make install +# Some forgotten headers? +cp fficonfig.h $PREFIX/include/ +cp include/ffi_common.h $PREFIX/include/ From 8172368e9550aa054c4f1480dee2574e80019379 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 6 Dec 2024 12:47:56 +0100 Subject: [PATCH 2/5] Add news --- .../next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst diff --git a/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst new file mode 100644 index 00000000000000..52ee84f5d6f6c3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst @@ -0,0 +1 @@ +Emscripten builds now include ctypes support. From a3f4891d172264959e1025aed1c4b91007f3e8ff Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 9 Dec 2024 19:39:52 +0100 Subject: [PATCH 3/5] Use release libffi --- Tools/wasm/emscripten/__main__.py | 41 ++++++++++++++++++---------- Tools/wasm/emscripten/make_libffi.sh | 1 - 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index a8dd6dff8f71ab..73901f0910f002 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -9,6 +9,7 @@ import sys import sysconfig import tempfile +from urllib.request import urlopen from pathlib import Path from textwrap import dedent @@ -161,20 +162,17 @@ def make_build_python(context, working_dir): @subdir(HOST_DIR, clean_ok=True) def make_emscripten_libffi(context, working_dir): - shutil.rmtree(HOST_DIR/"libffi", ignore_errors=True) - - call([ - "git", - "clone", - "https://github.com/libffi/libffi", - "--depth=1" - ], - quiet=context.quiet, - ) + shutil.rmtree(HOST_DIR, ignore_errors=True) + with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file: + with urlopen( + "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz" + ) as response: + shutil.copyfileobj(response, tmp_file) + shutil.unpack_archive(tmp_file.name, HOST_DIR) call( [EMSCRIPTEN_DIR / "make_libffi.sh"], env=updated_env({"PREFIX": HOST_DIR}), - cwd=HOST_DIR/"libffi", + cwd=HOST_DIR / "libffi-3.4.6", quiet=context.quiet, ) @@ -204,13 +202,17 @@ def configure_emscripten_python(context, working_dir): host_runner = context.host_runner pkg_config_path_dir = (working_dir / "lib/pkgconfig/").resolve() - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner, "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir)} + env_additions = { + "CONFIG_SITE": config_site, + "HOSTRUNNER": host_runner, + "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), + } build_python = os.fsdecode(build_python_path()) configure = [ "emconfigure", os.path.relpath(CHECKOUT / "configure", working_dir), "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", - 'PKG_CONFIG=pkg-config', + "PKG_CONFIG=pkg-config", f"--host={HOST_TRIPLE}", f"--build={build_platform()}", f"--with-build-python={build_python}", @@ -325,11 +327,20 @@ def main(): "configure-host", help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)", ) - make_host = subcommands.add_parser("make-host", help="Run `make` for the host/emscripten") + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/emscripten" + ) clean = subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) - for subcommand in build, configure_build, make_libffi_cmd, make_build, configure_host, make_host: + for subcommand in ( + build, + configure_build, + make_libffi_cmd, + make_build, + configure_host, + make_host, + ): subcommand.add_argument( "--quiet", action="store_true", diff --git a/Tools/wasm/emscripten/make_libffi.sh b/Tools/wasm/emscripten/make_libffi.sh index aa22a8a7f04819..3c75c4d5127102 100755 --- a/Tools/wasm/emscripten/make_libffi.sh +++ b/Tools/wasm/emscripten/make_libffi.sh @@ -12,7 +12,6 @@ export EM_PKG_CONFIG_PATH="$PKG_CONFIG_PATH" # Specific variables for cross-compilation export CHOST="wasm32-unknown-linux" # wasm32-unknown-emscripten -autoreconf -fi emconfigure ./configure --host=$CHOST --prefix="$PREFIX" --enable-static --disable-shared --disable-dependency-tracking \ --disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs From 556de1e773839e63a1678630958dd9781b370f58 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 9 Dec 2024 20:04:26 +0100 Subject: [PATCH 4/5] Distinguish between `build` and `prefix` directories --- Tools/wasm/emscripten/__main__.py | 32 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 73901f0910f002..4a53e0bd1bee1b 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -23,9 +23,13 @@ CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent CROSS_BUILD_DIR = CHECKOUT / "cross-build" -BUILD_DIR = CROSS_BUILD_DIR / "build" +NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build" HOST_TRIPLE = "wasm32-emscripten" -HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE + +DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_DIR = HOST_BUILD_DIR / "python" +PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") @@ -119,16 +123,16 @@ def build_platform(): def build_python_path(): """The path to the build Python binary.""" - binary = BUILD_DIR / "python" + binary = NATIVE_BUILD_DIR / "python" if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " f"{BUILD_DIR}") + raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") return binary -@subdir(BUILD_DIR, clean_ok=True) +@subdir(NATIVE_BUILD_DIR, clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" if LOCAL_SETUP.exists(): @@ -144,7 +148,7 @@ def configure_build_python(context, working_dir): call(configure, quiet=context.quiet) -@subdir(BUILD_DIR) +@subdir(NATIVE_BUILD_DIR) def make_build_python(context, working_dir): """Make/build the build Python.""" call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) @@ -160,19 +164,19 @@ def make_build_python(context, working_dir): print(f"🎉 {binary} {version}") -@subdir(HOST_DIR, clean_ok=True) +@subdir(HOST_BUILD_DIR, clean_ok=True) def make_emscripten_libffi(context, working_dir): - shutil.rmtree(HOST_DIR, ignore_errors=True) + shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True) with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file: with urlopen( "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz" ) as response: shutil.copyfileobj(response, tmp_file) - shutil.unpack_archive(tmp_file.name, HOST_DIR) + shutil.unpack_archive(tmp_file.name, working_dir) call( [EMSCRIPTEN_DIR / "make_libffi.sh"], - env=updated_env({"PREFIX": HOST_DIR}), - cwd=HOST_DIR / "libffi-3.4.6", + env=updated_env({"PREFIX": PREFIX_DIR}), + cwd=working_dir / "libffi-3.4.6", quiet=context.quiet, ) @@ -186,7 +190,7 @@ def configure_emscripten_python(context, working_dir): emscripten_build_dir = working_dir.relative_to(CHECKOUT) - python_build_dir = BUILD_DIR / "build" + python_build_dir = NATIVE_BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) assert ( len(lib_dirs) == 1 @@ -201,7 +205,7 @@ def configure_emscripten_python(context, working_dir): sysconfig_data += "-pydebug" host_runner = context.host_runner - pkg_config_path_dir = (working_dir / "lib/pkgconfig/").resolve() + pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() env_additions = { "CONFIG_SITE": config_site, "HOSTRUNNER": host_runner, @@ -221,7 +225,7 @@ def configure_emscripten_python(context, working_dir): "--disable-ipv6", "--enable-big-digits=30", "--enable-wasm-dynamic-linking", - f"--prefix={HOST_DIR}", + f"--prefix={PREFIX_DIR}", ] if pydebug: configure.append("--with-pydebug") From f670c2153cc6a70203ea96bf6504e6febf9a0255 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 9 Dec 2024 20:05:47 +0100 Subject: [PATCH 5/5] Add separate make-libffi step to readme --- Tools/wasm/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 4802d9683de52e..2e0fa2fb533d67 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -61,6 +61,7 @@ or you can break it out into four separate steps: ```shell python Tools/wasm/emscripten configure-build-python python Tools/wasm/emscripten make-build-python +python Tools/wasm/emscripten make-libffi python Tools/wasm/emscripten configure-host python Tools/wasm/emscripten make-host ```