From 13ac19d9fb778fb7beca76fa52bda0361d6cd428 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 7 Oct 2025 19:25:06 -0700 Subject: [PATCH 1/2] GH-139590: Run `ruff format` on pre-commit for Tools/wasm (#139591) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> (cherry picked from commit a15aeec29efa5b3d5d5568278c13bb3fc45f52ef) --- .pre-commit-config.yaml | 4 + Tools/wasm/.ruff.toml | 28 + Tools/wasm/emscripten/__main__.py | 491 ++++++++++++++++++ .../wasm/emscripten/prepare_external_wasm.py | 53 ++ Tools/wasm/wasi.py | 289 +++++++---- Tools/wasm/wasi/__main__.py | 491 ++++++++++++++++++ Tools/wasm/wasm_build.py | 27 +- Tools/wasm/wasm_webserver.py | 19 +- 8 files changed, 1281 insertions(+), 121 deletions(-) create mode 100644 Tools/wasm/.ruff.toml create mode 100644 Tools/wasm/emscripten/__main__.py create mode 100644 Tools/wasm/emscripten/prepare_external_wasm.py create mode 100644 Tools/wasm/wasi/__main__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e35b0880a4370e..977b4bd6b33a8f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,10 @@ repos: name: Run Ruff (format) on Doc/ args: [--check] files: ^Doc/ + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--check, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror rev: 25.1.0 diff --git a/Tools/wasm/.ruff.toml b/Tools/wasm/.ruff.toml new file mode 100644 index 00000000000000..aabcf8dc4f502e --- /dev/null +++ b/Tools/wasm/.ruff.toml @@ -0,0 +1,28 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + "E501", # Line too long + "F541", # f-string without any placeholders + "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` + "PYI025", # Use `from collections.abc import Set as AbstractSet` +] diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py new file mode 100644 index 00000000000000..fdf3142c0a3b1a --- /dev/null +++ b/Tools/wasm/emscripten/__main__.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os +import shutil +import subprocess +import sys +import sysconfig +import hashlib +import tempfile +from urllib.request import urlopen +from pathlib import Path +from textwrap import dedent + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count + + +EMSCRIPTEN_DIR = Path(__file__).parent +CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build" +HOST_TRIPLE = "wasm32-emscripten" + +DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_DIR = HOST_BUILD_DIR / "python" +PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode( + "utf-8" +) + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("๐ŸŒŽ Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("โŽฏ" * terminal_width) + print("๐Ÿ“", working_dir) + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): + print("๐Ÿšฎ Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, quiet, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + print("โฏ", " ".join(map(str, command))) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + prefix="cpython-emscripten-", + suffix=".log", + ) + stderr = subprocess.STDOUT + print(f"๐Ÿ“ Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_platform(): + """The name of the build/host platform.""" + # Can also be found via `config.guess`.` + return sysconfig.get_config_var("BUILD_GNU_TYPE") + + +def build_python_path(): + """The path to the build Python binary.""" + binary = NATIVE_BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}" + ) + + return binary + + +@subdir(NATIVE_BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...") + else: + print(f"๐Ÿ“ Touching {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, quiet=context.quiet) + + +@subdir(NATIVE_BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) + + binary = build_python_path() + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"๐ŸŽ‰ {binary} {version}") + + +def check_shasum(file: str, expected_shasum: str): + with open(file, "rb") as f: + digest = hashlib.file_digest(f, "sha256") + if digest.hexdigest() != expected_shasum: + raise RuntimeError(f"Unexpected shasum for {file}") + + +def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): + with tempfile.NamedTemporaryFile( + suffix=".tar.gz", delete_on_close=False + ) as tmp_file: + with urlopen(url) as response: + shutil.copyfileobj(response, tmp_file) + tmp_file.close() + check_shasum(tmp_file.name, expected_shasum) + shutil.unpack_archive(tmp_file.name, working_dir) + + +@subdir(HOST_BUILD_DIR, clean_ok=True) +def make_emscripten_libffi(context, working_dir): + ver = "3.4.6" + libffi_dir = working_dir / f"libffi-{ver}" + shutil.rmtree(libffi_dir, ignore_errors=True) + download_and_unpack( + working_dir, + f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", + "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e", + ) + call( + [EMSCRIPTEN_DIR / "make_libffi.sh"], + env=updated_env({"PREFIX": PREFIX_DIR}), + cwd=libffi_dir, + quiet=context.quiet, + ) + + +@subdir(HOST_BUILD_DIR, clean_ok=True) +def make_mpdec(context, working_dir): + ver = "4.0.1" + mpdec_dir = working_dir / f"mpdecimal-{ver}" + shutil.rmtree(mpdec_dir, ignore_errors=True) + download_and_unpack( + working_dir, + f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", + "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", + ) + call( + [ + "emconfigure", + mpdec_dir / "configure", + "CFLAGS=-fPIC", + "--prefix", + PREFIX_DIR, + "--disable-shared", + ], + cwd=mpdec_dir, + quiet=context.quiet, + ) + call( + ["make", "install"], + cwd=mpdec_dir, + quiet=context.quiet, + ) + + +@subdir(HOST_DIR, clean_ok=True) +def configure_emscripten_python(context, working_dir): + """Configure the emscripten/host build.""" + config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten") + + emscripten_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = NATIVE_BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) + lib_dir = os.fsdecode(lib_dirs[0]) + pydebug = lib_dir.endswith("-pydebug") + python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + sysconfig_data = ( + f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" + ) + if pydebug: + sysconfig_data += "-pydebug" + + host_runner = context.host_runner + if node_version := os.environ.get("PYTHON_NODE_VERSION", None): + res = subprocess.run( + [ + "bash", + "-c", + f"source ~/.nvm/nvm.sh && nvm which {node_version}", + ], + text=True, + capture_output=True, + ) + host_runner = res.stdout.strip() + pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() + env_additions = { + "CONFIG_SITE": config_site, + "HOSTRUNNER": host_runner, + "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), + } + build_python = os.fsdecode(build_python_path()) + configure = [ + "emconfigure", + os.path.relpath(CHECKOUT / "configure", working_dir), + "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", + "PKG_CONFIG=pkg-config", + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}", + "--without-pymalloc", + "--disable-shared", + "--disable-ipv6", + "--enable-big-digits=30", + "--enable-wasm-dynamic-linking", + f"--prefix={PREFIX_DIR}", + ] + if pydebug: + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions), + quiet=context.quiet, + ) + + shutil.copy( + EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs" + ) + + node_entry = working_dir / "node_entry.mjs" + exec_script = working_dir / "python.sh" + exec_script.write_text( + dedent( + f"""\ + #!/bin/sh + + # Macs come with FreeBSD coreutils which doesn't have the -s option + # so feature detect and work around it. + if which grealpath > /dev/null 2>&1; then + # It has brew installed gnu core utils, use that + REALPATH="grealpath -s" + elif which realpath > /dev/null 2>&1 && realpath --version > /dev/null 2>&1 && realpath --version | grep GNU > /dev/null 2>&1; then + # realpath points to GNU realpath so use it. + REALPATH="realpath -s" + else + # Shim for macs without GNU coreutils + abs_path () {{ + echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" + }} + REALPATH=abs_path + fi + + # Before node 24, --experimental-wasm-jspi uses different API, + # After node 24 JSPI is on by default. + ARGS=$({host_runner} -e "$(cat <<"EOF" + const major_version = Number(process.version.split(".")[0].slice(1)); + if (major_version === 24) {{ + process.stdout.write("--experimental-wasm-jspi"); + }} + EOF + )") + + # We compute our own path, not following symlinks and pass it in so that + # node_entry.mjs can set sys.executable correctly. + # Intentionally allow word splitting on NODEFLAGS. + exec {host_runner} $NODEFLAGS $ARGS {node_entry} --this-program="$($REALPATH "$0")" "$@" + """ + ) + ) + exec_script.chmod(0o755) + print(f"๐Ÿƒโ€โ™€๏ธ Created {exec_script} ... ") + sys.stdout.flush() + + +@subdir(HOST_DIR) +def make_emscripten_python(context, working_dir): + """Run `make` for the emscripten/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + quiet=context.quiet, + ) + + exec_script = working_dir / "python.sh" + subprocess.check_call([exec_script, "--version"]) + + +def build_all(context): + """Build everything.""" + steps = [ + configure_build_python, + make_build_python, + make_emscripten_libffi, + make_mpdec, + configure_emscripten_python, + make_emscripten_python, + ] + for step in steps: + step(context) + + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + print(f"๐Ÿงน Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + with LOCAL_SETUP.open("rb") as file: + if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: + print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...") + + +def main(): + default_host_runner = "node" + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_mpdec_cmd = subcommands.add_parser( + "make-mpdec", + help="Clone mpdec repo, configure and build it for emscripten", + ) + make_libffi_cmd = subcommands.add_parser( + "make-libffi", + help="Clone libffi repo, configure and build it for emscripten", + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/emscripten" + ) + clean = subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_libffi_cmd, + make_mpdec_cmd, + make_build, + configure_host, + make_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + for subcommand in configure_build, configure_host: + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + for subcommand in build, configure_host: + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the emscripten host" + f"`{default_host_runner}`)", + ) + + context = parser.parse_args() + + dispatch = { + "make-libffi": make_emscripten_libffi, + "make-mpdec": make_mpdec, + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_emscripten_python, + "make-host": make_emscripten_python, + "build": build_all, + "clean": clean_contents, + } + + if not context.subcommand: + # No command provided, display help and exit + print( + "Expected one of", + ", ".join(sorted(dispatch.keys())), + file=sys.stderr, + ) + parser.print_help(sys.stderr) + sys.exit(1) + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/emscripten/prepare_external_wasm.py b/Tools/wasm/emscripten/prepare_external_wasm.py new file mode 100644 index 00000000000000..1b0a9de4b1fe8d --- /dev/null +++ b/Tools/wasm/emscripten/prepare_external_wasm.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +JS_TEMPLATE = """ +#include "emscripten.h" + +EM_JS(void, {function_name}, (void), {{ + return new WebAssembly.Module(hexStringToUTF8Array("{hex_string}")); +}} +function hexStringToUTF8Array(hex) {{ + const bytes = []; + for (let i = 0; i < hex.length; i += 2) {{ + bytes.push(parseInt(hex.substr(i, 2), 16)); + }} + return new Uint8Array(bytes); +}}); +""" + + +def prepare_wasm(input_file, output_file, function_name): + # Read the compiled WASM as binary and convert to hex + wasm_bytes = Path(input_file).read_bytes() + + hex_string = "".join(f"{byte:02x}" for byte in wasm_bytes) + + # Generate JavaScript module + js_content = JS_TEMPLATE.format( + function_name=function_name, hex_string=hex_string + ) + Path(output_file).write_text(js_content) + + print(f"Successfully compiled {input_file} and generated {output_file}") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Compile WebAssembly text files using wasm-as" + ) + parser.add_argument("input_file", help="Input .wat file to compile") + parser.add_argument("output_file", help="Output file name") + parser.add_argument("function_name", help="Name of the export function") + + args = parser.parse_args() + + return prepare_wasm(args.input_file, args.output_file, args.function_name) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index c8f48a26f2a8be..34051bd7351a37 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -4,6 +4,7 @@ import contextlib import functools import os + try: from os import process_cpu_count as cpu_count except ImportError: @@ -37,7 +38,9 @@ def updated_env(updates={}): # https://reproducible-builds.org/docs/source-date-epoch/ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() env_defaults["SOURCE_DATE_EPOCH"] = epoch except subprocess.CalledProcessError: pass # Might be building from a tarball. @@ -58,6 +61,7 @@ def updated_env(updates={}): def subdir(working_dir, *, clean_ok=False): """Decorator to change to a working directory.""" + def decorator(func): @functools.wraps(func) def wrapper(context): @@ -66,16 +70,20 @@ def wrapper(context): if callable(working_dir): working_dir = working_dir(context) try: - tput_output = subprocess.check_output(["tput", "cols"], - encoding="utf-8") + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) except subprocess.CalledProcessError: terminal_width = 80 else: terminal_width = int(tput_output.strip()) print("โŽฏ" * terminal_width) print("๐Ÿ“", working_dir) - if (clean_ok and getattr(context, "clean", False) and - working_dir.exists()): + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): print(f"๐Ÿšฎ Deleting directory (--clean)...") shutil.rmtree(working_dir) @@ -99,10 +107,13 @@ def call(command, *, quiet, **kwargs): stdout = None stderr = None else: - stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", - delete=False, - prefix="cpython-wasi-", - suffix=".log") + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + prefix="cpython-wasi-", + suffix=".log", + ) stderr = subprocess.STDOUT print(f"๐Ÿ“ Logging output to {stdout.name} (--quiet)...") @@ -121,8 +132,9 @@ def build_python_path(): if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{BUILD_DIR}") + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) return binary @@ -136,7 +148,7 @@ def configure_build_python(context, working_dir): print(f"๐Ÿ“ Touching {LOCAL_SETUP} ...") LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] if context.args: configure.extend(context.args) @@ -146,13 +158,15 @@ def configure_build_python(context, working_dir): @subdir(BUILD_DIR) def make_build_python(context, working_dir): """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], - quiet=context.quiet) + call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) binary = build_python_path() - cmd = [binary, "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] version = subprocess.check_output(cmd, encoding="utf-8").strip() print(f"๐ŸŽ‰ {binary} {version}") @@ -170,8 +184,13 @@ def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", - "AR": "llvm-ar", "RANLIB": "ranlib"} + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } for env_var, binary_name in list(env.items()): env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) @@ -182,16 +201,20 @@ def wasi_sdk_env(context): env["PKG_CONFIG_PATH"] = "" env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map(os.fsdecode, - [sysroot / "lib" / "pkgconfig", - sysroot / "share" / "pkgconfig"])) + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) env["WASI_SYSROOT"] = os.fsdecode(sysroot) - env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"]]) + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) return env @@ -200,18 +223,24 @@ def wasi_sdk_env(context): def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk") + raise ValueError( + "WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) - config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") + config_site = os.fsdecode( + CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi" + ) wasi_build_dir = working_dir.relative_to(CHECKOUT) python_build_dir = BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) lib_dir = os.fsdecode(lib_dirs[0]) pydebug = lib_dir.endswith("-pydebug") python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] @@ -221,36 +250,44 @@ def configure_wasi_python(context, working_dir): # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. - args = {"GUEST_DIR": "/", - "HOST_DIR": CHECKOUT, - "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data}", - "PYTHON_WASM": working_dir / "python.wasm"} + args = { + "GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data}", + "PYTHON_WASM": working_dir / "python.wasm", + } # Check dynamically for wasmtime in case it was specified manually via # `--host-runner`. if WASMTIME_HOST_RUNNER_VAR in context.host_runner: if wasmtime := shutil.which("wasmtime"): args[WASMTIME_VAR_NAME] = wasmtime else: - raise FileNotFoundError("wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime") + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime" + ) host_runner = context.host_runner.format_map(args) env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} build_python = os.fsdecode(build_python_path()) # The path to `configure` MUST be relative, else `python.wasm` is unable # to find the stdlib due to Python not recognizing that it's being # executed from within a checkout. - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={context.host_triple}", - f"--build={build_platform()}", - f"--with-build-python={build_python}"] + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={build_platform()}", + f"--with-build-python={build_python}", + ] if pydebug: configure.append("--with-pydebug") if context.args: configure.extend(context.args) - call(configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - quiet=context.quiet) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + quiet=context.quiet, + ) python_wasm = working_dir / "python.wasm" exec_script = working_dir / "python.sh" @@ -264,9 +301,11 @@ def configure_wasi_python(context, working_dir): @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" - call(["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - quiet=context.quiet) + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + quiet=context.quiet, + ) exec_script = working_dir / "python.sh" subprocess.check_call([exec_script, "--version"]) @@ -274,11 +313,16 @@ def make_wasi_python(context, working_dir): def build_all(context): """Build everything.""" - steps = [configure_build_python, make_build_python, configure_wasi_python, - make_wasi_python] + steps = [ + configure_build_python, + make_build_python, + configure_wasi_python, + make_wasi_python, + ] for step in steps: step(context) + def clean_contents(context): """Delete all files created by this script.""" if CROSS_BUILD_DIR.exists(): @@ -292,74 +336,109 @@ def clean_contents(context): def main(): - default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " - # Make sure the stack size will work for a pydebug - # build. - # The 8388608 value comes from `ulimit -s` under Linux - # which equates to 8291 KiB. - "--wasm max-wasm-stack=8388608 " - # Use WASI 0.2 primitives. - "--wasi preview2 " - # Enable thread support; causes use of preview1. - #"--wasm threads=y --wasi threads=y " - # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") + default_host_runner = ( + f"{WASMTIME_HOST_RUNNER_VAR} run " + # Make sure the stack size will work for a pydebug + # build. + # The 8388608 value comes from `ulimit -s` under Linux + # which equates to 8291 KiB. + "--wasm max-wasm-stack=8388608 " + # Use WASI 0.2 primitives. + "--wasi preview2 " + # Enable thread support; causes use of preview1. + # "--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}" + ) parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build-python", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build-python", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for the " - "host/WASI (pydebug builds " - "are inferred from the build " - "Python)") - make_host = subcommands.add_parser("make-host", - help="Run `make` for the host/WASI") - clean = subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") - for subcommand in build, configure_build, make_build, configure_host, make_host: - subcommand.add_argument("--quiet", action="store_true", default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/WASI" + ) + clean = subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_build, + configure_host, + make_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) for subcommand in configure_build, configure_host: - subcommand.add_argument("--clean", action="store_true", default=False, - dest="clean", - help="Delete any relevant directories before building") + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) for subcommand in build, configure_build, configure_host: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) for subcommand in build, configure_host: - subcommand.add_argument("--wasi-sdk", type=pathlib.Path, - dest="wasi_sdk_path", - default=find_wasi_sdk(), - help="Path to wasi-sdk; defaults to " - "$WASI_SDK_PATH or /opt/wasi-sdk") - subcommand.add_argument("--host-runner", action="store", - default=default_host_runner, dest="host_runner", - help="Command template for running the WASI host " - "(default designed for wasmtime 14 or newer: " - f"`{default_host_runner}`)") + subcommand.add_argument( + "--wasi-sdk", + type=pathlib.Path, + dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk", + ) + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the WASI host " + "(default designed for wasmtime 14 or newer: " + f"`{default_host_runner}`)", + ) for subcommand in build, configure_host, make_host: - subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1", - help="The target triple for the WASI host build") + subcommand.add_argument( + "--host-triple", + action="store", + default="wasm32-wasip1", + help="The target triple for the WASI host build", + ) context = parser.parse_args() - dispatch = {"configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build": build_all, - "clean": clean_contents} + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all, + "clean": clean_contents, + } dispatch[context.subcommand](context) -if __name__ == "__main__": +if __name__ == "__main__": main() diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py new file mode 100644 index 00000000000000..a0658cb351a86f --- /dev/null +++ b/Tools/wasm/wasi/__main__.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count +import pathlib +import shutil +import subprocess +import sys +import sysconfig +import tempfile + + +CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent +assert (CHECKOUT / "configure").is_file(), ( + "Please update the location of the file" +) + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +# Build platform can also be found via `config.guess`. +BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = ( + "# Generated by Tools/wasm/wasi .\n" + "# Required to statically build extension modules." +).encode("utf-8") + +WASI_SDK_VERSION = 24 + +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("๐ŸŒŽ Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + nonlocal working_dir + + if callable(working_dir): + working_dir = working_dir(context) + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + except subprocess.CalledProcessError: + terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) + print("โŽฏ" * terminal_width) + print("๐Ÿ“", working_dir) + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): + print("๐Ÿšฎ Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, context=None, quiet=False, logdir=None, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + if context is not None: + quiet = context.quiet + logdir = context.logdir + elif quiet and logdir is None: + raise ValueError("When quiet is True, logdir must be specified") + + print("โฏ", " ".join(map(str, command))) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=logdir, + prefix="cpython-wasi-", + suffix=".log", + ) + stderr = subprocess.STDOUT + print(f"๐Ÿ“ Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) + + return binary + + +def build_python_is_pydebug(): + """Find out if the build Python is a pydebug build.""" + test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" + result = subprocess.run( + [build_python_path(), "-c", test], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return bool(result.returncode) + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...") + else: + print(f"โš ๏ธ {LOCAL_SETUP} exists, but has unexpected contents") + else: + print(f"๐Ÿ“ Creating {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, context=context) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], context=context) + + binary = build_python_path() + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"๐ŸŽ‰ {binary} {version}") + + +def find_wasi_sdk(): + """Find the path to the WASI SDK.""" + if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): + return pathlib.Path(wasi_sdk_path) + + opt_path = pathlib.Path("/opt") + # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team + # has said they don't plan to ever do a point release and all of their Git tags + # lack the ``.0`` suffix. + # Starting with WASI SDK 23, the tarballs went from containing a directory named + # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. + # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. + potential_sdks = [ + path + for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") + if path.is_dir() + ] + if len(potential_sdks) == 1: + return potential_sdks[0] + elif (default_path := opt_path / "wasi-sdk").is_dir(): + return default_path + + +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = context.wasi_sdk_path + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) + + return env + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError( + "WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) + + config_site = os.fsdecode( + CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi" + ) + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) + lib_dir = os.fsdecode(lib_dirs[0]) + python_version = lib_dir.rpartition("-")[-1] + sysconfig_data_dir = ( + f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + ) + + # Use PYTHONPATH to include sysconfig data which must be anchored to the + # WASI guest's `/` directory. + args = { + "GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data_dir}", + "PYTHON_WASM": working_dir / "python.wasm", + } + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if WASMTIME_HOST_RUNNER_VAR in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args[WASMTIME_VAR_NAME] = wasmtime + else: + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime" + ) + host_runner = context.host_runner.format_map(args) + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={BUILD_DIR.name}", + f"--with-build-python={build_python}", + ] + if build_python_is_pydebug(): + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + context=context, + ) + + python_wasm = working_dir / "python.wasm" + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') + exec_script.chmod(0o755) + print(f"๐Ÿƒโ€โ™€๏ธ Created {exec_script} (--host-runner)... ") + sys.stdout.flush() + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + context=context, + ) + + exec_script = working_dir / "python.sh" + call([exec_script, "--version"], quiet=False) + print( + f"๐ŸŽ‰ Use `{exec_script.relative_to(context.init_dir)}` " + "to run CPython w/ the WASI host specified by --host-runner" + ) + + +def build_all(context): + """Build everything.""" + steps = [ + configure_build_python, + make_build_python, + configure_wasi_python, + make_wasi_python, + ] + for step in steps: + step(context) + + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + print(f"๐Ÿงน Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...") + + +def main(): + default_host_triple = "wasm32-wasip1" + default_wasi_sdk = find_wasi_sdk() + default_host_runner = ( + f"{WASMTIME_HOST_RUNNER_VAR} run " + # Make sure the stack size will work for a pydebug + # build. + # Use 16 MiB stack. + "--wasm max-wasm-stack=16777216 " + # Enable thread support; causes use of preview1. + # "--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}" + ) + default_logdir = pathlib.Path(tempfile.gettempdir()) + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/WASI" + ) + subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_build, + configure_host, + make_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + subcommand.add_argument( + "--logdir", + type=pathlib.Path, + default=default_logdir, + help=f"Directory to store log files; defaults to {default_logdir}", + ) + for subcommand in configure_build, configure_host: + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + for subcommand in build, configure_host: + subcommand.add_argument( + "--wasi-sdk", + type=pathlib.Path, + dest="wasi_sdk_path", + default=default_wasi_sdk, + help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", + ) + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the WASI host; defaults to " + f"`{default_host_runner}`", + ) + for subcommand in build, configure_host, make_host: + subcommand.add_argument( + "--host-triple", + action="store", + default=default_host_triple, + help="The target triple for the WASI host build; " + f"defaults to {default_host_triple}", + ) + + context = parser.parse_args() + context.init_dir = pathlib.Path().absolute() + + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all, + "clean": clean_contents, + } + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 47a0abb8b5feef..5cc1c06f4a2796 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -21,6 +21,7 @@ ./Tools/wasm/wasm_builder.py --clean build build """ + import argparse import enum import dataclasses @@ -67,7 +68,9 @@ # path to Emscripten SDK config file. # auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh". -EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) +EM_CONFIG = pathlib.Path( + os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten") +) EMSDK_MIN_VERSION = (3, 1, 19) EMSDK_BROKEN_VERSION = { (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338", @@ -261,8 +264,7 @@ def _check_emscripten() -> None: # git / upstream / tot-upstream installation version = version[:-4] version_tuple = cast( - Tuple[int, int, int], - tuple(int(v) for v in version.split(".")) + Tuple[int, int, int], tuple(int(v) for v in version.split(".")) ) if version_tuple < EMSDK_MIN_VERSION: raise ConditionError( @@ -518,7 +520,7 @@ def make_cmd(self) -> List[str]: def getenv(self) -> Dict[str, Any]: """Generate environ dict for platform""" env = os.environ.copy() - if hasattr(os, 'process_cpu_count'): + if hasattr(os, "process_cpu_count"): cpu_count = os.process_cpu_count() else: cpu_count = os.cpu_count() @@ -596,7 +598,9 @@ def run_py(self, *args: str) -> int: """Run Python with hostrunner""" self._check_execute() return self.run_make( - "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run" + "--eval", + f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", + "run", ) def run_browser(self, bind: str = "127.0.0.1", port: int = 8000) -> None: @@ -666,9 +670,12 @@ def build_emports(self, force: bool = False) -> None: # Pre-build libbz2, libsqlite3, libz, and some system libs. ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) # Multi-threaded sqlite3 has different suffix - embuilder_cmd.extend( - ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"] - ) + embuilder_cmd.extend([ + "build", + "bzip2", + "sqlite3-mt" if self.pthreads else "sqlite3", + "zlib", + ]) self._run_cmd(embuilder_cmd, cwd=SRCDIR) @@ -817,7 +824,9 @@ def build_emports(self, force: bool = False) -> None: # Don't list broken and experimental variants in help platforms_choices = list(p.name for p in _profiles) + ["cleanall"] -platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"] +platforms_help = list(p.name for p in _profiles if p.support_level) + [ + "cleanall" +] parser.add_argument( "platform", metavar="PLATFORM", diff --git a/Tools/wasm/wasm_webserver.py b/Tools/wasm/wasm_webserver.py index 3d1d5d42a1e8c4..fe565db4cb4db3 100755 --- a/Tools/wasm/wasm_webserver.py +++ b/Tools/wasm/wasm_webserver.py @@ -6,20 +6,24 @@ description="Start a local webserver with a Python terminal." ) parser.add_argument( - "--port", type=int, default=8000, help="port for the http server to listen on" + "--port", + type=int, + default=8000, + help="port for the http server to listen on", ) parser.add_argument( - "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" + "--bind", + type=str, + default="127.0.0.1", + help="Bind address (empty for all)", ) class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy() - extensions_map.update( - { - ".wasm": "application/wasm", - } - ) + extensions_map.update({ + ".wasm": "application/wasm", + }) def end_headers(self) -> None: self.send_my_headers() @@ -42,5 +46,6 @@ def main() -> None: bind=args.bind, ) + if __name__ == "__main__": main() From eee8f27f2161411bbb1c07149c372f9e9c267fc8 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 7 Oct 2025 20:25:11 -0700 Subject: [PATCH 2/2] Remove files --- Tools/wasm/emscripten/__main__.py | 491 ------------------ .../wasm/emscripten/prepare_external_wasm.py | 53 -- Tools/wasm/wasi/__main__.py | 491 ------------------ 3 files changed, 1035 deletions(-) delete mode 100644 Tools/wasm/emscripten/__main__.py delete mode 100644 Tools/wasm/emscripten/prepare_external_wasm.py delete mode 100644 Tools/wasm/wasi/__main__.py diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py deleted file mode 100644 index fdf3142c0a3b1a..00000000000000 --- a/Tools/wasm/emscripten/__main__.py +++ /dev/null @@ -1,491 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import contextlib -import functools -import os -import shutil -import subprocess -import sys -import sysconfig -import hashlib -import tempfile -from urllib.request import urlopen -from pathlib import Path -from textwrap import dedent - -try: - from os import process_cpu_count as cpu_count -except ImportError: - from os import cpu_count - - -EMSCRIPTEN_DIR = Path(__file__).parent -CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent - -CROSS_BUILD_DIR = CHECKOUT / "cross-build" -NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build" -HOST_TRIPLE = "wasm32-emscripten" - -DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" -HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" -HOST_DIR = HOST_BUILD_DIR / "python" -PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" - -LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode( - "utf-8" -) - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output( - git_epoch_cmd, encoding="utf-8" - ).strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - print("๐ŸŒŽ Environment changes:") - for key in sorted(env_diff.keys()): - print(f" {key}={env_diff[key]}") - - return environment - - -def subdir(working_dir, *, clean_ok=False): - """Decorator to change to a working directory.""" - - def decorator(func): - @functools.wraps(func) - def wrapper(context): - try: - tput_output = subprocess.check_output( - ["tput", "cols"], encoding="utf-8" - ) - terminal_width = int(tput_output.strip()) - except subprocess.CalledProcessError: - terminal_width = 80 - print("โŽฏ" * terminal_width) - print("๐Ÿ“", working_dir) - if ( - clean_ok - and getattr(context, "clean", False) - and working_dir.exists() - ): - print("๐Ÿšฎ Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, quiet, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - print("โฏ", " ".join(map(str, command))) - if not quiet: - stdout = None - stderr = None - else: - stdout = tempfile.NamedTemporaryFile( - "w", - encoding="utf-8", - delete=False, - prefix="cpython-emscripten-", - suffix=".log", - ) - stderr = subprocess.STDOUT - print(f"๐Ÿ“ Logging output to {stdout.name} (--quiet)...") - - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - - -def build_platform(): - """The name of the build/host platform.""" - # Can also be found via `config.guess`.` - return sysconfig.get_config_var("BUILD_GNU_TYPE") - - -def build_python_path(): - """The path to the build Python binary.""" - binary = NATIVE_BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError( - f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}" - ) - - return binary - - -@subdir(NATIVE_BUILD_DIR, clean_ok=True) -def configure_build_python(context, working_dir): - """Configure the build/host Python.""" - if LOCAL_SETUP.exists(): - print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...") - else: - print(f"๐Ÿ“ Touching {LOCAL_SETUP} ...") - LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - - configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] - if context.args: - configure.extend(context.args) - - call(configure, quiet=context.quiet) - - -@subdir(NATIVE_BUILD_DIR) -def make_build_python(context, working_dir): - """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) - - binary = build_python_path() - cmd = [ - binary, - "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')", - ] - version = subprocess.check_output(cmd, encoding="utf-8").strip() - - print(f"๐ŸŽ‰ {binary} {version}") - - -def check_shasum(file: str, expected_shasum: str): - with open(file, "rb") as f: - digest = hashlib.file_digest(f, "sha256") - if digest.hexdigest() != expected_shasum: - raise RuntimeError(f"Unexpected shasum for {file}") - - -def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): - with tempfile.NamedTemporaryFile( - suffix=".tar.gz", delete_on_close=False - ) as tmp_file: - with urlopen(url) as response: - shutil.copyfileobj(response, tmp_file) - tmp_file.close() - check_shasum(tmp_file.name, expected_shasum) - shutil.unpack_archive(tmp_file.name, working_dir) - - -@subdir(HOST_BUILD_DIR, clean_ok=True) -def make_emscripten_libffi(context, working_dir): - ver = "3.4.6" - libffi_dir = working_dir / f"libffi-{ver}" - shutil.rmtree(libffi_dir, ignore_errors=True) - download_and_unpack( - working_dir, - f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", - "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e", - ) - call( - [EMSCRIPTEN_DIR / "make_libffi.sh"], - env=updated_env({"PREFIX": PREFIX_DIR}), - cwd=libffi_dir, - quiet=context.quiet, - ) - - -@subdir(HOST_BUILD_DIR, clean_ok=True) -def make_mpdec(context, working_dir): - ver = "4.0.1" - mpdec_dir = working_dir / f"mpdecimal-{ver}" - shutil.rmtree(mpdec_dir, ignore_errors=True) - download_and_unpack( - working_dir, - f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", - "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", - ) - call( - [ - "emconfigure", - mpdec_dir / "configure", - "CFLAGS=-fPIC", - "--prefix", - PREFIX_DIR, - "--disable-shared", - ], - cwd=mpdec_dir, - quiet=context.quiet, - ) - call( - ["make", "install"], - cwd=mpdec_dir, - quiet=context.quiet, - ) - - -@subdir(HOST_DIR, clean_ok=True) -def configure_emscripten_python(context, working_dir): - """Configure the emscripten/host build.""" - config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten") - - emscripten_build_dir = working_dir.relative_to(CHECKOUT) - - python_build_dir = NATIVE_BUILD_DIR / "build" - lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, ( - f"Expected a single lib.* directory in {python_build_dir}" - ) - lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] - sysconfig_data = ( - f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" - ) - if pydebug: - sysconfig_data += "-pydebug" - - host_runner = context.host_runner - if node_version := os.environ.get("PYTHON_NODE_VERSION", None): - res = subprocess.run( - [ - "bash", - "-c", - f"source ~/.nvm/nvm.sh && nvm which {node_version}", - ], - text=True, - capture_output=True, - ) - host_runner = res.stdout.strip() - pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() - env_additions = { - "CONFIG_SITE": config_site, - "HOSTRUNNER": host_runner, - "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), - } - build_python = os.fsdecode(build_python_path()) - configure = [ - "emconfigure", - os.path.relpath(CHECKOUT / "configure", working_dir), - "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", - "PKG_CONFIG=pkg-config", - f"--host={HOST_TRIPLE}", - f"--build={build_platform()}", - f"--with-build-python={build_python}", - "--without-pymalloc", - "--disable-shared", - "--disable-ipv6", - "--enable-big-digits=30", - "--enable-wasm-dynamic-linking", - f"--prefix={PREFIX_DIR}", - ] - if pydebug: - configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) - call( - configure, - env=updated_env(env_additions), - quiet=context.quiet, - ) - - shutil.copy( - EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs" - ) - - node_entry = working_dir / "node_entry.mjs" - exec_script = working_dir / "python.sh" - exec_script.write_text( - dedent( - f"""\ - #!/bin/sh - - # Macs come with FreeBSD coreutils which doesn't have the -s option - # so feature detect and work around it. - if which grealpath > /dev/null 2>&1; then - # It has brew installed gnu core utils, use that - REALPATH="grealpath -s" - elif which realpath > /dev/null 2>&1 && realpath --version > /dev/null 2>&1 && realpath --version | grep GNU > /dev/null 2>&1; then - # realpath points to GNU realpath so use it. - REALPATH="realpath -s" - else - # Shim for macs without GNU coreutils - abs_path () {{ - echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" - }} - REALPATH=abs_path - fi - - # Before node 24, --experimental-wasm-jspi uses different API, - # After node 24 JSPI is on by default. - ARGS=$({host_runner} -e "$(cat <<"EOF" - const major_version = Number(process.version.split(".")[0].slice(1)); - if (major_version === 24) {{ - process.stdout.write("--experimental-wasm-jspi"); - }} - EOF - )") - - # We compute our own path, not following symlinks and pass it in so that - # node_entry.mjs can set sys.executable correctly. - # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} $NODEFLAGS $ARGS {node_entry} --this-program="$($REALPATH "$0")" "$@" - """ - ) - ) - exec_script.chmod(0o755) - print(f"๐Ÿƒโ€โ™€๏ธ Created {exec_script} ... ") - sys.stdout.flush() - - -@subdir(HOST_DIR) -def make_emscripten_python(context, working_dir): - """Run `make` for the emscripten/host build.""" - call( - ["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - quiet=context.quiet, - ) - - exec_script = working_dir / "python.sh" - subprocess.check_call([exec_script, "--version"]) - - -def build_all(context): - """Build everything.""" - steps = [ - configure_build_python, - make_build_python, - make_emscripten_libffi, - make_mpdec, - configure_emscripten_python, - make_emscripten_python, - ] - for step in steps: - step(context) - - -def clean_contents(context): - """Delete all files created by this script.""" - if CROSS_BUILD_DIR.exists(): - print(f"๐Ÿงน Deleting {CROSS_BUILD_DIR} ...") - shutil.rmtree(CROSS_BUILD_DIR) - - if LOCAL_SETUP.exists(): - with LOCAL_SETUP.open("rb") as file: - if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: - print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...") - - -def main(): - default_host_runner = "node" - - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser( - "configure-build-python", help="Run `configure` for the build Python" - ) - make_mpdec_cmd = subcommands.add_parser( - "make-mpdec", - help="Clone mpdec repo, configure and build it for emscripten", - ) - make_libffi_cmd = subcommands.add_parser( - "make-libffi", - help="Clone libffi repo, configure and build it for emscripten", - ) - make_build = subcommands.add_parser( - "make-build-python", help="Run `make` for the build Python" - ) - configure_host = subcommands.add_parser( - "configure-host", - help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)", - ) - make_host = subcommands.add_parser( - "make-host", help="Run `make` for the host/emscripten" - ) - clean = subcommands.add_parser( - "clean", help="Delete files and directories created by this script" - ) - for subcommand in ( - build, - configure_build, - make_libffi_cmd, - make_mpdec_cmd, - make_build, - configure_host, - make_host, - ): - subcommand.add_argument( - "--quiet", - action="store_true", - default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file", - ) - for subcommand in configure_build, configure_host: - subcommand.add_argument( - "--clean", - action="store_true", - default=False, - dest="clean", - help="Delete any relevant directories before building", - ) - for subcommand in build, configure_build, configure_host: - subcommand.add_argument( - "args", nargs="*", help="Extra arguments to pass to `configure`" - ) - for subcommand in build, configure_host: - subcommand.add_argument( - "--host-runner", - action="store", - default=default_host_runner, - dest="host_runner", - help="Command template for running the emscripten host" - f"`{default_host_runner}`)", - ) - - context = parser.parse_args() - - dispatch = { - "make-libffi": make_emscripten_libffi, - "make-mpdec": make_mpdec, - "configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_emscripten_python, - "make-host": make_emscripten_python, - "build": build_all, - "clean": clean_contents, - } - - if not context.subcommand: - # No command provided, display help and exit - print( - "Expected one of", - ", ".join(sorted(dispatch.keys())), - file=sys.stderr, - ) - parser.print_help(sys.stderr) - sys.exit(1) - dispatch[context.subcommand](context) - - -if __name__ == "__main__": - main() diff --git a/Tools/wasm/emscripten/prepare_external_wasm.py b/Tools/wasm/emscripten/prepare_external_wasm.py deleted file mode 100644 index 1b0a9de4b1fe8d..00000000000000 --- a/Tools/wasm/emscripten/prepare_external_wasm.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys -from pathlib import Path - -JS_TEMPLATE = """ -#include "emscripten.h" - -EM_JS(void, {function_name}, (void), {{ - return new WebAssembly.Module(hexStringToUTF8Array("{hex_string}")); -}} -function hexStringToUTF8Array(hex) {{ - const bytes = []; - for (let i = 0; i < hex.length; i += 2) {{ - bytes.push(parseInt(hex.substr(i, 2), 16)); - }} - return new Uint8Array(bytes); -}}); -""" - - -def prepare_wasm(input_file, output_file, function_name): - # Read the compiled WASM as binary and convert to hex - wasm_bytes = Path(input_file).read_bytes() - - hex_string = "".join(f"{byte:02x}" for byte in wasm_bytes) - - # Generate JavaScript module - js_content = JS_TEMPLATE.format( - function_name=function_name, hex_string=hex_string - ) - Path(output_file).write_text(js_content) - - print(f"Successfully compiled {input_file} and generated {output_file}") - return 0 - - -def main(): - parser = argparse.ArgumentParser( - description="Compile WebAssembly text files using wasm-as" - ) - parser.add_argument("input_file", help="Input .wat file to compile") - parser.add_argument("output_file", help="Output file name") - parser.add_argument("function_name", help="Name of the export function") - - args = parser.parse_args() - - return prepare_wasm(args.input_file, args.output_file, args.function_name) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py deleted file mode 100644 index a0658cb351a86f..00000000000000 --- a/Tools/wasm/wasi/__main__.py +++ /dev/null @@ -1,491 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import contextlib -import functools -import os - -try: - from os import process_cpu_count as cpu_count -except ImportError: - from os import cpu_count -import pathlib -import shutil -import subprocess -import sys -import sysconfig -import tempfile - - -CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent -assert (CHECKOUT / "configure").is_file(), ( - "Please update the location of the file" -) - -CROSS_BUILD_DIR = CHECKOUT / "cross-build" -# Build platform can also be found via `config.guess`. -BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") - -LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = ( - "# Generated by Tools/wasm/wasi .\n" - "# Required to statically build extension modules." -).encode("utf-8") - -WASI_SDK_VERSION = 24 - -WASMTIME_VAR_NAME = "WASMTIME" -WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output( - git_epoch_cmd, encoding="utf-8" - ).strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - print("๐ŸŒŽ Environment changes:") - for key in sorted(env_diff.keys()): - print(f" {key}={env_diff[key]}") - - return environment - - -def subdir(working_dir, *, clean_ok=False): - """Decorator to change to a working directory.""" - - def decorator(func): - @functools.wraps(func) - def wrapper(context): - nonlocal working_dir - - if callable(working_dir): - working_dir = working_dir(context) - try: - tput_output = subprocess.check_output( - ["tput", "cols"], encoding="utf-8" - ) - except subprocess.CalledProcessError: - terminal_width = 80 - else: - terminal_width = int(tput_output.strip()) - print("โŽฏ" * terminal_width) - print("๐Ÿ“", working_dir) - if ( - clean_ok - and getattr(context, "clean", False) - and working_dir.exists() - ): - print("๐Ÿšฎ Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, context=None, quiet=False, logdir=None, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - if context is not None: - quiet = context.quiet - logdir = context.logdir - elif quiet and logdir is None: - raise ValueError("When quiet is True, logdir must be specified") - - print("โฏ", " ".join(map(str, command))) - if not quiet: - stdout = None - stderr = None - else: - stdout = tempfile.NamedTemporaryFile( - "w", - encoding="utf-8", - delete=False, - dir=logdir, - prefix="cpython-wasi-", - suffix=".log", - ) - stderr = subprocess.STDOUT - print(f"๐Ÿ“ Logging output to {stdout.name} (--quiet)...") - - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - - -def build_python_path(): - """The path to the build Python binary.""" - binary = BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError( - f"Unable to find `python(.exe)` in {BUILD_DIR}" - ) - - return binary - - -def build_python_is_pydebug(): - """Find out if the build Python is a pydebug build.""" - test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" - result = subprocess.run( - [build_python_path(), "-c", test], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - return bool(result.returncode) - - -@subdir(BUILD_DIR, clean_ok=True) -def configure_build_python(context, working_dir): - """Configure the build/host Python.""" - if LOCAL_SETUP.exists(): - if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: - print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...") - else: - print(f"โš ๏ธ {LOCAL_SETUP} exists, but has unexpected contents") - else: - print(f"๐Ÿ“ Creating {LOCAL_SETUP} ...") - LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - - configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] - if context.args: - configure.extend(context.args) - - call(configure, context=context) - - -@subdir(BUILD_DIR) -def make_build_python(context, working_dir): - """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], context=context) - - binary = build_python_path() - cmd = [ - binary, - "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')", - ] - version = subprocess.check_output(cmd, encoding="utf-8").strip() - - print(f"๐ŸŽ‰ {binary} {version}") - - -def find_wasi_sdk(): - """Find the path to the WASI SDK.""" - if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): - return pathlib.Path(wasi_sdk_path) - - opt_path = pathlib.Path("/opt") - # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team - # has said they don't plan to ever do a point release and all of their Git tags - # lack the ``.0`` suffix. - # Starting with WASI SDK 23, the tarballs went from containing a directory named - # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. - # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. - potential_sdks = [ - path - for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") - if path.is_dir() - ] - if len(potential_sdks) == 1: - return potential_sdks[0] - elif (default_path := opt_path / "wasi-sdk").is_dir(): - return default_path - - -def wasi_sdk_env(context): - """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path - sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = { - "CC": "clang", - "CPP": "clang-cpp", - "CXX": "clang++", - "AR": "llvm-ar", - "RANLIB": "ranlib", - } - - for env_var, binary_name in list(env.items()): - env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) - - if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): - for compiler in ["CC", "CPP", "CXX"]: - env[compiler] += f" --sysroot={sysroot}" - - env["PKG_CONFIG_PATH"] = "" - env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map( - os.fsdecode, - [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], - ) - ) - env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) - - env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) - env["WASI_SYSROOT"] = os.fsdecode(sysroot) - - env["PATH"] = os.pathsep.join([ - os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"], - ]) - - return env - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) -def configure_wasi_python(context, working_dir): - """Configure the WASI/host build.""" - if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError( - "WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk" - ) - - config_site = os.fsdecode( - CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi" - ) - - wasi_build_dir = working_dir.relative_to(CHECKOUT) - - python_build_dir = BUILD_DIR / "build" - lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, ( - f"Expected a single lib.* directory in {python_build_dir}" - ) - lib_dir = os.fsdecode(lib_dirs[0]) - python_version = lib_dir.rpartition("-")[-1] - sysconfig_data_dir = ( - f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" - ) - - # Use PYTHONPATH to include sysconfig data which must be anchored to the - # WASI guest's `/` directory. - args = { - "GUEST_DIR": "/", - "HOST_DIR": CHECKOUT, - "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data_dir}", - "PYTHON_WASM": working_dir / "python.wasm", - } - # Check dynamically for wasmtime in case it was specified manually via - # `--host-runner`. - if WASMTIME_HOST_RUNNER_VAR in context.host_runner: - if wasmtime := shutil.which("wasmtime"): - args[WASMTIME_VAR_NAME] = wasmtime - else: - raise FileNotFoundError( - "wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime" - ) - host_runner = context.host_runner.format_map(args) - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} - build_python = os.fsdecode(build_python_path()) - # The path to `configure` MUST be relative, else `python.wasm` is unable - # to find the stdlib due to Python not recognizing that it's being - # executed from within a checkout. - configure = [ - os.path.relpath(CHECKOUT / "configure", working_dir), - f"--host={context.host_triple}", - f"--build={BUILD_DIR.name}", - f"--with-build-python={build_python}", - ] - if build_python_is_pydebug(): - configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) - call( - configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - context=context, - ) - - python_wasm = working_dir / "python.wasm" - exec_script = working_dir / "python.sh" - with exec_script.open("w", encoding="utf-8") as file: - file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') - exec_script.chmod(0o755) - print(f"๐Ÿƒโ€โ™€๏ธ Created {exec_script} (--host-runner)... ") - sys.stdout.flush() - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) -def make_wasi_python(context, working_dir): - """Run `make` for the WASI/host build.""" - call( - ["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - context=context, - ) - - exec_script = working_dir / "python.sh" - call([exec_script, "--version"], quiet=False) - print( - f"๐ŸŽ‰ Use `{exec_script.relative_to(context.init_dir)}` " - "to run CPython w/ the WASI host specified by --host-runner" - ) - - -def build_all(context): - """Build everything.""" - steps = [ - configure_build_python, - make_build_python, - configure_wasi_python, - make_wasi_python, - ] - for step in steps: - step(context) - - -def clean_contents(context): - """Delete all files created by this script.""" - if CROSS_BUILD_DIR.exists(): - print(f"๐Ÿงน Deleting {CROSS_BUILD_DIR} ...") - shutil.rmtree(CROSS_BUILD_DIR) - - if LOCAL_SETUP.exists(): - if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: - print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...") - - -def main(): - default_host_triple = "wasm32-wasip1" - default_wasi_sdk = find_wasi_sdk() - default_host_runner = ( - f"{WASMTIME_HOST_RUNNER_VAR} run " - # Make sure the stack size will work for a pydebug - # build. - # Use 16 MiB stack. - "--wasm max-wasm-stack=16777216 " - # Enable thread support; causes use of preview1. - # "--wasm threads=y --wasi threads=y " - # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}" - ) - default_logdir = pathlib.Path(tempfile.gettempdir()) - - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser( - "configure-build-python", help="Run `configure` for the build Python" - ) - make_build = subcommands.add_parser( - "make-build-python", help="Run `make` for the build Python" - ) - configure_host = subcommands.add_parser( - "configure-host", - help="Run `configure` for the " - "host/WASI (pydebug builds " - "are inferred from the build " - "Python)", - ) - make_host = subcommands.add_parser( - "make-host", help="Run `make` for the host/WASI" - ) - subcommands.add_parser( - "clean", help="Delete files and directories created by this script" - ) - for subcommand in ( - build, - configure_build, - make_build, - configure_host, - make_host, - ): - subcommand.add_argument( - "--quiet", - action="store_true", - default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file", - ) - subcommand.add_argument( - "--logdir", - type=pathlib.Path, - default=default_logdir, - help=f"Directory to store log files; defaults to {default_logdir}", - ) - for subcommand in configure_build, configure_host: - subcommand.add_argument( - "--clean", - action="store_true", - default=False, - dest="clean", - help="Delete any relevant directories before building", - ) - for subcommand in build, configure_build, configure_host: - subcommand.add_argument( - "args", nargs="*", help="Extra arguments to pass to `configure`" - ) - for subcommand in build, configure_host: - subcommand.add_argument( - "--wasi-sdk", - type=pathlib.Path, - dest="wasi_sdk_path", - default=default_wasi_sdk, - help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", - ) - subcommand.add_argument( - "--host-runner", - action="store", - default=default_host_runner, - dest="host_runner", - help="Command template for running the WASI host; defaults to " - f"`{default_host_runner}`", - ) - for subcommand in build, configure_host, make_host: - subcommand.add_argument( - "--host-triple", - action="store", - default=default_host_triple, - help="The target triple for the WASI host build; " - f"defaults to {default_host_triple}", - ) - - context = parser.parse_args() - context.init_dir = pathlib.Path().absolute() - - dispatch = { - "configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build": build_all, - "clean": clean_contents, - } - dispatch[context.subcommand](context) - - -if __name__ == "__main__": - main()