diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e35b0880a4370e..977b4bd6b33a8f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,10 @@ repos: name: Run Ruff (format) on Doc/ args: [--check] files: ^Doc/ + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--check, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror rev: 25.1.0 diff --git a/Tools/wasm/.ruff.toml b/Tools/wasm/.ruff.toml new file mode 100644 index 00000000000000..aabcf8dc4f502e --- /dev/null +++ b/Tools/wasm/.ruff.toml @@ -0,0 +1,28 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + "E501", # Line too long + "F541", # f-string without any placeholders + "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` + "PYI025", # Use `from collections.abc import Set as AbstractSet` +] diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index c8f48a26f2a8be..34051bd7351a37 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -4,6 +4,7 @@ import contextlib import functools import os + try: from os import process_cpu_count as cpu_count except ImportError: @@ -37,7 +38,9 @@ def updated_env(updates={}): # https://reproducible-builds.org/docs/source-date-epoch/ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() env_defaults["SOURCE_DATE_EPOCH"] = epoch except subprocess.CalledProcessError: pass # Might be building from a tarball. @@ -58,6 +61,7 @@ def updated_env(updates={}): def subdir(working_dir, *, clean_ok=False): """Decorator to change to a working directory.""" + def decorator(func): @functools.wraps(func) def wrapper(context): @@ -66,16 +70,20 @@ def wrapper(context): if callable(working_dir): working_dir = working_dir(context) try: - tput_output = subprocess.check_output(["tput", "cols"], - encoding="utf-8") + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) except subprocess.CalledProcessError: terminal_width = 80 else: terminal_width = int(tput_output.strip()) print("⎯" * terminal_width) print("📁", working_dir) - if (clean_ok and getattr(context, "clean", False) and - working_dir.exists()): + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): print(f"🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) @@ -99,10 +107,13 @@ def call(command, *, quiet, **kwargs): stdout = None stderr = None else: - stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", - delete=False, - prefix="cpython-wasi-", - suffix=".log") + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + prefix="cpython-wasi-", + suffix=".log", + ) stderr = subprocess.STDOUT print(f"📝 Logging output to {stdout.name} (--quiet)...") @@ -121,8 +132,9 @@ def build_python_path(): if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{BUILD_DIR}") + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) return binary @@ -136,7 +148,7 @@ def configure_build_python(context, working_dir): print(f"📝 Touching {LOCAL_SETUP} ...") LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] if context.args: configure.extend(context.args) @@ -146,13 +158,15 @@ def configure_build_python(context, working_dir): @subdir(BUILD_DIR) def make_build_python(context, working_dir): """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], - quiet=context.quiet) + call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) binary = build_python_path() - cmd = [binary, "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] version = subprocess.check_output(cmd, encoding="utf-8").strip() print(f"🎉 {binary} {version}") @@ -170,8 +184,13 @@ def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", - "AR": "llvm-ar", "RANLIB": "ranlib"} + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } for env_var, binary_name in list(env.items()): env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) @@ -182,16 +201,20 @@ def wasi_sdk_env(context): env["PKG_CONFIG_PATH"] = "" env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map(os.fsdecode, - [sysroot / "lib" / "pkgconfig", - sysroot / "share" / "pkgconfig"])) + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) env["WASI_SYSROOT"] = os.fsdecode(sysroot) - env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"]]) + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) return env @@ -200,18 +223,24 @@ def wasi_sdk_env(context): def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk") + raise ValueError( + "WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) - config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") + config_site = os.fsdecode( + CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi" + ) wasi_build_dir = working_dir.relative_to(CHECKOUT) python_build_dir = BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) lib_dir = os.fsdecode(lib_dirs[0]) pydebug = lib_dir.endswith("-pydebug") python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] @@ -221,36 +250,44 @@ def configure_wasi_python(context, working_dir): # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. - args = {"GUEST_DIR": "/", - "HOST_DIR": CHECKOUT, - "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data}", - "PYTHON_WASM": working_dir / "python.wasm"} + args = { + "GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data}", + "PYTHON_WASM": working_dir / "python.wasm", + } # Check dynamically for wasmtime in case it was specified manually via # `--host-runner`. if WASMTIME_HOST_RUNNER_VAR in context.host_runner: if wasmtime := shutil.which("wasmtime"): args[WASMTIME_VAR_NAME] = wasmtime else: - raise FileNotFoundError("wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime") + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime" + ) host_runner = context.host_runner.format_map(args) env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} build_python = os.fsdecode(build_python_path()) # The path to `configure` MUST be relative, else `python.wasm` is unable # to find the stdlib due to Python not recognizing that it's being # executed from within a checkout. - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={context.host_triple}", - f"--build={build_platform()}", - f"--with-build-python={build_python}"] + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={build_platform()}", + f"--with-build-python={build_python}", + ] if pydebug: configure.append("--with-pydebug") if context.args: configure.extend(context.args) - call(configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - quiet=context.quiet) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + quiet=context.quiet, + ) python_wasm = working_dir / "python.wasm" exec_script = working_dir / "python.sh" @@ -264,9 +301,11 @@ def configure_wasi_python(context, working_dir): @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" - call(["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - quiet=context.quiet) + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + quiet=context.quiet, + ) exec_script = working_dir / "python.sh" subprocess.check_call([exec_script, "--version"]) @@ -274,11 +313,16 @@ def make_wasi_python(context, working_dir): def build_all(context): """Build everything.""" - steps = [configure_build_python, make_build_python, configure_wasi_python, - make_wasi_python] + steps = [ + configure_build_python, + make_build_python, + configure_wasi_python, + make_wasi_python, + ] for step in steps: step(context) + def clean_contents(context): """Delete all files created by this script.""" if CROSS_BUILD_DIR.exists(): @@ -292,74 +336,109 @@ def clean_contents(context): def main(): - default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " - # Make sure the stack size will work for a pydebug - # build. - # The 8388608 value comes from `ulimit -s` under Linux - # which equates to 8291 KiB. - "--wasm max-wasm-stack=8388608 " - # Use WASI 0.2 primitives. - "--wasi preview2 " - # Enable thread support; causes use of preview1. - #"--wasm threads=y --wasi threads=y " - # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") + default_host_runner = ( + f"{WASMTIME_HOST_RUNNER_VAR} run " + # Make sure the stack size will work for a pydebug + # build. + # The 8388608 value comes from `ulimit -s` under Linux + # which equates to 8291 KiB. + "--wasm max-wasm-stack=8388608 " + # Use WASI 0.2 primitives. + "--wasi preview2 " + # Enable thread support; causes use of preview1. + # "--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}" + ) parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build-python", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build-python", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for the " - "host/WASI (pydebug builds " - "are inferred from the build " - "Python)") - make_host = subcommands.add_parser("make-host", - help="Run `make` for the host/WASI") - clean = subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") - for subcommand in build, configure_build, make_build, configure_host, make_host: - subcommand.add_argument("--quiet", action="store_true", default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/WASI" + ) + clean = subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_build, + configure_host, + make_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) for subcommand in configure_build, configure_host: - subcommand.add_argument("--clean", action="store_true", default=False, - dest="clean", - help="Delete any relevant directories before building") + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) for subcommand in build, configure_build, configure_host: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) for subcommand in build, configure_host: - subcommand.add_argument("--wasi-sdk", type=pathlib.Path, - dest="wasi_sdk_path", - default=find_wasi_sdk(), - help="Path to wasi-sdk; defaults to " - "$WASI_SDK_PATH or /opt/wasi-sdk") - subcommand.add_argument("--host-runner", action="store", - default=default_host_runner, dest="host_runner", - help="Command template for running the WASI host " - "(default designed for wasmtime 14 or newer: " - f"`{default_host_runner}`)") + subcommand.add_argument( + "--wasi-sdk", + type=pathlib.Path, + dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk", + ) + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the WASI host " + "(default designed for wasmtime 14 or newer: " + f"`{default_host_runner}`)", + ) for subcommand in build, configure_host, make_host: - subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1", - help="The target triple for the WASI host build") + subcommand.add_argument( + "--host-triple", + action="store", + default="wasm32-wasip1", + help="The target triple for the WASI host build", + ) context = parser.parse_args() - dispatch = {"configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build": build_all, - "clean": clean_contents} + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all, + "clean": clean_contents, + } dispatch[context.subcommand](context) -if __name__ == "__main__": +if __name__ == "__main__": main() diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 47a0abb8b5feef..5cc1c06f4a2796 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -21,6 +21,7 @@ ./Tools/wasm/wasm_builder.py --clean build build """ + import argparse import enum import dataclasses @@ -67,7 +68,9 @@ # path to Emscripten SDK config file. # auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh". -EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) +EM_CONFIG = pathlib.Path( + os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten") +) EMSDK_MIN_VERSION = (3, 1, 19) EMSDK_BROKEN_VERSION = { (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338", @@ -261,8 +264,7 @@ def _check_emscripten() -> None: # git / upstream / tot-upstream installation version = version[:-4] version_tuple = cast( - Tuple[int, int, int], - tuple(int(v) for v in version.split(".")) + Tuple[int, int, int], tuple(int(v) for v in version.split(".")) ) if version_tuple < EMSDK_MIN_VERSION: raise ConditionError( @@ -518,7 +520,7 @@ def make_cmd(self) -> List[str]: def getenv(self) -> Dict[str, Any]: """Generate environ dict for platform""" env = os.environ.copy() - if hasattr(os, 'process_cpu_count'): + if hasattr(os, "process_cpu_count"): cpu_count = os.process_cpu_count() else: cpu_count = os.cpu_count() @@ -596,7 +598,9 @@ def run_py(self, *args: str) -> int: """Run Python with hostrunner""" self._check_execute() return self.run_make( - "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run" + "--eval", + f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", + "run", ) def run_browser(self, bind: str = "127.0.0.1", port: int = 8000) -> None: @@ -666,9 +670,12 @@ def build_emports(self, force: bool = False) -> None: # Pre-build libbz2, libsqlite3, libz, and some system libs. ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) # Multi-threaded sqlite3 has different suffix - embuilder_cmd.extend( - ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"] - ) + embuilder_cmd.extend([ + "build", + "bzip2", + "sqlite3-mt" if self.pthreads else "sqlite3", + "zlib", + ]) self._run_cmd(embuilder_cmd, cwd=SRCDIR) @@ -817,7 +824,9 @@ def build_emports(self, force: bool = False) -> None: # Don't list broken and experimental variants in help platforms_choices = list(p.name for p in _profiles) + ["cleanall"] -platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"] +platforms_help = list(p.name for p in _profiles if p.support_level) + [ + "cleanall" +] parser.add_argument( "platform", metavar="PLATFORM", diff --git a/Tools/wasm/wasm_webserver.py b/Tools/wasm/wasm_webserver.py index 3d1d5d42a1e8c4..fe565db4cb4db3 100755 --- a/Tools/wasm/wasm_webserver.py +++ b/Tools/wasm/wasm_webserver.py @@ -6,20 +6,24 @@ description="Start a local webserver with a Python terminal." ) parser.add_argument( - "--port", type=int, default=8000, help="port for the http server to listen on" + "--port", + type=int, + default=8000, + help="port for the http server to listen on", ) parser.add_argument( - "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" + "--bind", + type=str, + default="127.0.0.1", + help="Bind address (empty for all)", ) class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy() - extensions_map.update( - { - ".wasm": "application/wasm", - } - ) + extensions_map.update({ + ".wasm": "application/wasm", + }) def end_headers(self) -> None: self.send_my_headers() @@ -42,5 +46,6 @@ def main() -> None: bind=args.bind, ) + if __name__ == "__main__": main()