Finish out of tree build system (except xbuildenv deploy)

pyodide-build/pyodide_build/common.py

@@ -229,11 +229,14 @@ def init_environment() -> None:
         os.environ["PYODIDE_ROOT"] = str(search_pyodide_root(os.getcwd()))
 
     os.environ.update(get_make_environment_vars())
-    hostsitepackages = get_hostsitepackages()
-    pythonpath = [
-        hostsitepackages,
-    ]
-    os.environ["PYTHONPATH"] = ":".join(pythonpath)
+    try:
+        hostsitepackages = get_hostsitepackages()
+        pythonpath = [
+            hostsitepackages,
+        ]
+        os.environ["PYTHONPATH"] = ":".join(pythonpath)
+    except KeyError:
+        pass
     os.environ["BASH_ENV"] = ""
     get_unisolated_packages()
 
@@ -251,13 +254,18 @@ def get_unisolated_packages() -> list[str]:
     if "UNISOLATED_PACKAGES" in os.environ:
         return json.loads(os.environ["UNISOLATED_PACKAGES"])
     PYODIDE_ROOT = get_pyodide_root()
-    unisolated_packages = []
-    for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"):
-        config = parse_package_config(pkg, check=False)
-        if config.get("build", {}).get("cross-build-env", False):
-            unisolated_packages.append(config["package"]["name"])
-    # TODO: remove setuptools_rust from this when they release the next version.
-    unisolated_packages.append("setuptools_rust")
+    unisolated_file = PYODIDE_ROOT / "unisolated.txt"
+    if unisolated_file.exists():
+        # in xbuild env, read from file
+        unisolated_packages = unisolated_file.read_text().splitlines()
+    else:
+        unisolated_packages = []
+        for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"):
+            config = parse_package_config(pkg, check=False)
+            if config.get("build", {}).get("cross-build-env", False):
+                unisolated_packages.append(config["package"]["name"])
+        # TODO: remove setuptools_rust from this when they release the next version.
+        unisolated_packages.append("setuptools_rust")
     os.environ["UNISOLATED_PACKAGES"] = json.dumps(unisolated_packages)
     return unisolated_packages
 

pyodide-build/pyodide_build/create_xbuildenv.py

@@ -3,7 +3,7 @@
 import subprocess
 from pathlib import Path
 
-from .common import get_make_flag, get_pyodide_root
+from .common import get_make_flag, get_pyodide_root, get_unisolated_packages
 from .io import parse_package_config
 
 
@@ -44,7 +44,7 @@ def copy_wasm_libs(xbuildenv_path: Path) -> None:
     wasm_lib_dir = get_relative_path(pyodide_root, "WASM_LIBRARY_DIR")
     sysconfig_dir = get_relative_path(pyodide_root, "SYSCONFIGDATA_DIR")
     xbuildenv_root = xbuildenv_path / "pyodide-root"
-    xbuildenv_path.mkdir()
+    xbuildenv_path.mkdir(exist_ok=True)
     to_copy: list[Path] = [
         pythoninclude,
         sysconfig_dir,
@@ -89,3 +89,6 @@ def main(args: argparse.Namespace) -> None:
         stdout=subprocess.PIPE,
     )
     (xbuildenv_path / "requirements.txt").write_bytes(res.stdout)
+    (xbuildenv_path / "pyodide-root/unisolated.txt").write_text(
+        "\n".join(get_unisolated_packages())
+    )

pyodide-build/pyodide_build/install_xbuildenv.py

@@ -15,12 +15,27 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
         "on numpy or scipy.\n"
         "Note: this is a private endpoint that should not be used outside of the Pyodide Makefile."
     )
+    parser.add_argument("--download", action="store_true", help="Download xbuild env")
     parser.add_argument("xbuild_env", type=str, nargs=1)
     return parser
 
 
-def main(args: argparse.Namespace) -> None:
-    xbuildenv_path = Path(args.xbuild_env[0])
+def download_xbuild_env(version: str, xbuildenv_path: Path) -> None:
+    from shutil import rmtree, unpack_archive
+    from tempfile import NamedTemporaryFile
+    from urllib.request import urlretrieve
+
+    rmtree(xbuildenv_path, ignore_errors=True)
+    with NamedTemporaryFile(suffix=".tar") as f:
+        urlretrieve(
+            f"http://pyodide-cache.s3-website-us-east-1.amazonaws.com/xbuildenv/{version}.tar",
+            f.name,
+        )
+        unpack_archive(f.name, xbuildenv_path)
+
+
+def install_xbuild_env(xbuildenv_path: Path) -> None:
+    xbuildenv_path = xbuildenv_path / "xbuildenv"
     pyodide_root = get_pyodide_root()
     xbuildenv_root = xbuildenv_path / "pyodide-root"
     host_site_packages = xbuildenv_root / Path(
@@ -42,3 +57,11 @@ def main(args: argparse.Namespace) -> None:
     shutil.copytree(
         xbuildenv_path / "site-packages-extras", host_site_packages, dirs_exist_ok=True
     )
+
+
+def main(args: argparse.Namespace) -> None:
+    xbuildenv_path = Path(args.xbuild_env[0])
+    version = "2"
+    if args.download:
+        download_xbuild_env(version, xbuildenv_path)
+    install_xbuild_env(xbuildenv_path)

pyodide-build/pyodide_build/out_of_tree/__main__.py

@@ -0,0 +1,36 @@
+import argparse
+import os
+from pathlib import Path
+
+from . import build
+
+
+def ensure_env_installed(env: Path) -> None:
+    if env.exists():
+        return
+    from ..install_xbuildenv import download_xbuild_env, install_xbuild_env
+
+    version = "2"
+    download_xbuild_env(version, env)
+    install_xbuild_env(env)
+
+
+def main():
+    main_parser = argparse.ArgumentParser(prog="pywasmbuild")
+    main_parser.description = "Tools for creating Python extension modules for the wasm32-unknown-emscripten platform"
+    subparsers = main_parser.add_subparsers(help="action")
+    for module in [build]:
+        modname = module.__name__.rpartition(".")[-1]
+        parser = module.make_parser(subparsers.add_parser(modname))
+        parser.set_defaults(func=module.main)
+
+    env = Path(".pyodide-xbuildenv")
+    os.environ["PYODIDE_ROOT"] = str(env / "xbuildenv/pyodide-root")
+    ensure_env_installed(env)
+
+    args = main_parser.parse_args()
+    if hasattr(args, "func"):
+        # run the selected action
+        args.func(args)
+    else:
+        main_parser.print_help()

pyodide-build/pyodide_build/out_of_tree/build.py

@@ -0,0 +1,47 @@
+import argparse
+import os
+
+from .. import common, pypabuild, pywasmcross
+
+
+def run(exports, args):
+    cflags = common.get_make_flag("SIDE_MODULE_CFLAGS")
+    cflags += f" {os.environ.get('CFLAGS', '')}"
+    cxxflags = common.get_make_flag("SIDE_MODULE_CXXFLAGS")
+    cxxflags += f" {os.environ.get('CXXFLAGS', '')}"
+    ldflags = common.get_make_flag("SIDE_MODULE_LDFLAGS")
+    ldflags += f" {os.environ.get('LDFLAGS', '')}"
+    build_env_ctx = pywasmcross.get_build_env(
+        env=os.environ.copy(),
+        pkgname="",
+        cflags=cflags,
+        cxxflags=cxxflags,
+        ldflags=ldflags,
+        target_install_dir="",
+        exports=exports,
+    )
+
+    with build_env_ctx as env:
+        pypabuild.build(env, " ".join(args))
+
+
+def main(parser_args: argparse.Namespace) -> None:
+    run(parser_args.exports, parser_args.backend_args)
+
+
+def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
+    parser.description = "Use pypa/build to build a Python package."
+    parser.add_argument(
+        "--exports",
+        choices=["pyinit", "requested", "whole_archive"],
+        default="requested",
+        help="Which symbols should be exported when linking .so files?",
+    )
+    parser.add_argument(
+        "backend_args",
+        metavar="args",
+        type=str,
+        nargs=argparse.REMAINDER,
+        help="Arguments to pass on to the backend",
+    )
+    return parser

pyodide-build/setup.py

@@ -5,6 +5,7 @@
     setuptools.setup(
         entry_points={
             "console_scripts": [
+                "pywasm = pyodide_build.out_of_tree.__main__:main",
                 "_pywasmcross = pyodide_build.pywasmcross:compiler_main",
             ]
         }