diff --git a/interpreters/python/Kconfig b/interpreters/python/Kconfig index dfc3f5ee809..dfc5fd4cc00 100644 --- a/interpreters/python/Kconfig +++ b/interpreters/python/Kconfig @@ -37,4 +37,21 @@ config INTERPRETERS_CPYTHON_PROGNAME ---help--- This is the name of the program that will be used from the nsh. +config INTERPRETERS_CPYTHON_ENABLE_PIP + bool "Enable bundled pip" + default n + ---help--- + Enable bundling pip into the CPython module image. When enabled, the + build downloads the pip wheel and pre-installs it through a + site-packages .pth entry that points to ensurepip's bundled wheel. + Disable this to skip pip wheel download/integration entirely. + + +config INTERPRETERS_CPYTHON_PYTHONPATH + string "CPython Python path" + default "/tmp" + ---help--- + This is the Python default search path for modules files. This is + required to be a writable path. + endif diff --git a/interpreters/python/Make.defs b/interpreters/python/Make.defs index c02973e5a67..dceeb526726 100644 --- a/interpreters/python/Make.defs +++ b/interpreters/python/Make.defs @@ -27,6 +27,8 @@ CPYTHON_VERSION_MINOR=$(basename $(CPYTHON_VERSION)) EXTRA_LIBPATHS += -L$(APPDIR)/interpreters/python/install/target EXTRA_LIBS += -lpython$(CPYTHON_VERSION_MINOR) +EXTRA_LIBS += $(APPDIR)/interpreters/python/build/target/Modules/_hacl/libHacl_Hash_SHA2.a +EXTRA_LIBS += $(APPDIR)/interpreters/python/build/target/Modules/expat/libexpat.a CONFIGURED_APPS += $(APPDIR)/interpreters/python endif diff --git a/interpreters/python/Makefile b/interpreters/python/Makefile index 954f07b3075..a339bbe0256 100644 --- a/interpreters/python/Makefile +++ b/interpreters/python/Makefile @@ -84,6 +84,10 @@ $(CPYTHON_UNPACKNAME): $(CPYTHON_ZIP) $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0013-transform-functions-used-by-NuttX-to-lowercase.patch $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0014-insert-prefix-to-list_length-to-avoid-symbol-collisi.patch + $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0015-keep-ensurepip-in-stdlib-archive.patch + $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0016-fix-timezone-offset-check-when-time-t-is-unsigned.patch + $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0017-stdlib-zip-keep-pydecimal-and-trim-tooling-extras.patch + $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0018-ignore-chmod-on-nuttx-like-wasi.patch $(HOSTPYTHON): mkdir -p $(HOSTBUILD) @@ -92,6 +96,7 @@ $(HOSTPYTHON): cd $(HOSTBUILD) && $(CPYTHON_PATH)/configure \ --with-pydebug \ --prefix=$(HOSTINSTALL) \ + --disable-test-modules \ ) $(MAKE) -C $(HOSTBUILD) install @@ -152,7 +157,7 @@ $(TARGETBUILD)/Makefile: $(HOSTPYTHON) $(CONFIG_SITE) $(SETUP_LOCAL) AR="$(AR)" \ ARFLAGS=" " \ MACHDEP="$(MACHDEP)" \ - OPT="-g -O0 -Wall" \ + OPT="-O3" \ CONFIG_SITE="$(CONFIG_SITE)" \ $(CPYTHON_PATH)/configure \ --prefix=${TARGETINSTALL} \ @@ -163,13 +168,44 @@ $(TARGETBUILD)/Makefile: $(HOSTPYTHON) $(CONFIG_SITE) $(SETUP_LOCAL) --without-mimalloc \ --without-pymalloc \ --disable-test-modules \ + --with-ensurepip=no \ ) + $(Q) sed -i 's/^#define HAVE_LIBB2 1/\/* #undef HAVE_LIBB2 *\//' $(TARGETBUILD)/pyconfig.h + $(Q) sed -i 's/-lb2//g' $(TARGETBUILD)/Makefile $(TARGETLIBPYTHON): $(TARGETBUILD)/Makefile +ifeq ($(CONFIG_INTERPRETERS_CPYTHON_ENABLE_PIP),y) + $(Q) mkdir -p $(CPYTHON_PATH)/Lib/ensurepip/_bundled + $(Q) ( \ + PIP_WHEEL_VERSION=$$($(HOSTPYTHON) -c "import ensurepip; print(ensurepip._PIP_VERSION)"); \ + PIP_WHEEL=$(CPYTHON_PATH)/Lib/ensurepip/_bundled/pip-$${PIP_WHEEL_VERSION}-py3-none-any.whl; \ + if [ ! -f "$${PIP_WHEEL}" ]; then \ + echo "Fetching pip wheel $${PIP_WHEEL_VERSION} for ensurepip bundle"; \ + $(HOSTPYTHON) -m pip download --only-binary=:all: --no-deps --dest $(CPYTHON_PATH)/Lib/ensurepip/_bundled pip==$${PIP_WHEEL_VERSION}; \ + fi; \ + echo "Pre-compiling pip wheel with build Python (must match embedded CPython version)"; \ + $(HOSTPYTHON) $(CURDIR)/repack_wheel_add_pyc.py "$${PIP_WHEEL}"; \ + ) +endif $(MAKE) -C $(TARGETBUILD) regen-frozen $(MAKE) -C $(TARGETBUILD) libpython$(CPYTHON_VERSION_MINOR).a wasm_stdlib $(Q) ( cp $(TARGETBUILD)/libpython$(CPYTHON_VERSION_MINOR).a $(TARGETLIBPYTHON) ) $(Q) $(UNPACK) $(TARGETMODULESPACK) -d $(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR) +ifeq ($(CONFIG_INTERPRETERS_CPYTHON_ENABLE_PIP),y) + $(Q) mkdir -p $(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR)/site-packages + $(Q) ( \ + set -e; \ + BUNDLED_DIR=$(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR)/ensurepip/_bundled; \ + SITE_PACKAGES=$(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR)/site-packages; \ + : > "$${SITE_PACKAGES}/bundled_wheels.pth"; \ + for wheel in $${BUNDLED_DIR}/*.whl; do \ + [ -f "$${wheel}" ] || continue; \ + whl_name=$$(basename "$${wheel}"); \ + echo "Pre-installing wheel via zipimport into target sys.path: $${whl_name}"; \ + echo "../ensurepip/_bundled/$${whl_name}" >> "$${SITE_PACKAGES}/bundled_wheels.pth"; \ + done; \ + ) +endif MODULE = $(CONFIG_INTERPRETERS_CPYTHON) diff --git a/interpreters/python/Setup.local.in b/interpreters/python/Setup.local.in index 5ff7cd3fa48..0bd116574ec 100644 --- a/interpreters/python/Setup.local.in +++ b/interpreters/python/Setup.local.in @@ -4,7 +4,6 @@ *disabled* _asyncio -_blake2 _bz2 _codecs_cn _codecs_hk @@ -15,19 +14,12 @@ _codecs_tw _ctypes _decimal _elementtree -_hashlib _heapq _interpchannels _interpqueues _lsprof _lzma -_md5 _multibytecodec -_sha1 -_sha2 -_sha2 -_sha3 -_sha3 _sqlite3 _ssl _statistics @@ -41,9 +33,7 @@ _testlimitedcapi _uuid _xxtestfuzz _zoneinfo -mmap pwd -pyexpat readline resource xxsubtype diff --git a/interpreters/python/config.site.in b/interpreters/python/config.site.in index eb37e5a88c9..52c04725ae9 100644 --- a/interpreters/python/config.site.in +++ b/interpreters/python/config.site.in @@ -21,4 +21,13 @@ export ac_cv_func_pipe="yes" export ac_cv_enable_strict_prototypes_warning="no" export ac_cv_func_getnameinfo="yes" export ac_cv_func_poll="yes" -export ac_cv_func_gethostname="yes" \ No newline at end of file +export ac_cv_func_gethostname="yes" +export ac_cv_func_lstat="yes" +export ac_cv_func_readlink="yes" +export ac_cv_func_realpath="yes" +export ac_cv_func_getpid="yes" +export ac_cv_func_utime="yes" +export ac_cv_func_utimes="yes" +export ac_cv_func_getuid="yes" +export ac_cv_func_sysconf="yes" +export ac_cv_func_umask="yes" diff --git a/interpreters/python/patch/0015-keep-ensurepip-in-stdlib-archive.patch b/interpreters/python/patch/0015-keep-ensurepip-in-stdlib-archive.patch new file mode 100644 index 00000000000..a74e892ef4c --- /dev/null +++ b/interpreters/python/patch/0015-keep-ensurepip-in-stdlib-archive.patch @@ -0,0 +1,25 @@ +--- a/Tools/wasm/wasm_assets.py ++++ b/Tools/wasm/wasm_assets.py +@@ -40,7 +40,6 @@ OMIT_FILES = ( + # regression tests + "test/", + # package management +- "ensurepip/", + "venv/", + # other platforms + "_aix_support.py", +@@ -148,6 +147,13 @@ def create_stdlib_zip( + if entry.name.endswith(".py") or entry.is_dir(): + # writepy() writes .pyc files (bytecode). + pzf.writepy(entry, filterfunc=filterfunc) ++ ++ # Preserve ensurepip wheel payloads so `python -m ensurepip` can ++ # bootstrap pip on targets that consume this stdlib zip archive. ++ bundled_wheels = args.srcdir_lib / "ensurepip" / "_bundled" ++ if bundled_wheels.is_dir(): ++ for wheel in sorted(bundled_wheels.glob("*.whl")): ++ pzf.write(wheel, arcname=f"ensurepip/_bundled/{wheel.name}") + + + def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: + diff --git a/interpreters/python/patch/0016-fix-timezone-offset-check-when-time-t-is-unsigned.patch b/interpreters/python/patch/0016-fix-timezone-offset-check-when-time-t-is-unsigned.patch new file mode 100644 index 00000000000..81924297bed --- /dev/null +++ b/interpreters/python/patch/0016-fix-timezone-offset-check-when-time-t-is-unsigned.patch @@ -0,0 +1,21 @@ +--- a/Modules/timemodule.c ++++ b/Modules/timemodule.c +@@ -1800,15 +1800,15 @@ static int + static const time_t YEAR = (365 * 24 + 6) * 3600; + time_t t; + struct tm p; +- time_t janzone_t, julyzone_t; ++ long long janzone_t, julyzone_t; + char janname[10], julyname[10]; + t = (time((time_t *)0) / YEAR) * YEAR; + _PyTime_localtime(t, &p); + get_zone(janname, 9, &p); +- janzone_t = -get_gmtoff(t, &p); ++ janzone_t = -(long long)get_gmtoff(t, &p); + janname[9] = '\0'; + t += YEAR/2; + _PyTime_localtime(t, &p); + get_zone(julyname, 9, &p); +- julyzone_t = -get_gmtoff(t, &p); ++ julyzone_t = -(long long)get_gmtoff(t, &p); + julyname[9] = '\0'; diff --git a/interpreters/python/patch/0017-stdlib-zip-keep-pydecimal-and-trim-tooling-extras.patch b/interpreters/python/patch/0017-stdlib-zip-keep-pydecimal-and-trim-tooling-extras.patch new file mode 100644 index 00000000000..333bb983ad5 --- /dev/null +++ b/interpreters/python/patch/0017-stdlib-zip-keep-pydecimal-and-trim-tooling-extras.patch @@ -0,0 +1,35 @@ +--- a/Tools/wasm/wasm_assets.py ++++ b/Tools/wasm/wasm_assets.py +@@ -47,13 +47,20 @@ + # webbrowser + "antigravity.py", + "webbrowser.py", +- # Pure Python implementations of C extensions +- "_pydecimal.py", ++ # Pure Python implementations of C extensions. ++ # NOTE: keep "_pydecimal.py" so decimal.py can fall back to it when the ++ # _decimal C extension is not built (NuttX targets do not link libmpdec). + "_pyio.py", + # concurrent threading + "concurrent/futures/thread.py", + # Misc unused or large files + "pydoc_data/", ++ # Tooling/REPL extras not needed on a constrained embedded target. ++ "unittest/", ++ "_pyrepl/", ++ "idlelib/", ++ "turtledemo/", ++ "wsgiref/", + ) + + # Synchronous network I/O and protocols are not supported; for example, +@@ -80,7 +87,8 @@ + "_asyncio": ["asyncio/"], + "_curses": ["curses/"], + "_ctypes": ["ctypes/"], +- "_decimal": ["decimal.py"], ++ # decimal.py is intentionally NOT omitted here: it ships a pure-Python ++ # fallback (_pydecimal) used when the _decimal C ext is unavailable. + "_dbm": ["dbm/ndbm.py"], + "_gdbm": ["dbm/gnu.py"], + "_json": ["json/"], diff --git a/interpreters/python/patch/0018-ignore-chmod-on-nuttx-like-wasi.patch b/interpreters/python/patch/0018-ignore-chmod-on-nuttx-like-wasi.patch new file mode 100644 index 00000000000..4e2afb41030 --- /dev/null +++ b/interpreters/python/patch/0018-ignore-chmod-on-nuttx-like-wasi.patch @@ -0,0 +1,25 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Tiago Medicci +Date: Thu, 7 May 2026 14:00:00 -0300 +Subject: [PATCH] posixmodule: ignore chmod on NuttX like WASI + +NuttX's tmpfs does not implement chstat, so chmod fails with ENOSYS. +Apply the same workaround already used for WASI: silently succeed +when HAVE_CHMOD is not defined. + +--- + Modules/posixmodule.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -3608,8 +3608,8 @@ + #ifdef HAVE_CHMOD + result = chmod(path->narrow, mode); +-#elif defined(__wasi__) +- // WASI SDK 15.0 does not support chmod. ++#elif defined(__wasi__) || defined(__NuttX__) ++ // WASI SDK 15.0 and NuttX do not fully support chmod. + // Ignore missing syscall for now. + result = 0; + #else diff --git a/interpreters/python/python_wrapper.c b/interpreters/python/python_wrapper.c index 6617bae98d4..be7454514df 100644 --- a/interpreters/python/python_wrapper.c +++ b/interpreters/python/python_wrapper.c @@ -198,5 +198,7 @@ int main(int argc, FAR char *argv[]) setenv("PYTHON_BASIC_REPL", "1", 1); + setenv("PYTHONPATH", CONFIG_INTERPRETERS_CPYTHON_PYTHONPATH, 1); + return py_bytesmain(argc, argv); } diff --git a/interpreters/python/repack_wheel_add_pyc.py b/interpreters/python/repack_wheel_add_pyc.py new file mode 100644 index 00000000000..fc2d913f7ed --- /dev/null +++ b/interpreters/python/repack_wheel_add_pyc.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# +# Repack a wheel in-place: compile pip/*.py to legacy sibling *.pyc (compileall +# -b: required for zipimport, which does not read PEP 3147 __pycache__/ names), +# remove the .py sources, and rewrite *.dist-info/RECORD. + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import shutil +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + + +def wheel_record_hash(data: bytes) -> str: + digest = hashlib.sha256(data).digest() + return base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + + +def wheel_has_pip_py_sources(zf: zipfile.ZipFile) -> bool: + return any(n.startswith("pip/") and n.endswith(".py") for n in zf.namelist()) + + +def wheel_has_legacy_pip_bytecode(zf: zipfile.ZipFile) -> bool: + return "pip/__init__.pyc" in zf.namelist() + + +def strip_pip_py_sources(pip_dir: Path) -> int: + """Remove pip/**/*.py after sibling legacy *.pyc exists (compileall -b output).""" + removed = 0 + for path in sorted(pip_dir.rglob("*.py")): + if not path.is_file(): + continue + pyc = path.with_suffix(".pyc") + if not pyc.is_file(): + rel = path.relative_to(pip_dir) + raise SystemExit( + f"missing legacy .pyc for pip/{rel.as_posix()}, refusing to delete source" + ) + path.unlink() + removed += 1 + return removed + + +def rebuild_record(root: Path) -> None: + dist_infos = sorted(root.glob("*.dist-info")) + if len(dist_infos) != 1: + raise SystemExit( + f"expected one *.dist-info, got {[p.name for p in dist_infos]}" + ) + di = dist_infos[0] + record_path = di / "RECORD" + record_rel = f"{di.name}/RECORD" + lines: list[str] = [] + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(root).as_posix() + if rel == record_rel: + continue + body = path.read_bytes() + lines.append(f"{rel},sha256={wheel_record_hash(body)},{len(body)}") + lines.append(f"{record_rel},,") + record_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def repack(whl_path: Path, *, force: bool) -> None: + whl_path = whl_path.resolve() + if not whl_path.is_file(): + raise SystemExit(f"missing wheel: {whl_path}") + + with zipfile.ZipFile(whl_path) as zf: + has_py = wheel_has_pip_py_sources(zf) + if not has_py: + if not wheel_has_legacy_pip_bytecode(zf): + raise SystemExit( + "repack_wheel_add_pyc: wheel has no pip/*.py and no pip/__init__.pyc " + "(corrupt or old tool output). Delete the bundled pip-*.whl and rebuild." + ) + if not force: + print( + f"repack_wheel_add_pyc: skip (pip already bytecode-only): {whl_path.name}" + ) + return + + tmpdir = tempfile.mkdtemp(prefix="pip-whl-pyc-") + try: + root = Path(tmpdir) + with zipfile.ZipFile(whl_path) as zf: + zf.extractall(root) + + pip_dir = root / "pip" + if not pip_dir.is_dir(): + raise SystemExit("wheel has no pip/ top-level package") + + subprocess.run( + [sys.executable, "-m", "compileall", "-q", "-f", "-b", str(pip_dir)], + cwd=str(root), + check=True, + ) + n_py = strip_pip_py_sources(pip_dir) + rebuild_record(root) + + out_path = whl_path.with_suffix(whl_path.suffix + ".tmp") + with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as out: + for path in sorted(root.rglob("*")): + if path.is_file(): + arcname = path.relative_to(root).as_posix() + out.write(path, arcname) + + out_path.replace(whl_path) + print( + f"repack_wheel_add_pyc: bytecode-only pip ({n_py} .py removed) -> {whl_path.name}" + ) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("wheel", type=Path, help="path to .whl (updated in place)") + ap.add_argument( + "-f", + "--force", + action="store_true", + help="repack even if pip is already .pyc-only", + ) + args = ap.parse_args() + repack(args.wheel, force=args.force) + + +if __name__ == "__main__": + main()