webassembly: Add proxying between Python and JavaScript objects, and change top-level interface for PyScript use

.github/workflows/biome.yml

@@ -0,0 +1,16 @@
+name: JavaScript code lint and formatting with Biome
+
+on: [push, pull_request]
+
+jobs:
+  eslint:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Biome
+      uses: biomejs/setup-biome@v2
+      with:
+        version: 1.5.3
+    - name: Run Biome
+      run: biome ci --indent-style=space --indent-width=4 tests/ ports/webassembly

ports/webassembly/Makefile

@@ -1,57 +1,153 @@
-include ../../py/mkenv.mk
+################################################################################
+# Initial setup of Makefile environment.
+
+# Select the variant to build for:
+ifdef VARIANT_DIR
+# Custom variant path - remove trailing slash and get the final component of
+# the path as the variant name.
+VARIANT ?= $(notdir $(VARIANT_DIR:/=))
+else
+# If not given on the command line, then default to standard.
+VARIANT ?= standard
+VARIANT_DIR ?= variants/$(VARIANT)
+endif
+
+ifeq ($(wildcard $(VARIANT_DIR)/.),)
+$(error Invalid VARIANT specified: $(VARIANT_DIR))
+endif
+
+# If the build directory is not given, make it reflect the variant name.
+BUILD ?= build-$(VARIANT)
 
-CROSS = 0
+include ../../py/mkenv.mk
+include $(VARIANT_DIR)/mpconfigvariant.mk
 
+# Qstr definitions (must come before including py.mk).
 QSTR_DEFS = qstrdefsport.h
 
+# Include py core make definitions.
 include $(TOP)/py/py.mk
 include $(TOP)/extmod/extmod.mk
 
+################################################################################
+# Project specific settings and compiler/linker flags.
+
 CC = emcc
 LD = emcc
+NODE ?= node
+TERSER ?= npx terser
 
 INC += -I.
 INC += -I$(TOP)
 INC += -I$(BUILD)
+INC += -I$(VARIANT_DIR)
 
 CFLAGS += -std=c99 -Wall -Werror -Wdouble-promotion -Wfloat-conversion
 CFLAGS += -Os -DNDEBUG
 CFLAGS += $(INC)
 
+EXPORTED_FUNCTIONS_EXTRA += ,\
+	_mp_js_do_exec,\
+	_mp_js_do_exec_async,\
+	_mp_js_do_import,\
+	_mp_js_register_js_module,\
+	_proxy_c_init,\
+	_proxy_c_to_js_call,\
+	_proxy_c_to_js_delete_attr,\
+	_proxy_c_to_js_dir,\
+	_proxy_c_to_js_get_array,\
+	_proxy_c_to_js_get_dict,\
+	_proxy_c_to_js_get_type,\
+	_proxy_c_to_js_has_attr,\
+	_proxy_c_to_js_lookup_attr,\
+	_proxy_c_to_js_resume,\
+	_proxy_c_to_js_store_attr,\
+	_proxy_convert_mp_to_js_obj_cside
+
+EXPORTED_RUNTIME_METHODS_EXTRA += ,\
+	PATH,\
+	PATH_FS,\
+	UTF8ToString,\
+	getValue,\
+	lengthBytesUTF8,\
+	setValue,\
+	stringToUTF8
+
+JSFLAGS += -s EXPORTED_FUNCTIONS="\
+	_free,\
+	_malloc,\
+	_mp_js_init,\
+	_mp_js_repl_init,\
+	_mp_js_repl_process_char,\
+	_mp_hal_get_interrupt_char,\
+	_mp_sched_keyboard_interrupt$(EXPORTED_FUNCTIONS_EXTRA)"
+JSFLAGS += -s EXPORTED_RUNTIME_METHODS="\
+	ccall,\
+	cwrap,\
+	FS$(EXPORTED_RUNTIME_METHODS_EXTRA)"
+JSFLAGS += --js-library library.js
+JSFLAGS += -s SUPPORT_LONGJMP=emscripten
+JSFLAGS += -s MODULARIZE -s EXPORT_NAME=_createMicroPythonModule
+
+################################################################################
+# Source files and libraries.
+
 SRC_SHARED = $(addprefix shared/,\
 	runtime/interrupt_char.c \
 	runtime/stdout_helpers.c \
 	runtime/pyexec.c \
 	readline/readline.c \
+	timeutils/timeutils.c \
 	)
 
-SRC_C = \
+SRC_C += \
+	lexer_dedent.c \
 	main.c \
+	modjs.c \
+	modjsffi.c \
 	mphalport.c \
+	objjsproxy.c \
+	proxy_c.c \
 
+# List of sources for qstr extraction.
 SRC_QSTR += $(SRC_C) $(SRC_SHARED)
 
+SRC_JS += \
+	api.js \
+	objpyproxy.js \
+	proxy_js.js \
+
 OBJ += $(PY_O)
 OBJ += $(addprefix $(BUILD)/, $(SRC_SHARED:.c=.o))
 OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o))
 
-JSFLAGS += -s ASYNCIFY
-JSFLAGS += -s EXPORTED_FUNCTIONS="['_mp_js_init', '_mp_js_init_repl', '_mp_js_do_str', '_mp_js_process_char', '_mp_hal_get_interrupt_char', '_mp_sched_keyboard_interrupt']"
-JSFLAGS += -s EXPORTED_RUNTIME_METHODS="['ccall', 'cwrap', 'FS']"
-JSFLAGS += --js-library library.js
+################################################################################
+# Main targets.
+
+.PHONY: all repl min test test_min
+
+all: $(BUILD)/micropython.mjs
+
+$(BUILD)/micropython.mjs: $(OBJ) library.js $(SRC_JS)
+	$(ECHO) "LINK $@"
+	$(Q)emcc $(LDFLAGS) -o $@ $(OBJ) $(JSFLAGS)
+	$(Q)cat $(SRC_JS) >> $@
+
+$(BUILD)/micropython.min.mjs: $(BUILD)/micropython.mjs
+	$(TERSER) $< --compress --module -o $@
+
+repl: $(BUILD)/micropython.mjs
+	$(NODE) $<
 
-all: $(BUILD)/micropython.js
+min: $(BUILD)/micropython.min.mjs
 
-$(BUILD)/micropython.js: $(OBJ) library.js wrapper.js
-	$(ECHO) "LINK $(BUILD)/firmware.js"
-	$(Q)emcc $(LDFLAGS) -o $(BUILD)/firmware.js $(OBJ) $(JSFLAGS)
-	cat wrapper.js $(BUILD)/firmware.js > $@
+test: $(BUILD)/micropython.mjs $(TOP)/tests/run-tests.py
+	cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly
 
-min: $(BUILD)/micropython.js
-	uglifyjs $< -c -o $(BUILD)/micropython.min.js
+test_min: $(BUILD)/micropython.min.mjs $(TOP)/tests/run-tests.py
+	cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly
 
-test: $(BUILD)/micropython.js $(TOP)/tests/run-tests.py
-	$(eval DIRNAME=ports/$(notdir $(CURDIR)))
-	cd $(TOP)/tests && MICROPY_MICROPYTHON=../ports/webassembly/node_run.sh ./run-tests.py -j1
+################################################################################
+# Remaining make rules.
 
 include $(TOP)/py/mkrules.mk

ports/webassembly/README.md

@@ -6,22 +6,22 @@ MicroPython for [WebAssembly](https://webassembly.org/).
 Dependencies
 ------------
 
-Building webassembly port bears the same requirements as the standard
-MicroPython ports with the addition of Emscripten (and uglify-js for the
-minified file).
+Building the webassembly port bears the same requirements as the standard
+MicroPython ports with the addition of Emscripten, and optionally terser for
+the minified file.
 
-The output includes `micropython.js` (a JavaScript wrapper for the
-MicroPython runtime) and `firmware.wasm` (actual MicroPython compiled to
+The output includes `micropython.mjs` (a JavaScript wrapper for the
+MicroPython runtime) and `micropython.wasm` (actual MicroPython compiled to
 WASM).
 
 Build instructions
 ------------------
 
-In order to build micropython.js, run:
+In order to build `micropython.mjs`, run:
 
     $ make
 
-To generate the minified file micropython.min.js, run:
+To generate the minified file `micropython.min.mjs`, run:
 
     $ make min
 
@@ -30,55 +30,90 @@ Running with Node.js
 
 Access the repl with:
 
-    $ node build/micropython.js
+    $ make repl
 
-Stack size may be modified using:
+This is the same as running:
 
-    $ node build/micropython.js -X stack=64K
+    $ node build-standard/micropython.mjs
 
-Where stack size may be represented in Bytes, KiB or MiB.
+The initial MicroPython GC heap size may be modified using:
+
+    $ node build-standard/micropython.mjs -X heapsize=64k
+
+Where stack size may be represented in bytes, or have a `k` or `m` suffix.
 
 MicroPython scripts may be executed using:
 
-    $ node build/micropython.js hello.py
+    $ node build-standard/micropython.mjs hello.py
 
-Alternatively micropython.js may by accessed by other javascript programs in node
+Alternatively `micropython.mjs` may by accessed by other JavaScript programs in node
 using the require command and the general API outlined below. For example:
 
 ```javascript
-var mp_js = require('./build/micropython.js');
+const mp_mjs = await import("micropython.mjs");
+const mp = await mp_mjs.loadMicroPython();
+
+mp.runPython("print('hello world')");
+```
 
-mp_js_init(64 * 1024);
-await mp_js_do_str("print('hello world')\n");
+Or without await notation:
+
+```javascript
+import("micropython.mjs").then((mp_mjs) => {
+    mp_mjs.loadMicroPython().then((mp) => {
+        mp.runPython("print('hello world')");
+    });
+});
 ```
 
 Running with HTML
 -----------------
 
-The prerequisite for browser operation of micropython.js is to listen to the
-`micropython-print` event, which is passed data when MicroPython code prints
-something to stdout.  The following code demonstrates basic functionality:
+The following code demonstrates the simplest way to load `micropython.mjs` in a
+browser, create an interpreter context, and run some Python code:
+
+```html
+<!doctype html>
+<html>
+  <head>
+    <script src="build-standard/micropython.mjs" type="module"></script>
+  </head>
+  <body>
+    <script type="module">
+      const mp = await loadMicroPython();
+      mp.runPython("print('hello world')");
+    </script>
+  </body>
+</html>
+```
+
+The output in the above example will go to the JavaScript console.  It's possible
+to instead capture the output and print it somewhere else, for example in an
+HTML element.  The following example shows how to do this, and also demonstrates
+the use of top-level await and the `js` module:
 
 ```html
 <!doctype html>
 <html>
   <head>
-    <script src="build/micropython.js"></script>
+    <script src="build-standard/micropython.mjs" type="module"></script>
   </head>
   <body>
     <pre id="micropython-stdout"></pre>
-    <script>
-      document.addEventListener("micropython-print", function(e) {
-        let output = document.getElementById("micropython-stdout");
-        output.innerText += new TextDecoder().decode(e.detail);
-      }, false);
-
-      var mp_js_startup = Module["onRuntimeInitialized"];
-      Module["onRuntimeInitialized"] = async function() {
-        mp_js_startup();
-        mp_js_init(64 * 1024);
-        await mp_js_do_str("print('hello world')");
+    <script type="module">
+      const stdoutWriter = (line) => {
+        document.getElementById("micropython-stdout").innerText += line + "\n";
       };
+      const mp = await loadMicroPython({stdout:stdoutWriter});
+      await mp.runPythonAsync(`
+        import js
+        url = "https://api.github.com/users/micropython"
+        print(f"fetching {url}...")
+        res = await js.fetch(url)
+        json = await res.json()
+        for i in dir(json):
+          print(f"{i}: {json[i]}")
+      `);
     </script>
   </body>
 </html>
@@ -98,31 +133,41 @@ Run the test suite using:
 API
 ---
 
-The following functions have been exposed to javascript.
+The following functions have been exposed to JavaScript through the interpreter
+context, created and returned by `loadMicroPython()`.
 
-```
-mp_js_init(stack_size)
-```
+- `PyProxy`: the type of the object that proxies Python objects.
 
-Initialize MicroPython with the given stack size in bytes. This must be
-called before attempting to interact with MicroPython.
+- `FS`: the Emscripten filesystem object.
 
-```
-await mp_js_do_str(code)
-```
+- `globals`: an object exposing the globals from the Python `__main__` module,
+  with methods `get(key)`, `set(key, value)` and `delete(key)`.
 
-Execute the input code. `code` must be a `string`.
+- `registerJsModule(name, module)`: register a JavaScript object as importable
+  from Python with the given name.
 
-```
-mp_js_init_repl()
-```
+- `pyimport`: import a Python module and return it.
 
-Initialize MicroPython repl. Must be called before entering characters into
-the repl.
+- `runPython(code)`: execute Python code and return the result.
 
-```
-await mp_js_process_char(char)
-```
+- `runPythonAsync(code)`: execute Python code and return the result, allowing for
+  top-level await expressions (this call must be await'ed on the JavaScript side).
+
+- `replInit()`: initialise the REPL.
+
+- `replProcessChar(chr)`: process an incoming character at the REPL.
+
+- `replProcessCharWithAsyncify(chr)`: process an incoming character at the REPL,
+  for use when ASYNCIFY is enabled.
+
+Proxying
+--------
+
+A Python `dict` instance is proxied such that:
+
+    for (const key in dict) {
+        print(key, dict[key]);
+    }
 
-Input character into MicroPython repl. `char` must be of type `number`. This
-will execute MicroPython code when necessary.
+works as expected on the JavaScript side and iterates through the keys of the
+Python `dict`.