ENH Rework streams handling

Makefile

@@ -124,8 +124,8 @@ node_modules/.installed : src/js/package.json src/js/package-lock.json
 	ln -sfn src/js/node_modules/ node_modules
 	touch node_modules/.installed
 
-dist/pyodide.js src/js/_pyodide.out.js: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts node_modules/.installed
-	cd src/js && npm run tsc && node esbuild.config.mjs && cd -
+dist/pyodide.js src/js/_pyodide.out.js: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts src/js/generated_struct_info32.gen.json node_modules/.installed
+	cd src/js && node esbuild.config.mjs && npm run tsc && cd -
 
 dist/package.json : src/js/package.json
 	cp $< $@
@@ -134,13 +134,19 @@ dist/package.json : src/js/package.json
 npm-link: dist/package.json
 	cd src/test-js && npm ci && npm link ../../dist
 
-dist/pyodide.d.ts dist/pyodide/ffi.d.ts: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts
-	npx dts-bundle-generator src/js/{pyodide,ffi}.ts --export-referenced-types false --project src/js/tsconfig.json
+dist/pyodide.d.ts dist/pyodide/ffi.d.ts: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts src/js/generated_struct_info32.gen.json
+	node src/js/esbuild.config.mjs
+	# See https://github.com/timocov/dts-bundle-generator/issues/255
+	echo "export declare const defines: {[k:string] : number};" > src/js/generated_struct_info32.gen.json.d.ts
+	cd src/js && npx dts-bundle-generator {pyodide,ffi}.ts --export-referenced-types false --project tsconfig.json
 	mv src/js/{pyodide,ffi}.d.ts dist
 	python3 tools/fixup-type-definitions.py dist/pyodide.d.ts
 	python3 tools/fixup-type-definitions.py dist/ffi.d.ts
 
 
+src/js/generated_struct_info32.gen.json : emsdk/emsdk/.complete
+	cp emsdk/emsdk/upstream/emscripten/src/generated_struct_info32.json $@
+
 
 src/js/error_handling.gen.ts : src/core/error_handling.ts
 	cp $< $@

conftest.py

@@ -5,6 +5,7 @@
 import pathlib
 import re
 import sys
+from collections.abc import Sequence
 
 import pytest
 
@@ -250,3 +251,18 @@ def extra_checks_test_wrapper(browser, trace_hiwire_refs, trace_pyproxies):
 
 def package_is_built(package_name):
     return _package_is_built(package_name, pytest.pyodide_dist_dir)
+
+
+def strip_assertions_stderr(messages: Sequence[str]) -> list[str]:
+    """Strip additional messages on stderr included when ASSERTIONS=1"""
+    res = []
+    for msg in messages:
+        if msg.strip() in [
+            "sigaction: signal type not supported: this is a no-op.",
+            "Calling stub instead of siginterrupt()",
+            "warning: no blob constructor, cannot create blobs with mimetypes",
+            "warning: no BlobBuilder",
+        ]:
+            continue
+        res.append(msg)
+    return res

docs/conf.py

@@ -61,7 +61,7 @@
 autodoc_default_flags = ["members", "inherited-members"]
 
 intersphinx_mapping = {
-    "python": ("https://docs.python.org/3.10", None),
+    "python": ("https://docs.python.org/3.11", None),
     "micropip": (f"https://micropip.pyodide.org/en/v{micropip.__version__}/", None),
     "numpy": ("https://numpy.org/doc/stable/", None),
 }
@@ -259,9 +259,36 @@ def ensure_typedoc_on_path():
     )
 
 
+def get_emscripten_version():
+    prefix = "export PYODIDE_EMSCRIPTEN_VERSION ?= "
+    for line in Path("../Makefile.envs").read_text().splitlines():
+        if line.startswith(prefix):
+            return line.removeprefix(prefix)
+
+
+def create_struct_info():
+    struct_info_path = Path("../src/js/generated_struct_info32.gen.json")
+    if struct_info_path.exists():
+        return
+    src_path = Path(
+        "../emsdk/emsdk/upstream/emscripten/src/generated_struct_info32.json"
+    )
+    if src_path.exists():
+        shutil.copy(src_path, struct_info_path)
+        return
+    import urllib
+
+    urllib.request.urlretrieve(
+        f"https://raw.githubusercontent.com/emscripten-core/emscripten/{get_emscripten_version()}/src/generated_struct_info32.json",
+        struct_info_path,
+    )
+
+
 def create_generated_typescript_files(app):
     shutil.copy("../src/core/pyproxy.ts", "../src/js/pyproxy.gen.ts")
     shutil.copy("../src/core/error_handling.ts", "../src/js/error_handling.gen.ts")
+    shutil.copy("../src/core/error_handling.ts", "../src/js/error_handling.gen.ts")
+    create_struct_info()
     app.config.js_source_path = [str(x) for x in Path("../src/js").glob("*.ts")]
 
     def remove_pyproxy_gen_ts():

docs/sphinx_pyodide/sphinx_pyodide/lexers.py

@@ -22,9 +22,12 @@ class PyodideLexer(JavascriptLexer):
         ],
         "python-code": [
             (
-                rf"({quotemark})((?:\\\\|\\[^\\]|[^{quotemark}\\])*)({quotemark})",
+                rf"([A-Za-z.]*)({quotemark})((?:\\\\|\\[^\\]|[^{quotemark}\\])*)({quotemark})",
                 bygroups(
-                    Token.Literal.String, using(PythonLexer), Token.Literal.String
+                    using(JavascriptLexer),
+                    Token.Literal.String,
+                    using(PythonLexer),
+                    Token.Literal.String,
                 ),
                 "#pop",
             )