Feature: Native Emscripten support

.eslintrc.yml

@@ -0,0 +1,7 @@
+env:
+  es2020 : true
+  worker: true
+rules: {}
+extends: 
+-  eslint:recommended
+

.github/workflows/ci.yml

@@ -81,6 +81,10 @@ jobs:
             os: ubuntu-20.04  # CPython 3.9.2 is not available for ubuntu-22.04.
             experimental: false
             nox-session: test-3.9
+          - python-version: "3.11"
+            os: ubuntu-latest
+            nox-session: emscripten
+            experimental: true
         exclude:
           # Ubuntu 22.04 comes with OpenSSL 3.0, so only CPython 3.9+ is compatible with it
           # https://github.com/python/cpython/issues/83001
@@ -104,6 +108,12 @@ jobs:
       - name: "Install dependencies"
         run: python -m pip install --upgrade pip setuptools nox
 
+      - name: "Install Chrome"
+        uses: browser-actions/setup-chrome@11cef13cde73820422f9263a707fb8029808e191 # v1.3.0
+        if: ${{ matrix.nox-session == 'emscripten' }}
+      - name: "Install Firefox"
+        uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead # v1.4.0
+        if: ${{ matrix.nox-session == 'emscripten' }}
       - name: "Run tests"
         # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version
         run: nox -s ${NOX_SESSION:-test-$PYTHON_VERSION} --error-on-missing-interpreters

.pre-commit-config.yaml

@@ -21,3 +21,14 @@ repos:
     hooks:
       - id: flake8
         additional_dependencies: [flake8-2020]
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v3.1.0"
+    hooks:
+      - id: prettier
+        types_or: [javascript]
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v8.53.0
+    hooks:
+      - id: eslint
+        args: ["--fix"]

changelog/2951.feature.rst

@@ -0,0 +1 @@
+Added support for Emscripten, including streaming support in cross-origin isolated browser environments where threading is enabled.

dummyserver/handlers.py

@@ -247,6 +247,7 @@ def echo(self, request: httputil.HTTPServerRequest) -> Response:
 
     def echo_json(self, request: httputil.HTTPServerRequest) -> Response:
         "Echo back the JSON"
+        print("ECHO JSON:", request.body)
         return Response(json=request.body, headers=list(request.headers.items()))
 
     def echo_uri(self, request: httputil.HTTPServerRequest) -> Response:

emscripten-requirements.txt

@@ -0,0 +1,3 @@
+pytest-pyodide==0.54.0
+pyodide-build==0.24.1
+webdriver-manager==4.0.1

noxfile.py

@@ -3,6 +3,8 @@
 import os
 import shutil
 import sys
+import typing
+from pathlib import Path
 
 import nox
 
@@ -14,6 +16,7 @@ def tests_impl(
     # https://github.com/python-hyper/h2/issues/1236
     byte_string_comparisons: bool = False,
     integration: bool = False,
+    pytest_extra_args: list[str] = [],
 ) -> None:
     # Install deps and the package itself.
     session.install("-r", "dev-requirements.txt")
@@ -53,6 +56,7 @@ def tests_impl(
         "--durations=10",
         "--strict-config",
         "--strict-markers",
+        *pytest_extra_args,
         *(session.posargs or ("test/",)),
         env={"PYTHONWARNINGS": "always::DeprecationWarning"},
     )
@@ -152,6 +156,108 @@ def lint(session: nox.Session) -> None:
     mypy(session)
 
 
+# TODO: node support is not tested yet - it should work if you require('xmlhttprequest') before
+# loading pyodide, but there is currently no nice way to do this with pytest-pyodide
+# because you can't override the test runner properties easily - see
+# https://github.com/pyodide/pytest-pyodide/issues/118 for more
+@nox.session(python="3.11")
+@nox.parametrize("runner", ["firefox", "chrome"])
+def emscripten(session: nox.Session, runner: str) -> None:
+    """Test on Emscripten with Pyodide & Chrome / Firefox"""
+    session.install("-r", "emscripten-requirements.txt")
+    # build wheel into dist folder
+    session.run("python", "-m", "build")
+    # make sure we have a dist dir for pyodide
+    dist_dir = None
+    if "PYODIDE_ROOT" in os.environ:
+        # we have a pyodide build tree checked out
+        # use the dist directory from that
+        dist_dir = Path(os.environ["PYODIDE_ROOT"]) / "dist"
+    else:
+        # we don't have a build tree, get one
+        # that matches the version of pyodide build
+        pyodide_version = typing.cast(
+            str,
+            session.run(
+                "python",
+                "-c",
+                "import pyodide_build;print(pyodide_build.__version__)",
+                silent=True,
+            ),
+        ).strip()
+
+        pyodide_artifacts_path = Path(session.cache_dir) / f"pyodide-{pyodide_version}"
+        if not pyodide_artifacts_path.exists():
+            print("Fetching pyodide build artifacts")
+            session.run(
+                "wget",
+                f"https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/pyodide-{pyodide_version}.tar.bz2",
+                "-O",
+                f"{pyodide_artifacts_path}.tar.bz2",
+            )
+            pyodide_artifacts_path.mkdir(parents=True)
+            session.run(
+                "tar",
+                "-xjf",
+                f"{pyodide_artifacts_path}.tar.bz2",
+                "-C",
+                str(pyodide_artifacts_path),
+                "--strip-components",
+                "1",
+            )
+
+        dist_dir = pyodide_artifacts_path
+    assert dist_dir is not None
+    assert dist_dir.exists()
+    if runner == "chrome":
+        # install chrome webdriver and add it to path
+        driver = typing.cast(
+            str,
+            session.run(
+                "python",
+                "-c",
+                "from webdriver_manager.chrome import ChromeDriverManager;print(ChromeDriverManager().install())",
+                silent=True,
+            ),
+        ).strip()
+        session.env["PATH"] = f"{Path(driver).parent}:{session.env['PATH']}"
+
+        tests_impl(
+            session,
+            pytest_extra_args=[
+                "--rt",
+                "chrome-no-host",
+                "--dist-dir",
+                str(dist_dir),
+                "test",
+            ],
+        )
+    elif runner == "firefox":
+        driver = typing.cast(
+            str,
+            session.run(
+                "python",
+                "-c",
+                "from webdriver_manager.firefox import GeckoDriverManager;print(GeckoDriverManager().install())",
+                silent=True,
+            ),
+        ).strip()
+        session.env["PATH"] = f"{Path(driver).parent}:{session.env['PATH']}"
+
+        tests_impl(
+            session,
+            pytest_extra_args=[
+                "--rt",
+                "firefox-no-host",
+                "--dist-dir",
+                str(dist_dir),
+                "test",
+            ],
+        )
+    else:
+        raise ValueError(f"Unknown runnner: {runner}")
+
+
 @nox.session(python="3.12")
 def mypy(session: nox.Session) -> None:
     """Run mypy."""

src/urllib3/__init__.py

@@ -6,6 +6,7 @@
 
 # Set default logging handler to avoid "No handler found" warnings.
 import logging
+import sys
 import typing
 import warnings
 from logging import NullHandler
@@ -202,3 +203,9 @@ def request(
         timeout=timeout,
         json=json,
     )
+
+
+if sys.platform == "emscripten":
+    from .contrib.emscripten import inject_into_urllib3  # noqa: 401
+
+    inject_into_urllib3()

src/urllib3/connectionpool.py

@@ -543,6 +543,8 @@ def _make_request(
         response._connection = response_conn  # type: ignore[attr-defined]
         response._pool = self  # type: ignore[attr-defined]
 
+        # emscripten connection doesn't have _http_vsn_str
+        http_version = getattr(conn, "_http_vsn_str", "HTTP/?")
         log.debug(
             '%s://%s:%s "%s %s %s" %s %s',
             self.scheme,
@@ -551,7 +553,7 @@ def _make_request(
             method,
             url,
             # HTTP version
-            conn._http_vsn_str,  # type: ignore[attr-defined]
+            http_version,
             response.status,
             response.length_remaining,  # type: ignore[attr-defined]
         )

src/urllib3/contrib/emscripten/__init__.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+import urllib3.connection
+
+from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection
+
+
+def inject_into_urllib3() -> None:
+    # override connection classes to use emscripten specific classes
+    # n.b. mypy complains about the overriding of classes below
+    # if it isn't ignored
+    HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection
+    HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection
+    urllib3.connection.HTTPConnection = EmscriptenHTTPConnection  # type: ignore[misc,assignment]
+    urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection  # type: ignore[misc,assignment]