ENH Add Pyodide support

.github/workflows/ci-cd.yml

@@ -269,6 +269,82 @@ jobs:
             steps.python-install.outputs.python-version
           }}
 
+  test-pyodide:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+
+    name: Test
+    needs: gen_llhttp
+    runs-on: ubuntu-22.04
+    env:
+      PYODIDE_VERSION: 0.25.0a1
+      # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION.
+      # The appropriate versions can be found in the Pyodide repodata.json
+      # "info" field, or in Makefile.envs:
+      # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2
+      PYTHON_VERSION: 3.11.2
+      EMSCRIPTEN_VERSION: 3.1.32
+      NODE_VERSION: 18
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Setup Python ${{ matrix.pyver }}
+      id: python-install
+      uses: actions/setup-python@v4
+      with:
+        allow-prereleases: true
+        python-version: ${{ matrix.pyver }}
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"    # - name: Cache
+    - name: Cache PyPI
+      uses: actions/cache@v3.3.2
+      with:
+        key: pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }}
+        path: ${{ steps.pip-cache.outputs.dir }}
+        restore-keys: |
+            pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-
+    - name: Update pip, wheel, setuptools, build, twine
+      run: |
+        python -m pip install -U pip wheel setuptools build twine
+    - name: Install dependencies
+      run: |
+        python -m pip install -r requirements/test.in -c requirements/test.txt
+    - name: Restore llhttp generated files
+      if: ${{ matrix.no-extensions == '' }}
+      uses: actions/download-artifact@v3
+      with:
+        name: llhttp
+        path: vendor/llhttp/build/
+    - name: Cythonize
+      if: ${{ matrix.no-extensions == '' }}
+      run: |
+        make cythonize
+    - uses: mymindstorm/setup-emsdk@ab889da2abbcbb280f91ec4c215d3bb4f3a8f775 # v12
+      with:
+        version: ${{ env.EMSCRIPTEN_VERSION }}
+        actions-cache-folder: emsdk-cache
+    - name: Install pyodide-build
+      run: pip install "pydantic<2" pyodide-build==$PYODIDE_VERSION
+    - name: Build
+      run: |
+        CFLAGS=-g2 LDFLAGS=-g2 pyodide build
+
+    # - name: set up node
+    #   uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1
+    #   with:
+    #     node-version: ${{ env.NODE_VERSION }}
+
+    # - name: Set up Pyodide virtual environment
+    #   run: |
+    #     pyodide venv .venv-pyodide
+    #     source .venv-pyodide/bin/activate
+    #     pip install dist/*.whl
+    #     python -c "import sys; print(sys.platform)"
+
   check:  # This job does nothing and is only used for the branch protection
     if: always()
 

.pre-commit-config.yaml

@@ -29,6 +29,9 @@ repos:
   rev: v1.5.0
   hooks:
   - id: yesqa
+    additional_dependencies:
+      - flake8-docstrings==1.6.0
+      - flake8-requirements==1.7.8
 - repo: https://github.com/PyCQA/isort
   rev: '5.12.0'
   hooks:

aiohttp/client.py

@@ -66,17 +66,26 @@
     ClientRequest,
     ClientResponse,
     Fingerprint,
+    PyodideClientRequest,
+    PyodideClientResponse,
     RequestInfo,
 )
 from .client_ws import (
     DEFAULT_WS_CLIENT_TIMEOUT,
     ClientWebSocketResponse,
     ClientWSTimeout,
 )
-from .connector import BaseConnector, NamedPipeConnector, TCPConnector, UnixConnector
+from .connector import (
+    BaseConnector,
+    NamedPipeConnector,
+    PyodideConnector,
+    TCPConnector,
+    UnixConnector,
+)
 from .cookiejar import CookieJar
 from .helpers import (
     _SENTINEL,
+    IS_PYODIDE,
     BasicAuth,
     TimeoutHandle,
     ceil_timeout,
@@ -207,8 +216,8 @@ def __init__(
         skip_auto_headers: Optional[Iterable[str]] = None,
         auth: Optional[BasicAuth] = None,
         json_serialize: JSONEncoder = json.dumps,
-        request_class: Type[ClientRequest] = ClientRequest,
-        response_class: Type[ClientResponse] = ClientResponse,
+        request_class: Type[ClientRequest] = None,
+        response_class: Type[ClientResponse] = None,
         ws_response_class: Type[ClientWebSocketResponse] = ClientWebSocketResponse,
         version: HttpVersion = http.HttpVersion11,
         cookie_jar: Optional[AbstractCookieJar] = None,
@@ -237,7 +246,7 @@ def __init__(
         loop = asyncio.get_running_loop()
 
         if connector is None:
-            connector = TCPConnector()
+            connector = PyodideConnector() if IS_PYODIDE else TCPConnector()
 
         # Initialize these three attrs before raising any exception,
         # they are used in __del__
@@ -287,6 +296,11 @@ def __init__(
         else:
             self._skip_auto_headers = frozenset()
 
+        if request_class is None:
+            request_class = PyodideClientRequest if IS_PYODIDE else ClientRequest
+        if response_class is None:
+            response_class = PyodideClientResponse if IS_PYODIDE else ClientResponse
+
         self._request_class = request_class
         self._response_class = response_class
         self._ws_response_class = ws_response_class

aiohttp/client_reqrep.py

@@ -44,6 +44,7 @@
 from .formdata import FormData
 from .hdrs import CONTENT_TYPE
 from .helpers import (
+    IS_PYODIDE,
     BaseTimerContext,
     BasicAuth,
     HeadersMixin,
@@ -585,7 +586,7 @@
             await writer.write_eof()
             protocol.start_timeout()
 
-    async def send(self, conn: "Connection") -> "ClientResponse":
+    def _path(self) -> str:
         # Specify request target:
         # - CONNECT request must send authority form URI
         # - not CONNECT proxy must send absolute form URI
@@ -602,7 +603,10 @@
             path = self.url.raw_path
             if self.url.raw_query_string:
                 path += "?" + self.url.raw_query_string
+        return path
 
+    async def send(self, conn: "Connection") -> "ClientResponse":
+        path = self._path()
         protocol = conn.protocol
         assert protocol is not None
         writer = StreamWriter(
@@ -689,6 +693,52 @@
             await trace.send_request_headers(method, url, headers)
 
 
+class PyodideClientRequest(ClientRequest):
+    async def send(self, conn: "Connection") -> "ClientResponse":
+        if not IS_PYODIDE:
+            raise RuntimeError("PyodideClientRequest only works in Pyodide")
+
+        path = self._path()
+        protocol = conn.protocol
+        assert protocol is not None
+        from js import Headers, fetch  # noqa: I900
+        from pyodide.ffi import to_js  # noqa: I900
+
+        body = None
+        if self.body:
+            if isinstance(self.body, payload.Payload):
+                body = to_js(self.body._value)
+            else:
+                if isinstance(self.body, (bytes, bytearray)):
+                    body = (self.body,)  # type: ignore[assignment]
+
+                raise NotImplementedError("OOPS")
+        response_future = fetch(
+            path,
+            method=self.method,
+            headers=Headers.new(self.headers.items()),
+            body=body,
+            signal=protocol.abortcontroller.signal,
+        )
+        response_class = self.response_class
+        assert response_class is not None
+        assert issubclass(response_class, PyodideClientResponse)
+        self.response = response_class(
+            self.method,
+            self.original_url,
+            writer=None,
+            continue100=self._continue,
+            timer=self._timer,
+            request_info=self.request_info,
+            traces=self._traces,
+            loop=self.loop,
+            session=self._session,
+            response_future=response_future,
+        )
+        self.response.version = self.version
+        return self.response
+
+
 class ClientResponse(HeadersMixin):
     # Some of these attributes are None when created,
     # but will be set by the start() method.
@@ -1124,3 +1174,32 @@
         # if state is broken
         self.release()
         await self.wait_for_close()
+
+
+class PyodideClientResponse(ClientResponse):
+    def __init__(self, *args, response_future, **kwargs):
+        if not IS_PYODIDE:
+            raise RuntimeError("PyodideClientResponse only works in Pyodide")
+        self.response_future = response_future
+        super().__init__(*args, **kwargs)
+
+    async def start(self, connection: "Connection") -> "ClientResponse":
+        from .streams import DataQueue
+
+        self._connection = connection
+        self._protocol = connection.protocol
+        jsresp = await self.response_future
+        self.status = jsresp.status
+        self.reason = jsresp.statusText
+        # This is not quite correct in handling of repeated headers
+        self._headers = CIMultiDict(jsresp.headers)
+        self._raw_headers = tuple(tuple(e) for e in jsresp.headers)
+        self.content = DataQueue(self._loop)
+
+        def done_callback(fut):
+            data = fut.result().to_bytes()
+            self.content.feed_data(data, len(data))
+            self.content.feed_eof()
+
+        jsresp.arrayBuffer().add_done_callback(done_callback)
+        return self

aiohttp/connector.py

@@ -47,7 +47,14 @@
 )
 from .client_proto import ResponseHandler
 from .client_reqrep import SSL_ALLOWED_TYPES, ClientRequest, Fingerprint
-from .helpers import _SENTINEL, ceil_timeout, is_ip_address, sentinel, set_result
+from .helpers import (
+    _SENTINEL,
+    IS_PYODIDE,
+    ceil_timeout,
+    is_ip_address,
+    sentinel,
+    set_result,
+)
 from .locks import EventResultOrError
 from .resolver import DefaultResolver
 
@@ -1384,3 +1391,68 @@
             raise ClientConnectorError(req.connection_key, exc) from exc
 
         return cast(ResponseHandler, proto)
+
+
+IN_PYODIDE = "pyodide" in sys.modules or "emscripten" in sys.platform
+
+
+class PyodideProtocol(ResponseHandler):
+    def __init__(self, loop: asyncio.AbstractEventLoop):
+        from js import AbortController  # noqa: I900
+
+        super().__init__(loop)
+        self.abortcontroller = AbortController.new()
+        self.closed = loop.create_future()
+        # asyncio.Transport "raises NotImplemented for every method"
+        self.transport = asyncio.Transport()
+
+    def close(self):
+        self.abortcontroller.abort()
+        self.closed.set_result(None)
+
+
+class PyodideConnector(BaseConnector):
+    """Named pipe connector.
+
+    Only supported by the proactor event loop.
+    See also: https://docs.python.org/3/library/asyncio-eventloop.html
+
+    path - Windows named pipe path.
+    keepalive_timeout - (optional) Keep-alive timeout.
+    force_close - Set to True to force close and do reconnect
+        after each request (and between redirects).
+    limit - The total number of simultaneous connections.
+    limit_per_host - Number of simultaneous connections to one host.
+    loop - Optional event loop.
+    """
+
+    def __init__(
+        self,
+        *,
+        keepalive_timeout: Union[_SENTINEL, None, float] = sentinel,
+        force_close: bool = False,
+        limit: int = 100,
+        limit_per_host: int = 0,
+        enable_cleanup_closed: bool = False,
+        timeout_ceil_threshold: float = 5,
+    ) -> None:
+        super().__init__(
+            keepalive_timeout=keepalive_timeout,
+            force_close=force_close,
+            limit=limit,
+            limit_per_host=limit_per_host,
+            enable_cleanup_closed=enable_cleanup_closed,
+            timeout_ceil_threshold=timeout_ceil_threshold,
+        )
+        if not IS_PYODIDE:
+            raise RuntimeError("PyodideConnector only works in Pyodide")
+
+    @property
+    def path(self) -> str:
+        """Path to the named pipe."""
+        return self._path
+
+    async def _create_connection(
+        self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout"
+    ) -> ResponseHandler:
+        return PyodideProtocol(self._loop)

aiohttp/helpers.py

@@ -108,6 +108,8 @@
 }
 TOKEN = CHAR ^ CTL ^ SEPARATORS
 
+IS_PYODIDE = "pyodide" in sys.modules
+
 
 class noop:
     def __await__(self) -> Generator[None, None, None]: