diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index e961250bf67..67e4c2adc7b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -269,6 +269,82 @@ jobs: steps.python-install.outputs.python-version }} + test-pyodide: + permissions: + contents: read # to fetch code (actions/checkout) + + name: Test pyodide + needs: gen_llhttp + runs-on: ubuntu-22.04 + env: + PYODIDE_VERSION: 0.25.0a1 + # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. + # The appropriate versions can be found in the Pyodide repodata.json + # "info" field, or in Makefile.envs: + # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 + PYTHON_VERSION: 3.11.4 + EMSCRIPTEN_VERSION: 3.1.45 + NODE_VERSION: 18 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Python ${{ env.PYTHON_VERSION }} + id: python-install + uses: actions/setup-python@v4 + with: + allow-prereleases: true + python-version: ${{ env.PYTHON_VERSION }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" # - name: Cache + - name: Cache PyPI + uses: actions/cache@v3.3.2 + with: + key: pip-ci-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }} + path: ${{ steps.pip-cache.outputs.dir }} + restore-keys: | + pip-ci-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ matrix.no-extensions }}- + - name: Update pip, wheel, setuptools, build, twine + run: | + python -m pip install -U pip wheel setuptools build twine + - name: Install dependencies + run: | + python -m pip install -r requirements/test.in -c requirements/test.txt + - name: Restore llhttp generated files + if: ${{ matrix.no-extensions == '' }} + uses: actions/download-artifact@v3 + with: + name: llhttp + path: vendor/llhttp/build/ + - name: Cythonize + if: ${{ matrix.no-extensions == '' }} + run: | + make cythonize + - uses: mymindstorm/setup-emsdk@v12 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: emsdk-cache + - name: Install pyodide-build + run: pip install "pydantic<2" pyodide-build==$PYODIDE_VERSION + - name: Build + run: | + CFLAGS=-g2 LDFLAGS=-g2 pyodide build + + - uses: pyodide/pyodide-actions/download-pyodide@v1 + with: + version: ${{ env.PYODIDE_VERSION }} + to: pyodide-dist + + - uses: pyodide/pyodide-actions/install-browser@v1 + + - name: Test + run: | + pip install pytest-pyodide + pytest tests/test_pyodide.py --rt chrome --dist-dir ./pyodide-dist + check: # This job does nothing and is only used for the branch protection if: always() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be1fbae1ffc..6f6a1d22ac6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,9 @@ repos: rev: v1.5.0 hooks: - id: yesqa + additional_dependencies: + - flake8-docstrings==1.6.0 + - flake8-requirements==1.7.8 - repo: https://github.com/PyCQA/isort rev: '5.12.0' hooks: diff --git a/CHANGES/7803.feature b/CHANGES/7803.feature new file mode 100644 index 00000000000..68e0e864b9d --- /dev/null +++ b/CHANGES/7803.feature @@ -0,0 +1 @@ +Added basic support for using aiohttp in Pyodide. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 539a8807689..d95c3493b6d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -141,6 +141,7 @@ Hans Adema Harmon Y. Harry Liu Hiroshi Ogawa +Hood Chatham Hrishikesh Paranjape Hu Bo Hugh Young diff --git a/aiohttp/client.py b/aiohttp/client.py index b0920446d4b..3473c1bfe00 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -68,6 +68,8 @@ ClientRequest, ClientResponse, Fingerprint, + PyodideClientRequest, + PyodideClientResponse, RequestInfo, ) from .client_ws import ( @@ -75,10 +77,17 @@ ClientWebSocketResponse, ClientWSTimeout, ) -from .connector import BaseConnector, NamedPipeConnector, TCPConnector, UnixConnector +from .connector import ( + BaseConnector, + NamedPipeConnector, + PyodideConnector, + TCPConnector, + UnixConnector, +) from .cookiejar import CookieJar from .helpers import ( _SENTINEL, + IS_PYODIDE, BasicAuth, TimeoutHandle, ceil_timeout, @@ -211,8 +220,8 @@ def __init__( skip_auto_headers: Optional[Iterable[str]] = None, auth: Optional[BasicAuth] = None, json_serialize: JSONEncoder = json.dumps, - request_class: Type[ClientRequest] = ClientRequest, - response_class: Type[ClientResponse] = ClientResponse, + request_class: Optional[Type[ClientRequest]] = None, + response_class: Optional[Type[ClientResponse]] = None, ws_response_class: Type[ClientWebSocketResponse] = ClientWebSocketResponse, version: HttpVersion = http.HttpVersion11, cookie_jar: Optional[AbstractCookieJar] = None, @@ -241,7 +250,7 @@ def __init__( loop = asyncio.get_running_loop() if connector is None: - connector = TCPConnector() + connector = PyodideConnector() if IS_PYODIDE else TCPConnector() # Initialize these three attrs before raising any exception, # they are used in __del__ @@ -291,6 +300,11 @@ def __init__( else: self._skip_auto_headers = frozenset() + if request_class is None: + request_class = PyodideClientRequest if IS_PYODIDE else ClientRequest + if response_class is None: + response_class = PyodideClientResponse if IS_PYODIDE else ClientResponse + self._request_class = request_class self._response_class = response_class self._ws_response_class = ws_response_class diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 417294a5cfa..6bf04a831b7 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -44,9 +44,13 @@ from .formdata import FormData from .hdrs import CONTENT_TYPE from .helpers import ( + IS_PYODIDE, BaseTimerContext, BasicAuth, HeadersMixin, + JsArrayBuffer, + JsRequest, + JsResponse, TimerNoop, basicauth_from_netrc, is_expected_content_type, @@ -86,7 +90,7 @@ if TYPE_CHECKING: from .client import ClientSession - from .connector import Connection + from .connector import Connection, PyodideConnection from .tracing import Trace @@ -689,6 +693,73 @@ async def _on_headers_request_sent( await trace.send_request_headers(method, url, headers) +def _make_js_request( + path: str, *, method: str, headers: CIMultiDict[str], signal: Any, body: Any +) -> JsRequest: + from js import Headers, Request # type:ignore[import-not-found] # noqa: I900 + from pyodide.ffi import to_js # type:ignore[import-not-found] # noqa: I900 + + if method.lower() in ["get", "head"]: + body = None + # TODO: to_js does an unnecessary copy. + elif isinstance(body, payload.Payload): + body = to_js(body._value) + elif isinstance(body, (bytes, bytearray)): + body = to_js(body) + else: + # What else can happen here? Maybe body could be a list of + # bytes? In that case we should turn it into a Blob. + raise NotImplementedError("OOPS") + + return cast( + JsRequest, + Request.new( + path, + method=method, + headers=Headers.new(headers.items()), + body=body, + signal=signal, + ), + ) + + +class PyodideClientRequest(ClientRequest): + async def send( + self, conn: "PyodideConnection" # type:ignore[override] + ) -> "ClientResponse": + if not IS_PYODIDE: + raise RuntimeError("PyodideClientRequest only works in Pyodide") + + protocol = conn.protocol + assert protocol is not None + + request = _make_js_request( + str(self.url), + method=self.method, + headers=self.headers, + body=self.body, + signal=protocol.abortcontroller.signal, + ) + response_future = protocol.fetch_handler(request) + response_class = self.response_class + assert response_class is not None + assert issubclass(response_class, PyodideClientResponse) + self.response = response_class( + self.method, + self.original_url, + writer=None, # type:ignore[arg-type] + continue100=self._continue, + timer=self._timer, + request_info=self.request_info, + traces=self._traces, + loop=self.loop, + session=self._session, + response_future=response_future, + ) + self.response.version = self.version + return self.response + + class ClientResponse(HeadersMixin): # Some of these attributes are None when created, # but will be set by the start() method. @@ -1123,3 +1194,57 @@ async def __aexit__( # if state is broken self.release() await self.wait_for_close() + + +class PyodideClientResponse(ClientResponse): + def __init__( + self, + method: str, + url: URL, + *, + writer: "asyncio.Task[None]", + continue100: Optional["asyncio.Future[bool]"], + timer: Optional[BaseTimerContext], + request_info: RequestInfo, + traces: List["Trace"], + loop: asyncio.AbstractEventLoop, + session: "ClientSession", + response_future: "asyncio.Future[JsResponse]", + ): + if not IS_PYODIDE: + raise RuntimeError("PyodideClientResponse only works in Pyodide") + self.response_future = response_future + super().__init__( + method, + url, + writer=writer, + continue100=continue100, + timer=timer, + request_info=request_info, + traces=traces, + loop=loop, + session=session, + ) + + async def start(self, connection: "Connection") -> "ClientResponse": + from .streams import DataQueue + + self._connection = connection + self._protocol = connection.protocol + jsresp = await self.response_future + self.status = jsresp.status + self.reason = jsresp.statusText + # This is not quite correct in handling of repeated headers + self._headers = cast(CIMultiDictProxy[str], CIMultiDict(jsresp.headers)) + self._raw_headers = tuple( + (e[0].encode(), e[1].encode()) for e in jsresp.headers + ) + self.content = DataQueue(self._loop) # type:ignore[assignment] + + def done_callback(fut: "asyncio.Future[JsArrayBuffer]") -> None: + data = fut.result().to_bytes() + self.content.feed_data(data, len(data)) + self.content.feed_eof() + + jsresp.arrayBuffer().add_done_callback(done_callback) + return self diff --git a/aiohttp/connector.py b/aiohttp/connector.py index fa96c592f56..bb42c15964f 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -24,6 +24,7 @@ List, Literal, Optional, + Protocol, Set, Tuple, Type, @@ -47,7 +48,16 @@ ) from .client_proto import ResponseHandler from .client_reqrep import SSL_ALLOWED_TYPES, ClientRequest, Fingerprint -from .helpers import _SENTINEL, ceil_timeout, is_ip_address, sentinel, set_result +from .helpers import ( + _SENTINEL, + IS_PYODIDE, + JsRequest, + JsResponse, + ceil_timeout, + is_ip_address, + sentinel, + set_result, +) from .locks import EventResultOrError from .resolver import DefaultResolver @@ -1388,3 +1398,78 @@ async def _create_connection( raise ClientConnectorError(req.connection_key, exc) from exc return cast(ResponseHandler, proto) + + +IN_PYODIDE = "pyodide" in sys.modules or "emscripten" in sys.platform + + +class PyodideProtocol(ResponseHandler): + def __init__( + self, + loop: asyncio.AbstractEventLoop, + fetch_handler: Callable[[JsRequest], "asyncio.Future[JsResponse]"], + ): + from js import AbortController # type:ignore[import-not-found] # noqa: I900 + + super().__init__(loop) + self.abortcontroller = AbortController.new() + self.closed = loop.create_future() + # asyncio.Transport "raises NotImplemented for every method" + self.transport = asyncio.Transport() + self.fetch_handler = fetch_handler + + def close(self) -> None: + self.abortcontroller.abort() + self.closed.set_result(None) + + +class PyodideConnection(Connection): + _protocol: PyodideProtocol + + @property + def protocol(self) -> Optional[PyodideProtocol]: + return self._protocol + + +class PyodideConnector(BaseConnector): + """Pyodide connector + + Dummy connector that + """ + + protocol: PyodideProtocol + + def __init__( + self, + *, + fetch_handler: Optional[ + Callable[[JsRequest], "asyncio.Future[JsResponse]"] + ] = None, + keepalive_timeout: Union[_SENTINEL, None, float] = sentinel, + force_close: bool = False, + limit: int = 100, + limit_per_host: int = 0, + enable_cleanup_closed: bool = False, + timeout_ceil_threshold: float = 5, + ) -> None: + super().__init__( + keepalive_timeout=keepalive_timeout, + force_close=force_close, + limit=limit, + limit_per_host=limit_per_host, + enable_cleanup_closed=enable_cleanup_closed, + timeout_ceil_threshold=timeout_ceil_threshold, + ) + if fetch_handler is None: + from js import fetch # noqa: I900 + + fetch_handler = fetch + + self.fetch_handler = fetch_handler + if not IS_PYODIDE: + raise RuntimeError("PyodideConnector only works in Pyodide") + + async def _create_connection( + self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + return PyodideProtocol(self._loop, self.fetch_handler) diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 5435e2f9e07..a3a4834a205 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -108,6 +108,8 @@ } TOKEN = CHAR ^ CTL ^ SEPARATORS +IS_PYODIDE = "pyodide" in sys.modules + class noop: def __await__(self) -> Generator[None, None, None]: @@ -1090,3 +1092,21 @@ def should_remove_content_length(method: str, code: int) -> bool: or 100 <= code < 200 or (200 <= code < 300 and method.upper() == hdrs.METH_CONNECT) ) + + +class JsRequest(Protocol): + pass + + +class JsArrayBuffer(Protocol): + def to_bytes(self) -> bytes: + ... + + +class JsResponse(Protocol): + status: int + statusText: str + headers: List[Tuple[str, str]] + + def arrayBuffer(self) -> "asyncio.Future[JsArrayBuffer]": + ... diff --git a/setup.cfg b/setup.cfg index 13efc0b7796..600431a3714 100644 --- a/setup.cfg +++ b/setup.cfg @@ -126,8 +126,7 @@ addopts = --showlocals # `pytest-cov`: - --cov=aiohttp - --cov=tests/ + # run tests that are not marked with dev_mode -m "not dev_mode" diff --git a/tests/test_pyodide.py b/tests/test_pyodide.py new file mode 100644 index 00000000000..9412d47c082 --- /dev/null +++ b/tests/test_pyodide.py @@ -0,0 +1,156 @@ +import shutil +import sys +from asyncio import Future +from collections.abc import Mapping +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +from pytest import fixture + +try: + from pytest_pyodide import run_in_pyodide # noqa: I900 +except ImportError: + run_in_pyodide = pytest.mark.skip("pytest-pyodide not installed") + +from aiohttp import ClientSession, client, client_reqrep, connector +from aiohttp.connector import PyodideConnector + + +class JsAbortController: + @staticmethod + def new() -> "JsAbortController": + return JsAbortController() + + def abort(self) -> None: + pass + + @property + def signal(self) -> str: + return "AbortSignal" + + +class JsHeaders: + def __init__(self, items: Mapping): + self.items = dict(items) + + @staticmethod + def new(items: Mapping) -> "JsHeaders": + return JsHeaders(items) + + +class JsRequest: + def __init__(self, path: str, **kwargs: Any): + self.path = path + self.kwargs = kwargs + + @staticmethod + def new(path, **kwargs) -> "JsRequest": + return JsRequest(path, **kwargs) + + +class JsBuffer: + def __init__(self, content: bytes): + self.content = content + + def to_bytes(self) -> bytes: + return self.content + + +class JsResponse: + def __init__( + self, status: int, statusText: str, headers: JsHeaders, body: bytes | None + ): + self.status = status + self.statusText = statusText + self.headers = headers + self.body = body + + def arrayBuffer(self): + fut = Future() + fut.set_result(self.body) + return fut + + +@fixture +def mock_pyodide_env(monkeypatch: Any): + monkeypatch.setattr(client, "IS_PYODIDE", True) + monkeypatch.setattr(connector, "IS_PYODIDE", True) + monkeypatch.setattr(client_reqrep, "IS_PYODIDE", True) + jsmod = ModuleType("js") + jsmod.AbortController = JsAbortController + jsmod.Headers = JsHeaders + jsmod.Request = JsRequest + + monkeypatch.setitem(sys.modules, "js", jsmod) + + +async def test_pyodide_mock(mock_pyodide_env: Any) -> None: + def fetch_handler(request: JsRequest) -> Future[JsResponse]: + assert request.path == "http://example.com" + assert request.kwargs["method"] == "GET" + assert request.kwargs["headers"].items == { + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Host": "example.com", + "User-Agent": "Python/3.11 aiohttp/4.0.0a2.dev0", + } + assert request.kwargs["signal"] == "AbortSignal" + assert request.kwargs["body"] is None + fut = Future() + resp = JsResponse( + 200, "OK", [["Content-type", "text/html; charset=utf-8"]], JsBuffer(b"abc") + ) + fut.set_result(resp) + return fut + + c = PyodideConnector(fetch_handler=fetch_handler) + async with ClientSession(connector=c) as session: + async with session.get("http://example.com") as response: + assert response.status == 200 + assert response.headers["content-type"] == "text/html; charset=utf-8" + html = await response.text() + assert html == "abc" + + +@fixture +def install_aiohttp(selenium, request): + wheel = next(Path("dist").glob("*.whl")) + dist_dir = request.config.option.dist_dir + dist_wheel = dist_dir / wheel.name + shutil.copyfile(wheel, dist_wheel) + selenium.load_package(["multidict", "yarl", "aiosignal"]) + selenium.load_package(wheel.name) + try: + yield + finally: + dist_wheel.unlink() + + +@fixture +async def url_to_fetch(request, web_server_main): + target_file = Path(request.config.option.dist_dir) / "test.txt" + target_file.write_text("hello there!") + server_host, server_port, _ = web_server_main + try: + yield f"http://{server_host}:{server_port}/{target_file.name}" + finally: + target_file.unlink() + + +@fixture +async def loop_wrapper(loop): + return None + + +@run_in_pyodide +async def test_pyodide(selenium, install_aiohttp, url_to_fetch, loop_wrapper) -> None: + from aiohttp import ClientSession + + async with ClientSession() as session: + async with session.get(url_to_fetch) as response: + assert response.status == 200 + assert response.headers["content-type"] == "text/plain" + html = await response.text() + assert html == "hello there!"