From 89bdcd67073b5a0c789e94b1d8b0e13c2ea9978e Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 10 Nov 2023 19:32:03 +0000 Subject: [PATCH 01/49] first version of emscripten fetch streaming not working yet --- src/urllib3/__init__.py | 4 + src/urllib3/contrib/emscripten/__init__.py | 8 + src/urllib3/contrib/emscripten/connection.py | 186 +++++++++ src/urllib3/contrib/emscripten/fetch.py | 376 +++++++++++++++++++ src/urllib3/contrib/emscripten/request.py | 23 ++ src/urllib3/contrib/emscripten/response.py | 85 +++++ test/with_dummyserver/emscripten_fixtures.py | 159 ++++++++ test/with_dummyserver/test_emscripten.py | 156 ++++++++ 8 files changed, 997 insertions(+) create mode 100644 src/urllib3/contrib/emscripten/__init__.py create mode 100644 src/urllib3/contrib/emscripten/connection.py create mode 100644 src/urllib3/contrib/emscripten/fetch.py create mode 100644 src/urllib3/contrib/emscripten/request.py create mode 100644 src/urllib3/contrib/emscripten/response.py create mode 100644 test/with_dummyserver/emscripten_fixtures.py create mode 100644 test/with_dummyserver/test_emscripten.py diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 26fc5770e4..7505b5b18d 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -147,3 +147,7 @@ def request( timeout=timeout, json=json, ) + +import sys +if sys.platform == "emscripten": + from .contrib.emscripten import * diff --git a/src/urllib3/contrib/emscripten/__init__.py b/src/urllib3/contrib/emscripten/__init__.py new file mode 100644 index 0000000000..8d8323dacf --- /dev/null +++ b/src/urllib3/contrib/emscripten/__init__.py @@ -0,0 +1,8 @@ +from .connection import EmscriptenHTTPConnection,EmscriptenHTTPSConnection +from ...connectionpool import HTTPConnectionPool,HTTPSConnectionPool +HTTPConnectionPool.ConnectionCls=EmscriptenHTTPConnection +HTTPSConnectionPool.ConnectionCls=EmscriptenHTTPSConnection + +import urllib3.connection +urllib3.connection.HTTPConnection=EmscriptenHTTPConnection +urllib3.connection.HTTPSConnection=EmscriptenHTTPSConnection diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py new file mode 100644 index 0000000000..ce79af3ccd --- /dev/null +++ b/src/urllib3/contrib/emscripten/connection.py @@ -0,0 +1,186 @@ +from ..._base_connection import _TYPE_BODY, _TYPE_SOCKET_OPTIONS +from ...connection import HTTPConnection, ProxyConfig, port_by_scheme +from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT +from ...util.url import Url + +from .fetch import send_streaming_request, send_request + +from .request import EmscriptenRequest +from .response import EmscriptenHttpResponseWrapper +from ...response import BaseHTTPResponse + +import typing + + +class EmscriptenHTTPConnection(HTTPConnection): + host: str + port: int + timeout: None | (float) + blocksize: int + source_address: tuple[str, int] | None + socket_options: _TYPE_SOCKET_OPTIONS | None + + proxy: Url | None + proxy_config: ProxyConfig | None + + is_verified: bool + proxy_is_verified: bool | None + + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 8192, + socket_options: _TYPE_SOCKET_OPTIONS | None = ..., + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + ) -> None: + # ignore everything else because we don't have + # control over that stuff + self.host = host + self.port = port + self.timeout = timeout + + def set_tunnel( + self, + host: str, + port: int | None = None, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: + pass + + def connect(self) -> None: + pass + + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + # We know *at least* botocore is depending on the order of the + # first 3 parameters so to be safe we only mark the later ones + # as keyword-only to ensure we have space to extend. + *, + chunked: bool = False, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> None: + request = EmscriptenRequest(url=url, method=method) + request.set_body(body) + if headers: + for k, v in headers.items(): + request.set_header(k, v) + self._response = send_streaming_request(request) + if self._response == None: + self._response = send_request(request) + + def getresponse(self) -> BaseHTTPResponse: + return EmscriptenHttpResponseWrapper(self._response) + + def close(self) -> None: + ... + + @property + def is_closed(self) -> bool: + """Whether the connection either is brand new or has been previously closed. + If this property is True then both ``is_connected`` and ``has_connected_to_proxy`` + properties must be False. + """ + + @property + def is_connected(self) -> bool: + """Whether the connection is actively connected to any origin (proxy or target)""" + + @property + def has_connected_to_proxy(self) -> bool: + """Whether the connection has successfully connected to its proxy. + This returns False if no proxy is in use. Used to determine whether + errors are coming from the proxy layer or from tunnelling to the target origin. + """ + + +class EmscriptenHTTPSConnection(EmscriptenHTTPConnection): + default_port = port_by_scheme["https"] # type: ignore[misc] + + cert_reqs: int | str | None = None + ca_certs: str | None = None + ca_cert_dir: str | None = None + ca_cert_data: None | str | bytes = None + ssl_version: int | str | None = None + ssl_minimum_version: int | None = None + ssl_maximum_version: int | None = None + assert_fingerprint: str | None = None + + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: None + | _TYPE_SOCKET_OPTIONS = HTTPConnection.default_socket_options, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + cert_reqs: int | str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + server_hostname: str | None = None, + ssl_context: typing.Any | None = None, + ca_certs: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + ssl_version: int | str | None = None, # Deprecated + cert_file: str | None = None, + key_file: str | None = None, + key_password: str | None = None, + ) -> None: + super().__init__( + host, + port=port, + timeout=timeout, + source_address=source_address, + blocksize=blocksize, + socket_options=socket_options, + proxy=proxy, + proxy_config=proxy_config, + ) + + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password + self.ssl_context = ssl_context + self.server_hostname = server_hostname + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + self.ssl_version = ssl_version + self.ssl_minimum_version = ssl_minimum_version + self.ssl_maximum_version = ssl_maximum_version + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + self.ca_cert_data = ca_cert_data + + self.cert_reqs = None + + def set_cert( + self, + key_file: str | None = None, + cert_file: str | None = None, + cert_reqs: int | str | None = None, + key_password: str | None = None, + ca_certs: str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ) -> None: + pass diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py new file mode 100644 index 0000000000..53f76c2fcd --- /dev/null +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -0,0 +1,376 @@ +""" +Support for streaming http requests in emscripten. + +A couple of caveats - + +Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed. +Streaming only works if you're running pyodide in a web worker. + +Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch +operation, so it requires that you have crossOriginIsolation enabled, by serving over https +(or from localhost) with the two headers below set: + + Cross-Origin-Opener-Policy: same-origin + Cross-Origin-Embedder-Policy: require-corp + +You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in +javascript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole +request into a buffer and then returning it. it shows a warning in the javascript console in this case. +""" +import io +import json +import js +from pyodide.ffi import to_js +from .request import EmscriptenRequest +from .response import EmscriptenResponse + +from email.parser import Parser + +""" +There are some headers that trigger unintended CORS preflight requests. +See also https://github.com/koenvo/pyodide-http/issues/22 +""" +HEADERS_TO_IGNORE = ("user-agent",) + +SUCCESS_HEADER = -1 +SUCCESS_EOF = -2 +ERROR_TIMEOUT = -3 +ERROR_EXCEPTION = -4 + +_STREAMING_WORKER_CODE = """ +let SUCCESS_HEADER = -1 +let SUCCESS_EOF = -2 +let ERROR_TIMEOUT = -3 +let ERROR_EXCEPTION = -4 + +let connections = {}; +let nextConnectionID = 1; +self.addEventListener("message", async function (event) { + if(event.data.close) + { + let connectionID=event.data.close; + delete connections[connectionID]; + return; + }else if (event.data.getMore) { + let connectionID = event.data.getMore; + let { curOffset, value, reader,intBuffer,byteBuffer } = connections[connectionID]; + // if we still have some in buffer, then just send it back straight away + if (!value || curOffset >= value.length) { + // read another buffer if required + try + { + let readResponse = await reader.read(); + + if (readResponse.done) { + // read everything - clear connection and return + delete connections[connectionID]; + Atomics.store(intBuffer, 0, SUCCESS_EOF); + Atomics.notify(intBuffer, 0); + // finished reading successfully + // return from event handler + return; + } + curOffset = 0; + connections[connectionID].value = readResponse.value; + value=readResponse.value; + }catch(error) + { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } + + // send as much buffer as we can + let curLen = value.length - curOffset; + if (curLen > byteBuffer.length) { + curLen = byteBuffer.length; + } + byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0) + Atomics.store(intBuffer, 0, curLen);// store current length in bytes + Atomics.notify(intBuffer, 0); + curOffset+=curLen; + connections[connectionID].curOffset = curOffset; + + return; + } else { + // start fetch + let connectionID = nextConnectionID; + nextConnectionID += 1; + const encoder = new TextEncoder(); + const intBuffer = new Int32Array(event.data.buffer); + const byteBuffer = new Uint8Array(event.data.buffer, 8) + try { + const response = await fetch(event.data.url, event.data.fetchParams); + // return the headers first via textencoder + var headers = []; + for (const pair of response.headers.entries()) { + headers.push([pair[0], pair[1]]); + } + headerObj = { headers: headers, status: response.status, connectionID }; + const headerText = JSON.stringify(headerObj); + let headerBytes = encoder.encode(headerText); + let written = headerBytes.length; + byteBuffer.set(headerBytes); + intBuffer[1] = written; + // make a connection + connections[connectionID] = { reader:response.body.getReader(),intBuffer:intBuffer,byteBuffer:byteBuffer,value:undefined,curOffset:0 }; + // set header ready + Atomics.store(intBuffer, 0, SUCCESS_HEADER); + Atomics.notify(intBuffer, 0); + // all fetching after this goes through a new postmessage call with getMore + // this allows for parallel requests + } + catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } +}); +""" + + +def _obj_from_dict(dict_val: dict) -> any: + return to_js(dict_val, dict_converter=js.Object.fromEntries) + + +class _ReadStream(io.RawIOBase): + def __init__(self, int_buffer, byte_buffer, timeout, worker, connection_id): + self.int_buffer = int_buffer + self.byte_buffer = byte_buffer + self.read_pos = 0 + self.read_len = 0 + self.connection_id = connection_id + self.worker = worker + self.timeout = int(1000 * timeout) if timeout > 0 else None + self.closed = False + + def __del__(self): + self.close() + + def close(self): + if not self.closed: + self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) + self.closed = True + + def readable(self) -> bool: + return True + + def writeable(self) -> bool: + return False + + def seekable(self) -> bool: + return False + + def readinto(self, byte_obj) -> bool: + if not self.int_buffer: + return 0 + if self.read_len == 0: + # wait for the worker to send something + js.Atomics.store(self.int_buffer, 0, 0) + self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id})) + if js.Atomics.wait(self.int_buffer, 0, 0, self.timeout) == "timed-out": + from ._core import _StreamingTimeout + + raise _StreamingTimeout + data_len = self.int_buffer[0] + if data_len > 0: + self.read_len = data_len + self.read_pos = 0 + elif data_len == ERROR_EXCEPTION: + from ._core import _StreamingError + + raise _StreamingError + else: + # EOF, free the buffers and return zero + self.read_len = 0 + self.read_pos = 0 + self.int_buffer = None + self.byte_buffer = None + return 0 + # copy from int32array to python bytes + ret_length = min(self.read_len, len(byte_obj)) + self.byte_buffer.subarray(self.read_pos, self.read_pos + ret_length).assign_to( + byte_obj[0:ret_length] + ) + self.read_len -= ret_length + self.read_pos += ret_length + return ret_length + + +class _StreamingFetcher: + def __init__(self): + # make web-worker and data buffer on startup + dataBlob = js.globalThis.Blob.new( + [_STREAMING_WORKER_CODE], _obj_from_dict({"type": "application/javascript"}) + ) + print("Make worker") + dataURL = js.URL.createObjectURL(dataBlob) + self._worker = js.Worker.new(dataURL) + print("Initialized worker") + + def send(self, request): + headers = { + k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE + } + + body = request.body + fetch_data = {"headers": headers, "body": to_js(body), "method": request.method} + # start the request off in the worker + timeout = int(1000 * request.timeout) if request.timeout > 0 else None + shared_buffer = js.SharedArrayBuffer.new(1048576) + int_buffer = js.Int32Array.new(shared_buffer) + byte_buffer = js.Uint8Array.new(shared_buffer, 8) + + js.Atomics.store(int_buffer, 0, 0) + js.Atomics.notify(int_buffer, 0) + absolute_url = js.URL.new(request.url, js.location).href + js.console.log( + _obj_from_dict( + { + "buffer": shared_buffer, + "url": absolute_url, + "fetchParams": fetch_data, + } + ) + ) + self._worker.postMessage( + _obj_from_dict( + { + "buffer": shared_buffer, + "url": absolute_url, + "fetchParams": fetch_data, + } + ) + ) + # wait for the worker to send something + js.Atomics.wait(int_buffer, 0, 0, timeout) + if int_buffer[0] == 0: + from ._core import _StreamingTimeout + + raise _StreamingTimeout( + "Timeout connecting to streaming request", + request=request, + response=None, + ) + if int_buffer[0] == SUCCESS_HEADER: + # got response + # header length is in second int of intBuffer + string_len = int_buffer[1] + # decode the rest to a JSON string + decoder = js.TextDecoder.new() + # this does a copy (the slice) because decode can't work on shared array + # for some silly reason + json_str = decoder.decode(byte_buffer.slice(0, string_len)) + # get it as an object + response_obj = json.loads(json_str) + return EmscriptenResponse( + status_code=response_obj["status"], + headers=response_obj["headers"], + body=io.BufferedReader( + _ReadStream( + int_buffer, + byte_buffer, + request.timeout, + self._worker, + response_obj["connectionID"], + ), + buffer_size=1048576, + ), + ) + if int_buffer[0] == ERROR_EXCEPTION: + string_len = int_buffer[1] + # decode the error string + decoder = js.TextDecoder.new() + json_str = decoder.decode(byte_buffer.slice(0, string_len)) + from ._core import _StreamingError + + raise _StreamingError( + f"Exception thrown in fetch: {json_str}", request=request, response=None + ) + + +# check if we are in a worker or not +def is_in_browser_main_thread() -> bool: + return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window + +def is_cross_origin_isolated(): + print("COI:",js.crossOriginIsolated) + return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated + + +def is_in_node(): + return ( + hasattr(js, "process") + and hasattr(js.process, "release") + and hasattr(js.process.release, "name") + and js.process.release.name == "node" + ) + +def is_worker_available(): + return hasattr(js,"Worker") and hasattr(js,"Blob") + +if (is_worker_available() and ((is_cross_origin_isolated() and not is_in_browser_main_thread()) or is_in_node())): + _fetcher = _StreamingFetcher() +else: + _fetcher = None + +def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse: + if _fetcher: + return _fetcher.send(request) + else: + _show_streaming_warning() + return None + + +_SHOWN_WARNING = False + + +def _show_streaming_warning(): + global _SHOWN_WARNING + if not _SHOWN_WARNING: + _SHOWN_WARNING = True + message = "Can't stream HTTP requests because: \n" + if not is_cross_origin_isolated(): + message += " Page is not cross-origin isolated\n" + if is_in_browser_main_thread(): + message += " Python is running in main browser thread\n" + if not is_worker_available(): + message += " Worker or Blob classes are not available in this environment." + from js import console + + console.warn(message) + + +def send_request(request:EmscriptenRequest)->EmscriptenResponse: + xhr = js.XMLHttpRequest.new() + xhr.timeout = int(request.timeout * 1000) + + if not is_in_browser_main_thread(): + xhr.responseType = "arraybuffer" + else: + xhr.overrideMimeType("text/plain; charset=ISO-8859-15") + + xhr.open(request.method, request.url, False) + for name, value in request.headers.items(): + if name.lower() not in HEADERS_TO_IGNORE: + xhr.setRequestHeader(name, value) + + xhr.send(to_js(request.body)) + + headers = dict(Parser().parsestr(xhr.getAllResponseHeaders())) + + if not is_in_browser_main_thread(): + body = xhr.response.to_py().tobytes() + else: + body = xhr.response.encode("ISO-8859-15") + return EmscriptenResponse(status_code=xhr.status, headers=headers, body=body) diff --git a/src/urllib3/contrib/emscripten/request.py b/src/urllib3/contrib/emscripten/request.py new file mode 100644 index 0000000000..d0fe61e323 --- /dev/null +++ b/src/urllib3/contrib/emscripten/request.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass,field +from typing import Dict + +@dataclass +class EmscriptenRequest: + method: str + url: str + params: dict[str, str]|None = None + body: bytes|None = None + headers: dict[str, str] = field(default_factory=dict) + timeout: int = 0 + + def set_header(self, name: str, value: str): + self.headers[name.capitalize()] = value + + def set_body(self, body: bytes): + self.body = body + + def set_json(self, body: dict): + self.set_header("Content-Type", "application/json; charset=utf-8") + self.set_body(json.dumps(body).encode("utf-8")) + + diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py new file mode 100644 index 0000000000..e3f8225722 --- /dev/null +++ b/src/urllib3/contrib/emscripten/response.py @@ -0,0 +1,85 @@ +from dataclasses import dataclass +from io import IOBase,BytesIO +from itertools import takewhile +import typing + +from ...connection import HTTPConnection +from ...response import HTTPResponse +from ...util.retry import Retry + +@dataclass +class EmscriptenResponse: + status_code: int + headers: dict[str, str] + body: IOBase | bytes + + +class EmscriptenHttpResponseWrapper(HTTPResponse): + def __init__(self, internal_response: EmscriptenResponse, url: str = None, connection=None): + self._response = internal_response + self._url = url + self._connection = connection + super().__init__( + headers=internal_response.headers, + status=internal_response.status_code, + request_url=url, + version=0, + reason="", + decode_content=True + ) + + @property + def url(self) -> str | None: + return self._url + + @url.setter + def url(self, url: str | None) -> None: + self._url = url + + @property + def connection(self) -> HTTPConnection | None: + return self._connection + + @property + def retries(self) -> Retry | None: + return self._retries + + @retries.setter + def retries(self, retries: Retry | None) -> None: + # Override the request_url if retries has a redirect location. + if retries is not None and retries.history: + self.url = retries.history[-1].redirect_location + self._retries = retries + + def read( + self, + amt: int | None = None, + decode_content: bool | None = None, + cache_content: bool = False, + ) -> bytes: + if not isinstance(self._response.body,IOBase): + # wrap body in IOStream + self._response.body=BytesIO(self._response.body) + return self._response.body.read(amt) + + def read_chunked( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> typing.Iterator[bytes]: + return self.read(amt,decode_content) + + def release_conn(self) -> None: + if not self._pool or not self._connection: + return None + + self._pool._put_conn(self._connection) + self._connection = None + + def drain_conn(self) -> None: + self.close() + + def close(self) -> None: + if isinstance(self._response.body,IOBase): + self._response.body.close() + diff --git a/test/with_dummyserver/emscripten_fixtures.py b/test/with_dummyserver/emscripten_fixtures.py new file mode 100644 index 0000000000..82f90b0cd3 --- /dev/null +++ b/test/with_dummyserver/emscripten_fixtures.py @@ -0,0 +1,159 @@ +import asyncio +import contextlib +import os + +from urllib.parse import urlsplit +import textwrap +import mimetypes +import pytest +from tornado import web +from dummyserver.server import ( + run_loop_in_thread, + run_tornado_app, +) + +from pathlib import Path +from dummyserver.handlers import TestingApp +from dummyserver.handlers import Response +from dummyserver.testcase import HTTPDummyProxyTestCase + + +@pytest.fixture(scope="module") +def testserver_http(request): + dist_dir = Path(os.getcwd(), request.config.getoption("--dist-dir")) + server = PyodideDummyServerTestCase + server.setup_class(dist_dir) + print( + f"Server:{server.http_host}:{server.http_port},https({server.https_port}) [{dist_dir}]" + ) + yield server + print("Server teardown") + server.teardown_class() + + +class _FromServerRunner: + def __init__(self, host, port, selenium): + self.host = host + self.port = port + self.selenium = selenium + + def run_webworker(self, code): + if isinstance(code, str) and code.startswith("\n"): + # we have a multiline string, fix indentation + code = textwrap.dedent(code) + + return self.selenium.run_js( + """ + let worker = new Worker('{}'); + let p = new Promise((res, rej) => {{ + worker.onerror = e => res(e); + worker.onmessage = e => {{ + if (e.data.results) {{ + res(e.data.results); + }} else {{ + res(e.data.error); + }} + }}; + worker.postMessage({{ python: {!r} }}); + }}); + return await p + """.format( + f"https://{self.host}:{self.port}/pyodide/webworker_dev.js", + code, + ), + pyodide_checks=False, + ) + +# run pyodide on our test server instead of on the default +# pytest-pyodide one - this makes it so that +# we are at the same origin as web requests to server_host +@pytest.fixture() +def run_from_server(selenium, testserver_http): + addr = f"https://{testserver_http.http_host}:{testserver_http.https_port}/pyodide/test.html" + selenium.goto(addr) +# import time +# time.sleep(100) + selenium.javascript_setup() + selenium.load_pyodide() + selenium.initialize_pyodide() + selenium.save_state() + selenium.restore_state() + # install the wheel, which is served at /wheel/* + selenium.run_js( + """ +await pyodide.loadPackage('/wheel/dist.whl') +""" + ) + yield _FromServerRunner( + testserver_http.http_host, testserver_http.https_port, selenium + ) + + +class PyodideTestingApp(TestingApp): + pyodide_dist_dir: str = "" + + def set_default_headers(self) -> None: + """Allow cross-origin requests for emscripten""" + self.set_header("Access-Control-Allow-Origin", "*") + self.set_header("Cross-Origin-Opener-Policy", "same-origin") + self.set_header("Cross-Origin-Embedder-Policy", "require-corp") + self.add_header("Feature-Policy", "sync-xhr *;") + + def bigfile(self,req): + print("Bigfile requested") + return Response(b"WOOO YAY BOOYAKAH") + + def pyodide(self, req): + path = req.path[:] + if not path.startswith("/"): + path = urlsplit(path).path + path = path.split("/") + file_path = Path(PyodideTestingApp.pyodide_dist_dir, *path[2:]) + if file_path.exists(): + mime_type, encoding = mimetypes.guess_type(file_path) + print(file_path,mime_type) + if not mime_type: + mime_type = "text/plain" + self.set_header("Content-Type",mime_type) + return Response( + body=file_path.read_bytes(), headers=[("Access-Control-Allow-Origin", "*")] + ) + else: + return Response(status=404) + + def wheel(self, req): + # serve our wheel + wheel_folder = Path(__file__).parent.parent.parent / "dist" + print(wheel_folder) + wheels = list(wheel_folder.glob("*.whl")) + print(wheels) + if len(wheels) > 0: + resp = Response( + body=wheels[0].read_bytes(), + headers=[ + ("Content-Disposition", f"inline; filename='{wheels[0].name}'") + ], + ) + return resp + + +class PyodideDummyServerTestCase(HTTPDummyProxyTestCase): + @classmethod + def setup_class(cls, pyodide_dist_dir) -> None: + PyodideTestingApp.pyodide_dist_dir = pyodide_dist_dir + with contextlib.ExitStack() as stack: + io_loop = stack.enter_context(run_loop_in_thread()) + + async def run_app() -> None: + app = web.Application([(r".*", PyodideTestingApp)]) + cls.http_server, cls.http_port = run_tornado_app( + app, None, "http", cls.http_host + ) + + app = web.Application([(r".*", PyodideTestingApp)]) + cls.https_server, cls.https_port = run_tornado_app( + app, cls.https_certs, "https", cls.http_host + ) + + asyncio.run_coroutine_threadsafe(run_app(), io_loop.asyncio_loop).result() # type: ignore[attr-defined] + cls._stack = stack.pop_all() diff --git a/test/with_dummyserver/test_emscripten.py b/test/with_dummyserver/test_emscripten.py new file mode 100644 index 0000000000..f0c904b155 --- /dev/null +++ b/test/with_dummyserver/test_emscripten.py @@ -0,0 +1,156 @@ +import pytest_pyodide +pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") + +from pytest_pyodide import run_in_pyodide +from .emscripten_fixtures import testserver_http,run_from_server +from pytest_pyodide.decorator import copy_files_to_pyodide + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) +def test_index(selenium, testserver_http): + @run_in_pyodide + def pyodide_test(selenium, host, port): + import urllib3.contrib.emscripten + from urllib3.response import HTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + method = "GET" + path = "/" + url = f"http://{host}:{port}{path}" + conn.request(method, url) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + data=response.data + assert data.decode("utf-8") == "Dummy server!" + + pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) +def test_index_https(selenium, testserver_http): + @run_in_pyodide + def pyodide_test(selenium, host, port): + from urllib3.response import HTTPResponse + from urllib3.connection import HTTPSConnection + + conn = HTTPSConnection(host, port) + method = "GET" + path = "/" + url = f"https://{host}:{port}{path}" + conn.request(method, url) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + data=response.data + assert data.decode("utf-8") == "Dummy server!" + + pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + + +def test_specific_method(selenium, testserver_http,run_from_server): + print("Running from server") + @run_in_pyodide + def pyodide_test(selenium, host, port): + from urllib3 import HTTPConnectionPool + from urllib3.response import HTTPResponse + with HTTPConnectionPool(host, port) as pool: + method = "POST" + path = "/specific_method?method=POST" + response = pool.request(method,path) + assert isinstance(response, HTTPResponse) + assert(response.status==200) + + method = "PUT" + path = "/specific_method?method=POST" + response = pool.request(method,path) + assert isinstance(response, HTTPResponse) + assert(response.status==400) + + pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + +def test_upload(selenium, testserver_http,run_from_server): + @run_in_pyodide + def pyodide_test(selenium, host, port): + from urllib3 import HTTPConnectionPool + + data = "I'm in ur multipart form-data, hazing a cheezburgr" + fields: dict[str, _TYPE_FIELD_VALUE_TUPLE] = { + "upload_param": "filefield", + "upload_filename": "lolcat.txt", + "filefield": ("lolcat.txt", data), + } + fields["upload_size"] = len(data) # type: ignore + with HTTPConnectionPool(host, port) as pool: + r = pool.request("POST", "/upload", fields=fields) + assert r.status == 200, r.data + + pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) +def test_index_https(selenium, testserver_http): + @run_in_pyodide + def pyodide_test(selenium, host, port): + from urllib3.response import HTTPResponse + from urllib3.connection import HTTPSConnection + + conn = HTTPSConnection(host, port) + method = "GET" + path = "/" + url = f"https://{host}:{port}{path}" + conn.request(method, url) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + data=response.data + assert data.decode("utf-8") == "Dummy server!" + + pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + + +def test_specific_method(selenium, testserver_http,run_from_server): + print("Running from server") + @run_in_pyodide + def pyodide_test(selenium, host, port): + from urllib3 import HTTPConnectionPool + from urllib3.response import HTTPResponse + with HTTPConnectionPool(host, port) as pool: + method = "POST" + path = "/specific_method?method=POST" + response = pool.request(method,path) + assert isinstance(response, HTTPResponse) + assert(response.status==200) + + method = "PUT" + path = "/specific_method?method=POST" + response = pool.request(method,path) + assert isinstance(response, HTTPResponse) + assert(response.status==400) + + pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) +def test_streaming_download(selenium, testserver_http,run_from_server): + # test streaming download, which must be in a webworker + # as you can't do it on main thread + worker_code = f""" + try: + import micropip + await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) + import urllib3.contrib.emscripten + from urllib3.response import HTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.https_port}) + method = "GET" + url = "https://{testserver_http.http_host}:{testserver_http.https_port}/bigfile" + conn.request(method, url) + #response = conn.getresponse() + # assert isinstance(response, HTTPResponse) + # data=response.data + 1 + except BaseException as e: + str(e) + """ +# import time +# time.sleep(100) +# selenium.driver.implicitly_wait(100) + result=run_from_server.run_webworker(worker_code) + print(result) +# run_from_server.run_webworker(worker_code) From 2a77ea86318902c133e863010ddb1b171e68db4a Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Wed, 15 Nov 2023 12:42:15 +0000 Subject: [PATCH 02/49] streaming works and tests okay --- src/urllib3/contrib/emscripten/__init__.py | 14 +- src/urllib3/contrib/emscripten/connection.py | 4 +- src/urllib3/contrib/emscripten/fetch.py | 116 +++++++++---- src/urllib3/contrib/emscripten/request.py | 9 +- src/urllib3/contrib/emscripten/response.py | 18 +- test/with_dummyserver/emscripten_fixtures.py | 40 +++-- test/with_dummyserver/test_emscripten.py | 167 ++++++++++++++----- 7 files changed, 265 insertions(+), 103 deletions(-) diff --git a/src/urllib3/contrib/emscripten/__init__.py b/src/urllib3/contrib/emscripten/__init__.py index 8d8323dacf..4e42c6ac4d 100644 --- a/src/urllib3/contrib/emscripten/__init__.py +++ b/src/urllib3/contrib/emscripten/__init__.py @@ -1,8 +1,10 @@ -from .connection import EmscriptenHTTPConnection,EmscriptenHTTPSConnection -from ...connectionpool import HTTPConnectionPool,HTTPSConnectionPool -HTTPConnectionPool.ConnectionCls=EmscriptenHTTPConnection -HTTPSConnectionPool.ConnectionCls=EmscriptenHTTPSConnection +from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection +from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool + +HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection +HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection import urllib3.connection -urllib3.connection.HTTPConnection=EmscriptenHTTPConnection -urllib3.connection.HTTPSConnection=EmscriptenHTTPSConnection + +urllib3.connection.HTTPConnection = EmscriptenHTTPConnection +urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index ce79af3ccd..a2c3753670 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -76,7 +76,9 @@ def request( if headers: for k, v in headers.items(): request.set_header(k, v) - self._response = send_streaming_request(request) + self._response = None + if not preload_content: + self._response = send_streaming_request(request) if self._response == None: self._response = send_request(request) diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index 53f76c2fcd..d48015c63d 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -21,6 +21,7 @@ import json import js from pyodide.ffi import to_js +from pyodide.code import run_js from .request import EmscriptenRequest from .response import EmscriptenResponse @@ -37,6 +38,23 @@ ERROR_TIMEOUT = -3 ERROR_EXCEPTION = -4 + +class _RequestError(Exception): + def __init__(self, message=None, *, request=None, response=None): + self.request = request + self.response = response + self.message = message + super().__init__(self.message) + + +class _StreamingError(_RequestError): + pass + + +class _StreamingTimeout(_StreamingError): + pass + + _STREAMING_WORKER_CODE = """ let SUCCESS_HEADER = -1 let SUCCESS_EOF = -2 @@ -45,6 +63,7 @@ let connections = {}; let nextConnectionID = 1; + self.addEventListener("message", async function (event) { if(event.data.close) { @@ -91,6 +110,7 @@ curLen = byteBuffer.length; } byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0) + Atomics.store(intBuffer, 0, curLen);// store current length in bytes Atomics.notify(intBuffer, 0); curOffset+=curLen; @@ -136,6 +156,7 @@ } } }); +self.postMessage({inited:true}) """ @@ -152,15 +173,16 @@ def __init__(self, int_buffer, byte_buffer, timeout, worker, connection_id): self.connection_id = connection_id self.worker = worker self.timeout = int(1000 * timeout) if timeout > 0 else None - self.closed = False + self.is_live = True def __del__(self): self.close() def close(self): - if not self.closed: + if self.is_live: self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) - self.closed = True + self.is_live = False + super().close() def readable(self) -> bool: return True @@ -179,16 +201,12 @@ def readinto(self, byte_obj) -> bool: js.Atomics.store(self.int_buffer, 0, 0) self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id})) if js.Atomics.wait(self.int_buffer, 0, 0, self.timeout) == "timed-out": - from ._core import _StreamingTimeout - raise _StreamingTimeout data_len = self.int_buffer[0] if data_len > 0: self.read_len = data_len self.read_pos = 0 elif data_len == ERROR_EXCEPTION: - from ._core import _StreamingError - raise _StreamingError else: # EOF, free the buffers and return zero @@ -199,9 +217,10 @@ def readinto(self, byte_obj) -> bool: return 0 # copy from int32array to python bytes ret_length = min(self.read_len, len(byte_obj)) - self.byte_buffer.subarray(self.read_pos, self.read_pos + ret_length).assign_to( - byte_obj[0:ret_length] - ) + subarray = self.byte_buffer.subarray( + self.read_pos, self.read_pos + ret_length + ).to_py() + byte_obj[0:ret_length] = subarray self.read_len -= ret_length self.read_pos += ret_length return ret_length @@ -210,13 +229,26 @@ def readinto(self, byte_obj) -> bool: class _StreamingFetcher: def __init__(self): # make web-worker and data buffer on startup - dataBlob = js.globalThis.Blob.new( + self.streaming_ready = False + + dataBlob = js.Blob.new( [_STREAMING_WORKER_CODE], _obj_from_dict({"type": "application/javascript"}) ) - print("Make worker") + + def promise_resolver(res, rej): + def onMsg(e): + self.streaming_ready = True + res(e) + + def onErr(e): + rej(e) + + self._worker.onmessage = onMsg + self._worker.onerror = onErr + dataURL = js.URL.createObjectURL(dataBlob) - self._worker = js.Worker.new(dataURL) - print("Initialized worker") + self._worker = js.globalThis.Worker.new(dataURL) + self._worker_ready_promise = js.globalThis.Promise.new(promise_resolver) def send(self, request): headers = { @@ -234,15 +266,15 @@ def send(self, request): js.Atomics.store(int_buffer, 0, 0) js.Atomics.notify(int_buffer, 0) absolute_url = js.URL.new(request.url, js.location).href - js.console.log( - _obj_from_dict( - { - "buffer": shared_buffer, - "url": absolute_url, - "fetchParams": fetch_data, - } - ) - ) + # js.console.log( + # _obj_from_dict( + # { + # "buffer": shared_buffer, + # "url": absolute_url, + # "fetchParams": fetch_data, + # } + # ) + # ) self._worker.postMessage( _obj_from_dict( { @@ -255,8 +287,6 @@ def send(self, request): # wait for the worker to send something js.Atomics.wait(int_buffer, 0, 0, timeout) if int_buffer[0] == 0: - from ._core import _StreamingTimeout - raise _StreamingTimeout( "Timeout connecting to streaming request", request=request, @@ -292,8 +322,6 @@ def send(self, request): # decode the error string decoder = js.TextDecoder.new() json_str = decoder.decode(byte_buffer.slice(0, string_len)) - from ._core import _StreamingError - raise _StreamingError( f"Exception thrown in fetch: {json_str}", request=request, response=None ) @@ -303,8 +331,8 @@ def send(self, request): def is_in_browser_main_thread() -> bool: return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window + def is_cross_origin_isolated(): - print("COI:",js.crossOriginIsolated) return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated @@ -316,16 +344,21 @@ def is_in_node(): and js.process.release.name == "node" ) + def is_worker_available(): - return hasattr(js,"Worker") and hasattr(js,"Blob") + return hasattr(js, "Worker") and hasattr(js, "Blob") + -if (is_worker_available() and ((is_cross_origin_isolated() and not is_in_browser_main_thread()) or is_in_node())): +if is_worker_available() and ( + (is_cross_origin_isolated() and not is_in_browser_main_thread()) or is_in_node() +): _fetcher = _StreamingFetcher() else: _fetcher = None + def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse: - if _fetcher: + if _fetcher and streaming_ready(): return _fetcher.send(request) else: _show_streaming_warning() @@ -346,12 +379,16 @@ def _show_streaming_warning(): message += " Python is running in main browser thread\n" if not is_worker_available(): message += " Worker or Blob classes are not available in this environment." + if streaming_ready() == False: + message += """ Streaming fetch worker isn't ready. If you want to be sure that streamig fetch +is working, you need to call: 'await urllib3.contrib.emscripten.fetc.wait_for_streaming_ready()`""" from js import console console.warn(message) + print(message) -def send_request(request:EmscriptenRequest)->EmscriptenResponse: +def send_request(request: EmscriptenRequest) -> EmscriptenResponse: xhr = js.XMLHttpRequest.new() xhr.timeout = int(request.timeout * 1000) @@ -374,3 +411,18 @@ def send_request(request:EmscriptenRequest)->EmscriptenResponse: else: body = xhr.response.encode("ISO-8859-15") return EmscriptenResponse(status_code=xhr.status, headers=headers, body=body) + + +def streaming_ready(): + if _fetcher: + return _fetcher.streaming_ready + else: + return None # no fetcher, return None to signify that + + +async def wait_for_streaming_ready(): + if _fetcher: + await _fetcher._worker_ready_promise + return True + else: + return False diff --git a/src/urllib3/contrib/emscripten/request.py b/src/urllib3/contrib/emscripten/request.py index d0fe61e323..2ba53a09ad 100644 --- a/src/urllib3/contrib/emscripten/request.py +++ b/src/urllib3/contrib/emscripten/request.py @@ -1,12 +1,13 @@ -from dataclasses import dataclass,field +from dataclasses import dataclass, field from typing import Dict + @dataclass class EmscriptenRequest: method: str url: str - params: dict[str, str]|None = None - body: bytes|None = None + params: dict[str, str] | None = None + body: bytes | None = None headers: dict[str, str] = field(default_factory=dict) timeout: int = 0 @@ -19,5 +20,3 @@ def set_body(self, body: bytes): def set_json(self, body: dict): self.set_header("Content-Type", "application/json; charset=utf-8") self.set_body(json.dumps(body).encode("utf-8")) - - diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index e3f8225722..e8c92946ab 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from io import IOBase,BytesIO +from io import IOBase, BytesIO from itertools import takewhile import typing @@ -7,6 +7,7 @@ from ...response import HTTPResponse from ...util.retry import Retry + @dataclass class EmscriptenResponse: status_code: int @@ -15,7 +16,9 @@ class EmscriptenResponse: class EmscriptenHttpResponseWrapper(HTTPResponse): - def __init__(self, internal_response: EmscriptenResponse, url: str = None, connection=None): + def __init__( + self, internal_response: EmscriptenResponse, url: str = None, connection=None + ): self._response = internal_response self._url = url self._connection = connection @@ -25,7 +28,7 @@ def __init__(self, internal_response: EmscriptenResponse, url: str = None, conne request_url=url, version=0, reason="", - decode_content=True + decode_content=True, ) @property @@ -57,9 +60,9 @@ def read( decode_content: bool | None = None, cache_content: bool = False, ) -> bytes: - if not isinstance(self._response.body,IOBase): + if not isinstance(self._response.body, IOBase): # wrap body in IOStream - self._response.body=BytesIO(self._response.body) + self._response.body = BytesIO(self._response.body) return self._response.body.read(amt) def read_chunked( @@ -67,7 +70,7 @@ def read_chunked( amt: int | None = None, decode_content: bool | None = None, ) -> typing.Iterator[bytes]: - return self.read(amt,decode_content) + return self.read(amt, decode_content) def release_conn(self) -> None: if not self._pool or not self._connection: @@ -80,6 +83,5 @@ def drain_conn(self) -> None: self.close() def close(self) -> None: - if isinstance(self._response.body,IOBase): + if isinstance(self._response.body, IOBase): self._response.body.close() - diff --git a/test/with_dummyserver/emscripten_fixtures.py b/test/with_dummyserver/emscripten_fixtures.py index 82f90b0cd3..a906998cef 100644 --- a/test/with_dummyserver/emscripten_fixtures.py +++ b/test/with_dummyserver/emscripten_fixtures.py @@ -46,17 +46,18 @@ def run_webworker(self, code): """ let worker = new Worker('{}'); let p = new Promise((res, rej) => {{ - worker.onerror = e => res(e); + worker.onmessageerror = e => rej(e); + worker.onerror = e => rej(e); worker.onmessage = e => {{ if (e.data.results) {{ res(e.data.results); }} else {{ - res(e.data.error); + rej(e.data.error); }} }}; worker.postMessage({{ python: {!r} }}); }}); - return await p + return await p; """.format( f"https://{self.host}:{self.port}/pyodide/webworker_dev.js", code, @@ -64,6 +65,7 @@ def run_webworker(self, code): pyodide_checks=False, ) + # run pyodide on our test server instead of on the default # pytest-pyodide one - this makes it so that # we are at the same origin as web requests to server_host @@ -71,8 +73,8 @@ def run_webworker(self, code): def run_from_server(selenium, testserver_http): addr = f"https://{testserver_http.http_host}:{testserver_http.https_port}/pyodide/test.html" selenium.goto(addr) -# import time -# time.sleep(100) + # import time + # time.sleep(100) selenium.javascript_setup() selenium.load_pyodide() selenium.initialize_pyodide() @@ -99,9 +101,12 @@ def set_default_headers(self) -> None: self.set_header("Cross-Origin-Embedder-Policy", "require-corp") self.add_header("Feature-Policy", "sync-xhr *;") - def bigfile(self,req): + def bigfile(self, req): print("Bigfile requested") - return Response(b"WOOO YAY BOOYAKAH") + # great big text file, should force streaming + # if supported + bigdata = 1048576 * b"WOOO YAY BOOYAKAH" + return Response(bigdata) def pyodide(self, req): path = req.path[:] @@ -111,16 +116,31 @@ def pyodide(self, req): file_path = Path(PyodideTestingApp.pyodide_dist_dir, *path[2:]) if file_path.exists(): mime_type, encoding = mimetypes.guess_type(file_path) - print(file_path,mime_type) + print(file_path, mime_type) if not mime_type: mime_type = "text/plain" - self.set_header("Content-Type",mime_type) + self.set_header("Content-Type", mime_type) return Response( - body=file_path.read_bytes(), headers=[("Access-Control-Allow-Origin", "*")] + body=file_path.read_bytes(), + headers=[("Access-Control-Allow-Origin", "*")], ) else: return Response(status=404) + def worker_template(self, req): + return Response( + """ + + + + + worker loader""", + headers=[("Content-type", "text/html")], + ) + return + def wheel(self, req): # serve our wheel wheel_folder = Path(__file__).parent.parent.parent / "dist" diff --git a/test/with_dummyserver/test_emscripten.py b/test/with_dummyserver/test_emscripten.py index f0c904b155..630a9be31d 100644 --- a/test/with_dummyserver/test_emscripten.py +++ b/test/with_dummyserver/test_emscripten.py @@ -1,10 +1,12 @@ import pytest_pyodide + pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") from pytest_pyodide import run_in_pyodide -from .emscripten_fixtures import testserver_http,run_from_server +from .emscripten_fixtures import testserver_http, run_from_server from pytest_pyodide.decorator import copy_files_to_pyodide + @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) def test_index(selenium, testserver_http): @run_in_pyodide @@ -20,11 +22,12 @@ def pyodide_test(selenium, host, port): conn.request(method, url) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data=response.data + data = response.data assert data.decode("utf-8") == "Dummy server!" pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) def test_index_https(selenium, testserver_http): @run_in_pyodide @@ -39,34 +42,83 @@ def pyodide_test(selenium, host, port): conn.request(method, url) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data=response.data + data = response.data + assert data.decode("utf-8") == "Dummy server!" + + pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) +def test_non_streaming_no_fallback_warning(selenium, testserver_http): + @run_in_pyodide + def pyodide_test(selenium, host, port): + import urllib3.contrib.emscripten.fetch + from urllib3.response import HTTPResponse + from urllib3.connection import HTTPSConnection + + conn = HTTPSConnection(host, port) + method = "GET" + path = "/" + url = f"https://{host}:{port}{path}" + conn.request(method, url, preload_content=True) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + data = response.data + assert data.decode("utf-8") == "Dummy server!" + # no console warnings because we didn't ask it to stream the response + assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING == False + + pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) +def test_streaming_fallback_warning(selenium, testserver_http): + @run_in_pyodide + def pyodide_test(selenium, host, port): + import urllib3.contrib.emscripten.fetch + from urllib3.response import HTTPResponse + from urllib3.connection import HTTPSConnection + + conn = HTTPSConnection(host, port) + method = "GET" + path = "/" + url = f"https://{host}:{port}{path}" + conn.request(method, url, preload_content=False) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + data = response.data assert data.decode("utf-8") == "Dummy server!" + # check that it has warned about falling back to non-streaming fetch + assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING == True pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -def test_specific_method(selenium, testserver_http,run_from_server): +def test_specific_method(selenium, testserver_http, run_from_server): print("Running from server") + @run_in_pyodide def pyodide_test(selenium, host, port): from urllib3 import HTTPConnectionPool from urllib3.response import HTTPResponse + with HTTPConnectionPool(host, port) as pool: method = "POST" path = "/specific_method?method=POST" - response = pool.request(method,path) + response = pool.request(method, path) assert isinstance(response, HTTPResponse) - assert(response.status==200) + assert response.status == 200 method = "PUT" path = "/specific_method?method=POST" - response = pool.request(method,path) + response = pool.request(method, path) assert isinstance(response, HTTPResponse) - assert(response.status==400) + assert response.status == 400 pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -def test_upload(selenium, testserver_http,run_from_server): + +def test_upload(selenium, testserver_http, run_from_server): @run_in_pyodide def pyodide_test(selenium, host, port): from urllib3 import HTTPConnectionPool @@ -84,6 +136,7 @@ def pyodide_test(selenium, host, port): pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) def test_index_https(selenium, testserver_http): @run_in_pyodide @@ -98,59 +151,91 @@ def pyodide_test(selenium, host, port): conn.request(method, url) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data=response.data + data = response.data assert data.decode("utf-8") == "Dummy server!" pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -def test_specific_method(selenium, testserver_http,run_from_server): +def test_specific_method(selenium, testserver_http, run_from_server): print("Running from server") + @run_in_pyodide def pyodide_test(selenium, host, port): from urllib3 import HTTPConnectionPool from urllib3.response import HTTPResponse + with HTTPConnectionPool(host, port) as pool: method = "POST" path = "/specific_method?method=POST" - response = pool.request(method,path) + response = pool.request(method, path) assert isinstance(response, HTTPResponse) - assert(response.status==200) + assert response.status == 200 method = "PUT" path = "/specific_method?method=POST" - response = pool.request(method,path) + response = pool.request(method, path) assert isinstance(response, HTTPResponse) - assert(response.status==400) + assert response.status == 400 pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_streaming_download(selenium, testserver_http,run_from_server): +def test_streaming_download(selenium, testserver_http, run_from_server): # test streaming download, which must be in a webworker # as you can't do it on main thread - worker_code = f""" - try: - import micropip - await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) - import urllib3.contrib.emscripten - from urllib3.response import HTTPResponse - from urllib3.connection import HTTPConnection - - conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.https_port}) - method = "GET" - url = "https://{testserver_http.http_host}:{testserver_http.https_port}/bigfile" - conn.request(method, url) - #response = conn.getresponse() - # assert isinstance(response, HTTPResponse) - # data=response.data - 1 - except BaseException as e: - str(e) - """ -# import time -# time.sleep(100) -# selenium.driver.implicitly_wait(100) - result=run_from_server.run_webworker(worker_code) - print(result) -# run_from_server.run_webworker(worker_code) + + # this should return the 17mb big file, and + # should not log any warning about falling back + bigfile_url = ( + url + ) = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + worker_code = f"""import micropip +await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) +import urllib3.contrib.emscripten.fetch +await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() +from urllib3.response import HTTPResponse +from urllib3.connection import HTTPConnection +import js + +conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) +method = "GET" +url = "{bigfile_url}" +conn.request(method, url,preload_content=False) +response = conn.getresponse() +assert isinstance(response, HTTPResponse) +assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==False +data=response.data.decode('utf-8') +data +""" + result = run_from_server.run_webworker(worker_code) + assert len(result) == 17825792 + + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) +def test_streaming_notready_warning(selenium, testserver_http, run_from_server): + # test streaming download but don't wait for + # worker to be ready - should fallback to non-streaming + # and log a warning + bigfile_url = ( + url + ) = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + worker_code = f"""import micropip +await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) +import urllib3.contrib.emscripten.fetch +from urllib3.response import HTTPResponse +from urllib3.connection import HTTPConnection + +conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) +method = "GET" +url = "{bigfile_url}" +conn.request(method, url,preload_content=False) +response = conn.getresponse() +assert isinstance(response, HTTPResponse) +data=response.data.decode('utf-8') +assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==True +data +""" + result = run_from_server.run_webworker(worker_code) + assert len(result) == 17825792 From e6e9bb4f19f486943fa66910b08ecb130ed1d4ac Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Wed, 15 Nov 2023 13:21:32 +0000 Subject: [PATCH 03/49] made emscripten tests skip if pytest-pyodide is not installed --- test/with_dummyserver/test_emscripten.py | 86 +++++++++++------------- 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/test/with_dummyserver/test_emscripten.py b/test/with_dummyserver/test_emscripten.py index 630a9be31d..e84f9d6a92 100644 --- a/test/with_dummyserver/test_emscripten.py +++ b/test/with_dummyserver/test_emscripten.py @@ -1,12 +1,14 @@ -import pytest_pyodide - +import pytest +# only run these tests if pytest_pyodide is installed +# so we don't break non-emscripten pytest running +pytest_pyodide=pytest.importorskip("pytest_pyodide") +# hack to make our ssl certificates work in chrome pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") from pytest_pyodide import run_in_pyodide -from .emscripten_fixtures import testserver_http, run_from_server +from .emscripten_fixtures import testserver_http,run_from_server from pytest_pyodide.decorator import copy_files_to_pyodide - @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) def test_index(selenium, testserver_http): @run_in_pyodide @@ -22,12 +24,12 @@ def pyodide_test(selenium, host, port): conn.request(method, url) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data = response.data + data=response.data assert data.decode("utf-8") == "Dummy server!" + 1 pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) - @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) def test_index_https(selenium, testserver_http): @run_in_pyodide @@ -42,12 +44,12 @@ def pyodide_test(selenium, host, port): conn.request(method, url) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data = response.data + data=response.data assert data.decode("utf-8") == "Dummy server!" + 1 pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) - @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) def test_non_streaming_no_fallback_warning(selenium, testserver_http): @run_in_pyodide @@ -60,13 +62,13 @@ def pyodide_test(selenium, host, port): method = "GET" path = "/" url = f"https://{host}:{port}{path}" - conn.request(method, url, preload_content=True) + conn.request(method, url,preload_content=True) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data = response.data + data=response.data assert data.decode("utf-8") == "Dummy server!" # no console warnings because we didn't ask it to stream the response - assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING == False + assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==False pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) @@ -83,42 +85,40 @@ def pyodide_test(selenium, host, port): method = "GET" path = "/" url = f"https://{host}:{port}{path}" - conn.request(method, url, preload_content=False) + conn.request(method, url,preload_content=False) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data = response.data + data=response.data assert data.decode("utf-8") == "Dummy server!" # check that it has warned about falling back to non-streaming fetch - assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING == True + assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==True pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -def test_specific_method(selenium, testserver_http, run_from_server): - print("Running from server") +def test_specific_method(selenium, testserver_http,run_from_server): + print("Running from server") @run_in_pyodide def pyodide_test(selenium, host, port): from urllib3 import HTTPConnectionPool from urllib3.response import HTTPResponse - with HTTPConnectionPool(host, port) as pool: method = "POST" path = "/specific_method?method=POST" - response = pool.request(method, path) + response = pool.request(method,path) assert isinstance(response, HTTPResponse) - assert response.status == 200 + assert(response.status==200) method = "PUT" path = "/specific_method?method=POST" - response = pool.request(method, path) + response = pool.request(method,path) assert isinstance(response, HTTPResponse) - assert response.status == 400 + assert(response.status==400) pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) - -def test_upload(selenium, testserver_http, run_from_server): +def test_upload(selenium, testserver_http,run_from_server): @run_in_pyodide def pyodide_test(selenium, host, port): from urllib3 import HTTPConnectionPool @@ -136,7 +136,6 @@ def pyodide_test(selenium, host, port): pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) - @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) def test_index_https(selenium, testserver_http): @run_in_pyodide @@ -151,46 +150,41 @@ def pyodide_test(selenium, host, port): conn.request(method, url) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data = response.data + data=response.data assert data.decode("utf-8") == "Dummy server!" pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -def test_specific_method(selenium, testserver_http, run_from_server): +def test_specific_method(selenium, testserver_http,run_from_server): print("Running from server") - @run_in_pyodide def pyodide_test(selenium, host, port): from urllib3 import HTTPConnectionPool from urllib3.response import HTTPResponse - with HTTPConnectionPool(host, port) as pool: method = "POST" path = "/specific_method?method=POST" - response = pool.request(method, path) + response = pool.request(method,path) assert isinstance(response, HTTPResponse) - assert response.status == 200 + assert(response.status==200) method = "PUT" path = "/specific_method?method=POST" - response = pool.request(method, path) + response = pool.request(method,path) assert isinstance(response, HTTPResponse) - assert response.status == 400 + assert(response.status==400) pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) - @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_streaming_download(selenium, testserver_http, run_from_server): +def test_streaming_download(selenium, testserver_http,run_from_server): # test streaming download, which must be in a webworker # as you can't do it on main thread # this should return the 17mb big file, and # should not log any warning about falling back - bigfile_url = ( - url - ) = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + bigfile_url=url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" worker_code = f"""import micropip await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) import urllib3.contrib.emscripten.fetch @@ -209,18 +203,15 @@ def test_streaming_download(selenium, testserver_http, run_from_server): data=response.data.decode('utf-8') data """ - result = run_from_server.run_webworker(worker_code) - assert len(result) == 17825792 - + result=run_from_server.run_webworker(worker_code) + assert(len(result)==17825792) @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_streaming_notready_warning(selenium, testserver_http, run_from_server): - # test streaming download but don't wait for +def test_streaming_notready_warning(selenium, testserver_http,run_from_server): + # test streaming download but don't wait for # worker to be ready - should fallback to non-streaming # and log a warning - bigfile_url = ( - url - ) = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + bigfile_url=url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" worker_code = f"""import micropip await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) import urllib3.contrib.emscripten.fetch @@ -237,5 +228,6 @@ def test_streaming_notready_warning(selenium, testserver_http, run_from_server): assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==True data """ - result = run_from_server.run_webworker(worker_code) - assert len(result) == 17825792 + result=run_from_server.run_webworker(worker_code) + assert(len(result)==17825792) + From 0cc345cd0f60bb6f6d7041ecf53888e7b7c2f03e Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Wed, 15 Nov 2023 19:50:54 +0000 Subject: [PATCH 04/49] tests --- noxfile.py | 34 +++++++++++++++++++++++- test/with_dummyserver/test_emscripten.py | 4 +-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index c0b0b9f0e6..40da4ae825 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,7 +3,7 @@ import os import shutil import sys - +from pathlib import Path import nox @@ -11,6 +11,7 @@ def tests_impl( session: nox.Session, extras: str = "socks,brotli,zstd", byte_string_comparisons: bool = True, + pytest_extra_args: list[str] = [] ) -> None: # Install deps and the package itself. session.install("-r", "dev-requirements.txt") @@ -50,6 +51,7 @@ def tests_impl( "--strict-config", "--strict-markers", *(session.posargs or ("test/",)), + *pytest_extra_args, env={"PYTHONWARNINGS": "always::DeprecationWarning"}, ) @@ -142,6 +144,36 @@ def lint(session: nox.Session) -> None: mypy(session) +@nox.session(python="3.11") +def emscripten(session: nox.Session) -> None: + """install emscripten extras""" + session.install("build") + # build wheel into dist folder + session.run("python", "-m", "build") + # make sure we have a dist dir for pyodide + dist_dir = None + if "PYODIDE_ROOT" in os.environ: + # we have a pyodide build tree checked out + # use the dist directory from that + dist_dir = Path(os.environ["PYODIDE_ROOT"]) + else: + + # we don't have a build tree, get one + # that matches the version of pyodide build + import pyodide_build + pyodide_version=pyodide_build.__version__ + pyodide_artifacts_path = (Path(session.cache_dir) / f"pyodide-{pyodide_version}") + if not pyodide_artifacts_path.exists(): + print("Fetching pyodide build artifacts") + session.run("wget",f"https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/pyodide-{pyodide_version}.tar.bz2","-O",f"{pyodide_artifacts_path}.tar.bz2") + pyodide_artifacts_path.mkdir(parents=True) + session.run("tar","-xjf",f"{pyodide_artifacts_path}.tar.bz2","-C",pyodide_artifacts_path,"--strip-components","1") + + dist_dir=pyodide_artifacts_path + assert dist_dir != None + assert dist_dir.exists() + tests_impl(session,"emscripten-test",pytest_extra_args=["--rt","chrome-no-host","--dist-dir",dist_dir,"test"]) + @nox.session(python="3.12") def mypy(session: nox.Session) -> None: """Run mypy.""" diff --git a/test/with_dummyserver/test_emscripten.py b/test/with_dummyserver/test_emscripten.py index e84f9d6a92..8c2957405b 100644 --- a/test/with_dummyserver/test_emscripten.py +++ b/test/with_dummyserver/test_emscripten.py @@ -2,7 +2,7 @@ # only run these tests if pytest_pyodide is installed # so we don't break non-emscripten pytest running pytest_pyodide=pytest.importorskip("pytest_pyodide") -# hack to make our ssl certificates work in chrome +# make our ssl certificates work in chrome pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") from pytest_pyodide import run_in_pyodide @@ -210,7 +210,7 @@ def test_streaming_download(selenium, testserver_http,run_from_server): def test_streaming_notready_warning(selenium, testserver_http,run_from_server): # test streaming download but don't wait for # worker to be ready - should fallback to non-streaming - # and log a warning + # and log a warning bigfile_url=url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" worker_code = f"""import micropip await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) From ec011f26756bbf0a2020e61167916c61528330c2 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Wed, 15 Nov 2023 19:50:54 +0000 Subject: [PATCH 05/49] tests --- noxfile.py | 34 +++++++++++++++++++++++- pyproject.toml | 4 +++ test/with_dummyserver/test_emscripten.py | 4 +-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index c0b0b9f0e6..40da4ae825 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,7 +3,7 @@ import os import shutil import sys - +from pathlib import Path import nox @@ -11,6 +11,7 @@ def tests_impl( session: nox.Session, extras: str = "socks,brotli,zstd", byte_string_comparisons: bool = True, + pytest_extra_args: list[str] = [] ) -> None: # Install deps and the package itself. session.install("-r", "dev-requirements.txt") @@ -50,6 +51,7 @@ def tests_impl( "--strict-config", "--strict-markers", *(session.posargs or ("test/",)), + *pytest_extra_args, env={"PYTHONWARNINGS": "always::DeprecationWarning"}, ) @@ -142,6 +144,36 @@ def lint(session: nox.Session) -> None: mypy(session) +@nox.session(python="3.11") +def emscripten(session: nox.Session) -> None: + """install emscripten extras""" + session.install("build") + # build wheel into dist folder + session.run("python", "-m", "build") + # make sure we have a dist dir for pyodide + dist_dir = None + if "PYODIDE_ROOT" in os.environ: + # we have a pyodide build tree checked out + # use the dist directory from that + dist_dir = Path(os.environ["PYODIDE_ROOT"]) + else: + + # we don't have a build tree, get one + # that matches the version of pyodide build + import pyodide_build + pyodide_version=pyodide_build.__version__ + pyodide_artifacts_path = (Path(session.cache_dir) / f"pyodide-{pyodide_version}") + if not pyodide_artifacts_path.exists(): + print("Fetching pyodide build artifacts") + session.run("wget",f"https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/pyodide-{pyodide_version}.tar.bz2","-O",f"{pyodide_artifacts_path}.tar.bz2") + pyodide_artifacts_path.mkdir(parents=True) + session.run("tar","-xjf",f"{pyodide_artifacts_path}.tar.bz2","-C",pyodide_artifacts_path,"--strip-components","1") + + dist_dir=pyodide_artifacts_path + assert dist_dir != None + assert dist_dir.exists() + tests_impl(session,"emscripten-test",pytest_extra_args=["--rt","chrome-no-host","--dist-dir",dist_dir,"test"]) + @nox.session(python="3.12") def mypy(session: nox.Session) -> None: """Run mypy.""" diff --git a/pyproject.toml b/pyproject.toml index 141b263a83..a8fd31979a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,10 @@ zstd = [ socks = [ "PySocks>=1.5.6,<2.0,!=1.5.7", ] +emscripten_test = [ + "pytest-pyodide>=0.54.0", + "pyodide-build>=0.24.0" +] [project.urls] "Changelog" = "https://github.com/urllib3/urllib3/blob/main/CHANGES.rst" diff --git a/test/with_dummyserver/test_emscripten.py b/test/with_dummyserver/test_emscripten.py index e84f9d6a92..8c2957405b 100644 --- a/test/with_dummyserver/test_emscripten.py +++ b/test/with_dummyserver/test_emscripten.py @@ -2,7 +2,7 @@ # only run these tests if pytest_pyodide is installed # so we don't break non-emscripten pytest running pytest_pyodide=pytest.importorskip("pytest_pyodide") -# hack to make our ssl certificates work in chrome +# make our ssl certificates work in chrome pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") from pytest_pyodide import run_in_pyodide @@ -210,7 +210,7 @@ def test_streaming_download(selenium, testserver_http,run_from_server): def test_streaming_notready_warning(selenium, testserver_http,run_from_server): # test streaming download but don't wait for # worker to be ready - should fallback to non-streaming - # and log a warning + # and log a warning bigfile_url=url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" worker_code = f"""import micropip await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) From 0819efed9afc332f721d416f573b30873109fc62 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 16 Nov 2023 14:40:55 +0000 Subject: [PATCH 06/49] mypy and lint --- noxfile.py | 47 ++++- src/urllib3/__init__.py | 7 +- src/urllib3/contrib/emscripten/__init__.py | 20 +- src/urllib3/contrib/emscripten/connection.py | 44 ++-- src/urllib3/contrib/emscripten/fetch.py | 109 ++++++---- src/urllib3/contrib/emscripten/request.py | 15 +- src/urllib3/contrib/emscripten/response.py | 23 ++- test/contrib/emscripten/__init__.py | 0 .../emscripten/conftest.py} | 68 +++---- .../emscripten}/test_emscripten.py | 188 ++++++++---------- 10 files changed, 288 insertions(+), 233 deletions(-) create mode 100644 test/contrib/emscripten/__init__.py rename test/{with_dummyserver/emscripten_fixtures.py => contrib/emscripten/conftest.py} (80%) rename test/{with_dummyserver => contrib/emscripten}/test_emscripten.py (57%) diff --git a/noxfile.py b/noxfile.py index 40da4ae825..4fd48c616f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,6 +4,7 @@ import shutil import sys from pathlib import Path + import nox @@ -11,7 +12,7 @@ def tests_impl( session: nox.Session, extras: str = "socks,brotli,zstd", byte_string_comparisons: bool = True, - pytest_extra_args: list[str] = [] + pytest_extra_args: list[str] = [], ) -> None: # Install deps and the package itself. session.install("-r", "dev-requirements.txt") @@ -157,22 +158,46 @@ def emscripten(session: nox.Session) -> None: # use the dist directory from that dist_dir = Path(os.environ["PYODIDE_ROOT"]) else: - # we don't have a build tree, get one # that matches the version of pyodide build - import pyodide_build - pyodide_version=pyodide_build.__version__ - pyodide_artifacts_path = (Path(session.cache_dir) / f"pyodide-{pyodide_version}") + import pyodide_build # type: ignore[import] + + pyodide_version = pyodide_build.__version__ + pyodide_artifacts_path = Path(session.cache_dir) / f"pyodide-{pyodide_version}" if not pyodide_artifacts_path.exists(): print("Fetching pyodide build artifacts") - session.run("wget",f"https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/pyodide-{pyodide_version}.tar.bz2","-O",f"{pyodide_artifacts_path}.tar.bz2") + session.run( + "wget", + f"https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/pyodide-{pyodide_version}.tar.bz2", + "-O", + f"{pyodide_artifacts_path}.tar.bz2", + ) pyodide_artifacts_path.mkdir(parents=True) - session.run("tar","-xjf",f"{pyodide_artifacts_path}.tar.bz2","-C",pyodide_artifacts_path,"--strip-components","1") - - dist_dir=pyodide_artifacts_path - assert dist_dir != None + session.run( + "tar", + "-xjf", + f"{pyodide_artifacts_path}.tar.bz2", + "-C", + str(pyodide_artifacts_path), + "--strip-components", + "1", + ) + + dist_dir = pyodide_artifacts_path + assert dist_dir is not None assert dist_dir.exists() - tests_impl(session,"emscripten-test",pytest_extra_args=["--rt","chrome-no-host","--dist-dir",dist_dir,"test"]) + tests_impl( + session, + "emscripten-test", + pytest_extra_args=[ + "--rt", + "chrome-no-host", + "--dist-dir", + str(dist_dir), + "test", + ], + ) + @nox.session(python="3.12") def mypy(session: nox.Session) -> None: diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 7505b5b18d..f1e014251a 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -6,6 +6,7 @@ # Set default logging handler to avoid "No handler found" warnings. import logging +import sys import typing import warnings from logging import NullHandler @@ -148,6 +149,8 @@ def request( json=json, ) -import sys + if sys.platform == "emscripten": - from .contrib.emscripten import * + from .contrib.emscripten import _override_connections_for_emscripten # noqa: 401 + + _override_connections_for_emscripten() diff --git a/src/urllib3/contrib/emscripten/__init__.py b/src/urllib3/contrib/emscripten/__init__.py index 4e42c6ac4d..10a33f3e47 100644 --- a/src/urllib3/contrib/emscripten/__init__.py +++ b/src/urllib3/contrib/emscripten/__init__.py @@ -1,10 +1,16 @@ -from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection -from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool - -HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection -HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection +from __future__ import annotations import urllib3.connection -urllib3.connection.HTTPConnection = EmscriptenHTTPConnection -urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection +from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection + + +def _override_connections_for_emscripten() -> None: + # override connection classes to use emscripten specific classes + # n.b. mypy complains about the overriding of classes below + # if it isn't ignored + HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection + HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection + urllib3.connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment] + urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment] diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index a2c3753670..16ab00da27 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -1,21 +1,23 @@ -from ..._base_connection import _TYPE_BODY, _TYPE_SOCKET_OPTIONS +from __future__ import annotations + +import os +import typing +from http.client import ResponseNotReady + +from ..._base_connection import _TYPE_BODY from ...connection import HTTPConnection, ProxyConfig, port_by_scheme +from ...response import BaseHTTPResponse +from ...util.connection import _TYPE_SOCKET_OPTIONS from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT from ...util.url import Url - -from .fetch import send_streaming_request, send_request - +from .fetch import send_request, send_streaming_request from .request import EmscriptenRequest from .response import EmscriptenHttpResponseWrapper -from ...response import BaseHTTPResponse - -import typing class EmscriptenHTTPConnection(HTTPConnection): host: str port: int - timeout: None | (float) blocksize: int source_address: tuple[str, int] | None socket_options: _TYPE_SOCKET_OPTIONS | None @@ -29,12 +31,12 @@ class EmscriptenHTTPConnection(HTTPConnection): def __init__( self, host: str, - port: int | None = None, + port: int = 0, *, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, source_address: tuple[str, int] | None = None, blocksize: int = 8192, - socket_options: _TYPE_SOCKET_OPTIONS | None = ..., + socket_options: _TYPE_SOCKET_OPTIONS | None = None, proxy: Url | None = None, proxy_config: ProxyConfig | None = None, ) -> None: @@ -42,12 +44,12 @@ def __init__( # control over that stuff self.host = host self.port = port - self.timeout = timeout + self.timeout = timeout if isinstance(timeout, float) else 0.0 def set_tunnel( self, host: str, - port: int | None = None, + port: int | None = 0, headers: typing.Mapping[str, str] | None = None, scheme: str = "http", ) -> None: @@ -56,7 +58,7 @@ def set_tunnel( def connect(self) -> None: pass - def request( + def request( # type: ignore[override] self, method: str, url: str, @@ -79,14 +81,17 @@ def request( self._response = None if not preload_content: self._response = send_streaming_request(request) - if self._response == None: + if self._response is None: self._response = send_request(request) - def getresponse(self) -> BaseHTTPResponse: - return EmscriptenHttpResponseWrapper(self._response) + def getresponse(self) -> BaseHTTPResponse: # type: ignore[override] + if self._response is not None: + return EmscriptenHttpResponseWrapper(self._response) + else: + raise ResponseNotReady() def close(self) -> None: - ... + pass @property def is_closed(self) -> bool: @@ -94,10 +99,12 @@ def is_closed(self) -> bool: If this property is True then both ``is_connected`` and ``has_connected_to_proxy`` properties must be False. """ + return False @property def is_connected(self) -> bool: """Whether the connection is actively connected to any origin (proxy or target)""" + return True @property def has_connected_to_proxy(self) -> bool: @@ -105,6 +112,7 @@ def has_connected_to_proxy(self) -> bool: This returns False if no proxy is in use. Used to determine whether errors are coming from the proxy layer or from tunnelling to the target origin. """ + return False class EmscriptenHTTPSConnection(EmscriptenHTTPConnection): @@ -122,7 +130,7 @@ class EmscriptenHTTPSConnection(EmscriptenHTTPConnection): def __init__( self, host: str, - port: int | None = None, + port: int = 0, *, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, source_address: tuple[str, int] | None = None, diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index d48015c63d..3f64bc2c03 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -1,32 +1,41 @@ """ -Support for streaming http requests in emscripten. +Support for streaming http requests in emscripten. -A couple of caveats - +A few caveats - Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed. Streaming only works if you're running pyodide in a web worker. Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch -operation, so it requires that you have crossOriginIsolation enabled, by serving over https +operation, so it requires that you have crossOriginIsolation enabled, by serving over https (or from localhost) with the two headers below set: Cross-Origin-Opener-Policy: same-origin Cross-Origin-Embedder-Policy: require-corp -You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in +You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in javascript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole request into a buffer and then returning it. it shows a warning in the javascript console in this case. + +Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once +control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch. """ +from __future__ import annotations + import io import json -import js -from pyodide.ffi import to_js -from pyodide.code import run_js +from email.parser import Parser +from typing import TYPE_CHECKING, Any + +import js # type: ignore[import] +from pyodide.ffi import JsArray, JsProxy, to_js # type: ignore[import] + +if TYPE_CHECKING: + from typing_extensions import Buffer + from .request import EmscriptenRequest from .response import EmscriptenResponse -from email.parser import Parser - """ There are some headers that trigger unintended CORS preflight requests. See also https://github.com/koenvo/pyodide-http/issues/22 @@ -40,7 +49,13 @@ class _RequestError(Exception): - def __init__(self, message=None, *, request=None, response=None): + def __init__( + self, + message: str | None = None, + *, + request: EmscriptenRequest | None = None, + response: EmscriptenResponse | None = None, + ): self.request = request self.response = response self.message = message @@ -79,14 +94,14 @@ class _StreamingTimeout(_StreamingError): try { let readResponse = await reader.read(); - + if (readResponse.done) { // read everything - clear connection and return delete connections[connectionID]; Atomics.store(intBuffer, 0, SUCCESS_EOF); Atomics.notify(intBuffer, 0); // finished reading successfully - // return from event handler + // return from event handler return; } curOffset = 0; @@ -100,11 +115,11 @@ class _StreamingTimeout(_StreamingError): byteBuffer.set(errorBytes); intBuffer[1] = written; Atomics.store(intBuffer, 0, ERROR_EXCEPTION); - Atomics.notify(intBuffer, 0); + Atomics.notify(intBuffer, 0); } } - // send as much buffer as we can + // send as much buffer as we can let curLen = value.length - curOffset; if (curLen > byteBuffer.length) { curLen = byteBuffer.length; @@ -160,12 +175,19 @@ class _StreamingTimeout(_StreamingError): """ -def _obj_from_dict(dict_val: dict) -> any: +def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy: return to_js(dict_val, dict_converter=js.Object.fromEntries) class _ReadStream(io.RawIOBase): - def __init__(self, int_buffer, byte_buffer, timeout, worker, connection_id): + def __init__( + self, + int_buffer: JsArray, + byte_buffer: JsArray, + timeout: float, + worker: JsProxy, + connection_id: int, + ): self.int_buffer = int_buffer self.byte_buffer = byte_buffer self.read_pos = 0 @@ -175,10 +197,10 @@ def __init__(self, int_buffer, byte_buffer, timeout, worker, connection_id): self.timeout = int(1000 * timeout) if timeout > 0 else None self.is_live = True - def __del__(self): + def __del__(self) -> None: self.close() - def close(self): + def close(self) -> None: if self.is_live: self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) self.is_live = False @@ -193,7 +215,7 @@ def writeable(self) -> bool: def seekable(self) -> bool: return False - def readinto(self, byte_obj) -> bool: + def readinto(self, byte_obj: Buffer) -> int: if not self.int_buffer: return 0 if self.read_len == 0: @@ -216,18 +238,18 @@ def readinto(self, byte_obj) -> bool: self.byte_buffer = None return 0 # copy from int32array to python bytes - ret_length = min(self.read_len, len(byte_obj)) + ret_length = min(self.read_len, len(memoryview(byte_obj))) subarray = self.byte_buffer.subarray( self.read_pos, self.read_pos + ret_length ).to_py() - byte_obj[0:ret_length] = subarray + memoryview(byte_obj)[0:ret_length] = subarray self.read_len -= ret_length self.read_pos += ret_length return ret_length class _StreamingFetcher: - def __init__(self): + def __init__(self) -> None: # make web-worker and data buffer on startup self.streaming_ready = False @@ -235,12 +257,12 @@ def __init__(self): [_STREAMING_WORKER_CODE], _obj_from_dict({"type": "application/javascript"}) ) - def promise_resolver(res, rej): - def onMsg(e): + def promise_resolver(res: JsProxy, rej: JsProxy) -> None: + def onMsg(e: JsProxy) -> None: self.streaming_ready = True res(e) - def onErr(e): + def onErr(e: JsProxy) -> None: rej(e) self._worker.onmessage = onMsg @@ -250,7 +272,7 @@ def onErr(e): self._worker = js.globalThis.Worker.new(dataURL) self._worker_ready_promise = js.globalThis.Promise.new(promise_resolver) - def send(self, request): + def send(self, request: EmscriptenRequest) -> EmscriptenResponse: headers = { k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE } @@ -266,15 +288,6 @@ def send(self, request): js.Atomics.store(int_buffer, 0, 0) js.Atomics.notify(int_buffer, 0) absolute_url = js.URL.new(request.url, js.location).href - # js.console.log( - # _obj_from_dict( - # { - # "buffer": shared_buffer, - # "url": absolute_url, - # "fetchParams": fetch_data, - # } - # ) - # ) self._worker.postMessage( _obj_from_dict( { @@ -317,7 +330,7 @@ def send(self, request): buffer_size=1048576, ), ) - if int_buffer[0] == ERROR_EXCEPTION: + elif int_buffer[0] == ERROR_EXCEPTION: string_len = int_buffer[1] # decode the error string decoder = js.TextDecoder.new() @@ -325,6 +338,12 @@ def send(self, request): raise _StreamingError( f"Exception thrown in fetch: {json_str}", request=request, response=None ) + else: + raise _StreamingError( + f"Unknown status from worker in fetch: {int_buffer[0]}", + request=request, + response=None, + ) # check if we are in a worker or not @@ -332,11 +351,11 @@ def is_in_browser_main_thread() -> bool: return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window -def is_cross_origin_isolated(): +def is_cross_origin_isolated() -> bool: return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated -def is_in_node(): +def is_in_node() -> bool: return ( hasattr(js, "process") and hasattr(js.process, "release") @@ -345,10 +364,12 @@ def is_in_node(): ) -def is_worker_available(): +def is_worker_available() -> bool: return hasattr(js, "Worker") and hasattr(js, "Blob") +_fetcher: _StreamingFetcher | None = None + if is_worker_available() and ( (is_cross_origin_isolated() and not is_in_browser_main_thread()) or is_in_node() ): @@ -357,7 +378,7 @@ def is_worker_available(): _fetcher = None -def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse: +def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None: if _fetcher and streaming_ready(): return _fetcher.send(request) else: @@ -368,7 +389,7 @@ def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse: _SHOWN_WARNING = False -def _show_streaming_warning(): +def _show_streaming_warning() -> None: global _SHOWN_WARNING if not _SHOWN_WARNING: _SHOWN_WARNING = True @@ -379,7 +400,7 @@ def _show_streaming_warning(): message += " Python is running in main browser thread\n" if not is_worker_available(): message += " Worker or Blob classes are not available in this environment." - if streaming_ready() == False: + if streaming_ready() is False: message += """ Streaming fetch worker isn't ready. If you want to be sure that streamig fetch is working, you need to call: 'await urllib3.contrib.emscripten.fetc.wait_for_streaming_ready()`""" from js import console @@ -413,14 +434,14 @@ def send_request(request: EmscriptenRequest) -> EmscriptenResponse: return EmscriptenResponse(status_code=xhr.status, headers=headers, body=body) -def streaming_ready(): +def streaming_ready() -> bool | None: if _fetcher: return _fetcher.streaming_ready else: return None # no fetcher, return None to signify that -async def wait_for_streaming_ready(): +async def wait_for_streaming_ready() -> bool: if _fetcher: await _fetcher._worker_ready_promise return True diff --git a/src/urllib3/contrib/emscripten/request.py b/src/urllib3/contrib/emscripten/request.py index 2ba53a09ad..0eef079ec0 100644 --- a/src/urllib3/contrib/emscripten/request.py +++ b/src/urllib3/contrib/emscripten/request.py @@ -1,5 +1,10 @@ +from __future__ import annotations + +import json from dataclasses import dataclass, field -from typing import Dict +from typing import Any + +from ..._base_connection import _TYPE_BODY @dataclass @@ -7,16 +12,16 @@ class EmscriptenRequest: method: str url: str params: dict[str, str] | None = None - body: bytes | None = None + body: _TYPE_BODY | None = None headers: dict[str, str] = field(default_factory=dict) timeout: int = 0 - def set_header(self, name: str, value: str): + def set_header(self, name: str, value: str) -> None: self.headers[name.capitalize()] = value - def set_body(self, body: bytes): + def set_body(self, body: _TYPE_BODY | None) -> None: self.body = body - def set_json(self, body: dict): + def set_json(self, body: dict[str, Any]) -> None: self.set_header("Content-Type", "application/json; charset=utf-8") self.set_body(json.dumps(body).encode("utf-8")) diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index e8c92946ab..3eb494f7fe 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -1,7 +1,8 @@ -from dataclasses import dataclass -from io import IOBase, BytesIO -from itertools import takewhile +from __future__ import annotations + import typing +from dataclasses import dataclass +from io import BytesIO, IOBase from ...connection import HTTPConnection from ...response import HTTPResponse @@ -17,7 +18,10 @@ class EmscriptenResponse: class EmscriptenHttpResponseWrapper(HTTPResponse): def __init__( - self, internal_response: EmscriptenResponse, url: str = None, connection=None + self, + internal_response: EmscriptenResponse, + url: str | None = None, + connection: HTTPConnection | None = None, ): self._response = internal_response self._url = url @@ -63,14 +67,19 @@ def read( if not isinstance(self._response.body, IOBase): # wrap body in IOStream self._response.body = BytesIO(self._response.body) - return self._response.body.read(amt) + + return typing.cast(bytes, self._response.body.read(amt)) def read_chunked( self, amt: int | None = None, decode_content: bool | None = None, - ) -> typing.Iterator[bytes]: - return self.read(amt, decode_content) + ) -> typing.Generator[bytes, None, None]: + while True: + bytes = self.read(amt, decode_content) + if not bytes: + break + yield bytes def release_conn(self) -> None: if not self._pool or not self._connection: diff --git a/test/contrib/emscripten/__init__.py b/test/contrib/emscripten/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/with_dummyserver/emscripten_fixtures.py b/test/contrib/emscripten/conftest.py similarity index 80% rename from test/with_dummyserver/emscripten_fixtures.py rename to test/contrib/emscripten/conftest.py index a906998cef..9a6ece0d73 100644 --- a/test/with_dummyserver/emscripten_fixtures.py +++ b/test/contrib/emscripten/conftest.py @@ -1,28 +1,28 @@ +from __future__ import annotations + import asyncio import contextlib +import mimetypes import os - -from urllib.parse import urlsplit import textwrap -import mimetypes +from pathlib import Path +from urllib.parse import urlsplit + import pytest from tornado import web -from dummyserver.server import ( - run_loop_in_thread, - run_tornado_app, -) -from pathlib import Path -from dummyserver.handlers import TestingApp -from dummyserver.handlers import Response +from tornado.httputil import HTTPServerRequest +from dummyserver.handlers import Response, TestingApp +from dummyserver.server import run_loop_in_thread, run_tornado_app from dummyserver.testcase import HTTPDummyProxyTestCase +from typing import Generator,Any @pytest.fixture(scope="module") -def testserver_http(request): +def testserver_http(request:pytest.FixtureRequest)->Generator[PyodideServerInfo,None,None]: dist_dir = Path(os.getcwd(), request.config.getoption("--dist-dir")) server = PyodideDummyServerTestCase - server.setup_class(dist_dir) + server.setup_class(str(dist_dir)) print( f"Server:{server.http_host}:{server.http_port},https({server.https_port}) [{dist_dir}]" ) @@ -31,13 +31,13 @@ def testserver_http(request): server.teardown_class() -class _FromServerRunner: - def __init__(self, host, port, selenium): +class ServerRunnerInfo: + def __init__(self, host:str, port:int, selenium:Any)->None: self.host = host self.port = port self.selenium = selenium - def run_webworker(self, code): + def run_webworker(self, code:str)->Any: if isinstance(code, str) and code.startswith("\n"): # we have a multiline string, fix indentation code = textwrap.dedent(code) @@ -70,7 +70,7 @@ def run_webworker(self, code): # pytest-pyodide one - this makes it so that # we are at the same origin as web requests to server_host @pytest.fixture() -def run_from_server(selenium, testserver_http): +def run_from_server(selenium:Any, testserver_http:PyodideServerInfo)->Generator[ServerRunnerInfo,None,None]: addr = f"https://{testserver_http.http_host}:{testserver_http.https_port}/pyodide/test.html" selenium.goto(addr) # import time @@ -86,7 +86,7 @@ def run_from_server(selenium, testserver_http): await pyodide.loadPackage('/wheel/dist.whl') """ ) - yield _FromServerRunner( + yield ServerRunnerInfo( testserver_http.http_host, testserver_http.https_port, selenium ) @@ -101,19 +101,19 @@ def set_default_headers(self) -> None: self.set_header("Cross-Origin-Embedder-Policy", "require-corp") self.add_header("Feature-Policy", "sync-xhr *;") - def bigfile(self, req): + def bigfile(self, req:HTTPServerRequest)->Response: print("Bigfile requested") # great big text file, should force streaming # if supported bigdata = 1048576 * b"WOOO YAY BOOYAKAH" return Response(bigdata) - def pyodide(self, req): + def pyodide(self, req:HTTPServerRequest)->Response: path = req.path[:] if not path.startswith("/"): path = urlsplit(path).path - path = path.split("/") - file_path = Path(PyodideTestingApp.pyodide_dist_dir, *path[2:]) + path_split = path.split("/") + file_path = Path(PyodideTestingApp.pyodide_dist_dir, *path_split[2:]) if file_path.exists(): mime_type, encoding = mimetypes.guess_type(file_path) print(file_path, mime_type) @@ -125,25 +125,11 @@ def pyodide(self, req): headers=[("Access-Control-Allow-Origin", "*")], ) else: - return Response(status=404) + return Response(status="404 NOT FOUND") - def worker_template(self, req): - return Response( - """ - - - - - worker loader""", - headers=[("Content-type", "text/html")], - ) - return - - def wheel(self, req): + def wheel(self, _req:HTTPServerRequest)->Response: # serve our wheel - wheel_folder = Path(__file__).parent.parent.parent / "dist" + wheel_folder = Path(__file__).parent.parent.parent.parent / "dist" print(wheel_folder) wheels = list(wheel_folder.glob("*.whl")) print(wheels) @@ -155,11 +141,13 @@ def wheel(self, req): ], ) return resp + else: + return Response(status="404 NOT FOUND") class PyodideDummyServerTestCase(HTTPDummyProxyTestCase): @classmethod - def setup_class(cls, pyodide_dist_dir) -> None: + def setup_class(cls, pyodide_dist_dir:str) -> None: # type:ignore[override] PyodideTestingApp.pyodide_dist_dir = pyodide_dist_dir with contextlib.ExitStack() as stack: io_loop = stack.enter_context(run_loop_in_thread()) @@ -177,3 +165,5 @@ async def run_app() -> None: asyncio.run_coroutine_threadsafe(run_app(), io_loop.asyncio_loop).result() # type: ignore[attr-defined] cls._stack = stack.pop_all() + +PyodideServerInfo=type[PyodideDummyServerTestCase] \ No newline at end of file diff --git a/test/with_dummyserver/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py similarity index 57% rename from test/with_dummyserver/test_emscripten.py rename to test/contrib/emscripten/test_emscripten.py index 8c2957405b..492dcd85e2 100644 --- a/test/with_dummyserver/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -1,21 +1,29 @@ +from __future__ import annotations + import pytest + # only run these tests if pytest_pyodide is installed # so we don't break non-emscripten pytest running -pytest_pyodide=pytest.importorskip("pytest_pyodide") +pytest_pyodide = pytest.importorskip("pytest_pyodide") # make our ssl certificates work in chrome pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") -from pytest_pyodide import run_in_pyodide -from .emscripten_fixtures import testserver_http,run_from_server -from pytest_pyodide.decorator import copy_files_to_pyodide +from pytest_pyodide import run_in_pyodide # type: ignore[import] +from pytest_pyodide.decorator import copy_files_to_pyodide # type: ignore[import] -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_index(selenium, testserver_http): - @run_in_pyodide - def pyodide_test(selenium, host, port): - import urllib3.contrib.emscripten - from urllib3.response import HTTPResponse +import typing + +from urllib3.fields import _TYPE_FIELD_VALUE_TUPLE + +from .conftest import PyodideDummyServerTestCase, PyodideServerInfo, ServerRunnerInfo + + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_index(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPConnection + from urllib3.response import HTTPResponse conn = HTTPConnection(host, port) method = "GET" @@ -24,18 +32,19 @@ def pyodide_test(selenium, host, port): conn.request(method, url) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data=response.data + data = response.data assert data.decode("utf-8") == "Dummy server!" 1 pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_index_https(selenium, testserver_http): - @run_in_pyodide - def pyodide_test(selenium, host, port): - from urllib3.response import HTTPResponse + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_index_https(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPSConnection + from urllib3.response import HTTPResponse conn = HTTPSConnection(host, port) method = "GET" @@ -44,83 +53,69 @@ def pyodide_test(selenium, host, port): conn.request(method, url) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data=response.data + data = response.data assert data.decode("utf-8") == "Dummy server!" 1 pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_non_streaming_no_fallback_warning(selenium, testserver_http): - @run_in_pyodide - def pyodide_test(selenium, host, port): + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_non_streaming_no_fallback_warning( + selenium: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] import urllib3.contrib.emscripten.fetch - from urllib3.response import HTTPResponse from urllib3.connection import HTTPSConnection + from urllib3.response import HTTPResponse conn = HTTPSConnection(host, port) method = "GET" path = "/" url = f"https://{host}:{port}{path}" - conn.request(method, url,preload_content=True) + conn.request(method, url, preload_content=True) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data=response.data + data = response.data assert data.decode("utf-8") == "Dummy server!" # no console warnings because we didn't ask it to stream the response - assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==False + assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING == False pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_streaming_fallback_warning(selenium, testserver_http): - @run_in_pyodide - def pyodide_test(selenium, host, port): +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_streaming_fallback_warning( + selenium: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] import urllib3.contrib.emscripten.fetch - from urllib3.response import HTTPResponse from urllib3.connection import HTTPSConnection + from urllib3.response import HTTPResponse conn = HTTPSConnection(host, port) method = "GET" path = "/" url = f"https://{host}:{port}{path}" - conn.request(method, url,preload_content=False) + conn.request(method, url, preload_content=False) response = conn.getresponse() assert isinstance(response, HTTPResponse) - data=response.data + data = response.data assert data.decode("utf-8") == "Dummy server!" # check that it has warned about falling back to non-streaming fetch - assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==True + assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING == True pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -def test_specific_method(selenium, testserver_http,run_from_server): - print("Running from server") - @run_in_pyodide - def pyodide_test(selenium, host, port): - from urllib3 import HTTPConnectionPool - from urllib3.response import HTTPResponse - with HTTPConnectionPool(host, port) as pool: - method = "POST" - path = "/specific_method?method=POST" - response = pool.request(method,path) - assert isinstance(response, HTTPResponse) - assert(response.status==200) - - method = "PUT" - path = "/specific_method?method=POST" - response = pool.request(method,path) - assert isinstance(response, HTTPResponse) - assert(response.status==400) - - pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) - -def test_upload(selenium, testserver_http,run_from_server): - @run_in_pyodide - def pyodide_test(selenium, host, port): +def test_upload( + selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server:ServerRunnerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3 import HTTPConnectionPool data = "I'm in ur multipart form-data, hazing a cheezburgr" @@ -136,57 +131,46 @@ def pyodide_test(selenium, host, port): pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_index_https(selenium, testserver_http): - @run_in_pyodide - def pyodide_test(selenium, host, port): - from urllib3.response import HTTPResponse - from urllib3.connection import HTTPSConnection - - conn = HTTPSConnection(host, port) - method = "GET" - path = "/" - url = f"https://{host}:{port}{path}" - conn.request(method, url) - response = conn.getresponse() - assert isinstance(response, HTTPResponse) - data=response.data - assert data.decode("utf-8") == "Dummy server!" - - pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) - - -def test_specific_method(selenium, testserver_http,run_from_server): +def test_specific_method( + selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server:ServerRunnerInfo +) -> None: print("Running from server") - @run_in_pyodide - def pyodide_test(selenium, host, port): + + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3 import HTTPConnectionPool from urllib3.response import HTTPResponse + with HTTPConnectionPool(host, port) as pool: method = "POST" path = "/specific_method?method=POST" - response = pool.request(method,path) + response = pool.request(method, path) assert isinstance(response, HTTPResponse) - assert(response.status==200) + assert response.status == 200 method = "PUT" path = "/specific_method?method=POST" - response = pool.request(method,path) + response = pool.request(method, path) assert isinstance(response, HTTPResponse) - assert(response.status==400) + assert response.status == 400 pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_streaming_download(selenium, testserver_http,run_from_server): + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_streaming_download( + selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server:ServerRunnerInfo +) -> None: # test streaming download, which must be in a webworker # as you can't do it on main thread # this should return the 17mb big file, and - # should not log any warning about falling back - bigfile_url=url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + # should not log typing.Any warning about falling back + bigfile_url = ( + url + ) = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" worker_code = f"""import micropip -await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) +await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-typing.Any.whl',deps=False) import urllib3.contrib.emscripten.fetch await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() from urllib3.response import HTTPResponse @@ -203,17 +187,22 @@ def test_streaming_download(selenium, testserver_http,run_from_server): data=response.data.decode('utf-8') data """ - result=run_from_server.run_webworker(worker_code) - assert(len(result)==17825792) + result = run_from_server.run_webworker(worker_code) + assert len(result) == 17825792 + -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) -def test_streaming_notready_warning(selenium, testserver_http,run_from_server): - # test streaming download but don't wait for +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_streaming_notready_warning( + selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server:ServerRunnerInfo +) -> None: + # test streaming download but don't wait for # worker to be ready - should fallback to non-streaming - # and log a warning - bigfile_url=url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + # and log a warning + bigfile_url = ( + url + ) = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" worker_code = f"""import micropip -await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-any.whl',deps=False) +await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-typing.Any.whl',deps=False) import urllib3.contrib.emscripten.fetch from urllib3.response import HTTPResponse from urllib3.connection import HTTPConnection @@ -228,6 +217,5 @@ def test_streaming_notready_warning(selenium, testserver_http,run_from_server): assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==True data """ - result=run_from_server.run_webworker(worker_code) - assert(len(result)==17825792) - + result = run_from_server.run_webworker(worker_code) + assert len(result) == 17825792 From 667d54fc04b41812cf02d99dbef9b31726050a98 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 16 Nov 2023 14:46:53 +0000 Subject: [PATCH 07/49] changelog --- changelog/2951.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/2951.feature.rst diff --git a/changelog/2951.feature.rst b/changelog/2951.feature.rst new file mode 100644 index 0000000000..43bba90dee --- /dev/null +++ b/changelog/2951.feature.rst @@ -0,0 +1 @@ +Added support for Emscripten, including streaming support in cross-origin isolated browser environments where threading is enabled. \ No newline at end of file From a0a5709f773876ef1d411eecb41d93863fbddfcf Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 16 Nov 2023 14:58:42 +0000 Subject: [PATCH 08/49] mypy updates for changes in main --- test/contrib/emscripten/conftest.py | 31 ++++---- test/contrib/emscripten/test_emscripten.py | 85 ++++++++++++---------- 2 files changed, 66 insertions(+), 50 deletions(-) diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 9a6ece0d73..e83cdaa291 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -6,20 +6,22 @@ import os import textwrap from pathlib import Path +from typing import Any, Generator from urllib.parse import urlsplit import pytest from tornado import web - from tornado.httputil import HTTPServerRequest + from dummyserver.handlers import Response, TestingApp -from dummyserver.server import run_loop_in_thread, run_tornado_app from dummyserver.testcase import HTTPDummyProxyTestCase +from dummyserver.tornadoserver import run_tornado_app, run_tornado_loop_in_thread -from typing import Generator,Any @pytest.fixture(scope="module") -def testserver_http(request:pytest.FixtureRequest)->Generator[PyodideServerInfo,None,None]: +def testserver_http( + request: pytest.FixtureRequest, +) -> Generator[PyodideServerInfo, None, None]: dist_dir = Path(os.getcwd(), request.config.getoption("--dist-dir")) server = PyodideDummyServerTestCase server.setup_class(str(dist_dir)) @@ -32,12 +34,12 @@ def testserver_http(request:pytest.FixtureRequest)->Generator[PyodideServerInfo, class ServerRunnerInfo: - def __init__(self, host:str, port:int, selenium:Any)->None: + def __init__(self, host: str, port: int, selenium: Any) -> None: self.host = host self.port = port self.selenium = selenium - def run_webworker(self, code:str)->Any: + def run_webworker(self, code: str) -> Any: if isinstance(code, str) and code.startswith("\n"): # we have a multiline string, fix indentation code = textwrap.dedent(code) @@ -70,7 +72,9 @@ def run_webworker(self, code:str)->Any: # pytest-pyodide one - this makes it so that # we are at the same origin as web requests to server_host @pytest.fixture() -def run_from_server(selenium:Any, testserver_http:PyodideServerInfo)->Generator[ServerRunnerInfo,None,None]: +def run_from_server( + selenium: Any, testserver_http: PyodideServerInfo +) -> Generator[ServerRunnerInfo, None, None]: addr = f"https://{testserver_http.http_host}:{testserver_http.https_port}/pyodide/test.html" selenium.goto(addr) # import time @@ -101,14 +105,14 @@ def set_default_headers(self) -> None: self.set_header("Cross-Origin-Embedder-Policy", "require-corp") self.add_header("Feature-Policy", "sync-xhr *;") - def bigfile(self, req:HTTPServerRequest)->Response: + def bigfile(self, req: HTTPServerRequest) -> Response: print("Bigfile requested") # great big text file, should force streaming # if supported bigdata = 1048576 * b"WOOO YAY BOOYAKAH" return Response(bigdata) - def pyodide(self, req:HTTPServerRequest)->Response: + def pyodide(self, req: HTTPServerRequest) -> Response: path = req.path[:] if not path.startswith("/"): path = urlsplit(path).path @@ -127,7 +131,7 @@ def pyodide(self, req:HTTPServerRequest)->Response: else: return Response(status="404 NOT FOUND") - def wheel(self, _req:HTTPServerRequest)->Response: + def wheel(self, _req: HTTPServerRequest) -> Response: # serve our wheel wheel_folder = Path(__file__).parent.parent.parent.parent / "dist" print(wheel_folder) @@ -147,10 +151,10 @@ def wheel(self, _req:HTTPServerRequest)->Response: class PyodideDummyServerTestCase(HTTPDummyProxyTestCase): @classmethod - def setup_class(cls, pyodide_dist_dir:str) -> None: # type:ignore[override] + def setup_class(cls, pyodide_dist_dir: str) -> None: # type:ignore[override] PyodideTestingApp.pyodide_dist_dir = pyodide_dist_dir with contextlib.ExitStack() as stack: - io_loop = stack.enter_context(run_loop_in_thread()) + io_loop = stack.enter_context(run_tornado_loop_in_thread()) async def run_app() -> None: app = web.Application([(r".*", PyodideTestingApp)]) @@ -166,4 +170,5 @@ async def run_app() -> None: asyncio.run_coroutine_threadsafe(run_app(), io_loop.asyncio_loop).result() # type: ignore[attr-defined] cls._stack = stack.pop_all() -PyodideServerInfo=type[PyodideDummyServerTestCase] \ No newline at end of file + +PyodideServerInfo = type[PyodideDummyServerTestCase] diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 492dcd85e2..ff499b6f2f 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -1,27 +1,30 @@ from __future__ import annotations +import typing + import pytest +from urllib3.fields import _TYPE_FIELD_VALUE_TUPLE + # only run these tests if pytest_pyodide is installed # so we don't break non-emscripten pytest running pytest_pyodide = pytest.importorskip("pytest_pyodide") -# make our ssl certificates work in chrome -pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") -from pytest_pyodide import run_in_pyodide # type: ignore[import] -from pytest_pyodide.decorator import copy_files_to_pyodide # type: ignore[import] +from pytest_pyodide import run_in_pyodide # type: ignore[import] # noqa: E402 +from pytest_pyodide.decorator import ( # type: ignore[import] # noqa: E402 + copy_files_to_pyodide, +) -import typing +from .conftest import PyodideServerInfo, ServerRunnerInfo # noqa: E402 -from urllib3.fields import _TYPE_FIELD_VALUE_TUPLE - -from .conftest import PyodideDummyServerTestCase, PyodideServerInfo, ServerRunnerInfo +# make our ssl certificates work in chrome +pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] def test_index(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: - @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPConnection from urllib3.response import HTTPResponse @@ -39,10 +42,10 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unty pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] def test_index_https(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: - @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPSConnection from urllib3.response import HTTPResponse @@ -60,12 +63,12 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unty pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] def test_non_streaming_no_fallback_warning( selenium: typing.Any, testserver_http: PyodideServerInfo ) -> None: - @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] import urllib3.contrib.emscripten.fetch from urllib3.connection import HTTPSConnection from urllib3.response import HTTPResponse @@ -80,17 +83,17 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unty data = response.data assert data.decode("utf-8") == "Dummy server!" # no console warnings because we didn't ask it to stream the response - assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING == False + assert not urllib3.contrib.emscripten.fetch._SHOWN_WARNING pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] def test_streaming_fallback_warning( selenium: typing.Any, testserver_http: PyodideServerInfo ) -> None: - @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] import urllib3.contrib.emscripten.fetch from urllib3.connection import HTTPSConnection from urllib3.response import HTTPResponse @@ -105,17 +108,18 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unty data = response.data assert data.decode("utf-8") == "Dummy server!" # check that it has warned about falling back to non-streaming fetch - assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING == True + assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) - def test_upload( - selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server:ServerRunnerInfo + selenium: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, ) -> None: - @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3 import HTTPConnectionPool data = "I'm in ur multipart form-data, hazing a cheezburgr" @@ -131,13 +135,16 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unty pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + def test_specific_method( - selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server:ServerRunnerInfo + selenium: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, ) -> None: print("Running from server") - @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3 import HTTPConnectionPool from urllib3.response import HTTPResponse @@ -157,9 +164,11 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unty pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] def test_streaming_download( - selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server:ServerRunnerInfo + selenium: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, ) -> None: # test streaming download, which must be in a webworker # as you can't do it on main thread @@ -167,8 +176,8 @@ def test_streaming_download( # this should return the 17mb big file, and # should not log typing.Any warning about falling back bigfile_url = ( - url - ) = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + ) worker_code = f"""import micropip await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-typing.Any.whl',deps=False) import urllib3.contrib.emscripten.fetch @@ -191,16 +200,18 @@ def test_streaming_download( assert len(result) == 17825792 -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] def test_streaming_notready_warning( - selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server:ServerRunnerInfo + selenium: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, ) -> None: # test streaming download but don't wait for # worker to be ready - should fallback to non-streaming # and log a warning bigfile_url = ( - url - ) = f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + ) worker_code = f"""import micropip await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-typing.Any.whl',deps=False) import urllib3.contrib.emscripten.fetch From 5dd3c4ae157f85354b2beb236bc979f9bd793369 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 16 Nov 2023 18:45:43 +0000 Subject: [PATCH 09/49] error handling and tests for it --- src/urllib3/contrib/emscripten/connection.py | 20 ++- src/urllib3/contrib/emscripten/fetch.py | 76 ++++++---- src/urllib3/contrib/emscripten/request.py | 2 +- test/contrib/emscripten/conftest.py | 6 + test/contrib/emscripten/test_emscripten.py | 141 ++++++++++++++++++- 5 files changed, 208 insertions(+), 37 deletions(-) diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index 16ab00da27..43478c3a5d 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -6,11 +6,12 @@ from ..._base_connection import _TYPE_BODY from ...connection import HTTPConnection, ProxyConfig, port_by_scheme +from ...exceptions import ResponseError, TimeoutError from ...response import BaseHTTPResponse from ...util.connection import _TYPE_SOCKET_OPTIONS from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT from ...util.url import Url -from .fetch import send_request, send_streaming_request +from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request from .request import EmscriptenRequest from .response import EmscriptenHttpResponseWrapper @@ -73,16 +74,23 @@ def request( # type: ignore[override] decode_content: bool = True, enforce_content_length: bool = True, ) -> None: - request = EmscriptenRequest(url=url, method=method) + request = EmscriptenRequest( + url=url, method=method, timeout=self.timeout if self.timeout else 0 + ) request.set_body(body) if headers: for k, v in headers.items(): request.set_header(k, v) self._response = None - if not preload_content: - self._response = send_streaming_request(request) - if self._response is None: - self._response = send_request(request) + try: + if not preload_content: + self._response = send_streaming_request(request) + if self._response is None: + self._response = send_request(request) + except _TimeoutError as e: + raise TimeoutError(e.message) + except _RequestError as e: + raise ResponseError(e.message) def getresponse(self) -> BaseHTTPResponse: # type: ignore[override] if self._response is not None: diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index 3f64bc2c03..3a6be63846 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING, Any import js # type: ignore[import] -from pyodide.ffi import JsArray, JsProxy, to_js # type: ignore[import] +from pyodide.ffi import JsArray, JsException, JsProxy, to_js # type: ignore[import] if TYPE_CHECKING: from typing_extensions import Buffer @@ -66,7 +66,7 @@ class _StreamingError(_RequestError): pass -class _StreamingTimeout(_StreamingError): +class _TimeoutError(_RequestError): pass @@ -223,7 +223,7 @@ def readinto(self, byte_obj: Buffer) -> int: js.Atomics.store(self.int_buffer, 0, 0) self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id})) if js.Atomics.wait(self.int_buffer, 0, 0, self.timeout) == "timed-out": - raise _StreamingTimeout + raise _TimeoutError data_len = self.int_buffer[0] if data_len > 0: self.read_len = data_len @@ -300,7 +300,7 @@ def send(self, request: EmscriptenRequest) -> EmscriptenResponse: # wait for the worker to send something js.Atomics.wait(int_buffer, 0, 0, timeout) if int_buffer[0] == 0: - raise _StreamingTimeout( + raise _TimeoutError( "Timeout connecting to streaming request", request=request, response=None, @@ -386,13 +386,24 @@ def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | N return None -_SHOWN_WARNING = False +_SHOWN_TIMEOUT_WARNING = False + + +def _show_timeout_warning() -> None: + global _SHOWN_TIMEOUT_WARNING + if not _SHOWN_TIMEOUT_WARNING: + _SHOWN_TIMEOUT_WARNING = True + message = "Warning: Timeout is not available on main browser thread" + js.console.warn(message) + + +_SHOWN_STREAMING_WARNING = False def _show_streaming_warning() -> None: - global _SHOWN_WARNING - if not _SHOWN_WARNING: - _SHOWN_WARNING = True + global _SHOWN_STREAMING_WARNING + if not _SHOWN_STREAMING_WARNING: + _SHOWN_STREAMING_WARNING = True message = "Can't stream HTTP requests because: \n" if not is_cross_origin_isolated(): message += " Page is not cross-origin isolated\n" @@ -406,32 +417,45 @@ def _show_streaming_warning() -> None: from js import console console.warn(message) - print(message) def send_request(request: EmscriptenRequest) -> EmscriptenResponse: - xhr = js.XMLHttpRequest.new() - xhr.timeout = int(request.timeout * 1000) + try: + xhr = js.XMLHttpRequest.new() - if not is_in_browser_main_thread(): - xhr.responseType = "arraybuffer" - else: - xhr.overrideMimeType("text/plain; charset=ISO-8859-15") + if not is_in_browser_main_thread(): + xhr.responseType = "arraybuffer" + if request.timeout: + xhr.timeout = int(request.timeout * 1000) + else: + xhr.overrideMimeType("text/plain; charset=ISO-8859-15") + if request.timeout: + # timeout isn't available on the main thread - show a warning in console + # if it is set + _show_timeout_warning() - xhr.open(request.method, request.url, False) - for name, value in request.headers.items(): - if name.lower() not in HEADERS_TO_IGNORE: - xhr.setRequestHeader(name, value) + xhr.open(request.method, request.url, False) + for name, value in request.headers.items(): + if name.lower() not in HEADERS_TO_IGNORE: + xhr.setRequestHeader(name, value) - xhr.send(to_js(request.body)) + xhr.send(to_js(request.body)) - headers = dict(Parser().parsestr(xhr.getAllResponseHeaders())) + headers = dict(Parser().parsestr(xhr.getAllResponseHeaders())) - if not is_in_browser_main_thread(): - body = xhr.response.to_py().tobytes() - else: - body = xhr.response.encode("ISO-8859-15") - return EmscriptenResponse(status_code=xhr.status, headers=headers, body=body) + if not is_in_browser_main_thread(): + body = xhr.response.to_py().tobytes() + else: + body = xhr.response.encode("ISO-8859-15") + return EmscriptenResponse(status_code=xhr.status, headers=headers, body=body) + except JsException as err: + if err.name == "TimeoutError": + raise _TimeoutError(err.message, request=request) + elif err.name == "NetworkError": + raise _RequestError(err.message, request=request) + else: + # general http error + raise _RequestError(err.message, request=request) def streaming_ready() -> bool | None: diff --git a/src/urllib3/contrib/emscripten/request.py b/src/urllib3/contrib/emscripten/request.py index 0eef079ec0..a4de04abcf 100644 --- a/src/urllib3/contrib/emscripten/request.py +++ b/src/urllib3/contrib/emscripten/request.py @@ -14,7 +14,7 @@ class EmscriptenRequest: params: dict[str, str] | None = None body: _TYPE_BODY | None = None headers: dict[str, str] = field(default_factory=dict) - timeout: int = 0 + timeout: float = 0 def set_header(self, name: str, value: str) -> None: self.headers[name.capitalize()] = value diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index e83cdaa291..8ecb19f64a 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -105,6 +105,12 @@ def set_default_headers(self) -> None: self.set_header("Cross-Origin-Embedder-Policy", "require-corp") self.add_header("Feature-Policy", "sync-xhr *;") + def slow(self, _req: HTTPServerRequest) -> Response: + import time + + time.sleep(10) + return Response("TEN SECONDS LATER") + def bigfile(self, req: HTTPServerRequest) -> Response: print("Bigfile requested") # great big text file, should force streaming diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index ff499b6f2f..45dd3296b8 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -42,6 +42,139 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) +# wrong protocol / protocol error etc. should raise an exception of urllib3.exceptions.ResponseError +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_wrong_protocol( + selenium: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3.exceptions + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + method = "GET" + path = "/" + url = f"http://{host}:{port}{path}" + try: + conn.request(method, url) + conn.getresponse() + pytest.fail("Should have thrown ResponseError here") + except BaseException as ex: + assert isinstance(ex, urllib3.exceptions.ResponseError) + + pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + + +# no connection - should raise +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_no_response(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: + @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3.exceptions + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + method = "GET" + path = "/" + url = f"http://{host}:{port}{path}" + try: + conn.request(method, url) + _ = conn.getresponse() + pytest.fail("No response, should throw exception.") + except BaseException as ex: + assert isinstance(ex, urllib3.exceptions.ResponseError) + + import socket + + sock = socket.socket() + sock.bind(("", 0)) + free_port = sock.getsockname()[1] + sock.close() + + pyodide_test(selenium, testserver_http.http_host, free_port) + + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_404(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + from urllib3.response import HTTPResponse + + conn = HTTPConnection(host, port) + method = "GET" + path = "/status?status=404 NOT FOUND" + url = f"http://{host}:{port}{path}" + conn.request(method, url) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + assert response.status == 404 + 1 + + pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + + +# setting timeout should show a warning to js console +# if we're on the ui thread, because XMLHttpRequest doesn't +# support timeout in async mode if globalThis == Window +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_timeout_warning( + selenium: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide() # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port, timeout=1.0) + method = "GET" + path = "/" + url = f"http://{host}:{port}{path}" + conn.request(method, url) + conn.getresponse() + assert urllib3.contrib.emscripten.fetch._SHOWN_TIMEOUT_WARNING + 1 + + pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_timeout_in_worker( + selenium: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + worker_code = f""" + import micropip + await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-typing.Any.whl',deps=False) + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.exceptions import TimeoutError + from urllib3.connection import HTTPConnection + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + method = "GET" + url = "http://{testserver_http.http_host}:{testserver_http.http_port}/slow" + result=-1 + try: + conn.request(method, url) + _response = conn.getresponse() + result=-3 + except TimeoutError as e: + result=1 # we've got the correct exception + except BaseException as e: + result=-2 + result +""" + result = run_from_server.run_webworker(worker_code) + # result == 1 = success, -2 = wrong exception, -3 = no exception thrown + assert result == 1 + + @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] def test_index_https(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] @@ -83,7 +216,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt data = response.data assert data.decode("utf-8") == "Dummy server!" # no console warnings because we didn't ask it to stream the response - assert not urllib3.contrib.emscripten.fetch._SHOWN_WARNING + assert not urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) @@ -108,7 +241,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt data = response.data assert data.decode("utf-8") == "Dummy server!" # check that it has warned about falling back to non-streaming fetch - assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING + assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) @@ -192,7 +325,7 @@ def test_streaming_download( conn.request(method, url,preload_content=False) response = conn.getresponse() assert isinstance(response, HTTPResponse) -assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==False +assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==False data=response.data.decode('utf-8') data """ @@ -225,7 +358,7 @@ def test_streaming_notready_warning( response = conn.getresponse() assert isinstance(response, HTTPResponse) data=response.data.decode('utf-8') -assert urllib3.contrib.emscripten.fetch._SHOWN_WARNING==True +assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True data """ result = run_from_server.run_webworker(worker_code) From 034ad0ec5d0340d3d1535a9b88754f71a6be9337 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 12:44:19 +0000 Subject: [PATCH 10/49] fixed upload bugs --- src/urllib3/contrib/emscripten/connection.py | 16 +++- src/urllib3/contrib/emscripten/fetch.py | 5 +- src/urllib3/contrib/emscripten/request.py | 1 + src/urllib3/contrib/emscripten/response.py | 37 ++++++++- test/contrib/emscripten/test_emscripten.py | 84 ++++++++++++-------- 5 files changed, 103 insertions(+), 40 deletions(-) diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index 43478c3a5d..149ea865b5 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -46,6 +46,7 @@ def __init__( self.host = host self.port = port self.timeout = timeout if isinstance(timeout, float) else 0.0 + self.scheme = "http" def set_tunnel( self, @@ -74,8 +75,14 @@ def request( # type: ignore[override] decode_content: bool = True, enforce_content_length: bool = True, ) -> None: + if url.startswith("/"): + # no scheme / host / port included, make a full url + url = f"{self.scheme}://{self.host}:{self.port}" + url request = EmscriptenRequest( - url=url, method=method, timeout=self.timeout if self.timeout else 0 + url=url, + method=method, + timeout=self.timeout if self.timeout else 0, + decode_content=decode_content, ) request.set_body(body) if headers: @@ -94,7 +101,11 @@ def request( # type: ignore[override] def getresponse(self) -> BaseHTTPResponse: # type: ignore[override] if self._response is not None: - return EmscriptenHttpResponseWrapper(self._response) + return EmscriptenHttpResponseWrapper( + internal_response=self._response, + url=self._response.request.url, + connection=self, + ) else: raise ResponseNotReady() @@ -172,6 +183,7 @@ def __init__( proxy=proxy, proxy_config=proxy_config, ) + self.scheme = "https" self.key_file = key_file self.cert_file = cert_file diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index 3a6be63846..e578678736 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -317,6 +317,7 @@ def send(self, request: EmscriptenRequest) -> EmscriptenResponse: # get it as an object response_obj = json.loads(json_str) return EmscriptenResponse( + request=request, status_code=response_obj["status"], headers=response_obj["headers"], body=io.BufferedReader( @@ -447,7 +448,9 @@ def send_request(request: EmscriptenRequest) -> EmscriptenResponse: body = xhr.response.to_py().tobytes() else: body = xhr.response.encode("ISO-8859-15") - return EmscriptenResponse(status_code=xhr.status, headers=headers, body=body) + return EmscriptenResponse( + status_code=xhr.status, headers=headers, body=body, request=request + ) except JsException as err: if err.name == "TimeoutError": raise _TimeoutError(err.message, request=request) diff --git a/src/urllib3/contrib/emscripten/request.py b/src/urllib3/contrib/emscripten/request.py index a4de04abcf..4544ac66ce 100644 --- a/src/urllib3/contrib/emscripten/request.py +++ b/src/urllib3/contrib/emscripten/request.py @@ -15,6 +15,7 @@ class EmscriptenRequest: body: _TYPE_BODY | None = None headers: dict[str, str] = field(default_factory=dict) timeout: float = 0 + decode_content: bool = True def set_header(self, name: str, value: str) -> None: self.headers[name.capitalize()] = value diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 3eb494f7fe..66e23f4cd2 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json as _json import typing from dataclasses import dataclass from io import BytesIO, IOBase @@ -7,6 +8,7 @@ from ...connection import HTTPConnection from ...response import HTTPResponse from ...util.retry import Retry +from .request import EmscriptenRequest @dataclass @@ -14,6 +16,7 @@ class EmscriptenResponse: status_code: int headers: dict[str, str] body: IOBase | bytes + request: EmscriptenRequest class EmscriptenHttpResponseWrapper(HTTPResponse): @@ -23,6 +26,7 @@ def __init__( url: str | None = None, connection: HTTPConnection | None = None, ): + self._body = None self._response = internal_response self._url = url self._connection = connection @@ -61,14 +65,21 @@ def retries(self, retries: Retry | None) -> None: def read( self, amt: int | None = None, - decode_content: bool | None = None, + decode_content: bool | None = None, # ignored because browser decodes always cache_content: bool = False, ) -> bytes: if not isinstance(self._response.body, IOBase): # wrap body in IOStream self._response.body = BytesIO(self._response.body) - - return typing.cast(bytes, self._response.body.read(amt)) + if amt is not None: + # don't cache partial content + cache_content = False + return typing.cast(bytes, self._response.body.read(amt)) + else: + data = self._response.body.read(None) + if cache_content: + self._body = data + return typing.cast(bytes, data) def read_chunked( self, @@ -91,6 +102,26 @@ def release_conn(self) -> None: def drain_conn(self) -> None: self.close() + @property + def data(self) -> bytes: + if self._body: + return self._body # type: ignore[return-value] + else: + return self.read(cache_content=True) + + def json(self) -> typing.Any: + """ + Parses the body of the HTTP response as JSON. + + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to the decoder. + + This method can raise either `UnicodeDecodeError` or `json.JSONDecodeError`. + + Read more :ref:`here `. + """ + data = self.data.decode("utf-8") + return _json.loads(data) + def close(self) -> None: if isinstance(self._response.body, IOBase): self._response.body.close() diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 45dd3296b8..8acbdf5a21 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -37,7 +37,6 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt assert isinstance(response, HTTPResponse) data = response.data assert data.decode("utf-8") == "Dummy server!" - 1 pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) @@ -114,7 +113,6 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt response = conn.getresponse() assert isinstance(response, HTTPResponse) assert response.status == 404 - 1 pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) @@ -138,7 +136,6 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt conn.request(method, url) conn.getresponse() assert urllib3.contrib.emscripten.fetch._SHOWN_TIMEOUT_WARNING - 1 pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) @@ -191,7 +188,6 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt assert isinstance(response, HTTPResponse) data = response.data assert data.decode("utf-8") == "Dummy server!" - 1 pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) @@ -246,52 +242,24 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -def test_upload( - selenium: typing.Any, - testserver_http: PyodideServerInfo, - run_from_server: ServerRunnerInfo, -) -> None: - @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] - from urllib3 import HTTPConnectionPool - - data = "I'm in ur multipart form-data, hazing a cheezburgr" - fields: dict[str, _TYPE_FIELD_VALUE_TUPLE] = { - "upload_param": "filefield", - "upload_filename": "lolcat.txt", - "filefield": ("lolcat.txt", data), - } - fields["upload_size"] = len(data) # type: ignore - with HTTPConnectionPool(host, port) as pool: - r = pool.request("POST", "/upload", fields=fields) - assert r.status == 200, r.data - - pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) - - def test_specific_method( selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server: ServerRunnerInfo, ) -> None: - print("Running from server") - @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] - from urllib3 import HTTPConnectionPool - from urllib3.response import HTTPResponse + from urllib3 import HTTPSConnectionPool - with HTTPConnectionPool(host, port) as pool: + with HTTPSConnectionPool(host, port) as pool: method = "POST" path = "/specific_method?method=POST" response = pool.request(method, path) - assert isinstance(response, HTTPResponse) assert response.status == 200 method = "PUT" path = "/specific_method?method=POST" response = pool.request(method, path) - assert isinstance(response, HTTPResponse) assert response.status == 400 pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) @@ -363,3 +331,51 @@ def test_streaming_notready_warning( """ result = run_from_server.run_webworker(worker_code) assert len(result) == 17825792 + + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_post_receive_json( + selenium: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import json + + from urllib3.connection import HTTPConnection + from urllib3.response import HTTPResponse + + json_data = { + "Bears": "like", + "to": {"eat": "buns", "with": ["marmalade", "and custard"]}, + } + conn = HTTPConnection(host, port) + method = "POST" + path = "/echo_json" + url = f"http://{host}:{port}{path}" + conn.request(method, url, body=json.dumps(json_data).encode("utf-8")) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + data = response.json() + assert data == json_data + + pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + + +@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +def test_upload(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3 import HTTPConnectionPool + + data = "I'm in ur multipart form-data, hazing a cheezburgr" + fields: dict[str, _TYPE_FIELD_VALUE_TUPLE] = { + "upload_param": "filefield", + "upload_filename": "lolcat.txt", + "filefield": ("lolcat.txt", data), + } + fields["upload_size"] = len(data) # type: ignore + with HTTPConnectionPool(host, port) as pool: + r = pool.request("POST", "/upload", fields=fields) + assert r.status == 200 + + pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) From 51364a4a99d24035d1db46c2d7b68e3f7f02a986 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 14:59:28 +0000 Subject: [PATCH 11/49] ci updates --- .github/workflows/ci.yml | 7 +++++ noxfile.py | 37 ++++++++++++++++--------- pyproject.toml | 3 +- src/urllib3/contrib/emscripten/fetch.py | 3 +- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a58a92c1e..9e46826d37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,9 @@ jobs: os: ubuntu-20.04 # CPython 3.9.2 is not available for ubuntu-22.04. experimental: false nox-session: test-3.9 + - python-version: "3.11" + os: ubuntu-latest + nox-session: emscripten exclude: # Ubuntu 22.04 comes with OpenSSL 3.0, so only CPython 3.9+ is compatible with it # https://github.com/python/cpython/issues/83001 @@ -104,6 +107,10 @@ jobs: - name: "Install dependencies" run: python -m pip install --upgrade pip setuptools nox + - name: "Install chrome" + uses: browser-actions/setup-chrome@v1 + if: ${{ matrix.nox-session == "emscripten" }} + - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version run: nox -s ${NOX_SESSION:-test-$PYTHON_VERSION} --error-on-missing-interpreters diff --git a/noxfile.py b/noxfile.py index eb7b4908b4..cc1d9ba2a9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -7,6 +7,7 @@ import nox +import json def tests_impl( session: nox.Session, @@ -155,9 +156,12 @@ def lint(session: nox.Session) -> None: mypy(session) +# TODO: node support - should work if you require('xmlhttprequest') before +# loading pyodide @nox.session(python="3.11") -def emscripten(session: nox.Session) -> None: - """install emscripten extras""" +@nox.parametrize('runner', ['chrome']) +def emscripten(session: nox.Session,runner: str) -> None: + """Test on Emscripten with Pyodide & Chrome""" session.install("build") # build wheel into dist folder session.run("python", "-m", "build") @@ -196,17 +200,24 @@ def emscripten(session: nox.Session) -> None: dist_dir = pyodide_artifacts_path assert dist_dir is not None assert dist_dir.exists() - tests_impl( - session, - "emscripten-test", - pytest_extra_args=[ - "--rt", - "chrome-no-host", - "--dist-dir", - str(dist_dir), - "test", - ], - ) + if runner=='chrome': + # install chrome webdriver and add it to path + import webdriver_manager.chrome + from webdriver_manager.chrome import ChromeDriverManager + driver = ChromeDriverManager().install() + session.env["PATH"]=f"{Path(driver).parent}:{session.env['PATH']}" + + tests_impl( + session, + "emscripten-test", + pytest_extra_args=[ + "--rt", + "chrome-no-host", + "--dist-dir", + str(dist_dir), + "test", + ], + ) @nox.session(python="3.12") diff --git a/pyproject.toml b/pyproject.toml index 7c12e8f58a..e87326af52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,8 @@ socks = [ ] emscripten_test = [ "pytest-pyodide>=0.54.0", - "pyodide-build>=0.24.0" + "pyodide-build>=0.24.0", + "webdriver-manager>=4.0.0" ] [project.urls] diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index e578678736..457e85c577 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -368,11 +368,10 @@ def is_in_node() -> bool: def is_worker_available() -> bool: return hasattr(js, "Worker") and hasattr(js, "Blob") - _fetcher: _StreamingFetcher | None = None if is_worker_available() and ( - (is_cross_origin_isolated() and not is_in_browser_main_thread()) or is_in_node() + (is_cross_origin_isolated() and not is_in_browser_main_thread()) and (not is_in_node()) ): _fetcher = _StreamingFetcher() else: From b45cf16b25086fcaf3d75c0cfbb1f074763837bc Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 15:50:33 +0000 Subject: [PATCH 12/49] comment --- noxfile.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index cc1d9ba2a9..c16e32e54d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -156,8 +156,10 @@ def lint(session: nox.Session) -> None: mypy(session) -# TODO: node support - should work if you require('xmlhttprequest') before -# loading pyodide +# TODO: node support is not tested yet - it should work if you require('xmlhttprequest') before +# loading pyodide, but there is currently no nice way to do this with pytest-pyodide +# because you can't override the test runner properties - +# https://github.com/pyodide/pytest-pyodide/issues/118 @nox.session(python="3.11") @nox.parametrize('runner', ['chrome']) def emscripten(session: nox.Session,runner: str) -> None: From 1a00b78b751a93ab3de41d1bfeb1fb16899ca5a5 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 15:52:17 +0000 Subject: [PATCH 13/49] lint --- noxfile.py | 15 +++++++-------- src/urllib3/contrib/emscripten/fetch.py | 4 +++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/noxfile.py b/noxfile.py index c16e32e54d..9c212fb725 100644 --- a/noxfile.py +++ b/noxfile.py @@ -7,7 +7,6 @@ import nox -import json def tests_impl( session: nox.Session, @@ -159,10 +158,10 @@ def lint(session: nox.Session) -> None: # TODO: node support is not tested yet - it should work if you require('xmlhttprequest') before # loading pyodide, but there is currently no nice way to do this with pytest-pyodide # because you can't override the test runner properties - -# https://github.com/pyodide/pytest-pyodide/issues/118 +# https://github.com/pyodide/pytest-pyodide/issues/118 @nox.session(python="3.11") -@nox.parametrize('runner', ['chrome']) -def emscripten(session: nox.Session,runner: str) -> None: +@nox.parametrize("runner", ["chrome"]) +def emscripten(session: nox.Session, runner: str) -> None: """Test on Emscripten with Pyodide & Chrome""" session.install("build") # build wheel into dist folder @@ -202,12 +201,12 @@ def emscripten(session: nox.Session,runner: str) -> None: dist_dir = pyodide_artifacts_path assert dist_dir is not None assert dist_dir.exists() - if runner=='chrome': + if runner == "chrome": # install chrome webdriver and add it to path - import webdriver_manager.chrome - from webdriver_manager.chrome import ChromeDriverManager + from webdriver_manager.chrome import ChromeDriverManager # type: ignore[import] + driver = ChromeDriverManager().install() - session.env["PATH"]=f"{Path(driver).parent}:{session.env['PATH']}" + session.env["PATH"] = f"{Path(driver).parent}:{session.env['PATH']}" tests_impl( session, diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index 457e85c577..f468feb5b1 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -368,10 +368,12 @@ def is_in_node() -> bool: def is_worker_available() -> bool: return hasattr(js, "Worker") and hasattr(js, "Blob") + _fetcher: _StreamingFetcher | None = None if is_worker_available() and ( - (is_cross_origin_isolated() and not is_in_browser_main_thread()) and (not is_in_node()) + (is_cross_origin_isolated() and not is_in_browser_main_thread()) + and (not is_in_node()) ): _fetcher = _StreamingFetcher() else: From 09feff0860bd059176abdc22433aaafb32f81b62 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 15:56:26 +0000 Subject: [PATCH 14/49] ci updates --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e46826d37..fa5c045975 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,7 @@ jobs: - python-version: "3.11" os: ubuntu-latest nox-session: emscripten + experimental: true exclude: # Ubuntu 22.04 comes with OpenSSL 3.0, so only CPython 3.9+ is compatible with it # https://github.com/python/cpython/issues/83001 From 30d8b695cdf4eea5eadff221ab57091d11143a6a Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 15:57:27 +0000 Subject: [PATCH 15/49] fix to if statement --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa5c045975..04529271eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,8 +110,7 @@ jobs: - name: "Install chrome" uses: browser-actions/setup-chrome@v1 - if: ${{ matrix.nox-session == "emscripten" }} - + if: matrix.nox-session == "emscripten" - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version run: nox -s ${NOX_SESSION:-test-$PYTHON_VERSION} --error-on-missing-interpreters From a45b32ffc289e44c2925e5e3884754ceea9e14e6 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 15:58:48 +0000 Subject: [PATCH 16/49] actions fix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04529271eb..9b448fb19e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,7 +110,7 @@ jobs: - name: "Install chrome" uses: browser-actions/setup-chrome@v1 - if: matrix.nox-session == "emscripten" + if: ${{ matrix.nox-session == 'emscripten' }} - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version run: nox -s ${NOX_SESSION:-test-$PYTHON_VERSION} --error-on-missing-interpreters From d8fb8adac6e8856823b411c622aa7633a1a686b8 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 16:00:32 +0000 Subject: [PATCH 17/49] install pyodide_build for nox --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 9c212fb725..07a4af5b0f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -163,7 +163,7 @@ def lint(session: nox.Session) -> None: @nox.parametrize("runner", ["chrome"]) def emscripten(session: nox.Session, runner: str) -> None: """Test on Emscripten with Pyodide & Chrome""" - session.install("build") + session.install("build","pyodide_build") # build wheel into dist folder session.run("python", "-m", "build") # make sure we have a dist dir for pyodide From 3c4bb6624dcc561cc2aa2cf8e0206b76a00289cf Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 18:27:01 +0000 Subject: [PATCH 18/49] hopefully fix noxfile pyodide_build dependency --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 07a4af5b0f..b2c0ca8ddc 100644 --- a/noxfile.py +++ b/noxfile.py @@ -163,7 +163,7 @@ def lint(session: nox.Session) -> None: @nox.parametrize("runner", ["chrome"]) def emscripten(session: nox.Session, runner: str) -> None: """Test on Emscripten with Pyodide & Chrome""" - session.install("build","pyodide_build") + session.install("build","pyodide-build") # build wheel into dist folder session.run("python", "-m", "build") # make sure we have a dist dir for pyodide From 2f89bae19c081d1eb5539c669bec7a9414f1253d Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 18:43:23 +0000 Subject: [PATCH 19/49] 3.8 fixes --- noxfile.py | 2 +- test/contrib/emscripten/conftest.py | 9 ++++++++- test/contrib/emscripten/test_emscripten.py | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index b2c0ca8ddc..a48d3819da 100644 --- a/noxfile.py +++ b/noxfile.py @@ -163,7 +163,7 @@ def lint(session: nox.Session) -> None: @nox.parametrize("runner", ["chrome"]) def emscripten(session: nox.Session, runner: str) -> None: """Test on Emscripten with Pyodide & Chrome""" - session.install("build","pyodide-build") + session.install("build", "pyodide-build") # build wheel into dist folder session.run("python", "-m", "build") # make sure we have a dist dir for pyodide diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 8ecb19f64a..3c7b0b9694 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -5,6 +5,7 @@ import mimetypes import os import textwrap +import typing from pathlib import Path from typing import Any, Generator from urllib.parse import urlsplit @@ -17,6 +18,12 @@ from dummyserver.testcase import HTTPDummyProxyTestCase from dummyserver.tornadoserver import run_tornado_app, run_tornado_loop_in_thread +# only run this stuff if pytest_pyodide is installed +# pytest_pyodide = pytest.importorskip("pytest_pyodide") + +# if sys.version_info<(3,11): +# pytest.skip() + @pytest.fixture(scope="module") def testserver_http( @@ -177,4 +184,4 @@ async def run_app() -> None: cls._stack = stack.pop_all() -PyodideServerInfo = type[PyodideDummyServerTestCase] +PyodideServerInfo = PyodideDummyServerTestCase diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 8acbdf5a21..284d09e371 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -1,11 +1,16 @@ from __future__ import annotations +import sys import typing import pytest from urllib3.fields import _TYPE_FIELD_VALUE_TUPLE +if sys.version_info < (3, 11): + # pyodide only works on 3.11+ + pytest.skip(allow_module_level=True) + # only run these tests if pytest_pyodide is installed # so we don't break non-emscripten pytest running pytest_pyodide = pytest.importorskip("pytest_pyodide") From 7862c11e314aa67235a7917f8898efa55233d654 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 18:54:07 +0000 Subject: [PATCH 20/49] get pyodide_build version without requiring it --- noxfile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index a48d3819da..066a752e0b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -175,9 +175,8 @@ def emscripten(session: nox.Session, runner: str) -> None: else: # we don't have a build tree, get one # that matches the version of pyodide build - import pyodide_build # type: ignore[import] + pyodide_version=session.run("python","-c","import pyodide_build;print(pyodide_build.__version__)",silent=True).strip() - pyodide_version = pyodide_build.__version__ pyodide_artifacts_path = Path(session.cache_dir) / f"pyodide-{pyodide_version}" if not pyodide_artifacts_path.exists(): print("Fetching pyodide build artifacts") From 05c97ddcd6700aa94548a16bb88fff649ed09471 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 19:11:49 +0000 Subject: [PATCH 21/49] 3.8 compatible type checking --- noxfile.py | 25 ++++++++++++++++++++----- test/contrib/emscripten/conftest.py | 5 ++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/noxfile.py b/noxfile.py index 066a752e0b..86eb7d8dc8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,6 +3,7 @@ import os import shutil import sys +import typing from pathlib import Path import nox @@ -163,7 +164,7 @@ def lint(session: nox.Session) -> None: @nox.parametrize("runner", ["chrome"]) def emscripten(session: nox.Session, runner: str) -> None: """Test on Emscripten with Pyodide & Chrome""" - session.install("build", "pyodide-build") + session.install("build", "pyodide-build", "webdriver_manager") # build wheel into dist folder session.run("python", "-m", "build") # make sure we have a dist dir for pyodide @@ -175,7 +176,15 @@ def emscripten(session: nox.Session, runner: str) -> None: else: # we don't have a build tree, get one # that matches the version of pyodide build - pyodide_version=session.run("python","-c","import pyodide_build;print(pyodide_build.__version__)",silent=True).strip() + pyodide_version = typing.cast( + str, + session.run( + "python", + "-c", + "import pyodide_build;print(pyodide_build.__version__)", + silent=True, + ), + ).strip() pyodide_artifacts_path = Path(session.cache_dir) / f"pyodide-{pyodide_version}" if not pyodide_artifacts_path.exists(): @@ -202,9 +211,15 @@ def emscripten(session: nox.Session, runner: str) -> None: assert dist_dir.exists() if runner == "chrome": # install chrome webdriver and add it to path - from webdriver_manager.chrome import ChromeDriverManager # type: ignore[import] - - driver = ChromeDriverManager().install() + driver = typing.cast( + str, + session.run( + "python", + "-c", + "from webdriver_manager.chrome import ChromeDriverManager;print(ChromeDriverManager().install())", + silent=True, + ), + ).strip() session.env["PATH"] = f"{Path(driver).parent}:{session.env['PATH']}" tests_impl( diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 3c7b0b9694..388d31f61b 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -5,9 +5,8 @@ import mimetypes import os import textwrap -import typing from pathlib import Path -from typing import Any, Generator +from typing import Any, Generator, Type from urllib.parse import urlsplit import pytest @@ -184,4 +183,4 @@ async def run_app() -> None: cls._stack = stack.pop_all() -PyodideServerInfo = PyodideDummyServerTestCase +PyodideServerInfo = Type[PyodideDummyServerTestCase] From 363a0f38f00177f82a6ee73c2ddc7c7388069535 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 22:36:48 +0000 Subject: [PATCH 22/49] review changes --- .eslintrc.yml | 4 + .pre-commit-config.yaml | 5 + emscripten-requirements.txt | 3 + noxfile.py | 5 +- pyproject.toml | 5 - src/urllib3/__init__.py | 4 +- src/urllib3/contrib/emscripten/__init__.py | 2 +- .../emscripten/emscripten_fetch_worker.js | 101 ++++++++++++++ src/urllib3/contrib/emscripten/fetch.py | 123 ++---------------- test/contrib/emscripten/conftest.py | 11 +- 10 files changed, 136 insertions(+), 127 deletions(-) create mode 100644 .eslintrc.yml create mode 100644 emscripten-requirements.txt create mode 100644 src/urllib3/contrib/emscripten/emscripten_fetch_worker.js diff --git a/.eslintrc.yml b/.eslintrc.yml new file mode 100644 index 0000000000..100af5bf92 --- /dev/null +++ b/.eslintrc.yml @@ -0,0 +1,4 @@ +env: + es2020 : true + worker: true +rules: {} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04e65f8cd4..ab2430c963 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,3 +21,8 @@ repos: hooks: - id: flake8 additional_dependencies: [flake8-2020] + + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.53.0 + hooks: + - id: eslint \ No newline at end of file diff --git a/emscripten-requirements.txt b/emscripten-requirements.txt new file mode 100644 index 0000000000..614a44bb0b --- /dev/null +++ b/emscripten-requirements.txt @@ -0,0 +1,3 @@ +pytest-pyodide>=0.54.0 +pyodide-build>=0.24.0 +webdriver-manager>=4.0.0 \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index 86eb7d8dc8..1b16447518 100644 --- a/noxfile.py +++ b/noxfile.py @@ -164,7 +164,7 @@ def lint(session: nox.Session) -> None: @nox.parametrize("runner", ["chrome"]) def emscripten(session: nox.Session, runner: str) -> None: """Test on Emscripten with Pyodide & Chrome""" - session.install("build", "pyodide-build", "webdriver_manager") + session.install("-r", "emscripten-requirements.txt") # build wheel into dist folder session.run("python", "-m", "build") # make sure we have a dist dir for pyodide @@ -224,7 +224,6 @@ def emscripten(session: nox.Session, runner: str) -> None: tests_impl( session, - "emscripten-test", pytest_extra_args=[ "--rt", "chrome-no-host", @@ -233,6 +232,8 @@ def emscripten(session: nox.Session, runner: str) -> None: "test", ], ) + else: + raise ValueError(f"Unknown runnner: {runner}") @nox.session(python="3.12") diff --git a/pyproject.toml b/pyproject.toml index e87326af52..6b850412a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,11 +49,6 @@ zstd = [ socks = [ "PySocks>=1.5.6,<2.0,!=1.5.7", ] -emscripten_test = [ - "pytest-pyodide>=0.54.0", - "pyodide-build>=0.24.0", - "webdriver-manager>=4.0.0" -] [project.urls] "Changelog" = "https://github.com/urllib3/urllib3/blob/main/CHANGES.rst" diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index ec18bded63..1e0bf37b33 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -206,6 +206,6 @@ def request( if sys.platform == "emscripten": - from .contrib.emscripten import _override_connections_for_emscripten # noqa: 401 + from .contrib.emscripten import inject_into_urllib3 # noqa: 401 - _override_connections_for_emscripten() + inject_into_urllib3() diff --git a/src/urllib3/contrib/emscripten/__init__.py b/src/urllib3/contrib/emscripten/__init__.py index 10a33f3e47..8a3c5bebdc 100644 --- a/src/urllib3/contrib/emscripten/__init__.py +++ b/src/urllib3/contrib/emscripten/__init__.py @@ -6,7 +6,7 @@ from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection -def _override_connections_for_emscripten() -> None: +def inject_into_urllib3() -> None: # override connection classes to use emscripten specific classes # n.b. mypy complains about the overriding of classes below # if it isn't ignored diff --git a/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js b/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js new file mode 100644 index 0000000000..063a862e8c --- /dev/null +++ b/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js @@ -0,0 +1,101 @@ + +let NO_RESULT_YET = 0; +let SUCCESS_HEADER = -1; +let SUCCESS_EOF = -2; +let ERROR_TIMEOUT = -3; +let ERROR_EXCEPTION = -4; + +let connections = {}; +let nextConnectionID = 1; + +self.addEventListener("message", async function (event) { + if (event.data.close) { + let connectionID = event.data.close; + delete connections[connectionID]; + return; + } else if (event.data.getMore) { + let connectionID = event.data.getMore; + let { curOffset, value, reader, intBuffer, byteBuffer } = connections[connectionID]; + // if we still have some in buffer, then just send it back straight away + if (!value || curOffset >= value.length) { + // read another buffer if required + try { + let readResponse = await reader.read(); + + if (readResponse.done) { + // read everything - clear connection and return + delete connections[connectionID]; + Atomics.store(intBuffer, 0, SUCCESS_EOF); + Atomics.notify(intBuffer, 0); + // finished reading successfully + // return from event handler + return; + } + curOffset = 0; + connections[connectionID].value = readResponse.value; + value = readResponse.value; + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } + + // send as much buffer as we can + let curLen = value.length - curOffset; + if (curLen > byteBuffer.length) { + curLen = byteBuffer.length; + } + byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0) + + Atomics.store(intBuffer, 0, curLen);// store current length in bytes + Atomics.notify(intBuffer, 0); + curOffset += curLen; + connections[connectionID].curOffset = curOffset; + + return; + } else { + // start fetch + let connectionID = nextConnectionID; + nextConnectionID += 1; + const encoder = new TextEncoder(); + const intBuffer = new Int32Array(event.data.buffer); + const byteBuffer = new Uint8Array(event.data.buffer, 8) + intBuffer[0] = NO_RESULT_YET; + try { + const response = await fetch(event.data.url, event.data.fetchParams); + // return the headers first via textencoder + var headers = []; + for (const pair of response.headers.entries()) { + headers.push([pair[0], pair[1]]); + } + headerObj = { headers: headers, status: response.status, connectionID }; + const headerText = JSON.stringify(headerObj); + let headerBytes = encoder.encode(headerText); + let written = headerBytes.length; + byteBuffer.set(headerBytes); + intBuffer[1] = written; + // make a connection + connections[connectionID] = { reader: response.body.getReader(), intBuffer: intBuffer, byteBuffer: byteBuffer, value: undefined, curOffset: 0 }; + // set header ready + Atomics.store(intBuffer, 0, SUCCESS_HEADER); + Atomics.notify(intBuffer, 0); + // all fetching after this goes through a new postmessage call with getMore + // this allows for parallel requests + } + catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } +}); +self.postMessage({ inited: true }) diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index f468feb5b1..d5dee67414 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -25,6 +25,7 @@ import io import json from email.parser import Parser +from importlib.resources import files from typing import TYPE_CHECKING, Any import js # type: ignore[import] @@ -42,11 +43,18 @@ """ HEADERS_TO_IGNORE = ("user-agent",) +NO_RESULT = 0 SUCCESS_HEADER = -1 SUCCESS_EOF = -2 ERROR_TIMEOUT = -3 ERROR_EXCEPTION = -4 +_STREAMING_WORKER_CODE = ( + files(__package__) + .joinpath("emscripten_fetch_worker.js") + .read_text(encoding="utf-8") +) + class _RequestError(Exception): def __init__( @@ -70,111 +78,6 @@ class _TimeoutError(_RequestError): pass -_STREAMING_WORKER_CODE = """ -let SUCCESS_HEADER = -1 -let SUCCESS_EOF = -2 -let ERROR_TIMEOUT = -3 -let ERROR_EXCEPTION = -4 - -let connections = {}; -let nextConnectionID = 1; - -self.addEventListener("message", async function (event) { - if(event.data.close) - { - let connectionID=event.data.close; - delete connections[connectionID]; - return; - }else if (event.data.getMore) { - let connectionID = event.data.getMore; - let { curOffset, value, reader,intBuffer,byteBuffer } = connections[connectionID]; - // if we still have some in buffer, then just send it back straight away - if (!value || curOffset >= value.length) { - // read another buffer if required - try - { - let readResponse = await reader.read(); - - if (readResponse.done) { - // read everything - clear connection and return - delete connections[connectionID]; - Atomics.store(intBuffer, 0, SUCCESS_EOF); - Atomics.notify(intBuffer, 0); - // finished reading successfully - // return from event handler - return; - } - curOffset = 0; - connections[connectionID].value = readResponse.value; - value=readResponse.value; - }catch(error) - { - console.log("Request exception:", error); - let errorBytes = encoder.encode(error.message); - let written = errorBytes.length; - byteBuffer.set(errorBytes); - intBuffer[1] = written; - Atomics.store(intBuffer, 0, ERROR_EXCEPTION); - Atomics.notify(intBuffer, 0); - } - } - - // send as much buffer as we can - let curLen = value.length - curOffset; - if (curLen > byteBuffer.length) { - curLen = byteBuffer.length; - } - byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0) - - Atomics.store(intBuffer, 0, curLen);// store current length in bytes - Atomics.notify(intBuffer, 0); - curOffset+=curLen; - connections[connectionID].curOffset = curOffset; - - return; - } else { - // start fetch - let connectionID = nextConnectionID; - nextConnectionID += 1; - const encoder = new TextEncoder(); - const intBuffer = new Int32Array(event.data.buffer); - const byteBuffer = new Uint8Array(event.data.buffer, 8) - try { - const response = await fetch(event.data.url, event.data.fetchParams); - // return the headers first via textencoder - var headers = []; - for (const pair of response.headers.entries()) { - headers.push([pair[0], pair[1]]); - } - headerObj = { headers: headers, status: response.status, connectionID }; - const headerText = JSON.stringify(headerObj); - let headerBytes = encoder.encode(headerText); - let written = headerBytes.length; - byteBuffer.set(headerBytes); - intBuffer[1] = written; - // make a connection - connections[connectionID] = { reader:response.body.getReader(),intBuffer:intBuffer,byteBuffer:byteBuffer,value:undefined,curOffset:0 }; - // set header ready - Atomics.store(intBuffer, 0, SUCCESS_HEADER); - Atomics.notify(intBuffer, 0); - // all fetching after this goes through a new postmessage call with getMore - // this allows for parallel requests - } - catch (error) { - console.log("Request exception:", error); - let errorBytes = encoder.encode(error.message); - let written = errorBytes.length; - byteBuffer.set(errorBytes); - intBuffer[1] = written; - Atomics.store(intBuffer, 0, ERROR_EXCEPTION); - Atomics.notify(intBuffer, 0); - } - } -}); -self.postMessage({inited:true}) -""" - - def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy: return to_js(dict_val, dict_converter=js.Object.fromEntries) @@ -257,13 +160,13 @@ def __init__(self) -> None: [_STREAMING_WORKER_CODE], _obj_from_dict({"type": "application/javascript"}) ) - def promise_resolver(res: JsProxy, rej: JsProxy) -> None: + def promise_resolver(resolve_fn: JsProxy, reject_fn: JsProxy) -> None: def onMsg(e: JsProxy) -> None: self.streaming_ready = True - res(e) + resolve_fn(e) def onErr(e: JsProxy) -> None: - rej(e) + reject_fn(e) self._worker.onmessage = onMsg self._worker.onerror = onErr @@ -299,13 +202,13 @@ def send(self, request: EmscriptenRequest) -> EmscriptenResponse: ) # wait for the worker to send something js.Atomics.wait(int_buffer, 0, 0, timeout) - if int_buffer[0] == 0: + if int_buffer[0] == NO_RESULT: raise _TimeoutError( "Timeout connecting to streaming request", request=request, response=None, ) - if int_buffer[0] == SUCCESS_HEADER: + elif int_buffer[0] == SUCCESS_HEADER: # got response # header length is in second int of intBuffer string_len = int_buffer[1] diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 388d31f61b..fffbab0d54 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -51,8 +51,8 @@ def run_webworker(self, code: str) -> Any: code = textwrap.dedent(code) return self.selenium.run_js( - """ - let worker = new Worker('{}'); + f""" + let worker = new Worker('https://{self.host}:{self.port}/pyodide/webworker_dev.js'); let p = new Promise((res, rej) => {{ worker.onmessageerror = e => rej(e); worker.onerror = e => rej(e); @@ -63,13 +63,10 @@ def run_webworker(self, code: str) -> Any: rej(e.data.error); }} }}; - worker.postMessage({{ python: {!r} }}); + worker.postMessage({{ python: {repr(code)} }}); }}); return await p; - """.format( - f"https://{self.host}:{self.port}/pyodide/webworker_dev.js", - code, - ), + """, pyodide_checks=False, ) From f871d98270eb0da6e07ce74c82be3136675edd29 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 23:10:03 +0000 Subject: [PATCH 23/49] prettier and eslint --- .eslintrc.yml | 3 + .pre-commit-config.yaml | 8 +- .../emscripten/emscripten_fetch_worker.js | 187 +++++++++--------- src/urllib3/contrib/emscripten/fetch.py | 14 +- 4 files changed, 116 insertions(+), 96 deletions(-) diff --git a/.eslintrc.yml b/.eslintrc.yml index 100af5bf92..b1be2badbd 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -2,3 +2,6 @@ env: es2020 : true worker: true rules: {} +extends: +- eslint:recommended + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab2430c963..7cb61dedcd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,13 @@ repos: - id: flake8 additional_dependencies: [flake8-2020] + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.1.0" + hooks: + - id: prettier + types_or: [javascript] - repo: https://github.com/pre-commit/mirrors-eslint rev: v8.53.0 hooks: - - id: eslint \ No newline at end of file + - id: eslint + args: ["--fix"] \ No newline at end of file diff --git a/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js b/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js index 063a862e8c..243b86222f 100644 --- a/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js +++ b/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js @@ -1,101 +1,110 @@ - -let NO_RESULT_YET = 0; -let SUCCESS_HEADER = -1; -let SUCCESS_EOF = -2; -let ERROR_TIMEOUT = -3; -let ERROR_EXCEPTION = -4; +let Status = { + SUCCESS_HEADER: -1, + SUCCESS_EOF: -2, + ERROR_TIMEOUT: -3, + ERROR_EXCEPTION: -4, +}; let connections = {}; let nextConnectionID = 1; +const encoder = new TextEncoder(); self.addEventListener("message", async function (event) { - if (event.data.close) { - let connectionID = event.data.close; - delete connections[connectionID]; - return; - } else if (event.data.getMore) { - let connectionID = event.data.getMore; - let { curOffset, value, reader, intBuffer, byteBuffer } = connections[connectionID]; - // if we still have some in buffer, then just send it back straight away - if (!value || curOffset >= value.length) { - // read another buffer if required - try { - let readResponse = await reader.read(); + if (event.data.close) { + let connectionID = event.data.close; + delete connections[connectionID]; + return; + } else if (event.data.getMore) { + let connectionID = event.data.getMore; + let { curOffset, value, reader, intBuffer, byteBuffer } = + connections[connectionID]; + // if we still have some in buffer, then just send it back straight away + if (!value || curOffset >= value.length) { + // read another buffer if required + try { + let readResponse = await reader.read(); - if (readResponse.done) { - // read everything - clear connection and return - delete connections[connectionID]; - Atomics.store(intBuffer, 0, SUCCESS_EOF); - Atomics.notify(intBuffer, 0); - // finished reading successfully - // return from event handler - return; - } - curOffset = 0; - connections[connectionID].value = readResponse.value; - value = readResponse.value; - } catch (error) { - console.log("Request exception:", error); - let errorBytes = encoder.encode(error.message); - let written = errorBytes.length; - byteBuffer.set(errorBytes); - intBuffer[1] = written; - Atomics.store(intBuffer, 0, ERROR_EXCEPTION); - Atomics.notify(intBuffer, 0); - } + if (readResponse.done) { + // read everything - clear connection and return + delete connections[connectionID]; + Atomics.store(intBuffer, 0, Status.SUCCESS_EOF); + Atomics.notify(intBuffer, 0); + // finished reading successfully + // return from event handler + return; } + curOffset = 0; + connections[connectionID].value = readResponse.value; + value = readResponse.value; + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } - // send as much buffer as we can - let curLen = value.length - curOffset; - if (curLen > byteBuffer.length) { - curLen = byteBuffer.length; - } - byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0) + // send as much buffer as we can + let curLen = value.length - curOffset; + if (curLen > byteBuffer.length) { + curLen = byteBuffer.length; + } + byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0); - Atomics.store(intBuffer, 0, curLen);// store current length in bytes - Atomics.notify(intBuffer, 0); - curOffset += curLen; - connections[connectionID].curOffset = curOffset; + Atomics.store(intBuffer, 0, curLen); // store current length in bytes + Atomics.notify(intBuffer, 0); + curOffset += curLen; + connections[connectionID].curOffset = curOffset; - return; - } else { - // start fetch - let connectionID = nextConnectionID; - nextConnectionID += 1; - const encoder = new TextEncoder(); - const intBuffer = new Int32Array(event.data.buffer); - const byteBuffer = new Uint8Array(event.data.buffer, 8) - intBuffer[0] = NO_RESULT_YET; - try { - const response = await fetch(event.data.url, event.data.fetchParams); - // return the headers first via textencoder - var headers = []; - for (const pair of response.headers.entries()) { - headers.push([pair[0], pair[1]]); - } - headerObj = { headers: headers, status: response.status, connectionID }; - const headerText = JSON.stringify(headerObj); - let headerBytes = encoder.encode(headerText); - let written = headerBytes.length; - byteBuffer.set(headerBytes); - intBuffer[1] = written; - // make a connection - connections[connectionID] = { reader: response.body.getReader(), intBuffer: intBuffer, byteBuffer: byteBuffer, value: undefined, curOffset: 0 }; - // set header ready - Atomics.store(intBuffer, 0, SUCCESS_HEADER); - Atomics.notify(intBuffer, 0); - // all fetching after this goes through a new postmessage call with getMore - // this allows for parallel requests - } - catch (error) { - console.log("Request exception:", error); - let errorBytes = encoder.encode(error.message); - let written = errorBytes.length; - byteBuffer.set(errorBytes); - intBuffer[1] = written; - Atomics.store(intBuffer, 0, ERROR_EXCEPTION); - Atomics.notify(intBuffer, 0); - } + return; + } else { + // start fetch + let connectionID = nextConnectionID; + nextConnectionID += 1; + const intBuffer = new Int32Array(event.data.buffer); + const byteBuffer = new Uint8Array(event.data.buffer, 8); + try { + const response = await fetch(event.data.url, event.data.fetchParams); + // return the headers first via textencoder + var headers = []; + for (const pair of response.headers.entries()) { + headers.push([pair[0], pair[1]]); + } + let headerObj = { + headers: headers, + status: response.status, + connectionID, + }; + const headerText = JSON.stringify(headerObj); + let headerBytes = encoder.encode(headerText); + let written = headerBytes.length; + byteBuffer.set(headerBytes); + intBuffer[1] = written; + // make a connection + connections[connectionID] = { + reader: response.body.getReader(), + intBuffer: intBuffer, + byteBuffer: byteBuffer, + value: undefined, + curOffset: 0, + }; + // set header ready + Atomics.store(intBuffer, 0, Status.SUCCESS_HEADER); + Atomics.notify(intBuffer, 0); + // all fetching after this goes through a new postmessage call with getMore + // this allows for parallel requests + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); } + } }); -self.postMessage({ inited: true }) +self.postMessage({ inited: true }); diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index d5dee67414..c5c09692f8 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -43,7 +43,6 @@ """ HEADERS_TO_IGNORE = ("user-agent",) -NO_RESULT = 0 SUCCESS_HEADER = -1 SUCCESS_EOF = -2 ERROR_TIMEOUT = -3 @@ -123,9 +122,12 @@ def readinto(self, byte_obj: Buffer) -> int: return 0 if self.read_len == 0: # wait for the worker to send something - js.Atomics.store(self.int_buffer, 0, 0) + js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT) self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id})) - if js.Atomics.wait(self.int_buffer, 0, 0, self.timeout) == "timed-out": + if ( + js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout) + == "timed-out" + ): raise _TimeoutError data_len = self.int_buffer[0] if data_len > 0: @@ -188,7 +190,7 @@ def send(self, request: EmscriptenRequest) -> EmscriptenResponse: int_buffer = js.Int32Array.new(shared_buffer) byte_buffer = js.Uint8Array.new(shared_buffer, 8) - js.Atomics.store(int_buffer, 0, 0) + js.Atomics.store(int_buffer, 0, ERROR_TIMEOUT) js.Atomics.notify(int_buffer, 0) absolute_url = js.URL.new(request.url, js.location).href self._worker.postMessage( @@ -201,8 +203,8 @@ def send(self, request: EmscriptenRequest) -> EmscriptenResponse: ) ) # wait for the worker to send something - js.Atomics.wait(int_buffer, 0, 0, timeout) - if int_buffer[0] == NO_RESULT: + js.Atomics.wait(int_buffer, 0, ERROR_TIMEOUT, timeout) + if int_buffer[0] == ERROR_TIMEOUT: raise _TimeoutError( "Timeout connecting to streaming request", request=request, From e0d973b76d08af59e62b37ed180c489a1a98248d Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 17 Nov 2023 23:39:16 +0000 Subject: [PATCH 24/49] made tests less verbose --- test/contrib/emscripten/test_emscripten.py | 79 ++++++---------------- 1 file changed, 21 insertions(+), 58 deletions(-) diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 284d09e371..e5ae4cdf58 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -7,6 +7,8 @@ from urllib3.fields import _TYPE_FIELD_VALUE_TUPLE +from ...port_helpers import find_unused_port + if sys.version_info < (3, 11): # pyodide only works on 3.11+ pytest.skip(allow_module_level=True) @@ -34,10 +36,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3.response import HTTPResponse conn = HTTPConnection(host, port) - method = "GET" - path = "/" - url = f"http://{host}:{port}{path}" - conn.request(method, url) + conn.request("GET", f"http://{host}:{port}/") response = conn.getresponse() assert isinstance(response, HTTPResponse) data = response.data @@ -59,11 +58,8 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3.connection import HTTPConnection conn = HTTPConnection(host, port) - method = "GET" - path = "/" - url = f"http://{host}:{port}{path}" try: - conn.request(method, url) + conn.request("GET", f"http://{host}:{port}/") conn.getresponse() pytest.fail("Should have thrown ResponseError here") except BaseException as ex: @@ -83,24 +79,14 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3.connection import HTTPConnection conn = HTTPConnection(host, port) - method = "GET" - path = "/" - url = f"http://{host}:{port}{path}" try: - conn.request(method, url) + conn.request("GET", f"http://{host}:{port}/") _ = conn.getresponse() pytest.fail("No response, should throw exception.") except BaseException as ex: assert isinstance(ex, urllib3.exceptions.ResponseError) - import socket - - sock = socket.socket() - sock.bind(("", 0)) - free_port = sock.getsockname()[1] - sock.close() - - pyodide_test(selenium, testserver_http.http_host, free_port) + pyodide_test(selenium, testserver_http.http_host, find_unused_port()) @copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] @@ -111,10 +97,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3.response import HTTPResponse conn = HTTPConnection(host, port) - method = "GET" - path = "/status?status=404 NOT FOUND" - url = f"http://{host}:{port}{path}" - conn.request(method, url) + conn.request("GET", f"http://{host}:{port}/status?status=404 NOT FOUND") response = conn.getresponse() assert isinstance(response, HTTPResponse) assert response.status == 404 @@ -135,10 +118,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3.connection import HTTPConnection conn = HTTPConnection(host, port, timeout=1.0) - method = "GET" - path = "/" - url = f"http://{host}:{port}{path}" - conn.request(method, url) + conn.request("GET", f"http://{host}:{port}/") conn.getresponse() assert urllib3.contrib.emscripten.fetch._SHOWN_TIMEOUT_WARNING @@ -159,11 +139,9 @@ def test_timeout_in_worker( from urllib3.exceptions import TimeoutError from urllib3.connection import HTTPConnection conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) - method = "GET" - url = "http://{testserver_http.http_host}:{testserver_http.http_port}/slow" result=-1 try: - conn.request(method, url) + conn.request("GET","http://{testserver_http.http_host}:{testserver_http.http_port}/slow") _response = conn.getresponse() result=-3 except TimeoutError as e: @@ -185,10 +163,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3.response import HTTPResponse conn = HTTPSConnection(host, port) - method = "GET" - path = "/" - url = f"https://{host}:{port}{path}" - conn.request(method, url) + conn.request("GET", f"https://{host}:{port}/") response = conn.getresponse() assert isinstance(response, HTTPResponse) data = response.data @@ -208,10 +183,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3.response import HTTPResponse conn = HTTPSConnection(host, port) - method = "GET" - path = "/" - url = f"https://{host}:{port}{path}" - conn.request(method, url, preload_content=True) + conn.request("GET", f"https://{host}:{port}/", preload_content=True) response = conn.getresponse() assert isinstance(response, HTTPResponse) data = response.data @@ -233,10 +205,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3.response import HTTPResponse conn = HTTPSConnection(host, port) - method = "GET" - path = "/" - url = f"https://{host}:{port}{path}" - conn.request(method, url, preload_content=False) + conn.request("GET", f"https://{host}:{port}/", preload_content=False) response = conn.getresponse() assert isinstance(response, HTTPResponse) data = response.data @@ -257,14 +226,11 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt from urllib3 import HTTPSConnectionPool with HTTPSConnectionPool(host, port) as pool: - method = "POST" path = "/specific_method?method=POST" - response = pool.request(method, path) + response = pool.request("POST", path) assert response.status == 200 - method = "PUT" - path = "/specific_method?method=POST" - response = pool.request(method, path) + response = pool.request("PUT", path) assert response.status == 400 pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) @@ -293,9 +259,7 @@ def test_streaming_download( import js conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) -method = "GET" -url = "{bigfile_url}" -conn.request(method, url,preload_content=False) +conn.request("GET", "{bigfile_url}",preload_content=False) response = conn.getresponse() assert isinstance(response, HTTPResponse) assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==False @@ -325,9 +289,7 @@ def test_streaming_notready_warning( from urllib3.connection import HTTPConnection conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) -method = "GET" -url = "{bigfile_url}" -conn.request(method, url,preload_content=False) +conn.request("GET", "{bigfile_url}",preload_content=False) response = conn.getresponse() assert isinstance(response, HTTPResponse) data=response.data.decode('utf-8') @@ -354,10 +316,11 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt "to": {"eat": "buns", "with": ["marmalade", "and custard"]}, } conn = HTTPConnection(host, port) - method = "POST" - path = "/echo_json" - url = f"http://{host}:{port}{path}" - conn.request(method, url, body=json.dumps(json_data).encode("utf-8")) + conn.request( + "POST", + f"http://{host}:{port}/echo_json", + body=json.dumps(json_data).encode("utf-8"), + ) response = conn.getresponse() assert isinstance(response, HTTPResponse) data = response.json() From 5aaaab6e8f97ab996d90ee8d64056fcc2990a86a Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sat, 18 Nov 2023 00:11:03 +0000 Subject: [PATCH 25/49] simplified multiple use of decorators --- test/contrib/emscripten/conftest.py | 6 -- test/contrib/emscripten/test_emscripten.py | 106 ++++++++++++--------- 2 files changed, 60 insertions(+), 52 deletions(-) diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index fffbab0d54..e21cf47640 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -17,12 +17,6 @@ from dummyserver.testcase import HTTPDummyProxyTestCase from dummyserver.tornadoserver import run_tornado_app, run_tornado_loop_in_thread -# only run this stuff if pytest_pyodide is installed -# pytest_pyodide = pytest.importorskip("pytest_pyodide") - -# if sys.version_info<(3,11): -# pytest.skip() - @pytest.fixture(scope="module") def testserver_http( diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index e5ae4cdf58..935cf9fd20 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -28,7 +28,18 @@ pytest_pyodide.runner.CHROME_FLAGS.append("ignore-certificate-errors") -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +# copy our wheel file to pyodide and install it +def install_urllib3_wheel() -> ( + typing.Callable[ + [typing.Callable[..., typing.Any]], typing.Callable[..., typing.Any] + ] +): + return copy_files_to_pyodide( # type: ignore[no-any-return] + file_list=[("dist/*.whl", "/tmp")], install_wheels=True + ) + + +@install_urllib3_wheel() def test_index(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] @@ -46,7 +57,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt # wrong protocol / protocol error etc. should raise an exception of urllib3.exceptions.ResponseError -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_wrong_protocol( selenium: typing.Any, testserver_http: PyodideServerInfo ) -> None: @@ -69,7 +80,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt # no connection - should raise -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_no_response(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] @@ -89,7 +100,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, find_unused_port()) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_404(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] @@ -108,7 +119,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt # setting timeout should show a warning to js console # if we're on the ui thread, because XMLHttpRequest doesn't # support timeout in async mode if globalThis == Window -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_timeout_warning( selenium: typing.Any, testserver_http: PyodideServerInfo ) -> None: @@ -125,15 +136,15 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_timeout_in_worker( selenium: typing.Any, testserver_http: PyodideServerInfo, run_from_server: ServerRunnerInfo, ) -> None: worker_code = f""" - import micropip - await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-typing.Any.whl',deps=False) + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) import urllib3.contrib.emscripten.fetch await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() from urllib3.exceptions import TimeoutError @@ -141,7 +152,7 @@ def test_timeout_in_worker( conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) result=-1 try: - conn.request("GET","http://{testserver_http.http_host}:{testserver_http.http_port}/slow") + conn.request("GET","/slow") _response = conn.getresponse() result=-3 except TimeoutError as e: @@ -155,7 +166,7 @@ def test_timeout_in_worker( assert result == 1 -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_index_https(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] @@ -172,7 +183,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_non_streaming_no_fallback_warning( selenium: typing.Any, testserver_http: PyodideServerInfo ) -> None: @@ -194,7 +205,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_streaming_fallback_warning( selenium: typing.Any, testserver_http: PyodideServerInfo ) -> None: @@ -236,7 +247,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_streaming_download( selenium: typing.Any, testserver_http: PyodideServerInfo, @@ -246,31 +257,33 @@ def test_streaming_download( # as you can't do it on main thread # this should return the 17mb big file, and - # should not log typing.Any warning about falling back + # should not log any warning about falling back bigfile_url = ( f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" ) - worker_code = f"""import micropip -await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-typing.Any.whl',deps=False) -import urllib3.contrib.emscripten.fetch -await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() -from urllib3.response import HTTPResponse -from urllib3.connection import HTTPConnection -import js - -conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) -conn.request("GET", "{bigfile_url}",preload_content=False) -response = conn.getresponse() -assert isinstance(response, HTTPResponse) -assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==False -data=response.data.decode('utf-8') -data + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import HTTPResponse + from urllib3.connection import HTTPConnection + import js + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{bigfile_url}",preload_content=False) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==False + data=response.data.decode('utf-8') + data """ result = run_from_server.run_webworker(worker_code) assert len(result) == 17825792 -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_streaming_notready_warning( selenium: typing.Any, testserver_http: PyodideServerInfo, @@ -282,25 +295,26 @@ def test_streaming_notready_warning( bigfile_url = ( f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" ) - worker_code = f"""import micropip -await micropip.install('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/urllib3-2.0.7-py3-none-typing.Any.whl',deps=False) -import urllib3.contrib.emscripten.fetch -from urllib3.response import HTTPResponse -from urllib3.connection import HTTPConnection - -conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) -conn.request("GET", "{bigfile_url}",preload_content=False) -response = conn.getresponse() -assert isinstance(response, HTTPResponse) -data=response.data.decode('utf-8') -assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True -data -""" + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + import urllib3 + from urllib3.response import HTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{bigfile_url}",preload_content=False) + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + data=response.data.decode('utf-8') + assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True + data + """ result = run_from_server.run_webworker(worker_code) assert len(result) == 17825792 -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_post_receive_json( selenium: typing.Any, testserver_http: PyodideServerInfo ) -> None: @@ -329,7 +343,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) -@copy_files_to_pyodide(file_list=[("dist/*.whl", "/tmp")], install_wheels=True) # type: ignore[misc] +@install_urllib3_wheel() def test_upload(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] From 5e4b9ae4eb1bb455e7ab25c076c1a2815c140124 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sat, 18 Nov 2023 00:27:56 +0000 Subject: [PATCH 26/49] simplified class passed to testserver_http --- test/contrib/emscripten/conftest.py | 15 ++++++++++++--- test/contrib/emscripten/test_emscripten.py | 1 - 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index e21cf47640..3b103b274d 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -5,8 +5,9 @@ import mimetypes import os import textwrap +from dataclasses import dataclass from pathlib import Path -from typing import Any, Generator, Type +from typing import Any, Generator from urllib.parse import urlsplit import pytest @@ -28,7 +29,11 @@ def testserver_http( print( f"Server:{server.http_host}:{server.http_port},https({server.https_port}) [{dist_dir}]" ) - yield server + yield PyodideServerInfo( + http_host=server.http_host, + http_port=server.http_port, + https_port=server.https_port, + ) print("Server teardown") server.teardown_class() @@ -174,4 +179,8 @@ async def run_app() -> None: cls._stack = stack.pop_all() -PyodideServerInfo = Type[PyodideDummyServerTestCase] +@dataclass +class PyodideServerInfo: + http_port: int + https_port: int + http_host: str diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 935cf9fd20..e6a3477a36 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -230,7 +230,6 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt def test_specific_method( selenium: typing.Any, testserver_http: PyodideServerInfo, - run_from_server: ServerRunnerInfo, ) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] From 319d0679daef225ef01f7eae8f255d850511f8ac Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sat, 18 Nov 2023 00:34:24 +0000 Subject: [PATCH 27/49] ci action updates --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b448fb19e..69f7dfa81b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,8 +108,8 @@ jobs: - name: "Install dependencies" run: python -m pip install --upgrade pip setuptools nox - - name: "Install chrome" - uses: browser-actions/setup-chrome@v1 + - name: "Install Chrome" + uses: browser-actions/setup-chrome@11cef13cde73820422f9263a707fb8029808e191 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version From 7918f0c052b275eef5fd9c345942f73e55be1f9f Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sat, 18 Nov 2023 00:55:21 +0000 Subject: [PATCH 28/49] firefox emscripten tests --- .github/workflows/ci.yml | 3 +++ noxfile.py | 28 +++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69f7dfa81b..a8018f49ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,9 @@ jobs: - name: "Install Chrome" uses: browser-actions/setup-chrome@11cef13cde73820422f9263a707fb8029808e191 if: ${{ matrix.nox-session == 'emscripten' }} + - name: "Install Firefox" + uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead + if: ${{ matrix.nox-session == 'emscripten' }} - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version run: nox -s ${NOX_SESSION:-test-$PYTHON_VERSION} --error-on-missing-interpreters diff --git a/noxfile.py b/noxfile.py index 1b16447518..c3e30793ab 100644 --- a/noxfile.py +++ b/noxfile.py @@ -161,9 +161,9 @@ def lint(session: nox.Session) -> None: # because you can't override the test runner properties - # https://github.com/pyodide/pytest-pyodide/issues/118 @nox.session(python="3.11") -@nox.parametrize("runner", ["chrome"]) +@nox.parametrize("runner", ["firefox", "chrome"]) def emscripten(session: nox.Session, runner: str) -> None: - """Test on Emscripten with Pyodide & Chrome""" + """Test on Emscripten with Pyodide & Chrome / Firefox""" session.install("-r", "emscripten-requirements.txt") # build wheel into dist folder session.run("python", "-m", "build") @@ -172,7 +172,7 @@ def emscripten(session: nox.Session, runner: str) -> None: if "PYODIDE_ROOT" in os.environ: # we have a pyodide build tree checked out # use the dist directory from that - dist_dir = Path(os.environ["PYODIDE_ROOT"]) + dist_dir = Path(os.environ["PYODIDE_ROOT"]) / "dist" else: # we don't have a build tree, get one # that matches the version of pyodide build @@ -232,6 +232,28 @@ def emscripten(session: nox.Session, runner: str) -> None: "test", ], ) + elif runner == "firefox": + driver = typing.cast( + str, + session.run( + "python", + "-c", + "from webdriver_manager.firefox import GeckoDriverManager;print(GeckoDriverManager().install())", + silent=True, + ), + ).strip() + session.env["PATH"] = f"{Path(driver).parent}:{session.env['PATH']}" + + tests_impl( + session, + pytest_extra_args=[ + "--rt", + "firefox-no-host", + "--dist-dir", + str(dist_dir), + "test", + ], + ) else: raise ValueError(f"Unknown runnner: {runner}") From 611e16df538fc5736496a77ccd0276a10c31adec Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sat, 18 Nov 2023 01:24:01 +0000 Subject: [PATCH 29/49] test checks console output --- test/contrib/emscripten/conftest.py | 4 -- test/contrib/emscripten/test_emscripten.py | 61 +++++++++++++++++++++- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 3b103b274d..9876160943 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -114,7 +114,6 @@ def slow(self, _req: HTTPServerRequest) -> Response: return Response("TEN SECONDS LATER") def bigfile(self, req: HTTPServerRequest) -> Response: - print("Bigfile requested") # great big text file, should force streaming # if supported bigdata = 1048576 * b"WOOO YAY BOOYAKAH" @@ -128,7 +127,6 @@ def pyodide(self, req: HTTPServerRequest) -> Response: file_path = Path(PyodideTestingApp.pyodide_dist_dir, *path_split[2:]) if file_path.exists(): mime_type, encoding = mimetypes.guess_type(file_path) - print(file_path, mime_type) if not mime_type: mime_type = "text/plain" self.set_header("Content-Type", mime_type) @@ -142,9 +140,7 @@ def pyodide(self, req: HTTPServerRequest) -> Response: def wheel(self, _req: HTTPServerRequest) -> Response: # serve our wheel wheel_folder = Path(__file__).parent.parent.parent.parent / "dist" - print(wheel_folder) wheels = list(wheel_folder.glob("*.whl")) - print(wheels) if len(wheels) > 0: resp = Response( body=wheels[0].read_bytes(), diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index e6a3477a36..7c85f025fc 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -125,12 +125,27 @@ def test_timeout_warning( ) -> None: @run_in_pyodide() # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js # type: ignore[import] + import urllib3.contrib.emscripten.fetch from urllib3.connection import HTTPConnection + old_log = js.console.warn + log_msgs = [] + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log + conn = HTTPConnection(host, port, timeout=1.0) conn.request("GET", f"http://{host}:{port}/") conn.getresponse() + js.console.warn = old_log + # should have shown timeout warning exactly once by now + print(log_msgs) + assert len([x for x in log_msgs if x.find("Warning: Timeout") != -1]) == 1 assert urllib3.contrib.emscripten.fetch._SHOWN_TIMEOUT_WARNING pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) @@ -189,17 +204,33 @@ def test_non_streaming_no_fallback_warning( ) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js + import urllib3.contrib.emscripten.fetch from urllib3.connection import HTTPSConnection from urllib3.response import HTTPResponse + log_msgs = [] + old_log = js.console.warn + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log conn = HTTPSConnection(host, port) conn.request("GET", f"https://{host}:{port}/", preload_content=True) response = conn.getresponse() + js.console.warn = old_log assert isinstance(response, HTTPResponse) data = response.data assert data.decode("utf-8") == "Dummy server!" # no console warnings because we didn't ask it to stream the response + # check no log messages + assert ( + len([x for x in log_msgs if x.find("Can't stream HTTP requests") != -1]) + == 0 + ) assert not urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) @@ -211,25 +242,43 @@ def test_streaming_fallback_warning( ) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js + import urllib3.contrib.emscripten.fetch from urllib3.connection import HTTPSConnection from urllib3.response import HTTPResponse + log_msgs = [] + old_log = js.console.warn + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log + conn = HTTPSConnection(host, port) conn.request("GET", f"https://{host}:{port}/", preload_content=False) response = conn.getresponse() + js.console.warn = old_log assert isinstance(response, HTTPResponse) data = response.data assert data.decode("utf-8") == "Dummy server!" - # check that it has warned about falling back to non-streaming fetch + # check that it has warned about falling back to non-streaming fetch exactly once + assert ( + len([x for x in log_msgs if x.find("Can't stream HTTP requests") != -1]) + == 1 + ) assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) +@install_urllib3_wheel() def test_specific_method( selenium: typing.Any, testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, ) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] @@ -297,15 +346,25 @@ def test_streaming_notready_warning( worker_code = f""" import pyodide_js as pjs await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + import js import urllib3 from urllib3.response import HTTPResponse from urllib3.connection import HTTPConnection + log_msgs=[] + old_log=js.console.warn + def capture_log(*args): + log_msgs.append(str(args)) + old_log(*args) + js.console.warn=capture_log + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) conn.request("GET", "{bigfile_url}",preload_content=False) + js.console.warn=old_log response = conn.getresponse() assert isinstance(response, HTTPResponse) data=response.data.decode('utf-8') + assert len([x for x in log_msgs if x.find("Can't stream HTTP requests")!=-1])==1 assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True data """ From 242af7b2ab2c0f4b4a39af9181e0009329ec831e Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sat, 18 Nov 2023 01:40:07 +0000 Subject: [PATCH 30/49] formatting and comments --- noxfile.py | 4 +- src/urllib3/contrib/emscripten/fetch.py | 93 +++++++++++++------------ 2 files changed, 50 insertions(+), 47 deletions(-) diff --git a/noxfile.py b/noxfile.py index c3e30793ab..612c914823 100644 --- a/noxfile.py +++ b/noxfile.py @@ -158,8 +158,8 @@ def lint(session: nox.Session) -> None: # TODO: node support is not tested yet - it should work if you require('xmlhttprequest') before # loading pyodide, but there is currently no nice way to do this with pytest-pyodide -# because you can't override the test runner properties - -# https://github.com/pyodide/pytest-pyodide/issues/118 +# because you can't override the test runner properties easily - see +# https://github.com/pyodide/pytest-pyodide/issues/118 for more @nox.session(python="3.11") @nox.parametrize("runner", ["firefox", "chrome"]) def emscripten(session: nox.Session, runner: str) -> None: diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index c5c09692f8..dd2890d48c 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -19,6 +19,9 @@ Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch. + +NB: in this code, there are a lot of javascript objects. They are named js_* +to make it clear what type of object they are. """ from __future__ import annotations @@ -158,24 +161,24 @@ def __init__(self) -> None: # make web-worker and data buffer on startup self.streaming_ready = False - dataBlob = js.Blob.new( + js_data_blob = js.Blob.new( [_STREAMING_WORKER_CODE], _obj_from_dict({"type": "application/javascript"}) ) - def promise_resolver(resolve_fn: JsProxy, reject_fn: JsProxy) -> None: + def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None: def onMsg(e: JsProxy) -> None: self.streaming_ready = True - resolve_fn(e) + js_resolve_fn(e) def onErr(e: JsProxy) -> None: - reject_fn(e) + js_reject_fn(e) - self._worker.onmessage = onMsg - self._worker.onerror = onErr + self.js_worker.onmessage = onMsg + self.js_worker.onerror = onErr - dataURL = js.URL.createObjectURL(dataBlob) - self._worker = js.globalThis.Worker.new(dataURL) - self._worker_ready_promise = js.globalThis.Promise.new(promise_resolver) + js_data_url = js.URL.createObjectURL(js_data_blob) + self.js_worker = js.globalThis.Worker.new(js_data_url) + self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver) def send(self, request: EmscriptenRequest) -> EmscriptenResponse: headers = { @@ -186,39 +189,39 @@ def send(self, request: EmscriptenRequest) -> EmscriptenResponse: fetch_data = {"headers": headers, "body": to_js(body), "method": request.method} # start the request off in the worker timeout = int(1000 * request.timeout) if request.timeout > 0 else None - shared_buffer = js.SharedArrayBuffer.new(1048576) - int_buffer = js.Int32Array.new(shared_buffer) - byte_buffer = js.Uint8Array.new(shared_buffer, 8) - - js.Atomics.store(int_buffer, 0, ERROR_TIMEOUT) - js.Atomics.notify(int_buffer, 0) - absolute_url = js.URL.new(request.url, js.location).href - self._worker.postMessage( + js_shared_buffer = js.SharedArrayBuffer.new(1048576) + js_int_buffer = js.Int32Array.new(js_shared_buffer) + js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8) + + js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT) + js.Atomics.notify(js_int_buffer, 0) + js_absolute_url = js.URL.new(request.url, js.location).href + self.js_worker.postMessage( _obj_from_dict( { - "buffer": shared_buffer, - "url": absolute_url, + "buffer": js_shared_buffer, + "url": js_absolute_url, "fetchParams": fetch_data, } ) ) # wait for the worker to send something - js.Atomics.wait(int_buffer, 0, ERROR_TIMEOUT, timeout) - if int_buffer[0] == ERROR_TIMEOUT: + js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout) + if js_int_buffer[0] == ERROR_TIMEOUT: raise _TimeoutError( "Timeout connecting to streaming request", request=request, response=None, ) - elif int_buffer[0] == SUCCESS_HEADER: + elif js_int_buffer[0] == SUCCESS_HEADER: # got response # header length is in second int of intBuffer - string_len = int_buffer[1] + string_len = js_int_buffer[1] # decode the rest to a JSON string - decoder = js.TextDecoder.new() + js_decoder = js.TextDecoder.new() # this does a copy (the slice) because decode can't work on shared array # for some silly reason - json_str = decoder.decode(byte_buffer.slice(0, string_len)) + json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len)) # get it as an object response_obj = json.loads(json_str) return EmscriptenResponse( @@ -227,26 +230,26 @@ def send(self, request: EmscriptenRequest) -> EmscriptenResponse: headers=response_obj["headers"], body=io.BufferedReader( _ReadStream( - int_buffer, - byte_buffer, + js_int_buffer, + js_byte_buffer, request.timeout, - self._worker, + self.js_worker, response_obj["connectionID"], ), buffer_size=1048576, ), ) - elif int_buffer[0] == ERROR_EXCEPTION: - string_len = int_buffer[1] + elif js_int_buffer[0] == ERROR_EXCEPTION: + string_len = js_int_buffer[1] # decode the error string - decoder = js.TextDecoder.new() - json_str = decoder.decode(byte_buffer.slice(0, string_len)) + js_decoder = js.TextDecoder.new() + json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len)) raise _StreamingError( f"Exception thrown in fetch: {json_str}", request=request, response=None ) else: raise _StreamingError( - f"Unknown status from worker in fetch: {int_buffer[0]}", + f"Unknown status from worker in fetch: {js_int_buffer[0]}", request=request, response=None, ) @@ -328,34 +331,34 @@ def _show_streaming_warning() -> None: def send_request(request: EmscriptenRequest) -> EmscriptenResponse: try: - xhr = js.XMLHttpRequest.new() + js_xhr = js.XMLHttpRequest.new() if not is_in_browser_main_thread(): - xhr.responseType = "arraybuffer" + js_xhr.responseType = "arraybuffer" if request.timeout: - xhr.timeout = int(request.timeout * 1000) + js_xhr.timeout = int(request.timeout * 1000) else: - xhr.overrideMimeType("text/plain; charset=ISO-8859-15") + js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15") if request.timeout: # timeout isn't available on the main thread - show a warning in console # if it is set _show_timeout_warning() - xhr.open(request.method, request.url, False) + js_xhr.open(request.method, request.url, False) for name, value in request.headers.items(): if name.lower() not in HEADERS_TO_IGNORE: - xhr.setRequestHeader(name, value) + js_xhr.setRequestHeader(name, value) - xhr.send(to_js(request.body)) + js_xhr.send(to_js(request.body)) - headers = dict(Parser().parsestr(xhr.getAllResponseHeaders())) + headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders())) if not is_in_browser_main_thread(): - body = xhr.response.to_py().tobytes() + body = js_xhr.response.to_py().tobytes() else: - body = xhr.response.encode("ISO-8859-15") + body = js_xhr.response.encode("ISO-8859-15") return EmscriptenResponse( - status_code=xhr.status, headers=headers, body=body, request=request + status_code=js_xhr.status, headers=headers, body=body, request=request ) except JsException as err: if err.name == "TimeoutError": @@ -376,7 +379,7 @@ def streaming_ready() -> bool | None: async def wait_for_streaming_ready() -> bool: if _fetcher: - await _fetcher._worker_ready_promise + await _fetcher.js_worker_ready_promise return True else: return False From 2771537af0b6715993ddc05ebbe0b37c1811a621 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sun, 19 Nov 2023 12:38:36 +0000 Subject: [PATCH 31/49] Update ci.yml with versions --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8018f49ac..9e244f5632 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,10 +109,10 @@ jobs: run: python -m pip install --upgrade pip setuptools nox - name: "Install Chrome" - uses: browser-actions/setup-chrome@11cef13cde73820422f9263a707fb8029808e191 + uses: browser-actions/setup-chrome@11cef13cde73820422f9263a707fb8029808e191 # v1.3.0 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Install Firefox" - uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead + uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead # v1.4.0 if: ${{ matrix.nox-session == 'emscripten' }} - name: "Run tests" # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version From 38be9dfa3f97f33bd7fe7a4202b32ed68315a67a Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Sun, 19 Nov 2023 12:41:50 +0000 Subject: [PATCH 32/49] pin emscripten-requirements.txt --- emscripten-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/emscripten-requirements.txt b/emscripten-requirements.txt index 614a44bb0b..7ed8739675 100644 --- a/emscripten-requirements.txt +++ b/emscripten-requirements.txt @@ -1,3 +1,3 @@ -pytest-pyodide>=0.54.0 -pyodide-build>=0.24.0 -webdriver-manager>=4.0.0 \ No newline at end of file +pytest-pyodide==0.54.0 +pyodide-build==0.24.1 +webdriver-manager==4.0.1 \ No newline at end of file From a5ead7f2926d0664fa05350767f13be0e87a3e75 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 20 Nov 2023 14:02:35 +0000 Subject: [PATCH 33/49] derive from BaseHTTPConnection etc. --- src/urllib3/connectionpool.py | 4 +- src/urllib3/contrib/emscripten/connection.py | 44 ++++++++++++++++---- src/urllib3/contrib/emscripten/response.py | 15 ++++--- src/urllib3/response.py | 5 ++- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 70048b7aed..27d3cfc395 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -543,6 +543,8 @@ def _make_request( response._connection = response_conn # type: ignore[attr-defined] response._pool = self # type: ignore[attr-defined] + # emscripten connection doesn't have _http_vsn_str + http_version = getattr(conn, "_http_vsn_str", "HTTP/?") log.debug( '%s://%s:%s "%s %s %s" %s %s', self.scheme, @@ -551,7 +553,7 @@ def _make_request( method, url, # HTTP version - conn._http_vsn_str, # type: ignore[attr-defined] + http_version, response.status, response.length_remaining, # type: ignore[attr-defined] ) diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index 149ea865b5..07ebf3b991 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -15,8 +15,27 @@ from .request import EmscriptenRequest from .response import EmscriptenHttpResponseWrapper +try: # Compiled with SSL? + import ssl + + BaseSSLError = ssl.SSLError +except (ImportError, AttributeError): + ssl = None # type: ignore[assignment] + + class BaseSSLError(BaseException): # type: ignore[no-redef] + pass + + +if typing.TYPE_CHECKING: + from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection + + +class EmscriptenHTTPConnection: + default_port: typing.ClassVar[int] = port_by_scheme["http"] + default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS] + + timeout: None | (float) -class EmscriptenHTTPConnection(HTTPConnection): host: str port: int blocksize: int @@ -26,8 +45,8 @@ class EmscriptenHTTPConnection(HTTPConnection): proxy: Url | None proxy_config: ProxyConfig | None - is_verified: bool - proxy_is_verified: bool | None + is_verified: bool = False + proxy_is_verified: bool | None = None def __init__( self, @@ -60,7 +79,7 @@ def set_tunnel( def connect(self) -> None: pass - def request( # type: ignore[override] + def request( self, method: str, url: str, @@ -99,7 +118,7 @@ def request( # type: ignore[override] except _RequestError as e: raise ResponseError(e.message) - def getresponse(self) -> BaseHTTPResponse: # type: ignore[override] + def getresponse(self) -> BaseHTTPResponse: if self._response is not None: return EmscriptenHttpResponseWrapper( internal_response=self._response, @@ -135,15 +154,20 @@ def has_connected_to_proxy(self) -> bool: class EmscriptenHTTPSConnection(EmscriptenHTTPConnection): - default_port = port_by_scheme["https"] # type: ignore[misc] - + default_port = port_by_scheme["https"] + # all this is basically ignored, as browser handles https cert_reqs: int | str | None = None ca_certs: str | None = None ca_cert_dir: str | None = None ca_cert_data: None | str | bytes = None + cert_file: str | None + key_file: str | None + key_password: str | None + ssl_context: ssl.SSLContext | None ssl_version: int | str | None = None ssl_minimum_version: int | None = None ssl_maximum_version: int | None = None + assert_hostname: None | str | typing.Literal[False] assert_fingerprint: str | None = None def __init__( @@ -214,3 +238,9 @@ def set_cert( ca_cert_data: None | str | bytes = None, ) -> None: pass + + +# verify that this class implements BaseHTTP(s) connection correctly +if typing.TYPE_CHECKING: + _supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0) + _supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0) diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 66e23f4cd2..7d58858783 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -5,11 +5,13 @@ from dataclasses import dataclass from io import BytesIO, IOBase -from ...connection import HTTPConnection -from ...response import HTTPResponse +from ...response import BaseHTTPResponse from ...util.retry import Retry from .request import EmscriptenRequest +if typing.TYPE_CHECKING: + from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection + @dataclass class EmscriptenResponse: @@ -19,13 +21,14 @@ class EmscriptenResponse: request: EmscriptenRequest -class EmscriptenHttpResponseWrapper(HTTPResponse): +class EmscriptenHttpResponseWrapper(BaseHTTPResponse): def __init__( self, internal_response: EmscriptenResponse, url: str | None = None, - connection: HTTPConnection | None = None, + connection: BaseHTTPConnection | BaseHTTPSConnection | None = None, ): + self._pool = None # set by pool class self._body = None self._response = internal_response self._url = url @@ -48,7 +51,7 @@ def url(self, url: str | None) -> None: self._url = url @property - def connection(self) -> HTTPConnection | None: + def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None: return self._connection @property @@ -105,7 +108,7 @@ def drain_conn(self) -> None: @property def data(self) -> bytes: if self._body: - return self._body # type: ignore[return-value] + return self._body else: return self.read(cache_content=True) diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 37936f9397..c096eff34f 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -14,6 +14,9 @@ from http.client import HTTPResponse as _HttplibHTTPResponse from socket import timeout as SocketTimeout +if typing.TYPE_CHECKING: + from ._base_connection import BaseHTTPConnection + try: try: import brotlicffi as brotli # type: ignore[import] @@ -366,7 +369,7 @@ def url(self, url: str | None) -> None: raise NotImplementedError() @property - def connection(self) -> HTTPConnection | None: + def connection(self) -> BaseHTTPConnection | None: raise NotImplementedError() @property From 8978367756a93ad8d17bd408a8536a49440f5e08 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 20 Nov 2023 14:05:34 +0000 Subject: [PATCH 34/49] make posargs override extra args --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 612c914823..d153d08e3e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -56,8 +56,8 @@ def tests_impl( "--durations=10", "--strict-config", "--strict-markers", - *(session.posargs or ("test/",)), *pytest_extra_args, + *(session.posargs or ("test/",)), env={"PYTHONWARNINGS": "always::DeprecationWarning"}, ) From 4321c9dc1ddf2e5eab9d8c6327ec2218f623ddd6 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 20 Nov 2023 16:19:11 +0000 Subject: [PATCH 35/49] fixed missing length_remaining --- src/urllib3/contrib/emscripten/response.py | 63 +++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 7d58858783..37ede24d39 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -41,6 +41,7 @@ def __init__( reason="", decode_content=True, ) + self.length_remaining=self._init_length() @property def url(self) -> str | None: @@ -65,6 +66,60 @@ def retries(self, retries: Retry | None) -> None: self.url = retries.history[-1].redirect_location self._retries = retries + def _init_length(self, request_method: str | None) -> int | None: + length: int | None + content_length: str | None = self.headers.get("content-length") + + if content_length is not None: + if self.chunked: + # This Response will fail with an IncompleteRead if it can't be + # received as chunked. This method falls back to attempt reading + # the response before raising an exception. + log.warning( + "Received response with both Content-Length and " + "Transfer-Encoding set. This is expressly forbidden " + "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " + "attempting to process response as Transfer-Encoding: " + "chunked." + ) + return None + + try: + # RFC 7230 section 3.3.2 specifies multiple content lengths can + # be sent in a single Content-Length header + # (e.g. Content-Length: 42, 42). This line ensures the values + # are all valid ints and that as long as the `set` length is 1, + # all values are the same. Otherwise, the header is invalid. + lengths = {int(val) for val in content_length.split(",")} + if len(lengths) > 1: + raise InvalidHeader( + "Content-Length contained multiple " + "unmatching values (%s)" % content_length + ) + length = lengths.pop() + except ValueError: + length = None + else: + if length < 0: + length = None + + else: # if content_length is None + length = None + + # Convert status to int for comparison + # In some cases, httplib returns a status of "_UNKNOWN" + try: + status = int(self.status) + except ValueError: + status = 0 + + # Check for responses that shouldn't include a body + if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD": + length = 0 + + return length + + def read( self, amt: int | None = None, @@ -72,16 +127,22 @@ def read( cache_content: bool = False, ) -> bytes: if not isinstance(self._response.body, IOBase): + self.length_remaining=len(self._response.body) # wrap body in IOStream self._response.body = BytesIO(self._response.body) if amt is not None: # don't cache partial content cache_content = False - return typing.cast(bytes, self._response.body.read(amt)) + data=self._response.body.read(amt) + if self.length_remaining>0: + self.length_remaining= max(self.length_remaining-len(data),0) + return typing.cast(bytes, read_data) else: data = self._response.body.read(None) if cache_content: self._body = data + if self.length_remaining>0: + self.length_remaining= max(self.length_remaining-len(data),0) return typing.cast(bytes, data) def read_chunked( From b5aa6e5dc0830e3bfa597e16a92842ec05193e63 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Tue, 21 Nov 2023 13:03:19 +0000 Subject: [PATCH 36/49] fix response exceptions etc. --- src/urllib3/contrib/emscripten/response.py | 21 ++++++++------ test/contrib/emscripten/test_emscripten.py | 32 +++++++++++----------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 37ede24d39..6022ddd571 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -1,10 +1,12 @@ from __future__ import annotations import json as _json +import logging import typing from dataclasses import dataclass from io import BytesIO, IOBase +from ...exceptions import InvalidHeader from ...response import BaseHTTPResponse from ...util.retry import Retry from .request import EmscriptenRequest @@ -12,6 +14,8 @@ if typing.TYPE_CHECKING: from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection +log = logging.getLogger(__name__) + @dataclass class EmscriptenResponse: @@ -41,7 +45,7 @@ def __init__( reason="", decode_content=True, ) - self.length_remaining=self._init_length() + self.length_remaining = self._init_length(self._response.request.method) @property def url(self) -> str | None: @@ -119,7 +123,6 @@ def _init_length(self, request_method: str | None) -> int | None: return length - def read( self, amt: int | None = None, @@ -127,22 +130,22 @@ def read( cache_content: bool = False, ) -> bytes: if not isinstance(self._response.body, IOBase): - self.length_remaining=len(self._response.body) + self.length_remaining = len(self._response.body) # wrap body in IOStream self._response.body = BytesIO(self._response.body) if amt is not None: # don't cache partial content cache_content = False - data=self._response.body.read(amt) - if self.length_remaining>0: - self.length_remaining= max(self.length_remaining-len(data),0) - return typing.cast(bytes, read_data) + data = self._response.body.read(amt) + if self.length_remaining: + self.length_remaining = max(self.length_remaining - len(data), 0) + return typing.cast(bytes, data) else: data = self._response.body.read(None) if cache_content: self._body = data - if self.length_remaining>0: - self.length_remaining= max(self.length_remaining-len(data),0) + if self.length_remaining: + self.length_remaining = max(self.length_remaining - len(data), 0) return typing.cast(bytes, data) def read_chunked( diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 7c85f025fc..6932fac595 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -44,12 +44,12 @@ def test_index(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPConnection - from urllib3.response import HTTPResponse + from urllib3.response import BaseHTTPResponse conn = HTTPConnection(host, port) conn.request("GET", f"http://{host}:{port}/") response = conn.getresponse() - assert isinstance(response, HTTPResponse) + assert isinstance(response, BaseHTTPResponse) data = response.data assert data.decode("utf-8") == "Dummy server!" @@ -105,12 +105,12 @@ def test_404(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPConnection - from urllib3.response import HTTPResponse + from urllib3.response import BaseHTTPResponse conn = HTTPConnection(host, port) conn.request("GET", f"http://{host}:{port}/status?status=404 NOT FOUND") response = conn.getresponse() - assert isinstance(response, HTTPResponse) + assert isinstance(response, BaseHTTPResponse) assert response.status == 404 pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) @@ -186,12 +186,12 @@ def test_index_https(selenium: typing.Any, testserver_http: PyodideServerInfo) - @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPSConnection - from urllib3.response import HTTPResponse + from urllib3.response import BaseHTTPResponse conn = HTTPSConnection(host, port) conn.request("GET", f"https://{host}:{port}/") response = conn.getresponse() - assert isinstance(response, HTTPResponse) + assert isinstance(response, BaseHTTPResponse) data = response.data assert data.decode("utf-8") == "Dummy server!" @@ -208,7 +208,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt import urllib3.contrib.emscripten.fetch from urllib3.connection import HTTPSConnection - from urllib3.response import HTTPResponse + from urllib3.response import BaseHTTPResponse log_msgs = [] old_log = js.console.warn @@ -222,7 +222,7 @@ def capture_log(*args): # type: ignore[no-untyped-def] conn.request("GET", f"https://{host}:{port}/", preload_content=True) response = conn.getresponse() js.console.warn = old_log - assert isinstance(response, HTTPResponse) + assert isinstance(response, BaseHTTPResponse) data = response.data assert data.decode("utf-8") == "Dummy server!" # no console warnings because we didn't ask it to stream the response @@ -246,7 +246,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt import urllib3.contrib.emscripten.fetch from urllib3.connection import HTTPSConnection - from urllib3.response import HTTPResponse + from urllib3.response import BaseHTTPResponse log_msgs = [] old_log = js.console.warn @@ -261,7 +261,7 @@ def capture_log(*args): # type: ignore[no-untyped-def] conn.request("GET", f"https://{host}:{port}/", preload_content=False) response = conn.getresponse() js.console.warn = old_log - assert isinstance(response, HTTPResponse) + assert isinstance(response, BaseHTTPResponse) data = response.data assert data.decode("utf-8") == "Dummy server!" # check that it has warned about falling back to non-streaming fetch exactly once @@ -315,14 +315,14 @@ def test_streaming_download( import urllib3.contrib.emscripten.fetch await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() - from urllib3.response import HTTPResponse + from urllib3.response import BaseHTTPResponse from urllib3.connection import HTTPConnection import js conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) conn.request("GET", "{bigfile_url}",preload_content=False) response = conn.getresponse() - assert isinstance(response, HTTPResponse) + assert isinstance(response, BaseHTTPResponse) assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==False data=response.data.decode('utf-8') data @@ -348,7 +348,7 @@ def test_streaming_notready_warning( await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) import js import urllib3 - from urllib3.response import HTTPResponse + from urllib3.response import BaseHTTPResponse from urllib3.connection import HTTPConnection log_msgs=[] @@ -362,7 +362,7 @@ def capture_log(*args): conn.request("GET", "{bigfile_url}",preload_content=False) js.console.warn=old_log response = conn.getresponse() - assert isinstance(response, HTTPResponse) + assert isinstance(response, BaseHTTPResponse) data=response.data.decode('utf-8') assert len([x for x in log_msgs if x.find("Can't stream HTTP requests")!=-1])==1 assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True @@ -381,7 +381,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt import json from urllib3.connection import HTTPConnection - from urllib3.response import HTTPResponse + from urllib3.response import BaseHTTPResponse json_data = { "Bears": "like", @@ -394,7 +394,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt body=json.dumps(json_data).encode("utf-8"), ) response = conn.getresponse() - assert isinstance(response, HTTPResponse) + assert isinstance(response, BaseHTTPResponse) data = response.json() assert data == json_data From 35994ad25d494f00ea9d2226d17682c153cf205f Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Wed, 22 Nov 2023 14:22:50 +0000 Subject: [PATCH 37/49] fixes for requests --- dummyserver/handlers.py | 1 + src/urllib3/contrib/emscripten/fetch.py | 5 ++++ src/urllib3/contrib/emscripten/response.py | 30 ++++++++++++++++++++++ test/contrib/emscripten/conftest.py | 3 +-- test/contrib/emscripten/test_emscripten.py | 18 +++++++++++++ 5 files changed, 55 insertions(+), 2 deletions(-) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 86201a116f..4425fb5aaa 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -247,6 +247,7 @@ def echo(self, request: httputil.HTTPServerRequest) -> Response: def echo_json(self, request: httputil.HTTPServerRequest) -> Response: "Echo back the JSON" + print("ECHO JSON:",request.body) return Response(json=request.body, headers=list(request.headers.items())) def echo_uri(self, request: httputil.HTTPServerRequest) -> Response: diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index dd2890d48c..cdc41761eb 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -101,11 +101,16 @@ def __init__( self.worker = worker self.timeout = int(1000 * timeout) if timeout > 0 else None self.is_live = True + self._is_closed = False def __del__(self) -> None: self.close() + def is_closed(self): + return self._is_closed + def close(self) -> None: + self._is_closed = True if self.is_live: self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) self.is_live = False diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 6022ddd571..b2deef2bf3 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -70,6 +70,36 @@ def retries(self, retries: Retry | None) -> None: self.url = retries.history[-1].redirect_location self._retries = retries + def stream( + self, amt: int | None = 2**16, decode_content: bool | None = None + ) -> typing.Generator[bytes, None, None]: + """ + A generator wrapper for the read() method. A call will block until + ``amt`` bytes have been read from the connection or until the + connection is closed. + + :param amt: + How much of the content to read. The generator will return up to + much data per iteration, but may return less. This is particularly + likely when using compressed data. However, the empty string will + never be returned. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + if self.chunked and self.supports_chunked_reads(): + yield from self.read_chunked(amt, decode_content=decode_content) + else: + while True: + data = self.read(amt=amt, decode_content=decode_content) + + if data: + yield data + else: + break + + def _init_length(self, request_method: str | None) -> int | None: length: int | None content_length: str | None = self.headers.get("content-length") diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 9876160943..3e15d916ab 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -79,8 +79,6 @@ def run_from_server( ) -> Generator[ServerRunnerInfo, None, None]: addr = f"https://{testserver_http.http_host}:{testserver_http.https_port}/pyodide/test.html" selenium.goto(addr) - # import time - # time.sleep(100) selenium.javascript_setup() selenium.load_pyodide() selenium.initialize_pyodide() @@ -106,6 +104,7 @@ def set_default_headers(self) -> None: self.set_header("Cross-Origin-Opener-Policy", "same-origin") self.set_header("Cross-Origin-Embedder-Policy", "require-corp") self.add_header("Feature-Policy", "sync-xhr *;") + self.add_header("Access-Control-Allow-Headers", "*") def slow(self, _req: HTTPServerRequest) -> Response: import time diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 6932fac595..0f692628ad 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -419,3 +419,21 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt assert r.status == 200 pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + +@install_urllib3_wheel() +def test_requests_with_micropip(selenium,testserver_http) -> None: + # this can't be @run_in_pyodide because of the async code + response=selenium.run_async(f""" + import micropip + await micropip.install("requests") + import requests + import json + r = requests.get("http://{testserver_http.http_host}:{testserver_http.http_port}/") + assert(r.status_code == 200) + assert(r.text == "Dummy server!") + json_data={{"woo":"yay"}} + # try posting some json with requests + r = requests.post("http://{testserver_http.http_host}:{testserver_http.http_port}/echo_json",json=json_data) + import js + assert(r.json() == json_data) + """) From f62e0a4829921c6eea47c3bd2501574a9bd5c3fc Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Wed, 22 Nov 2023 14:29:37 +0000 Subject: [PATCH 38/49] mypy and lint --- dummyserver/handlers.py | 2 +- src/urllib3/contrib/emscripten/fetch.py | 4 ++-- src/urllib3/contrib/emscripten/response.py | 16 ++++++---------- test/contrib/emscripten/test_emscripten.py | 11 ++++++++--- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 4425fb5aaa..84493ab82d 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -247,7 +247,7 @@ def echo(self, request: httputil.HTTPServerRequest) -> Response: def echo_json(self, request: httputil.HTTPServerRequest) -> Response: "Echo back the JSON" - print("ECHO JSON:",request.body) + print("ECHO JSON:", request.body) return Response(json=request.body, headers=list(request.headers.items())) def echo_uri(self, request: httputil.HTTPServerRequest) -> Response: diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index cdc41761eb..395b0a7a15 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -106,9 +106,9 @@ def __init__( def __del__(self) -> None: self.close() - def is_closed(self): + def is_closed(self) -> bool: return self._is_closed - + def close(self) -> None: self._is_closed = True if self.is_live: diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index b2deef2bf3..5fef8a7e07 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -88,17 +88,13 @@ def stream( If True, will attempt to decode the body based on the 'content-encoding' header. """ - if self.chunked and self.supports_chunked_reads(): - yield from self.read_chunked(amt, decode_content=decode_content) - else: - while True: - data = self.read(amt=amt, decode_content=decode_content) - - if data: - yield data - else: - break + while True: + data = self.read(amt=amt, decode_content=decode_content) + if data: + yield data + else: + break def _init_length(self, request_method: str | None) -> int | None: length: int | None diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 0f692628ad..94db37ca07 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -420,10 +420,14 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + @install_urllib3_wheel() -def test_requests_with_micropip(selenium,testserver_http) -> None: +def test_requests_with_micropip( + selenium: typing.Any, testserver_http: PyodideServerInfo +) -> None: # this can't be @run_in_pyodide because of the async code - response=selenium.run_async(f""" + selenium.run_async( + f""" import micropip await micropip.install("requests") import requests @@ -436,4 +440,5 @@ def test_requests_with_micropip(selenium,testserver_http) -> None: r = requests.post("http://{testserver_http.http_host}:{testserver_http.http_port}/echo_json",json=json_data) import js assert(r.json() == json_data) - """) + """ + ) From 62ad859de22fc1ca3dbcd71f3cc943b20ca9cd62 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Wed, 22 Nov 2023 16:27:55 +0000 Subject: [PATCH 39/49] added coverage support to emscripten --- test/contrib/emscripten/conftest.py | 90 +++++++++++++++++++--- test/contrib/emscripten/test_emscripten.py | 89 +++++++++++---------- 2 files changed, 124 insertions(+), 55 deletions(-) diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 3e15d916ab..08b13a8e17 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -5,6 +5,7 @@ import mimetypes import os import textwrap +import time from dataclasses import dataclass from pathlib import Path from typing import Any, Generator @@ -38,6 +39,45 @@ def testserver_http( server.teardown_class() +@pytest.fixture() +def selenium_coverage(selenium: typing.Any) -> Generator[typing.Any, None, None]: + def _install_coverage(self): + self.run_js( + """ + await pyodide.loadPackage("coverage") + await pyodide.runPythonAsync(`import coverage +_coverage= coverage.Coverage(source_pkgs=['urllib3']) +_coverage.start() + ` + )""" + ) + + setattr( + selenium, + "_install_coverage", + _install_coverage.__get__(selenium, selenium.__class__), + ) + selenium._install_coverage() + yield selenium + # on teardown, save _coverage output + coverage_out = selenium.run_js( + """ +return await pyodide.runPythonAsync(` +_coverage.stop() +_coverage.save() +datafile=open(".coverage","rb") +datafile.read() +`) + """ + ) + if isinstance(coverage_out, bytearray): + coverage_out_binary = coverage_out + elif isinstance(coverage_out, list): + coverage_out_binary = bytes(coverage_out) + with open(f".coverage.emscripten.{time.time()}", "wb") as outfile: + outfile.write(coverage_out_binary) + + class ServerRunnerInfo: def __init__(self, host: str, port: int, selenium: Any) -> None: self.host = host @@ -48,8 +88,27 @@ def run_webworker(self, code: str) -> Any: if isinstance(code, str) and code.startswith("\n"): # we have a multiline string, fix indentation code = textwrap.dedent(code) + # add coverage collection to this code + code = ( + textwrap.dedent( + """ + import coverage + _coverage= coverage.Coverage(source_pkgs=['urllib3']) + _coverage.start() + """ + ) + + code + ) + code += textwrap.dedent( + """ + _coverage.stop() + _coverage.save() + datafile=open(".coverage","rb") + str(list(datafile.read())) + """ + ) - return self.selenium.run_js( + coverage_out = self.selenium.run_js( f""" let worker = new Worker('https://{self.host}:{self.port}/pyodide/webworker_dev.js'); let p = new Promise((res, rej) => {{ @@ -68,6 +127,16 @@ def run_webworker(self, code: str) -> Any: """, pyodide_checks=False, ) + print(type(coverage_out)) + print(coverage_out[0:10]) + if isinstance(coverage_out,str): + coverage_out = eval(coverage_out) + if isinstance(coverage_out, bytearray): + coverage_out_binary = coverage_out + elif isinstance(coverage_out, list): + coverage_out_binary = bytes(coverage_out) + with open(f".coverage.emscripten.{time.time()}", "wb") as outfile: + outfile.write(coverage_out_binary) # run pyodide on our test server instead of on the default @@ -75,23 +144,24 @@ def run_webworker(self, code: str) -> Any: # we are at the same origin as web requests to server_host @pytest.fixture() def run_from_server( - selenium: Any, testserver_http: PyodideServerInfo + selenium_coverage: Any, testserver_http: PyodideServerInfo ) -> Generator[ServerRunnerInfo, None, None]: addr = f"https://{testserver_http.http_host}:{testserver_http.https_port}/pyodide/test.html" - selenium.goto(addr) - selenium.javascript_setup() - selenium.load_pyodide() - selenium.initialize_pyodide() - selenium.save_state() - selenium.restore_state() + selenium_coverage.goto(addr) + selenium_coverage.javascript_setup() + selenium_coverage.load_pyodide() + selenium_coverage.initialize_pyodide() + selenium_coverage.save_state() + selenium_coverage.restore_state() # install the wheel, which is served at /wheel/* - selenium.run_js( + selenium_coverage.run_js( """ await pyodide.loadPackage('/wheel/dist.whl') """ ) + selenium_coverage._install_coverage() yield ServerRunnerInfo( - testserver_http.http_host, testserver_http.https_port, selenium + testserver_http.http_host, testserver_http.https_port, selenium_coverage ) diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 94db37ca07..8a9f0d8854 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -40,9 +40,9 @@ def install_urllib3_wheel() -> ( @install_urllib3_wheel() -def test_index(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_index(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPConnection from urllib3.response import BaseHTTPResponse @@ -53,16 +53,16 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt data = response.data assert data.decode("utf-8") == "Dummy server!" - pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) # wrong protocol / protocol error etc. should raise an exception of urllib3.exceptions.ResponseError @install_urllib3_wheel() def test_wrong_protocol( - selenium: typing.Any, testserver_http: PyodideServerInfo + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] import pytest import urllib3.exceptions @@ -76,14 +76,14 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt except BaseException as ex: assert isinstance(ex, urllib3.exceptions.ResponseError) - pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) # no connection - should raise @install_urllib3_wheel() -def test_no_response(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_no_response(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] import pytest import urllib3.exceptions @@ -97,13 +97,13 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt except BaseException as ex: assert isinstance(ex, urllib3.exceptions.ResponseError) - pyodide_test(selenium, testserver_http.http_host, find_unused_port()) + pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) @install_urllib3_wheel() -def test_404(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_404(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPConnection from urllib3.response import BaseHTTPResponse @@ -113,7 +113,7 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt assert isinstance(response, BaseHTTPResponse) assert response.status == 404 - pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) # setting timeout should show a warning to js console @@ -121,10 +121,10 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt # support timeout in async mode if globalThis == Window @install_urllib3_wheel() def test_timeout_warning( - selenium: typing.Any, testserver_http: PyodideServerInfo + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: @run_in_pyodide() # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] import js # type: ignore[import] import urllib3.contrib.emscripten.fetch @@ -144,16 +144,15 @@ def capture_log(*args): # type: ignore[no-untyped-def] conn.getresponse() js.console.warn = old_log # should have shown timeout warning exactly once by now - print(log_msgs) assert len([x for x in log_msgs if x.find("Warning: Timeout") != -1]) == 1 assert urllib3.contrib.emscripten.fetch._SHOWN_TIMEOUT_WARNING - pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) @install_urllib3_wheel() def test_timeout_in_worker( - selenium: typing.Any, + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, run_from_server: ServerRunnerInfo, ) -> None: @@ -182,9 +181,9 @@ def test_timeout_in_worker( @install_urllib3_wheel() -def test_index_https(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_index_https(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPSConnection from urllib3.response import BaseHTTPResponse @@ -195,15 +194,15 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt data = response.data assert data.decode("utf-8") == "Dummy server!" - pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) @install_urllib3_wheel() def test_non_streaming_no_fallback_warning( - selenium: typing.Any, testserver_http: PyodideServerInfo + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] import js import urllib3.contrib.emscripten.fetch @@ -233,15 +232,15 @@ def capture_log(*args): # type: ignore[no-untyped-def] ) assert not urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING - pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) @install_urllib3_wheel() def test_streaming_fallback_warning( - selenium: typing.Any, testserver_http: PyodideServerInfo + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] import js import urllib3.contrib.emscripten.fetch @@ -271,17 +270,17 @@ def capture_log(*args): # type: ignore[no-untyped-def] ) assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING - pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) @install_urllib3_wheel() def test_specific_method( - selenium: typing.Any, + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, run_from_server: ServerRunnerInfo, ) -> None: @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3 import HTTPSConnectionPool with HTTPSConnectionPool(host, port) as pool: @@ -292,12 +291,12 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt response = pool.request("PUT", path) assert response.status == 400 - pyodide_test(selenium, testserver_http.http_host, testserver_http.https_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) @install_urllib3_wheel() def test_streaming_download( - selenium: typing.Any, + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, run_from_server: ServerRunnerInfo, ) -> None: @@ -325,15 +324,15 @@ def test_streaming_download( assert isinstance(response, BaseHTTPResponse) assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==False data=response.data.decode('utf-8') - data + assert len(data) == 17825792 """ - result = run_from_server.run_webworker(worker_code) - assert len(result) == 17825792 + run_from_server.run_webworker(worker_code) + @install_urllib3_wheel() def test_streaming_notready_warning( - selenium: typing.Any, + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, run_from_server: ServerRunnerInfo, ) -> None: @@ -366,18 +365,18 @@ def capture_log(*args): data=response.data.decode('utf-8') assert len([x for x in log_msgs if x.find("Can't stream HTTP requests")!=-1])==1 assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True - data + assert len(data) == 17825792 """ - result = run_from_server.run_webworker(worker_code) - assert len(result) == 17825792 + run_from_server.run_webworker(worker_code) + @install_urllib3_wheel() def test_post_receive_json( - selenium: typing.Any, testserver_http: PyodideServerInfo + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] import json from urllib3.connection import HTTPConnection @@ -398,13 +397,13 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt data = response.json() assert data == json_data - pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) @install_urllib3_wheel() -def test_upload(selenium: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_upload(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: @run_in_pyodide # type: ignore[misc] - def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-untyped-def] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3 import HTTPConnectionPool data = "I'm in ur multipart form-data, hazing a cheezburgr" @@ -418,15 +417,15 @@ def pyodide_test(selenium, host: str, port: int) -> None: # type: ignore[no-unt r = pool.request("POST", "/upload", fields=fields) assert r.status == 200 - pyodide_test(selenium, testserver_http.http_host, testserver_http.http_port) + pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) @install_urllib3_wheel() def test_requests_with_micropip( - selenium: typing.Any, testserver_http: PyodideServerInfo + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: # this can't be @run_in_pyodide because of the async code - selenium.run_async( + selenium_coverage.run_async( f""" import micropip await micropip.install("requests") From 30b306312e03c410061d19c5a1db570f35c2b25d Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 23 Nov 2023 09:38:42 +0000 Subject: [PATCH 40/49] mypy fixes --- test/contrib/emscripten/conftest.py | 10 ++-- test/contrib/emscripten/test_emscripten.py | 60 ++++++++++++++++------ 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 08b13a8e17..671d8863fc 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -40,8 +40,8 @@ def testserver_http( @pytest.fixture() -def selenium_coverage(selenium: typing.Any) -> Generator[typing.Any, None, None]: - def _install_coverage(self): +def selenium_coverage(selenium: Any) -> Generator[Any, None, None]: + def _install_coverage(self: Any) -> None: self.run_js( """ await pyodide.loadPackage("coverage") @@ -73,7 +73,7 @@ def _install_coverage(self): if isinstance(coverage_out, bytearray): coverage_out_binary = coverage_out elif isinstance(coverage_out, list): - coverage_out_binary = bytes(coverage_out) + coverage_out_binary = bytearray(coverage_out) with open(f".coverage.emscripten.{time.time()}", "wb") as outfile: outfile.write(coverage_out_binary) @@ -129,12 +129,12 @@ def run_webworker(self, code: str) -> Any: ) print(type(coverage_out)) print(coverage_out[0:10]) - if isinstance(coverage_out,str): + if isinstance(coverage_out, str): coverage_out = eval(coverage_out) if isinstance(coverage_out, bytearray): coverage_out_binary = coverage_out elif isinstance(coverage_out, list): - coverage_out_binary = bytes(coverage_out) + coverage_out_binary = bytearray(coverage_out) with open(f".coverage.emscripten.{time.time()}", "wb") as outfile: outfile.write(coverage_out_binary) diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 8a9f0d8854..48377c1ac3 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -40,7 +40,9 @@ def install_urllib3_wheel() -> ( @install_urllib3_wheel() -def test_index(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_index( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPConnection @@ -53,7 +55,9 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno data = response.data assert data.decode("utf-8") == "Dummy server!" - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) # wrong protocol / protocol error etc. should raise an exception of urllib3.exceptions.ResponseError @@ -76,12 +80,16 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno except BaseException as ex: assert isinstance(ex, urllib3.exceptions.ResponseError) - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) # no connection - should raise @install_urllib3_wheel() -def test_no_response(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_no_response( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] import pytest @@ -113,7 +121,9 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno assert isinstance(response, BaseHTTPResponse) assert response.status == 404 - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) # setting timeout should show a warning to js console @@ -147,7 +157,9 @@ def capture_log(*args): # type: ignore[no-untyped-def] assert len([x for x in log_msgs if x.find("Warning: Timeout") != -1]) == 1 assert urllib3.contrib.emscripten.fetch._SHOWN_TIMEOUT_WARNING - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) @install_urllib3_wheel() @@ -181,7 +193,9 @@ def test_timeout_in_worker( @install_urllib3_wheel() -def test_index_https(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_index_https( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3.connection import HTTPSConnection @@ -194,7 +208,9 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno data = response.data assert data.decode("utf-8") == "Dummy server!" - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) @install_urllib3_wheel() @@ -232,7 +248,9 @@ def capture_log(*args): # type: ignore[no-untyped-def] ) assert not urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) @install_urllib3_wheel() @@ -270,7 +288,9 @@ def capture_log(*args): # type: ignore[no-untyped-def] ) assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) @install_urllib3_wheel() @@ -291,7 +311,9 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno response = pool.request("PUT", path) assert response.status == 400 - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.https_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) @install_urllib3_wheel() @@ -327,7 +349,6 @@ def test_streaming_download( assert len(data) == 17825792 """ run_from_server.run_webworker(worker_code) - @install_urllib3_wheel() @@ -365,10 +386,9 @@ def capture_log(*args): data=response.data.decode('utf-8') assert len([x for x in log_msgs if x.find("Can't stream HTTP requests")!=-1])==1 assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True - assert len(data) == 17825792 + assert len(data) == 17825792 """ run_from_server.run_webworker(worker_code) - @install_urllib3_wheel() @@ -397,11 +417,15 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno data = response.json() assert data == json_data - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) @install_urllib3_wheel() -def test_upload(selenium_coverage: typing.Any, testserver_http: PyodideServerInfo) -> None: +def test_upload( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] from urllib3 import HTTPConnectionPool @@ -417,7 +441,9 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno r = pool.request("POST", "/upload", fields=fields) assert r.status == 200 - pyodide_test(selenium_coverage, testserver_http.http_host, testserver_http.http_port) + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) @install_urllib3_wheel() From ea6731682fa8c3f349828042f952133967971320 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 12:50:16 +0000 Subject: [PATCH 41/49] coverage updates --- src/urllib3/contrib/emscripten/connection.py | 34 +- src/urllib3/contrib/emscripten/fetch.py | 65 ++- src/urllib3/contrib/emscripten/request.py | 6 - src/urllib3/contrib/emscripten/response.py | 141 ++++-- test/contrib/emscripten/conftest.py | 32 +- test/contrib/emscripten/test_emscripten.py | 466 ++++++++++++++++++- 6 files changed, 628 insertions(+), 116 deletions(-) diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index 07ebf3b991..007e3ce35f 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -13,18 +13,7 @@ from ...util.url import Url from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request from .request import EmscriptenRequest -from .response import EmscriptenHttpResponseWrapper - -try: # Compiled with SSL? - import ssl - - BaseSSLError = ssl.SSLError -except (ImportError, AttributeError): - ssl = None # type: ignore[assignment] - - class BaseSSLError(BaseException): # type: ignore[no-redef] - pass - +from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse if typing.TYPE_CHECKING: from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection @@ -48,6 +37,8 @@ class EmscriptenHTTPConnection: is_verified: bool = False proxy_is_verified: bool | None = None + _response: EmscriptenResponse | None + def __init__( self, host: str, @@ -60,12 +51,19 @@ def __init__( proxy: Url | None = None, proxy_config: ProxyConfig | None = None, ) -> None: - # ignore everything else because we don't have - # control over that stuff self.host = host self.port = port self.timeout = timeout if isinstance(timeout, float) else 0.0 self.scheme = "http" + self._closed = True + self._response = None + # ignore these things because we don't + # have control over that stuff + self.proxy = None + self.proxy_config = None + self.blocksize = blocksize + self.source_address = None + self.socket_options = None def set_tunnel( self, @@ -94,6 +92,7 @@ def request( decode_content: bool = True, enforce_content_length: bool = True, ) -> None: + self._closed = False if url.startswith("/"): # no scheme / host / port included, make a full url url = f"{self.scheme}://{self.host}:{self.port}" + url @@ -129,7 +128,8 @@ def getresponse(self) -> BaseHTTPResponse: raise ResponseNotReady() def close(self) -> None: - pass + self._closed = True + self._response = None @property def is_closed(self) -> bool: @@ -137,7 +137,7 @@ def is_closed(self) -> bool: If this property is True then both ``is_connected`` and ``has_connected_to_proxy`` properties must be False. """ - return False + return self._closed @property def is_connected(self) -> bool: @@ -163,7 +163,7 @@ class EmscriptenHTTPSConnection(EmscriptenHTTPConnection): cert_file: str | None key_file: str | None key_password: str | None - ssl_context: ssl.SSLContext | None + ssl_context: typing.Any | None ssl_version: int | str | None = None ssl_minimum_version: int | None = None ssl_maximum_version: int | None = None diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index 395b0a7a15..38f92ef414 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -92,6 +92,7 @@ def __init__( timeout: float, worker: JsProxy, connection_id: int, + request: EmscriptenRequest, ): self.int_buffer = int_buffer self.byte_buffer = byte_buffer @@ -102,19 +103,32 @@ def __init__( self.timeout = int(1000 * timeout) if timeout > 0 else None self.is_live = True self._is_closed = False + self.request: EmscriptenRequest | None = request def __del__(self) -> None: self.close() + # this is compatible with _base_connection def is_closed(self) -> bool: return self._is_closed + # for compatibility with RawIOBase + @property + def closed(self) -> bool: + return self.is_closed() + def close(self) -> None: - self._is_closed = True - if self.is_live: - self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) - self.is_live = False - super().close() + if not self.is_closed(): + self.read_len = 0 + self.read_pos = 0 + self.int_buffer = None + self.byte_buffer = None + self._is_closed = True + self.request = None + if self.is_live: + self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) + self.is_live = False + super().close() def readable(self) -> bool: return True @@ -127,7 +141,11 @@ def seekable(self) -> bool: def readinto(self, byte_obj: Buffer) -> int: if not self.int_buffer: - return 0 + raise _StreamingError( + "No buffer for stream in _ReadStream.readinto", + request=self.request, + response=None, + ) if self.read_len == 0: # wait for the worker to send something js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT) @@ -142,13 +160,20 @@ def readinto(self, byte_obj: Buffer) -> int: self.read_len = data_len self.read_pos = 0 elif data_len == ERROR_EXCEPTION: - raise _StreamingError + string_len = self.int_buffer[1] + # decode the error string + js_decoder = js.TextDecoder.new() + json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len)) + raise _StreamingError( + f"Exception thrown in fetch: {json_str}", + request=self.request, + response=None, + ) else: # EOF, free the buffers and return zero - self.read_len = 0 - self.read_pos = 0 - self.int_buffer = None - self.byte_buffer = None + # and free the request + self.is_live = False + self.close() return 0 # copy from int32array to python bytes ret_length = min(self.read_len, len(memoryview(byte_obj))) @@ -233,15 +258,13 @@ def send(self, request: EmscriptenRequest) -> EmscriptenResponse: request=request, status_code=response_obj["status"], headers=response_obj["headers"], - body=io.BufferedReader( - _ReadStream( - js_int_buffer, - js_byte_buffer, - request.timeout, - self.js_worker, - response_obj["connectionID"], - ), - buffer_size=1048576, + body=_ReadStream( + js_int_buffer, + js_byte_buffer, + request.timeout, + self.js_worker, + response_obj["connectionID"], + request, ), ) elif js_int_buffer[0] == ERROR_EXCEPTION: @@ -328,7 +351,7 @@ def _show_streaming_warning() -> None: message += " Worker or Blob classes are not available in this environment." if streaming_ready() is False: message += """ Streaming fetch worker isn't ready. If you want to be sure that streamig fetch -is working, you need to call: 'await urllib3.contrib.emscripten.fetc.wait_for_streaming_ready()`""" +is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`""" from js import console console.warn(message) diff --git a/src/urllib3/contrib/emscripten/request.py b/src/urllib3/contrib/emscripten/request.py index 4544ac66ce..e692e692bd 100644 --- a/src/urllib3/contrib/emscripten/request.py +++ b/src/urllib3/contrib/emscripten/request.py @@ -1,8 +1,6 @@ from __future__ import annotations -import json from dataclasses import dataclass, field -from typing import Any from ..._base_connection import _TYPE_BODY @@ -22,7 +20,3 @@ def set_header(self, name: str, value: str) -> None: def set_body(self, body: _TYPE_BODY | None) -> None: self.body = body - - def set_json(self, body: dict[str, Any]) -> None: - self.set_header("Content-Type", "application/json; charset=utf-8") - self.set_body(json.dumps(body).encode("utf-8")) diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 5fef8a7e07..9165711e4c 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -3,10 +3,11 @@ import json as _json import logging import typing +from contextlib import contextmanager from dataclasses import dataclass from io import BytesIO, IOBase -from ...exceptions import InvalidHeader +from ...exceptions import InvalidHeader, ResponseError, TimeoutError from ...response import BaseHTTPResponse from ...util.retry import Retry from .request import EmscriptenRequest @@ -37,6 +38,7 @@ def __init__( self._response = internal_response self._url = url self._connection = connection + self._closed = False super().__init__( headers=internal_response.headers, status=internal_response.status_code, @@ -46,6 +48,7 @@ def __init__( decode_content=True, ) self.length_remaining = self._init_length(self._response.request.method) + self.length_is_certain = False @property def url(self) -> str | None: @@ -101,19 +104,6 @@ def _init_length(self, request_method: str | None) -> int | None: content_length: str | None = self.headers.get("content-length") if content_length is not None: - if self.chunked: - # This Response will fail with an IncompleteRead if it can't be - # received as chunked. This method falls back to attempt reading - # the response before raising an exception. - log.warning( - "Received response with both Content-Length and " - "Transfer-Encoding set. This is expressly forbidden " - "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " - "attempting to process response as Transfer-Encoding: " - "chunked." - ) - return None - try: # RFC 7230 section 3.3.2 specifies multiple content lengths can # be sent in a single Content-Length header @@ -136,15 +126,12 @@ def _init_length(self, request_method: str | None) -> int | None: else: # if content_length is None length = None - # Convert status to int for comparison - # In some cases, httplib returns a status of "_UNKNOWN" - try: - status = int(self.status) - except ValueError: - status = 0 - # Check for responses that shouldn't include a body - if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD": + if ( + self.status in (204, 304) + or 100 <= self.status < 200 + or request_method == "HEAD" + ): length = 0 return length @@ -155,30 +142,51 @@ def read( decode_content: bool | None = None, # ignored because browser decodes always cache_content: bool = False, ) -> bytes: - if not isinstance(self._response.body, IOBase): - self.length_remaining = len(self._response.body) - # wrap body in IOStream - self._response.body = BytesIO(self._response.body) - if amt is not None: - # don't cache partial content - cache_content = False - data = self._response.body.read(amt) - if self.length_remaining: - self.length_remaining = max(self.length_remaining - len(data), 0) - return typing.cast(bytes, data) - else: - data = self._response.body.read(None) - if cache_content: - self._body = data - if self.length_remaining: - self.length_remaining = max(self.length_remaining - len(data), 0) - return typing.cast(bytes, data) + if ( + self._closed + or self._response is None + or (isinstance(self._response.body, IOBase) and self._response.body.closed) + ): + return b"" + + with self._error_catcher(): + # body has been preloaded as a string by XmlHttpRequest + if not isinstance(self._response.body, IOBase): + self.length_remaining = len(self._response.body) + self.length_is_certain = True + # wrap body in IOStream + self._response.body = BytesIO(self._response.body) + if amt is not None: + # don't cache partial content + cache_content = False + data = self._response.body.read(amt) + if self.length_remaining is not None: + self.length_remaining = max(self.length_remaining - len(data), 0) + if (self.length_is_certain and self.length_remaining == 0) or len( + data + ) < amt: + # definitely finished reading, close response stream + self._response.body.close() + return typing.cast(bytes, data) + else: # read all we can (and cache it) + data = self._response.body.read(None) + if cache_content: + self._body = data + if self.length_remaining is not None: + self.length_remaining = max(self.length_remaining - len(data), 0) + if len(data) == 0 or ( + self.length_is_certain and self.length_remaining == 0 + ): + # definitely finished reading, close response stream + self._response.body.close() + return typing.cast(bytes, data) def read_chunked( self, amt: int | None = None, decode_content: bool | None = None, ) -> typing.Generator[bytes, None, None]: + # chunked is handled by browser while True: bytes = self.read(amt, decode_content) if not bytes: @@ -216,5 +224,54 @@ def json(self) -> typing.Any: return _json.loads(data) def close(self) -> None: - if isinstance(self._response.body, IOBase): - self._response.body.close() + if not self._closed: + if isinstance(self._response.body, IOBase): + self._response.body.close() + if self._connection: + self._connection.close() + self._connection = None + self._closed = True + + @contextmanager + def _error_catcher(self) -> typing.Generator[None, None, None]: + """ + Catch Emscripten specific exceptions thrown by fetch.py, + instead re-raising urllib3 variants, so that low-level exceptions + are not leaked in the high-level api. + + On exit, release the connection back to the pool. + """ + from .fetch import _RequestError, _TimeoutError # avoid circular import + + clean_exit = False + + try: + yield + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True + except _TimeoutError as e: + raise TimeoutError(e.message) + except _RequestError as e: + raise ResponseError(e.message) + finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + # The response may not be closed but we're not going to use it + # anymore so close it now + if ( + isinstance(self._response.body, IOBase) + and not self._response.body.closed + ): + self._response.body.close() + # release the connection back to the pool + self.release_conn() + else: + # If we have read everything from the response stream, + # return the connection back to the pool. + if ( + isinstance(self._response.body, IOBase) + and self._response.body.closed + ): + self.release_conn() diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 671d8863fc..849ab28b72 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -4,8 +4,8 @@ import contextlib import mimetypes import os +import random import textwrap -import time from dataclasses import dataclass from pathlib import Path from typing import Any, Generator @@ -19,6 +19,15 @@ from dummyserver.testcase import HTTPDummyProxyTestCase from dummyserver.tornadoserver import run_tornado_app, run_tornado_loop_in_thread +_coverage_count = 0 + + +def _get_coverage_filename(prefix: str) -> str: + global _coverage_count + _coverage_count += 1 + rand_part = "".join([random.choice("1234567890") for x in range(20)]) + return prefix + rand_part + f".{_coverage_count}" + @pytest.fixture(scope="module") def testserver_http( @@ -70,11 +79,8 @@ def _install_coverage(self: Any) -> None: `) """ ) - if isinstance(coverage_out, bytearray): - coverage_out_binary = coverage_out - elif isinstance(coverage_out, list): - coverage_out_binary = bytearray(coverage_out) - with open(f".coverage.emscripten.{time.time()}", "wb") as outfile: + coverage_out_binary = bytes(coverage_out) + with open(f"{_get_coverage_filename('.coverage.emscripten.')}", "wb") as outfile: outfile.write(coverage_out_binary) @@ -127,15 +133,11 @@ def run_webworker(self, code: str) -> Any: """, pyodide_checks=False, ) - print(type(coverage_out)) - print(coverage_out[0:10]) - if isinstance(coverage_out, str): - coverage_out = eval(coverage_out) - if isinstance(coverage_out, bytearray): - coverage_out_binary = coverage_out - elif isinstance(coverage_out, list): - coverage_out_binary = bytearray(coverage_out) - with open(f".coverage.emscripten.{time.time()}", "wb") as outfile: + coverage_out = eval(coverage_out) + coverage_out_binary = bytes(coverage_out) + with open( + f"{_get_coverage_filename('.coverage.emscripten.worker.')}", "wb" + ) as outfile: outfile.write(coverage_out_binary) diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 48377c1ac3..46fad7745a 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -49,23 +49,98 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno from urllib3.response import BaseHTTPResponse conn = HTTPConnection(host, port) - conn.request("GET", f"http://{host}:{port}/") + url = f"http://{host}:{port}/" + conn.request("GET", url) response = conn.getresponse() + # check methods of response assert isinstance(response, BaseHTTPResponse) - data = response.data - assert data.decode("utf-8") == "Dummy server!" + assert response.url == url + response.url = "http://woo" + assert response.url == "http://woo" + assert response.connection == conn + assert response.retries is None + data1 = response.data + decoded1 = data1.decode("utf-8") + data2 = response.data # check that getting data twice works + decoded2 = data2.decode("utf-8") + assert decoded1 == decoded2 == "Dummy server!" pyodide_test( selenium_coverage, testserver_http.http_host, testserver_http.http_port ) +@install_urllib3_wheel() +def test_pool_requests( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int, https_port: int) -> None: # type: ignore[no-untyped-def] + # first with PoolManager + import urllib3 + + http = urllib3.PoolManager() + resp = http.request("GET", f"http://{host}:{port}/") + assert resp.data.decode("utf-8") == "Dummy server!" + + resp2 = http.request("GET", f"http://{host}:{port}/index") + assert resp2.data.decode("utf-8") == "Dummy server!" + + # should all have come from one pool + assert len(http.pools) == 1 + + resp3 = http.request("GET", f"https://{host}:{https_port}/") + assert resp2.data.decode("utf-8") == "Dummy server!" + + # one http pool + one https pool + assert len(http.pools) == 2 + + # now with ConnectionPool + # because block == True, this will fail if the connection isn't + # returned to the pool correctly after the first request + pool = urllib3.HTTPConnectionPool(host, port, maxsize=1, block=True) + resp3 = pool.urlopen("GET", "/index") + assert resp3.data.decode("utf-8") == "Dummy server!" + + resp4 = pool.urlopen("GET", "/") + assert resp4.data.decode("utf-8") == "Dummy server!" + + # now with manual release of connection + # first - connection should be released once all + # data is read + pool2 = urllib3.HTTPConnectionPool(host, port, maxsize=1, block=True) + + resp5 = pool2.urlopen("GET", "/index", preload_content=False) + assert pool2.pool is not None + # at this point, the connection should not be in the pool + assert pool2.pool.qsize() == 0 + assert resp5.data.decode("utf-8") == "Dummy server!" + # now we've read all the data, connection should be back to the pool + assert pool2.pool.qsize() == 1 + resp6 = pool2.urlopen("GET", "/index", preload_content=False) + assert pool2.pool.qsize() == 0 + # force it back to the pool + resp6.release_conn() + assert pool2.pool.qsize() == 1 + read_str = resp6.read() + # for consistency with urllib3, this still returns the correct data even though + # we are in theory not using the connection any more + assert read_str.decode("utf-8") == "Dummy server!" + + pyodide_test( + selenium_coverage, + testserver_http.http_host, + testserver_http.http_port, + testserver_http.https_port, + ) + + # wrong protocol / protocol error etc. should raise an exception of urllib3.exceptions.ResponseError @install_urllib3_wheel() def test_wrong_protocol( selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: - @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] + @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] import pytest @@ -73,12 +148,29 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno from urllib3.connection import HTTPConnection conn = HTTPConnection(host, port) - try: + with pytest.raises(urllib3.exceptions.ResponseError): conn.request("GET", f"http://{host}:{port}/") - conn.getresponse() - pytest.fail("Should have thrown ResponseError here") - except BaseException as ex: - assert isinstance(ex, urllib3.exceptions.ResponseError) + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +# wrong protocol / protocol error etc. should raise an exception of urllib3.exceptions.ResponseError +@install_urllib3_wheel() +def test_bad_method( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3.exceptions + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + with pytest.raises(urllib3.exceptions.ResponseError): + conn.request("TRACE", f"http://{host}:{port}/") pyodide_test( selenium_coverage, testserver_http.http_host, testserver_http.https_port @@ -163,7 +255,33 @@ def capture_log(*args): # type: ignore[no-untyped-def] @install_urllib3_wheel() -def test_timeout_in_worker( +def test_timeout_in_worker_non_streaming( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + from urllib3.exceptions import TimeoutError + from urllib3.connection import HTTPConnection + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + result=-1 + try: + conn.request("GET","/slow") + _response = conn.getresponse() + result=-3 + except TimeoutError as e: + result=1 # we've got the correct exception + except BaseException as e: + result=-2 + assert result == 1 +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_timeout_in_worker_streaming( selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, run_from_server: ServerRunnerInfo, @@ -178,18 +296,16 @@ def test_timeout_in_worker( conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) result=-1 try: - conn.request("GET","/slow") + conn.request("GET","/slow",preload_content=False) _response = conn.getresponse() result=-3 except TimeoutError as e: result=1 # we've got the correct exception except BaseException as e: result=-2 - result + assert result == 1 """ - result = run_from_server.run_webworker(worker_code) - # result == 1 = success, -2 = wrong exception, -3 = no exception thrown - assert result == 1 + run_from_server.run_webworker(worker_code) @install_urllib3_wheel() @@ -265,6 +381,10 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno from urllib3.connection import HTTPSConnection from urllib3.response import BaseHTTPResponse + # monkeypatch is_cross_origin_isolated so that it warns about that + # even if we're serving it so it is fine + urllib3.contrib.emscripten.fetch.is_cross_origin_isolated = lambda: False + log_msgs = [] old_log = js.console.warn @@ -351,6 +471,110 @@ def test_streaming_download( run_from_server.run_webworker(worker_code) +@install_urllib3_wheel() +def test_streaming_close( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # test streaming download, which must be in a webworker + # as you can't do it on main thread + + # this should return the 17mb big file, and + # should not log any warning about falling back + url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + from io import RawIOBase + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{url}",preload_content=False) + response = conn.getresponse() + # check body is a RawIOBase stream and isn't seekable, writeable + body_internal = response._response.body.raw + assert(isinstance(body_internal,RawIOBase)) + assert(body_internal.writable() is False) + assert(body_internal.seekable() is False) + + response.drain_conn() + x=response.read() + assert(x==None) + response.close() + conn.close() + # try and make destructor be covered + # by killing everything + del response + del body_internal + del conn +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_streaming_bad_url( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # this should cause an error + # because the protocol is bad + bad_url = f"hsffsdfttp://{testserver_http.http_host}:{testserver_http.http_port}/" + # this must be in a webworker + # as you can't do it on main thread + worker_code = f""" + import pytest + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + from urllib3.exceptions import ResponseError + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + with pytest.raises(ResponseError): + conn.request("GET", "{bad_url}",preload_content=False) +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_streaming_bad_method( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # this should cause an error + # because the protocol is bad + bad_url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" + # this must be in a webworker + # as you can't do it on main thread + worker_code = f""" + import pytest + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + from urllib3.exceptions import ResponseError + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + with pytest.raises(ResponseError): + # TRACE method should throw SecurityError in Javascript + conn.request("TRACE", "{bad_url}",preload_content=False) +""" + run_from_server.run_webworker(worker_code) + + @install_urllib3_wheel() def test_streaming_notready_warning( selenium_coverage: typing.Any, @@ -446,6 +670,22 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno ) +@install_urllib3_wheel() +def test_streaming_not_ready_in_browser( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + # streaming ready should always be false + # if we're in the main browser thread + selenium_coverage.run_async( + """ + import urllib3.contrib.emscripten.fetch + result=await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + assert(result is False) + assert(urllib3.contrib.emscripten.fetch.streaming_ready() is None ) + """ + ) + + @install_urllib3_wheel() def test_requests_with_micropip( selenium_coverage: typing.Any, testserver_http: PyodideServerInfo @@ -467,3 +707,199 @@ def test_requests_with_micropip( assert(r.json() == json_data) """ ) + + +@install_urllib3_wheel() +def test_open_close( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from http.client import ResponseNotReady + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + # initially connection should be closed + assert conn.is_closed is True + # connection should have no response + with pytest.raises(ResponseNotReady): + response = conn.getresponse() + # now make the response + conn.request("GET", f"http://{host}:{port}/") + # we never connect to proxy (or if we do, browser handles it) + assert conn.has_connected_to_proxy is False + # now connection should be open + assert conn.is_closed is False + # and should have a response + response = conn.getresponse() + assert response is not None + conn.close() + # now it is closed + assert conn.is_closed is True + # closed connection shouldn't have any response + with pytest.raises(ResponseNotReady): + conn.getresponse() + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +# check that various ways that the worker may be broken +# throw exceptions nicely, by deliberately breaking things +# this is for coverage +@install_urllib3_wheel() +def test_break_worker_streaming( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + worker_code = f""" + import pyodide_js as pjs + await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) + import pytest + import urllib3.contrib.emscripten.fetch + import js + + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.exceptions import TimeoutError,ResponseError + from urllib3.connection import HTTPConnection + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + # make the fetch worker return a bad response by: + # 1) Clearing the int buffer + # in the receive stream + with pytest.raises(ResponseError): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + body_internal = response._response.body.raw + assert(body_internal.int_buffer!=None) + body_internal.int_buffer=None + data=response.read() + # 2) Monkeypatch postMessage so that it just sets an + # exception status + old_pm= body_internal.worker.postMessage + with pytest.raises(ResponseError): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + # make posted messages set an exception + body_internal = response._response.body.raw + def set_exception(*args): + body_internal.int_buffer[1]=5 + body_internal.int_buffer[2]=ord("W") + body_internal.int_buffer[3]=ord("O") + body_internal.int_buffer[4]=ord("O") + body_internal.int_buffer[5]=ord("!") + body_internal.int_buffer[6]=0 + js.Atomics.store(body_internal.int_buffer, 0, -4) + js.Atomics.notify(body_internal.int_buffer,0) + body_internal.worker.postMessage = set_exception + data=response.read() + body_internal.worker.postMessage = old_pm + # 3) Stopping the worker receiving any messages which should cause a timeout error + # in the receive stream + with pytest.raises(TimeoutError): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + # make posted messages not be send + body_internal = response._response.body.raw + def ignore_message(*args): + pass + old_pm= body_internal.worker.postMessage + body_internal.worker.postMessage = ignore_message + data=response.read() + body_internal.worker.postMessage = old_pm + +""" + run_from_server.run_webworker(worker_code) + + +@install_urllib3_wheel() +def test_response_init_length( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3.exceptions + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + # head shouldn't have length + length = response._init_length("HEAD") + assert length == 0 + # multiple inconsistent lengths - should raise invalid header + with pytest.raises(urllib3.exceptions.InvalidHeader): + response.headers["Content-Length"] = "4,5,6" + length = response._init_length("GET") + # non-numeric length - should return None + response.headers["Content-Length"] = "anna" + length = response._init_length("GET") + assert length is None + # numeric length - should return it + response.headers["Content-Length"] = "54" + length = response._init_length("GET") + assert length == 54 + # negative length - should return None + response.headers["Content-Length"] = "-12" + length = response._init_length("GET") + assert length is None + # none -> None + del response.headers["Content-Length"] + length = response._init_length("GET") + assert length is None + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_response_close_connection( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + response.close() + assert conn.is_closed + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@install_urllib3_wheel() +def test_read_chunked( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/bigfile", preload_content=False) + response = conn.getresponse() + count = 0 + for x in response.read_chunked(64): + count += 0 + assert len(x) == 64 + if count > 10: + break + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) From 64d4ab6e6a884fb9513be238c944a1f2a20181a6 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 13:26:57 +0000 Subject: [PATCH 42/49] fixes to firefox coverage checking --- test/contrib/emscripten/conftest.py | 35 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index 849ab28b72..fe2d59bfc3 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -69,17 +69,23 @@ def _install_coverage(self: Any) -> None: selenium._install_coverage() yield selenium # on teardown, save _coverage output - coverage_out = selenium.run_js( - """ + coverage_out_binary = bytes( + selenium.run_js( + """ return await pyodide.runPythonAsync(` _coverage.stop() _coverage.save() -datafile=open(".coverage","rb") -datafile.read() +_coverage_datafile = open(".coverage","rb") +_coverage_outdata = _coverage_datafile.read() +# avoid polluting main namespace too much +import js as _coverage_js +# convert to js Array (as default conversion is TypedArray which does +# bad things in firefox) +_coverage_js.Array.from_(_coverage_outdata) `) """ + ) ) - coverage_out_binary = bytes(coverage_out) with open(f"{_get_coverage_filename('.coverage.emscripten.')}", "wb") as outfile: outfile.write(coverage_out_binary) @@ -109,13 +115,19 @@ def run_webworker(self, code: str) -> Any: """ _coverage.stop() _coverage.save() - datafile=open(".coverage","rb") - str(list(datafile.read())) + _coverage_datafile = open(".coverage","rb") + _coverage_outdata = _coverage_datafile.read() + # avoid polluting main namespace too much + import js as _coverage_js + # convert to js Array (as default conversion is TypedArray which does + # bad things in firefox) + _coverage_js.Array.from_(_coverage_outdata) """ ) - coverage_out = self.selenium.run_js( - f""" + coverage_out_binary = bytes( + self.selenium.run_js( + f""" let worker = new Worker('https://{self.host}:{self.port}/pyodide/webworker_dev.js'); let p = new Promise((res, rej) => {{ worker.onmessageerror = e => rej(e); @@ -131,10 +143,9 @@ def run_webworker(self, code: str) -> Any: }}); return await p; """, - pyodide_checks=False, + pyodide_checks=False, + ) ) - coverage_out = eval(coverage_out) - coverage_out_binary = bytes(coverage_out) with open( f"{_get_coverage_filename('.coverage.emscripten.worker.')}", "wb" ) as outfile: From bc8c0964686008dba6fe67790d010bb1f02191a2 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 14:42:51 +0000 Subject: [PATCH 43/49] fixes for tests --- src/urllib3/contrib/emscripten/connection.py | 7 +- src/urllib3/contrib/emscripten/response.py | 9 +-- test/contrib/emscripten/test_emscripten.py | 83 ++++++++++++++------ 3 files changed, 68 insertions(+), 31 deletions(-) diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py index 007e3ce35f..25d7baa42f 100644 --- a/src/urllib3/contrib/emscripten/connection.py +++ b/src/urllib3/contrib/emscripten/connection.py @@ -2,11 +2,14 @@ import os import typing + +# use http.client.HTTPException for consistency with non-emscripten +from http.client import HTTPException as HTTPException # noqa: F401 from http.client import ResponseNotReady from ..._base_connection import _TYPE_BODY from ...connection import HTTPConnection, ProxyConfig, port_by_scheme -from ...exceptions import ResponseError, TimeoutError +from ...exceptions import TimeoutError from ...response import BaseHTTPResponse from ...util.connection import _TYPE_SOCKET_OPTIONS from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT @@ -115,7 +118,7 @@ def request( except _TimeoutError as e: raise TimeoutError(e.message) except _RequestError as e: - raise ResponseError(e.message) + raise HTTPException(e.message) def getresponse(self) -> BaseHTTPResponse: if self._response is not None: diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index 9165711e4c..cce89e596f 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -5,9 +5,10 @@ import typing from contextlib import contextmanager from dataclasses import dataclass +from http.client import HTTPException as HTTPException # noqa: F401 from io import BytesIO, IOBase -from ...exceptions import InvalidHeader, ResponseError, TimeoutError +from ...exceptions import InvalidHeader, TimeoutError from ...response import BaseHTTPResponse from ...util.retry import Retry from .request import EmscriptenRequest @@ -69,8 +70,6 @@ def retries(self) -> Retry | None: @retries.setter def retries(self, retries: Retry | None) -> None: # Override the request_url if retries has a redirect location. - if retries is not None and retries.history: - self.url = retries.history[-1].redirect_location self._retries = retries def stream( @@ -169,7 +168,7 @@ def read( self._response.body.close() return typing.cast(bytes, data) else: # read all we can (and cache it) - data = self._response.body.read(None) + data = self._response.body.read() if cache_content: self._body = data if self.length_remaining is not None: @@ -253,7 +252,7 @@ def _error_catcher(self) -> typing.Generator[None, None, None]: except _TimeoutError as e: raise TimeoutError(e.message) except _RequestError as e: - raise ResponseError(e.message) + raise HTTPException(e.message) finally: # If we didn't terminate cleanly, we need to throw away our # connection. diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 46fad7745a..ffcfe59e14 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -135,20 +135,21 @@ def pyodide_test(selenium_coverage, host: str, port: int, https_port: int) -> No ) -# wrong protocol / protocol error etc. should raise an exception of urllib3.exceptions.ResponseError +# wrong protocol / protocol error etc. should raise an exception of http.client.HTTPException @install_urllib3_wheel() def test_wrong_protocol( selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: @run_in_pyodide # type: ignore[misc] def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + import pytest - import urllib3.exceptions from urllib3.connection import HTTPConnection conn = HTTPConnection(host, port) - with pytest.raises(urllib3.exceptions.ResponseError): + with pytest.raises(http.client.HTTPException): conn.request("GET", f"http://{host}:{port}/") pyodide_test( @@ -156,20 +157,21 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno ) -# wrong protocol / protocol error etc. should raise an exception of urllib3.exceptions.ResponseError +# wrong protocol / protocol error etc. should raise an exception of http.client.HTTPException @install_urllib3_wheel() def test_bad_method( selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ) -> None: @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + import pytest - import urllib3.exceptions from urllib3.connection import HTTPConnection conn = HTTPConnection(host, port) - with pytest.raises(urllib3.exceptions.ResponseError): + with pytest.raises(http.client.HTTPException): conn.request("TRACE", f"http://{host}:{port}/") pyodide_test( @@ -184,18 +186,16 @@ def test_no_response( ) -> None: @run_in_pyodide(packages=("pytest",)) # type: ignore[misc] def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + import pytest - import urllib3.exceptions from urllib3.connection import HTTPConnection conn = HTTPConnection(host, port) - try: + with pytest.raises(http.client.HTTPException): conn.request("GET", f"http://{host}:{port}/") _ = conn.getresponse() - pytest.fail("No response, should throw exception.") - except BaseException as ex: - assert isinstance(ex, urllib3.exceptions.ResponseError) pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) @@ -458,7 +458,6 @@ def test_streaming_download( await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() from urllib3.response import BaseHTTPResponse from urllib3.connection import HTTPConnection - import js conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) conn.request("GET", "{bigfile_url}",preload_content=False) @@ -497,7 +496,7 @@ def test_streaming_close( conn.request("GET", "{url}",preload_content=False) response = conn.getresponse() # check body is a RawIOBase stream and isn't seekable, writeable - body_internal = response._response.body.raw + body_internal = response._response.body assert(isinstance(body_internal,RawIOBase)) assert(body_internal.writable() is False) assert(body_internal.seekable() is False) @@ -531,15 +530,14 @@ def test_streaming_bad_url( import pytest import pyodide_js as pjs await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) - + import http.client import urllib3.contrib.emscripten.fetch await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() from urllib3.response import BaseHTTPResponse from urllib3.connection import HTTPConnection - from urllib3.exceptions import ResponseError conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) - with pytest.raises(ResponseError): + with pytest.raises(http.client.HTTPException): conn.request("GET", "{bad_url}",preload_content=False) """ run_from_server.run_webworker(worker_code) @@ -558,6 +556,7 @@ def test_streaming_bad_method( # as you can't do it on main thread worker_code = f""" import pytest + import http.client import pyodide_js as pjs await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) @@ -565,10 +564,9 @@ def test_streaming_bad_method( await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() from urllib3.response import BaseHTTPResponse from urllib3.connection import HTTPConnection - from urllib3.exceptions import ResponseError conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) - with pytest.raises(ResponseError): + with pytest.raises(http.client.HTTPException): # TRACE method should throw SecurityError in Javascript conn.request("TRACE", "{bad_url}",preload_content=False) """ @@ -763,29 +761,30 @@ def test_break_worker_streaming( import pytest import urllib3.contrib.emscripten.fetch import js + import http.client await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() - from urllib3.exceptions import TimeoutError,ResponseError + from urllib3.exceptions import TimeoutError from urllib3.connection import HTTPConnection conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) # make the fetch worker return a bad response by: # 1) Clearing the int buffer # in the receive stream - with pytest.raises(ResponseError): + with pytest.raises(http.client.HTTPException): conn.request("GET","/",preload_content=False) response = conn.getresponse() - body_internal = response._response.body.raw + body_internal = response._response.body assert(body_internal.int_buffer!=None) body_internal.int_buffer=None data=response.read() # 2) Monkeypatch postMessage so that it just sets an # exception status old_pm= body_internal.worker.postMessage - with pytest.raises(ResponseError): + with pytest.raises(http.client.HTTPException): conn.request("GET","/",preload_content=False) response = conn.getresponse() # make posted messages set an exception - body_internal = response._response.body.raw + body_internal = response._response.body def set_exception(*args): body_internal.int_buffer[1]=5 body_internal.int_buffer[2]=ord("W") @@ -804,7 +803,7 @@ def set_exception(*args): conn.request("GET","/",preload_content=False) response = conn.getresponse() # make posted messages not be send - body_internal = response._response.body.raw + body_internal = response._response.body def ignore_message(*args): pass old_pm= body_internal.worker.postMessage @@ -903,3 +902,39 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno pyodide_test( selenium_coverage, testserver_http.http_host, testserver_http.http_port ) + + +@install_urllib3_wheel() +def test_retries( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3 + + pool = urllib3.HTTPConnectionPool( + host, + port, + maxsize=1, + block=True, + retries=urllib3.util.Retry(connect=5, read=5, redirect=5), + ) + + # monkeypatch connection class to count calls + old_request = urllib3.connection.HTTPConnection.request + count = 0 + + def count_calls(self, *args, **argv): # type: ignore[no-untyped-def] + nonlocal count + count += 1 + return old_request(self, *args, **argv) + + urllib3.connection.HTTPConnection.request = count_calls # type: ignore[method-assign] + with pytest.raises(urllib3.exceptions.MaxRetryError): + pool.urlopen("GET", "/") + # this should fail, but should have tried 6 times total + assert count == 6 + + pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) From 9de656b1fd0e0641f81ac45543af04a39740a35c Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 16:22:17 +0000 Subject: [PATCH 44/49] test fixes --- src/urllib3/contrib/emscripten/fetch.py | 10 +++--- src/urllib3/contrib/emscripten/response.py | 6 ++-- test/contrib/emscripten/conftest.py | 5 +++ test/contrib/emscripten/test_emscripten.py | 42 +++++++++++++--------- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index 38f92ef414..399a2a18dc 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -133,7 +133,7 @@ def close(self) -> None: def readable(self) -> bool: return True - def writeable(self) -> bool: + def writable(self) -> bool: return False def seekable(self) -> bool: @@ -200,7 +200,7 @@ def onMsg(e: JsProxy) -> None: self.streaming_ready = True js_resolve_fn(e) - def onErr(e: JsProxy) -> None: + def onErr(e: JsProxy) -> None: # pragma: no cover js_reject_fn(e) self.js_worker.onmessage = onMsg @@ -348,9 +348,11 @@ def _show_streaming_warning() -> None: if is_in_browser_main_thread(): message += " Python is running in main browser thread\n" if not is_worker_available(): - message += " Worker or Blob classes are not available in this environment." + # this would fire if we tested in node without + # correct packages loaded + message += " Worker or Blob classes are not available in this environment." # pragma: no cover if streaming_ready() is False: - message += """ Streaming fetch worker isn't ready. If you want to be sure that streamig fetch + message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`""" from js import console diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py index cce89e596f..303b4ee011 100644 --- a/src/urllib3/contrib/emscripten/response.py +++ b/src/urllib3/contrib/emscripten/response.py @@ -5,7 +5,7 @@ import typing from contextlib import contextmanager from dataclasses import dataclass -from http.client import HTTPException as HTTPException # noqa: F401 +from http.client import HTTPException as HTTPException from io import BytesIO, IOBase from ...exceptions import InvalidHeader, TimeoutError @@ -250,9 +250,9 @@ def _error_catcher(self) -> typing.Generator[None, None, None]: # unnecessarily. clean_exit = True except _TimeoutError as e: - raise TimeoutError(e.message) + raise TimeoutError(str(e)) except _RequestError as e: - raise HTTPException(e.message) + raise HTTPException(str(e)) finally: # If we didn't terminate cleanly, we need to throw away our # connection. diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py index fe2d59bfc3..abaa3bee6e 100644 --- a/test/contrib/emscripten/conftest.py +++ b/test/contrib/emscripten/conftest.py @@ -201,6 +201,11 @@ def bigfile(self, req: HTTPServerRequest) -> Response: bigdata = 1048576 * b"WOOO YAY BOOYAKAH" return Response(bigdata) + def mediumfile(self, req: HTTPServerRequest) -> Response: + # quite big file + bigdata = 1024 * b"WOOO YAY BOOYAKAH" + return Response(bigdata) + def pyodide(self, req: HTTPServerRequest) -> Response: path = req.path[:] if not path.startswith("/"): diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index ffcfe59e14..43bed754a7 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -500,10 +500,10 @@ def test_streaming_close( assert(isinstance(body_internal,RawIOBase)) assert(body_internal.writable() is False) assert(body_internal.seekable() is False) - + assert(body_internal.readable() is True) response.drain_conn() x=response.read() - assert(x==None) + assert(not x) response.close() conn.close() # try and make destructor be covered @@ -582,9 +582,7 @@ def test_streaming_notready_warning( # test streaming download but don't wait for # worker to be ready - should fallback to non-streaming # and log a warning - bigfile_url = ( - f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" - ) + file_url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" worker_code = f""" import pyodide_js as pjs await pjs.loadPackage('http://{testserver_http.http_host}:{testserver_http.http_port}/wheel/dist.whl',deps=False) @@ -601,7 +599,7 @@ def capture_log(*args): js.console.warn=capture_log conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) - conn.request("GET", "{bigfile_url}",preload_content=False) + conn.request("GET", "{file_url}",preload_content=False) js.console.warn=old_log response = conn.getresponse() assert isinstance(response, BaseHTTPResponse) @@ -658,7 +656,7 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno "upload_filename": "lolcat.txt", "filefield": ("lolcat.txt", data), } - fields["upload_size"] = len(data) # type: ignore + fields["upload_size"] = str(len(data)) with HTTPConnectionPool(host, port) as pool: r = pool.request("POST", "/upload", fields=fields) assert r.status == 200 @@ -786,17 +784,29 @@ def test_break_worker_streaming( # make posted messages set an exception body_internal = response._response.body def set_exception(*args): - body_internal.int_buffer[1]=5 - body_internal.int_buffer[2]=ord("W") - body_internal.int_buffer[3]=ord("O") - body_internal.int_buffer[4]=ord("O") - body_internal.int_buffer[5]=ord("!") - body_internal.int_buffer[6]=0 + body_internal.worker.postMessage = old_pm + body_internal.int_buffer[1]=4 + body_internal.byte_buffer[0]=ord("W") + body_internal.byte_buffer[1]=ord("O") + body_internal.byte_buffer[2]=ord("O") + body_internal.byte_buffer[3]=ord("!") + body_internal.byte_buffer[4]=0 js.Atomics.store(body_internal.int_buffer, 0, -4) js.Atomics.notify(body_internal.int_buffer,0) body_internal.worker.postMessage = set_exception data=response.read() - body_internal.worker.postMessage = old_pm + # monkeypatch so it returns an unknown value for the magic number on initial fetch call + with pytest.raises(http.client.HTTPException): + # make posted messages set an exception + worker=urllib3.contrib.emscripten.fetch._fetcher.js_worker + def set_exception(self,*args): + array=js.Int32Array.new(args[0].buffer) + array[0]=-1234 + worker.postMessage=set_exception.__get__(worker,worker.__class__) + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + data=response.read() + urllib3.contrib.emscripten.fetch._fetcher.js_worker.postMessage=old_pm # 3) Stopping the worker receiving any messages which should cause a timeout error # in the receive stream with pytest.raises(TimeoutError): @@ -890,11 +900,11 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno from urllib3.connection import HTTPConnection conn = HTTPConnection(host, port) - conn.request("GET", f"http://{host}:{port}/bigfile", preload_content=False) + conn.request("GET", f"http://{host}:{port}/mediumfile", preload_content=False) response = conn.getresponse() count = 0 for x in response.read_chunked(64): - count += 0 + count += 1 assert len(x) == 64 if count > 10: break From 68ff2d006f9004fa46def308d5d572fcf12acb68 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 16:30:59 +0000 Subject: [PATCH 45/49] test fixes --- test/contrib/emscripten/test_emscripten.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 43bed754a7..b68d3f4a9b 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -598,6 +598,8 @@ def capture_log(*args): old_log(*args) js.console.warn=capture_log + urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING = False + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) conn.request("GET", "{file_url}",preload_content=False) js.console.warn=old_log From 1af9fdd17ffe13100aa856136743a7325bece285 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 16:39:32 +0000 Subject: [PATCH 46/49] test bugfix --- test/contrib/emscripten/test_emscripten.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index b68d3f4a9b..0577951776 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -598,8 +598,6 @@ def capture_log(*args): old_log(*args) js.console.warn=capture_log - urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING = False - conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) conn.request("GET", "{file_url}",preload_content=False) js.console.warn=old_log @@ -608,7 +606,6 @@ def capture_log(*args): data=response.data.decode('utf-8') assert len([x for x in log_msgs if x.find("Can't stream HTTP requests")!=-1])==1 assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True - assert len(data) == 17825792 """ run_from_server.run_webworker(worker_code) From 19964c26aba33a557efaef5bf3f02756c94eee8f Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 19:52:27 +0000 Subject: [PATCH 47/49] mark defensive tests --- src/urllib3/contrib/emscripten/fetch.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index 399a2a18dc..b568df1e86 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -200,9 +200,8 @@ def onMsg(e: JsProxy) -> None: self.streaming_ready = True js_resolve_fn(e) - def onErr(e: JsProxy) -> None: # pragma: no cover - js_reject_fn(e) - + def onErr(e: JsProxy) -> None: + js_reject_fn(e) # Defensive: never happens in ci self.js_worker.onmessage = onMsg self.js_worker.onerror = onErr @@ -348,10 +347,8 @@ def _show_streaming_warning() -> None: if is_in_browser_main_thread(): message += " Python is running in main browser thread\n" if not is_worker_available(): - # this would fire if we tested in node without - # correct packages loaded - message += " Worker or Blob classes are not available in this environment." # pragma: no cover - if streaming_ready() is False: + message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in + if streaming_ready() is False: message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`""" from js import console From 985d8cafdab19910209318f25482233e57d9686c Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 19:56:21 +0000 Subject: [PATCH 48/49] fix tests --- src/urllib3/contrib/emscripten/fetch.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py index b568df1e86..ecf845d931 100644 --- a/src/urllib3/contrib/emscripten/fetch.py +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -201,7 +201,8 @@ def onMsg(e: JsProxy) -> None: js_resolve_fn(e) def onErr(e: JsProxy) -> None: - js_reject_fn(e) # Defensive: never happens in ci + js_reject_fn(e) # Defensive: never happens in ci + self.js_worker.onmessage = onMsg self.js_worker.onerror = onErr @@ -347,8 +348,8 @@ def _show_streaming_warning() -> None: if is_in_browser_main_thread(): message += " Python is running in main browser thread\n" if not is_worker_available(): - message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in - if streaming_ready() is False: + message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in + if streaming_ready() is False: message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`""" from js import console From 098473210f51c876100404e1f16036f5c2475669 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 24 Nov 2023 21:50:37 +0000 Subject: [PATCH 49/49] Tests --- test/contrib/emscripten/test_emscripten.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py index 0577951776..29b8cf08a2 100644 --- a/test/contrib/emscripten/test_emscripten.py +++ b/test/contrib/emscripten/test_emscripten.py @@ -902,11 +902,10 @@ def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: igno conn.request("GET", f"http://{host}:{port}/mediumfile", preload_content=False) response = conn.getresponse() count = 0 - for x in response.read_chunked(64): + for x in response.read_chunked(512): count += 1 - assert len(x) == 64 - if count > 10: - break + if count < 10: + assert len(x) == 512 pyodide_test( selenium_coverage, testserver_http.http_host, testserver_http.http_port