diff --git a/changelog/3284.feature.rst b/changelog/3284.feature.rst new file mode 100644 index 0000000000..3105c9a8e2 --- /dev/null +++ b/changelog/3284.feature.rst @@ -0,0 +1 @@ +Added rudimentary support for HTTP/2 via ``urllib3.contrib.h2`` and ``HTTP2Connection`` class. diff --git a/noxfile.py b/noxfile.py index 9c97b92069..3a29d7fd04 100644 --- a/noxfile.py +++ b/noxfile.py @@ -11,7 +11,7 @@ def tests_impl( session: nox.Session, - extras: str = "socks,brotli,zstd", + extras: str = "socks,brotli,zstd,h2", # hypercorn dependency h2 compares bytes and strings # https://github.com/python-hyper/h2/issues/1236 byte_string_comparisons: bool = False, diff --git a/pyproject.toml b/pyproject.toml index 962330940c..1fe82937ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,9 @@ zstd = [ socks = [ "PySocks>=1.5.6,<2.0,!=1.5.7", ] +h2 = [ + "h2>=4,<5" +] [project.urls] "Changelog" = "https://github.com/urllib3/urllib3/blob/main/CHANGES.rst" diff --git a/src/urllib3/http2.py b/src/urllib3/http2.py new file mode 100644 index 0000000000..3364500c48 --- /dev/null +++ b/src/urllib3/http2.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import contextlib +import threading +import typing + +import h2.config # type: ignore[import] +import h2.connection # type: ignore[import] +import h2.events # type: ignore[import] + +import urllib3.connection +import urllib3.util.ssl_ + +from ._collections import HTTPHeaderDict +from .connection import HTTPSConnection +from .connectionpool import HTTPSConnectionPool + +orig_HTTPSConnection = HTTPSConnection + + +class HTTP2Connection(HTTPSConnection): + def __init__( + self, host: str, port: int | None = None, **kwargs: typing.Any + ) -> None: + self._h2_lock = threading.RLock() + self._h2_conn = h2.connection.H2Connection( + config=h2.config.H2Configuration(client_side=True) + ) + self._h2_stream: int | None = None + self._h2_headers: list[tuple[bytes, bytes]] = [] + + if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive: + raise NotImplementedError("Proxies aren't supported with HTTP/2") + + super().__init__(host, port, **kwargs) + + @contextlib.contextmanager + def _lock_h2_conn(self) -> typing.Generator[h2.connection.H2Connection, None, None]: + with self._h2_lock: + yield self._h2_conn + + def connect(self) -> None: + super().connect() + + with self._lock_h2_conn() as h2_conn: + h2_conn.initiate_connection() + self.sock.sendall(h2_conn.data_to_send()) + + def putrequest( + self, + method: str, + url: str, + skip_host: bool = False, + skip_accept_encoding: bool = False, + ) -> None: + with self._lock_h2_conn() as h2_conn: + self._h2_stream = h2_conn.get_next_available_stream_id() + + if ":" in self.host: + authority = f"[{self.host}]:{self.port or 443}" + else: + authority = f"{self.host}:{self.port or 443}" + + self._h2_headers.extend( + ( + (b":scheme", b"https"), + (b":method", method.encode()), + (b":authority", authority.encode()), + (b":path", url.encode()), + ) + ) + + def putheader(self, header: str, *values: str) -> None: + for value in values: + self._h2_headers.append( + (header.encode("utf-8").lower(), value.encode("utf-8")) + ) + + def endheaders(self) -> None: # type: ignore[override] + with self._lock_h2_conn() as h2_conn: + h2_conn.send_headers( + stream_id=self._h2_stream, + headers=self._h2_headers, + end_stream=True, + ) + if data_to_send := h2_conn.data_to_send(): + self.sock.sendall(data_to_send) + + def send(self, data: bytes) -> None: # type: ignore[override] # Defensive: + if not data: + return + raise NotImplementedError("Sending data isn't supported yet") + + def getresponse( # type: ignore[override] + self, + ) -> HTTP2Response: + status = None + data = bytearray() + with self._lock_h2_conn() as h2_conn: + end_stream = False + while not end_stream: + # TODO: Arbitrary read value. + if received_data := self.sock.recv(65535): + events = h2_conn.receive_data(received_data) + for event in events: + if isinstance( + event, h2.events.InformationalResponseReceived + ): # Defensive: + continue # TODO: Does the stdlib do anything with these responses? + + elif isinstance(event, h2.events.ResponseReceived): + headers = HTTPHeaderDict() + for header, value in event.headers: + if header == b":status": + status = int(value.decode()) + else: + headers.add( + header.decode("ascii"), value.decode("ascii") + ) + + elif isinstance(event, h2.events.DataReceived): + data += event.data + h2_conn.acknowledge_received_data( + event.flow_controlled_length, event.stream_id + ) + + elif isinstance(event, h2.events.StreamEnded): + end_stream = True + + if data_to_send := h2_conn.data_to_send(): + self.sock.sendall(data_to_send) + + # We always close to not have to handle connection management. + self.close() + + assert status is not None + return HTTP2Response(status=status, headers=headers, data=bytes(data)) + + def close(self) -> None: + with self._lock_h2_conn() as h2_conn: + try: + self._h2_conn.close_connection() + if data := h2_conn.data_to_send(): + self.sock.sendall(data) + except Exception: + pass + + # Reset all our HTTP/2 connection state. + self._h2_conn = h2.connection.H2Connection( + config=h2.config.H2Configuration(client_side=True) + ) + self._h2_stream = None + self._h2_headers = [] + + super().close() + + +class HTTP2Response: + # TODO: This is a woefully incomplete response object, but works for non-streaming. + def __init__(self, status: int, headers: HTTPHeaderDict, data: bytes) -> None: + self.status = status + self.headers = headers + self.data = data + self.length_remaining = 0 + + def get_redirect_location(self) -> None: + return None + + +def inject_into_urllib3() -> None: + HTTPSConnectionPool.ConnectionCls = HTTP2Connection # type: ignore[assignment] + urllib3.connection.HTTPSConnection = HTTP2Connection # type: ignore[misc] + + # TODO: Offer 'http/1.1' as well, but for testing purposes this is handy. + urllib3.util.ssl_.ALPN_PROTOCOLS = ["h2"] + + +def extract_from_urllib3() -> None: + HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection + urllib3.connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc] + + urllib3.util.ssl_.ALPN_PROTOCOLS = ["http/1.1"] diff --git a/test/with_dummyserver/test_http2.py b/test/with_dummyserver/test_http2.py index fa05900c01..54f2582cb5 100644 --- a/test/with_dummyserver/test_http2.py +++ b/test/with_dummyserver/test_http2.py @@ -2,26 +2,72 @@ import subprocess from test import notWindows +from test.conftest import ServerConfig -from dummyserver.testcase import HypercornDummyServerTestCase +import pytest +import urllib3 +from dummyserver.socketserver import DEFAULT_CERTS +from dummyserver.testcase import HTTPSHypercornDummyServerTestCase + +DEFAULT_CERTS_HTTP2 = DEFAULT_CERTS.copy() +DEFAULT_CERTS_HTTP2["alpn_protocols"] = ["h2"] + + +def setup_module() -> None: + try: + from urllib3.http2 import inject_into_urllib3 + + inject_into_urllib3() + except ImportError as e: + pytest.skip(f"Could not import h2: {e!r}") + + +def teardown_module() -> None: + try: + from urllib3.http2 import extract_from_urllib3 + + extract_from_urllib3() + except ImportError: + pass + + +class TestHypercornDummyServerTestCase(HTTPSHypercornDummyServerTestCase): + certs = DEFAULT_CERTS_HTTP2 -class TestHypercornDummyServerTestCase(HypercornDummyServerTestCase): @classmethod def setup_class(cls) -> None: super().setup_class() - cls.base_url = f"http://{cls.host}:{cls.port}" + cls.base_url = f"https://{cls.host}:{cls.port}" @notWindows() # GitHub Actions Windows doesn't have HTTP/2 support. def test_hypercorn_server_http2(self) -> None: # This is a meta test to make sure our Hypercorn test server is actually using HTTP/2 # before urllib3 is capable of speaking HTTP/2. Thanks, Daniel! <3 output = subprocess.check_output( - ["curl", "-vvv", "--http2", self.base_url], stderr=subprocess.STDOUT + [ + "curl", + "-vvv", + "--http2", + "--cacert", + self.certs["ca_certs"], + self.base_url, + ], + stderr=subprocess.STDOUT, ) - # curl does HTTP/1.1 and upgrades to HTTP/2 without TLS which is fine - # for us. Hypercorn supports this thankfully, but we should try with - # HTTPS as well once that's available. assert b"< HTTP/2 200" in output assert output.endswith(b"Dummy server!") + + +def test_simple_http2(san_server: ServerConfig) -> None: + with urllib3.PoolManager(ca_certs=san_server.ca_certs) as http: + resp = http.request("HEAD", san_server.base_url, retries=False) + + assert resp.status == 200 + resp.headers.pop("date") + assert resp.headers == { + "content-type": "text/html; charset=utf-8", + "content-length": "13", + "server": "hypercorn-h2", + }