Add rudimentary HTTP/2 connection and response

changelog/3284.feature.rst

@@ -0,0 +1 @@
+Added rudimentary support for HTTP/2 via ``urllib3.contrib.h2`` and ``HTTP2Connection`` class.

noxfile.py

@@ -11,7 +11,7 @@
 
 def tests_impl(
     session: nox.Session,
-    extras: str = "socks,brotli,zstd",
+    extras: str = "socks,brotli,zstd,h2",
     # hypercorn dependency h2 compares bytes and strings
     # https://github.com/python-hyper/h2/issues/1236
     byte_string_comparisons: bool = False,

pyproject.toml

@@ -49,6 +49,9 @@ zstd = [
 socks = [
   "PySocks>=1.5.6,<2.0,!=1.5.7",
 ]
+h2 = [
+  "h2>=4,<5"
+]
 
 [project.urls]
 "Changelog" = "https://github.com/urllib3/urllib3/blob/main/CHANGES.rst"

src/urllib3/http2.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import contextlib
+import threading
+import typing
+
+import h2.config  # type: ignore[import]
+import h2.connection  # type: ignore[import]
+import h2.events  # type: ignore[import]
+
+import urllib3.connection
+import urllib3.util.ssl_
+
+from ._collections import HTTPHeaderDict
+from .connection import HTTPSConnection
+from .connectionpool import HTTPSConnectionPool
+
+orig_HTTPSConnection = HTTPSConnection
+
+
+class HTTP2Connection(HTTPSConnection):
+    def __init__(
+        self, host: str, port: int | None = None, **kwargs: typing.Any
+    ) -> None:
+        self._h2_lock = threading.RLock()
+        self._h2_conn = h2.connection.H2Connection(
+            config=h2.config.H2Configuration(client_side=True)
+        )
+        self._h2_stream: int | None = None
+        self._h2_headers: list[tuple[bytes, bytes]] = []
+
+        if "proxy" in kwargs or "proxy_config" in kwargs:  # Defensive:
+            raise NotImplementedError("Proxies aren't supported with HTTP/2")
+
+        super().__init__(host, port, **kwargs)
+
+    @contextlib.contextmanager
+    def _lock_h2_conn(self) -> typing.Generator[h2.connection.H2Connection, None, None]:
+        with self._h2_lock:
+            yield self._h2_conn
+
+    def connect(self) -> None:
+        super().connect()
+
+        with self._lock_h2_conn() as h2_conn:
+            h2_conn.initiate_connection()
+            self.sock.sendall(h2_conn.data_to_send())
+
+    def putrequest(
+        self,
+        method: str,
+        url: str,
+        skip_host: bool = False,
+        skip_accept_encoding: bool = False,
+    ) -> None:
+        with self._lock_h2_conn() as h2_conn:
+            self._h2_stream = h2_conn.get_next_available_stream_id()
+
+            if ":" in self.host:
+                authority = f"[{self.host}]:{self.port or 443}"
+            else:
+                authority = f"{self.host}:{self.port or 443}"
+
+            self._h2_headers.extend(
+                (
+                    (b":scheme", b"https"),
+                    (b":method", method.encode()),
+                    (b":authority", authority.encode()),
+                    (b":path", url.encode()),
+                )
+            )
+
+    def putheader(self, header: str, *values: str) -> None:
+        for value in values:
+            self._h2_headers.append(
+                (header.encode("utf-8").lower(), value.encode("utf-8"))
+            )
+
+    def endheaders(self) -> None:  # type: ignore[override]
+        with self._lock_h2_conn() as h2_conn:
+            h2_conn.send_headers(
+                stream_id=self._h2_stream,
+                headers=self._h2_headers,
+                end_stream=True,
+            )
+            if data_to_send := h2_conn.data_to_send():
+                self.sock.sendall(data_to_send)
+
+    def send(self, data: bytes) -> None:  # type: ignore[override]  # Defensive:
+        if not data:
+            return
+        raise NotImplementedError("Sending data isn't supported yet")
+
+    def getresponse(  # type: ignore[override]
+        self,
+    ) -> HTTP2Response:
+        status = None
+        data = bytearray()
+        with self._lock_h2_conn() as h2_conn:
+            end_stream = False
+            while not end_stream:
+                # TODO: Arbitrary read value.
+                if received_data := self.sock.recv(65535):
+                    events = h2_conn.receive_data(received_data)
+                    for event in events:
+                        if isinstance(
+                            event, h2.events.InformationalResponseReceived
+                        ):  # Defensive:
+                            continue  # TODO: Does the stdlib do anything with these responses?
+
+                        elif isinstance(event, h2.events.ResponseReceived):
+                            headers = HTTPHeaderDict()
+                            for header, value in event.headers:
+                                if header == b":status":
+                                    status = int(value.decode())
+                                else:
+                                    headers.add(
+                                        header.decode("ascii"), value.decode("ascii")
+                                    )
+
+                        elif isinstance(event, h2.events.DataReceived):
+                            data += event.data
+                            h2_conn.acknowledge_received_data(
+                                event.flow_controlled_length, event.stream_id
+                            )
+
+                        elif isinstance(event, h2.events.StreamEnded):
+                            end_stream = True
+
+                if data_to_send := h2_conn.data_to_send():
+                    self.sock.sendall(data_to_send)
+
+        # We always close to not have to handle connection management.
+        self.close()
+
+        assert status is not None
+        return HTTP2Response(status=status, headers=headers, data=bytes(data))
+
+    def close(self) -> None:
+        with self._lock_h2_conn() as h2_conn:
+            try:
+                self._h2_conn.close_connection()
+                if data := h2_conn.data_to_send():
+                    self.sock.sendall(data)
+            except Exception:
+                pass
+
+        # Reset all our HTTP/2 connection state.
+        self._h2_conn = h2.connection.H2Connection(
+            config=h2.config.H2Configuration(client_side=True)
+        )
+        self._h2_stream = None
+        self._h2_headers = []
+
+        super().close()
+
+
+class HTTP2Response:
+    # TODO: This is a woefully incomplete response object, but works for non-streaming.
+    def __init__(self, status: int, headers: HTTPHeaderDict, data: bytes) -> None:
+        self.status = status
+        self.headers = headers
+        self.data = data
+        self.length_remaining = 0
+
+    def get_redirect_location(self) -> None:
+        return None
+
+
+def inject_into_urllib3() -> None:
+    HTTPSConnectionPool.ConnectionCls = HTTP2Connection  # type: ignore[assignment]
+    urllib3.connection.HTTPSConnection = HTTP2Connection  # type: ignore[misc]
+
+    # TODO: Offer 'http/1.1' as well, but for testing purposes this is handy.
+    urllib3.util.ssl_.ALPN_PROTOCOLS = ["h2"]
+
+
+def extract_from_urllib3() -> None:
+    HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection
+    urllib3.connection.HTTPSConnection = orig_HTTPSConnection  # type: ignore[misc]
+
+    urllib3.util.ssl_.ALPN_PROTOCOLS = ["http/1.1"]

test/with_dummyserver/test_http2.py

@@ -2,26 +2,72 @@
 
 import subprocess
 from test import notWindows
+from test.conftest import ServerConfig
 
-from dummyserver.testcase import HypercornDummyServerTestCase
+import pytest
 
+import urllib3
+from dummyserver.socketserver import DEFAULT_CERTS
+from dummyserver.testcase import HTTPSHypercornDummyServerTestCase
+
+DEFAULT_CERTS_HTTP2 = DEFAULT_CERTS.copy()
+DEFAULT_CERTS_HTTP2["alpn_protocols"] = ["h2"]
+
+
+def setup_module() -> None:
+    try:
+        from urllib3.http2 import inject_into_urllib3
+
+        inject_into_urllib3()
+    except ImportError as e:
+        pytest.skip(f"Could not import h2: {e!r}")
+
+
+def teardown_module() -> None:
+    try:
+        from urllib3.http2 import extract_from_urllib3
+
+        extract_from_urllib3()
+    except ImportError:
+        pass
+
+
+class TestHypercornDummyServerTestCase(HTTPSHypercornDummyServerTestCase):
+    certs = DEFAULT_CERTS_HTTP2
 
-class TestHypercornDummyServerTestCase(HypercornDummyServerTestCase):
     @classmethod
     def setup_class(cls) -> None:
         super().setup_class()
-        cls.base_url = f"http://{cls.host}:{cls.port}"
+        cls.base_url = f"https://{cls.host}:{cls.port}"
 
     @notWindows()  # GitHub Actions Windows doesn't have HTTP/2 support.
     def test_hypercorn_server_http2(self) -> None:
         # This is a meta test to make sure our Hypercorn test server is actually using HTTP/2
         # before urllib3 is capable of speaking HTTP/2. Thanks, Daniel! <3
         output = subprocess.check_output(
-            ["curl", "-vvv", "--http2", self.base_url], stderr=subprocess.STDOUT
+            [
+                "curl",
+                "-vvv",
+                "--http2",
+                "--cacert",
+                self.certs["ca_certs"],
+                self.base_url,
+            ],
+            stderr=subprocess.STDOUT,
         )
 
-        # curl does HTTP/1.1 and upgrades to HTTP/2 without TLS which is fine
-        # for us. Hypercorn supports this thankfully, but we should try with
-        # HTTPS as well once that's available.
         assert b"< HTTP/2 200" in output
         assert output.endswith(b"Dummy server!")
+
+
+def test_simple_http2(san_server: ServerConfig) -> None:
+    with urllib3.PoolManager(ca_certs=san_server.ca_certs) as http:
+        resp = http.request("HEAD", san_server.base_url, retries=False)
+
+    assert resp.status == 200
+    resp.headers.pop("date")
+    assert resp.headers == {
+        "content-type": "text/html; charset=utf-8",
+        "content-length": "13",
+        "server": "hypercorn-h2",
+    }