http: Add support for proxies

not_my_board/_http.py

@@ -7,6 +7,9 @@
 import logging
 import ssl
 import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from typing import Optional, Union
 
 import h11
 import websockets
@@ -30,6 +33,9 @@ def __init__(self, ca_files=None):
         if ca_files:
             for ca_file in ca_files:
                 self._ssl_ctx.load_verify_locations(cafile=ca_file)
+        self._proxies = {
+            scheme: self._parse_url(url) for scheme, url in urllib.request.getproxies()
+        }
 
     async def get_json(self, url):
         return await self._request_json("GET", url)
@@ -40,7 +46,7 @@ async def post_form(self, url, params):
         return await self._request_json("POST", url, content_type, body)
 
     async def _request_json(self, method, url, content_type=None, body=None):
-        url = urllib.parse.urlsplit(url)
+        url = self._parse_url(url)
         headers = [
             ("Host", url.netloc),
             ("User-Agent", h11.PRODUCT_ID),
@@ -62,18 +68,7 @@ async def _request_json(self, method, url, content_type=None, body=None):
             to_send += conn.send(h11.Data(body))
         to_send += conn.send(h11.EndOfMessage())
 
-        if url.scheme == "https":
-            default_port = 443
-            ssl_ = self._ssl_ctx
-        elif url.scheme == "http":
-            default_port = 80
-            ssl_ = False
-        else:
-            raise ValueError(f'Unknown scheme "{url.scheme}"')
-
-        port = url.port or default_port
-
-        async with util.connect(url.hostname, port, ssl=ssl_) as (reader, writer):
+        async with self._connect(url) as (reader, writer):
             writer.write(to_send)
             await writer.drain()
 
@@ -105,7 +100,66 @@ async def receive_all():
         return json.loads(content)
 
     @contextlib.asynccontextmanager
-    async def open_tunnel(self, proxy_host, proxy_port, target_host, target_port):
+    async def _connect(self, url):
+        proxy = self._get_proxy(url)
+        if proxy:
+            tunnel = self.open_tunnel(
+                proxy.host, proxy.port, url.host, url.port, ssl_=proxy.ssl
+            )
+            async with tunnel as (reader, writer, trailing_data):
+                if trailing_data:
+                    raise ProtocolError("Unexpected trailing_data")
+                if url.ssl:
+                    await writer.start_tls(url.ssl, server_hostname=url.host)
+                yield reader, writer
+        else:
+            async with util.connect(url.host, url.port, ssl=url.ssl) as (
+                reader,
+                writer,
+            ):
+                yield reader, writer
+
+    def _get_proxy(self, url):
+        proxy = self._proxies.get(url.scheme)
+        if proxy and not urllib.request.proxy_bypass_environment(
+            url.host, self._proxies
+        ):
+            return proxy
+        return None
+
+    def _parse_url(self, url):
+        url = urllib.parse.urlsplit(url)
+        if url.scheme == "https":
+            default_port = 443
+            ssl_ = self._ssl_ctx
+        elif url.scheme == "http":
+            default_port = 80
+            ssl_ = False
+        else:
+            raise ValueError(f'Unknown scheme "{url.scheme}"')
+
+        port = url.port or default_port
+
+        if not url.hostname:
+            raise ValueError(f'No hostname in URL "{url}"')
+
+        return _ParsedURL(
+            url.scheme,
+            url.netloc,
+            url.hostname,
+            port,
+            url.path,
+            url.query,
+            url.fragment,
+            url.username,
+            url.password,
+            ssl_,
+        )
+
+    @contextlib.asynccontextmanager
+    async def open_tunnel(
+        self, proxy_host, proxy_port, target_host, target_port, ssl_=False
+    ):
         headers = [
             ("Host", f"{target_host}:{target_port}"),
             ("User-Agent", h11.PRODUCT_ID),
@@ -118,7 +172,7 @@ async def open_tunnel(self, proxy_host, proxy_port, target_host, target_port):
             )
         )
 
-        async with util.connect(proxy_host, proxy_port) as (reader, writer):
+        async with util.connect(proxy_host, proxy_port, ssl=ssl_) as (reader, writer):
             writer.write(to_send)
             writer.write(conn.send(h11.EndOfMessage()))
             await writer.drain()
@@ -284,3 +338,17 @@ async def _send_protocol_data(self):
             else:
                 if self._writer.can_write_eof():
                     self._writer.write_eof()
+
+
+@dataclass
+class _ParsedURL:
+    scheme: str
+    netloc: str
+    host: str
+    port: int
+    path: str
+    query: str
+    fragment: str
+    username: Optional[str]
+    password: Optional[str]
+    ssl: Union[bool, ssl.SSLContext]