From bb60c65dd8244b2985c8d70a783b1c2a2351faeb Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Tue, 21 Dec 2021 22:32:10 +0530
Subject: [PATCH 01/22] Merge pull request #896 from
 abhinavsingh/missed-scenarios

Avoid registering invalid FD with selectors
---
 .github/workflows/test-library.yml | 14 +++++++++-----
 proxy/core/acceptor/threadless.py  |  2 +-
 proxy/http/proxy/server.py         | 28 ++++++++++++----------------
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml
index e849773a9e..1b7928e0f3 100644
--- a/.github/workflows/test-library.yml
+++ b/.github/workflows/test-library.yml
@@ -485,12 +485,13 @@ jobs:
         path: ${{ steps.pip-cache.outputs.dir }}
         key: >-
           ${{ runner.os }}-pip-${{
-          steps.calc-cache-key-py.outputs.py-hash-key }}-${{
-          hashFiles('tox.ini', 'requirements.txt', 'requirements-testing.txt')
+            steps.calc-cache-key-py.outputs.py-hash-key
+          }}-${{
+            hashFiles('tox.ini', 'requirements**.txt')
           }}
         restore-keys: |
           ${{ runner.os }}-pip-${{
-              steps.calc-cache-key-py.outputs.py-hash-key
+            steps.calc-cache-key-py.outputs.py-hash-key
           }}-
           ${{ runner.os }}-pip-
     - name: Install tox
@@ -746,8 +747,11 @@ jobs:
     - name: Tag latest on GHCR
       if: >-
         github.event_name == 'push' &&
-        github.ref == format(
-          'refs/heads/{0}', github.event.repository.default_branch
+        (
+          github.ref == format(
+            'refs/heads/{0}', github.event.repository.default_branch
+          ) ||
+          github.ref == 'refs/heads/master'
         )
       run: >-
         REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py
index 7e96c3a646..f36f9c0a98 100644
--- a/proxy/core/acceptor/threadless.py
+++ b/proxy/core/acceptor/threadless.py
@@ -180,7 +180,7 @@ async def _update_work_events(self, work_id: int) -> None:
                 # else:
                 #     logger.info(
                 #         'fd#{0} by work#{1} not modified'.format(fileno, work_id))
-            else:
+            elif fileno != -1:
                 # Can throw ValueError: Invalid file descriptor: -1
                 #
                 # A guard within Work classes may not help here due to
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index 0a54c4d5f7..3e41b15625 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -770,11 +770,15 @@ def generate_upstream_certificate(
 
     def intercept(self) -> Union[socket.socket, bool]:
         # Perform SSL/TLS handshake with upstream
-        self.wrap_server()
+        teardown = self.wrap_server()
+        if teardown:
+            return teardown
         # Generate certificate and perform handshake with client
         # wrap_client also flushes client data before wrapping
         # sending to client can raise, handle expected exceptions
-        self.wrap_client()
+        teardown = self.wrap_client()
+        if teardown:
+            return teardown
         # Update all plugin connection reference
         # TODO(abhinavsingh): Is this required?
         for plugin in self.plugins.values():
@@ -813,13 +817,9 @@ def wrap_server(self) -> bool:
                     ), exc_info=e,
                 )
             do_close = True
-        finally:
-            if do_close:
-                raise HttpProtocolException(
-                    'Exception when wrapping server for interception',
-                )
-        assert isinstance(self.upstream.connection, ssl.SSLSocket)
-        return False
+        if not do_close:
+            assert isinstance(self.upstream.connection, ssl.SSLSocket)
+        return do_close
 
     def wrap_client(self) -> bool:
         assert self.upstream is not None and self.flags.ca_signing_key_file is not None
@@ -883,13 +883,9 @@ def wrap_client(self) -> bool:
                 ), exc_info=e,
             )
             do_close = True
-        finally:
-            if do_close:
-                raise HttpProtocolException(
-                    'Exception when wrapping client for interception',
-                )
-        logger.debug('TLS intercepting using %s', generated_cert)
-        return False
+        if not do_close:
+            logger.debug('TLS intercepting using %s', generated_cert)
+        return do_close
 
     #
     # Event emitter callbacks

From da1795d2ed78175addb4da16852903e8e9b1ccd3 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Wed, 22 Dec 2021 01:48:15 +0530
Subject: [PATCH 02/22] [ProxyPool] Add support for basic authorization with
 upstream proxies (#897)

* Use `Url` class to parse proxy pool entries

* Add support for parsing user:pass from raw url bytes

* Add `httpHeaders.PROXY_AUTHORIZATION` headers for upstream proxies

* Add support for httpHeaders enum

* Send base64 encoded proxy authorization header to upstream proxies

* mypy fixes

* Document proxy pool authentication support usage info
---
 README.md                                     | 17 +++---
 proxy/common/constants.py                     | 22 +++++--
 proxy/http/__init__.py                        |  2 +
 proxy/http/headers.py                         | 30 ++++++++++
 proxy/http/proxy/auth.py                      |  6 +-
 proxy/http/proxy/server.py                    |  6 +-
 proxy/http/url.py                             | 38 +++++++++---
 proxy/plugin/proxy_pool.py                    | 59 +++++++++++++------
 .../exceptions/test_http_proxy_auth_failed.py |  8 +--
 tests/http/test_protocol_handler.py           | 10 ++--
 tests/http/test_url.py                        | 20 +++++++
 11 files changed, 168 insertions(+), 50 deletions(-)
 create mode 100644 proxy/http/headers.py

diff --git a/README.md b/README.md
index 992cde04f2..b87bfff2f6 100644
--- a/README.md
+++ b/README.md
@@ -760,9 +760,8 @@ Response body `Hello from man in the middle` is sent by our plugin.
 
 Forward incoming proxy requests to a set of upstream proxy servers.
 
-Let's start upstream proxies first.
-
-Start `proxy.py` on port `9000` and `9001`
+Let's start 2 upstream proxies first.  To simulate upstream proxies,
+start `proxy.py` on port `9000` and `9001`
 
 ```console
 ❯ proxy --port 9000
@@ -789,6 +788,10 @@ Make a curl request via `8899` proxy:
 Verify that `8899` proxy forwards requests to upstream proxies
 by checking respective logs.
 
+If an upstream proxy require credentials, pass them as arguments. Example:
+
+`--proxy-pool user:pass@upstream.proxy:port`
+
 ### FilterByClientIpPlugin
 
 Reject traffic from specific IP addresses. By default this
@@ -2092,7 +2095,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
-proxy.py v2.4.0rc3.dev33+gc341594.d20211214
+proxy.py v2.4.0b4.dev12+g19e6881.d20211221
 
 options:
   -h, --help            show this help message and exit
@@ -2175,9 +2178,9 @@ options:
                         generated HTTPS certificates. If used, must also pass
                         --ca-cert-file and --ca-signing-key-file
   --ca-cert-dir CA_CERT_DIR
-                        Default: ~/.proxy.py. Directory to store dynamically
-                        generated certificates. Also see --ca-key-file, --ca-
-                        cert-file and --ca-signing-key-file
+                        Default: ~/.proxy/certificates. Directory to store
+                        dynamically generated certificates. Also see --ca-key-
+                        file, --ca-cert-file and --ca-signing-key-file
   --ca-cert-file CA_CERT_FILE
                         Default: None. Signing certificate to use for signing
                         dynamically generated HTTPS certificates. If used,
diff --git a/proxy/common/constants.py b/proxy/common/constants.py
index 745281165e..7b5a947d99 100644
--- a/proxy/common/constants.py
+++ b/proxy/common/constants.py
@@ -43,10 +43,24 @@ def _env_threadless_compliant() -> bool:
 COMMA = b','
 DOT = b'.'
 SLASH = b'/'
-HTTP_1_0 = b'HTTP/1.0'
-HTTP_1_1 = b'HTTP/1.1'
-HTTP_URL_PREFIX = b'http://'
-HTTPS_URL_PREFIX = b'https://'
+AT = b'@'
+HTTP_PROTO = b'http'
+HTTPS_PROTO = HTTP_PROTO + b's'
+HTTP_1_0 = HTTP_PROTO.upper() + SLASH + b'1.0'
+HTTP_1_1 = HTTP_PROTO.upper() + SLASH + b'1.1'
+HTTP_URL_PREFIX = HTTP_PROTO + COLON + SLASH + SLASH
+HTTPS_URL_PREFIX = HTTPS_PROTO + COLON + SLASH + SLASH
+
+LOCAL_INTERFACE_HOSTNAMES = (
+    b'localhost',
+    b'127.0.0.1',
+    b'::1',
+)
+
+ANY_INTERFACE_HOSTNAMES = (
+    b'0.0.0.0',
+    b'::',
+)
 
 PROXY_AGENT_HEADER_KEY = b'Proxy-agent'
 PROXY_AGENT_HEADER_VALUE = b'proxy.py v' + \
diff --git a/proxy/http/__init__.py b/proxy/http/__init__.py
index f8d2e4fa4f..b918c3ecf9 100644
--- a/proxy/http/__init__.py
+++ b/proxy/http/__init__.py
@@ -12,6 +12,7 @@
 from .plugin import HttpProtocolHandlerPlugin
 from .codes import httpStatusCodes
 from .methods import httpMethods
+from .headers import httpHeaders
 from .url import Url
 
 __all__ = [
@@ -19,5 +20,6 @@
     'HttpProtocolHandlerPlugin',
     'httpStatusCodes',
     'httpMethods',
+    'httpHeaders',
     'Url',
 ]
diff --git a/proxy/http/headers.py b/proxy/http/headers.py
new file mode 100644
index 0000000000..d042067c02
--- /dev/null
+++ b/proxy/http/headers.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+
+    .. spelling::
+
+       http
+       iterable
+"""
+from typing import NamedTuple
+
+
+# Ref: https://www.iana.org/assignments/http-methods/http-methods.xhtml
+HttpHeaders = NamedTuple(
+    'HttpHeaders', [
+        ('PROXY_AUTHORIZATION', bytes),
+        ('PROXY_CONNECTION', bytes),
+    ],
+)
+
+httpHeaders = HttpHeaders(
+    b'proxy-authorization',
+    b'proxy-connection',
+)
diff --git a/proxy/http/proxy/auth.py b/proxy/http/proxy/auth.py
index be0bccd79b..a309d194c7 100644
--- a/proxy/http/proxy/auth.py
+++ b/proxy/http/proxy/auth.py
@@ -19,6 +19,8 @@
 
 from ...common.flag import flags
 from ...common.constants import DEFAULT_BASIC_AUTH
+
+from ...http import httpHeaders
 from ...http.parser import HttpParser
 from ...http.proxy import HttpProxyBasePlugin
 
@@ -39,9 +41,9 @@ def before_upstream_connection(
             self, request: HttpParser,
     ) -> Optional[HttpParser]:
         if self.flags.auth_code and request.headers:
-            if b'proxy-authorization' not in request.headers:
+            if httpHeaders.PROXY_AUTHORIZATION not in request.headers:
                 raise ProxyAuthenticationFailed()
-            parts = request.headers[b'proxy-authorization'][1].split()
+            parts = request.headers[httpHeaders.PROXY_AUTHORIZATION][1].split()
             if len(parts) != 2 \
                     or parts[0].lower() != b'basic' \
                     or parts[1] != self.flags.auth_code:
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index 3e41b15625..2fda84f8d5 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -26,6 +26,7 @@
 
 from .plugin import HttpProxyBasePlugin
 
+from ..headers import httpHeaders
 from ..methods import httpMethods
 from ..codes import httpStatusCodes
 from ..plugin import HttpProtocolHandlerPlugin
@@ -557,7 +558,10 @@ def on_request_complete(self) -> Union[socket.socket, bool]:
                 #   officially documented in any specification, drop it.
                 # - proxy-authorization is of no use for upstream, remove it.
                 self.request.del_headers(
-                    [b'proxy-authorization', b'proxy-connection'],
+                    [
+                        httpHeaders.PROXY_AUTHORIZATION,
+                        httpHeaders.PROXY_CONNECTION,
+                    ],
                 )
                 # - For HTTP/1.0, connection header defaults to close
                 # - For HTTP/1.1, connection header defaults to keep-alive
diff --git a/proxy/http/url.py b/proxy/http/url.py
index a7fc4390cb..9a5db36611 100644
--- a/proxy/http/url.py
+++ b/proxy/http/url.py
@@ -15,7 +15,7 @@
 """
 from typing import Optional, Tuple
 
-from ..common.constants import COLON, SLASH, HTTP_URL_PREFIX, HTTPS_URL_PREFIX
+from ..common.constants import COLON, SLASH, HTTP_URL_PREFIX, HTTPS_URL_PREFIX, AT
 from ..common.utils import text_
 
 
@@ -28,15 +28,24 @@ class Url:
     def __init__(
             self,
             scheme: Optional[bytes] = None,
+            username: Optional[bytes] = None,
+            password: Optional[bytes] = None,
             hostname: Optional[bytes] = None,
             port: Optional[int] = None,
             remainder: Optional[bytes] = None,
     ) -> None:
         self.scheme: Optional[bytes] = scheme
+        self.username: Optional[bytes] = username
+        self.password: Optional[bytes] = password
         self.hostname: Optional[bytes] = hostname
         self.port: Optional[int] = port
         self.remainder: Optional[bytes] = remainder
 
+    @property
+    def has_credentials(self) -> bool:
+        """Returns true if both username and password components are present."""
+        return self.username is not None and self.password is not None
+
     def __str__(self) -> str:
         url = ''
         if self.scheme:
@@ -74,29 +83,40 @@ def from_bytes(cls, raw: bytes) -> 'Url':
                 if is_https \
                 else raw[len(b'http://'):]
             parts = rest.split(SLASH, 1)
-            host, port = Url.parse_host_and_port(parts[0])
+            username, password, host, port = Url._parse(parts[0])
             return cls(
                 scheme=b'https' if is_https else b'http',
+                username=username,
+                password=password,
                 hostname=host,
                 port=port,
                 remainder=None if len(parts) == 1 else (
                     SLASH + parts[1]
                 ),
             )
-        host, port = Url.parse_host_and_port(raw)
-        return cls(hostname=host, port=port)
+        username, password, host, port = Url._parse(raw)
+        return cls(username=username, password=password, hostname=host, port=port)
 
     @staticmethod
-    def parse_host_and_port(raw: bytes) -> Tuple[bytes, Optional[int]]:
-        parts = raw.split(COLON, 2)
+    def _parse(raw: bytes) -> Tuple[
+            Optional[bytes],
+            Optional[bytes],
+            bytes,
+            Optional[int],
+    ]:
+        split_at = raw.split(AT, 1)
+        username, password = None, None
+        if len(split_at) == 2:
+            username, password = split_at[0].split(COLON)
+        parts = split_at[-1].split(COLON, 2)
         num_parts = len(parts)
         port: Optional[int] = None
         # No port found
         if num_parts == 1:
-            return parts[0], None
+            return username, password, parts[0], None
         # Host and port found
         if num_parts == 2:
-            return COLON.join(parts[:-1]), int(parts[-1])
+            return username, password, COLON.join(parts[:-1]), int(parts[-1])
         # More than a single COLON i.e. IPv6 scenario
         try:
             # Try to resolve last part as an int port
@@ -114,4 +134,4 @@ def parse_host_and_port(raw: bytes) -> Tuple[bytes, Optional[int]]:
                 rhost[0] != '[' and \
                 rhost[-1] != ']':
             host = b'[' + host + b']'
-        return host, port
+        return username, password, host, port
diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py
index c39c6af01d..bfe00aaafa 100644
--- a/proxy/plugin/proxy_pool.py
+++ b/proxy/plugin/proxy_pool.py
@@ -8,6 +8,7 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+import base64
 import random
 import logging
 import ipaddress
@@ -15,9 +16,10 @@
 from typing import Dict, List, Optional, Any
 
 from ..common.flag import flags
-from ..common.utils import text_
+from ..common.utils import text_, bytes_
+from ..common.constants import COLON, LOCAL_INTERFACE_HOSTNAMES, ANY_INTERFACE_HOSTNAMES
 
-from ..http import Url, httpMethods
+from ..http import Url, httpMethods, httpHeaders
 from ..http.parser import HttpParser
 from ..http.exception import HttpProtocolException
 from ..http.proxy import HttpProxyBasePlugin
@@ -67,8 +69,9 @@ class ProxyPoolPlugin(TcpUpstreamConnectionHandler, HttpProxyBasePlugin):
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
+        self._endpoint: Url = self._select_proxy()
         # Cached attributes to be used during access log override
-        self.request_host_port_path_method: List[Any] = [
+        self._metadata: List[Any] = [
             None, None, None, None,
         ]
 
@@ -94,20 +97,22 @@ def before_upstream_connection(
                 return request
         except ValueError:
             pass
-        # Choose a random proxy from the pool
-        # TODO: Implement your own logic here e.g. round-robin, least connection etc.
-        endpoint = random.choice(self.flags.proxy_pool)[0].split(':', 1)
-        if endpoint[0] == 'localhost' and endpoint[1] == '8899':
+        # If chosen proxy is the local instance, bypass upstream proxies
+        assert self._endpoint.port and self._endpoint.hostname
+        if self._endpoint.port == self.flags.port and \
+                self._endpoint.hostname in LOCAL_INTERFACE_HOSTNAMES + ANY_INTERFACE_HOSTNAMES:
             return request
-        logger.debug('Using endpoint: {0}:{1}'.format(*endpoint))
-        self.initialize_upstream(endpoint[0], int(endpoint[1]))
+        # Establish connection to chosen upstream proxy
+        endpoint_tuple = (text_(self._endpoint.hostname), self._endpoint.port)
+        logger.debug('Using endpoint: {0}:{1}'.format(*endpoint_tuple))
+        self.initialize_upstream(*endpoint_tuple)
         assert self.upstream
         try:
             self.upstream.connect()
         except TimeoutError:
             logger.info(
                 'Timed out connecting to upstream proxy {0}:{1}'.format(
-                    *endpoint,
+                    *endpoint_tuple,
                 ),
             )
             raise HttpProtocolException()
@@ -121,13 +126,13 @@ def before_upstream_connection(
             # check.
             logger.info(
                 'Connection refused by upstream proxy {0}:{1}'.format(
-                    *endpoint,
+                    *endpoint_tuple,
                 ),
             )
             raise HttpProtocolException()
         logger.debug(
             'Established connection to upstream proxy {0}:{1}'.format(
-                *endpoint,
+                *endpoint_tuple,
             ),
         )
         return None
@@ -154,10 +159,21 @@ def handle_client_request(
                 443 if request.is_https_tunnel else 80
             )
         path = None if not request.path else request.path.decode()
-        self.request_host_port_path_method = [
+        self._metadata = [
             host, port, path, request.method,
         ]
-        # Queue original request to upstream proxy
+        # Queue original request optionally with auth headers to upstream proxy
+        if self._endpoint.has_credentials:
+            assert self._endpoint.username and self._endpoint.password
+            request.add_header(
+                httpHeaders.PROXY_AUTHORIZATION,
+                b'Basic ' +
+                base64.b64encode(
+                    self._endpoint.username +
+                    COLON +
+                    self._endpoint.password,
+                ),
+            )
         self.upstream.queue(memoryview(request.build(for_proxy=True)))
         return request
 
@@ -189,9 +205,9 @@ def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]:
         context.update({
             'upstream_proxy_host': addr,
             'upstream_proxy_port': port,
-            'server_host': self.request_host_port_path_method[0],
-            'server_port': self.request_host_port_path_method[1],
-            'request_path': self.request_host_port_path_method[2],
+            'server_host': self._metadata[0],
+            'server_port': self._metadata[1],
+            'request_path': self._metadata[2],
             'response_bytes': self.total_size,
         })
         self.access_log(context)
@@ -199,7 +215,14 @@ def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]:
 
     def access_log(self, log_attrs: Dict[str, Any]) -> None:
         access_log_format = DEFAULT_HTTPS_ACCESS_LOG_FORMAT
-        request_method = self.request_host_port_path_method[3]
+        request_method = self._metadata[3]
         if request_method and request_method != httpMethods.CONNECT:
             access_log_format = DEFAULT_HTTP_ACCESS_LOG_FORMAT
         logger.info(access_log_format.format_map(log_attrs))
+
+    def _select_proxy(self) -> Url:
+        """Choose a random proxy from the pool.
+
+        TODO: Implement your own logic here e.g. round-robin, least connection etc.
+        """
+        return Url.from_bytes(bytes_(random.choice(self.flags.proxy_pool)[0]))
diff --git a/tests/http/exceptions/test_http_proxy_auth_failed.py b/tests/http/exceptions/test_http_proxy_auth_failed.py
index 9b4feb1bc5..fdf1b9d6da 100644
--- a/tests/http/exceptions/test_http_proxy_auth_failed.py
+++ b/tests/http/exceptions/test_http_proxy_auth_failed.py
@@ -15,7 +15,7 @@
 
 from proxy.common.flag import FlagParser
 from proxy.http.exception.proxy_auth_failed import ProxyAuthenticationFailed
-from proxy.http import HttpProtocolHandler
+from proxy.http import HttpProtocolHandler, httpHeaders
 from proxy.core.connection import TcpClientConnection
 from proxy.common.utils import build_http_request
 
@@ -77,7 +77,7 @@ async def test_proxy_auth_fails_with_invalid_cred(self) -> None:
             b'GET', b'http://upstream.host/not-found.html',
             headers={
                 b'Host': b'upstream.host',
-                b'Proxy-Authorization': b'Basic hello',
+                httpHeaders.PROXY_AUTHORIZATION: b'Basic hello',
             },
         )
         self.mock_selector.return_value.select.side_effect = [
@@ -105,7 +105,7 @@ async def test_proxy_auth_works_with_valid_cred(self) -> None:
             b'GET', b'http://upstream.host/not-found.html',
             headers={
                 b'Host': b'upstream.host',
-                b'Proxy-Authorization': b'Basic dXNlcjpwYXNz',
+                httpHeaders.PROXY_AUTHORIZATION: b'Basic dXNlcjpwYXNz',
             },
         )
         self.mock_selector.return_value.select.side_effect = [
@@ -129,7 +129,7 @@ async def test_proxy_auth_works_with_mixed_case_basic_string(self) -> None:
             b'GET', b'http://upstream.host/not-found.html',
             headers={
                 b'Host': b'upstream.host',
-                b'Proxy-Authorization': b'bAsIc dXNlcjpwYXNz',
+                httpHeaders.PROXY_AUTHORIZATION: b'bAsIc dXNlcjpwYXNz',
             },
         )
         self.mock_selector.return_value.select.side_effect = [
diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py
index ca6cce944e..c5f1ff1163 100644
--- a/tests/http/test_protocol_handler.py
+++ b/tests/http/test_protocol_handler.py
@@ -26,7 +26,7 @@
 from proxy.http.proxy import HttpProxyPlugin
 from proxy.http.parser import httpParserStates, httpParserTypes
 from proxy.http.exception import ProxyAuthenticationFailed, ProxyConnectionFailed
-from proxy.http import HttpProtocolHandler
+from proxy.http import HttpProtocolHandler, httpHeaders
 
 from ..test_assertions import Assertions
 
@@ -321,8 +321,8 @@ async def test_authenticated_proxy_http_get(self) -> None:
             b'User-Agent: proxy.py/%s' % bytes_(__version__),
             b'Host: localhost:%d' % self.http_server_port,
             b'Accept: */*',
-            b'Proxy-Connection: Keep-Alive',
-            b'Proxy-Authorization: Basic dXNlcjpwYXNz',
+            httpHeaders.PROXY_CONNECTION + b': Keep-Alive',
+            httpHeaders.PROXY_AUTHORIZATION + b': Basic dXNlcjpwYXNz',
             CRLF,
         ])
         await self.assert_data_queued(server)
@@ -354,8 +354,8 @@ async def test_authenticated_proxy_http_tunnel(self) -> None:
             b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port,
             b'Host: localhost:%d' % self.http_server_port,
             b'User-Agent: proxy.py/%s' % bytes_(__version__),
-            b'Proxy-Connection: Keep-Alive',
-            b'Proxy-Authorization: Basic dXNlcjpwYXNz',
+            httpHeaders.PROXY_CONNECTION + b': Keep-Alive',
+            httpHeaders.PROXY_AUTHORIZATION + b': Basic dXNlcjpwYXNz',
             CRLF,
         ])
         await self.assert_tunnel_response(server)
diff --git a/tests/http/test_url.py b/tests/http/test_url.py
index de8ec0e71a..a640f74cf3 100644
--- a/tests/http/test_url.py
+++ b/tests/http/test_url.py
@@ -114,3 +114,23 @@ def test_trailing_slash_url(self) -> None:
         self.assertEqual(url.hostname, b'localhost')
         self.assertEqual(url.port, 12345)
         self.assertEqual(url.remainder, b'/v1/users/')
+        self.assertEqual(url.username, None)
+        self.assertEqual(url.password, None)
+
+    def test_username_password(self) -> None:
+        url = Url.from_bytes(b'http://user:pass@localhost:12345/v1/users/')
+        self.assertEqual(url.scheme, b'http')
+        self.assertEqual(url.hostname, b'localhost')
+        self.assertEqual(url.port, 12345)
+        self.assertEqual(url.remainder, b'/v1/users/')
+        self.assertEqual(url.username, b'user')
+        self.assertEqual(url.password, b'pass')
+
+    def test_username_password_without_proto_prefix(self) -> None:
+        url = Url.from_bytes('user:pass@å∫ç.com'.encode('utf-8'))
+        self.assertEqual(url.scheme, None)
+        self.assertEqual(url.hostname, 'å∫ç.com'.encode('utf-8'))
+        self.assertEqual(url.port, None)
+        self.assertEqual(url.remainder, None)
+        self.assertEqual(url.username, b'user')
+        self.assertEqual(url.password, b'pass')

From 37c779ade9d06bc17d5f67fadec4ae72850d6799 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Wed, 22 Dec 2021 02:56:52 +0530
Subject: [PATCH 03/22] Add `conn_close` kwarg to packet builder utilities
 (#898)

* Add `conn_close` kwarg to packet builder utilities, passing True will automatically add `Connection: close` header

* Add `conn_close` to `HttpRequestRejected` responses
---
 README.md                                     |  3 +--
 examples/https_connect_tunnel.py              |  2 +-
 proxy/common/utils.py                         | 20 ++++++++++++-------
 proxy/dashboard/dashboard.py                  |  2 +-
 proxy/http/exception/http_request_rejected.py |  1 +
 proxy/http/exception/proxy_auth_failed.py     |  2 +-
 proxy/http/exception/proxy_conn_failed.py     |  2 +-
 proxy/http/server/web.py                      |  6 +++---
 proxy/plugin/filter_by_client_ip.py           |  6 ++----
 proxy/plugin/filter_by_upstream.py            |  6 ++----
 proxy/plugin/filter_by_url_regex.py           |  1 -
 proxy/plugin/shortlink.py                     |  4 ++--
 .../exceptions/test_http_request_rejected.py  |  8 ++++++--
 tests/http/test_http_parser.py                |  4 +---
 tests/http/test_web_server.py                 |  9 ++++++---
 tests/plugin/test_http_proxy_plugins.py       |  6 ++----
 16 files changed, 43 insertions(+), 39 deletions(-)

diff --git a/README.md b/README.md
index b87bfff2f6..e8743686c5 100644
--- a/README.md
+++ b/README.md
@@ -1512,8 +1512,7 @@ As a decorator:
 - Generate HTTP GET request with headers
 
   ```python
-  >>> build_http_request(b'GET', b'/',
-          headers={b'Connection': b'close'})
+  >>> build_http_request(b'GET', b'/', conn_close=True)
   b'GET / HTTP/1.1\r\nConnection: close\r\n\r\n'
   ```
 
diff --git a/examples/https_connect_tunnel.py b/examples/https_connect_tunnel.py
index 135a472b6b..a20a4b3433 100644
--- a/examples/https_connect_tunnel.py
+++ b/examples/https_connect_tunnel.py
@@ -31,8 +31,8 @@ class HttpsConnectTunnelHandler(BaseTcpTunnelHandler):
     PROXY_TUNNEL_UNSUPPORTED_SCHEME = memoryview(
         build_http_response(
             httpStatusCodes.BAD_REQUEST,
-            headers={b'Connection': b'close'},
             reason=b'Unsupported protocol scheme',
+            conn_close=True,
         ),
     )
 
diff --git a/proxy/common/utils.py b/proxy/common/utils.py
index 54a2cb4162..3a51dbf0f4 100644
--- a/proxy/common/utils.py
+++ b/proxy/common/utils.py
@@ -75,12 +75,14 @@ def build_http_request(
     protocol_version: bytes = HTTP_1_1,
     headers: Optional[Dict[bytes, bytes]] = None,
     body: Optional[bytes] = None,
+    conn_close: bool = False,
 ) -> bytes:
     """Build and returns a HTTP request packet."""
-    if headers is None:
-        headers = {}
     return build_http_pkt(
-        [method, url, protocol_version], headers, body,
+        [method, url, protocol_version],
+        headers or {},
+        body,
+        conn_close,
     )
 
 
@@ -90,6 +92,7 @@ def build_http_response(
     reason: Optional[bytes] = None,
     headers: Optional[Dict[bytes, bytes]] = None,
     body: Optional[bytes] = None,
+    conn_close: bool = False,
 ) -> bytes:
     """Build and returns a HTTP response packet."""
     line = [protocol_version, bytes_(status_code)]
@@ -108,7 +111,7 @@ def build_http_response(
             not has_transfer_encoding and \
             not has_content_length:
         headers[b'Content-Length'] = bytes_(len(body))
-    return build_http_pkt(line, headers, body)
+    return build_http_pkt(line, headers, body, conn_close)
 
 
 def build_http_header(k: bytes, v: bytes) -> bytes:
@@ -120,12 +123,15 @@ def build_http_pkt(
     line: List[bytes],
     headers: Optional[Dict[bytes, bytes]] = None,
     body: Optional[bytes] = None,
+    conn_close: bool = False,
 ) -> bytes:
     """Build and returns a HTTP request or response packet."""
     pkt = WHITESPACE.join(line) + CRLF
-    if headers is not None:
-        for k, v in headers.items():
-            pkt += build_http_header(k, v) + CRLF
+    headers = headers or {}
+    if conn_close:
+        headers[b'Connection'] = b'close'
+    for k, v in headers.items():
+        pkt += build_http_header(k, v) + CRLF
     pkt += CRLF
     if body:
         pkt += body
diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py
index 9bb2a3eef3..cfdb5c8c65 100644
--- a/proxy/dashboard/dashboard.py
+++ b/proxy/dashboard/dashboard.py
@@ -83,8 +83,8 @@ def handle_request(self, request: HttpParser) -> None:
                         headers={
                             b'Location': b'/dashboard/',
                             b'Content-Length': b'0',
-                            b'Connection': b'close',
                         },
+                        conn_close=True,
                     ),
                 ),
             )
diff --git a/proxy/http/exception/http_request_rejected.py b/proxy/http/exception/http_request_rejected.py
index a0fa810fc1..1b3b1fd742 100644
--- a/proxy/http/exception/http_request_rejected.py
+++ b/proxy/http/exception/http_request_rejected.py
@@ -41,6 +41,7 @@ def response(self, _request: HttpParser) -> Optional[memoryview]:
                     reason=self.reason,
                     headers=self.headers,
                     body=self.body,
+                    conn_close=True,
                 ),
             )
         return None
diff --git a/proxy/http/exception/proxy_auth_failed.py b/proxy/http/exception/proxy_auth_failed.py
index d82211d092..6c9649880b 100644
--- a/proxy/http/exception/proxy_auth_failed.py
+++ b/proxy/http/exception/proxy_auth_failed.py
@@ -33,9 +33,9 @@ class ProxyAuthenticationFailed(HttpProtocolException):
             headers={
                 PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
                 b'Proxy-Authenticate': b'Basic',
-                b'Connection': b'close',
             },
             body=b'Proxy Authentication Required',
+            conn_close=True,
         ),
     )
 
diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py
index cfba1d26dd..3c90557e8e 100644
--- a/proxy/http/exception/proxy_conn_failed.py
+++ b/proxy/http/exception/proxy_conn_failed.py
@@ -30,9 +30,9 @@ class ProxyConnectionFailed(HttpProtocolException):
             reason=b'Bad Gateway',
             headers={
                 PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
-                b'Connection': b'close',
             },
             body=b'Bad Gateway',
+            conn_close=True,
         ),
     )
 
diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py
index bf7a301c63..c645267f57 100644
--- a/proxy/http/server/web.py
+++ b/proxy/http/server/web.py
@@ -81,8 +81,8 @@ class HttpWebServerPlugin(HttpProtocolHandlerPlugin):
             headers={
                 b'Server': PROXY_AGENT_HEADER_VALUE,
                 b'Content-Length': b'0',
-                b'Connection': b'close',
             },
+            conn_close=True,
         ),
     )
 
@@ -93,8 +93,8 @@ class HttpWebServerPlugin(HttpProtocolHandlerPlugin):
             headers={
                 b'Server': PROXY_AGENT_HEADER_VALUE,
                 b'Content-Length': b'0',
-                b'Connection': b'close',
             },
+            conn_close=True,
         ),
     )
 
@@ -147,7 +147,6 @@ def read_and_build_static_file_response(path: str, min_compression_limit: int) -
             headers = {
                 b'Content-Type': bytes_(content_type),
                 b'Cache-Control': b'max-age=86400',
-                b'Connection': b'close',
             }
             do_compress = len(content) > min_compression_limit
             if do_compress:
@@ -160,6 +159,7 @@ def read_and_build_static_file_response(path: str, min_compression_limit: int) -
                     reason=b'OK',
                     headers=headers,
                     body=gzip.compress(content) if do_compress else content,
+                    conn_close=True,
                 ),
             )
         except FileNotFoundError:
diff --git a/proxy/plugin/filter_by_client_ip.py b/proxy/plugin/filter_by_client_ip.py
index 4b3724e96d..fba981012d 100644
--- a/proxy/plugin/filter_by_client_ip.py
+++ b/proxy/plugin/filter_by_client_ip.py
@@ -39,9 +39,7 @@ def before_upstream_connection(
         assert not self.flags.unix_socket_path and self.client.addr
         if self.client.addr[0] in self.flags.filtered_client_ips.split(','):
             raise HttpRequestRejected(
-                status_code=httpStatusCodes.I_AM_A_TEAPOT, reason=b'I\'m a tea pot',
-                headers={
-                    b'Connection': b'close',
-                },
+                status_code=httpStatusCodes.I_AM_A_TEAPOT,
+                reason=b'I\'m a tea pot',
             )
         return request
diff --git a/proxy/plugin/filter_by_upstream.py b/proxy/plugin/filter_by_upstream.py
index 0970fa6eac..a257fdf286 100644
--- a/proxy/plugin/filter_by_upstream.py
+++ b/proxy/plugin/filter_by_upstream.py
@@ -35,9 +35,7 @@ def before_upstream_connection(
     ) -> Optional[HttpParser]:
         if text_(request.host) in self.flags.filtered_upstream_hosts.split(','):
             raise HttpRequestRejected(
-                status_code=httpStatusCodes.I_AM_A_TEAPOT, reason=b'I\'m a tea pot',
-                headers={
-                    b'Connection': b'close',
-                },
+                status_code=httpStatusCodes.I_AM_A_TEAPOT,
+                reason=b'I\'m a tea pot',
             )
         return request
diff --git a/proxy/plugin/filter_by_url_regex.py b/proxy/plugin/filter_by_url_regex.py
index 557a8cff3c..c9659e3732 100644
--- a/proxy/plugin/filter_by_url_regex.py
+++ b/proxy/plugin/filter_by_url_regex.py
@@ -88,7 +88,6 @@ def handle_client_request(
                 # list
                 raise HttpRequestRejected(
                     status_code=httpStatusCodes.NOT_FOUND,
-                    headers={b'Connection': b'close'},
                     reason=b'Blocked',
                 )
                 # stop looping through filter list
diff --git a/proxy/plugin/shortlink.py b/proxy/plugin/shortlink.py
index 5b1ff48ffa..43e2534ea6 100644
--- a/proxy/plugin/shortlink.py
+++ b/proxy/plugin/shortlink.py
@@ -72,8 +72,8 @@ def handle_client_request(
                             headers={
                                 b'Location': b'http://' + self.SHORT_LINKS[request.host] + path,
                                 b'Content-Length': b'0',
-                                b'Connection': b'close',
                             },
+                            conn_close=True,
                         ),
                     ),
                 )
@@ -84,8 +84,8 @@ def handle_client_request(
                             httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND',
                             headers={
                                 b'Content-Length': b'0',
-                                b'Connection': b'close',
                             },
+                            conn_close=True,
                         ),
                     ),
                 )
diff --git a/tests/http/exceptions/test_http_request_rejected.py b/tests/http/exceptions/test_http_request_rejected.py
index de8f736bc4..69a9611339 100644
--- a/tests/http/exceptions/test_http_request_rejected.py
+++ b/tests/http/exceptions/test_http_request_rejected.py
@@ -31,19 +31,23 @@ def test_status_code_response(self) -> None:
         self.assertEqual(
             e.response(self.request), CRLF.join([
                 b'HTTP/1.1 200 OK',
+                b'Connection: close',
                 CRLF,
             ]),
         )
 
     def test_body_response(self) -> None:
         e = HttpRequestRejected(
-            status_code=httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND',
+            status_code=httpStatusCodes.NOT_FOUND,
+            reason=b'NOT FOUND',
             body=b'Nothing here',
         )
         self.assertEqual(
             e.response(self.request),
             build_http_response(
                 httpStatusCodes.NOT_FOUND,
-                reason=b'NOT FOUND', body=b'Nothing here',
+                reason=b'NOT FOUND',
+                body=b'Nothing here',
+                conn_close=True,
             ),
         )
diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py
index 7e3c974377..774bf9f3d2 100644
--- a/tests/http/test_http_parser.py
+++ b/tests/http/test_http_parser.py
@@ -694,9 +694,7 @@ def test_is_not_http_1_1_keep_alive_with_close_header(self) -> None:
         self.parser.parse(
             build_http_request(
                 httpMethods.GET, b'/',
-                headers={
-                    b'Connection': b'close',
-                },
+                conn_close=True,
             ),
         )
         self.assertFalse(self.parser.is_http_1_1_keep_alive)
diff --git a/tests/http/test_web_server.py b/tests/http/test_web_server.py
index 54a1d49060..3a59fa63af 100644
--- a/tests/http/test_web_server.py
+++ b/tests/http/test_web_server.py
@@ -181,10 +181,13 @@ async def test_pac_file_served_from_disk(self) -> None:
         )
         self._conn.send.called_once_with(
             build_http_response(
-                200, reason=b'OK', headers={
+                200,
+                reason=b'OK',
+                headers={
                     b'Content-Type': b'application/x-ns-proxy-autoconfig',
-                    b'Connection': b'close',
-                }, body=self.expected_response,
+                },
+                body=self.expected_response,
+                conn_close=True,
             ),
         )
 
diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py
index 7102c3ef7a..03a32527ed 100644
--- a/tests/plugin/test_http_proxy_plugins.py
+++ b/tests/plugin/test_http_proxy_plugins.py
@@ -242,9 +242,7 @@ async def test_filter_by_upstream_host_plugin(self) -> None:
             build_http_response(
                 status_code=httpStatusCodes.I_AM_A_TEAPOT,
                 reason=b'I\'m a tea pot',
-                headers={
-                    b'Connection': b'close',
-                },
+                conn_close=True,
             ),
         )
 
@@ -376,6 +374,6 @@ async def test_filter_by_url_regex_plugin(self) -> None:
             build_http_response(
                 status_code=httpStatusCodes.NOT_FOUND,
                 reason=b'Blocked',
-                headers={b'Connection': b'close'},
+                conn_close=True,
             ),
         )

From 32fb72985ceb40145401769ed1f82f0c931bf8e1 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Wed, 22 Dec 2021 05:24:31 +0530
Subject: [PATCH 04/22] Raise `HttpProtocolException` instead of `ValueError` 
 (#899)

* Raise `HttpProtocolException` instead of `ValueError` for clean teardown of the offending connection

* Fix circular import errors

* Fix tests

* fix lint errors

* Avoid containerizing until check has passed
---
 .github/workflows/test-library.yml            | 42 +++++++++----------
 proxy/core/acceptor/threadless.py             | 14 ++++---
 proxy/http/exception/base.py                  |  7 ++--
 proxy/http/exception/http_request_rejected.py |  9 ++--
 proxy/http/exception/proxy_auth_failed.py     |  8 +++-
 proxy/http/exception/proxy_conn_failed.py     |  8 +++-
 proxy/http/handler.py                         |  2 +-
 proxy/http/parser/parser.py                   |  6 +--
 proxy/http/parser/protocol.py                 |  6 ++-
 tests/http/test_http_parser.py                |  5 ++-
 tests/http/test_proxy_protocol.py             |  3 +-
 11 files changed, 64 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml
index 1b7928e0f3..c344b67610 100644
--- a/.github/workflows/test-library.yml
+++ b/.github/workflows/test-library.yml
@@ -668,12 +668,32 @@ jobs:
         make ca-certificates
         python3 -m proxy --version
 
+  check:  # This job does nothing and is only used for the branch protection
+    if: always()
+
+    needs:
+    - analyze
+    - test
+    - lint
+    - dashboard
+    - brew
+    - developer
+
+    runs-on: Ubuntu-latest
+
+    steps:
+    - name: Decide whether the needed jobs succeeded or failed
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{ toJSON(needs) }}
+
   docker:
     runs-on: Ubuntu-latest
     permissions:
       packages: write
+    if: success()
     needs:
-    - build
+    - check
     - pre-setup  # transitive, for accessing settings
     name: 🐳 containerize
     strategy:
@@ -786,26 +806,6 @@ jobs:
         }}'
         -t $CONTAINER_TAG .
 
-  check:  # This job does nothing and is only used for the branch protection
-    if: always()
-
-    needs:
-    - analyze
-    - test
-    - lint
-    - docker
-    - dashboard
-    - brew
-    - developer
-
-    runs-on: Ubuntu-latest
-
-    steps:
-    - name: Decide whether the needed jobs succeeded or failed
-      uses: re-actors/alls-green@release/v1
-      with:
-        jobs: ${{ toJSON(needs) }}
-
   publish-pypi:
     name: Publish 🐍📦 ${{ needs.pre-setup.outputs.git-tag }} to PyPI
     needs:
diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py
index f36f9c0a98..cfd2f12086 100644
--- a/proxy/core/acceptor/threadless.py
+++ b/proxy/core/acceptor/threadless.py
@@ -180,13 +180,15 @@ async def _update_work_events(self, work_id: int) -> None:
                 # else:
                 #     logger.info(
                 #         'fd#{0} by work#{1} not modified'.format(fileno, work_id))
+            # Can throw ValueError: Invalid file descriptor: -1
+            #
+            # A guard within Work classes may not help here due to
+            # asynchronous nature.  Hence, threadless will handle
+            # ValueError exceptions raised by selector.register
+            # for invalid fd.
+            #
+            # TODO: Also remove offending work from pool to avoid spin loop.
             elif fileno != -1:
-                # Can throw ValueError: Invalid file descriptor: -1
-                #
-                # A guard within Work classes may not help here due to
-                # asynchronous nature.  Hence, threadless will handle
-                # ValueError exceptions raised by selector.register
-                # for invalid fd.
                 self.selector.register(
                     fileno, events=mask,
                     data=work_id,
diff --git a/proxy/http/exception/base.py b/proxy/http/exception/base.py
index 37817c9265..d39a640392 100644
--- a/proxy/http/exception/base.py
+++ b/proxy/http/exception/base.py
@@ -12,9 +12,10 @@
 
        http
 """
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 
-from ..parser import HttpParser
+if TYPE_CHECKING:
+    from ..parser import HttpParser
 
 
 class HttpProtocolException(Exception):
@@ -25,5 +26,5 @@ class HttpProtocolException(Exception):
     ``response()`` method to optionally return custom response to client.
     """
 
-    def response(self, request: HttpParser) -> Optional[memoryview]:
+    def response(self, request: 'HttpParser') -> Optional[memoryview]:
         return None  # pragma: no cover
diff --git a/proxy/http/exception/http_request_rejected.py b/proxy/http/exception/http_request_rejected.py
index 1b3b1fd742..6d095481fe 100644
--- a/proxy/http/exception/http_request_rejected.py
+++ b/proxy/http/exception/http_request_rejected.py
@@ -8,12 +8,15 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-from typing import Optional, Dict
+from typing import Optional, Dict, TYPE_CHECKING
 
 from .base import HttpProtocolException
-from ..parser import HttpParser
+
 from ...common.utils import build_http_response
 
+if TYPE_CHECKING:
+    from ..parser import HttpParser
+
 
 class HttpRequestRejected(HttpProtocolException):
     """Generic exception that can be used to reject the client requests.
@@ -33,7 +36,7 @@ def __init__(
         self.headers: Optional[Dict[bytes, bytes]] = headers
         self.body: Optional[bytes] = body
 
-    def response(self, _request: HttpParser) -> Optional[memoryview]:
+    def response(self, _request: 'HttpParser') -> Optional[memoryview]:
         if self.status_code:
             return memoryview(
                 build_http_response(
diff --git a/proxy/http/exception/proxy_auth_failed.py b/proxy/http/exception/proxy_auth_failed.py
index 6c9649880b..ecb9a2994d 100644
--- a/proxy/http/exception/proxy_auth_failed.py
+++ b/proxy/http/exception/proxy_auth_failed.py
@@ -13,14 +13,18 @@
        auth
        http
 """
+from typing import TYPE_CHECKING
+
 from .base import HttpProtocolException
 
 from ..codes import httpStatusCodes
-from ..parser import HttpParser
 
 from ...common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY
 from ...common.utils import build_http_response
 
+if TYPE_CHECKING:
+    from ..parser import HttpParser
+
 
 class ProxyAuthenticationFailed(HttpProtocolException):
     """Exception raised when HTTP Proxy auth is enabled and
@@ -39,5 +43,5 @@ class ProxyAuthenticationFailed(HttpProtocolException):
         ),
     )
 
-    def response(self, _request: HttpParser) -> memoryview:
+    def response(self, _request: 'HttpParser') -> memoryview:
         return self.RESPONSE_PKT
diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py
index 3c90557e8e..c091bf50be 100644
--- a/proxy/http/exception/proxy_conn_failed.py
+++ b/proxy/http/exception/proxy_conn_failed.py
@@ -12,14 +12,18 @@
 
        conn
 """
+from typing import TYPE_CHECKING
+
 from .base import HttpProtocolException
 
 from ..codes import httpStatusCodes
-from ..parser import HttpParser
 
 from ...common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY
 from ...common.utils import build_http_response
 
+if TYPE_CHECKING:
+    from ..parser import HttpParser
+
 
 class ProxyConnectionFailed(HttpProtocolException):
     """Exception raised when ``HttpProxyPlugin`` is unable to establish connection to upstream server."""
@@ -41,5 +45,5 @@ def __init__(self, host: str, port: int, reason: str):
         self.port: int = port
         self.reason: str = reason
 
-    def response(self, _request: HttpParser) -> memoryview:
+    def response(self, _request: 'HttpParser') -> memoryview:
         return self.RESPONSE_PKT
diff --git a/proxy/http/handler.py b/proxy/http/handler.py
index 4a03aa5915..c033e8b022 100644
--- a/proxy/http/handler.py
+++ b/proxy/http/handler.py
@@ -237,7 +237,7 @@ def handle_data(self, data: memoryview) -> Optional[bool]:
                         break
                     data = optional_data
         except HttpProtocolException as e:
-            logger.debug('HttpProtocolException raised')
+            logger.exception('HttpProtocolException raised', exc_info=e)
             response: Optional[memoryview] = e.response(self.request)
             if response:
                 self.work.queue(response)
diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py
index c446b34b15..5ed707246a 100644
--- a/proxy/http/parser/parser.py
+++ b/proxy/http/parser/parser.py
@@ -22,6 +22,7 @@
 
 from ..url import Url
 from ..methods import httpMethods
+from ..exception import HttpProtocolException
 
 from .protocol import ProxyProtocol
 from .chunk import ChunkParser, chunkParserStates
@@ -363,10 +364,7 @@ def _process_line(self, raw: bytes) -> Tuple[bool, bytes]:
                     break
                 # To avoid a possible attack vector, we raise exception
                 # if parser receives an invalid request line.
-                #
-                # TODO: Better to use raise HttpProtocolException,
-                # but we should solve circular import problem first.
-                raise ValueError('Invalid request line')
+                raise HttpProtocolException('Invalid request line %r' % raw)
             parts = line.split(WHITESPACE, 2)
             self.version = parts[0]
             self.code = parts[1]
diff --git a/proxy/http/parser/protocol.py b/proxy/http/parser/protocol.py
index d749fdc08a..c44192f0e5 100644
--- a/proxy/http/parser/protocol.py
+++ b/proxy/http/parser/protocol.py
@@ -10,6 +10,8 @@
 """
 from typing import Optional, Tuple
 
+from ..exception import HttpProtocolException
+
 from ...common.constants import WHITESPACE
 
 PROXY_PROTOCOL_V2_SIGNATURE = b'\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A'
@@ -43,4 +45,6 @@ def parse(self, raw: bytes) -> None:
             self.version = 2
             raise NotImplementedError()
         else:
-            raise ValueError('Neither a v1 or v2 proxy protocol packet')
+            raise HttpProtocolException(
+                'Neither a v1 or v2 proxy protocol packet',
+            )
diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py
index 774bf9f3d2..5f771bd708 100644
--- a/tests/http/test_http_parser.py
+++ b/tests/http/test_http_parser.py
@@ -15,6 +15,7 @@
 from proxy.common.utils import find_http_line, bytes_
 
 from proxy.http import httpStatusCodes, httpMethods
+from proxy.http.exception import HttpProtocolException
 from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates
 
 
@@ -24,10 +25,10 @@ def setUp(self) -> None:
         self.parser = HttpParser(httpParserTypes.REQUEST_PARSER)
 
     def test_issue_127(self) -> None:
-        with self.assertRaises(ValueError):
+        with self.assertRaises(HttpProtocolException):
             self.parser.parse(CRLF)
 
-        with self.assertRaises(ValueError):
+        with self.assertRaises(HttpProtocolException):
             raw = b'qwqrqw!@!#@!#ad adfad\r\n'
             while True:
                 self.parser.parse(raw)
diff --git a/tests/http/test_proxy_protocol.py b/tests/http/test_proxy_protocol.py
index b6701abfb7..f8b609b1b9 100644
--- a/tests/http/test_proxy_protocol.py
+++ b/tests/http/test_proxy_protocol.py
@@ -11,6 +11,7 @@
 import unittest
 
 from proxy.http.parser import ProxyProtocol, PROXY_PROTOCOL_V2_SIGNATURE
+from proxy.http.exception import HttpProtocolException
 
 
 class TestProxyProtocol(unittest.TestCase):
@@ -81,6 +82,6 @@ def test_v2_not_implemented(self) -> None:
             self.assertEqual(self.protocol.version, 2)
 
     def test_unknown_value_error(self) -> None:
-        with self.assertRaises(ValueError):
+        with self.assertRaises(HttpProtocolException):
             self.protocol.parse(PROXY_PROTOCOL_V2_SIGNATURE[:10])
             self.assertEqual(self.protocol.version, None)

From 2d8385720c33ffb0233b73a1f1ac5e993c8525f6 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Wed, 22 Dec 2021 09:20:06 +0530
Subject: [PATCH 05/22] Ensure message for every `HttpProtocolException` raised
 (#900)

---
 proxy/http/exception/base.py                  |  5 ++++-
 proxy/http/exception/http_request_rejected.py | 20 +++++++++++++------
 proxy/http/exception/proxy_auth_failed.py     |  5 ++++-
 proxy/http/exception/proxy_conn_failed.py     |  5 +++--
 proxy/http/handler.py                         |  2 +-
 proxy/http/proxy/server.py                    |  3 +--
 proxy/http/server/web.py                      |  6 ++----
 proxy/plugin/proxy_pool.py                    |  6 ++----
 proxy/plugin/reverse_proxy.py                 |  3 +--
 9 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/proxy/http/exception/base.py b/proxy/http/exception/base.py
index d39a640392..17a69bb689 100644
--- a/proxy/http/exception/base.py
+++ b/proxy/http/exception/base.py
@@ -12,7 +12,7 @@
 
        http
 """
-from typing import Optional, TYPE_CHECKING
+from typing import Any, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from ..parser import HttpParser
@@ -26,5 +26,8 @@ class HttpProtocolException(Exception):
     ``response()`` method to optionally return custom response to client.
     """
 
+    def __init__(self, message: Optional[str] = None, **kwargs: Any) -> None:
+        super().__init__(message or 'Reason unknown')
+
     def response(self, request: 'HttpParser') -> Optional[memoryview]:
         return None  # pragma: no cover
diff --git a/proxy/http/exception/http_request_rejected.py b/proxy/http/exception/http_request_rejected.py
index 6d095481fe..baec9e997a 100644
--- a/proxy/http/exception/http_request_rejected.py
+++ b/proxy/http/exception/http_request_rejected.py
@@ -8,7 +8,7 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-from typing import Optional, Dict, TYPE_CHECKING
+from typing import Any, Optional, Dict, TYPE_CHECKING
 
 from .base import HttpProtocolException
 
@@ -25,16 +25,24 @@ class HttpRequestRejected(HttpProtocolException):
     HTTP status code can be returned."""
 
     def __init__(
-        self,
-        status_code: Optional[int] = None,
-        reason: Optional[bytes] = None,
-        headers: Optional[Dict[bytes, bytes]] = None,
-        body: Optional[bytes] = None,
+            self,
+            status_code: Optional[int] = None,
+            reason: Optional[bytes] = None,
+            headers: Optional[Dict[bytes, bytes]] = None,
+            body: Optional[bytes] = None,
+            **kwargs: Any,
     ):
         self.status_code: Optional[int] = status_code
         self.reason: Optional[bytes] = reason
         self.headers: Optional[Dict[bytes, bytes]] = headers
         self.body: Optional[bytes] = body
+        klass_name = self.__class__.__name__
+        super().__init__(
+            message='%s %r' % (klass_name, reason)
+            if reason
+            else klass_name,
+            **kwargs,
+        )
 
     def response(self, _request: 'HttpParser') -> Optional[memoryview]:
         if self.status_code:
diff --git a/proxy/http/exception/proxy_auth_failed.py b/proxy/http/exception/proxy_auth_failed.py
index ecb9a2994d..60782e0c23 100644
--- a/proxy/http/exception/proxy_auth_failed.py
+++ b/proxy/http/exception/proxy_auth_failed.py
@@ -13,7 +13,7 @@
        auth
        http
 """
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from .base import HttpProtocolException
 
@@ -43,5 +43,8 @@ class ProxyAuthenticationFailed(HttpProtocolException):
         ),
     )
 
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(self.__class__.__name__, **kwargs)
+
     def response(self, _request: 'HttpParser') -> memoryview:
         return self.RESPONSE_PKT
diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py
index c091bf50be..d0eae48192 100644
--- a/proxy/http/exception/proxy_conn_failed.py
+++ b/proxy/http/exception/proxy_conn_failed.py
@@ -12,7 +12,7 @@
 
        conn
 """
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from .base import HttpProtocolException
 
@@ -40,10 +40,11 @@ class ProxyConnectionFailed(HttpProtocolException):
         ),
     )
 
-    def __init__(self, host: str, port: int, reason: str):
+    def __init__(self, host: str, port: int, reason: str, **kwargs: Any):
         self.host: str = host
         self.port: int = port
         self.reason: str = reason
+        super().__init__('%s %s' % (self.__class__.__name__, reason), **kwargs)
 
     def response(self, _request: 'HttpParser') -> memoryview:
         return self.RESPONSE_PKT
diff --git a/proxy/http/handler.py b/proxy/http/handler.py
index c033e8b022..38429df7de 100644
--- a/proxy/http/handler.py
+++ b/proxy/http/handler.py
@@ -237,7 +237,7 @@ def handle_data(self, data: memoryview) -> Optional[bool]:
                         break
                     data = optional_data
         except HttpProtocolException as e:
-            logger.exception('HttpProtocolException raised', exc_info=e)
+            logger.info('HttpProtocolException: %s' % e)
             response: Optional[memoryview] = e.response(self.request)
             if response:
                 self.work.queue(response)
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index 2fda84f8d5..462cf0fc83 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -657,8 +657,7 @@ def connect_upstream(self) -> None:
                     text_(host), port, repr(e),
                 ) from e
         else:
-            logger.exception('Both host and port must exist')
-            raise HttpProtocolException()
+            raise HttpProtocolException('Both host and port must exist')
 
     #
     # Interceptor related methods
diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py
index c645267f57..520f170d53 100644
--- a/proxy/http/server/web.py
+++ b/proxy/http/server/web.py
@@ -263,10 +263,9 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]:
                 # TODO: Tear down if invalid protocol exception
                 remaining = frame.parse(remaining)
                 if frame.opcode == websocketOpcodes.CONNECTION_CLOSE:
-                    logger.warning(
+                    raise HttpProtocolException(
                         'Client sent connection close packet',
                     )
-                    raise HttpProtocolException()
                 else:
                     assert self.route
                     self.route.on_websocket_message(frame)
@@ -287,10 +286,9 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]:
             if self.pipeline_request.is_complete:
                 self.route.handle_request(self.pipeline_request)
                 if not self.pipeline_request.is_http_1_1_keep_alive:
-                    logger.error(
+                    raise HttpProtocolException(
                         'Pipelined request is not keep-alive, will tear down request...',
                     )
-                    raise HttpProtocolException()
                 self.pipeline_request = None
         return raw
 
diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py
index bfe00aaafa..cfc8017820 100644
--- a/proxy/plugin/proxy_pool.py
+++ b/proxy/plugin/proxy_pool.py
@@ -110,12 +110,11 @@ def before_upstream_connection(
         try:
             self.upstream.connect()
         except TimeoutError:
-            logger.info(
+            raise HttpProtocolException(
                 'Timed out connecting to upstream proxy {0}:{1}'.format(
                     *endpoint_tuple,
                 ),
             )
-            raise HttpProtocolException()
         except ConnectionRefusedError:
             # TODO(abhinavsingh): Try another choice, when all (or max configured) choices have
             # exhausted, retry for configured number of times before giving up.
@@ -124,12 +123,11 @@ def before_upstream_connection(
             # A periodic health check must put them back in the pool.  This can be achieved
             # using a datastructure without having to spawn separate thread/process for health
             # check.
-            logger.info(
+            raise HttpProtocolException(
                 'Connection refused by upstream proxy {0}:{1}'.format(
                     *endpoint_tuple,
                 ),
             )
-            raise HttpProtocolException()
         logger.debug(
             'Established connection to upstream proxy {0}:{1}'.format(
                 *endpoint_tuple,
diff --git a/proxy/plugin/reverse_proxy.py b/proxy/plugin/reverse_proxy.py
index 0715d184f8..26ebbb6aae 100644
--- a/proxy/plugin/reverse_proxy.py
+++ b/proxy/plugin/reverse_proxy.py
@@ -101,12 +101,11 @@ def handle_request(self, request: HttpParser) -> None:
                 )
             self.upstream.queue(memoryview(request.build()))
         except ConnectionRefusedError:
-            logger.info(
+            raise HttpProtocolException(
                 'Connection refused by upstream server {0}:{1}'.format(
                     text_(self.choice.hostname), port,
                 ),
             )
-            raise HttpProtocolException()
 
     def on_client_connection_close(self) -> None:
         if self.upstream and not self.upstream.closed:

From c936f7e52d030d76240c373d8c7f442ff633ea75 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Sat, 25 Dec 2021 00:30:40 +0530
Subject: [PATCH 06/22] Response Packet Utilities (#903)

* Add response pkt utility

* Unused import

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix tests as some content is now by default gzipped based upon min compression config

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Remove unused

* Update necessary tests to use `okResponse` utility

* Add option to explicitly disable compression

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 .github/workflows/test-library.yml            |  14 +-
 examples/https_connect_tunnel.py              |  31 +---
 proxy/dashboard/dashboard.py                  |  18 +--
 proxy/http/exception/proxy_auth_failed.py     |  20 +--
 proxy/http/exception/proxy_conn_failed.py     |  19 +--
 proxy/http/proxy/server.py                    |  15 +-
 proxy/http/responses.py                       | 138 ++++++++++++++++++
 proxy/http/server/pac_plugin.py               |  19 ++-
 proxy/http/server/web.py                      |  58 ++------
 proxy/plugin/man_in_the_middle.py             |  11 +-
 proxy/plugin/mock_rest_api.py                 |  27 +---
 proxy/plugin/shortlink.py                     |  26 +---
 proxy/plugin/web_server_route.py              |  19 +--
 .../exceptions/test_http_proxy_auth_failed.py |   8 +-
 tests/http/test_http_parser.py                |  54 +++----
 .../http/test_http_proxy_tls_interception.py  |   9 +-
 tests/http/test_protocol_handler.py           |  14 +-
 tests/http/test_web_server.py                 |   6 +-
 tests/plugin/test_http_proxy_plugins.py       |  38 +++--
 ...ttp_proxy_plugins_with_tls_interception.py |  29 ++--
 tests/testing/test_embed.py                   |   4 +-
 21 files changed, 294 insertions(+), 283 deletions(-)
 create mode 100644 proxy/http/responses.py

diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml
index c344b67610..e6fc703fa9 100644
--- a/.github/workflows/test-library.yml
+++ b/.github/workflows/test-library.yml
@@ -247,8 +247,10 @@ jobs:
         path: ${{ steps.pip-cache.outputs.dir }}
         key: >-
           ${{ runner.os }}-pip-${{
-          steps.calc-cache-key-py.outputs.py-hash-key }}-${{
-          hashFiles('tox.ini') }}
+            steps.calc-cache-key-py.outputs.py-hash-key
+          }}-${{
+            hashFiles('tox.ini')
+          }}
         restore-keys: |
           ${{ runner.os }}-pip-${{
               steps.calc-cache-key-py.outputs.py-hash-key
@@ -370,8 +372,10 @@ jobs:
         path: ${{ steps.pip-cache.outputs.dir }}
         key: >-
           ${{ runner.os }}-pip-${{
-          steps.calc-cache-key-py.outputs.py-hash-key }}-${{
-          hashFiles('tox.ini') }}
+            steps.calc-cache-key-py.outputs.py-hash-key
+          }}-${{
+            hashFiles('tox.ini')
+          }}
         restore-keys: |
           ${{ runner.os }}-pip-${{
               steps.calc-cache-key-py.outputs.py-hash-key
@@ -701,6 +705,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.inputs.release-commitish }}
     - name: Download all the dists
       uses: actions/download-artifact@v2
       with:
diff --git a/examples/https_connect_tunnel.py b/examples/https_connect_tunnel.py
index a20a4b3433..21ef67e3e3 100644
--- a/examples/https_connect_tunnel.py
+++ b/examples/https_connect_tunnel.py
@@ -13,29 +13,18 @@
 from typing import Any, Optional
 
 from proxy import Proxy
-from proxy.common.utils import build_http_response
-from proxy.http import httpStatusCodes
+
+from proxy.http.responses import (
+    PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+    PROXY_TUNNEL_UNSUPPORTED_SCHEME,
+)
+
 from proxy.core.base import BaseTcpTunnelHandler
 
 
 class HttpsConnectTunnelHandler(BaseTcpTunnelHandler):
     """A https CONNECT tunnel."""
 
-    PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview(
-        build_http_response(
-            httpStatusCodes.OK,
-            reason=b'Connection established',
-        ),
-    )
-
-    PROXY_TUNNEL_UNSUPPORTED_SCHEME = memoryview(
-        build_http_response(
-            httpStatusCodes.BAD_REQUEST,
-            reason=b'Unsupported protocol scheme',
-            conn_close=True,
-        ),
-    )
-
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
@@ -50,9 +39,7 @@ def handle_data(self, data: memoryview) -> Optional[bool]:
 
         # Drop the request if not a CONNECT request
         if not self.request.is_https_tunnel:
-            self.work.queue(
-                HttpsConnectTunnelHandler.PROXY_TUNNEL_UNSUPPORTED_SCHEME,
-            )
+            self.work.queue(PROXY_TUNNEL_UNSUPPORTED_SCHEME)
             return True
 
         # CONNECT requests are short and we need not worry about
@@ -63,9 +50,7 @@ def handle_data(self, data: memoryview) -> Optional[bool]:
         self.connect_upstream()
 
         # Queue tunnel established response to client
-        self.work.queue(
-            HttpsConnectTunnelHandler.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
-        )
+        self.work.queue(PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT)
 
         return None
 
diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py
index cfdb5c8c65..196ff16ac8 100644
--- a/proxy/dashboard/dashboard.py
+++ b/proxy/dashboard/dashboard.py
@@ -15,9 +15,9 @@
 
 from .plugin import ProxyDashboardWebsocketPlugin
 
-from ..common.utils import build_http_response, bytes_
+from ..common.utils import bytes_
 
-from ..http import httpStatusCodes
+from ..http.responses import permanentRedirectResponse
 from ..http.parser import HttpParser
 from ..http.websocket import WebsocketFrame
 from ..http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes
@@ -69,25 +69,13 @@ def handle_request(self, request: HttpParser) -> None:
                         self.flags.static_server_dir,
                         'dashboard', 'proxy.html',
                     ),
-                    self.flags.min_compression_limit,
                 ),
             )
         elif request.path in (
                 b'/dashboard',
                 b'/dashboard/proxy.html',
         ):
-            self.client.queue(
-                memoryview(
-                    build_http_response(
-                        httpStatusCodes.PERMANENT_REDIRECT, reason=b'Permanent Redirect',
-                        headers={
-                            b'Location': b'/dashboard/',
-                            b'Content-Length': b'0',
-                        },
-                        conn_close=True,
-                    ),
-                ),
-            )
+            self.client.queue(permanentRedirectResponse(b'/dashboard/'))
 
     def on_websocket_open(self) -> None:
         logger.info('app ws opened')
diff --git a/proxy/http/exception/proxy_auth_failed.py b/proxy/http/exception/proxy_auth_failed.py
index 60782e0c23..76bdcf227c 100644
--- a/proxy/http/exception/proxy_auth_failed.py
+++ b/proxy/http/exception/proxy_auth_failed.py
@@ -17,10 +17,7 @@
 
 from .base import HttpProtocolException
 
-from ..codes import httpStatusCodes
-
-from ...common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY
-from ...common.utils import build_http_response
+from ..responses import PROXY_AUTH_FAILED_RESPONSE_PKT
 
 if TYPE_CHECKING:
     from ..parser import HttpParser
@@ -30,21 +27,8 @@ class ProxyAuthenticationFailed(HttpProtocolException):
     """Exception raised when HTTP Proxy auth is enabled and
     incoming request doesn't present necessary credentials."""
 
-    RESPONSE_PKT = memoryview(
-        build_http_response(
-            httpStatusCodes.PROXY_AUTH_REQUIRED,
-            reason=b'Proxy Authentication Required',
-            headers={
-                PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
-                b'Proxy-Authenticate': b'Basic',
-            },
-            body=b'Proxy Authentication Required',
-            conn_close=True,
-        ),
-    )
-
     def __init__(self, **kwargs: Any) -> None:
         super().__init__(self.__class__.__name__, **kwargs)
 
     def response(self, _request: 'HttpParser') -> memoryview:
-        return self.RESPONSE_PKT
+        return PROXY_AUTH_FAILED_RESPONSE_PKT
diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py
index d0eae48192..d6af42131e 100644
--- a/proxy/http/exception/proxy_conn_failed.py
+++ b/proxy/http/exception/proxy_conn_failed.py
@@ -16,10 +16,7 @@
 
 from .base import HttpProtocolException
 
-from ..codes import httpStatusCodes
-
-from ...common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY
-from ...common.utils import build_http_response
+from ..responses import BAD_GATEWAY_RESPONSE_PKT
 
 if TYPE_CHECKING:
     from ..parser import HttpParser
@@ -28,18 +25,6 @@
 class ProxyConnectionFailed(HttpProtocolException):
     """Exception raised when ``HttpProxyPlugin`` is unable to establish connection to upstream server."""
 
-    RESPONSE_PKT = memoryview(
-        build_http_response(
-            httpStatusCodes.BAD_GATEWAY,
-            reason=b'Bad Gateway',
-            headers={
-                PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
-            },
-            body=b'Bad Gateway',
-            conn_close=True,
-        ),
-    )
-
     def __init__(self, host: str, port: int, reason: str, **kwargs: Any):
         self.host: str = host
         self.port: int = port
@@ -47,4 +32,4 @@ def __init__(self, host: str, port: int, reason: str, **kwargs: Any):
         super().__init__('%s %s' % (self.__class__.__name__, reason), **kwargs)
 
     def response(self, _request: 'HttpParser') -> memoryview:
-        return self.RESPONSE_PKT
+        return BAD_GATEWAY_RESPONSE_PKT
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index 462cf0fc83..7c2c41083c 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -28,10 +28,10 @@
 
 from ..headers import httpHeaders
 from ..methods import httpMethods
-from ..codes import httpStatusCodes
 from ..plugin import HttpProtocolHandlerPlugin
 from ..exception import HttpProtocolException, ProxyConnectionFailed
 from ..parser import HttpParser, httpParserStates, httpParserTypes
+from ..responses import PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT
 
 from ...common.types import Readables, Writables
 from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE
@@ -40,7 +40,7 @@
 from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS
 from ...common.constants import DEFAULT_HTTP_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_ACCESS_LOG_FORMAT
 from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH
-from ...common.utils import build_http_response, text_
+from ...common.utils import text_
 from ...common.pki import gen_public_key, gen_csr, sign_csr
 
 from ...core.event import eventNames
@@ -136,13 +136,6 @@
 class HttpProxyPlugin(HttpProtocolHandlerPlugin):
     """HttpProtocolHandler plugin which implements HttpProxy specifications."""
 
-    PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview(
-        build_http_response(
-            httpStatusCodes.OK,
-            reason=b'Connection established',
-        ),
-    )
-
     # Used to synchronization during certificate generation and
     # connection pool operations.
     lock = threading.Lock()
@@ -546,9 +539,7 @@ def on_request_complete(self) -> Union[socket.socket, bool]:
         # Optionally, setup interceptor if TLS interception is enabled.
         if self.upstream:
             if self.request.is_https_tunnel:
-                self.client.queue(
-                    HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
-                )
+                self.client.queue(PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT)
                 if self.tls_interception_enabled():
                     return self.intercept()
             # If an upstream server connection was established for http request,
diff --git a/proxy/http/responses.py b/proxy/http/responses.py
new file mode 100644
index 0000000000..9e865cd274
--- /dev/null
+++ b/proxy/http/responses.py
@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import gzip
+from typing import Any, Dict, Optional
+
+from ..common.flag import flags
+from ..common.utils import build_http_response
+from ..common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY
+
+from .codes import httpStatusCodes
+
+
+PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview(
+    build_http_response(
+        httpStatusCodes.OK,
+        reason=b'Connection established',
+    ),
+)
+
+PROXY_TUNNEL_UNSUPPORTED_SCHEME = memoryview(
+    build_http_response(
+        httpStatusCodes.BAD_REQUEST,
+        reason=b'Unsupported protocol scheme',
+        conn_close=True,
+    ),
+)
+
+PROXY_AUTH_FAILED_RESPONSE_PKT = memoryview(
+    build_http_response(
+        httpStatusCodes.PROXY_AUTH_REQUIRED,
+        reason=b'Proxy Authentication Required',
+        headers={
+            PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
+            b'Proxy-Authenticate': b'Basic',
+        },
+        body=b'Proxy Authentication Required',
+        conn_close=True,
+    ),
+)
+
+NOT_FOUND_RESPONSE_PKT = memoryview(
+    build_http_response(
+        httpStatusCodes.NOT_FOUND,
+        reason=b'NOT FOUND',
+        headers={
+            b'Server': PROXY_AGENT_HEADER_VALUE,
+            b'Content-Length': b'0',
+        },
+        conn_close=True,
+    ),
+)
+
+NOT_IMPLEMENTED_RESPONSE_PKT = memoryview(
+    build_http_response(
+        httpStatusCodes.NOT_IMPLEMENTED,
+        reason=b'NOT IMPLEMENTED',
+        headers={
+            b'Server': PROXY_AGENT_HEADER_VALUE,
+            b'Content-Length': b'0',
+        },
+        conn_close=True,
+    ),
+)
+
+BAD_GATEWAY_RESPONSE_PKT = memoryview(
+    build_http_response(
+        httpStatusCodes.BAD_GATEWAY,
+        reason=b'Bad Gateway',
+        headers={
+            PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
+        },
+        body=b'Bad Gateway',
+        conn_close=True,
+    ),
+)
+
+
+def okResponse(
+        content: Optional[bytes] = None,
+        headers: Optional[Dict[bytes, bytes]] = None,
+        compress: bool = True,
+        **kwargs: Any,
+) -> memoryview:
+    do_compress: bool = False
+    if compress and flags.args and content and len(content) > flags.args.min_compression_limit:
+        do_compress = True
+        if not headers:
+            headers = {}
+        headers.update({
+            b'Content-Encoding': b'gzip',
+        })
+    return memoryview(
+        build_http_response(
+            200,
+            reason=b'OK',
+            headers=headers,
+            body=gzip.compress(content)
+            if do_compress and content
+            else content,
+            **kwargs,
+        ),
+    )
+
+
+def permanentRedirectResponse(location: bytes) -> memoryview:
+    return memoryview(
+        build_http_response(
+            httpStatusCodes.PERMANENT_REDIRECT,
+            reason=b'Permanent Redirect',
+            headers={
+                b'Location': location,
+                b'Content-Length': b'0',
+            },
+            conn_close=True,
+        ),
+    )
+
+
+def seeOthersResponse(location: bytes) -> memoryview:
+    return memoryview(
+        build_http_response(
+            httpStatusCodes.SEE_OTHER,
+            reason=b'See Other',
+            headers={
+                b'Location': location,
+                b'Content-Length': b'0',
+            },
+            conn_close=True,
+        ),
+    )
diff --git a/proxy/http/server/pac_plugin.py b/proxy/http/server/pac_plugin.py
index f6858cf452..2e6e233872 100644
--- a/proxy/http/server/pac_plugin.py
+++ b/proxy/http/server/pac_plugin.py
@@ -12,16 +12,15 @@
 
        pac
 """
-import gzip
-
 from typing import List, Tuple, Optional, Any
 
 from .plugin import HttpWebServerBasePlugin
 from .protocols import httpProtocolTypes
 
 from ..parser import HttpParser
+from ..responses import okResponse
 
-from ...common.utils import bytes_, text_, build_http_response
+from ...common.utils import bytes_, text_
 from ...common.flag import flags
 from ...common.constants import DEFAULT_PAC_FILE, DEFAULT_PAC_FILE_URL_PATH
 
@@ -69,11 +68,11 @@ def cache_pac_file_response(self) -> None:
                     content = f.read()
             except IOError:
                 content = bytes_(self.flags.pac_file)
-            self.pac_file_response = memoryview(
-                build_http_response(
-                    200, reason=b'OK', headers={
-                        b'Content-Type': b'application/x-ns-proxy-autoconfig',
-                        b'Content-Encoding': b'gzip',
-                    }, body=gzip.compress(content),
-                ),
+            self.pac_file_response = okResponse(
+                content=content,
+                headers={
+                    b'Content-Type': b'application/x-ns-proxy-autoconfig',
+                    b'Content-Encoding': b'gzip',
+                },
+                conn_close=True,
             )
diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py
index 520f170d53..89ed43c618 100644
--- a/proxy/http/server/web.py
+++ b/proxy/http/server/web.py
@@ -9,7 +9,6 @@
     :license: BSD, see LICENSE for more details.
 """
 import re
-import gzip
 import time
 import socket
 import logging
@@ -17,18 +16,18 @@
 
 from typing import List, Tuple, Optional, Dict, Union, Any, Pattern
 
-from ...common.constants import DEFAULT_STATIC_SERVER_DIR, PROXY_AGENT_HEADER_VALUE
+from ...common.constants import DEFAULT_STATIC_SERVER_DIR
 from ...common.constants import DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_ENABLE_WEB_SERVER
 from ...common.constants import DEFAULT_MIN_COMPRESSION_LIMIT, DEFAULT_WEB_ACCESS_LOG_FORMAT
-from ...common.utils import bytes_, text_, build_http_response, build_websocket_handshake_response
+from ...common.utils import bytes_, text_, build_websocket_handshake_response
 from ...common.types import Readables, Writables
 from ...common.flag import flags
 
-from ..codes import httpStatusCodes
 from ..exception import HttpProtocolException
 from ..plugin import HttpProtocolHandlerPlugin
 from ..websocket import WebsocketFrame, websocketOpcodes
 from ..parser import HttpParser, httpParserTypes
+from ..responses import NOT_FOUND_RESPONSE_PKT, NOT_IMPLEMENTED_RESPONSE_PKT, okResponse
 
 from .plugin import HttpWebServerBasePlugin
 from .protocols import httpProtocolTypes
@@ -74,30 +73,6 @@
 class HttpWebServerPlugin(HttpProtocolHandlerPlugin):
     """HttpProtocolHandler plugin which handles incoming requests to local web server."""
 
-    DEFAULT_404_RESPONSE = memoryview(
-        build_http_response(
-            httpStatusCodes.NOT_FOUND,
-            reason=b'NOT FOUND',
-            headers={
-                b'Server': PROXY_AGENT_HEADER_VALUE,
-                b'Content-Length': b'0',
-            },
-            conn_close=True,
-        ),
-    )
-
-    DEFAULT_501_RESPONSE = memoryview(
-        build_http_response(
-            httpStatusCodes.NOT_IMPLEMENTED,
-            reason=b'NOT IMPLEMENTED',
-            headers={
-                b'Server': PROXY_AGENT_HEADER_VALUE,
-                b'Content-Length': b'0',
-            },
-            conn_close=True,
-        ),
-    )
-
     def __init__(
             self,
             *args: Any, **kwargs: Any,
@@ -137,7 +112,7 @@ def encryption_enabled(self) -> bool:
             self.flags.certfile is not None
 
     @staticmethod
-    def read_and_build_static_file_response(path: str, min_compression_limit: int) -> memoryview:
+    def read_and_build_static_file_response(path: str) -> memoryview:
         try:
             with open(path, 'rb') as f:
                 content = f.read()
@@ -148,22 +123,14 @@ def read_and_build_static_file_response(path: str, min_compression_limit: int) -
                 b'Content-Type': bytes_(content_type),
                 b'Cache-Control': b'max-age=86400',
             }
-            do_compress = len(content) > min_compression_limit
-            if do_compress:
-                headers.update({
-                    b'Content-Encoding': b'gzip',
-                })
-            return memoryview(
-                build_http_response(
-                    httpStatusCodes.OK,
-                    reason=b'OK',
-                    headers=headers,
-                    body=gzip.compress(content) if do_compress else content,
-                    conn_close=True,
-                ),
+            return okResponse(
+                content=content,
+                headers=headers,
+                # TODO: Should we really close or take advantage of keep-alive?
+                conn_close=True,
             )
         except FileNotFoundError:
-            return HttpWebServerPlugin.DEFAULT_404_RESPONSE
+            return NOT_FOUND_RESPONSE_PKT
 
     def try_upgrade(self) -> bool:
         if self.request.has_header(b'connection') and \
@@ -181,7 +148,7 @@ def try_upgrade(self) -> bool:
                 )
                 self.switched_protocol = httpProtocolTypes.WEBSOCKET
             else:
-                self.client.queue(self.DEFAULT_501_RESPONSE)
+                self.client.queue(NOT_IMPLEMENTED_RESPONSE_PKT)
                 return True
         return False
 
@@ -223,12 +190,11 @@ def on_request_complete(self) -> Union[socket.socket, bool]:
             self.client.queue(
                 self.read_and_build_static_file_response(
                     self.flags.static_server_dir + path,
-                    self.flags.min_compression_limit,
                 ),
             )
             return True
         # Catch all unhandled web server requests, return 404
-        self.client.queue(self.DEFAULT_404_RESPONSE)
+        self.client.queue(NOT_FOUND_RESPONSE_PKT)
         return True
 
     def get_descriptors(self) -> Tuple[List[int], List[int]]:
diff --git a/proxy/plugin/man_in_the_middle.py b/proxy/plugin/man_in_the_middle.py
index 970547e6f2..a6d6220dfd 100644
--- a/proxy/plugin/man_in_the_middle.py
+++ b/proxy/plugin/man_in_the_middle.py
@@ -8,8 +8,7 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-from ..common.utils import build_http_response
-from ..http import httpStatusCodes
+from ..http.responses import okResponse
 from ..http.proxy import HttpProxyBasePlugin
 
 
@@ -17,10 +16,4 @@ class ManInTheMiddlePlugin(HttpProxyBasePlugin):
     """Modifies upstream server responses."""
 
     def handle_upstream_chunk(self, chunk: memoryview) -> memoryview:
-        return memoryview(
-            build_http_response(
-                httpStatusCodes.OK,
-                reason=b'OK',
-                body=b'Hello from man in the middle',
-            ),
-        )
+        return okResponse(content=b'Hello from man in the middle')
diff --git a/proxy/plugin/mock_rest_api.py b/proxy/plugin/mock_rest_api.py
index 12e3543deb..56cac4cb27 100644
--- a/proxy/plugin/mock_rest_api.py
+++ b/proxy/plugin/mock_rest_api.py
@@ -15,9 +15,9 @@
 import json
 from typing import Optional
 
-from ..common.utils import bytes_, build_http_response, text_
+from ..common.utils import bytes_, text_
 
-from ..http import httpStatusCodes
+from ..http.responses import okResponse, NOT_FOUND_RESPONSE_PKT
 from ..http.parser import HttpParser
 from ..http.proxy import HttpProxyBasePlugin
 
@@ -75,26 +75,15 @@ def handle_client_request(
         assert request.path
         if request.path in self.REST_API_SPEC:
             self.client.queue(
-                memoryview(
-                    build_http_response(
-                        httpStatusCodes.OK,
-                        reason=b'OK',
-                        headers={b'Content-Type': b'application/json'},
-                        body=bytes_(
-                            json.dumps(
-                                self.REST_API_SPEC[request.path],
-                            ),
+                okResponse(
+                    headers={b'Content-Type': b'application/json'},
+                    content=bytes_(
+                        json.dumps(
+                            self.REST_API_SPEC[request.path],
                         ),
                     ),
                 ),
             )
         else:
-            self.client.queue(
-                memoryview(
-                    build_http_response(
-                        httpStatusCodes.NOT_FOUND,
-                        reason=b'NOT FOUND', body=b'Not Found',
-                    ),
-                ),
-            )
+            self.client.queue(NOT_FOUND_RESPONSE_PKT)
         return None
diff --git a/proxy/plugin/shortlink.py b/proxy/plugin/shortlink.py
index 43e2534ea6..11b7a7d823 100644
--- a/proxy/plugin/shortlink.py
+++ b/proxy/plugin/shortlink.py
@@ -15,9 +15,8 @@
 from typing import Optional
 
 from ..common.constants import DOT, SLASH
-from ..common.utils import build_http_response
 
-from ..http import httpStatusCodes
+from ..http.responses import NOT_FOUND_RESPONSE_PKT, seeOthersResponse
 from ..http.parser import HttpParser
 from ..http.proxy import HttpProxyBasePlugin
 
@@ -66,28 +65,11 @@ def handle_client_request(
             if request.host in self.SHORT_LINKS:
                 path = SLASH if not request.path else request.path
                 self.client.queue(
-                    memoryview(
-                        build_http_response(
-                            httpStatusCodes.SEE_OTHER, reason=b'See Other',
-                            headers={
-                                b'Location': b'http://' + self.SHORT_LINKS[request.host] + path,
-                                b'Content-Length': b'0',
-                            },
-                            conn_close=True,
-                        ),
+                    seeOthersResponse(
+                        b'http://' + self.SHORT_LINKS[request.host] + path,
                     ),
                 )
             else:
-                self.client.queue(
-                    memoryview(
-                        build_http_response(
-                            httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND',
-                            headers={
-                                b'Content-Length': b'0',
-                            },
-                            conn_close=True,
-                        ),
-                    ),
-                )
+                self.client.queue(NOT_FOUND_RESPONSE_PKT)
             return None
         return request
diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py
index bfbe4bd345..fe5e95707c 100644
--- a/proxy/plugin/web_server_route.py
+++ b/proxy/plugin/web_server_route.py
@@ -11,27 +11,16 @@
 import logging
 from typing import List, Tuple
 
-from ..common.utils import build_http_response
-
-from ..http import httpStatusCodes
+from ..http.responses import okResponse
 from ..http.parser import HttpParser
 from ..http.websocket import WebsocketFrame
 from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes
 
-HTTP_RESPONSE = memoryview(
-    build_http_response(
-        httpStatusCodes.OK, body=b'HTTP route response',
-    ),
-)
-
-HTTPS_RESPONSE = memoryview(
-    build_http_response(
-        httpStatusCodes.OK, body=b'HTTPS route response',
-    ),
-)
-
 logger = logging.getLogger(__name__)
 
+HTTP_RESPONSE = okResponse(content=b'HTTP route response')
+HTTPS_RESPONSE = okResponse(content=b'HTTP route response')
+
 
 class WebServerPlugin(HttpWebServerBasePlugin):
     """Demonstrates inbuilt web server routing using plugin."""
diff --git a/tests/http/exceptions/test_http_proxy_auth_failed.py b/tests/http/exceptions/test_http_proxy_auth_failed.py
index fdf1b9d6da..23a3b4dcd5 100644
--- a/tests/http/exceptions/test_http_proxy_auth_failed.py
+++ b/tests/http/exceptions/test_http_proxy_auth_failed.py
@@ -14,8 +14,8 @@
 from pytest_mock import MockerFixture
 
 from proxy.common.flag import FlagParser
-from proxy.http.exception.proxy_auth_failed import ProxyAuthenticationFailed
 from proxy.http import HttpProtocolHandler, httpHeaders
+from proxy.http.responses import PROXY_AUTH_FAILED_RESPONSE_PKT
 from proxy.core.connection import TcpClientConnection
 from proxy.common.utils import build_http_request
 
@@ -67,7 +67,8 @@ async def test_proxy_auth_fails_without_cred(self) -> None:
         self.mock_server_conn.assert_not_called()
         self.assertEqual(self.protocol_handler.work.has_buffer(), True)
         self.assertEqual(
-            self.protocol_handler.work.buffer[0], ProxyAuthenticationFailed.RESPONSE_PKT,
+            self.protocol_handler.work.buffer[0],
+            PROXY_AUTH_FAILED_RESPONSE_PKT,
         )
         self._conn.send.assert_not_called()
 
@@ -95,7 +96,8 @@ async def test_proxy_auth_fails_with_invalid_cred(self) -> None:
         self.mock_server_conn.assert_not_called()
         self.assertEqual(self.protocol_handler.work.has_buffer(), True)
         self.assertEqual(
-            self.protocol_handler.work.buffer[0], ProxyAuthenticationFailed.RESPONSE_PKT,
+            self.protocol_handler.work.buffer[0],
+            PROXY_AUTH_FAILED_RESPONSE_PKT,
         )
         self._conn.send.assert_not_called()
 
diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py
index 5f771bd708..0489b85741 100644
--- a/tests/http/test_http_parser.py
+++ b/tests/http/test_http_parser.py
@@ -11,12 +11,13 @@
 import unittest
 
 from proxy.common.constants import CRLF, HTTP_1_0
-from proxy.common.utils import build_http_request, build_http_response, build_http_header
+from proxy.common.utils import build_http_request, build_http_header
 from proxy.common.utils import find_http_line, bytes_
 
-from proxy.http import httpStatusCodes, httpMethods
+from proxy.http import httpMethods
 from proxy.http.exception import HttpProtocolException
 from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates
+from proxy.http.responses import okResponse
 
 
 class TestHttpParser(unittest.TestCase):
@@ -141,18 +142,16 @@ def test_build_request(self) -> None:
 
     def test_build_response(self) -> None:
         self.assertEqual(
-            build_http_response(
-                200, reason=b'OK', protocol_version=b'HTTP/1.1',
-            ),
+            okResponse(protocol_version=b'HTTP/1.1'),
             CRLF.join([
                 b'HTTP/1.1 200 OK',
                 CRLF,
             ]),
         )
         self.assertEqual(
-            build_http_response(
-                200, reason=b'OK', protocol_version=b'HTTP/1.1',
+            okResponse(
                 headers={b'key': b'value'},
+                protocol_version=b'HTTP/1.1',
             ),
             CRLF.join([
                 b'HTTP/1.1 200 OK',
@@ -164,10 +163,10 @@ def test_build_response(self) -> None:
     def test_build_response_adds_content_length_header(self) -> None:
         body = b'Hello world!!!'
         self.assertEqual(
-            build_http_response(
-                200, reason=b'OK', protocol_version=b'HTTP/1.1',
+            okResponse(
                 headers={b'key': b'value'},
-                body=body,
+                content=body,
+                protocol_version=b'HTTP/1.1',
             ),
             CRLF.join([
                 b'HTTP/1.1 200 OK',
@@ -610,29 +609,30 @@ def test_chunked_response_parse(self) -> None:
         self.assertEqual(self.parser.state, httpParserStates.COMPLETE)
 
     def test_pipelined_response_parse(self) -> None:
-        response = build_http_response(
-            httpStatusCodes.OK, reason=b'OK',
-            headers={
-                b'Content-Length': b'15',
-            },
-            body=b'{"key":"value"}',
+        self.assert_pipeline_response(
+            okResponse(
+                headers={
+                    b'Content-Length': b'15',
+                },
+                content=b'{"key":"value"}',
+            ),
         )
-        self.assert_pipeline_response(response)
 
     def test_pipelined_chunked_response_parse(self) -> None:
-        response = build_http_response(
-            httpStatusCodes.OK, reason=b'OK',
-            headers={
-                b'Transfer-Encoding': b'chunked',
-                b'Content-Type': b'application/json',
-            },
-            body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n',
+        self.assert_pipeline_response(
+            okResponse(
+                headers={
+                    b'Transfer-Encoding': b'chunked',
+                    b'Content-Type': b'application/json',
+                },
+                content=b'f\r\n{"key":"value"}\r\n0\r\n\r\n',
+                compress=False,
+            ),
         )
-        self.assert_pipeline_response(response)
 
-    def assert_pipeline_response(self, response: bytes) -> None:
+    def assert_pipeline_response(self, response: memoryview) -> None:
         self.parser = HttpParser(httpParserTypes.RESPONSE_PARSER)
-        self.parser.parse(response + response)
+        self.parser.parse(response.tobytes() + response.tobytes())
         self.assertEqual(self.parser.state, httpParserStates.COMPLETE)
         self.assertEqual(self.parser.body, b'{"key":"value"}')
         self.assertEqual(self.parser.buffer, response)
diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py
index f569afbdc3..23ae05faa9 100644
--- a/tests/http/test_http_proxy_tls_interception.py
+++ b/tests/http/test_http_proxy_tls_interception.py
@@ -19,11 +19,12 @@
 from unittest import mock
 from proxy.common.constants import DEFAULT_CA_FILE
 
-from proxy.core.connection import TcpClientConnection, TcpServerConnection
-from proxy.http import HttpProtocolHandler, httpMethods
-from proxy.http.proxy import HttpProxyPlugin
 from proxy.common.utils import build_http_request, bytes_
 from proxy.common.flag import FlagParser
+from proxy.http import HttpProtocolHandler, httpMethods
+from proxy.http.proxy import HttpProxyPlugin
+from proxy.http.responses import PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT
+from proxy.core.connection import TcpClientConnection, TcpServerConnection
 
 from ..test_assertions import Assertions
 
@@ -178,7 +179,7 @@ async def asyncReturnBool(val: bool) -> bool:
             ssl_connection,
         )
         self._conn.send.assert_called_with(
-            HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+            PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
         )
         assert self.flags.ca_cert_dir is not None
         self.mock_ssl_wrap.assert_called_with(
diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py
index c5f1ff1163..766208c959 100644
--- a/tests/http/test_protocol_handler.py
+++ b/tests/http/test_protocol_handler.py
@@ -24,8 +24,12 @@
 from proxy.core.connection import TcpClientConnection
 from proxy.http.parser import HttpParser
 from proxy.http.proxy import HttpProxyPlugin
+from proxy.http.responses import (
+    BAD_GATEWAY_RESPONSE_PKT,
+    PROXY_AUTH_FAILED_RESPONSE_PKT,
+    PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+)
 from proxy.http.parser import httpParserStates, httpParserTypes
-from proxy.http.exception import ProxyAuthenticationFailed, ProxyConnectionFailed
 from proxy.http import HttpProtocolHandler, httpHeaders
 
 from ..test_assertions import Assertions
@@ -80,7 +84,7 @@ async def test_proxy_connection_failed(self) -> None:
         await self.protocol_handler._run_once()
         self.assertEqual(
             self.protocol_handler.work.buffer[0],
-            ProxyConnectionFailed.RESPONSE_PKT,
+            BAD_GATEWAY_RESPONSE_PKT,
         )
 
     @pytest.mark.asyncio    # type: ignore[misc]
@@ -108,7 +112,7 @@ async def test_proxy_authentication_failed(self) -> None:
         await self.protocol_handler._run_once()
         self.assertEqual(
             self.protocol_handler.work.buffer[0],
-            ProxyAuthenticationFailed.RESPONSE_PKT,
+            PROXY_AUTH_FAILED_RESPONSE_PKT,
         )
 
 
@@ -191,7 +195,7 @@ async def assert_tunnel_response(
         )
         self.assertEqual(
             self.protocol_handler.work.buffer[0],
-            HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+            PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
         )
         self.mock_server_connection.assert_called_once()
         server.connect.assert_called_once()
@@ -432,7 +436,7 @@ async def assert_data_queued_to_server(self, server: mock.Mock) -> None:
         assert self.http_server_port is not None
         self.assertEqual(
             self._conn.send.call_args[0][0],
-            HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+            PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
         )
 
         pkt = CRLF.join([
diff --git a/tests/http/test_web_server.py b/tests/http/test_web_server.py
index 3a59fa63af..96083356f8 100644
--- a/tests/http/test_web_server.py
+++ b/tests/http/test_web_server.py
@@ -25,7 +25,7 @@
 from proxy.http.parser import HttpParser, httpParserStates, httpParserTypes
 from proxy.common.utils import build_http_response, build_http_request, bytes_
 from proxy.common.constants import CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PAC_FILE, PLUGIN_WEB_SERVER, PROXY_PY_DIR
-from proxy.http.server import HttpWebServerPlugin
+from proxy.http.responses import NOT_FOUND_RESPONSE_PKT
 
 from ..test_assertions import Assertions
 
@@ -309,7 +309,7 @@ async def test_static_web_server_serves_404(self) -> None:
         self.assertEqual(self._conn.send.call_count, 1)
         self.assertEqual(
             self._conn.send.call_args[0][0],
-            HttpWebServerPlugin.DEFAULT_404_RESPONSE,
+            NOT_FOUND_RESPONSE_PKT,
         )
 
 
@@ -368,5 +368,5 @@ async def test_default_web_server_returns_404(self) -> None:
         )
         self.assertEqual(
             self.protocol_handler.work.buffer[0],
-            HttpWebServerPlugin.DEFAULT_404_RESPONSE,
+            NOT_FOUND_RESPONSE_PKT,
         )
diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py
index 03a32527ed..50d2055888 100644
--- a/tests/plugin/test_http_proxy_plugins.py
+++ b/tests/plugin/test_http_proxy_plugins.py
@@ -8,6 +8,7 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+import gzip
 import json
 import pytest
 import selectors
@@ -23,6 +24,7 @@
 from proxy.http import HttpProtocolHandler
 from proxy.http import httpStatusCodes
 from proxy.http.proxy import HttpProxyPlugin
+from proxy.http.parser import HttpParser, httpParserTypes
 from proxy.common.utils import build_http_request, bytes_, build_http_response
 from proxy.common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_HTTP_PORT
 from proxy.plugin import ProposedRestApiPlugin, RedirectToCustomServerPlugin
@@ -151,15 +153,22 @@ async def test_proposed_rest_api_plugin(self) -> None:
         await self.protocol_handler._run_once()
 
         self.mock_server_conn.assert_not_called()
+        response = HttpParser(httpParserTypes.RESPONSE_PARSER)
+        response.parse(self.protocol_handler.work.buffer[0].tobytes())
+        assert response.body
         self.assertEqual(
-            self.protocol_handler.work.buffer[0].tobytes(),
-            build_http_response(
-                httpStatusCodes.OK, reason=b'OK',
-                headers={b'Content-Type': b'application/json'},
-                body=bytes_(
-                    json.dumps(
-                        ProposedRestApiPlugin.REST_API_SPEC[path],
-                    ),
+            response.header(b'content-type'),
+            b'application/json',
+        )
+        self.assertEqual(
+            response.header(b'content-encoding'),
+            b'gzip',
+        )
+        self.assertEqual(
+            gzip.decompress(response.body),
+            bytes_(
+                json.dumps(
+                    ProposedRestApiPlugin.REST_API_SPEC[path],
                 ),
             ),
         )
@@ -329,15 +338,16 @@ def closed() -> bool:
         server.recv.return_value = \
             build_http_response(
                 httpStatusCodes.OK,
-                reason=b'OK', body=b'Original Response From Upstream',
+                reason=b'OK',
+                body=b'Original Response From Upstream',
             )
         await self.protocol_handler._run_once()
+        response = HttpParser(httpParserTypes.RESPONSE_PARSER)
+        response.parse(self.protocol_handler.work.buffer[0].tobytes())
+        assert response.body
         self.assertEqual(
-            self.protocol_handler.work.buffer[0].tobytes(),
-            build_http_response(
-                httpStatusCodes.OK,
-                reason=b'OK', body=b'Hello from man in the middle',
-            ),
+            gzip.decompress(response.body),
+            b'Hello from man in the middle',
         )
 
     @pytest.mark.asyncio    # type: ignore[misc]
diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py
index 232b0dd954..9845980201 100644
--- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py
+++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py
@@ -9,6 +9,7 @@
     :license: BSD, see LICENSE for more details.
 """
 import ssl
+import gzip
 import socket
 import pytest
 import selectors
@@ -17,12 +18,13 @@
 from typing import Any, cast
 
 from proxy.common.flag import FlagParser
-from proxy.common.utils import bytes_, build_http_request, build_http_response
+from proxy.common.utils import bytes_, build_http_request
 from proxy.core.connection import TcpClientConnection, TcpServerConnection
 
-from proxy.http import httpMethods, httpStatusCodes, HttpProtocolHandler
+from proxy.http import httpMethods, HttpProtocolHandler
 from proxy.http.proxy import HttpProxyPlugin
-from proxy.http.parser import HttpParser
+from proxy.http.parser import HttpParser, httpParserTypes
+from proxy.http.responses import PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, okResponse
 
 from .utils import get_plugin_by_test_name
 
@@ -174,7 +176,7 @@ async def test_modify_post_data_plugin(self) -> None:
         )
         self.assertEqual(self.server.connection, self.server_ssl_connection)
         self._conn.send.assert_called_with(
-            HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+            PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
         )
         self.assertFalse(self.protocol_handler.work.has_buffer())
 
@@ -229,7 +231,7 @@ async def test_man_in_the_middle_plugin(self) -> None:
         )
         self.assertEqual(self.server.connection, self.server_ssl_connection)
         self._conn.send.assert_called_with(
-            HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+            PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
         )
         self.assertFalse(self.protocol_handler.work.has_buffer())
         #
@@ -250,17 +252,14 @@ async def test_man_in_the_middle_plugin(self) -> None:
         self.server.flush.assert_called_once()
 
         # Server read
-        self.server.recv.return_value = memoryview(
-            build_http_response(
-                httpStatusCodes.OK,
-                reason=b'OK', body=b'Original Response From Upstream',
-            ),
+        self.server.recv.return_value = okResponse(
+            content=b'Original Response From Upstream',
         )
         await self.protocol_handler._run_once()
+        response = HttpParser(httpParserTypes.RESPONSE_PARSER)
+        response.parse(self.protocol_handler.work.buffer[0].tobytes())
+        assert response.body
         self.assertEqual(
-            self.protocol_handler.work.buffer[0].tobytes(),
-            build_http_response(
-                httpStatusCodes.OK,
-                reason=b'OK', body=b'Hello from man in the middle',
-            ),
+            gzip.decompress(response.body),
+            b'Hello from man in the middle',
         )
diff --git a/tests/testing/test_embed.py b/tests/testing/test_embed.py
index 6db0458e81..1ab6f404e0 100644
--- a/tests/testing/test_embed.py
+++ b/tests/testing/test_embed.py
@@ -19,7 +19,7 @@
 from proxy.common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, PROXY_AGENT_HEADER_VALUE
 from proxy.common.utils import socket_connection, build_http_request
 from proxy.http import httpMethods
-from proxy.http.server import HttpWebServerPlugin
+from proxy.http.responses import NOT_FOUND_RESPONSE_PKT
 
 
 @pytest.mark.skipif(
@@ -49,7 +49,7 @@ def test_with_proxy(self) -> None:
             response = conn.recv(DEFAULT_CLIENT_RECVBUF_SIZE)
         self.assertEqual(
             response,
-            HttpWebServerPlugin.DEFAULT_404_RESPONSE.tobytes(),
+            NOT_FOUND_RESPONSE_PKT.tobytes(),
         )
 
     def test_proxy_vcr(self) -> None:

From 13c64b53573510b0df924faf7d6203a337af7818 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Sat, 25 Dec 2021 12:40:25 +0530
Subject: [PATCH 07/22] Introduce `ProgramNamePlugin` plugin (#904)

* Add `ProgramNamePlugin`

* Update readme

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Remove `_compat.py`

* Add suggestions coming from #659

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 .flake8                               |  1 -
 .gitignore                            |  2 +
 .pylintrc                             |  1 -
 README.md                             | 49 +++++++++++++++----
 proxy/common/_compat.py               | 23 ---------
 proxy/common/constants.py             | 12 +++--
 proxy/common/flag.py                  |  3 +-
 proxy/common/types.py                 |  1 +
 proxy/common/utils.py                 |  6 ++-
 proxy/core/event/dispatcher.py        |  6 +--
 proxy/core/event/subscriber.py        |  4 +-
 proxy/http/proxy/server.py            |  6 +--
 proxy/http/server/web.py              |  4 +-
 proxy/plugin/__init__.py              |  2 +
 proxy/plugin/filter_by_url_regex.py   |  7 +--
 proxy/plugin/man_in_the_middle.py     |  2 +-
 proxy/plugin/program_name.py          | 68 +++++++++++++++++++++++++++
 proxy/plugin/web_server_route.py      | 10 ----
 tests/core/test_listener.py           |  2 +-
 tests/http/test_http2.py              |  4 +-
 tests/integration/test_integration.py |  6 +--
 tests/test_set_open_file_limit.py     |  2 +-
 tests/testing/test_embed.py           |  3 +-
 23 files changed, 142 insertions(+), 82 deletions(-)
 delete mode 100644 proxy/common/_compat.py
 create mode 100644 proxy/plugin/program_name.py

diff --git a/.flake8 b/.flake8
index b18b1a582e..0999bbdc5d 100644
--- a/.flake8
+++ b/.flake8
@@ -148,7 +148,6 @@ extend-ignore =
   WPS420  # FIXME: pointless keyword like `pass`
   WPS421  # FIXME: call to `print()`
   WPS425  # FIXME: bool non-keyword arg
-  WPS427  # FIXME: unreachable code
   WPS428  # FIXME: pointless statement
   WPS430  # FIXME: nested func
   WPS431  # FIXME: nested class
diff --git a/.gitignore b/.gitignore
index 217ce78fd2..09bd936373 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,8 @@ proxy.py.iml
 *.crt
 *.key
 *.pem
+
+.venv*
 venv*
 
 cover
diff --git a/.pylintrc b/.pylintrc
index 1203c8dec9..25434cea23 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -127,7 +127,6 @@ disable=raw-checker-failed,
         too-many-return-statements,
         too-many-statements,
         unnecessary-pass,
-        unreachable,
         unused-argument,
         useless-return,
         useless-super-delegation,
diff --git a/README.md b/README.md
index e8743686c5..4446ff21be 100644
--- a/README.md
+++ b/README.md
@@ -54,11 +54,12 @@
     - [Cache Responses Plugin](#cacheresponsesplugin)
     - [Man-In-The-Middle Plugin](#maninthemiddleplugin)
     - [Proxy Pool Plugin](#proxypoolplugin)
-    - [FilterByClientIpPlugin](#filterbyclientipplugin)
-    - [ModifyChunkResponsePlugin](#modifychunkresponseplugin)
-    - [CloudflareDnsResolverPlugin](#cloudflarednsresolverplugin)
-    - [CustomDnsResolverPlugin](#customdnsresolverplugin)
-    - [CustomNetworkInterface](#customnetworkinterface)
+    - [Filter By Client IP Plugin](#filterbyclientipplugin)
+    - [Modify Chunk Response Plugin](#modifychunkresponseplugin)
+    - [Cloudflare DNS Resolver Plugin](#cloudflarednsresolverplugin)
+    - [Custom DNS Resolver Plugin](#customdnsresolverplugin)
+    - [Custom Network Interface](#customnetworkinterface)
+    - [Program Name Plugin](#programnameplugin)
   - [HTTP Web Server Plugins](#http-web-server-plugins)
     - [Reverse Proxy](#reverse-proxy)
     - [Web Server Route](#web-server-route)
@@ -578,7 +579,7 @@ Verify mock API response using `curl -x localhost:8899 http://api.example.com/v1
 Verify the same by inspecting `proxy.py` logs:
 
 ```console
-2019-09-27 12:44:02,212 - INFO - pid:7077 - access_log:1210 - ::1:64792 - GET None:None/v1/users/ - None None - 0 byte
+... [redacted] ... - access_log:1210 - ::1:64792 - GET None:None/v1/users/ - None None - 0 byte
 ```
 
 Access log shows `None:None` as server `ip:port`. `None` simply means that
@@ -618,8 +619,8 @@ Verify the same by inspecting the logs for `proxy.py`.
 Along with the proxy request log, you must also see a http web server request log.
 
 ```
-2019-09-24 19:09:33,602 - INFO - pid:49996 - access_log:1241 - ::1:49525 - GET /
-2019-09-24 19:09:33,603 - INFO - pid:49995 - access_log:1157 - ::1:49524 - GET localhost:8899/ - 404 NOT FOUND - 70 bytes
+... [redacted] ... - access_log:1241 - ::1:49525 - GET /
+... [redacted] ... - access_log:1157 - ::1:49524 - GET localhost:8899/ - 404 NOT FOUND - 70 bytes
 ```
 
 ### FilterByUpstreamHostPlugin
@@ -650,10 +651,10 @@ Above `418 I'm a tea pot` is sent by our plugin.
 Verify the same by inspecting logs for `proxy.py`:
 
 ```console
-2019-09-24 19:21:37,893 - ERROR - pid:50074 - handle_readables:1347 - HttpProtocolException type raised
+... [redacted] ... - handle_readables:1347 - HttpProtocolException type raised
 Traceback (most recent call last):
 ... [redacted] ...
-2019-09-24 19:21:37,897 - INFO - pid:50074 - access_log:1157 - ::1:49911 - GET None:None/ - None None - 0 bytes
+... [redacted] ... - access_log:1157 - ::1:49911 - GET None:None/ - None None - 0 bytes
 ```
 
 ### CacheResponsesPlugin
@@ -887,6 +888,34 @@ for more details.
 PS: There is no plugin named, but [CustomDnsResolverPlugin](#customdnsresolverplugin)
 can be easily customized according to your needs.
 
+### ProgramNamePlugin
+
+Attempts to resolve program `(application)` name for proxy requests originating from the local machine.
+If identified, client IP in the access logs is replaced with program name.
+
+Start `proxy.py` as:
+
+```console
+❯ proxy \
+    --plugins proxy.plugin.ProgramNamePlugin
+```
+
+Make a request using `curl`:
+
+```console
+❯ curl -v -x localhost:8899 https://httpbin.org/get
+```
+
+You must see log lines like this:
+
+```console
+... [redacted] ... - [I] server.access_log:419 - curl:58096 - CONNECT httpbin.org:443 - 6010 bytes - 1824.62ms
+```
+
+Notice `curl` in-place of `::1` or `127.0.0.1` as client IP.
+
+[![WARNING](https://img.shields.io/static/v1?label=Compatibility&message=warning&color=red)](#programnameplugin) If `ProgramNamePlugin` does not work reliably on your operating system, kindly contribute by sending a pull request and/or open an issue.  Thank you!!!
+
 ## HTTP Web Server Plugins
 
 ### Reverse Proxy
diff --git a/proxy/common/_compat.py b/proxy/common/_compat.py
deleted file mode 100644
index c3ec75e411..0000000000
--- a/proxy/common/_compat.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    proxy.py
-    ~~~~~~~~
-    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
-    Network monitoring, controls & Application development, testing, debugging.
-
-    :copyright: (c) 2013-present by Abhinav Singh and contributors.
-    :license: BSD, see LICENSE for more details.
-
-    Compatibility code for using Proxy.py across various versions of Python.
-
-    .. spelling::
-
-        compat
-        py
-"""
-
-import platform
-
-
-SYS_PLATFORM = platform.system()
-IS_WINDOWS = SYS_PLATFORM == 'Windows'
diff --git a/proxy/common/constants.py b/proxy/common/constants.py
index 7b5a947d99..80d12ac29d 100644
--- a/proxy/common/constants.py
+++ b/proxy/common/constants.py
@@ -13,14 +13,17 @@
 import time
 import secrets
 import pathlib
+import platform
 import sysconfig
 import ipaddress
 
 from typing import Any, List
 
-from ._compat import IS_WINDOWS  # noqa: WPS436
 from .version import __version__
 
+SYS_PLATFORM = platform.system()
+IS_WINDOWS = SYS_PLATFORM == 'Windows'
+
 
 def _env_threadless_compliant() -> bool:
     """Returns true for Python 3.8+ across all platforms
@@ -96,12 +99,13 @@ def _env_threadless_compliant() -> bool:
 DEFAULT_LOG_FILE = None
 DEFAULT_LOG_FORMAT = '%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(module)s.%(funcName)s:%(lineno)d - %(message)s'
 DEFAULT_LOG_LEVEL = 'INFO'
-DEFAULT_WEB_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - {request_method} {request_path} - {connection_time_ms}ms'
-DEFAULT_HTTP_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
+DEFAULT_WEB_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' \
+    '{request_method} {request_path} - {request_ua} - {connection_time_ms}ms'
+DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
     '{request_method} {server_host}:{server_port}{request_path} - ' + \
     '{response_code} {response_reason} - {response_bytes} bytes - ' + \
     '{connection_time_ms}ms'
-DEFAULT_HTTPS_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
+DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
     '{request_method} {server_host}:{server_port} - ' + \
     '{response_bytes} bytes - {connection_time_ms}ms'
 DEFAULT_NUM_ACCEPTORS = 0
diff --git a/proxy/common/flag.py b/proxy/common/flag.py
index 3c7f88b855..6161dbd0b6 100644
--- a/proxy/common/flag.py
+++ b/proxy/common/flag.py
@@ -19,7 +19,6 @@
 
 from typing import Optional, List, Any, cast
 
-from ._compat import IS_WINDOWS  # noqa: WPS436
 from .plugins import Plugins
 from .types import IpAddress
 from .utils import bytes_, is_py2, is_threadless, set_open_file_limit
@@ -27,7 +26,7 @@
 from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE
 from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, DEFAULT_MIN_COMPRESSION_LIMIT
 from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE
-from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH
+from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH, IS_WINDOWS
 from .logger import Logger
 
 from .version import __version__
diff --git a/proxy/common/types.py b/proxy/common/types.py
index a95a49e8d2..3d226829bd 100644
--- a/proxy/common/types.py
+++ b/proxy/common/types.py
@@ -22,6 +22,7 @@
 else:
     from typing_extensions import Protocol
 
+
 if TYPE_CHECKING:
     DictQueueType = queue.Queue[Dict[str, Any]]    # pragma: no cover
 else:
diff --git a/proxy/common/utils.py b/proxy/common/utils.py
index 3a51dbf0f4..0bc2bc9929 100644
--- a/proxy/common/utils.py
+++ b/proxy/common/utils.py
@@ -23,8 +23,10 @@
 from types import TracebackType
 from typing import Optional, Dict, Any, List, Tuple, Type, Callable
 
-from ._compat import IS_WINDOWS  # noqa: WPS436
-from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT, DEFAULT_THREADLESS
+from .constants import (
+    HTTP_1_1, COLON, WHITESPACE, CRLF,
+    DEFAULT_TIMEOUT, DEFAULT_THREADLESS, IS_WINDOWS,
+)
 
 if not IS_WINDOWS:
     import resource
diff --git a/proxy/core/event/dispatcher.py b/proxy/core/event/dispatcher.py
index 1bcb40201d..d26340d2ea 100644
--- a/proxy/core/event/dispatcher.py
+++ b/proxy/core/event/dispatcher.py
@@ -83,11 +83,7 @@ def run(self) -> None:
                     self.run_once()
                 except queue.Empty:
                     pass
-        except BrokenPipeError:
-            pass
-        except EOFError:
-            pass
-        except KeyboardInterrupt:
+        except (BrokenPipeError, EOFError, KeyboardInterrupt):
             pass
         except Exception as e:
             logger.exception('Dispatcher exception', exc_info=e)
diff --git a/proxy/core/event/subscriber.py b/proxy/core/event/subscriber.py
index c92803074f..14b9e8f1a5 100644
--- a/proxy/core/event/subscriber.py
+++ b/proxy/core/event/subscriber.py
@@ -103,9 +103,7 @@ def unsubscribe(self) -> None:
             return
         try:
             self.event_queue.unsubscribe(self.relay_sub_id)
-        except BrokenPipeError:
-            pass
-        except EOFError:
+        except (BrokenPipeError, EOFError):
             pass
         finally:
             # self.relay_sub_id = None
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index 7c2c41083c..55604abf63 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -38,7 +38,7 @@
 from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE
 from ...common.constants import COMMA, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CERT_FILE
 from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS
-from ...common.constants import DEFAULT_HTTP_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_ACCESS_LOG_FORMAT
+from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT
 from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH
 from ...common.utils import text_
 from ...common.pki import gen_public_key, gen_csr, sign_csr
@@ -413,9 +413,9 @@ def on_client_connection_close(self) -> None:
             )
 
     def access_log(self, log_attrs: Dict[str, Any]) -> None:
-        access_log_format = DEFAULT_HTTPS_ACCESS_LOG_FORMAT
+        access_log_format = DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT
         if not self.request.is_https_tunnel:
-            access_log_format = DEFAULT_HTTP_ACCESS_LOG_FORMAT
+            access_log_format = DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT
         logger.info(access_log_format.format_map(log_attrs))
 
     def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]:
diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py
index 89ed43c618..eda6f7c40b 100644
--- a/proxy/http/server/web.py
+++ b/proxy/http/server/web.py
@@ -272,10 +272,10 @@ def on_client_connection_close(self) -> None:
             'request_method': text_(self.request.method),
             'request_path': text_(self.request.path),
             'request_bytes': self.request.total_size,
-            'request_ua': self.request.header(b'user-agent')
+            'request_ua': text_(self.request.header(b'user-agent'))
             if self.request.has_header(b'user-agent')
             else None,
-            'request_version': self.request.version,
+            'request_version': None if not self.request.version else text_(self.request.version),
             # Response
             #
             # TODO: Track and inject web server specific response attributes
diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py
index cd6b3a205f..578bc3bcc4 100644
--- a/proxy/plugin/__init__.py
+++ b/proxy/plugin/__init__.py
@@ -27,6 +27,7 @@
 from .modify_chunk_response import ModifyChunkResponsePlugin
 from .custom_dns_resolver import CustomDnsResolverPlugin
 from .cloudflare_dns import CloudflareDnsResolverPlugin
+from .program_name import ProgramNamePlugin
 
 __all__ = [
     'CacheResponsesPlugin',
@@ -45,4 +46,5 @@
     'FilterByURLRegexPlugin',
     'CustomDnsResolverPlugin',
     'CloudflareDnsResolverPlugin',
+    'ProgramNamePlugin',
 ]
diff --git a/proxy/plugin/filter_by_url_regex.py b/proxy/plugin/filter_by_url_regex.py
index c9659e3732..3d12ad342c 100644
--- a/proxy/plugin/filter_by_url_regex.py
+++ b/proxy/plugin/filter_by_url_regex.py
@@ -72,8 +72,7 @@ def handle_client_request(
             request.path,
         )
         # check URL against list
-        rule_number = 1
-        for blocked_entry in self.filters:
+        for rule_number, blocked_entry in enumerate(self.filters, start=1):
             # if regex matches on URL
             if re.search(text_(blocked_entry['regex']), text_(url)):
                 # log that the request has been filtered
@@ -90,8 +89,4 @@ def handle_client_request(
                     status_code=httpStatusCodes.NOT_FOUND,
                     reason=b'Blocked',
                 )
-                # stop looping through filter list
-                break
-            # increment rule number
-            rule_number += 1
         return request
diff --git a/proxy/plugin/man_in_the_middle.py b/proxy/plugin/man_in_the_middle.py
index a6d6220dfd..7975820ec0 100644
--- a/proxy/plugin/man_in_the_middle.py
+++ b/proxy/plugin/man_in_the_middle.py
@@ -15,5 +15,5 @@
 class ManInTheMiddlePlugin(HttpProxyBasePlugin):
     """Modifies upstream server responses."""
 
-    def handle_upstream_chunk(self, chunk: memoryview) -> memoryview:
+    def handle_upstream_chunk(self, _chunk: memoryview) -> memoryview:
         return okResponse(content=b'Hello from man in the middle')
diff --git a/proxy/plugin/program_name.py b/proxy/plugin/program_name.py
new file mode 100644
index 0000000000..b8e205b4df
--- /dev/null
+++ b/proxy/plugin/program_name.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import os
+import subprocess
+
+from typing import Any, Dict, Optional
+
+from ..common.utils import text_
+from ..common.constants import IS_WINDOWS
+
+from ..http.parser import HttpParser
+from ..http.proxy import HttpProxyBasePlugin
+
+
+class ProgramNamePlugin(HttpProxyBasePlugin):
+    """Tries to identify the application connecting to the
+    proxy instance.  This is only possible when connection
+    itself originates from the same machine where the proxy
+    instance is running."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.program_name: Optional[str] = None
+
+    def before_upstream_connection(
+            self, request: HttpParser,
+    ) -> Optional[HttpParser]:
+        if IS_WINDOWS:
+            raise NotImplementedError()
+        assert self.client.addr
+        if self.client.addr[0] in ('::1', '127.0.0.1'):
+            assert self.client.addr
+            port = self.client.addr[1]
+            ls = subprocess.Popen(
+                ('lsof', '-i', '-P', '-n'),
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            try:
+                output = subprocess.check_output(
+                    ('grep', '{0}'.format(port)),
+                    stdin=ls.stdout,
+                )
+                port_programs = output.splitlines()
+                for program in port_programs:
+                    parts = program.split()
+                    if int(parts[1]) != os.getpid():
+                        self.program_name = text_(parts[0])
+                        break
+            except subprocess.CalledProcessError:
+                pass
+            finally:
+                ls.wait(timeout=1)
+        if self.program_name is None:
+            self.program_name = self.client.addr[0]
+        return request
+
+    def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        context.update({'client_ip': self.program_name})
+        return context
diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py
index fe5e95707c..5f881a68f7 100644
--- a/proxy/plugin/web_server_route.py
+++ b/proxy/plugin/web_server_route.py
@@ -13,7 +13,6 @@
 
 from ..http.responses import okResponse
 from ..http.parser import HttpParser
-from ..http.websocket import WebsocketFrame
 from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes
 
 logger = logging.getLogger(__name__)
@@ -37,12 +36,3 @@ def handle_request(self, request: HttpParser) -> None:
             self.client.queue(HTTP_RESPONSE)
         elif request.path == b'/https-route-example':
             self.client.queue(HTTPS_RESPONSE)
-
-    def on_websocket_open(self) -> None:
-        logger.info('Websocket open')
-
-    def on_websocket_message(self, frame: WebsocketFrame) -> None:
-        logger.info(frame.data)
-
-    def on_client_connection_close(self) -> None:
-        logger.debug('Client connection close')
diff --git a/tests/core/test_listener.py b/tests/core/test_listener.py
index 5ca29e58f9..8d5706953b 100644
--- a/tests/core/test_listener.py
+++ b/tests/core/test_listener.py
@@ -18,7 +18,7 @@
 import pytest
 
 from proxy.core.acceptor import Listener
-from proxy.common._compat import IS_WINDOWS  # noqa: WPS436
+from proxy.common.constants import IS_WINDOWS
 from proxy.common.flag import FlagParser
 
 
diff --git a/tests/http/test_http2.py b/tests/http/test_http2.py
index dc15740462..942e5edf8b 100644
--- a/tests/http/test_http2.py
+++ b/tests/http/test_http2.py
@@ -8,10 +8,10 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-import pytest
 import httpx
+import pytest
 
-from proxy.common._compat import IS_WINDOWS  # noqa: WPS436
+from proxy.common.constants import IS_WINDOWS
 from proxy import TestCase
 
 
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
index 9df24fa8d1..bf7df0b889 100644
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -10,14 +10,14 @@
 
     Test the simplest proxy use scenario for smoke.
 """
+import pytest
+
 from pathlib import Path
 from subprocess import check_output, Popen
 from typing import Generator, Any
 
-import pytest
-
+from proxy.common.constants import IS_WINDOWS
 from proxy.common.utils import get_available_port
-from proxy.common._compat import IS_WINDOWS  # noqa: WPS436
 
 
 # FIXME: Ignore is necessary for as long as pytest hasn't figured out
diff --git a/tests/test_set_open_file_limit.py b/tests/test_set_open_file_limit.py
index c785b5adb6..f02e3c2215 100644
--- a/tests/test_set_open_file_limit.py
+++ b/tests/test_set_open_file_limit.py
@@ -13,7 +13,7 @@
 
 import pytest
 
-from proxy.common._compat import IS_WINDOWS  # noqa: WPS436
+from proxy.common.constants import IS_WINDOWS
 from proxy.common.utils import set_open_file_limit
 
 if not IS_WINDOWS:
diff --git a/tests/testing/test_embed.py b/tests/testing/test_embed.py
index 1ab6f404e0..a1f485bf3e 100644
--- a/tests/testing/test_embed.py
+++ b/tests/testing/test_embed.py
@@ -15,8 +15,7 @@
 import pytest
 
 from proxy import TestCase
-from proxy.common._compat import IS_WINDOWS  # noqa: WPS436
-from proxy.common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, PROXY_AGENT_HEADER_VALUE
+from proxy.common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, PROXY_AGENT_HEADER_VALUE, IS_WINDOWS
 from proxy.common.utils import socket_connection, build_http_request
 from proxy.http import httpMethods
 from proxy.http.responses import NOT_FOUND_RESPONSE_PKT

From 904617eca49b8af584129d96661ae3628255333a Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Sat, 25 Dec 2021 17:41:15 +0530
Subject: [PATCH 08/22] Update defaults for `--hostname` and `--local-executor`
 (#905)

* do it

* Change defaults

* integrate with ci/cd updates

* Fix ci

* Add dockerfile `SKIP_OPENSSL` option which will allow us to build container without openssl

* Skip openssl for latest tag, add another openssl tag for images with openssl support

* Push separate openssl image to GHCR for every PR
---
 .github/workflows/test-library.yml |  40 ++++++++
 Dockerfile                         |  10 +-
 Makefile                           |  11 ++-
 README.md                          | 152 ++++++++++++++++++++++++-----
 docs/spelling_wordlist.txt         |   2 +
 proxy/common/constants.py          |   2 +-
 proxy/core/acceptor/listener.py    |   6 +-
 tests/core/test_acceptor.py        |   3 +-
 tests/test_main.py                 |   6 +-
 9 files changed, 196 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml
index e6fc703fa9..38d9913cbb 100644
--- a/.github/workflows/test-library.yml
+++ b/.github/workflows/test-library.yml
@@ -766,6 +766,7 @@ jobs:
         --platform ${{
           needs.pre-setup.outputs.container-platforms
         }}
+        --build-arg SKIP_OPENSSL=1
         --build-arg PROXYPY_PKG_PATH='dist/${{
           needs.pre-setup.outputs.wheel-artifact-name
         }}'
@@ -787,6 +788,43 @@ jobs:
         --platform ${{
           needs.pre-setup.outputs.container-platforms
         }}
+        --build-arg SKIP_OPENSSL=1
+        --build-arg PROXYPY_PKG_PATH='dist/${{
+          needs.pre-setup.outputs.wheel-artifact-name
+        }}'
+        -t $LATEST_TAG .
+    - name: Push openssl to GHCR
+      run: >-
+        REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
+        CONTAINER_TAG=$REGISTRY_URL:${{
+          needs.pre-setup.outputs.container-version
+        }}-openssl;
+        docker buildx build
+        --push
+        --platform ${{
+          needs.pre-setup.outputs.container-platforms
+        }}
+        --build-arg PROXYPY_PKG_PATH='dist/${{
+          needs.pre-setup.outputs.wheel-artifact-name
+        }}'
+        -t $CONTAINER_TAG .
+    - name: Tag openssl on GHCR
+      if: >-
+        github.event_name == 'push' &&
+        (
+          github.ref == format(
+            'refs/heads/{0}', github.event.repository.default_branch
+          ) ||
+          github.ref == 'refs/heads/master'
+        )
+      run: >-
+        REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
+        LATEST_TAG=$REGISTRY_URL:openssl;
+        docker buildx build
+        --push
+        --platform ${{
+          needs.pre-setup.outputs.container-platforms
+        }}
         --build-arg PROXYPY_PKG_PATH='dist/${{
           needs.pre-setup.outputs.wheel-artifact-name
         }}'
@@ -796,6 +834,7 @@ jobs:
       with:
         username: abhinavsingh
         password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+    # TODO: openssl image is not published on DockerHub
     - name: Push to DockerHub
       run: >-
         REGISTRY_URL="abhinavsingh/proxy.py";
@@ -807,6 +846,7 @@ jobs:
         --platform ${{
           needs.pre-setup.outputs.container-platforms
         }}
+        --build-arg SKIP_OPENSSL=1
         --build-arg PROXYPY_PKG_PATH='dist/${{
           needs.pre-setup.outputs.wheel-artifact-name
         }}'
diff --git a/Dockerfile b/Dockerfile
index 830fbcc6db..1b99e7151f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,5 @@
 FROM python:3.10-alpine as base
+
 LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \
   com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \
   😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \
@@ -8,11 +9,15 @@ LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \
   com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \
   com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" \
   org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py"
+
 ENV PYTHONUNBUFFERED 1
+
+ARG SKIP_OPENSSL
 ARG PROXYPY_PKG_PATH
 
 COPY README.md /
 COPY $PROXYPY_PKG_PATH /
+
 RUN pip install --upgrade pip && \
   pip install \
   --no-index \
@@ -20,9 +25,8 @@ RUN pip install --upgrade pip && \
   proxy.py && \
   rm *.whl
 
-# Install openssl to enable TLS interception & HTTPS proxy options within container
-# NOTE: You can comment out this line if you don't intend to use those features.
-RUN apk update && apk add openssl
+# Use `--build-arg SKIP_OPENSSL=1` to disable openssl installation
+RUN if [[ -z "$SKIP_OPENSSL" ]]; then apk update && apk add openssl; fi
 
 EXPOSE 8899/tcp
 ENTRYPOINT [ "proxy" ]
diff --git a/Makefile b/Makefile
index 5da6167be9..989b642d4c 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@ endif
 .PHONY: lib-release-test lib-release lib-profile lib-doc
 .PHONY: lib-dep lib-flake8 lib-mypy lib-speedscope container-buildx-all-platforms
 .PHONY: container container-run container-release container-build container-buildx
-.PHONY: devtools dashboard dashboard-clean
+.PHONY: devtools dashboard dashboard-clean container-without-openssl
 
 all: lib-test
 
@@ -175,12 +175,15 @@ dashboard-clean:
 	if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi
 
 container: lib-package
-	$(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl)
+	docker build \
+		-t $(PROXYPY_CONTAINER_TAG) \
+		--build-arg PROXYPY_PKG_PATH=$$(ls dist/*.whl) .
 
-container-build:
+container-without-openssl: lib-package
 	docker build \
 		-t $(PROXYPY_CONTAINER_TAG) \
-		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
+		--build-arg SKIP_OPENSSL=1 \
+		--build-arg PROXYPY_PKG_PATH=$$(ls dist/*.whl) .
 
 # Usage:
 #
diff --git a/README.md b/README.md
index 4446ff21be..daade52a4b 100644
--- a/README.md
+++ b/README.md
@@ -12,15 +12,17 @@
 [![Android, Android Emulator](https://img.shields.io/static/v1?label=tested%20with&message=Android%20%F0%9F%93%B1%20%7C%20Android%20Emulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/)
 [![iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=iOS%20%F0%9F%93%B1%20%7C%20iOS%20Simulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/)
 
-[![pypi version](https://img.shields.io/pypi/v/proxy.py)](https://pypi.org/project/proxy.py/)
-[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue)](https://www.python.org/)
-[![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue)](http://mypy-lang.org/)
-[![lib](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml/badge.svg?branch=develop&event=push)](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml)
+[![pypi version](https://img.shields.io/pypi/v/proxy.py?style=flat-square)](https://pypi.org/project/proxy.py/)
+[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue&style=flat-square)](https://www.python.org/)
+[![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue&style=flat-square)](http://mypy-lang.org/)
+
+[![doc](https://img.shields.io/readthedocs/proxypy/latest?style=flat-square&color=darkgreen)](https://proxypy.readthedocs.io/)
 [![codecov](https://codecov.io/gh/abhinavsingh/proxy.py/branch/develop/graph/badge.svg?token=Zh9J7b4la2)](https://codecov.io/gh/abhinavsingh/proxy.py)
+[![lib](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml/badge.svg?branch=develop&event=push)](https://github.com/abhinavsingh/proxy.py/actions/workflows/test-library.yml)
 
-[![Contributions Welcome](https://img.shields.io/static/v1?label=Contributions&message=Welcome%20%F0%9F%91%8D&color=darkgreen)](https://github.com/abhinavsingh/proxy.py/issues)
-[![Need Help](https://img.shields.io/static/v1?label=Need%20Help%3F&message=Ask&color=darkgreen)](https://twitter.com/imoracle)
-[![Sponsored by Jaxl Innovations Private Limited](https://img.shields.io/static/v1?label=Sponsored%20By&message=Jaxl%20Innovations%20Private%20Limited&color=darkgreen)](https://github.com/jaxl-innovations-private-limited)
+[![Contributions Welcome](https://img.shields.io/static/v1?label=Contributions&message=Welcome%20%F0%9F%91%8D&color=darkgreen&style=flat-square)](https://github.com/abhinavsingh/proxy.py/issues)
+[![Need Help](https://img.shields.io/static/v1?label=Need%20Help%3F&message=Ask&color=darkgreen&style=flat-square)](https://twitter.com/imoracle)
+[![Sponsored by Jaxl Innovations Private Limited](https://img.shields.io/static/v1?label=Sponsored%20By&message=Jaxl%20Innovations%20Private%20Limited&color=darkgreen&style=flat-square)](https://github.com/jaxl-innovations-private-limited)
 
 # Table of Contents
 
@@ -93,6 +95,11 @@
   - [Inspect Traffic](#inspect-traffic)
 - [Chrome DevTools Protocol](#chrome-devtools-protocol)
 - [Frequently Asked Questions](#frequently-asked-questions)
+  - [Deploying proxy.py in production](#deploying-proxypy-in-production)
+    - [What not to do?](#what-not-to-do)
+    - [Via Requirements](#via-requirements)
+    - [Via Docker Container](#via-docker-container)
+    - [Integrate your CI/CD with proxy.py](#integrate-your-cicd-with-proxypy)
   - [Stable vs Develop](#stable-vs-develop)
     - [Release Schedule](#release-schedule)
   - [Threads vs Threadless](#threads-vs-threadless)
@@ -124,8 +131,12 @@
 # Features
 - Fast & Scalable
 
-  - Scales by using all available cores on the system
+  - Scale up by using all available cores on the system
+    - Use `--num-acceptors` flag to control number of cores
+
   - Threadless executions using asyncio
+    - Use `--threaded` for synchronous thread based execution mode
+
   - Made to handle `tens-of-thousands` connections / sec
 
     ```console
@@ -175,35 +186,40 @@
         [200] 100000 responses
     ```
 
-  - See [Benchmark](https://github.com/abhinavsingh/proxy.py/tree/develop/benchmark#readme) for more details and how to run them locally.
+    See [Benchmark](https://github.com/abhinavsingh/proxy.py/tree/develop/benchmark#readme) for more details and for how to run benchmarks locally.
 
 - Lightweight
-  - Uses `~5-20 MB` RAM
-  - Compressed containers size is `~18.04 MB`
+  - Uses only `~5-20 MB` RAM
+    - No memory leaks
+    - Start once and forget, no restarts required
+  - Compressed containers size is only `~25 MB`
   - No external dependency other than standard Python library
+
 - Programmable
   - Customize proxy behavior using [Proxy Server Plugins](#http-proxy-plugins). Example:
     - `--plugins proxy.plugin.ProxyPoolPlugin`
-  - Optionally, enable builtin [Web Server Plugins](#http-web-server-plugins). Example:
-    - `--plugins proxy.plugin.ReverseProxyPlugin`
-  - Plugin API is currently in development phase, expect breaking changes
+  - Optionally, enable builtin [Web Server](#http-web-server-plugins). Example:
+    - `--enable-web-server --plugins proxy.plugin.ReverseProxyPlugin`
+  - Plugin API is currently in *development phase*. Expect breaking changes. See [Deploying proxy.py in production](#deploying-proxypy-in-production) on how to ensure reliability across code changes.
 - Real-time Dashboard
   - Optionally, enable [proxy.py dashboard](#run-dashboard).
-    - Available at `http://localhost:8899/dashboard`.
+    - Use `--enable-dashboard`
+    - Then, visit `http://localhost:8899/dashboard`
   - [Inspect, Monitor, Control and Configure](#inspect-traffic) `proxy.py` at runtime
   - [Chrome DevTools Protocol](#chrome-devtools-protocol) support
-  - Extend dashboard using plugins
-  - Dashboard is currently in development phase, expect breaking changes
+  - Extend dashboard frontend using `typescript` based [plugins](https://github.com/abhinavsingh/proxy.py/tree/develop/dashboard/src/plugins)
+  - Dashboard is currently in *development phase*  Expect breaking changes.
 - Secure
   - Enable end-to-end encryption between clients and `proxy.py`
   - See [End-to-End Encryption](#end-to-end-encryption)
 - Private
-  - Everyone deserves privacy. Browse with malware and adult content protection
+  - Protection against DNS based traffic blockers
+  - Browse with malware and adult content protection enabled
   - See [DNS-over-HTTPS](#cloudflarednsresolverplugin)
 - Man-In-The-Middle
   - Can decrypt TLS traffic between clients and upstream servers
   - See [TLS Interception](#tls-interception)
-- Supported proxy protocols
+- Supported http protocols for proxy requests
   - `http(s)`
     - `http1`
     - `http1.1` with pipeline
@@ -226,6 +242,8 @@
 
 # Install
 
+Consult [Deploying proxy.py in production](#deploying-proxypy-in-production) when deploying production grade applications using `proxy.py`.
+
 ## Using PIP
 
 ### Stable Version with PIP
@@ -1738,6 +1756,96 @@ Now point your CDT instance to `ws://localhost:8899/devtools`.
 
 # Frequently Asked Questions
 
+## Deploying proxy.py in production
+
+Listed below are a few strategies for using `proxy.py` in your private/production/corporate projects.
+
+### What not to do?
+
+> You MUST `avoid forking` the repository *"just"* to put your plugin code in `proxy/plugin` directory.  Forking is recommended workflow for project contributors, NOT for project users.
+
+Instead, use one of the suggested approaches from below.  Then load your plugins using `--plugin`, `--plugins` flags or `plugin` kwargs.
+
+### Via Requirements
+
+It is *highly* recommended that you use `proxy.py` via `requirements.txt` or similar dependency management setups. This will allow you to take advantages of regular performance updates, bug fixes, security patches and other improvements happening in the `proxy.py` ecosystem.  Example:
+
+1. Use `--pre` option to depend upon last `pre-release`
+
+    ```console
+    ❯ pip install proxy.py --pre
+    ```
+
+    Pre-releases are similar to depending upon `develop` branch code, just that pre-releases may not point to the `HEAD`.  This could happen because pre-releases are NOT made available on `PyPi` after every PR merge.
+
+2. Use `TestPyPi` with `--pre` option to depend upon `develop` branch code
+
+    ```console
+    ❯ pip install -i https://test.pypi.org/simple/ proxy.py --pre
+    ```
+
+    A pre-release is made available on `TestPyPi` after every PR merge.
+
+3. Use last `stable` release code
+
+    As usual, simply use:
+
+    ```console
+    ❯ pip install proxy.py
+    ```
+
+### Via Docker Container
+
+If you are into deploying containers, then simply build your image from base `proxy.py` container images.
+
+1. Use `GHCR` to build from `develop` branch code:
+
+    ```console
+    FROM ghcr.io/abhinavsingh/proxy.py:latest as base
+    ```
+
+    *PS: I use GHCR latest for several production level projects*
+
+2. Use `DockerHub` to build from last `stable` release code:
+
+    ```console
+    FROM abhinavsingh/proxy.py:latest as base
+    ```
+
+PS: IMHO, container based strategy is *the best approach* and the only strategy that *I use myself*.
+
+### Integrate your CI/CD with proxy.py
+
+*Hey, but you keep making breaking changes in the develop branch.*
+
+I hear you.  And hence, for your production grade applications, you *MUST* integrate application CI/CD with `proxy.py`.  You must make sure that your application builds and passes its tests for every PR merge into the `proxy.py` upstream repo.
+
+If your application repository is public, in certain scenarios, PR authors may send patch PRs for all dependents to maintain backward incompatibility and green CI/CD.
+
+CI/CD integration ensure your app continues to build with latest `proxy.py` code.  Depending upon where you host your code, use the strategy listed below:
+
+- GitHub
+
+    TBD
+
+- Google Cloud Build
+
+    TBD
+
+- AWS
+
+    TBD
+
+- Azure
+
+    TBD
+
+- Others
+
+    TBD
+
+> At some stage, we'll deprecate `master` branch segregation and simply maintain a `develop` branch.  As dependents can maintain stability via CI/CD integrations. Currently, it's hard for a production grade project to blindly depend upon `develop` branch.
+
 ## Stable vs Develop
 
 - `master` branch contains latest `stable` code and is available via `PyPi` repository and `Docker` containers via `docker.io` and `ghcr.io` registries.
@@ -2123,7 +2231,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
-proxy.py v2.4.0b4.dev12+g19e6881.d20211221
+proxy.py v2.4.0rc5.dev11+ga872675.d20211225
 
 options:
   -h, --help            show this help message and exit
@@ -2140,7 +2248,7 @@ options:
                         handle each client connection.
   --num-workers NUM_WORKERS
                         Defaults to number of CPU cores.
-  --local-executor      Default: False. Disabled by default. When enabled
+  --local-executor      Default: True. Disabled by default. When enabled
                         acceptors will make use of local (same process)
                         executor instead of distributing load across remote
                         (other process) executors. Enable this option to
@@ -2149,7 +2257,7 @@ options:
                         algorithm.
   --backlog BACKLOG     Default: 100. Maximum number of pending connections to
                         proxy server
-  --hostname HOSTNAME   Default: ::1. Server IP address.
+  --hostname HOSTNAME   Default: 127.0.0.1. Server IP address.
   --port PORT           Default: 8899. Server port.
   --unix-socket-path UNIX_SOCKET_PATH
                         Default: None. Unix socket path to use. When provided
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 7c61e32077..404de054ed 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -13,7 +13,9 @@ faq
 html
 http
 https
+integrations
 iterable
+pre
 readables
 scm
 sexualized
diff --git a/proxy/common/constants.py b/proxy/common/constants.py
index 80d12ac29d..0db89afd16 100644
--- a/proxy/common/constants.py
+++ b/proxy/common/constants.py
@@ -120,7 +120,7 @@ def _env_threadless_compliant() -> bool:
 DEFAULT_STATIC_SERVER_DIR = os.path.join(PROXY_PY_DIR, "public")
 DEFAULT_MIN_COMPRESSION_LIMIT = 20  # In bytes
 DEFAULT_THREADLESS = _env_threadless_compliant()
-DEFAULT_LOCAL_EXECUTOR = False
+DEFAULT_LOCAL_EXECUTOR = True
 DEFAULT_TIMEOUT = 10.0
 DEFAULT_VERSION = False
 DEFAULT_HTTP_PORT = 80
diff --git a/proxy/core/acceptor/listener.py b/proxy/core/acceptor/listener.py
index ac3b211ff5..c137502db0 100644
--- a/proxy/core/acceptor/listener.py
+++ b/proxy/core/acceptor/listener.py
@@ -20,7 +20,7 @@
 from typing import Optional, Any
 
 from ...common.flag import flags
-from ...common.constants import DEFAULT_BACKLOG, DEFAULT_IPV6_HOSTNAME, DEFAULT_PORT
+from ...common.constants import DEFAULT_BACKLOG, DEFAULT_IPV4_HOSTNAME, DEFAULT_PORT
 
 
 flags.add_argument(
@@ -33,8 +33,8 @@
 flags.add_argument(
     '--hostname',
     type=str,
-    default=str(DEFAULT_IPV6_HOSTNAME),
-    help='Default: ::1. Server IP address.',
+    default=str(DEFAULT_IPV4_HOSTNAME),
+    help='Default: 127.0.0.1. Server IP address.',
 )
 
 flags.add_argument(
diff --git a/tests/core/test_acceptor.py b/tests/core/test_acceptor.py
index a923a97357..69b6ecfba0 100644
--- a/tests/core/test_acceptor.py
+++ b/tests/core/test_acceptor.py
@@ -26,6 +26,7 @@ def setUp(self) -> None:
         self.flags = FlagParser.initialize(
             threaded=True,
             work_klass=mock.MagicMock(),
+            local_executor=False,
         )
         self.acceptor = Acceptor(
             idd=self.acceptor_id,
@@ -93,7 +94,7 @@ def test_accepts_client_from_server_socket(
         mock_recv_handle.assert_called_with(self.pipe[1])
         mock_fromfd.assert_called_with(
             fileno,
-            family=socket.AF_INET6,
+            family=socket.AF_INET,
             type=socket.SOCK_STREAM,
         )
         self.flags.work_klass.assert_called_with(
diff --git a/tests/test_main.py b/tests/test_main.py
index e39cbb3100..22b9feb0dd 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -228,7 +228,8 @@ def test_enable_dashboard(
         mock_args = mock_parse_args.return_value
         self.mock_default_args(mock_args)
         mock_args.enable_dashboard = True
-        main(enable_dashboard=True)
+        mock_args.local_executor = False
+        main(enable_dashboard=True, local_executor=False)
         mock_load_plugins.assert_called()
         self.assertEqual(
             mock_load_plugins.call_args_list[0][0][0], [
@@ -273,7 +274,8 @@ def test_enable_devtools(
         mock_args = mock_parse_args.return_value
         self.mock_default_args(mock_args)
         mock_args.enable_devtools = True
-        main(enable_devtools=True)
+        mock_args.local_executor = False
+        main(enable_devtools=True, local_executor=False)
         mock_load_plugins.assert_called()
         self.assertEqual(
             mock_load_plugins.call_args_list[0][0][0], [

From f921f568036c940e0bbfae8b142619ec90a0eb32 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Sun, 26 Dec 2021 20:51:31 +0530
Subject: [PATCH 09/22] `Work` can also be `TcpServerConnection`, not just
 `TcpClientConnection` (#906)

* Work can also be TcpServerConnection, not just TcpClientConnection.  More over, it can be any generic work type

* Add py_class_role and py_obj

* Port internal integration tests into public repo

* Fix proxy py addr

* Use cross-platform compat shasum
---
 docs/conf.py                          |  2 +
 examples/web_scraper.py               |  3 +-
 proxy/core/acceptor/executors.py      |  2 +-
 proxy/core/acceptor/threadless.py     |  4 +-
 proxy/core/acceptor/work.py           |  9 +--
 proxy/core/base/tcp_server.py         |  3 +-
 tests/integration/test_integration.sh | 91 ++++++++++++++++++++-------
 7 files changed, 83 insertions(+), 31 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index e71c4723a9..8172d57f55 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -293,6 +293,7 @@
     (_py_class_role, 'proxy.core.pool.AcceptorPool'),
     (_py_class_role, 'proxy.core.executors.ThreadlessPool'),
     (_py_class_role, 'proxy.core.acceptor.threadless.T'),
+    (_py_class_role, 'proxy.core.acceptor.work.T'),
     (_py_class_role, 'queue.Queue[Any]'),
     (_py_class_role, 'TcpClientConnection'),
     (_py_class_role, 'TcpServerConnection'),
@@ -303,4 +304,5 @@
     (_py_class_role, 'WebsocketFrame'),
     (_py_class_role, 'Work'),
     (_py_obj_role, 'proxy.core.acceptor.threadless.T'),
+    (_py_obj_role, 'proxy.core.acceptor.work.T'),
 ]
diff --git a/examples/web_scraper.py b/examples/web_scraper.py
index 4b925876c5..b3dae1aa2d 100644
--- a/examples/web_scraper.py
+++ b/examples/web_scraper.py
@@ -14,10 +14,11 @@
 
 from proxy import Proxy
 from proxy.core.acceptor import Work
+from proxy.core.connection import TcpClientConnection
 from proxy.common.types import Readables, Writables
 
 
-class WebScraper(Work):
+class WebScraper(Work[TcpClientConnection]):
     """Demonstrates how to orchestrate a generic work acceptors and executors
     workflow using proxy.py core.
 
diff --git a/proxy/core/acceptor/executors.py b/proxy/core/acceptor/executors.py
index e4e165b97b..9512762671 100644
--- a/proxy/core/acceptor/executors.py
+++ b/proxy/core/acceptor/executors.py
@@ -125,7 +125,7 @@ def start_threaded_work(
             addr: Optional[Tuple[str, int]],
             event_queue: Optional[EventQueue] = None,
             publisher_id: Optional[str] = None,
-    ) -> Tuple[Work, threading.Thread]:
+    ) -> Tuple[Work[TcpClientConnection], threading.Thread]:
         """Utility method to start a work in a new thread."""
         work = flags.work_klass(
             TcpClientConnection(conn, addr),
diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py
index cfd2f12086..5140c9345b 100644
--- a/proxy/core/acceptor/threadless.py
+++ b/proxy/core/acceptor/threadless.py
@@ -18,7 +18,7 @@
 import multiprocessing
 
 from abc import abstractmethod, ABC
-from typing import Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union
+from typing import Any, Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union
 
 from ...common.logger import Logger
 from ...common.types import Readables, Writables
@@ -71,7 +71,7 @@ def __init__(
         self.event_queue = event_queue
 
         self.running = multiprocessing.Event()
-        self.works: Dict[int, Work] = {}
+        self.works: Dict[int, Work[Any]] = {}
         self.selector: Optional[selectors.DefaultSelector] = None
         # If we remove single quotes for typing hint below,
         # runtime exceptions will occur for < Python 3.9.
diff --git a/proxy/core/acceptor/work.py b/proxy/core/acceptor/work.py
index 0152e1b2a6..5a7ba0723d 100644
--- a/proxy/core/acceptor/work.py
+++ b/proxy/core/acceptor/work.py
@@ -16,19 +16,20 @@
 
 from abc import ABC, abstractmethod
 from uuid import uuid4
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, TypeVar, Generic
 
 from ..event import eventNames, EventQueue
-from ..connection import TcpClientConnection
 from ...common.types import Readables, Writables
 
+T = TypeVar('T')
 
-class Work(ABC):
+
+class Work(ABC, Generic[T]):
     """Implement Work to hook into the event loop provided by Threadless process."""
 
     def __init__(
             self,
-            work: TcpClientConnection,
+            work: T,
             flags: argparse.Namespace,
             event_queue: Optional[EventQueue] = None,
             uid: Optional[str] = None,
diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py
index 236d5b764e..80f795e36f 100644
--- a/proxy/core/base/tcp_server.py
+++ b/proxy/core/base/tcp_server.py
@@ -19,12 +19,13 @@
 from typing import Dict, Any, Optional
 
 from ...core.acceptor import Work
+from ...core.connection import TcpClientConnection
 from ...common.types import Readables, Writables
 
 logger = logging.getLogger(__name__)
 
 
-class BaseTcpServerHandler(Work):
+class BaseTcpServerHandler(Work[TcpClientConnection]):
     """BaseTcpServerHandler implements Work interface.
 
     BaseTcpServerHandler lifecycle is controlled by Threadless core
diff --git a/tests/integration/test_integration.sh b/tests/integration/test_integration.sh
index b31b8452bf..64cc3ae007 100755
--- a/tests/integration/test_integration.sh
+++ b/tests/integration/test_integration.sh
@@ -15,6 +15,8 @@ if [[ -z "$PROXY_PY_PORT" ]]; then
   exit 1
 fi
 
+PROXY_URL="127.0.0.1:$PROXY_PY_PORT"
+
 # Wait for server to come up
 WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '"
 while true; do
@@ -31,8 +33,8 @@ while true; do
     curl -v \
         --max-time 1 \
         --connect-timeout 1 \
-        -x 127.0.0.1:$PROXY_PY_PORT \
-        http://127.0.0.1:$PROXY_PY_PORT/ 2>/dev/null
+        -x $PROXY_URL \
+        http://$PROXY_URL/ 2>/dev/null
     if [[ $? == 0 ]]; then
         break
     fi
@@ -40,6 +42,23 @@ while true; do
     sleep 1
 done
 
+verify_response() {
+    if [ "$1" == "" ];
+    then
+        echo "Empty response";
+        return 1;
+    else
+        if [ "$1" == "$2" ];
+        then
+            echo "Ok";
+            return 0;
+        else
+            echo "Invalid response: '$1', expected: '$2'";
+            return 1;
+        fi
+    fi;
+}
+
 # Check if proxy was started with integration
 # testing web server plugin.  If detected, use
 # internal web server for integration testing.
@@ -47,28 +66,56 @@ done
 # If integration testing plugin is not found,
 # detect if we have internet access.  If we do,
 # then use httpbin.org for integration testing.
-curl -v \
-    -x 127.0.0.1:$PROXY_PY_PORT \
-    http://httpbin.org/get
-if [[ $? != 0 ]]; then
-    echo "http request failed"
-    exit 1
-fi
+read -r -d '' ROBOTS_RESPONSE << EOM
+User-agent: *
+Disallow: /deny
+EOM
 
-curl -v \
-    -x 127.0.0.1:$PROXY_PY_PORT \
-    https://httpbin.org/get
-if [[ $? != 0 ]]; then
-    echo "https request failed"
-    exit 1
-fi
+echo "[Test HTTP Request via Proxy]"
+CMD="curl -v -x $PROXY_URL http://httpbin.org/robots.txt"
+RESPONSE=$($CMD 2> /dev/null)
+verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
+VERIFIED1=$?
+
+echo "[Test HTTPS Request via Proxy]"
+CMD="curl -v -x $PROXY_URL https://httpbin.org/robots.txt"
+RESPONSE=$($CMD 2> /dev/null)
+verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
+VERIFIED2=$?
 
+echo "[Test Internal Web Server via Proxy]"
 curl -v \
-    -x 127.0.0.1:$PROXY_PY_PORT \
-    http://127.0.0.1:$PROXY_PY_PORT/
-if [[ $? != 0 ]]; then
-    echo "http request to built in webserver failed"
-    exit 1
+    -x $PROXY_URL \
+    http://$PROXY_URL/
+VERIFIED3=$?
+
+SHASUM=sha256sum
+if [ "$(uname)" = "Darwin" ];
+then
+    SHASUM="shasum -a 256"
 fi
 
-exit 0
+echo "[Test Download File Hash Verifies 1]"
+touch downloaded.hash
+echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc  -" > downloaded.hash
+curl -vL \
+    -o downloaded.whl \
+    -x $PROXY_URL \
+    https://files.pythonhosted.org/packages/88/78/e642316313b1cd6396e4b85471a316e003eff968f29773e95ea191ea1d08/proxy.py-2.4.0rc4-py3-none-any.whl#sha256=3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc
+cat downloaded.whl | $SHASUM -c downloaded.hash
+VERIFIED4=$?
+rm downloaded.whl downloaded.hash
+
+echo "[Test Download File Hash Verifies 2]"
+touch downloaded.hash
+echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8  -" > downloaded.hash
+curl -vL \
+    -o downloaded.whl \
+    -x $PROXY_URL \
+    https://files.pythonhosted.org/packages/20/9a/e5d9ec41927401e41aea8af6d16e78b5e612bca4699d417f646a9610a076/Jinja2-3.0.3-py3-none-any.whl#sha256=077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8
+cat downloaded.whl | $SHASUM -c downloaded.hash
+VERIFIED5=$?
+rm downloaded.whl downloaded.hash
+
+EXIT_CODE=$(( $VERIFIED1 || $VERIFIED2 || $VERIFIED3 || $VERIFIED4 || $VERIFIED5 ))
+exit $EXIT_CODE

From 99856a6aa7d9878f6815a53af2c0ebd69d813ed8 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Mon, 27 Dec 2021 10:23:38 +0530
Subject: [PATCH 10/22] Change `--local-executor` flag semantics (#907)

* Convert `--local-executor` in an integer flag, defaults to 1 i.e. enabled, use 0 to disable

* Consider any value other than 1 as remote mode

* Use integer to disable local executor mode in integration tests

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix mypy errors

* Update flags in readme

* Check for type

* Remove pid file check for now

* Update `--local-executor` flag usage

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 .github/workflows/test-library.yml    |  3 ++-
 Dockerfile                            |  1 -
 Makefile                              |  2 --
 README.md                             | 20 +++++++++++---------
 benchmark/_proxy.py                   |  2 +-
 proxy/common/flag.py                  |  2 +-
 proxy/core/acceptor/acceptor.py       | 16 ++++++++--------
 proxy/core/acceptor/pool.py           | 12 +++++++++++-
 proxy/proxy.py                        | 12 ++++++------
 tests/core/test_acceptor.py           |  2 +-
 tests/integration/test_integration.py |  2 +-
 tests/test_main.py                    | 16 ++++++++--------
 12 files changed, 50 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml
index 38d9913cbb..46b01bb7c1 100644
--- a/.github/workflows/test-library.yml
+++ b/.github/workflows/test-library.yml
@@ -754,7 +754,8 @@ jobs:
         $CONTAINER_TAG
         --hostname 0.0.0.0
         --enable-web-server
-        --local-executor && ./tests/integration/test_integration.sh 8899
+        &&
+        ./tests/integration/test_integration.sh 8899
     - name: Push to GHCR
       run: >-
         REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
diff --git a/Dockerfile b/Dockerfile
index 1b99e7151f..dcfb3611b1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -32,5 +32,4 @@ EXPOSE 8899/tcp
 ENTRYPOINT [ "proxy" ]
 CMD [ \
   "--hostname=0.0.0.0" \
-  "--local-executor" \
   ]
diff --git a/Makefile b/Makefile
index 989b642d4c..f5fb25a8e1 100644
--- a/Makefile
+++ b/Makefile
@@ -143,7 +143,6 @@ lib-profile:
 			--num-workers 1 \
 			--enable-web-server \
 			--plugin proxy.plugin.WebServerPlugin \
-			--local-executor \
 			--backlog 65536 \
 			--open-file-limit 65536 \
 			--log-file /dev/null
@@ -160,7 +159,6 @@ lib-speedscope:
 			--num-workers 1 \
 			--enable-web-server \
 			--plugin proxy.plugin.WebServerPlugin \
-			--local-executor \
 			--backlog 65536 \
 			--open-file-limit 65536 \
 			--log-file /dev/null
diff --git a/README.md b/README.md
index daade52a4b..6050d85a2f 100644
--- a/README.md
+++ b/README.md
@@ -2204,8 +2204,9 @@ To run standalone benchmark for `proxy.py`, use the following command from repo
 ```console
 ❯ proxy -h
 usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
-          [--threaded] [--num-workers NUM_WORKERS] [--local-executor]
-          [--backlog BACKLOG] [--hostname HOSTNAME] [--port PORT]
+          [--threaded] [--num-workers NUM_WORKERS]
+          [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG]
+          [--hostname HOSTNAME] [--port PORT]
           [--unix-socket-path UNIX_SOCKET_PATH]
           [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL]
           [--log-file LOG_FILE] [--log-format LOG_FORMAT]
@@ -2248,13 +2249,14 @@ options:
                         handle each client connection.
   --num-workers NUM_WORKERS
                         Defaults to number of CPU cores.
-  --local-executor      Default: True. Disabled by default. When enabled
-                        acceptors will make use of local (same process)
-                        executor instead of distributing load across remote
-                        (other process) executors. Enable this option to
-                        achieve CPU affinity between acceptors and executors,
-                        instead of using underlying OS kernel scheduling
-                        algorithm.
+  --local-executor LOCAL_EXECUTOR
+                        Default: 1. Enabled by default. Use 0 to disable. When
+                        enabled acceptors will make use of local (same
+                        process) executor instead of distributing load across
+                        remote (other process) executors. Enable this option
+                        to achieve CPU affinity between acceptors and
+                        executors, instead of using underlying OS kernel
+                        scheduling algorithm.
   --backlog BACKLOG     Default: 100. Maximum number of pending connections to
                         proxy server
   --hostname HOSTNAME   Default: 127.0.0.1. Server IP address.
diff --git a/benchmark/_proxy.py b/benchmark/_proxy.py
index 363684f29f..838918ee5b 100644
--- a/benchmark/_proxy.py
+++ b/benchmark/_proxy.py
@@ -23,7 +23,7 @@
             enable_web_server=True,
             disable_proxy_server=False,
             num_acceptors=10,
-            local_executor=True,
+            local_executor=1,
             log_file='/dev/null',
     ) as _:
         while True:
diff --git a/proxy/common/flag.py b/proxy/common/flag.py
index 6161dbd0b6..f0c7f7c7fb 100644
--- a/proxy/common/flag.py
+++ b/proxy/common/flag.py
@@ -340,7 +340,7 @@ def initialize(
         )
         args.timeout = cast(int, opts.get('timeout', args.timeout))
         args.local_executor = cast(
-            bool,
+            int,
             opts.get(
                 'local_executor',
                 args.local_executor,
diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py
index c600f00766..8efd1f0bc4 100644
--- a/proxy/core/acceptor/acceptor.py
+++ b/proxy/core/acceptor/acceptor.py
@@ -40,11 +40,11 @@
 
 flags.add_argument(
     '--local-executor',
-    action='store_true',
-    default=DEFAULT_LOCAL_EXECUTOR,
-    help='Default: ' + ('True' if DEFAULT_LOCAL_EXECUTOR else 'False') + '.  ' +
-    'Disabled by default.  When enabled acceptors will make use of ' +
-    'local (same process) executor instead of distributing load across ' +
+    type=int,
+    default=int(DEFAULT_LOCAL_EXECUTOR),
+    help='Default: ' + ('1' if DEFAULT_LOCAL_EXECUTOR else '0') + '.  ' +
+    'Enabled by default.  Use 0 to disable.  When enabled acceptors ' +
+    'will make use of local (same process) executor instead of distributing load across ' +
     'remote (other process) executors.  Enable this option to achieve CPU affinity between ' +
     'acceptors and executors, instead of using underlying OS kernel scheduling algorithm.',
 )
@@ -149,7 +149,7 @@ def run_once(self) -> None:
                 if locked:
                     self.lock.release()
             for work in works:
-                if self.flags.local_executor:
+                if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR):
                     assert self._local_work_queue
                     self._local_work_queue.put(work)
                 else:
@@ -172,7 +172,7 @@ def run(self) -> None:
             type=socket.SOCK_STREAM,
         )
         try:
-            if self.flags.local_executor:
+            if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR):
                 self._start_local()
             self.selector.register(self.sock, selectors.EVENT_READ)
             while not self.running.is_set():
@@ -181,7 +181,7 @@ def run(self) -> None:
             pass
         finally:
             self.selector.unregister(self.sock)
-            if self.flags.local_executor:
+            if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR):
                 self._stop_local()
             self.sock.close()
             logger.debug('Acceptor#%d shutdown', self.idd)
diff --git a/proxy/core/acceptor/pool.py b/proxy/core/acceptor/pool.py
index 90e7c77560..a5ae95329a 100644
--- a/proxy/core/acceptor/pool.py
+++ b/proxy/core/acceptor/pool.py
@@ -98,7 +98,17 @@ def __exit__(self, *args: Any) -> None:
     def setup(self) -> None:
         """Setup acceptors."""
         self._start()
-        logger.info('Started %d acceptors' % self.flags.num_acceptors)
+        execution_mode = (
+            'threadless (local)'
+            if self.flags.local_executor
+            else 'threadless (remote)'
+        ) if self.flags.threadless else 'threaded'
+        logger.info(
+            'Started %d acceptors in %s mode' % (
+                self.flags.num_acceptors,
+                execution_mode,
+            ),
+        )
         # Send file descriptor to all acceptor processes.
         fd = self.listener.fileno()
         for index in range(self.flags.num_acceptors):
diff --git a/proxy/proxy.py b/proxy/proxy.py
index c35e9e0b4a..1f52fab4df 100644
--- a/proxy/proxy.py
+++ b/proxy/proxy.py
@@ -19,7 +19,7 @@
 from .core.event import EventManager
 from .common.utils import bytes_
 from .common.flag import FlagParser, flags
-from .common.constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL
+from .common.constants import DEFAULT_LOCAL_EXECUTOR, DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL
 from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PLUGINS, DEFAULT_VERSION
 from .common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_WORK_KLASS, DEFAULT_PID_FILE
 
@@ -205,19 +205,19 @@ def shutdown(self) -> None:
         self._delete_pid_file()
 
     def _write_pid_file(self) -> None:
-        if self.flags.pid_file is not None:
-            # NOTE: Multiple instances of proxy.py running on
-            # same host machine will currently result in overwriting the PID file
+        if self.flags.pid_file:
             with open(self.flags.pid_file, 'wb') as pid_file:
                 pid_file.write(bytes_(os.getpid()))
 
     def _delete_pid_file(self) -> None:
-        if self.flags.pid_file and os.path.exists(self.flags.pid_file):
+        if self.flags.pid_file \
+                and os.path.exists(self.flags.pid_file):
             os.remove(self.flags.pid_file)
 
     @property
     def remote_executors_enabled(self) -> bool:
-        return self.flags.threadless and not self.flags.local_executor
+        return self.flags.threadless \
+            and not (self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR))
 
 
 def main(**opts: Any) -> None:
diff --git a/tests/core/test_acceptor.py b/tests/core/test_acceptor.py
index 69b6ecfba0..2a4d089898 100644
--- a/tests/core/test_acceptor.py
+++ b/tests/core/test_acceptor.py
@@ -26,7 +26,7 @@ def setUp(self) -> None:
         self.flags = FlagParser.initialize(
             threaded=True,
             work_klass=mock.MagicMock(),
-            local_executor=False,
+            local_executor=0,
         )
         self.acceptor = Acceptor(
             idd=self.acceptor_id,
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
index bf7df0b889..8962350f27 100644
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -59,7 +59,7 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]:
     'proxy_py_subprocess',
     (
         ('--threadless'),
-        ('--threadless --local-executor'),
+        ('--threadless --local-executor 0'),
         ('--threaded'),
     ),
     indirect=True,
diff --git a/tests/test_main.py b/tests/test_main.py
index 22b9feb0dd..380ffd65b1 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -68,7 +68,7 @@ def mock_default_args(mock_args: mock.Mock) -> None:
         mock_args.enable_events = DEFAULT_ENABLE_EVENTS
         mock_args.enable_dashboard = DEFAULT_ENABLE_DASHBOARD
         mock_args.work_klass = DEFAULT_WORK_KLASS
-        mock_args.local_executor = DEFAULT_LOCAL_EXECUTOR
+        mock_args.local_executor = int(DEFAULT_LOCAL_EXECUTOR)
 
     @mock.patch('os.remove')
     @mock.patch('os.path.exists')
@@ -93,7 +93,7 @@ def test_entry_point(
     ) -> None:
         pid_file = os.path.join(tempfile.gettempdir(), 'pid')
         mock_sleep.side_effect = KeyboardInterrupt()
-        mock_initialize.return_value.local_executor = False
+        mock_initialize.return_value.local_executor = 0
         mock_initialize.return_value.enable_events = False
         mock_initialize.return_value.pid_file = pid_file
         entry_point()
@@ -141,7 +141,7 @@ def test_main_with_no_flags(
             mock_sleep: mock.Mock,
     ) -> None:
         mock_sleep.side_effect = KeyboardInterrupt()
-        mock_initialize.return_value.local_executor = False
+        mock_initialize.return_value.local_executor = 0
         mock_initialize.return_value.enable_events = False
         main()
         mock_event_manager.assert_not_called()
@@ -181,7 +181,7 @@ def test_enable_events(
             mock_sleep: mock.Mock,
     ) -> None:
         mock_sleep.side_effect = KeyboardInterrupt()
-        mock_initialize.return_value.local_executor = False
+        mock_initialize.return_value.local_executor = 0
         mock_initialize.return_value.enable_events = True
         main()
         mock_event_manager.assert_called_once()
@@ -228,8 +228,8 @@ def test_enable_dashboard(
         mock_args = mock_parse_args.return_value
         self.mock_default_args(mock_args)
         mock_args.enable_dashboard = True
-        mock_args.local_executor = False
-        main(enable_dashboard=True, local_executor=False)
+        mock_args.local_executor = 0
+        main(enable_dashboard=True, local_executor=0)
         mock_load_plugins.assert_called()
         self.assertEqual(
             mock_load_plugins.call_args_list[0][0][0], [
@@ -274,8 +274,8 @@ def test_enable_devtools(
         mock_args = mock_parse_args.return_value
         self.mock_default_args(mock_args)
         mock_args.enable_devtools = True
-        mock_args.local_executor = False
-        main(enable_devtools=True, local_executor=False)
+        mock_args.local_executor = 0
+        main(enable_devtools=True, local_executor=0)
         mock_load_plugins.assert_called()
         self.assertEqual(
             mock_load_plugins.call_args_list[0][0][0], [

From d4449b8644e75897719305f34193b34e258357e4 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Mon, 27 Dec 2021 14:28:01 +0530
Subject: [PATCH 11/22] Clean shutdown on `SIGINT`, `SIGHUP`, `SIGTERM`,
 `SIGQUIT` (#908)

* sys.exit on SIGINT, SIGHUP, SIGTERM

* Add todo for pending signal actions

* SIGHUP is win only

* Remove frametype signature as it causes lint issues and we are not using it anyways

* SIGQUIT is not on Win
---
 proxy/proxy.py | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/proxy/proxy.py b/proxy/proxy.py
index 1f52fab4df..43a90593af 100644
--- a/proxy/proxy.py
+++ b/proxy/proxy.py
@@ -11,6 +11,7 @@
 import os
 import sys
 import time
+import signal
 import logging
 
 from typing import List, Optional, Any
@@ -19,7 +20,7 @@
 from .core.event import EventManager
 from .common.utils import bytes_
 from .common.flag import FlagParser, flags
-from .common.constants import DEFAULT_LOCAL_EXECUTOR, DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL
+from .common.constants import DEFAULT_LOCAL_EXECUTOR, DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, IS_WINDOWS
 from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PLUGINS, DEFAULT_VERSION
 from .common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_WORK_KLASS, DEFAULT_PID_FILE
 
@@ -190,6 +191,7 @@ def setup(self) -> None:
         )
         self.acceptors.setup()
         # TODO: May be close listener fd as we don't need it now
+        self._register_signals()
 
     def shutdown(self) -> None:
         assert self.acceptors
@@ -204,6 +206,11 @@ def shutdown(self) -> None:
         self.listener.shutdown()
         self._delete_pid_file()
 
+    @property
+    def remote_executors_enabled(self) -> bool:
+        return self.flags.threadless and \
+            not (self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR))
+
     def _write_pid_file(self) -> None:
         if self.flags.pid_file:
             with open(self.flags.pid_file, 'wb') as pid_file:
@@ -214,19 +221,32 @@ def _delete_pid_file(self) -> None:
                 and os.path.exists(self.flags.pid_file):
             os.remove(self.flags.pid_file)
 
-    @property
-    def remote_executors_enabled(self) -> bool:
-        return self.flags.threadless \
-            and not (self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR))
+    def _register_signals(self) -> None:
+        # TODO: Handle SIGINFO, SIGUSR1, SIGUSR2
+        signal.signal(signal.SIGINT, self._handle_exit_signal)
+        signal.signal(signal.SIGTERM, self._handle_exit_signal)
+        if not IS_WINDOWS:
+            signal.signal(signal.SIGHUP, self._handle_exit_signal)
+            # TODO: SIGQUIT is ideally meant for terminate with core dumps
+            signal.signal(signal.SIGQUIT, self._handle_exit_signal)
+
+    @staticmethod
+    def _handle_exit_signal(signum: int, _frame: Any) -> None:
+        logger.info('Received signal %d' % signum)
+        sys.exit(0)
+
+
+def sleep_loop() -> None:
+    while True:
+        try:
+            time.sleep(1)
+        except KeyboardInterrupt:
+            break
 
 
 def main(**opts: Any) -> None:
     with Proxy(sys.argv[1:], **opts):
-        while True:
-            try:
-                time.sleep(1)
-            except KeyboardInterrupt:
-                break
+        sleep_loop()
 
 
 def entry_point() -> None:

From 8e3d724cd6b684fcadca336d6b59fe1794c6f72e Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Tue, 28 Dec 2021 10:27:09 +0530
Subject: [PATCH 12/22] Fix `HttpWebServerPacFilePlugin` broken routes logic
 (#915)

* Fix `HttpWebServerPacFilePlugin` broken routes logic

* lint
---
 proxy/http/server/pac_plugin.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/proxy/http/server/pac_plugin.py b/proxy/http/server/pac_plugin.py
index 2e6e233872..befe483a9b 100644
--- a/proxy/http/server/pac_plugin.py
+++ b/proxy/http/server/pac_plugin.py
@@ -52,8 +52,16 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
     def routes(self) -> List[Tuple[int, str]]:
         if self.flags.pac_file_url_path:
             return [
-                (httpProtocolTypes.HTTP, text_(self.flags.pac_file_url_path)),
-                (httpProtocolTypes.HTTPS, text_(self.flags.pac_file_url_path)),
+                (
+                    httpProtocolTypes.HTTP, r'{0}$'.format(
+                        text_(self.flags.pac_file_url_path),
+                    ),
+                ),
+                (
+                    httpProtocolTypes.HTTPS, r'{0}$'.format(
+                        text_(self.flags.pac_file_url_path),
+                    ),
+                ),
             ]
         return []   # pragma: no cover
 

From ea662808271841f1db7ac79015fef836d785816b Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Tue, 28 Dec 2021 11:47:34 +0530
Subject: [PATCH 13/22] Proxy Auto-Configuration (PAC) file should not be
 compressed (#916)

---
 proxy/http/responses.py         | 2 +-
 proxy/http/server/pac_plugin.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/proxy/http/responses.py b/proxy/http/responses.py
index 9e865cd274..65b548b3bf 100644
--- a/proxy/http/responses.py
+++ b/proxy/http/responses.py
@@ -90,7 +90,7 @@ def okResponse(
         **kwargs: Any,
 ) -> memoryview:
     do_compress: bool = False
-    if compress and flags.args and content and len(content) > flags.args.min_compression_limit:
+    if flags.args and compress and content and len(content) > flags.args.min_compression_limit:
         do_compress = True
         if not headers:
             headers = {}
diff --git a/proxy/http/server/pac_plugin.py b/proxy/http/server/pac_plugin.py
index befe483a9b..8cc0a4d0f0 100644
--- a/proxy/http/server/pac_plugin.py
+++ b/proxy/http/server/pac_plugin.py
@@ -80,7 +80,7 @@ def cache_pac_file_response(self) -> None:
                 content=content,
                 headers={
                     b'Content-Type': b'application/x-ns-proxy-autoconfig',
-                    b'Content-Encoding': b'gzip',
                 },
                 conn_close=True,
+                compress=False,
             )

From f6214a46c9822e47acbf0bd1dc94afc02036eb26 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Tue, 28 Dec 2021 13:51:20 +0530
Subject: [PATCH 14/22] Move `UpstreamConnectionPool` lifecycle within
 `Threadless` (#917)

* Tie connection pool into Threadless

* Pass upstream conn pool reference to work instances

* Mark upstream conn pool as optional

* spellcheck

* Fix unused import
---
 docs/conf.py                      |  1 +
 proxy/core/acceptor/executors.py  |  1 +
 proxy/core/acceptor/threadless.py |  6 +++++-
 proxy/core/acceptor/work.py       |  7 ++++++-
 proxy/core/connection/__init__.py |  4 ++--
 proxy/core/connection/pool.py     | 19 ++++++++++++++-----
 proxy/http/handler.py             |  1 +
 proxy/http/plugin.py              |  7 ++++++-
 proxy/http/proxy/server.py        | 18 +++++++++---------
 proxy/plugin/proxy_pool.py        |  2 +-
 tests/core/test_acceptor.py       |  1 +
 tests/core/test_conn_pool.py      |  6 +++---
 12 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 8172d57f55..402e263b2d 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -300,6 +300,7 @@
     (_py_class_role, 'unittest.case.TestCase'),
     (_py_class_role, 'unittest.result.TestResult'),
     (_py_class_role, 'UUID'),
+    (_py_class_role, 'UpstreamConnectionPool'),
     (_py_class_role, 'Url'),
     (_py_class_role, 'WebsocketFrame'),
     (_py_class_role, 'Work'),
diff --git a/proxy/core/acceptor/executors.py b/proxy/core/acceptor/executors.py
index 9512762671..d0c5a912c3 100644
--- a/proxy/core/acceptor/executors.py
+++ b/proxy/core/acceptor/executors.py
@@ -131,6 +131,7 @@ def start_threaded_work(
             TcpClientConnection(conn, addr),
             flags=flags,
             event_queue=event_queue,
+            upstream_conn_pool=None,
         )
         # TODO: Keep reference to threads and join during shutdown.
         # This will ensure connections are not abruptly closed on shutdown
diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py
index 5140c9345b..9858712adc 100644
--- a/proxy/core/acceptor/threadless.py
+++ b/proxy/core/acceptor/threadless.py
@@ -25,7 +25,7 @@
 from ...common.constants import DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT
 from ...common.constants import DEFAULT_WAIT_FOR_TASKS_TIMEOUT
 
-from ..connection import TcpClientConnection
+from ..connection import TcpClientConnection, UpstreamConnectionPool
 from ..event import eventNames, EventQueue
 
 from .work import Work
@@ -87,6 +87,9 @@ def __init__(
         self.wait_timeout: float = DEFAULT_WAIT_FOR_TASKS_TIMEOUT
         self.cleanup_inactive_timeout: float = DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT
         self._total: int = 0
+        self._upstream_conn_pool: Optional[UpstreamConnectionPool] = None
+        if self.flags.enable_conn_pool:
+            self._upstream_conn_pool = UpstreamConnectionPool()
 
     @property
     @abstractmethod
@@ -134,6 +137,7 @@ def work_on_tcp_conn(
             flags=self.flags,
             event_queue=self.event_queue,
             uid=uid,
+            upstream_conn_pool=self._upstream_conn_pool,
         )
         self.works[fileno].publish_event(
             event_name=eventNames.WORK_STARTED,
diff --git a/proxy/core/acceptor/work.py b/proxy/core/acceptor/work.py
index 5a7ba0723d..37f0bf591b 100644
--- a/proxy/core/acceptor/work.py
+++ b/proxy/core/acceptor/work.py
@@ -16,11 +16,14 @@
 
 from abc import ABC, abstractmethod
 from uuid import uuid4
-from typing import Optional, Dict, Any, TypeVar, Generic
+from typing import Optional, Dict, Any, TypeVar, Generic, TYPE_CHECKING
 
 from ..event import eventNames, EventQueue
 from ...common.types import Readables, Writables
 
+if TYPE_CHECKING:
+    from ..connection import UpstreamConnectionPool
+
 T = TypeVar('T')
 
 
@@ -33,6 +36,7 @@ def __init__(
             flags: argparse.Namespace,
             event_queue: Optional[EventQueue] = None,
             uid: Optional[str] = None,
+            upstream_conn_pool: Optional['UpstreamConnectionPool'] = None,
     ) -> None:
         # Work uuid
         self.uid: str = uid if uid is not None else uuid4().hex
@@ -41,6 +45,7 @@ def __init__(
         self.event_queue = event_queue
         # Accept work
         self.work = work
+        self.upstream_conn_pool = upstream_conn_pool
 
     @abstractmethod
     async def get_events(self) -> Dict[int, int]:
diff --git a/proxy/core/connection/__init__.py b/proxy/core/connection/__init__.py
index 952ee08f9e..58d100a81b 100644
--- a/proxy/core/connection/__init__.py
+++ b/proxy/core/connection/__init__.py
@@ -16,7 +16,7 @@
 from .connection import TcpConnection, TcpConnectionUninitializedException
 from .client import TcpClientConnection
 from .server import TcpServerConnection
-from .pool import ConnectionPool
+from .pool import UpstreamConnectionPool
 from .types import tcpConnectionTypes
 
 __all__ = [
@@ -25,5 +25,5 @@
     'TcpServerConnection',
     'TcpClientConnection',
     'tcpConnectionTypes',
-    'ConnectionPool',
+    'UpstreamConnectionPool',
 ]
diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py
index 16cd5096b1..5f92066b9d 100644
--- a/proxy/core/connection/pool.py
+++ b/proxy/core/connection/pool.py
@@ -17,6 +17,9 @@
 from typing import Set, Dict, Tuple
 
 from ...common.flag import flags
+from ...common.types import Readables, Writables
+
+from ..acceptor.work import Work
 
 from .server import TcpServerConnection
 
@@ -31,10 +34,10 @@
 )
 
 
-class ConnectionPool:
+class UpstreamConnectionPool(Work[TcpServerConnection]):
     """Manages connection pool to upstream servers.
 
-    `ConnectionPool` avoids need to reconnect with the upstream
+    `UpstreamConnectionPool` avoids need to reconnect with the upstream
     servers repeatedly when a reusable connection is available
     in the pool.
 
@@ -47,16 +50,16 @@ class ConnectionPool:
     the pool users.  Example, if acquired connection
     is stale, reacquire.
 
-    TODO: Ideally, ConnectionPool must be shared across
+    TODO: Ideally, `UpstreamConnectionPool` must be shared across
     all cores to make SSL session cache to also work
     without additional out-of-bound synchronizations.
 
-    TODO: ConnectionPool currently WON'T work for
+    TODO: `UpstreamConnectionPool` currently WON'T work for
     HTTPS connection. This is because of missing support for
     session cache, session ticket, abbr TLS handshake
     and other necessary features to make it work.
 
-    NOTE: However, for all HTTP only connections, ConnectionPool
+    NOTE: However, for all HTTP only connections, `UpstreamConnectionPool`
     can be used to save upon connection setup time and
     speed-up performance of requests.
     """
@@ -113,3 +116,9 @@ def release(self, conn: TcpServerConnection) -> None:
             assert not conn.is_reusable()
             # Reset for reusability
             conn.reset()
+
+    async def get_events(self) -> Dict[int, int]:
+        return await super().get_events()
+
+    async def handle_events(self, readables: Readables, writables: Writables) -> bool:
+        return await super().handle_events(readables, writables)
diff --git a/proxy/http/handler.py b/proxy/http/handler.py
index 38429df7de..ae6c0d66ca 100644
--- a/proxy/http/handler.py
+++ b/proxy/http/handler.py
@@ -100,6 +100,7 @@ def initialize(self) -> None:
                     self.work,
                     self.request,
                     self.event_queue,
+                    self.upstream_conn_pool,
                 )
                 self.plugins[instance.name()] = instance
         logger.debug('Handling connection %r' % self.work.connection)
diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py
index eafcd0539d..0180e5f9d9 100644
--- a/proxy/http/plugin.py
+++ b/proxy/http/plugin.py
@@ -12,7 +12,7 @@
 import argparse
 
 from abc import ABC, abstractmethod
-from typing import Tuple, List, Union, Optional
+from typing import Tuple, List, Union, Optional, TYPE_CHECKING
 
 from .parser import HttpParser
 
@@ -20,6 +20,9 @@
 from ..core.event import EventQueue
 from ..core.connection import TcpClientConnection
 
+if TYPE_CHECKING:
+    from ..core.connection import UpstreamConnectionPool
+
 
 class HttpProtocolHandlerPlugin(ABC):
     """Base HttpProtocolHandler Plugin class.
@@ -50,12 +53,14 @@ def __init__(
             client: TcpClientConnection,
             request: HttpParser,
             event_queue: EventQueue,
+            upstream_conn_pool: Optional['UpstreamConnectionPool'] = None,
     ):
         self.uid: str = uid
         self.flags: argparse.Namespace = flags
         self.client: TcpClientConnection = client
         self.request: HttpParser = request
         self.event_queue = event_queue
+        self.upstream_conn_pool = upstream_conn_pool
         super().__init__()
 
     def name(self) -> str:
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index 55604abf63..228d8ecc34 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -44,7 +44,7 @@
 from ...common.pki import gen_public_key, gen_csr, sign_csr
 
 from ...core.event import eventNames
-from ...core.connection import TcpServerConnection, ConnectionPool
+from ...core.connection import TcpServerConnection
 from ...core.connection import TcpConnectionUninitializedException
 from ...common.flag import flags
 
@@ -140,9 +140,6 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
     # connection pool operations.
     lock = threading.Lock()
 
-    # Shared connection pool
-    pool = ConnectionPool()
-
     def __init__(
             self,
             *args: Any, **kwargs: Any,
@@ -200,10 +197,10 @@ def get_descriptors(self) -> Tuple[List[int], List[int]]:
 
     def _close_and_release(self) -> bool:
         if self.flags.enable_conn_pool:
-            assert self.upstream and not self.upstream.closed
+            assert self.upstream and not self.upstream.closed and self.upstream_conn_pool
             self.upstream.closed = True
             with self.lock:
-                self.pool.release(self.upstream)
+                self.upstream_conn_pool.release(self.upstream)
             self.upstream = None
         return True
 
@@ -391,9 +388,10 @@ def on_client_connection_close(self) -> None:
             return
 
         if self.flags.enable_conn_pool:
+            assert self.upstream_conn_pool
             # Release the connection for reusability
             with self.lock:
-                self.pool.release(self.upstream)
+                self.upstream_conn_pool.release(self.upstream)
             return
 
         try:
@@ -589,8 +587,9 @@ def connect_upstream(self) -> None:
         host, port = self.request.host, self.request.port
         if host and port:
             if self.flags.enable_conn_pool:
+                assert self.upstream_conn_pool
                 with self.lock:
-                    created, self.upstream = self.pool.acquire(
+                    created, self.upstream = self.upstream_conn_pool.acquire(
                         text_(host), port,
                     )
             else:
@@ -642,8 +641,9 @@ def connect_upstream(self) -> None:
                     ),
                 )
                 if self.flags.enable_conn_pool:
+                    assert self.upstream_conn_pool
                     with self.lock:
-                        self.pool.release(self.upstream)
+                        self.upstream_conn_pool.release(self.upstream)
                 raise ProxyConnectionFailed(
                     text_(host), port, repr(e),
                 ) from e
diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py
index cfc8017820..641d95d622 100644
--- a/proxy/plugin/proxy_pool.py
+++ b/proxy/plugin/proxy_pool.py
@@ -88,7 +88,7 @@ def before_upstream_connection(
         must be bootstrapped within it's own re-usable and garbage collected pool,
         to avoid establishing a new upstream proxy connection for each client request.
 
-        See :class:`~proxy.core.connection.pool.ConnectionPool` which is a work
+        See :class:`~proxy.core.connection.pool.UpstreamConnectionPool` which is a work
         in progress for SSL cache handling.
         """
         # We don't want to send private IP requests to remote proxies
diff --git a/tests/core/test_acceptor.py b/tests/core/test_acceptor.py
index 2a4d089898..89bbce46aa 100644
--- a/tests/core/test_acceptor.py
+++ b/tests/core/test_acceptor.py
@@ -101,6 +101,7 @@ def test_accepts_client_from_server_socket(
             mock_client.return_value,
             flags=self.flags,
             event_queue=None,
+            upstream_conn_pool=None,
         )
         mock_thread.assert_called_with(
             target=self.flags.work_klass.return_value.run,
diff --git a/tests/core/test_conn_pool.py b/tests/core/test_conn_pool.py
index 3eaad052f3..db3de3d7c0 100644
--- a/tests/core/test_conn_pool.py
+++ b/tests/core/test_conn_pool.py
@@ -12,14 +12,14 @@
 
 from unittest import mock
 
-from proxy.core.connection import ConnectionPool
+from proxy.core.connection import UpstreamConnectionPool
 
 
 class TestConnectionPool(unittest.TestCase):
 
     @mock.patch('proxy.core.connection.pool.TcpServerConnection')
     def test_acquire_and_release_and_reacquire(self, mock_tcp_server_connection: mock.Mock) -> None:
-        pool = ConnectionPool()
+        pool = UpstreamConnectionPool()
         addr = ('localhost', 1234)
         # Mock
         mock_conn = mock_tcp_server_connection.return_value
@@ -50,7 +50,7 @@ def test_acquire_and_release_and_reacquire(self, mock_tcp_server_connection: moc
     def test_closed_connections_are_removed_on_release(
             self, mock_tcp_server_connection: mock.Mock,
     ) -> None:
-        pool = ConnectionPool()
+        pool = UpstreamConnectionPool()
         addr = ('localhost', 1234)
         # Mock
         mock_conn = mock_tcp_server_connection.return_value

From 263c067301cd28c36de6ac902d3d950b41d4be4d Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Wed, 29 Dec 2021 00:30:32 +0530
Subject: [PATCH 15/22] Define work lifecycle events for pool (#918)

* Define work lifecycle events for pool

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Use isinstance

* Use mocker fixture to pass CI on 3.6 and 3.7

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 proxy/core/connection/pool.py   | 45 +++++++++++++++--------
 proxy/core/connection/server.py |  2 +-
 proxy/http/proxy/server.py      | 64 ++++++++++++++++-----------------
 tests/core/test_conn_pool.py    | 49 ++++++++++++++++++++++---
 4 files changed, 109 insertions(+), 51 deletions(-)

diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py
index 5f92066b9d..a9b0585a01 100644
--- a/proxy/core/connection/pool.py
+++ b/proxy/core/connection/pool.py
@@ -13,8 +13,9 @@
        reusability
 """
 import logging
+import selectors
 
-from typing import Set, Dict, Tuple
+from typing import TYPE_CHECKING, Set, Dict, Tuple
 
 from ...common.flag import flags
 from ...common.types import Readables, Writables
@@ -66,11 +67,21 @@ class UpstreamConnectionPool(Work[TcpServerConnection]):
 
     def __init__(self) -> None:
         # Pools of connection per upstream server
+        self.connections: Dict[int, TcpServerConnection] = {}
         self.pools: Dict[Tuple[str, int], Set[TcpServerConnection]] = {}
 
-    def acquire(self, host: str, port: int) -> Tuple[bool, TcpServerConnection]:
+    def add(self, addr: Tuple[str, int]) -> TcpServerConnection:
+        # Create new connection
+        new_conn = TcpServerConnection(addr[0], addr[1])
+        new_conn.connect()
+        if addr not in self.pools:
+            self.pools[addr] = set()
+        self.pools[addr].add(new_conn)
+        self.connections[new_conn.connection.fileno()] = new_conn
+        return new_conn
+
+    def acquire(self, addr: Tuple[str, int]) -> Tuple[bool, TcpServerConnection]:
         """Returns a connection for use with the server."""
-        addr = (host, port)
         # Return a reusable connection if available
         if addr in self.pools:
             for old_conn in self.pools[addr]:
@@ -78,18 +89,14 @@ def acquire(self, host: str, port: int) -> Tuple[bool, TcpServerConnection]:
                     old_conn.mark_inuse()
                     logger.debug(
                         'Reusing connection#{2} for upstream {0}:{1}'.format(
-                            host, port, id(old_conn),
+                            addr[0], addr[1], id(old_conn),
                         ),
                     )
                     return False, old_conn
-        # Create new connection
-        new_conn = TcpServerConnection(*addr)
-        if addr not in self.pools:
-            self.pools[addr] = set()
-        self.pools[addr].add(new_conn)
+        new_conn = self.add(addr)
         logger.debug(
             'Created new connection#{2} for upstream {0}:{1}'.format(
-                host, port, id(new_conn),
+                addr[0], addr[1], id(new_conn),
             ),
         )
         return True, new_conn
@@ -118,7 +125,17 @@ def release(self, conn: TcpServerConnection) -> None:
             conn.reset()
 
     async def get_events(self) -> Dict[int, int]:
-        return await super().get_events()
-
-    async def handle_events(self, readables: Readables, writables: Writables) -> bool:
-        return await super().handle_events(readables, writables)
+        events = {}
+        for connections in self.pools.values():
+            for conn in connections:
+                events[conn.connection.fileno()] = selectors.EVENT_READ
+        return events
+
+    async def handle_events(self, readables: Readables, _writables: Writables) -> bool:
+        for r in readables:
+            if TYPE_CHECKING:
+                assert isinstance(r, int)
+            conn = self.connections[r]
+            self.pools[conn.addr].remove(conn)
+            del self.connections[r]
+        return False
diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py
index 7aae5371cc..2c2af73f99 100644
--- a/proxy/core/connection/server.py
+++ b/proxy/core/connection/server.py
@@ -25,7 +25,7 @@ class TcpServerConnection(TcpConnection):
     def __init__(self, host: str, port: int) -> None:
         super().__init__(tcpConnectionTypes.SERVER)
         self._conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None
-        self.addr: Tuple[str, int] = (host, int(port))
+        self.addr: Tuple[str, int] = (host, port)
         self.closed = True
 
     @property
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index 228d8ecc34..fdb74d7176 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -586,29 +586,6 @@ def handle_pipeline_response(self, raw: memoryview) -> None:
     def connect_upstream(self) -> None:
         host, port = self.request.host, self.request.port
         if host and port:
-            if self.flags.enable_conn_pool:
-                assert self.upstream_conn_pool
-                with self.lock:
-                    created, self.upstream = self.upstream_conn_pool.acquire(
-                        text_(host), port,
-                    )
-            else:
-                created, self.upstream = True, TcpServerConnection(
-                    text_(host), port,
-                )
-            if not created:
-                # NOTE: Acquired connection might be in an unusable state.
-                #
-                # This can only be confirmed by reading from connection.
-                # For stale connections, we will receive None, indicating
-                # to drop the connection.
-                #
-                # If that happen, we must acquire a fresh connection.
-                logger.info(
-                    'Reusing connection to upstream %s:%d' %
-                    (text_(host), port),
-                )
-                return
             try:
                 logger.debug(
                     'Connecting to upstream %s:%d' %
@@ -622,14 +599,37 @@ def connect_upstream(self) -> None:
                     )
                     if upstream_ip or source_addr:
                         break
-                # Connect with overridden upstream IP and source address
-                # if any of the plugin returned a non-null value.
-                self.upstream.connect(
-                    addr=None if not upstream_ip else (
-                        upstream_ip, port,
-                    ), source_address=source_addr,
-                )
-                self.upstream.connection.setblocking(False)
+                if self.flags.enable_conn_pool:
+                    assert self.upstream_conn_pool
+                    with self.lock:
+                        created, self.upstream = self.upstream_conn_pool.acquire(
+                            (text_(host), port),
+                        )
+                else:
+                    created, self.upstream = True, TcpServerConnection(
+                        text_(host), port,
+                    )
+                    # Connect with overridden upstream IP and source address
+                    # if any of the plugin returned a non-null value.
+                    self.upstream.connect(
+                        addr=None if not upstream_ip else (
+                            upstream_ip, port,
+                        ), source_address=source_addr,
+                    )
+                    self.upstream.connection.setblocking(False)
+                if not created:
+                    # NOTE: Acquired connection might be in an unusable state.
+                    #
+                    # This can only be confirmed by reading from connection.
+                    # For stale connections, we will receive None, indicating
+                    # to drop the connection.
+                    #
+                    # If that happen, we must acquire a fresh connection.
+                    logger.info(
+                        'Reusing connection to upstream %s:%d' %
+                        (text_(host), port),
+                    )
+                    return
                 logger.debug(
                     'Connected to upstream %s:%s' %
                     (text_(host), port),
@@ -640,7 +640,7 @@ def connect_upstream(self) -> None:
                         text_(host), port, str(e),
                     ),
                 )
-                if self.flags.enable_conn_pool:
+                if self.flags.enable_conn_pool and self.upstream:
                     assert self.upstream_conn_pool
                     with self.lock:
                         self.upstream_conn_pool.release(self.upstream)
diff --git a/tests/core/test_conn_pool.py b/tests/core/test_conn_pool.py
index db3de3d7c0..e00436cd2f 100644
--- a/tests/core/test_conn_pool.py
+++ b/tests/core/test_conn_pool.py
@@ -8,9 +8,12 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+import pytest
 import unittest
+import selectors
 
 from unittest import mock
+from pytest_mock import MockerFixture
 
 from proxy.core.connection import UpstreamConnectionPool
 
@@ -28,7 +31,7 @@ def test_acquire_and_release_and_reacquire(self, mock_tcp_server_connection: moc
         ]
         mock_conn.closed = False
         # Acquire
-        created, conn = pool.acquire(*addr)
+        created, conn = pool.acquire(addr)
         self.assertTrue(created)
         mock_tcp_server_connection.assert_called_once_with(*addr)
         self.assertEqual(conn, mock_conn)
@@ -39,7 +42,7 @@ def test_acquire_and_release_and_reacquire(self, mock_tcp_server_connection: moc
         self.assertEqual(len(pool.pools[addr]), 1)
         self.assertTrue(conn in pool.pools[addr])
         # Reacquire
-        created, conn = pool.acquire(*addr)
+        created, conn = pool.acquire(addr)
         self.assertFalse(created)
         mock_conn.reset.assert_called_once()
         self.assertEqual(conn, mock_conn)
@@ -57,7 +60,7 @@ def test_closed_connections_are_removed_on_release(
         mock_conn.closed = True
         mock_conn.addr = addr
         # Acquire
-        created, conn = pool.acquire(*addr)
+        created, conn = pool.acquire(addr)
         self.assertTrue(created)
         mock_tcp_server_connection.assert_called_once_with(*addr)
         self.assertEqual(conn, mock_conn)
@@ -67,7 +70,45 @@ def test_closed_connections_are_removed_on_release(
         pool.release(conn)
         self.assertEqual(len(pool.pools[addr]), 0)
         # Acquire
-        created, conn = pool.acquire(*addr)
+        created, conn = pool.acquire(addr)
         self.assertTrue(created)
         self.assertEqual(mock_tcp_server_connection.call_count, 2)
         mock_conn.is_reusable.assert_not_called()
+
+
+class TestConnectionPoolAsync:
+
+    @pytest.mark.asyncio    # type: ignore[misc]
+    async def test_get_events(self, mocker: MockerFixture) -> None:
+        mock_tcp_server_connection = mocker.patch(
+            'proxy.core.connection.pool.TcpServerConnection',
+        )
+        pool = UpstreamConnectionPool()
+        addr = ('localhost', 1234)
+        mock_conn = mock_tcp_server_connection.return_value
+        pool.add(addr)
+        mock_tcp_server_connection.assert_called_once_with(*addr)
+        mock_conn.connect.assert_called_once()
+        events = await pool.get_events()
+        print(events)
+        assert events == {
+            mock_conn.connection.fileno.return_value: selectors.EVENT_READ,
+        }
+        assert pool.pools[addr].pop() == mock_conn
+        assert len(pool.pools[addr]) == 0
+        assert pool.connections[mock_conn.connection.fileno.return_value] == mock_conn
+
+    @pytest.mark.asyncio    # type: ignore[misc]
+    async def test_handle_events(self, mocker: MockerFixture) -> None:
+        mock_tcp_server_connection = mocker.patch(
+            'proxy.core.connection.pool.TcpServerConnection',
+        )
+        pool = UpstreamConnectionPool()
+        mock_conn = mock_tcp_server_connection.return_value
+        addr = mock_conn.addr
+        pool.add(addr)
+        assert len(pool.pools[addr]) == 1
+        assert len(pool.connections) == 1
+        await pool.handle_events([mock_conn.connection.fileno.return_value], [])
+        assert len(pool.pools[addr]) == 0
+        assert len(pool.connections) == 0

From 46c942f9477936a36b77aa6dd505d1f5e8e79550 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Wed, 29 Dec 2021 17:37:15 +0530
Subject: [PATCH 16/22] Hook `UpstreamConnectionPool` lifecycle within
 `Threadless` (#921)

* Hook connection pool lifecycle within threadless

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix test

* Fix spell

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 proxy/core/acceptor/threadless.py | 80 +++++++++++++++++++++++--------
 proxy/core/connection/pool.py     | 78 ++++++++++++++++++++----------
 proxy/core/connection/server.py   | 10 ++--
 proxy/http/handler.py             | 11 ++---
 proxy/http/parser/parser.py       |  2 +-
 proxy/http/proxy/server.py        | 14 +++---
 proxy/plugin/proxy_pool.py        |  2 +-
 tests/core/test_conn_pool.py      | 15 +++---
 tests/core/test_connection.py     |  5 +-
 9 files changed, 142 insertions(+), 75 deletions(-)

diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py
index 9858712adc..fa10721617 100644
--- a/proxy/core/acceptor/threadless.py
+++ b/proxy/core/acceptor/threadless.py
@@ -88,6 +88,7 @@ def __init__(
         self.cleanup_inactive_timeout: float = DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT
         self._total: int = 0
         self._upstream_conn_pool: Optional[UpstreamConnectionPool] = None
+        self._upstream_conn_filenos: Set[int] = set()
         if self.flags.enable_conn_pool:
             self._upstream_conn_pool = UpstreamConnectionPool()
 
@@ -176,14 +177,25 @@ async def _update_work_events(self, work_id: int) -> None:
                         data=work_id,
                     )
                     self.registered_events_by_work_ids[work_id][fileno] = mask
-                    # logger.debug(
-                    #     'fd#{0} modified for mask#{1} by work#{2}'.format(
-                    #         fileno, mask, work_id,
-                    #     ),
-                    # )
+                    logger.debug(
+                        'fd#{0} modified for mask#{1} by work#{2}'.format(
+                            fileno, mask, work_id,
+                        ),
+                    )
                 # else:
                 #     logger.info(
                 #         'fd#{0} by work#{1} not modified'.format(fileno, work_id))
+            elif fileno in self._upstream_conn_filenos:
+                # Descriptor offered by work, but is already registered by connection pool
+                # Most likely because work has acquired a reusable connection.
+                self.selector.modify(fileno, events=mask, data=work_id)
+                self.registered_events_by_work_ids[work_id][fileno] = mask
+                self._upstream_conn_filenos.remove(fileno)
+                logger.debug(
+                    'fd#{0} borrowed with mask#{1} by work#{2}'.format(
+                        fileno, mask, work_id,
+                    ),
+                )
             # Can throw ValueError: Invalid file descriptor: -1
             #
             # A guard within Work classes may not help here due to
@@ -193,16 +205,33 @@ async def _update_work_events(self, work_id: int) -> None:
             #
             # TODO: Also remove offending work from pool to avoid spin loop.
             elif fileno != -1:
-                self.selector.register(
-                    fileno, events=mask,
-                    data=work_id,
-                )
+                self.selector.register(fileno, events=mask, data=work_id)
                 self.registered_events_by_work_ids[work_id][fileno] = mask
-                # logger.debug(
-                #     'fd#{0} registered for mask#{1} by work#{2}'.format(
-                #         fileno, mask, work_id,
-                #     ),
-                # )
+                logger.debug(
+                    'fd#{0} registered for mask#{1} by work#{2}'.format(
+                        fileno, mask, work_id,
+                    ),
+                )
+
+    async def _update_conn_pool_events(self) -> None:
+        if not self._upstream_conn_pool:
+            return
+        assert self.selector is not None
+        new_conn_pool_events = await self._upstream_conn_pool.get_events()
+        old_conn_pool_filenos = self._upstream_conn_filenos.copy()
+        self._upstream_conn_filenos.clear()
+        new_conn_pool_filenos = set(new_conn_pool_events.keys())
+        new_conn_pool_filenos.difference_update(old_conn_pool_filenos)
+        for fileno in new_conn_pool_filenos:
+            self.selector.register(
+                fileno,
+                events=new_conn_pool_events[fileno],
+                data=0,
+            )
+            self._upstream_conn_filenos.add(fileno)
+        old_conn_pool_filenos.difference_update(self._upstream_conn_filenos)
+        for fileno in old_conn_pool_filenos:
+            self.selector.unregister(fileno)
 
     async def _update_selector(self) -> None:
         assert self.selector is not None
@@ -215,6 +244,7 @@ async def _update_selector(self) -> None:
             if work_id in unfinished_work_ids:
                 continue
             await self._update_work_events(work_id)
+        await self._update_conn_pool_events()
 
     async def _selected_events(self) -> Tuple[
             Dict[int, Tuple[Readables, Writables]],
@@ -235,9 +265,6 @@ async def _selected_events(self) -> Tuple[
         """
         assert self.selector is not None
         await self._update_selector()
-        events = self.selector.select(
-            timeout=DEFAULT_SELECTOR_SELECT_TIMEOUT,
-        )
         # Keys are work_id and values are 2-tuple indicating
         # readables & writables that work_id is interested in
         # and are ready for IO.
@@ -248,6 +275,11 @@ async def _selected_events(self) -> Tuple[
             # When ``work_queue_fileno`` returns None,
             # always return True for the boolean value.
             new_work_available = True
+
+        events = self.selector.select(
+            timeout=DEFAULT_SELECTOR_SELECT_TIMEOUT,
+        )
+
         for key, mask in events:
             if not new_work_available and wqfileno is not None and key.fileobj == wqfileno:
                 assert mask & selectors.EVENT_READ
@@ -302,9 +334,17 @@ def _create_tasks(
         assert self.loop
         tasks: Set['asyncio.Task[bool]'] = set()
         for work_id in work_by_ids:
-            task = self.loop.create_task(
-                self.works[work_id].handle_events(*work_by_ids[work_id]),
-            )
+            if work_id == 0:
+                assert self._upstream_conn_pool
+                task = self.loop.create_task(
+                    self._upstream_conn_pool.handle_events(
+                        *work_by_ids[work_id],
+                    ),
+                )
+            else:
+                task = self.loop.create_task(
+                    self.works[work_id].handle_events(*work_by_ids[work_id]),
+                )
             task._work_id = work_id     # type: ignore[attr-defined]
             # task.set_name(work_id)
             tasks.add(task)
diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py
index a9b0585a01..e51ce3fb5f 100644
--- a/proxy/core/connection/pool.py
+++ b/proxy/core/connection/pool.py
@@ -12,6 +12,7 @@
 
        reusability
 """
+import socket
 import logging
 import selectors
 
@@ -45,11 +46,19 @@ class UpstreamConnectionPool(Work[TcpServerConnection]):
     A separate pool is maintained for each upstream server.
     So internally, it's a pool of pools.
 
-    TODO: Listen for read events from the connections
-    to remove them from the pool when peer closes the
-    connection.  This can also be achieved lazily by
-    the pool users.  Example, if acquired connection
-    is stale, reacquire.
+    Internal data structure maintains references to connection objects
+    that pool owns or has borrowed.  Borrowed connections are marked as
+    NOT reusable.
+
+    For reusable connections only, pool listens for read events
+    to detect broken connections.  This can happen if pool has opened
+    a connection, which was never used and eventually reaches
+    upstream server timeout limit.
+
+    When a borrowed connection is returned back to the pool,
+    the connection is marked as reusable again.  However, if
+    returned connection has already been closed, it is removed
+    from the internal data structure.
 
     TODO: Ideally, `UpstreamConnectionPool` must be shared across
     all cores to make SSL session cache to also work
@@ -60,29 +69,25 @@ class UpstreamConnectionPool(Work[TcpServerConnection]):
     session cache, session ticket, abbr TLS handshake
     and other necessary features to make it work.
 
-    NOTE: However, for all HTTP only connections, `UpstreamConnectionPool`
-    can be used to save upon connection setup time and
-    speed-up performance of requests.
+    NOTE: However, currently for all HTTP only upstream connections,
+    `UpstreamConnectionPool` can be used to remove slow starts.
     """
 
     def __init__(self) -> None:
-        # Pools of connection per upstream server
         self.connections: Dict[int, TcpServerConnection] = {}
         self.pools: Dict[Tuple[str, int], Set[TcpServerConnection]] = {}
 
     def add(self, addr: Tuple[str, int]) -> TcpServerConnection:
-        # Create new connection
+        """Creates and add a new connection to the pool."""
         new_conn = TcpServerConnection(addr[0], addr[1])
         new_conn.connect()
-        if addr not in self.pools:
-            self.pools[addr] = set()
-        self.pools[addr].add(new_conn)
-        self.connections[new_conn.connection.fileno()] = new_conn
+        self._add(new_conn)
         return new_conn
 
     def acquire(self, addr: Tuple[str, int]) -> Tuple[bool, TcpServerConnection]:
-        """Returns a connection for use with the server."""
-        # Return a reusable connection if available
+        """Returns a reusable connection from the pool.
+
+        If none exists, will create and return a new connection."""
         if addr in self.pools:
             for old_conn in self.pools[addr]:
                 if old_conn.is_reusable():
@@ -102,40 +107,63 @@ def acquire(self, addr: Tuple[str, int]) -> Tuple[bool, TcpServerConnection]:
         return True, new_conn
 
     def release(self, conn: TcpServerConnection) -> None:
-        """Release the connection.
+        """Release a previously acquired connection.
 
         If the connection has not been closed,
         then it will be retained in the pool for reusability.
         """
+        assert not conn.is_reusable()
         if conn.closed:
             logger.debug(
                 'Removing connection#{2} from pool from upstream {0}:{1}'.format(
                     conn.addr[0], conn.addr[1], id(conn),
                 ),
             )
-            self.pools[conn.addr].remove(conn)
+            self._remove(conn.connection.fileno())
         else:
             logger.debug(
                 'Retaining connection#{2} to upstream {0}:{1}'.format(
                     conn.addr[0], conn.addr[1], id(conn),
                 ),
             )
-            assert not conn.is_reusable()
             # Reset for reusability
             conn.reset()
 
     async def get_events(self) -> Dict[int, int]:
+        """Returns read event flag for all reusable connections in the pool."""
         events = {}
         for connections in self.pools.values():
             for conn in connections:
-                events[conn.connection.fileno()] = selectors.EVENT_READ
+                if conn.is_reusable():
+                    events[conn.connection.fileno()] = selectors.EVENT_READ
         return events
 
     async def handle_events(self, readables: Readables, _writables: Writables) -> bool:
-        for r in readables:
+        """Removes reusable connection from the pool.
+
+        When pool is the owner of connection, we don't expect a read event from upstream
+        server.  A read event means either upstream closed the connection or connection
+        has somehow reached an illegal state e.g. upstream sending data for previous
+        connection acquisition lifecycle."""
+        for fileno in readables:
             if TYPE_CHECKING:
-                assert isinstance(r, int)
-            conn = self.connections[r]
-            self.pools[conn.addr].remove(conn)
-            del self.connections[r]
+                assert isinstance(fileno, int)
+            logger.debug('Upstream fd#{0} is read ready'.format(fileno))
+            self._remove(fileno)
         return False
+
+    def _add(self, conn: TcpServerConnection) -> None:
+        """Adds a new connection to internal data structure."""
+        if conn.addr not in self.pools:
+            self.pools[conn.addr] = set()
+        self.pools[conn.addr].add(conn)
+        self.connections[conn.connection.fileno()] = conn
+
+    def _remove(self, fileno: int) -> None:
+        """Remove a connection by descriptor from the internal data structure."""
+        conn = self.connections[fileno]
+        logger.debug('Removing conn#{0} from pool'.format(id(conn)))
+        conn.connection.shutdown(socket.SHUT_WR)
+        conn.close()
+        self.pools[conn.addr].remove(conn)
+        del self.connections[fileno]
diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py
index 2c2af73f99..109c238e30 100644
--- a/proxy/core/connection/server.py
+++ b/proxy/core/connection/server.py
@@ -39,11 +39,11 @@ def connect(
             addr: Optional[Tuple[str, int]] = None,
             source_address: Optional[Tuple[str, int]] = None,
     ) -> None:
-        if self._conn is None:
-            self._conn = new_socket_connection(
-                addr or self.addr, source_address=source_address,
-            )
-            self.closed = False
+        assert self._conn is None
+        self._conn = new_socket_connection(
+            addr or self.addr, source_address=source_address,
+        )
+        self.closed = False
 
     def wrap(self, hostname: str, ca_file: Optional[str]) -> None:
         ctx = ssl.create_default_context(
diff --git a/proxy/http/handler.py b/proxy/http/handler.py
index ae6c0d66ca..f46f594485 100644
--- a/proxy/http/handler.py
+++ b/proxy/http/handler.py
@@ -103,7 +103,7 @@ def initialize(self) -> None:
                     self.upstream_conn_pool,
                 )
                 self.plugins[instance.name()] = instance
-        logger.debug('Handling connection %r' % self.work.connection)
+        logger.debug('Handling connection %s' % self.work.address)
 
     def is_inactive(self) -> bool:
         if not self.work.has_buffer() and \
@@ -123,9 +123,8 @@ def shutdown(self) -> None:
             for plugin in self.plugins.values():
                 plugin.on_client_connection_close()
             logger.debug(
-                'Closing client connection %r '
-                'at address %s has buffer %s' %
-                (self.work.connection, self.work.address, self.work.has_buffer()),
+                'Closing client connection %s has buffer %s' %
+                (self.work.address, self.work.has_buffer()),
             )
             conn = self.work.connection
             # Unwrap if wrapped before shutdown.
@@ -247,7 +246,7 @@ def handle_data(self, data: memoryview) -> Optional[bool]:
 
     async def handle_writables(self, writables: Writables) -> bool:
         if self.work.connection.fileno() in writables and self.work.has_buffer():
-            logger.debug('Client is ready for writes, flushing buffer')
+            logger.debug('Client is write ready, flushing...')
             self.last_activity = time.time()
 
             # TODO(abhinavsingh): This hook could just reside within server recv block
@@ -277,7 +276,7 @@ async def handle_writables(self, writables: Writables) -> bool:
 
     async def handle_readables(self, readables: Readables) -> bool:
         if self.work.connection.fileno() in readables:
-            logger.debug('Client is ready for reads, reading')
+            logger.debug('Client is read ready, receiving...')
             self.last_activity = time.time()
             try:
                 teardown = await super().handle_readables(readables)
diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py
index 5ed707246a..197923caa1 100644
--- a/proxy/http/parser/parser.py
+++ b/proxy/http/parser/parser.py
@@ -77,7 +77,7 @@ def __init__(
         self.total_size: int = 0
         # Buffer to hold unprocessed bytes
         self.buffer: bytes = b''
-        # Internal headers datastructure:
+        # Internal headers data structure:
         # - Keys are lower case header names.
         # - Values are 2-tuple containing original
         #   header and it's value as received.
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index fdb74d7176..e6f66e1238 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -217,7 +217,7 @@ async def write_to_descriptors(self, w: Writables) -> bool:
                 self.upstream and not self.upstream.closed and \
                 self.upstream.has_buffer() and \
                 self.upstream.connection.fileno() in w:
-            logger.debug('Server is write ready, flushing buffer')
+            logger.debug('Server is write ready, flushing...')
             try:
                 self.upstream.flush()
             except ssl.SSLWantWriteError:
@@ -254,7 +254,7 @@ async def read_from_descriptors(self, r: Readables) -> bool:
                 and self.upstream \
                 and not self.upstream.closed \
                 and self.upstream.connection.fileno() in r:
-            logger.debug('Server is ready for reads, reading...')
+            logger.debug('Server is read ready, receiving...')
             try:
                 raw = self.upstream.recv(self.flags.server_recvbuf_size)
             except TimeoutError as e:
@@ -401,7 +401,7 @@ def on_client_connection_close(self) -> None:
                 pass
             finally:
                 # TODO: Unwrap if wrapped before close?
-                self.upstream.connection.close()
+                self.upstream.close()
         except TcpConnectionUninitializedException:
             pass
         finally:
@@ -587,10 +587,6 @@ def connect_upstream(self) -> None:
         host, port = self.request.host, self.request.port
         if host and port:
             try:
-                logger.debug(
-                    'Connecting to upstream %s:%d' %
-                    (text_(host), port),
-                )
                 # Invoke plugin.resolve_dns
                 upstream_ip, source_addr = None, None
                 for plugin in self.plugins.values():
@@ -599,6 +595,10 @@ def connect_upstream(self) -> None:
                     )
                     if upstream_ip or source_addr:
                         break
+                logger.debug(
+                    'Connecting to upstream %s:%d' %
+                    (text_(host), port),
+                )
                 if self.flags.enable_conn_pool:
                     assert self.upstream_conn_pool
                     with self.lock:
diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py
index 641d95d622..1ae13ed123 100644
--- a/proxy/plugin/proxy_pool.py
+++ b/proxy/plugin/proxy_pool.py
@@ -121,7 +121,7 @@ def before_upstream_connection(
             #
             # Failing upstream proxies, must be removed from the pool temporarily.
             # A periodic health check must put them back in the pool.  This can be achieved
-            # using a datastructure without having to spawn separate thread/process for health
+            # using a data structure without having to spawn separate thread/process for health
             # check.
             raise HttpProtocolException(
                 'Connection refused by upstream proxy {0}:{1}'.format(
diff --git a/tests/core/test_conn_pool.py b/tests/core/test_conn_pool.py
index e00436cd2f..ae2d2c718e 100644
--- a/tests/core/test_conn_pool.py
+++ b/tests/core/test_conn_pool.py
@@ -23,9 +23,9 @@ class TestConnectionPool(unittest.TestCase):
     @mock.patch('proxy.core.connection.pool.TcpServerConnection')
     def test_acquire_and_release_and_reacquire(self, mock_tcp_server_connection: mock.Mock) -> None:
         pool = UpstreamConnectionPool()
-        addr = ('localhost', 1234)
         # Mock
         mock_conn = mock_tcp_server_connection.return_value
+        addr = mock_conn.addr
         mock_conn.is_reusable.side_effect = [
             False, True, True,
         ]
@@ -33,7 +33,7 @@ def test_acquire_and_release_and_reacquire(self, mock_tcp_server_connection: moc
         # Acquire
         created, conn = pool.acquire(addr)
         self.assertTrue(created)
-        mock_tcp_server_connection.assert_called_once_with(*addr)
+        mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1])
         self.assertEqual(conn, mock_conn)
         self.assertEqual(len(pool.pools[addr]), 1)
         self.assertTrue(conn in pool.pools[addr])
@@ -54,26 +54,25 @@ def test_closed_connections_are_removed_on_release(
             self, mock_tcp_server_connection: mock.Mock,
     ) -> None:
         pool = UpstreamConnectionPool()
-        addr = ('localhost', 1234)
         # Mock
         mock_conn = mock_tcp_server_connection.return_value
         mock_conn.closed = True
-        mock_conn.addr = addr
+        addr = mock_conn.addr
         # Acquire
         created, conn = pool.acquire(addr)
         self.assertTrue(created)
-        mock_tcp_server_connection.assert_called_once_with(*addr)
+        mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1])
         self.assertEqual(conn, mock_conn)
         self.assertEqual(len(pool.pools[addr]), 1)
         self.assertTrue(conn in pool.pools[addr])
         # Release
+        mock_conn.is_reusable.return_value = False
         pool.release(conn)
         self.assertEqual(len(pool.pools[addr]), 0)
         # Acquire
         created, conn = pool.acquire(addr)
         self.assertTrue(created)
         self.assertEqual(mock_tcp_server_connection.call_count, 2)
-        mock_conn.is_reusable.assert_not_called()
 
 
 class TestConnectionPoolAsync:
@@ -84,10 +83,10 @@ async def test_get_events(self, mocker: MockerFixture) -> None:
             'proxy.core.connection.pool.TcpServerConnection',
         )
         pool = UpstreamConnectionPool()
-        addr = ('localhost', 1234)
         mock_conn = mock_tcp_server_connection.return_value
+        addr = mock_conn.addr
         pool.add(addr)
-        mock_tcp_server_connection.assert_called_once_with(*addr)
+        mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1])
         mock_conn.connect.assert_called_once()
         events = await pool.get_events()
         print(events)
diff --git a/tests/core/test_connection.py b/tests/core/test_connection.py
index 905ab56d2b..95bc000626 100644
--- a/tests/core/test_connection.py
+++ b/tests/core/test_connection.py
@@ -79,7 +79,7 @@ def testTcpServerEstablishesIPv6Connection(
         )
 
     @mock.patch('proxy.core.connection.server.new_socket_connection')
-    def testTcpServerIgnoresDoubleConnectSilently(
+    def testTcpServerWillNotIgnoreDoubleConnectAttemptsSilently(
             self,
             mock_new_socket_connection: mock.Mock,
     ) -> None:
@@ -87,7 +87,8 @@ def testTcpServerIgnoresDoubleConnectSilently(
             str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT,
         )
         conn.connect()
-        conn.connect()
+        with self.assertRaises(AssertionError):
+            conn.connect()
         mock_new_socket_connection.assert_called_once()
 
     @mock.patch('socket.socket')

From 48bdcc22a355fc02497d8b9ed5f7b0a5c048237b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 29 Dec 2021 22:29:07 +0530
Subject: [PATCH 17/22] pip prod(deps): bump sphinx from 4.3.1 to 4.3.2 (#902)

Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 4.3.1 to 4.3.2.
- [Release notes](https://github.com/sphinx-doc/sphinx/releases)
- [Changelog](https://github.com/sphinx-doc/sphinx/blob/4.x/CHANGES)
- [Commits](https://github.com/sphinx-doc/sphinx/compare/v4.3.1...v4.3.2)

---
updated-dependencies:
- dependency-name: sphinx
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
---
 docs/requirements.txt | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 7b12d2f2f0..4a87aa4f7d 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -31,7 +31,9 @@ charset-normalizer==2.0.7 \
 click==8.0.3 \
     --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \
     --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b
-    # via towncrier
+    # via
+    #   click-default-group
+    #   towncrier
 click-default-group==1.2.2 \
     --hash=sha256:d9560e8e8dfa44b3562fbc9425042a0fd6d21956fcc2db0077f63f34253ab904
     # via towncrier
@@ -53,6 +55,10 @@ imagesize==1.3.0 \
     --hash=sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c \
     --hash=sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d
     # via sphinx
+importlib-metadata==4.10.0 \
+    --hash=sha256:92a8b58ce734b2a4494878e0ecf7d79ccd7a128b5fc6014c401e0b61f006f0f6 \
+    --hash=sha256:b7cf7d3fef75f1e4c80a96ca660efbd51473d7e8f39b5ab9210febc7809012a4
+    # via click
 incremental==21.3.0 \
     --hash=sha256:02f5de5aff48f6b9f665d99d48bfc7ec03b6e3943210de7cfc88856d755d6f57 \
     --hash=sha256:92014aebc6a20b78a8084cdd5645eeaa7f74b8933f70fa3ada2cfbd1e3b54321
@@ -228,9 +234,9 @@ soupsieve==2.3.1 \
     --hash=sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb \
     --hash=sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9
     # via beautifulsoup4
-sphinx==4.3.1 \
-    --hash=sha256:048dac56039a5713f47a554589dc98a442b39226a2b9ed7f82797fcb2fe9253f \
-    --hash=sha256:32a5b3e9a1b176cc25ed048557d4d3d01af635e6b76c5bc7a43b0a34447fbd45
+sphinx==4.3.2 \
+    --hash=sha256:0a8836751a68306b3fe97ecbe44db786f8479c3bf4b80e3a7f5c838657b4698c \
+    --hash=sha256:6a11ea5dd0bdb197f9c2abc2e0ce73e01340464feaece525e64036546d24c851
     # via
     #   -r docs/requirements.in
     #   furo
@@ -281,6 +287,12 @@ towncrier==21.3.0 \
     --hash=sha256:6eed0bc924d72c98c000cb8a64de3bd566e5cb0d11032b73fcccf8a8f956ddfe \
     --hash=sha256:e6ccec65418bbcb8de5c908003e130e37fe0e9d6396cb77c1338241071edc082
     # via sphinxcontrib-towncrier
+typing-extensions==4.0.1 \
+    --hash=sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e \
+    --hash=sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b
+    # via
+    #   importlib-metadata
+    #   markdown-it-py
 uc-micro-py==1.0.1 \
     --hash=sha256:316cfb8b6862a0f1d03540f0ae6e7b033ff1fa0ddbe60c12cbe0d4cec846a69f \
     --hash=sha256:b7cdf4ea79433043ddfe2c82210208f26f7962c0cfbe3bacb05ee879a7fdb596
@@ -289,6 +301,10 @@ urllib3==1.26.7 \
     --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \
     --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844
     # via requests
+zipp==3.6.0 \
+    --hash=sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832 \
+    --hash=sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 setuptools==59.0.1 \

From 498a1bb84b4f8fc3fc56da379b1e9a5ea6976f48 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 29 Dec 2021 23:54:56 +0530
Subject: [PATCH 18/22] pip prod(deps): bump paramiko from 2.8.1 to 2.9.1
 (#923)

---
 requirements-tunnel.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-tunnel.txt b/requirements-tunnel.txt
index dca82fff04..dd57026e10 100644
--- a/requirements-tunnel.txt
+++ b/requirements-tunnel.txt
@@ -1,2 +1,2 @@
-paramiko==2.8.1
+paramiko==2.9.1
 types-paramiko==2.8.4

From d22d55116808bd62530966f4fdc6cb5c8db920b4 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Fri, 31 Dec 2021 15:58:36 +0530
Subject: [PATCH 19/22] Optimize how `HttpProtocolHandler` delegates to the
 core plugins (#925)

* Add `protocols` abstract static method to `HttpProtocolHandlerBase` which defines which HTTP specification is followed by the core plugin

* lint

* Fix tests

* Lint fixes

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 README.md                                     |   2 +-
 docs/conf.py                                  |   6 +-
 proxy/http/handler.py                         | 141 +++++++++++-------
 proxy/http/parser/parser.py                   |  10 +-
 proxy/http/plugin.py                          |  16 +-
 proxy/http/protocols.py                       |  33 ++++
 proxy/http/proxy/server.py                    |  27 ++--
 proxy/http/responses.py                       |  12 ++
 proxy/http/server/web.py                      |   9 +-
 tests/http/test_http_proxy.py                 |   4 +-
 .../http/test_http_proxy_tls_interception.py  |  67 +++++----
 tests/http/test_protocol_handler.py           |   2 +-
 tests/http/test_web_server.py                 |   3 +-
 13 files changed, 203 insertions(+), 129 deletions(-)
 create mode 100644 proxy/http/protocols.py

diff --git a/README.md b/README.md
index 6050d85a2f..e93f39b687 100644
--- a/README.md
+++ b/README.md
@@ -2232,7 +2232,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
-proxy.py v2.4.0rc5.dev11+ga872675.d20211225
+proxy.py v2.4.0rc5.dev26+gb2b1bdc.d20211230
 
 options:
   -h, --help            show this help message and exit
diff --git a/docs/conf.py b/docs/conf.py
index 402e263b2d..3b5c8bb0ba 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -219,7 +219,11 @@
 # -- Options for linkcheck builder -------------------------------------------
 
 linkcheck_ignore = [
-    r'http://localhost:\d+/',  # local URLs
+    # local URLs
+    r'http://localhost:\d+/',
+    # GHA sees "403 Client Error: Forbidden for url:"
+    # while the URL actually works
+    r'https://developers.cloudflare.com/',
 ]
 linkcheck_workers = 25
 
diff --git a/proxy/http/handler.py b/proxy/http/handler.py
index f46f594485..aede94746f 100644
--- a/proxy/http/handler.py
+++ b/proxy/http/handler.py
@@ -16,20 +16,21 @@
 import logging
 import selectors
 
-from typing import Tuple, List, Union, Optional, Dict, Any
+from typing import Tuple, List, Type, Union, Optional, Dict, Any
 
-from .plugin import HttpProtocolHandlerPlugin
-from .parser import HttpParser, httpParserStates, httpParserTypes
-from .exception import HttpProtocolException
-
-from ..common.types import Readables, Writables
+from ..common.flag import flags
 from ..common.utils import wrap_socket
 from ..core.base import BaseTcpServerHandler
+from ..common.types import Readables, Writables
 from ..core.connection import TcpClientConnection
-from ..common.flag import flags
 from ..common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_KEY_FILE
 from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT, DEFAULT_TIMEOUT
 
+from .exception import HttpProtocolException
+from .plugin import HttpProtocolHandlerPlugin
+from .responses import BAD_REQUEST_RESPONSE_PKT
+from .parser import HttpParser, httpParserStates, httpParserTypes
+
 
 logger = logging.getLogger(__name__)
 
@@ -78,7 +79,7 @@ def __init__(self, *args: Any, **kwargs: Any):
         self.selector: Optional[selectors.DefaultSelector] = None
         if not self.flags.threadless:
             self.selector = selectors.DefaultSelector()
-        self.plugins: Dict[str, HttpProtocolHandlerPlugin] = {}
+        self.plugin: Optional[HttpProtocolHandlerPlugin] = None
 
     ##
     # initialize, is_inactive, shutdown, get_events, handle_events
@@ -86,23 +87,16 @@ def __init__(self, *args: Any, **kwargs: Any):
     ##
 
     def initialize(self) -> None:
-        """Optionally upgrades connection to HTTPS, set ``conn`` in non-blocking mode and initializes plugins."""
+        """Optionally upgrades connection to HTTPS,
+        sets ``conn`` in non-blocking mode and initializes
+        HTTP protocol plugins.
+        """
         conn = self._optionally_wrap_socket(self.work.connection)
         conn.setblocking(False)
         # Update client connection reference if connection was wrapped
         if self._encryption_enabled():
             self.work = TcpClientConnection(conn=conn, addr=self.work.addr)
-        if b'HttpProtocolHandlerPlugin' in self.flags.plugins:
-            for klass in self.flags.plugins[b'HttpProtocolHandlerPlugin']:
-                instance: HttpProtocolHandlerPlugin = klass(
-                    self.uid,
-                    self.flags,
-                    self.work,
-                    self.request,
-                    self.event_queue,
-                    self.upstream_conn_pool,
-                )
-                self.plugins[instance.name()] = instance
+        # self._initialize_plugins()
         logger.debug('Handling connection %s' % self.work.address)
 
     def is_inactive(self) -> bool:
@@ -120,8 +114,8 @@ def shutdown(self) -> None:
             if self.selector and self.work.has_buffer():
                 self._flush()
             # Invoke plugin.on_client_connection_close
-            for plugin in self.plugins.values():
-                plugin.on_client_connection_close()
+            if self.plugin:
+                self.plugin.on_client_connection_close()
             logger.debug(
                 'Closing client connection %s has buffer %s' %
                 (self.work.address, self.work.has_buffer()),
@@ -153,8 +147,8 @@ async def get_events(self) -> Dict[int, int]:
         # Get default client events
         events: Dict[int, int] = await super().get_events()
         # HttpProtocolHandlerPlugin.get_descriptors
-        for plugin in self.plugins.values():
-            plugin_read_desc, plugin_write_desc = plugin.get_descriptors()
+        if self.plugin:
+            plugin_read_desc, plugin_write_desc = self.plugin.get_descriptors()
             for rfileno in plugin_read_desc:
                 if rfileno not in events:
                     events[rfileno] = selectors.EVENT_READ
@@ -179,8 +173,8 @@ async def handle_events(
         if teardown:
             return True
         # Invoke plugin.write_to_descriptors
-        for plugin in self.plugins.values():
-            teardown = await plugin.write_to_descriptors(writables)
+        if self.plugin:
+            teardown = await self.plugin.write_to_descriptors(writables)
             if teardown:
                 return True
         # Read from ready to read sockets
@@ -188,8 +182,8 @@ async def handle_events(
         if teardown:
             return True
         # Invoke plugin.read_from_descriptors
-        for plugin in self.plugins.values():
-            teardown = await plugin.read_from_descriptors(readables)
+        if self.plugin:
+            teardown = await self.plugin.read_from_descriptors(readables)
             if teardown:
                 return True
         return False
@@ -209,33 +203,13 @@ def handle_data(self, data: memoryview) -> Optional[bool]:
             # apply custom logic to handle request data sent after 1st
             # valid request.
             if self.request.state != httpParserStates.COMPLETE:
-                # Parse http request
-                #
-                # TODO(abhinavsingh): Remove .tobytes after parser is
-                # memoryview compliant
-                self.request.parse(data.tobytes())
-                if self.request.is_complete:
-                    # Invoke plugin.on_request_complete
-                    for plugin in self.plugins.values():
-                        upgraded_sock = plugin.on_request_complete()
-                        if isinstance(upgraded_sock, ssl.SSLSocket):
-                            logger.debug(
-                                'Updated client conn to %s', upgraded_sock,
-                            )
-                            self.work._conn = upgraded_sock
-                            for plugin_ in self.plugins.values():
-                                if plugin_ != plugin:
-                                    plugin_.client._conn = upgraded_sock
-                        elif isinstance(upgraded_sock, bool) and upgraded_sock is True:
-                            return True
+                if self._parse_first_request(data):
+                    return True
             else:
                 # HttpProtocolHandlerPlugin.on_client_data
                 # Can raise HttpProtocolException to tear down the connection
-                for plugin in self.plugins.values():
-                    optional_data = plugin.on_client_data(data)
-                    if optional_data is None:
-                        break
-                    data = optional_data
+                if self.plugin:
+                    data = self.plugin.on_client_data(data) or data
         except HttpProtocolException as e:
             logger.info('HttpProtocolException: %s' % e)
             response: Optional[memoryview] = e.response(self.request)
@@ -248,17 +222,13 @@ async def handle_writables(self, writables: Writables) -> bool:
         if self.work.connection.fileno() in writables and self.work.has_buffer():
             logger.debug('Client is write ready, flushing...')
             self.last_activity = time.time()
-
             # TODO(abhinavsingh): This hook could just reside within server recv block
             # instead of invoking when flushed to client.
             #
             # Invoke plugin.on_response_chunk
             chunk = self.work.buffer
-            for plugin in self.plugins.values():
-                chunk = plugin.on_response_chunk(chunk)
-                if chunk is None:
-                    break
-
+            if self.plugin:
+                chunk = self.plugin.on_response_chunk(chunk)
             try:
                 # Call super() for client flush
                 teardown = await super().handle_writables(writables)
@@ -305,6 +275,61 @@ async def handle_readables(self, readables: Readables) -> bool:
     # Internal methods
     ##
 
+    def _initialize_plugin(
+            self,
+            klass: Type['HttpProtocolHandlerPlugin'],
+    ) -> HttpProtocolHandlerPlugin:
+        """Initializes passed HTTP protocol handler plugin class."""
+        return klass(
+            self.uid,
+            self.flags,
+            self.work,
+            self.request,
+            self.event_queue,
+            self.upstream_conn_pool,
+        )
+
+    def _discover_plugin_klass(self, protocol: int) -> Optional[Type['HttpProtocolHandlerPlugin']]:
+        """Discovers and return matching HTTP handler plugin matching protocol."""
+        if b'HttpProtocolHandlerPlugin' in self.flags.plugins:
+            for klass in self.flags.plugins[b'HttpProtocolHandlerPlugin']:
+                k: Type['HttpProtocolHandlerPlugin'] = klass
+                if protocol in k.protocols():
+                    return k
+        return None
+
+    def _parse_first_request(self, data: memoryview) -> bool:
+        # Parse http request
+        #
+        # TODO(abhinavsingh): Remove .tobytes after parser is
+        # memoryview compliant
+        self.request.parse(data.tobytes())
+        if not self.request.is_complete:
+            return False
+        # Discover which HTTP handler plugin is capable of
+        # handling the current incoming request
+        klass = self._discover_plugin_klass(
+            self.request.http_handler_protocol,
+        )
+        if klass is None:
+            # No matching protocol class found.
+            # Return bad request response and
+            # close the connection.
+            self.work.queue(BAD_REQUEST_RESPONSE_PKT)
+            return True
+        assert klass is not None
+        self.plugin = self._initialize_plugin(klass)
+        # Invoke plugin.on_request_complete
+        output = self.plugin.on_request_complete()
+        if isinstance(output, bool):
+            return output
+        assert isinstance(output, ssl.SSLSocket)
+        logger.debug(
+            'Updated client conn to %s', output,
+        )
+        self.work._conn = output
+        return False
+
     def _encryption_enabled(self) -> bool:
         return self.flags.keyfile is not None and \
             self.flags.certfile is not None
diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py
index 197923caa1..ecb774ef13 100644
--- a/proxy/http/parser/parser.py
+++ b/proxy/http/parser/parser.py
@@ -22,6 +22,7 @@
 
 from ..url import Url
 from ..methods import httpMethods
+from ..protocols import httpProtocols
 from ..exception import HttpProtocolException
 
 from .protocol import ProxyProtocol
@@ -153,11 +154,10 @@ def set_url(self, url: bytes) -> None:
         self._url = Url.from_bytes(url)
         self._set_line_attributes()
 
-    def has_host(self) -> bool:
-        """Returns whether host line attribute was parsed or set.
-
-        NOTE: Host field WILL be None for incoming local WebServer requests."""
-        return self.host is not None
+    @property
+    def http_handler_protocol(self) -> int:
+        """Returns `HttpProtocols` that this request belongs to."""
+        return httpProtocols.HTTP_PROXY if self.host is not None else httpProtocols.WEB_SERVER
 
     @property
     def is_complete(self) -> bool:
diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py
index 0180e5f9d9..0b87d18c9c 100644
--- a/proxy/http/plugin.py
+++ b/proxy/http/plugin.py
@@ -14,12 +14,12 @@
 from abc import ABC, abstractmethod
 from typing import Tuple, List, Union, Optional, TYPE_CHECKING
 
-from .parser import HttpParser
-
 from ..common.types import Readables, Writables
 from ..core.event import EventQueue
 from ..core.connection import TcpClientConnection
 
+from .parser import HttpParser
+
 if TYPE_CHECKING:
     from ..core.connection import UpstreamConnectionPool
 
@@ -52,7 +52,7 @@ def __init__(
             flags: argparse.Namespace,
             client: TcpClientConnection,
             request: HttpParser,
-            event_queue: EventQueue,
+            event_queue: Optional[EventQueue],
             upstream_conn_pool: Optional['UpstreamConnectionPool'] = None,
     ):
         self.uid: str = uid
@@ -63,12 +63,10 @@ def __init__(
         self.upstream_conn_pool = upstream_conn_pool
         super().__init__()
 
-    def name(self) -> str:
-        """A unique name for your plugin.
-
-        Defaults to name of the class. This helps plugin developers to directly
-        access a specific plugin by its name."""
-        return self.__class__.__name__
+    @staticmethod
+    @abstractmethod
+    def protocols() -> List[int]:
+        raise NotImplementedError()
 
     @abstractmethod
     def get_descriptors(self) -> Tuple[List[int], List[int]]:
diff --git a/proxy/http/protocols.py b/proxy/http/protocols.py
new file mode 100644
index 0000000000..976a41852b
--- /dev/null
+++ b/proxy/http/protocols.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+
+    .. spelling::
+
+       http
+       iterable
+"""
+from typing import NamedTuple
+
+
+HttpProtocols = NamedTuple(
+    'HttpProtocols', [
+        # Web server handling HTTP/1.0, HTTP/1.1, HTTP/2, HTTP/3
+        # over plain Text or encrypted connection with clients
+        ('WEB_SERVER', int),
+        # Proxies handling HTTP/1.0, HTTP/1.1, HTTP/2 protocols
+        # over plain text connection or encrypted connection
+        # with clients
+        ('HTTP_PROXY', int),
+        # Proxies handling SOCKS4, SOCKS4a, SOCKS5 protocol
+        ('SOCKS_PROXY', int),
+    ],
+)
+
+httpProtocols = HttpProtocols(1, 2, 3)
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
index e6f66e1238..31b6a074c9 100644
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@@ -28,6 +28,7 @@
 
 from ..headers import httpHeaders
 from ..methods import httpMethods
+from ..protocols import httpProtocols
 from ..plugin import HttpProtocolHandlerPlugin
 from ..exception import HttpProtocolException, ProxyConnectionFailed
 from ..parser import HttpParser, httpParserStates, httpParserTypes
@@ -162,6 +163,10 @@ def __init__(
                 )
                 self.plugins[instance.name()] = instance
 
+    @staticmethod
+    def protocols() -> List[int]:
+        return [httpProtocols.HTTP_PROXY]
+
     def tls_interception_enabled(self) -> bool:
         return self.flags.ca_key_file is not None and \
             self.flags.ca_cert_dir is not None and \
@@ -169,8 +174,6 @@ def tls_interception_enabled(self) -> bool:
             self.flags.ca_cert_file is not None
 
     def get_descriptors(self) -> Tuple[List[int], List[int]]:
-        if not self.request.has_host():
-            return [], []
         r: List[int] = []
         w: List[int] = []
         if (
@@ -213,8 +216,7 @@ async def write_to_descriptors(self, w: Writables) -> bool:
                 teardown = plugin.write_to_descriptors(w)
                 if teardown:
                     return True
-        elif self.request.has_host() and \
-                self.upstream and not self.upstream.closed and \
+        elif self.upstream and not self.upstream.closed and \
                 self.upstream.has_buffer() and \
                 self.upstream.connection.fileno() in w:
             logger.debug('Server is write ready, flushing...')
@@ -250,8 +252,7 @@ async def read_from_descriptors(self, r: Readables) -> bool:
                 teardown = plugin.read_from_descriptors(r)
                 if teardown:
                     return True
-        elif self.request.has_host() \
-                and self.upstream \
+        elif self.upstream \
                 and not self.upstream.closed \
                 and self.upstream.connection.fileno() in r:
             logger.debug('Server is read ready, receiving...')
@@ -315,9 +316,6 @@ async def read_from_descriptors(self, r: Readables) -> bool:
         return False
 
     def on_client_connection_close(self) -> None:
-        if not self.request.has_host():
-            return
-
         context = {
             'client_ip': None if not self.client.addr else self.client.addr[0],
             'client_port': None if not self.client.addr else self.client.addr[1],
@@ -429,9 +427,6 @@ def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]:
 
     # Can return None to tear down connection
     def on_client_data(self, raw: memoryview) -> Optional[memoryview]:
-        if not self.request.has_host():
-            return raw
-
         # For scenarios when an upstream connection was never established,
         # let plugin do whatever they wish to.  These are special scenarios
         # where plugins are trying to do something magical.  Within the core
@@ -500,9 +495,6 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]:
         return raw
 
     def on_request_complete(self) -> Union[socket.socket, bool]:
-        if not self.request.has_host():
-            return False
-
         self.emit_request_complete()
 
         # Invoke plugin.before_upstream_connection
@@ -888,7 +880,7 @@ def wrap_client(self) -> bool:
     def emit_request_complete(self) -> None:
         if not self.flags.enable_events:
             return
-        assert self.request.port
+        assert self.request.port and self.event_queue
         self.event_queue.publish(
             request_id=self.uid,
             event_name=eventNames.REQUEST_COMPLETE,
@@ -923,6 +915,7 @@ def emit_response_events(self, chunk_size: int) -> None:
     def emit_response_headers_complete(self) -> None:
         if not self.flags.enable_events:
             return
+        assert self.event_queue
         self.event_queue.publish(
             request_id=self.uid,
             event_name=eventNames.RESPONSE_HEADERS_COMPLETE,
@@ -940,6 +933,7 @@ def emit_response_headers_complete(self) -> None:
     def emit_response_chunk_received(self, chunk_size: int) -> None:
         if not self.flags.enable_events:
             return
+        assert self.event_queue
         self.event_queue.publish(
             request_id=self.uid,
             event_name=eventNames.RESPONSE_CHUNK_RECEIVED,
@@ -953,6 +947,7 @@ def emit_response_chunk_received(self, chunk_size: int) -> None:
     def emit_response_complete(self) -> None:
         if not self.flags.enable_events:
             return
+        assert self.event_queue
         self.event_queue.publish(
             request_id=self.uid,
             event_name=eventNames.RESPONSE_COMPLETE,
diff --git a/proxy/http/responses.py b/proxy/http/responses.py
index 65b548b3bf..3eaca1e606 100644
--- a/proxy/http/responses.py
+++ b/proxy/http/responses.py
@@ -46,6 +46,18 @@
     ),
 )
 
+BAD_REQUEST_RESPONSE_PKT = memoryview(
+    build_http_response(
+        httpStatusCodes.BAD_REQUEST,
+        reason=b'BAD REQUEST',
+        headers={
+            b'Server': PROXY_AGENT_HEADER_VALUE,
+            b'Content-Length': b'0',
+        },
+        conn_close=True,
+    ),
+)
+
 NOT_FOUND_RESPONSE_PKT = memoryview(
     build_http_response(
         httpStatusCodes.NOT_FOUND,
diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py
index eda6f7c40b..2316e716e9 100644
--- a/proxy/http/server/web.py
+++ b/proxy/http/server/web.py
@@ -27,6 +27,7 @@
 from ..plugin import HttpProtocolHandlerPlugin
 from ..websocket import WebsocketFrame, websocketOpcodes
 from ..parser import HttpParser, httpParserTypes
+from ..protocols import httpProtocols
 from ..responses import NOT_FOUND_RESPONSE_PKT, NOT_IMPLEMENTED_RESPONSE_PKT, okResponse
 
 from .plugin import HttpWebServerBasePlugin
@@ -94,6 +95,10 @@ def __init__(
         if b'HttpWebServerBasePlugin' in self.flags.plugins:
             self._initialize_web_plugins()
 
+    @staticmethod
+    def protocols() -> List[int]:
+        return [httpProtocols.WEB_SERVER]
+
     def _initialize_web_plugins(self) -> None:
         for klass in self.flags.plugins[b'HttpWebServerBasePlugin']:
             instance: HttpWebServerBasePlugin = klass(
@@ -153,8 +158,6 @@ def try_upgrade(self) -> bool:
         return False
 
     def on_request_complete(self) -> Union[socket.socket, bool]:
-        if self.request.has_host():
-            return False
         path = self.request.path or b'/'
         # Routing for Http(s) requests
         protocol = httpProtocolTypes.HTTPS \
@@ -262,8 +265,6 @@ def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]:
         return chunk
 
     def on_client_connection_close(self) -> None:
-        if self.request.has_host():
-            return
         context = {
             'client_ip': None if not self.client.addr else self.client.addr[0],
             'client_port': None if not self.client.addr else self.client.addr[1],
diff --git a/tests/http/test_http_proxy.py b/tests/http/test_http_proxy.py
index b1bc964059..6c09776d2c 100644
--- a/tests/http/test_http_proxy.py
+++ b/tests/http/test_http_proxy.py
@@ -47,8 +47,8 @@ def _setUp(self, mocker: MockerFixture) -> None:
         )
         self.protocol_handler.initialize()
 
-    def test_proxy_plugin_initialized(self) -> None:
-        self.plugin.assert_called()
+    def test_proxy_plugin_not_initialized_unless_first_request_completes(self) -> None:
+        self.plugin.assert_not_called()
 
     @pytest.mark.asyncio    # type: ignore[misc]
     async def test_proxy_plugin_on_and_before_upstream_connection(self) -> None:
diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py
index 23ae05faa9..0d30dd55de 100644
--- a/tests/http/test_http_proxy_tls_interception.py
+++ b/tests/http/test_http_proxy_tls_interception.py
@@ -93,15 +93,8 @@ def mock_connection() -> Any:
         )
         self.protocol_handler.initialize()
 
-        self.plugin.assert_called()
-        self.assertEqual(self.plugin.call_args[0][1], self.flags)
-        self.assertEqual(self.plugin.call_args[0][2].connection, self._conn)
-        self.proxy_plugin.assert_called()
-        self.assertEqual(self.proxy_plugin.call_args[0][1], self.flags)
-        self.assertEqual(
-            self.proxy_plugin.call_args[0][2].connection,
-            self._conn,
-        )
+        self.plugin.assert_not_called()
+        self.proxy_plugin.assert_not_called()
 
         connect_request = build_http_request(
             httpMethods.CONNECT, bytes_(netloc),
@@ -112,15 +105,15 @@ def mock_connection() -> Any:
         self._conn.recv.return_value = connect_request
 
         # Prepare mocked HttpProtocolHandlerPlugin
-        async def asyncReturnBool(val: bool) -> bool:
-            return val
-        self.plugin.return_value.get_descriptors.return_value = ([], [])
-        self.plugin.return_value.write_to_descriptors.return_value = asyncReturnBool(False)
-        self.plugin.return_value.read_from_descriptors.return_value = asyncReturnBool(False)
-        self.plugin.return_value.on_client_data.side_effect = lambda raw: raw
-        self.plugin.return_value.on_request_complete.return_value = False
-        self.plugin.return_value.on_response_chunk.side_effect = lambda chunk: chunk
-        self.plugin.return_value.on_client_connection_close.return_value = None
+        # async def asyncReturnBool(val: bool) -> bool:
+        #     return val
+        # self.plugin.return_value.get_descriptors.return_value = ([], [])
+        # self.plugin.return_value.write_to_descriptors.return_value = asyncReturnBool(False)
+        # self.plugin.return_value.read_from_descriptors.return_value = asyncReturnBool(False)
+        # self.plugin.return_value.on_client_data.side_effect = lambda raw: raw
+        # self.plugin.return_value.on_request_complete.return_value = False
+        # self.plugin.return_value.on_response_chunk.side_effect = lambda chunk: chunk
+        # self.plugin.return_value.on_client_connection_close.return_value = None
 
         # Prepare mocked HttpProxyBasePlugin
         self.proxy_plugin.return_value.write_to_descriptors.return_value = False
@@ -143,15 +136,29 @@ async def asyncReturnBool(val: bool) -> bool:
 
         await self.protocol_handler._run_once()
 
+        # Assert correct plugin was initialized
+        self.plugin.assert_not_called()
+        self.proxy_plugin.assert_called_once()
+        self.assertEqual(self.proxy_plugin.call_args[0][1], self.flags)
+        # Actual call arg must be `_conn` object
+        # but because internally the reference is updated
+        # we assert it against `mock_ssl_wrap` which is
+        # called during proxy plugin initialization
+        # for interception
+        self.assertEqual(
+            self.proxy_plugin.call_args[0][2].connection,
+            self.mock_ssl_wrap.return_value,
+        )
+
         # Assert our mocked plugins invocations
-        self.plugin.return_value.get_descriptors.assert_called()
-        self.plugin.return_value.write_to_descriptors.assert_called_with([])
-        # on_client_data is only called after initial request has completed
-        self.plugin.return_value.on_client_data.assert_not_called()
-        self.plugin.return_value.on_request_complete.assert_called()
-        self.plugin.return_value.read_from_descriptors.assert_called_with([
-            self._conn.fileno(),
-        ])
+        # self.plugin.return_value.get_descriptors.assert_called()
+        # self.plugin.return_value.write_to_descriptors.assert_called_with([])
+        # # on_client_data is only called after initial request has completed
+        # self.plugin.return_value.on_client_data.assert_not_called()
+        # self.plugin.return_value.on_request_complete.assert_called()
+        # self.plugin.return_value.read_from_descriptors.assert_called_with([
+        #     self._conn.fileno(),
+        # ])
         self.proxy_plugin.return_value.before_upstream_connection.assert_called()
         self.proxy_plugin.return_value.handle_client_request.assert_called()
 
@@ -198,10 +205,10 @@ async def asyncReturnBool(val: bool) -> bool:
         )
 
         # Assert connection references for all other plugins is updated
-        self.assertEqual(
-            self.plugin.return_value.client._conn,
-            self.mock_ssl_wrap.return_value,
-        )
+        # self.assertEqual(
+        #     self.plugin.return_value.client._conn,
+        #     self.mock_ssl_wrap.return_value,
+        # )
         self.assertEqual(
             self.proxy_plugin.return_value.client._conn,
             self.mock_ssl_wrap.return_value,
diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py
index 766208c959..9adb0bb683 100644
--- a/tests/http/test_protocol_handler.py
+++ b/tests/http/test_protocol_handler.py
@@ -190,7 +190,7 @@ async def assert_tunnel_response(
         self.assertTrue(
             cast(
                 HttpProxyPlugin,
-                self.protocol_handler.plugins['HttpProxyPlugin'],
+                self.protocol_handler.plugin,
             ).upstream is not None,
         )
         self.assertEqual(
diff --git a/tests/http/test_web_server.py b/tests/http/test_web_server.py
index 96083356f8..f383b30a6a 100644
--- a/tests/http/test_web_server.py
+++ b/tests/http/test_web_server.py
@@ -51,12 +51,11 @@ def test_on_client_connection_called_on_teardown(mocker: MockerFixture) -> None:
         flags=flags,
     )
     protocol_handler.initialize()
-    plugin.assert_called()
+    plugin.assert_not_called()
     mock_run_once = mocker.patch.object(protocol_handler, '_run_once')
     mock_run_once.return_value = True
     protocol_handler.run()
     assert _conn.closed
-    plugin.return_value.on_client_connection_close.assert_called()
 
 
 def mock_selector_for_client_read(self: Any) -> None:

From db0c923447974bda5ba27d9e3c98784f818deb91 Mon Sep 17 00:00:00 2001
From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
Date: Fri, 31 Dec 2021 22:54:33 +0530
Subject: [PATCH 20/22] [TlsParser] Refactored implementation from #748 (#922)

* Refactored TlsParser based upon work done in #748

* Add missing `tls_server_hello.data`, thanks to @JerryKwan

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Pass `check.py`

* Run check.py locally

* Fix lint errors

* Fix indentation issue

* Ignore linkcheck for cloudflare links, GHA is getting a 403 reply, while the link actually works

* Fix lint

* codespell skip

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 proxy/http/parser/tls/__init__.py     |  18 ++
 proxy/http/parser/tls/certificate.py  |  54 +++++
 proxy/http/parser/tls/finished.py     |  25 ++
 proxy/http/parser/tls/handshake.py    | 122 ++++++++++
 proxy/http/parser/tls/hello.py        | 242 ++++++++++++++++++
 proxy/http/parser/tls/key_exchange.py |  40 +++
 proxy/http/parser/tls/pretty.py       |  16 ++
 proxy/http/parser/tls/tls.py          |  76 ++++++
 proxy/http/parser/tls/types.py        |  40 +++
 setup.cfg                             |   3 +
 tests/http/test_tls_parser.py         |  68 ++++++
 tests/http/tls_server_hello.data      | 337 ++++++++++++++++++++++++++
 12 files changed, 1041 insertions(+)
 create mode 100644 proxy/http/parser/tls/__init__.py
 create mode 100644 proxy/http/parser/tls/certificate.py
 create mode 100644 proxy/http/parser/tls/finished.py
 create mode 100644 proxy/http/parser/tls/handshake.py
 create mode 100644 proxy/http/parser/tls/hello.py
 create mode 100644 proxy/http/parser/tls/key_exchange.py
 create mode 100644 proxy/http/parser/tls/pretty.py
 create mode 100644 proxy/http/parser/tls/tls.py
 create mode 100644 proxy/http/parser/tls/types.py
 create mode 100644 tests/http/test_tls_parser.py
 create mode 100644 tests/http/tls_server_hello.data

diff --git a/proxy/http/parser/tls/__init__.py b/proxy/http/parser/tls/__init__.py
new file mode 100644
index 0000000000..d32e35bcd0
--- /dev/null
+++ b/proxy/http/parser/tls/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+from .tls import TlsParser
+from .types import tlsContentType, tlsHandshakeType
+
+__all__ = [
+    'TlsParser',
+    'tlsContentType',
+    'tlsHandshakeType',
+]
diff --git a/proxy/http/parser/tls/certificate.py b/proxy/http/parser/tls/certificate.py
new file mode 100644
index 0000000000..1004210af7
--- /dev/null
+++ b/proxy/http/parser/tls/certificate.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+from typing import Optional, Tuple
+
+
+class TlsCertificate:
+    """TLS Certificate"""
+
+    def __init__(self) -> None:
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        self.data = raw
+        return True, raw
+
+    def build(self) -> bytes:
+        assert self.data
+        return self.data
+
+
+class TlsCertificateRequest:
+    """TLS Certificate Request"""
+
+    def __init__(self) -> None:
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        return False, raw
+
+    def build(self) -> bytes:
+        assert self.data
+        return self.data
+
+
+class TlsCertificateVerify:
+    """TLS Certificate Verify"""
+
+    def __init__(self) -> None:
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        return False, raw
+
+    def build(self) -> bytes:
+        assert self.data
+        return self.data
diff --git a/proxy/http/parser/tls/finished.py b/proxy/http/parser/tls/finished.py
new file mode 100644
index 0000000000..4cc7273372
--- /dev/null
+++ b/proxy/http/parser/tls/finished.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+from typing import Optional, Tuple
+
+
+class TlsFinished:
+    """TLS Finished"""
+
+    def __init__(self) -> None:
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        return False, raw
+
+    def build(self) -> bytes:
+        assert self.data
+        return self.data
diff --git a/proxy/http/parser/tls/handshake.py b/proxy/http/parser/tls/handshake.py
new file mode 100644
index 0000000000..d859ceb589
--- /dev/null
+++ b/proxy/http/parser/tls/handshake.py
@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import struct
+import logging
+
+from typing import Optional, Tuple
+
+from .types import tlsHandshakeType
+from .hello import TlsHelloRequest, TlsClientHello, TlsServerHello, TlsServerHelloDone
+from .certificate import TlsCertificate, TlsCertificateRequest, TlsCertificateVerify
+from .key_exchange import TlsClientKeyExchange, TlsServerKeyExchange
+from .finished import TlsFinished
+
+logger = logging.getLogger(__name__)
+
+
+class TlsHandshake:
+    """TLS Handshake"""
+
+    def __init__(self) -> None:
+        self.msg_type: int = tlsHandshakeType.OTHER
+        self.length: Optional[bytes] = None
+        self.hello_request: Optional[TlsHelloRequest] = None
+        self.client_hello: Optional[TlsClientHello] = None
+        self.server_hello: Optional[TlsServerHello] = None
+        self.certificate: Optional[TlsCertificate] = None
+        self.server_key_exchange: Optional[TlsServerKeyExchange] = None
+        self.certificate_request: Optional[TlsCertificateRequest] = None
+        self.server_hello_done: Optional[TlsServerHelloDone] = None
+        self.certificate_verify: Optional[TlsCertificateVerify] = None
+        self.client_key_exchange: Optional[TlsClientKeyExchange] = None
+        self.finished: Optional[TlsFinished] = None
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        length = len(raw)
+        if length < 4:
+            logger.debug('invalid data, len(raw) = %s', length)
+            return False, raw
+        payload_length, = struct.unpack('!I', b'\x00' + raw[1:4])
+        self.length = payload_length
+        if length < 4 + payload_length:
+            logger.debug(
+                'incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length,
+            )
+            return False, raw
+        # parse
+        self.msg_type = raw[0]
+        self.length = raw[1:4]
+        self.data = raw[: 4 + payload_length]
+        payload = raw[4: 4 + payload_length]
+        if self.msg_type == tlsHandshakeType.HELLO_REQUEST:
+            # parse hello request
+            self.hello_request = TlsHelloRequest()
+            self.hello_request.parse(payload)
+        elif self.msg_type == tlsHandshakeType.CLIENT_HELLO:
+            # parse client hello
+            self.client_hello = TlsClientHello()
+            self.client_hello.parse(payload)
+        elif self.msg_type == tlsHandshakeType.SERVER_HELLO:
+            # parse server hello
+            self.server_hello = TlsServerHello()
+            self.server_hello.parse(payload)
+        elif self.msg_type == tlsHandshakeType.CERTIFICATE:
+            # parse certificate
+            self.certificate = TlsCertificate()
+            self.certificate.parse(payload)
+        elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE:
+            # parse server key exchange
+            self.server_key_exchange = TlsServerKeyExchange()
+            self.server_key_exchange.parse(payload)
+        elif self.msg_type == tlsHandshakeType.CERTIFICATE_REQUEST:
+            # parse certificate request
+            self.certificate_request = TlsCertificateRequest()
+            self.certificate_request.parse(payload)
+        elif self.msg_type == tlsHandshakeType.SERVER_HELLO_DONE:
+            # parse server hello done
+            self.server_hello_done = TlsServerHelloDone()
+            self.server_hello_done.parse(payload)
+        elif self.msg_type == tlsHandshakeType.CERTIFICATE_VERIFY:
+            # parse certificate verify
+            self.certificate_verify = TlsCertificateVerify()
+            self.certificate_verify.parse(payload)
+        elif self.msg_type == tlsHandshakeType.CLIENT_KEY_EXCHANGE:
+            # parse client key exchange
+            self.client_key_exchange = TlsClientKeyExchange()
+            self.client_key_exchange.parse(payload)
+        elif self.msg_type == tlsHandshakeType.FINISHED:
+            # parse finished
+            self.finished = TlsFinished()
+            self.finished.parse(payload)
+        return True, raw[4 + payload_length:]
+
+    def build(self) -> bytes:
+        data = b''
+        data += bytes([self.msg_type])
+        payload = b''
+        if self.msg_type == tlsHandshakeType.CLIENT_HELLO:
+            assert self.client_hello
+            payload = self.client_hello.build()
+        elif self.msg_type == tlsHandshakeType.SERVER_HELLO:
+            assert self.server_hello
+            payload = self.server_hello.build()
+        elif self.msg_type == tlsHandshakeType.CERTIFICATE:
+            assert self.certificate
+            payload = self.certificate.build()
+        elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE:
+            assert self.server_key_exchange
+            payload = self.server_key_exchange.build()
+        # calculate length
+        length = struct.pack('!I', len(payload))[1:]
+        data += length
+        data += payload
+        return data
diff --git a/proxy/http/parser/tls/hello.py b/proxy/http/parser/tls/hello.py
new file mode 100644
index 0000000000..80287109d0
--- /dev/null
+++ b/proxy/http/parser/tls/hello.py
@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import os
+import struct
+import logging
+
+from typing import Optional, Tuple
+
+from .pretty import pretty_hexlify
+
+logger = logging.getLogger(__name__)
+
+
+class TlsHelloRequest:
+    """TLS Hello Request"""
+
+    def __init__(self) -> None:
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> None:
+        self.data = raw
+
+    def build(self) -> bytes:
+        assert self.data
+        return self.data
+
+
+class TlsClientHello:
+    """TLS Client Hello"""
+
+    def __init__(self) -> None:
+        self.protocol_version: Optional[bytes] = None
+        self.random: Optional[bytes] = None
+        self.session_id: Optional[bytes] = None
+        self.cipher_suite: Optional[bytes] = None
+        self.compression_method: Optional[bytes] = None
+        self.extension: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        try:
+            idx = 0
+            length = len(raw)
+            self.protocol_version = raw[idx:idx + 2]
+            idx += 2
+            self.random = raw[idx:idx + 32]
+            idx += 32
+            session_length = raw[idx]
+            self.session_id = raw[idx: idx + 1 + session_length]
+            idx += 1 + session_length
+            cipher_suite_length, = struct.unpack('!H', raw[idx: idx + 2])
+            self.cipher_suite = raw[idx: idx + 2 + cipher_suite_length]
+            idx += 2 + cipher_suite_length
+            compression_method_length = raw[idx]
+            self.compression_method = raw[
+                idx: idx +
+                1 + compression_method_length
+            ]
+            idx += 1 + compression_method_length
+            # extension
+            if idx == length:
+                self.extension = b''
+            else:
+                extension_length, = struct.unpack('!H', raw[idx: idx + 2])
+                self.extension = raw[idx: idx + 2 + extension_length]
+                idx += 2 + extension_length
+            return True, raw[idx:]
+        except Exception as e:
+            logger.exception(e)
+            return False, raw
+
+    def build(self) -> bytes:
+        # calculate length
+        return b''.join([
+            bs for bs in (
+                self.protocol_version, self.random, self.session_id, self.cipher_suite,
+                self.compression_method, self.extension,
+            ) if bs is not None
+        ])
+
+    def format(self) -> str:
+        parts = []
+        parts.append(
+            'Protocol Version: %s' % (
+                pretty_hexlify(self.protocol_version)
+                if self.protocol_version is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Random: %s' % (
+                pretty_hexlify(self.random)
+                if self.random is not None else ''
+            ),
+        )
+        parts.append(
+            'Session ID: %s' % (
+                pretty_hexlify(self.session_id)
+                if self.session_id is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Cipher Suite: %s' % (
+                pretty_hexlify(self.cipher_suite)
+                if self.cipher_suite is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Compression Method: %s' % (
+                pretty_hexlify(self.compression_method)
+                if self.compression_method is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Extension: %s' % (
+                pretty_hexlify(self.extension)
+                if self.extension is not None
+                else ''
+            ),
+        )
+        return os.linesep.join(parts)
+
+
+class TlsServerHello:
+    """TLS Server Hello"""
+
+    def __init__(self) -> None:
+        self.protocol_version: Optional[bytes] = None
+        self.random: Optional[bytes] = None
+        self.session_id: Optional[bytes] = None
+        self.cipher_suite: Optional[bytes] = None
+        self.compression_method: Optional[bytes] = None
+        self.extension: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        try:
+            idx = 0
+            length = len(raw)
+            self.protocol_version = raw[idx:idx + 2]
+            idx += 2
+            self.random = raw[idx:idx + 32]
+            idx += 32
+            session_length = raw[idx]
+            self.session_id = raw[idx: idx + 1 + session_length]
+            idx += 1 + session_length
+            self.cipher_suite = raw[idx: idx + 2]
+            idx += 2
+            compression_method_length = raw[idx]
+            self.compression_method = raw[
+                idx: idx +
+                1 + compression_method_length
+            ]
+            idx += 1 + compression_method_length
+            # extension
+            if idx == length:
+                self.extension = b''
+            else:
+                extension_length, = struct.unpack('!H', raw[idx: idx + 2])
+                self.extension = raw[idx: idx + 2 + extension_length]
+                idx += 2 + extension_length
+            return True, raw[idx:]
+        except Exception as e:
+            logger.exception(e)
+            return False, raw
+
+    def build(self) -> bytes:
+        return b''.join([
+            bs for bs in (
+                self.protocol_version, self.random, self.session_id, self.cipher_suite,
+                self.compression_method, self.extension,
+            ) if bs is not None
+        ])
+
+    def format(self) -> str:
+        parts = []
+        parts.append(
+            'Protocol Version: %s' % (
+                pretty_hexlify(self.protocol_version)
+                if self.protocol_version is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Random: %s' % (
+                pretty_hexlify(self.random)
+                if self.random is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Session ID: %s' % (
+                pretty_hexlify(self.session_id)
+                if self.session_id is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Cipher Suite: %s' % (
+                pretty_hexlify(self.cipher_suite)
+                if self.cipher_suite is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Compression Method: %s' % (
+                pretty_hexlify(self.compression_method)
+                if self.compression_method is not None
+                else ''
+            ),
+        )
+        parts.append(
+            'Extension: %s' % (
+                pretty_hexlify(self.extension)
+                if self.extension is not None
+                else ''
+            ),
+        )
+        return os.linesep.join(parts)
+
+
+class TlsServerHelloDone:
+    """TLS Server Hello Done"""
+
+    def __init__(self) -> None:
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        return False, raw
+
+    def build(self) -> bytes:
+        assert self.data
+        return self.data
diff --git a/proxy/http/parser/tls/key_exchange.py b/proxy/http/parser/tls/key_exchange.py
new file mode 100644
index 0000000000..ce56562b19
--- /dev/null
+++ b/proxy/http/parser/tls/key_exchange.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+from typing import Optional, Tuple
+
+
+class TlsServerKeyExchange:
+    """TLS Server Key Exchange"""
+
+    def __init__(self) -> None:
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        self.data = raw
+        return True, raw
+
+    def build(self) -> bytes:
+        assert self.data
+        return self.data
+
+
+class TlsClientKeyExchange:
+    """TLS Client Key Exchange"""
+
+    def __init__(self) -> None:
+        self.data: Optional[bytes] = None
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        return False, raw
+
+    def build(self) -> bytes:
+        assert self.data
+        return self.data
diff --git a/proxy/http/parser/tls/pretty.py b/proxy/http/parser/tls/pretty.py
new file mode 100644
index 0000000000..200463e3dd
--- /dev/null
+++ b/proxy/http/parser/tls/pretty.py
@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import binascii
+
+
+def pretty_hexlify(raw: bytes) -> str:
+    hexlified = binascii.hexlify(raw).decode('utf-8')
+    return ' '.join([hexlified[i: i+2] for i in range(0, len(hexlified), 2)])
diff --git a/proxy/http/parser/tls/tls.py b/proxy/http/parser/tls/tls.py
new file mode 100644
index 0000000000..634c5b93ed
--- /dev/null
+++ b/proxy/http/parser/tls/tls.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import struct
+import logging
+
+from typing import Optional, Tuple
+
+from .types import tlsContentType
+from .certificate import TlsCertificate
+from .handshake import TlsHandshake
+
+logger = logging.getLogger(__name__)
+
+
+class TlsParser:
+    """TLS packet parser"""
+
+    def __init__(self) -> None:
+        self.content_type: int = tlsContentType.OTHER
+        self.protocol_version: Optional[bytes] = None
+        self.length: Optional[bytes] = None
+        # only parse hand shake payload temporary
+        self.handshake: Optional[TlsHandshake] = None
+        self.certificate: Optional[TlsCertificate]
+
+    def parse(self, raw: bytes) -> Tuple[bool, bytes]:
+        """Parse TLS fragmentation.
+
+        References
+
+        https://datatracker.ietf.org/doc/html/rfc5246#page-15
+        https://datatracker.ietf.org/doc/html/rfc5077#page-3
+        https://datatracker.ietf.org/doc/html/rfc8446#page-10
+        """
+        length = len(raw)
+        if length < 5:
+            logger.debug('invalid data, len(raw) = %s', length)
+            return False, raw
+        payload_length, = struct.unpack('!H', raw[3:5])
+        if length < 5 + payload_length:
+            logger.debug(
+                'incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length,
+            )
+            return False, raw
+        # parse
+        self.content_type = raw[0]
+        self.protocol_version = raw[1:3]
+        self.length = raw[3:5]
+        payload = raw[5:5 + payload_length]
+        if self.content_type == tlsContentType.HANDSHAKE:
+            # parse handshake
+            self.handshake = TlsHandshake()
+            self.handshake.parse(payload)
+        return True, raw[5 + payload_length:]
+
+    def build(self) -> bytes:
+        data = b''
+        data += bytes([self.content_type])
+        assert self.protocol_version
+        data += self.protocol_version
+        payload = b''
+        if self.content_type == tlsContentType.HANDSHAKE:
+            assert self.handshake
+            payload += self.handshake.build()
+        length = struct.pack('!H', len(payload))
+        data += length
+        data += payload
+        return data
diff --git a/proxy/http/parser/tls/types.py b/proxy/http/parser/tls/types.py
new file mode 100644
index 0000000000..9dd05df321
--- /dev/null
+++ b/proxy/http/parser/tls/types.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+from typing import NamedTuple
+
+TlsContentType = NamedTuple(
+    'TlsContentType', [
+        ('CHANGE_CIPHER_SPEC', int),
+        ('ALERT', int),
+        ('HANDSHAKE', int),
+        ('APPLICATION_DATA', int),
+        ('OTHER', int),
+    ],
+)
+tlsContentType = TlsContentType(20, 21, 22, 23, 255)
+
+
+TlsHandshakeType = NamedTuple(
+    'TlsHandshakeType', [
+        ('HELLO_REQUEST', int),
+        ('CLIENT_HELLO', int),
+        ('SERVER_HELLO', int),
+        ('CERTIFICATE', int),
+        ('SERVER_KEY_EXCHANGE', int),
+        ('CERTIFICATE_REQUEST', int),
+        ('SERVER_HELLO_DONE', int),
+        ('CERTIFICATE_VERIFY', int),
+        ('CLIENT_KEY_EXCHANGE', int),
+        ('FINISHED', int),
+        ('OTHER', int),
+    ],
+)
+tlsHandshakeType = TlsHandshakeType(0, 1, 2, 11, 12, 13, 14, 15, 16, 20, 255)
diff --git a/setup.cfg b/setup.cfg
index fe1fcf3b08..26a5e175d9 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -120,3 +120,6 @@ proxy =
 exclude =
   tests
   tests.*
+
+[codespell]
+skip = tests/http/tls_server_hello.data
diff --git a/tests/http/test_tls_parser.py b/tests/http/test_tls_parser.py
new file mode 100644
index 0000000000..1a5a0c93d3
--- /dev/null
+++ b/tests/http/test_tls_parser.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import re
+import unittest
+import binascii
+
+from pathlib import Path
+
+from proxy.http.parser.tls import TlsParser, tlsContentType, tlsHandshakeType
+
+
+class TestTlsParser(unittest.TestCase):
+    """Ref: https://tls.ulfheim.net/"""
+
+    def _unhexlify(self, raw: str) -> bytes:
+        return binascii.unhexlify(re.sub(r'\s', '', raw))
+
+    def test_parse_client_hello(self) -> None:
+        data = """
+                16 03 01 00 a5 01 00 00 a1 03 03 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14
+                15 16 17 18 19 1a 1b 1c 1d 1e 1f 00 00 20 cc a8 cc a9 c0 2f c0 30 c0 2b c0 2c c0 13 c0 09 c0 14
+                c0 0a 00 9c 00 9d 00 2f 00 35 c0 12 00 0a 01 00 00 58 00 00 00 18 00 16 00 00 13 65 78 61 6d 70
+                6c 65 2e 75 6c 66 68 65 69 6d 2e 6e 65 74 00 05 00 05 01 00 00 00 00 00 0a 00 0a 00 08 00 1d 00
+                17 00 18 00 19 00 0b 00 02 01 00 00 0d 00 12 00 10 04 01 04 03 05 01 05 03 06 01 06 03 02 01 02
+                03 ff 01 00 01 00 00 12 00 00
+                """
+        tls = TlsParser()
+        tls.parse(self._unhexlify(data))
+        self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE)
+        self.assertEqual(tls.length, b'\x00\xa5')
+        assert tls.handshake
+        self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.CLIENT_HELLO)
+        self.assertEqual(tls.handshake.length, b'\x00\x00\xa1')
+        self.assertEqual(len(tls.handshake.build()), 0xA1 + 0x04)
+        assert tls.handshake.client_hello
+        self.assertEqual(
+            tls.handshake.client_hello.protocol_version, b'\x03\x03',
+        )
+        self.assertEqual(
+            tls.handshake.client_hello.random, self._unhexlify(
+                '00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f',
+            ),
+        )
+
+    def test_parse_server_hello(self) -> None:
+        with open(Path(__file__).parent / 'tls_server_hello.data', 'rb') as f:
+            data = f.read()
+            tls = TlsParser()
+            tls.parse(self._unhexlify(data.decode('utf-8')))
+            self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE)
+            assert tls.handshake
+            self.assertEqual(
+                tls.handshake.msg_type,
+                tlsHandshakeType.SERVER_HELLO,
+            )
+            assert tls.handshake.server_hello
+            self.assertEqual(
+                tls.handshake.server_hello.protocol_version, b'\x03\x03',
+            )
+            print(tls.handshake.server_hello.format())
diff --git a/tests/http/tls_server_hello.data b/tests/http/tls_server_hello.data
new file mode 100644
index 0000000000..a22f74ad4a
--- /dev/null
+++ b/tests/http/tls_server_hello.data
@@ -0,0 +1,337 @@
+16 03 03 00 59 02 00 00 55 03 03 60 f9 54 e7 1b
+c2 60 2d 02 94 ed 2a 82 d2 1d 06 71 3c 52 d1 23
+0e 73 fe c9 93 8a 05 71 f5 48 46 20 b0 d9 d0 1c
+bd 6b 49 27 47 88 14 c6 ef 04 bc 48 3b 01 5f 00
+61 ae f8 a2 45 5e 7b fe fb ed f4 a8 c0 2f 00 00
+0d ff 01 00 01 00 00 0b 00 04 03 00 01 02 16 03
+03 13 4f 0b 00 13 4b 00 13 48 00 05 e0 30 82 05
+dc 30 82 04 c4 a0 03 02 01 02 02 10 01 58 f3 0a
+d5 9d 62 af 20 9f 15 ee 4f c2 bb 33 30 0d 06 09
+2a 86 48 86 f7 0d 01 01 0b 05 00 30 46 31 0b 30
+09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06 03
+55 04 0a 13 06 41 6d 61 7a 6f 6e 31 15 30 13 06
+03 55 04 0b 13 0c 53 65 72 76 65 72 20 43 41 20
+31 42 31 0f 30 0d 06 03 55 04 03 13 06 41 6d 61
+7a 6f 6e 30 1e 17 0d 32 31 31 31 32 31 30 30 30
+30 30 30 5a 17 0d 32 32 31 32 31 39 32 33 35 39
+35 39 5a 30 16 31 14 30 12 06 03 55 04 03 13 0b
+68 74 74 70 62 69 6e 2e 6f 72 67 30 82 01 22 30
+0d 06 09 2a 86 48 86 f7 0d 01 01 01 05 00 03 82
+01 0f 00 30 82 01 0a 02 82 01 01 00 84 e4 27 a5
+ec eb c0 0d 2f 1f 37 f8 ec f6 be 3b ce 1f 5a e7
+bf e7 ad 93 a9 0a d5 8a bb 5f fb 77 ec 19 07 77
+32 6c 27 21 df a4 b0 01 90 bd 63 78 33 5f e1 49
+e9 58 25 bd b8 ea 91 6a 18 1e ed f9 92 3e be 30
+58 1f 8f c9 e8 28 85 b0 ac 33 ba ca a4 48 5e b7
+7e d4 21 f0 ec 74 39 ab 01 f4 7b d1 94 e3 c7 4f
+f0 79 0e c9 21 f0 73 62 27 ed 65 13 14 99 11 a3
+bd ce c4 3d c0 ee b6 6e c1 87 b0 c5 ff 50 fd d5
+40 1c 97 2d 50 59 bb 40 cc 2a c9 76 84 66 94 37
+2f 7e 2e 06 6f e3 f8 95 a5 60 cf 43 5f a4 86 97
+f1 3f dc 75 1f e9 8c 95 45 81 81 83 ea ee 31 b9
+71 e4 db 96 d2 28 17 fb 14 40 f3 51 fb 22 88 05
+29 ee 88 84 97 14 73 bd 1c 49 e4 e6 98 7a d7 48
+56 99 54 0a 22 b0 80 3e c0 14 30 cf 66 eb e8 83
+6f ae 0d 8e b3 23 59 5f db 92 6d fa af e1 41 33
+3f 4b f0 77 c4 07 88 71 97 0c 20 e5 02 03 01 00
+01 a3 82 02 f4 30 82 02 f0 30 1f 06 03 55 1d 23
+04 18 30 16 80 14 59 a4 66 06 52 a0 7b 95 92 3c
+a3 94 07 27 96 74 5b f9 3d d0 30 1d 06 03 55 1d
+0e 04 16 04 14 cf 07 84 b8 03 48 ac c9 ac db 11
+65 0a 7d 29 ff d6 97 4b b3 30 25 06 03 55 1d 11
+04 1e 30 1c 82 0b 68 74 74 70 62 69 6e 2e 6f 72
+67 82 0d 2a 2e 68 74 74 70 62 69 6e 2e 6f 72 67
+30 0e 06 03 55 1d 0f 01 01 ff 04 04 03 02 05 a0
+30 1d 06 03 55 1d 25 04 16 30 14 06 08 2b 06 01
+05 05 07 03 01 06 08 2b 06 01 05 05 07 03 02 30
+3d 06 03 55 1d 1f 04 36 30 34 30 32 a0 30 a0 2e
+86 2c 68 74 74 70 3a 2f 2f 63 72 6c 2e 73 63 61
+31 62 2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63
+6f 6d 2f 73 63 61 31 62 2d 31 2e 63 72 6c 30 13
+06 03 55 1d 20 04 0c 30 0a 30 08 06 06 67 81 0c
+01 02 01 30 75 06 08 2b 06 01 05 05 07 01 01 04
+69 30 67 30 2d 06 08 2b 06 01 05 05 07 30 01 86
+21 68 74 74 70 3a 2f 2f 6f 63 73 70 2e 73 63 61
+31 62 2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63
+6f 6d 30 36 06 08 2b 06 01 05 05 07 30 02 86 2a
+68 74 74 70 3a 2f 2f 63 72 74 2e 73 63 61 31 62
+2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63 6f 6d
+2f 73 63 61 31 62 2e 63 72 74 30 0c 06 03 55 1d
+13 01 01 ff 04 02 30 00 30 82 01 7d 06 0a 2b 06
+01 04 01 d6 79 02 04 02 04 82 01 6d 04 82 01 69
+01 67 00 76 00 29 79 be f0 9e 39 39 21 f0 56 73
+9f 63 a5 77 e5 be 57 7d 9c 60 0a f8 f9 4d 5d 26
+5c 25 5d c7 84 00 00 01 7d 40 73 08 b5 00 00 04
+03 00 47 30 45 02 20 5d d8 f1 cc 14 99 9f 5c 99
+84 9c 94 da d2 60 e8 58 0d e0 b3 49 30 18 1f 0a
+48 86 9e a8 00 72 81 02 21 00 ff c9 ca c0 dd 06
+dd 2c 14 89 2d ff 35 dc e5 56 eb 9a 92 be 1a ee
+8b e2 03 33 7b cd 78 92 9b 37 00 76 00 51 a3 b0
+f5 fd 01 79 9c 56 6d b8 37 78 8f 0c a4 7a cc 1b
+27 cb f7 9e 88 42 9a 0d fe d4 8b 05 e5 00 00 01
+7d 40 73 08 c9 00 00 04 03 00 47 30 45 02 20 7e
+29 67 be 79 d6 34 0e 27 f9 55 db 4b 03 28 d0 96
+38 b7 6d f9 c4 b6 a6 35 08 14 4a 48 f2 7c 25 02
+21 00 8f 68 71 a6 2b c8 3e d5 fd b5 17 84 70 90
+9a 02 e0 b9 1a 1f 74 99 9c fd 95 64 32 ba c2 03
+d1 54 00 75 00 df a5 5e ab 68 82 4f 1f 6c ad ee
+b8 5f 4e 3e 5a ea cd a2 12 a4 6a 5e 8e 3b 12 c0
+20 44 5c 2a 73 00 00 01 7d 40 73 08 a2 00 00 04
+03 00 46 30 44 02 20 4c 9e 68 c3 ac cb 92 d2 b0
+a9 08 a1 dd 82 7c 13 0b dc 9f fe 0b 08 ef 25 e9
+1d 30 12 38 41 82 36 02 20 18 29 da 43 a6 d0 df
+b5 71 bb 19 23 b3 f1 28 29 a9 1b 54 a1 62 07 b3
+5c 28 4a 3e ee c0 42 70 bb 30 0d 06 09 2a 86 48
+86 f7 0d 01 01 0b 05 00 03 82 01 01 00 2f 64 1d
+d3 7d 3c 06 41 7b 6a 1c 94 60 99 31 92 b7 eb e1
+6c b2 ac ee d2 5b f4 ec 36 94 a6 c6 a3 c5 f8 3f
+d0 a9 07 01 b7 cb 4f 1d 68 35 a0 d1 1c 51 88 a7
+97 9a ad 03 89 32 1e aa 40 96 0c d8 97 4e 05 52
+94 a1 90 d0 e3 73 07 75 1a 29 20 98 d3 08 85 21
+b1 dc 41 da 41 ec 87 be b2 88 80 24 5e 4c ad 5f
+3c fc 43 bc 55 36 6a 8f 67 9c 47 e2 43 43 a8 b2
+c4 55 0b a1 1a 26 c4 3e fc 21 27 4e 2a f4 04 43
+e3 59 18 d6 d8 2f e0 f5 83 87 47 23 fd 3c ee 11
+ff 9f 0a 3b 2f 29 74 96 70 5a 43 b2 18 ac a5 65
+e9 ac 2f 2e ac 47 c8 77 50 4d 0a 5b 44 da c0 cc
+06 3b eb 23 59 4d 9e 07 fb 8e 27 a6 1c 68 04 b4
+a2 0d 52 ec da cb 0e a4 f7 1d 45 92 7f cd 60 ee
+a9 4d 78 2d e1 36 02 3c 7c cc 45 2c f4 e4 86 7f
+3e bf 6d b4 48 98 a9 8b a8 a0 9e 42 51 54 5a 02
+d5 a4 c5 a2 55 cc ab fa 0c a9 1a 94 1c 00 04 4d
+30 82 04 49 30 82 03 31 a0 03 02 01 02 02 13 06
+7f 94 57 85 87 e8 ac 77 de b2 53 32 5b bc 99 8b
+56 0d 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05
+00 30 39 31 0b 30 09 06 03 55 04 06 13 02 55 53
+31 0f 30 0d 06 03 55 04 0a 13 06 41 6d 61 7a 6f
+6e 31 19 30 17 06 03 55 04 03 13 10 41 6d 61 7a
+6f 6e 20 52 6f 6f 74 20 43 41 20 31 30 1e 17 0d
+31 35 31 30 32 32 30 30 30 30 30 30 5a 17 0d 32
+35 31 30 31 39 30 30 30 30 30 30 5a 30 46 31 0b
+30 09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06
+03 55 04 0a 13 06 41 6d 61 7a 6f 6e 31 15 30 13
+06 03 55 04 0b 13 0c 53 65 72 76 65 72 20 43 41
+20 31 42 31 0f 30 0d 06 03 55 04 03 13 06 41 6d
+61 7a 6f 6e 30 82 01 22 30 0d 06 09 2a 86 48 86
+f7 0d 01 01 01 05 00 03 82 01 0f 00 30 82 01 0a
+02 82 01 01 00 c2 4e 16 67 dd ce bc 6a c8 37 5a
+ec 3a 30 b0 1d e6 d1 12 e8 12 28 48 cc e8 29 c1
+b9 6e 53 d5 a3 eb 03 39 1a cc 77 87 f6 01 b9 d9
+70 cc cf 6b 8d e3 e3 03 71 86 99 6d cb a6 94 2a
+4e 13 d6 a7 bd 04 ec 0a 16 3c 0a eb 39 b1 c4 b5
+58 a3 b6 c7 56 25 ec 3e 52 7a a8 e3 29 16 07 b9
+6e 50 cf fb 5f 31 f8 1d ba 03 4a 62 89 03 ae 3e
+47 f2 0f 27 91 e3 14 20 85 f8 fa e9 8a 35 f5 5f
+9e 99 4d e7 6b 37 ef a4 50 3e 44 ec fa 5a 85 66
+07 9c 7e 17 6a 55 f3 17 8a 35 1e ee e9 ac c3 75
+4e 58 55 7d 53 6b 0a 6b 9b 14 42 d7 e5 ac 01 89
+b3 ea a3 fe cf c0 2b 0c 84 c2 d8 53 15 cb 67 f0
+d0 88 ca 3a d1 17 73 f5 5f 9a d4 c5 72 1e 7e 01
+f1 98 30 63 2a aa f2 7a 2d c5 e2 02 1a 86 e5 32
+3e 0e bd 11 b4 cf 3c 93 ef 17 50 10 9e 43 c2 06
+2a e0 0d 68 be d3 88 8b 4a 65 8c 4a d4 c3 2e 4c
+9b 55 f4 86 e5 02 03 01 00 01 a3 82 01 3b 30 82
+01 37 30 12 06 03 55 1d 13 01 01 ff 04 08 30 06
+01 01 ff 02 01 00 30 0e 06 03 55 1d 0f 01 01 ff
+04 04 03 02 01 86 30 1d 06 03 55 1d 0e 04 16 04
+14 59 a4 66 06 52 a0 7b 95 92 3c a3 94 07 27 96
+74 5b f9 3d d0 30 1f 06 03 55 1d 23 04 18 30 16
+80 14 84 18 cc 85 34 ec bc 0c 94 94 2e 08 59 9c
+c7 b2 10 4e 0a 08 30 7b 06 08 2b 06 01 05 05 07
+01 01 04 6f 30 6d 30 2f 06 08 2b 06 01 05 05 07
+30 01 86 23 68 74 74 70 3a 2f 2f 6f 63 73 70 2e
+72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74 72
+75 73 74 2e 63 6f 6d 30 3a 06 08 2b 06 01 05 05
+07 30 02 86 2e 68 74 74 70 3a 2f 2f 63 72 74 2e
+72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74 72
+75 73 74 2e 63 6f 6d 2f 72 6f 6f 74 63 61 31 2e
+63 65 72 30 3f 06 03 55 1d 1f 04 38 30 36 30 34
+a0 32 a0 30 86 2e 68 74 74 70 3a 2f 2f 63 72 6c
+2e 72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74
+72 75 73 74 2e 63 6f 6d 2f 72 6f 6f 74 63 61 31
+2e 63 72 6c 30 13 06 03 55 1d 20 04 0c 30 0a 30
+08 06 06 67 81 0c 01 02 01 30 0d 06 09 2a 86 48
+86 f7 0d 01 01 0b 05 00 03 82 01 01 00 85 92 be
+35 bb 79 cf a3 81 42 1c e4 e3 63 73 53 39 52 35
+e7 d1 ad fd ae 99 8a ac 89 12 2f bb e7 6f 9a d5
+4e 72 ea 20 30 61 f9 97 b2 cd a5 27 02 45 a8 ca
+76 3e 98 4a 83 9e b6 e6 45 e0 f2 43 f6 08 de 6d
+e8 6e db 31 07 13 f0 2f 31 0d 93 6d 61 37 7b 58
+f0 fc 51 98 91 28 02 4f 05 76 b7 d3 f0 1b c2 e6
+5e d0 66 85 11 0f 2e 81 c6 10 81 29 fe 20 60 48
+f3 f2 f0 84 13 53 65 35 15 11 6b 82 51 40 55 57
+5f 18 b5 b0 22 3e ad f2 5e a3 01 e3 c3 b3 f9 cb
+41 5a e6 52 91 bb e4 36 87 4f 2d a9 a4 07 68 35
+ba 94 72 cd 0e ea 0e 7d 57 f2 79 fc 37 c5 7b 60
+9e b2 eb c0 2d 90 77 0d 49 10 27 a5 38 ad c4 12
+a3 b4 a3 c8 48 b3 15 0b 1e e2 e2 19 dc c4 76 52
+c8 bc 8a 41 78 70 d9 6d 97 b3 4a 8b 78 2d 5e b4
+0f a3 4c 60 ca e1 47 cb 78 2d 12 17 b1 52 8b ca
+39 2c bd b5 2f c2 33 02 96 ab da 94 7f 00 04 96
+30 82 04 92 30 82 03 7a a0 03 02 01 02 02 13 06
+7f 94 4a 2a 27 cd f3 fa c2 ae 2b 01 f9 08 ee b9
+c4 c6 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05
+00 30 81 98 31 0b 30 09 06 03 55 04 06 13 02 55
+53 31 10 30 0e 06 03 55 04 08 13 07 41 72 69 7a
+6f 6e 61 31 13 30 11 06 03 55 04 07 13 0a 53 63
+6f 74 74 73 64 61 6c 65 31 25 30 23 06 03 55 04
+0a 13 1c 53 74 61 72 66 69 65 6c 64 20 54 65 63
+68 6e 6f 6c 6f 67 69 65 73 2c 20 49 6e 63 2e 31
+3b 30 39 06 03 55 04 03 13 32 53 74 61 72 66 69
+65 6c 64 20 53 65 72 76 69 63 65 73 20 52 6f 6f
+74 20 43 65 72 74 69 66 69 63 61 74 65 20 41 75
+74 68 6f 72 69 74 79 20 2d 20 47 32 30 1e 17 0d
+31 35 30 35 32 35 31 32 30 30 30 30 5a 17 0d 33
+37 31 32 33 31 30 31 30 30 30 30 5a 30 39 31 0b
+30 09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06
+03 55 04 0a 13 06 41 6d 61 7a 6f 6e 31 19 30 17
+06 03 55 04 03 13 10 41 6d 61 7a 6f 6e 20 52 6f
+6f 74 20 43 41 20 31 30 82 01 22 30 0d 06 09 2a
+86 48 86 f7 0d 01 01 01 05 00 03 82 01 0f 00 30
+82 01 0a 02 82 01 01 00 b2 78 80 71 ca 78 d5 e3
+71 af 47 80 50 74 7d 6e d8 d7 88 76 f4 99 68 f7
+58 21 60 f9 74 84 01 2f ac 02 2d 86 d3 a0 43 7a
+4e b2 a4 d0 36 ba 01 be 8d db 48 c8 07 17 36 4c
+f4 ee 88 23 c7 3e eb 37 f5 b5 19 f8 49 68 b0 de
+d7 b9 76 38 1d 61 9e a4 fe 82 36 a5 e5 4a 56 e4
+45 e1 f9 fd b4 16 fa 74 da 9c 9b 35 39 2f fa b0
+20 50 06 6c 7a d0 80 b2 a6 f9 af ec 47 19 8f 50
+38 07 dc a2 87 39 58 f8 ba d5 a9 f9 48 67 30 96
+ee 94 78 5e 6f 89 a3 51 c0 30 86 66 a1 45 66 ba
+54 eb a3 c3 91 f9 48 dc ff d1 e8 30 2d 7d 2d 74
+70 35 d7 88 24 f7 9e c4 59 6e bb 73 87 17 f2 32
+46 28 b8 43 fa b7 1d aa ca b4 f2 9f 24 0e 2d 4b
+f7 71 5c 5e 69 ff ea 95 02 cb 38 8a ae 50 38 6f
+db fb 2d 62 1b c5 c7 1e 54 e1 77 e0 67 c8 0f 9c
+87 23 d6 3f 40 20 7f 20 80 c4 80 4c 3e 3b 24 26
+8e 04 ae 6c 9a c8 aa 0d 02 03 01 00 01 a3 82 01
+31 30 82 01 2d 30 0f 06 03 55 1d 13 01 01 ff 04
+05 30 03 01 01 ff 30 0e 06 03 55 1d 0f 01 01 ff
+04 04 03 02 01 86 30 1d 06 03 55 1d 0e 04 16 04
+14 84 18 cc 85 34 ec bc 0c 94 94 2e 08 59 9c c7
+b2 10 4e 0a 08 30 1f 06 03 55 1d 23 04 18 30 16
+80 14 9c 5f 00 df aa 01 d7 30 2b 38 88 a2 b8 6d
+4a 9c f2 11 91 83 30 78 06 08 2b 06 01 05 05 07
+01 01 04 6c 30 6a 30 2e 06 08 2b 06 01 05 05 07
+30 01 86 22 68 74 74 70 3a 2f 2f 6f 63 73 70 2e
+72 6f 6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75
+73 74 2e 63 6f 6d 30 38 06 08 2b 06 01 05 05 07
+30 02 86 2c 68 74 74 70 3a 2f 2f 63 72 74 2e 72
+6f 6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75 73
+74 2e 63 6f 6d 2f 72 6f 6f 74 67 32 2e 63 65 72
+30 3d 06 03 55 1d 1f 04 36 30 34 30 32 a0 30 a0
+2e 86 2c 68 74 74 70 3a 2f 2f 63 72 6c 2e 72 6f
+6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75 73 74
+2e 63 6f 6d 2f 72 6f 6f 74 67 32 2e 63 72 6c 30
+11 06 03 55 1d 20 04 0a 30 08 30 06 06 04 55 1d
+20 00 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05
+00 03 82 01 01 00 62 37 42 5c bc 10 b5 3e 8b 2c
+e9 0c 9b 6c 45 e2 07 00 7a f9 c5 58 0b b9 08 8c
+3e ed b3 25 3c b5 6f 50 e4 cd 35 6a a7 93 34 96
+32 21 a9 48 44 ab 9c ed 3d b4 aa 73 6d e4 7f 16
+80 89 6c cf 28 03 18 83 47 79 a3 10 7e 30 5b ac
+3b b0 60 e0 77 d4 08 a6 e1 1d 7c 5e c0 bb f9 9a
+7b 22 9d a7 00 09 7e ac 46 17 83 dc 9c 26 57 99
+30 39 62 96 8f ed da de aa c5 cc 1b 3e ca 43 68
+6c 57 16 bc d5 0e 20 2e fe ff c2 6a 5d 2e a0 4a
+6d 14 58 87 94 e6 39 31 5f 7c 73 cb 90 88 6a 84
+11 96 27 a6 ed d9 81 46 a6 7e a3 72 00 0a 52 3e
+83 88 07 63 77 89 69 17 0f 39 85 d2 ab 08 45 4d
+d0 51 3a fd 5d 5d 37 64 4c 7e 30 b2 55 24 42 9d
+36 b0 5d 9c 17 81 61 f1 ca f9 10 02 24 ab eb 0d
+74 91 8d 7b 45 29 50 39 88 b2 a6 89 35 25 1e 14
+6a 47 23 31 2f 5c 9a fa ad 9a 0e 62 51 a4 2a a9
+c4 f9 34 9d 21 18 00 04 79 30 82 04 75 30 82 03
+5d a0 03 02 01 02 02 09 00 a7 0e 4a 4c 34 82 b7
+7f 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 00
+30 68 31 0b 30 09 06 03 55 04 06 13 02 55 53 31
+25 30 23 06 03 55 04 0a 13 1c 53 74 61 72 66 69
+65 6c 64 20 54 65 63 68 6e 6f 6c 6f 67 69 65 73
+2c 20 49 6e 63 2e 31 32 30 30 06 03 55 04 0b 13
+29 53 74 61 72 66 69 65 6c 64 20 43 6c 61 73 73
+20 32 20 43 65 72 74 69 66 69 63 61 74 69 6f 6e
+20 41 75 74 68 6f 72 69 74 79 30 1e 17 0d 30 39
+30 39 30 32 30 30 30 30 30 30 5a 17 0d 33 34 30
+36 32 38 31 37 33 39 31 36 5a 30 81 98 31 0b 30
+09 06 03 55 04 06 13 02 55 53 31 10 30 0e 06 03
+55 04 08 13 07 41 72 69 7a 6f 6e 61 31 13 30 11
+06 03 55 04 07 13 0a 53 63 6f 74 74 73 64 61 6c
+65 31 25 30 23 06 03 55 04 0a 13 1c 53 74 61 72
+66 69 65 6c 64 20 54 65 63 68 6e 6f 6c 6f 67 69
+65 73 2c 20 49 6e 63 2e 31 3b 30 39 06 03 55 04
+03 13 32 53 74 61 72 66 69 65 6c 64 20 53 65 72
+76 69 63 65 73 20 52 6f 6f 74 20 43 65 72 74 69
+66 69 63 61 74 65 20 41 75 74 68 6f 72 69 74 79
+20 2d 20 47 32 30 82 01 22 30 0d 06 09 2a 86 48
+86 f7 0d 01 01 01 05 00 03 82 01 0f 00 30 82 01
+0a 02 82 01 01 00 d5 0c 3a c4 2a f9 4e e2 f5 be
+19 97 5f 8e 88 53 b1 1f 3f cb cf 9f 20 13 6d 29
+3a c8 0f 7d 3c f7 6b 76 38 63 d9 36 60 a8 9b 5e
+5c 00 80 b2 2f 59 7f f6 87 f9 25 43 86 e7 69 1b
+52 9a 90 e1 71 e3 d8 2d 0d 4e 6f f6 c8 49 d9 b6
+f3 1a 56 ae 2b b6 74 14 eb cf fb 26 e3 1a ba 1d
+96 2e 6a 3b 58 94 89 47 56 ff 25 a0 93 70 53 83
+da 84 74 14 c3 67 9e 04 68 3a df 8e 40 5a 1d 4a
+4e cf 43 91 3b e7 56 d6 00 70 cb 52 ee 7b 7d ae
+3a e7 bc 31 f9 45 f6 c2 60 cf 13 59 02 2b 80 cc
+34 47 df b9 de 90 65 6d 02 cf 2c 91 a6 a6 e7 de
+85 18 49 7c 66 4e a3 3a 6d a9 b5 ee 34 2e ba 0d
+03 b8 33 df 47 eb b1 6b 8d 25 d9 9b ce 81 d1 45
+46 32 96 70 87 de 02 0e 49 43 85 b6 6c 73 bb 64
+ea 61 41 ac c9 d4 54 df 87 2f c7 22 b2 26 cc 9f
+59 54 68 9f fc be 2a 2f c4 55 1c 75 40 60 17 85
+02 55 39 8b 7f 05 02 03 01 00 01 a3 81 f0 30 81
+ed 30 0f 06 03 55 1d 13 01 01 ff 04 05 30 03 01
+01 ff 30 0e 06 03 55 1d 0f 01 01 ff 04 04 03 02
+01 86 30 1d 06 03 55 1d 0e 04 16 04 14 9c 5f 00
+df aa 01 d7 30 2b 38 88 a2 b8 6d 4a 9c f2 11 91
+83 30 1f 06 03 55 1d 23 04 18 30 16 80 14 bf 5f
+b7 d1 ce dd 1f 86 f4 5b 55 ac dc d7 10 c2 0e a9
+88 e7 30 4f 06 08 2b 06 01 05 05 07 01 01 04 43
+30 41 30 1c 06 08 2b 06 01 05 05 07 30 01 86 10
+68 74 74 70 3a 2f 2f 6f 2e 73 73 32 2e 75 73 2f
+30 21 06 08 2b 06 01 05 05 07 30 02 86 15 68 74
+74 70 3a 2f 2f 78 2e 73 73 32 2e 75 73 2f 78 2e
+63 65 72 30 26 06 03 55 1d 1f 04 1f 30 1d 30 1b
+a0 19 a0 17 86 15 68 74 74 70 3a 2f 2f 73 2e 73
+73 32 2e 75 73 2f 72 2e 63 72 6c 30 11 06 03 55
+1d 20 04 0a 30 08 30 06 06 04 55 1d 20 00 30 0d
+06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 03 82 01
+01 00 23 1d e3 8a 57 ca 7d e9 17 79 4c f1 1e 55
+fd cc 53 6e 3e 47 0f df c6 55 f2 b2 04 36 ed 80
+1f 53 c4 5d 34 28 6b be c7 55 fc 67 ea cb 3f 7f
+90 b2 33 cd 1b 58 10 82 02 f8 f8 2f f5 13 60 d4
+05 ce f1 81 08 c1 dd a7 75 97 4f 18 b9 6d de f7
+93 91 08 ba 7e 40 2c ed c1 ea bb 76 9e 33 06 77
+1d 0d 08 7f 53 dd 1b 64 ab 82 27 f1 69 d5 4d 5e
+ae f4 a1 c3 75 a7 58 44 2d f2 3c 70 98 ac ba 69
+b6 95 77 7f 0f 31 5e 2c fc a0 87 3a 47 69 f0 79
+5f f4 14 54 a4 95 5e 11 78 12 60 27 ce 9f c2 77
+ff 23 53 77 5d ba ff ea 59 e7 db cf af 92 96 ef
+24 9a 35 10 7a 9c 91 c6 0e 7d 99 f6 3f 19 df f5
+72 54 e1 15 a9 07 59 7b 83 bf 52 2e 46 8c b2 00
+64 76 1c 48 d3 d8 79 e8 6e 56 cc ae 2c 03 90 d7
+19 38 99 e4 ca 09 19 5b ff 07 96 b0 a8 7f 34 49
+df 56 a9 f7 b0 5f ed 33 ed 8c 47 b7 30 03 5d f4
+03 8c 16 03 03 01 4d 0c 00 01 49 03 00 17 41 04
+e4 b3 e7 a8 fe ec e3 99 76 95 8d a5 32 ce de bd
+ce 8c 2b 3d 31 f4 97 88 9c 39 e5 a0 1f 37 ec f9
+3d b1 e8 5c 5e a2 da 02 6d 91 e5 41 51 ca 72 6e
+68 61 77 c8 3a e6 c1 d2 84 c7 4b 64 de 05 06 70
+06 01 01 00 28 3b e1 f9 23 fd 39 af 3a fa d6 f2
+ad e9 92 ae 66 49 90 58 36 19 2d 21 7b 0b 71 48
+ef 61 3f fe a0 db 3c 61 70 7f 3f cf 14 b5 a9 a8
+5b fa 45 45 8c f2 3f 13 5e da 19 f4 91 dd 2d f1
+8a 29 f6 7a 04 66 d7 54 14 38 b6 19 9d 07 66 a0
+1d c2 a5 cc ac a4 50 f1 d0 71 7a 75 84 a5 87 bd
+7b f6 3d 75 6f 66 0f 28 0f 72 17 15 1b a4 fc 16
+9f 97 03 17 3a 88 a9 62 ce bc f3 cf 16 e1 e6 70
+d3 ad 65 dc fb 25 55 88 40 bf 5f b7 35 ac 4b 48
+9e f8 6b ab 6e b7 4d 10 41 a9 c8 19 d2 d7 76 51
+c9 d6 49 92 10 7e 7a 52 d8 ba 25 44 d3 4a 59 56
+15 49 aa 12 b0 ea 56 dc cf b1 43 07 ca cd 61 49
+e3 46 19 a8 1b 24 34 e5 c8 5f a5 cd 81 08 de b6
+6e 19 85 19 fc f0 d5 d1 98 19 1d 1a 09 fc d2 36
+87 c0 19 70 fc ca 1d 37 dc bd 1d 33 c3 8d 6f 2d
+5d 29 4a ec 7f f8 f5 cf ea c8 de 47 8f 6f 17 42
+c4 23 4f 76 16 03 03 00 04 0e 00 00 00

From 3921b90fc0e9502babe20ef880b57d3d712845d5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 31 Dec 2021 23:37:37 +0530
Subject: [PATCH 21/22] pip prod(deps): bump tox from 3.24.4 to 3.24.5 (#924)

Bumps [tox](https://github.com/tox-dev/tox) from 3.24.4 to 3.24.5.
- [Release notes](https://github.com/tox-dev/tox/releases)
- [Changelog](https://github.com/tox-dev/tox/blob/master/docs/changelog.rst)
- [Commits](https://github.com/tox-dev/tox/compare/3.24.4...3.24.5)

---
updated-dependencies:
- dependency-name: tox
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
---
 requirements-testing.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-testing.txt b/requirements-testing.txt
index d65ce96771..76485cd19e 100644
--- a/requirements-testing.txt
+++ b/requirements-testing.txt
@@ -11,7 +11,7 @@ autopep8==1.6.0
 mypy==0.920
 py-spy==0.3.11
 codecov==2.1.12
-tox==3.24.4
+tox==3.24.5
 mccabe==0.6.1
 pylint==2.12.2
 rope==0.22.0

From 4b915c9605ef4f18625c908ce5fc2676d94cd87c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 31 Dec 2021 23:39:55 +0530
Subject: [PATCH 22/22] pip prod(deps): bump twine from 3.7.0 to 3.7.1 (#927)

Bumps [twine](https://github.com/pypa/twine) from 3.7.0 to 3.7.1.
- [Release notes](https://github.com/pypa/twine/releases)
- [Changelog](https://github.com/pypa/twine/blob/main/docs/changelog.rst)
- [Commits](https://github.com/pypa/twine/compare/3.7.0...3.7.1)

---
updated-dependencies:
- dependency-name: twine
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>
---
 requirements-release.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-release.txt b/requirements-release.txt
index fb53c6e485..0abc33fb64 100644
--- a/requirements-release.txt
+++ b/requirements-release.txt
@@ -1,2 +1,2 @@
 setuptools-scm == 6.3.2
-twine==3.7.0
+twine==3.7.1