From 14c3ce29c58c62927139643b2a1feba7deb339c6 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Fri, 14 Jan 2022 19:10:57 +0530 Subject: [PATCH 01/15] Fix README instructions for embedded mode --- README.md | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 6e0e5b2c1a..86fa7fd850 100644 --- a/README.md +++ b/README.md @@ -1326,19 +1326,7 @@ if __name__ == '__main__': proxy.main() ``` -Customize startup flags by passing list of input arguments: - -```python -import proxy - -if __name__ == '__main__': - proxy.main([ - '--hostname', '::1', - '--port', '8899' - ]) -``` - -or, customize startup flags by passing them as kwargs: +Customize startup flags by passing them as kwargs: ```python import ipaddress @@ -1354,7 +1342,8 @@ if __name__ == '__main__': Note that: 1. Calling `main` is simply equivalent to starting `proxy.py` from command line. -2. `main` will block until `proxy.py` shuts down. +2. `main` doesn't accept any `*args`. It will automatically parse any available `sys.argv`. +3. `main` will block until `proxy.py` shuts down. ## Non-blocking Mode @@ -1365,7 +1354,7 @@ by using `Proxy` context manager: Example: import proxy if __name__ == '__main__': - with proxy.Proxy([]) as p: + with proxy.Proxy() as p: # ... your logic here ... ``` @@ -1375,8 +1364,8 @@ Note that: 2. Internally `Proxy` is a context manager. 3. It will start `proxy.py` when called and will shut it down once the scope ends. -4. Just like `main`, startup flags with `Proxy` - can be customized by either passing flags as list of +4. However, unlike `main`, startup flags with `Proxy` + can also be customized by either passing flags as list of input arguments e.g. `Proxy(['--port', '8899'])` or by using passing flags as kwargs e.g. `Proxy(port=8899)`. @@ -1390,7 +1379,7 @@ In embedded mode, you can access this port. Example: import proxy if __name__ == '__main__': - with proxy.Proxy([]) as p: + with proxy.Proxy() as p: print(p.flags.port) ``` @@ -1412,9 +1401,7 @@ Example, load a single plugin using `--plugins` flag: import proxy if __name__ == '__main__': - proxy.main([ - '--plugins', 'proxy.plugin.CacheResponsesPlugin', - ]) + proxy.main(plugins=['proxy.plugin.CacheResponsesPlugin']) ``` For simplicity, you can also pass the list of plugins as a keyword argument to `proxy.main` or the `Proxy` constructor. @@ -1426,7 +1413,7 @@ import proxy from proxy.plugin import FilterByUpstreamHostPlugin if __name__ == '__main__': - proxy.main([], plugins=[ + proxy.main(plugins=[ b'proxy.plugin.CacheResponsesPlugin', FilterByUpstreamHostPlugin, ]) @@ -1436,8 +1423,7 @@ if __name__ == '__main__': ## `proxy.TestCase` -To setup and tear down `proxy.py` for your Python `unittest` classes, -simply use `proxy.TestCase` instead of `unittest.TestCase`. +To setup and tear down `proxy.py` for your Python `unittest` classes, simply use `proxy.TestCase` instead of `unittest.TestCase`. Example: ```python From e7e0315ece1a46b2ed626e8384206b2dc3be3df9 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Fri, 14 Jan 2022 19:50:17 +0530 Subject: [PATCH 02/15] Expose sleep_loop --- README.md | 23 +++++++++++++---------- proxy/__init__.py | 4 +++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 86fa7fd850..11c1c4d088 100644 --- a/README.md +++ b/README.md @@ -1341,8 +1341,9 @@ if __name__ == '__main__': Note that: -1. Calling `main` is simply equivalent to starting `proxy.py` from command line. -2. `main` doesn't accept any `*args`. It will automatically parse any available `sys.argv`. +1. `main` is equivalent to starting `proxy.py` from command line. +2. `main` does not accept any `args` (only `kwargs`). +3. `main` will automatically consume any available `sys.argv` as `args`. 3. `main` will block until `proxy.py` shuts down. ## Non-blocking Mode @@ -1355,19 +1356,20 @@ import proxy if __name__ == '__main__': with proxy.Proxy() as p: - # ... your logic here ... + # Uncomment the line below and + # implement your app your logic here + proxy.sleep_loop() ``` Note that: -1. `Proxy` is similar to `main`, except `Proxy` does not block. -2. Internally `Proxy` is a context manager. -3. It will start `proxy.py` when called and will shut it down - once the scope ends. -4. However, unlike `main`, startup flags with `Proxy` - can also be customized by either passing flags as list of - input arguments e.g. `Proxy(['--port', '8899'])` or +1. `Proxy` is similar to `main`, except `Proxy` will not block. +2. Internally, `Proxy` is a context manager which will start + `proxy.py` when called and will shut it down once the scope ends. +3. Unlike `main`, startup flags with `Proxy` can also be customized + by using `args` and `kwargs`. e.g. `Proxy(['--port', '8899'])` or by using passing flags as kwargs e.g. `Proxy(port=8899)`. +4. Unlike `main`, `Proxy` will not inspect `sys.argv`. ## Ephemeral Port @@ -1381,6 +1383,7 @@ import proxy if __name__ == '__main__': with proxy.Proxy() as p: print(p.flags.port) + proxy.sleep_loop() ``` `flags.port` will give you access to the random port allocated by the kernel. diff --git a/proxy/__init__.py b/proxy/__init__.py index a2e0fa77ad..f08c2afce2 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .proxy import entry_point, main, Proxy +from .proxy import entry_point, main, Proxy, sleep_loop from .testing import TestCase __all__ = [ @@ -22,4 +22,6 @@ # https://github.com/abhinavsingh/proxy.py#unit-testing-with-proxypy 'TestCase', 'Proxy', + # Utility exposed for demos + 'sleep_loop', ] From d17dc9e448716d9f22cd0b6057cba3b1984fa177 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Sun, 16 Jan 2022 00:22:34 +0530 Subject: [PATCH 03/15] [SshTunnel] WIP (#992) [SshTunnel] WIP --- README.md | 18 ++++- docs/conf.py | 1 + proxy/common/constants.py | 1 + proxy/core/ssh/__init__.py | 8 +- proxy/core/ssh/client.py | 28 ------- proxy/core/ssh/handler.py | 34 +++++++++ proxy/core/ssh/listener.py | 135 ++++++++++++++++++++++++++++++++++ proxy/core/ssh/tunnel.py | 70 ------------------ proxy/core/work/threadless.py | 7 +- proxy/proxy.py | 31 +++++++- tests/test_main.py | 6 +- 11 files changed, 231 insertions(+), 108 deletions(-) delete mode 100644 proxy/core/ssh/client.py create mode 100644 proxy/core/ssh/handler.py create mode 100644 proxy/core/ssh/listener.py delete mode 100644 proxy/core/ssh/tunnel.py diff --git a/README.md b/README.md index 11c1c4d088..ec7f71713f 100644 --- a/README.md +++ b/README.md @@ -1272,8 +1272,15 @@ Start `proxy.py` as: --tunnel-username username \ --tunnel-hostname ip.address.or.domain.name \ --tunnel-port 22 \ - --tunnel-remote-host 127.0.0.1 - --tunnel-remote-port 8899 + --tunnel-remote-port 8899 \ + --tunnel-ssh-key /path/to/ssh/private.key \ + --tunnel-ssh-key-passphrase XXXXX +...[redacted]... [I] listener.setup:97 - Listening on 127.0.0.1:8899 +...[redacted]... [I] pool.setup:106 - Started 16 acceptors in threadless (local) mode +...[redacted]... [I] transport._log:1873 - Connected (version 2.0, client OpenSSH_7.6p1) +...[redacted]... [I] transport._log:1873 - Authentication (publickey) successful! +...[redacted]... [I] listener.setup:116 - SSH connection established to ip.address.or.domain.name:22... +...[redacted]... [I] listener.start_port_forward:91 - :8899 forwarding successful... ``` Make a HTTP proxy request on `remote` server and @@ -1312,6 +1319,13 @@ access_log:328 - remote:52067 - GET httpbin.org:80 FIREWALL (allow tcp/22) +Not planned. + +If you have a valid use case, kindly open an issue. You are always welcome to send +contributions via pull-requests to add this functionality :) + +> To proxy local requests remotely, make use of [Proxy Pool Plugin](#proxypoolplugin). + # Embed proxy.py ## Blocking Mode diff --git a/docs/conf.py b/docs/conf.py index 8543ee8014..8daf9fd725 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -284,6 +284,7 @@ (_py_class_role, '_asyncio.Task'), (_py_class_role, 'asyncio.events.AbstractEventLoop'), (_py_class_role, 'CacheStore'), + (_py_class_role, 'Channel'), (_py_class_role, 'HttpParser'), (_py_class_role, 'HttpProtocolHandlerPlugin'), (_py_class_role, 'HttpProxyBasePlugin'), diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 1da2f5e30a..37ef452f54 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -88,6 +88,7 @@ def _env_threadless_compliant() -> bool: DEFAULT_DISABLE_HEADERS: List[bytes] = [] DEFAULT_DISABLE_HTTP_PROXY = False DEFAULT_ENABLE_DASHBOARD = False +DEFAULT_ENABLE_SSH_TUNNEL = False DEFAULT_ENABLE_DEVTOOLS = False DEFAULT_ENABLE_EVENTS = False DEFAULT_EVENTS_QUEUE = None diff --git a/proxy/core/ssh/__init__.py b/proxy/core/ssh/__init__.py index e37310801c..2150e1557c 100644 --- a/proxy/core/ssh/__init__.py +++ b/proxy/core/ssh/__init__.py @@ -12,10 +12,10 @@ Submodules """ -from .client import SshClient -from .tunnel import Tunnel +from .handler import SshHttpProtocolHandler +from .listener import SshTunnelListener __all__ = [ - 'SshClient', - 'Tunnel', + 'SshHttpProtocolHandler', + 'SshTunnelListener', ] diff --git a/proxy/core/ssh/client.py b/proxy/core/ssh/client.py deleted file mode 100644 index 4657b1a3c1..0000000000 --- a/proxy/core/ssh/client.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on - Network monitoring, controls & Application development, testing, debugging. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -import socket -import ssl -from typing import Union - -from ..connection import TcpClientConnection - - -class SshClient(TcpClientConnection): - """Overrides TcpClientConnection. - - This is necessary because paramiko ``fileno()`` can be used for polling - but not for send / recv. - """ - - @property - def connection(self) -> Union[ssl.SSLSocket, socket.socket]: - # Dummy return to comply with - return socket.socket() diff --git a/proxy/core/ssh/handler.py b/proxy/core/ssh/handler.py new file mode 100644 index 0000000000..fdf6f3aa65 --- /dev/null +++ b/proxy/core/ssh/handler.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import argparse + +from typing import TYPE_CHECKING, Tuple + +if TYPE_CHECKING: + try: + from paramiko.channel import Channel + except ImportError: + pass + + +class SshHttpProtocolHandler: + """Handles incoming connections over forwarded SSH transport.""" + + def __init__(self, flags: argparse.Namespace) -> None: + self.flags = flags + + def on_connection( + self, + chan: 'Channel', + origin: Tuple[str, int], + server: Tuple[str, int], + ) -> None: + pass diff --git a/proxy/core/ssh/listener.py b/proxy/core/ssh/listener.py new file mode 100644 index 0000000000..47c9b41419 --- /dev/null +++ b/proxy/core/ssh/listener.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import argparse +import logging + +from typing import TYPE_CHECKING, Any, Callable, Optional, Set, Tuple + +try: + from paramiko import SSHClient, AutoAddPolicy + from paramiko.transport import Transport + if TYPE_CHECKING: + from paramiko.channel import Channel +except ImportError: + pass + +from ...common.flag import flags + +logger = logging.getLogger(__name__) + + +flags.add_argument( + '--tunnel-hostname', + type=str, + default=None, + help='Default: None. Remote hostname or IP address to which SSH tunnel will be established.', +) + +flags.add_argument( + '--tunnel-port', + type=int, + default=22, + help='Default: 22. SSH port of the remote host.', +) + +flags.add_argument( + '--tunnel-username', + type=str, + default=None, + help='Default: None. Username to use for establishing SSH tunnel.', +) + +flags.add_argument( + '--tunnel-ssh-key', + type=str, + default=None, + help='Default: None. Private key path in pem format', +) + +flags.add_argument( + '--tunnel-ssh-key-passphrase', + type=str, + default=None, + help='Default: None. Private key passphrase', +) + +flags.add_argument( + '--tunnel-remote-port', + type=int, + default=8899, + help='Default: 8899. Remote port which will be forwarded locally for proxy.', +) + + +class SshTunnelListener: + """Connects over SSH and forwards a remote port to local host. + + Incoming connections are delegated to provided callback.""" + + def __init__( + self, + flags: argparse.Namespace, + on_connection_callback: Callable[['Channel', Tuple[str, int], Tuple[str, int]], None], + ) -> None: + self.flags = flags + self.on_connection_callback = on_connection_callback + self.ssh: Optional[SSHClient] = None + self.transport: Optional[Transport] = None + self.forwarded: Set[Tuple[str, int]] = set() + + def start_port_forward(self, remote_addr: Tuple[str, int]) -> None: + assert self.transport is not None + self.transport.request_port_forward( + *remote_addr, + handler=self.on_connection_callback, + ) + self.forwarded.add(remote_addr) + logger.info('%s:%d forwarding successful...' % remote_addr) + + def stop_port_forward(self, remote_addr: Tuple[str, int]) -> None: + assert self.transport is not None + self.transport.cancel_port_forward(*remote_addr) + self.forwarded.remove(remote_addr) + + def __enter__(self) -> 'SshTunnelListener': + self.setup() + return self + + def __exit__(self, *args: Any) -> None: + self.shutdown() + + def setup(self) -> None: + self.ssh = SSHClient() + self.ssh.load_system_host_keys() + self.ssh.set_missing_host_key_policy(AutoAddPolicy()) + self.ssh.connect( + hostname=self.flags.tunnel_hostname, + port=self.flags.tunnel_port, + username=self.flags.tunnel_username, + key_filename=self.flags.tunnel_ssh_key, + passphrase=self.flags.tunnel_ssh_key_passphrase, + ) + logger.info( + 'SSH connection established to %s:%d...' % ( + self.flags.tunnel_hostname, + self.flags.tunnel_port, + ), + ) + self.transport = self.ssh.get_transport() + + def shutdown(self) -> None: + for remote_addr in list(self.forwarded): + self.stop_port_forward(remote_addr) + self.forwarded.clear() + if self.transport is not None: + self.transport.close() + if self.ssh is not None: + self.ssh.close() diff --git a/proxy/core/ssh/tunnel.py b/proxy/core/ssh/tunnel.py deleted file mode 100644 index 4a899543ae..0000000000 --- a/proxy/core/ssh/tunnel.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on - Network monitoring, controls & Application development, testing, debugging. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -import logging -import paramiko - -from typing import Optional, Tuple, Callable - -logger = logging.getLogger(__name__) - - -class Tunnel: - """Establishes a tunnel between local (machine where Tunnel is running) and remote host. - Once a tunnel has been established, remote host can route HTTP(s) traffic to - ``localhost`` over tunnel. - """ - - def __init__( - self, - ssh_username: str, - remote_addr: Tuple[str, int], - private_pem_key: str, - remote_proxy_port: int, - conn_handler: Callable[[paramiko.channel.Channel], None], - ) -> None: - self.remote_addr = remote_addr - self.ssh_username = ssh_username - self.private_pem_key = private_pem_key - self.remote_proxy_port = remote_proxy_port - self.conn_handler = conn_handler - - def run(self) -> None: - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - ssh.set_missing_host_key_policy(paramiko.WarningPolicy()) - try: - ssh.connect( - hostname=self.remote_addr[0], - port=self.remote_addr[1], - username=self.ssh_username, - key_filename=self.private_pem_key, - ) - logger.info('SSH connection established...') - transport: Optional[paramiko.transport.Transport] = ssh.get_transport( - ) - assert transport is not None - transport.request_port_forward('', self.remote_proxy_port) - logger.info('Tunnel port forward setup successful...') - while True: - conn: Optional[paramiko.channel.Channel] = transport.accept( - timeout=1, - ) - assert conn is not None - e = transport.get_exception() - if e: - raise e - if conn is None: - continue - self.conn_handler(conn) - except KeyboardInterrupt: - pass - finally: - ssh.close() diff --git a/proxy/core/work/threadless.py b/proxy/core/work/threadless.py index d713e94440..bf50507310 100644 --- a/proxy/core/work/threadless.py +++ b/proxy/core/work/threadless.py @@ -25,7 +25,7 @@ from ...common.constants import DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT from ...common.constants import DEFAULT_WAIT_FOR_TASKS_TIMEOUT -from ..connection import TcpClientConnection, UpstreamConnectionPool +from ..connection import TcpClientConnection from ..event import eventNames if TYPE_CHECKING: # pragma: no cover @@ -91,7 +91,10 @@ def __init__( self.wait_timeout: float = DEFAULT_WAIT_FOR_TASKS_TIMEOUT self.cleanup_inactive_timeout: float = DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT self._total: int = 0 - self._upstream_conn_pool: Optional[UpstreamConnectionPool] = None + # When put at the top, causes circular import error + # since integrated ssh tunnel was introduced. + from ..connection import UpstreamConnectionPool # pylint: disable=C0415 + self._upstream_conn_pool: Optional['UpstreamConnectionPool'] = None self._upstream_conn_filenos: Set[int] = set() if self.flags.enable_conn_pool: self._upstream_conn_pool = UpstreamConnectionPool() diff --git a/proxy/proxy.py b/proxy/proxy.py index b37e80a812..9aed833f69 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -16,15 +16,19 @@ from typing import List, Optional, Any +from proxy.core.ssh.listener import SshTunnelListener + from .core.work import ThreadlessPool from .core.event import EventManager +from .core.ssh import SshHttpProtocolHandler from .core.acceptor import AcceptorPool, Listener from .common.utils import bytes_ from .common.flag import FlagParser, flags -from .common.constants import DEFAULT_LOCAL_EXECUTOR, DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, IS_WINDOWS +from .common.constants import DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_LOCAL_EXECUTOR, DEFAULT_LOG_FILE from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PLUGINS, DEFAULT_VERSION from .common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_WORK_KLASS, DEFAULT_PID_FILE +from .common.constants import DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, IS_WINDOWS logger = logging.getLogger(__name__) @@ -96,6 +100,13 @@ help='Default: False. Enables proxy.py dashboard.', ) +flags.add_argument( + '--enable-ssh-tunnel', + action='store_true', + default=DEFAULT_ENABLE_SSH_TUNNEL, + help='Default: False. Enable SSH tunnel.', +) + flags.add_argument( '--work-klass', type=str, @@ -135,6 +146,8 @@ def __init__(self, input_args: Optional[List[str]] = None, **opts: Any) -> None: self.executors: Optional[ThreadlessPool] = None self.acceptors: Optional[AcceptorPool] = None self.event_manager: Optional[EventManager] = None + self.ssh_http_protocol_handler: Optional[SshHttpProtocolHandler] = None + self.ssh_tunnel_listener: Optional[SshTunnelListener] = None def __enter__(self) -> 'Proxy': self.setup() @@ -193,10 +206,26 @@ def setup(self) -> None: event_queue=event_queue, ) self.acceptors.setup() + # Start SSH tunnel acceptor if enabled + if self.flags.enable_ssh_tunnel: + self.ssh_http_protocol_handler = SshHttpProtocolHandler( + flags=self.flags, + ) + self.ssh_tunnel_listener = SshTunnelListener( + flags=self.flags, + on_connection_callback=self.ssh_http_protocol_handler.on_connection, + ) + self.ssh_tunnel_listener.setup() + self.ssh_tunnel_listener.start_port_forward( + ('', self.flags.tunnel_remote_port), + ) # TODO: May be close listener fd as we don't need it now self._register_signals() def shutdown(self) -> None: + if self.flags.enable_ssh_tunnel: + assert self.ssh_tunnel_listener is not None + self.ssh_tunnel_listener.shutdown() assert self.acceptors self.acceptors.shutdown() if self.remote_executors_enabled: diff --git a/tests/test_main.py b/tests/test_main.py index 8ac149fb53..6b465c47f9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -17,7 +17,7 @@ from proxy.proxy import main, entry_point from proxy.common.utils import bytes_ from proxy.common.constants import ( # noqa: WPS450 - DEFAULT_CA_CERT_DIR, DEFAULT_PORT, DEFAULT_PLUGINS, DEFAULT_TIMEOUT, DEFAULT_KEY_FILE, + DEFAULT_CA_CERT_DIR, DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_PORT, DEFAULT_PLUGINS, DEFAULT_TIMEOUT, DEFAULT_KEY_FILE, DEFAULT_LOG_FILE, DEFAULT_PAC_FILE, DEFAULT_PID_FILE, PLUGIN_DASHBOARD, DEFAULT_CERT_FILE, DEFAULT_LOG_LEVEL, DEFAULT_PORT_FILE, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, DEFAULT_BASIC_AUTH, @@ -75,6 +75,7 @@ def mock_default_args(mock_args: mock.Mock) -> None: mock_args.work_klass = DEFAULT_WORK_KLASS mock_args.local_executor = int(DEFAULT_LOCAL_EXECUTOR) mock_args.port_file = DEFAULT_PORT_FILE + mock_args.enable_ssh_tunnel = DEFAULT_ENABLE_SSH_TUNNEL @mock.patch('os.remove') @mock.patch('os.path.exists') @@ -103,6 +104,7 @@ def test_entry_point( mock_initialize.return_value.enable_events = False mock_initialize.return_value.pid_file = pid_file mock_initialize.return_value.port_file = None + mock_initialize.return_value.enable_ssh_tunnel = False entry_point() mock_event_manager.assert_not_called() mock_listener.assert_called_once_with( @@ -151,6 +153,7 @@ def test_main_with_no_flags( mock_initialize.return_value.local_executor = 0 mock_initialize.return_value.enable_events = False mock_initialize.return_value.port_file = None + mock_initialize.return_value.enable_ssh_tunnel = False main() mock_event_manager.assert_not_called() mock_listener.assert_called_once_with( @@ -192,6 +195,7 @@ def test_enable_events( mock_initialize.return_value.local_executor = 0 mock_initialize.return_value.enable_events = True mock_initialize.return_value.port_file = None + mock_initialize.return_value.enable_ssh_tunnel = False main() mock_event_manager.assert_called_once() mock_event_manager.return_value.setup.assert_called_once() From c6fceb639a5925994f04b15b333413693eb434eb Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Sun, 16 Jan 2022 03:22:55 +0530 Subject: [PATCH 04/15] [Middleware] Capability in the core to allow custom client connection classes (#993) * Move all TCP server related flags within `tcp_server.py` and also move the encryption functionality within TCP base server * Templatize `BaseTcpServerHandler` which now expects a client connection object bound to `TcpClientConnection`. This will allow for custom `HttpClientConnection` object in future to be used by `HttpProtocolHandler` * Pass necessary flags to allow self-signed certificates * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix https integration tests * Affected by https://github.com/abhinavsingh/proxy.py/issues/994 * Fix docs Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Makefile | 2 +- docs/conf.py | 1 + examples/ssl_echo_server.py | 2 +- examples/tcp_echo_server.py | 3 +- proxy/common/pki.py | 4 +- proxy/core/base/tcp_server.py | 87 ++++++++++++++++++++++++++- proxy/core/base/tcp_tunnel.py | 4 +- proxy/http/handler.py | 69 ++++----------------- proxy/http/proxy/server.py | 21 +------ proxy/plugin/web_server_route.py | 2 +- tests/common/test_pki.py | 2 +- tests/integration/test_integration.py | 42 +++++++++++++ tests/integration/test_integration.sh | 43 +++++++++---- 13 files changed, 180 insertions(+), 102 deletions(-) diff --git a/Makefile b/Makefile index 793edfad9c..03232dc2d0 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ sign-https-certificates: python -m proxy.common.pki sign_csr \ --csr-path $(HTTPS_CSR_FILE_PATH) \ --crt-path $(HTTPS_SIGNED_CERT_FILE_PATH) \ - --hostname example.com \ + --hostname localhost \ --private-key-path $(CA_KEY_FILE_PATH) \ --public-key-path $(CA_CERT_FILE_PATH) diff --git a/docs/conf.py b/docs/conf.py index 8daf9fd725..adabd032c0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -316,4 +316,5 @@ (_py_class_role, 'EventQueue'), (_py_obj_role, 'proxy.core.work.threadless.T'), (_py_obj_role, 'proxy.core.work.work.T'), + (_py_obj_role, 'proxy.core.base.tcp_server.T'), ] diff --git a/examples/ssl_echo_server.py b/examples/ssl_echo_server.py index 433af3878d..8103e01051 100644 --- a/examples/ssl_echo_server.py +++ b/examples/ssl_echo_server.py @@ -17,7 +17,7 @@ from proxy.core.connection import TcpClientConnection -class EchoSSLServerHandler(BaseTcpServerHandler): +class EchoSSLServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Wraps client socket during initialization.""" def initialize(self) -> None: diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index cd4924150f..309ac79e21 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -13,9 +13,10 @@ from proxy import Proxy from proxy.core.base import BaseTcpServerHandler +from proxy.core.connection import TcpClientConnection -class EchoServerHandler(BaseTcpServerHandler): +class EchoServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Sets client socket to non-blocking during initialization.""" def initialize(self) -> None: diff --git a/proxy/common/pki.py b/proxy/common/pki.py index 93aab09e46..0ccf695481 100644 --- a/proxy/common/pki.py +++ b/proxy/common/pki.py @@ -268,8 +268,8 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: parser.add_argument( '--subject', type=str, - default='/CN=example.com', - help='Subject to use for public key generation. Default: /CN=example.com', + default='/CN=localhost', + help='Subject to use for public key generation. Default: /CN=localhost', ) parser.add_argument( '--csr-path', diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index edc5361510..66804505b7 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -12,20 +12,75 @@ tcp """ +import ssl +import socket import logging import selectors from abc import abstractmethod -from typing import Any, Optional +from typing import Any, Optional, TypeVar, Union + +from ...common.flag import flags +from ...common.utils import wrap_socket +from ...common.types import Readables, SelectableEvents, Writables +from ...common.constants import DEFAULT_CERT_FILE, DEFAULT_CLIENT_RECVBUF_SIZE +from ...common.constants import DEFAULT_KEY_FILE, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_TIMEOUT from ...core.work import Work from ...core.connection import TcpClientConnection -from ...common.types import Readables, SelectableEvents, Writables logger = logging.getLogger(__name__) -class BaseTcpServerHandler(Work[TcpClientConnection]): +flags.add_argument( + '--key-file', + type=str, + default=DEFAULT_KEY_FILE, + help='Default: None. Server key file to enable end-to-end TLS encryption with clients. ' + 'If used, must also pass --cert-file.', +) + +flags.add_argument( + '--cert-file', + type=str, + default=DEFAULT_CERT_FILE, + help='Default: None. Server certificate to enable end-to-end TLS encryption with clients. ' + 'If used, must also pass --key-file.', +) + +flags.add_argument( + '--client-recvbuf-size', + type=int, + default=DEFAULT_CLIENT_RECVBUF_SIZE, + help='Default: ' + str(int(DEFAULT_CLIENT_RECVBUF_SIZE / 1024)) + + ' KB. Maximum amount of data received from the ' + 'client in a single recv() operation.', +) + +flags.add_argument( + '--server-recvbuf-size', + type=int, + default=DEFAULT_SERVER_RECVBUF_SIZE, + help='Default: ' + str(int(DEFAULT_SERVER_RECVBUF_SIZE / 1024)) + + ' KB. Maximum amount of data received from the ' + 'server in a single recv() operation.', +) + +flags.add_argument( + '--timeout', + type=int, + default=DEFAULT_TIMEOUT, + help='Default: ' + str(DEFAULT_TIMEOUT) + + '. Number of seconds after which ' + 'an inactive connection must be dropped. Inactivity is defined by no ' + 'data sent or received by the client.', +) + + +T = TypeVar('T', bound=TcpClientConnection) + + +class BaseTcpServerHandler(Work[T]): """BaseTcpServerHandler implements Work interface. BaseTcpServerHandler lifecycle is controlled by Threadless core @@ -56,6 +111,14 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.work.address, ) + def initialize(self) -> None: + """Optionally upgrades connection to HTTPS, + sets ``conn`` in non-blocking mode and initializes + HTTP protocol plugins.""" + conn = self._optionally_wrap_socket(self.work.connection) + conn.setblocking(False) + logger.debug('Handling connection %s' % self.work.address) + @abstractmethod def handle_data(self, data: memoryview) -> Optional[bool]: """Optionally return True to close client connection.""" @@ -139,3 +202,21 @@ async def handle_readables(self, readables: Readables) -> bool: else: teardown = True return teardown + + def _encryption_enabled(self) -> bool: + return self.flags.keyfile is not None and \ + self.flags.certfile is not None + + def _optionally_wrap_socket( + self, conn: socket.socket, + ) -> Union[ssl.SSLSocket, socket.socket]: + """Attempts to wrap accepted client connection using provided certificates. + + Shutdown and closes client connection upon error. + """ + if self._encryption_enabled(): + assert self.flags.keyfile and self.flags.certfile + # TODO(abhinavsingh): Insecure TLS versions must not be accepted by default + conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) + self.work._conn = conn + return conn diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index fb230ec0e7..20f0bbeff3 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -18,13 +18,13 @@ from ...common.types import Readables, SelectableEvents, Writables from ...common.utils import text_ -from ..connection import TcpServerConnection +from ..connection import TcpServerConnection, TcpClientConnection from .tcp_server import BaseTcpServerHandler logger = logging.getLogger(__name__) -class BaseTcpTunnelHandler(BaseTcpServerHandler): +class BaseTcpTunnelHandler(BaseTcpServerHandler[TcpClientConnection]): """BaseTcpTunnelHandler build on-top of BaseTcpServerHandler work class. On-top of BaseTcpServerHandler implementation, diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 7966174dbb..1d80b4e984 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -16,15 +16,12 @@ import logging import selectors -from typing import Tuple, List, Type, Union, Optional, Any +from typing import Tuple, List, Type, Optional, Any -from ..common.flag import flags -from ..common.utils import wrap_socket from ..core.base import BaseTcpServerHandler from ..core.connection import TcpClientConnection from ..common.types import Readables, SelectableEvents, Writables -from ..common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_KEY_FILE -from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT, DEFAULT_TIMEOUT +from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT from .exception import HttpProtocolException from .plugin import HttpProtocolHandlerPlugin @@ -35,33 +32,7 @@ logger = logging.getLogger(__name__) -flags.add_argument( - '--client-recvbuf-size', - type=int, - default=DEFAULT_CLIENT_RECVBUF_SIZE, - help='Default: ' + str(int(DEFAULT_CLIENT_RECVBUF_SIZE / 1024)) + - ' KB. Maximum amount of data received from the ' - 'client in a single recv() operation.', -) -flags.add_argument( - '--key-file', - type=str, - default=DEFAULT_KEY_FILE, - help='Default: None. Server key file to enable end-to-end TLS encryption with clients. ' - 'If used, must also pass --cert-file.', -) -flags.add_argument( - '--timeout', - type=int, - default=DEFAULT_TIMEOUT, - help='Default: ' + str(DEFAULT_TIMEOUT) + - '. Number of seconds after which ' - 'an inactive connection must be dropped. Inactivity is defined by no ' - 'data sent or received by the client.', -) - - -class HttpProtocolHandler(BaseTcpServerHandler): +class HttpProtocolHandler(BaseTcpServerHandler[TcpClientConnection]): """HTTP, HTTPS, HTTP2, WebSockets protocol handler. Accepts `Client` connection and delegates to HttpProtocolHandlerPlugin. @@ -86,17 +57,16 @@ def __init__(self, *args: Any, **kwargs: Any): ## def initialize(self) -> None: - """Optionally upgrades connection to HTTPS, - sets ``conn`` in non-blocking mode and initializes - HTTP protocol plugins. - """ - conn = self._optionally_wrap_socket(self.work.connection) - conn.setblocking(False) + super().initialize() # Update client connection reference if connection was wrapped + # This is here in `handler` and not `tcp_server` because + # `tcp_server` is agnostic to constructing TcpClientConnection + # objects. if self._encryption_enabled(): - self.work = TcpClientConnection(conn=conn, addr=self.work.addr) - # self._initialize_plugins() - logger.debug('Handling connection %s' % self.work.address) + self.work = TcpClientConnection( + conn=self.work.connection, + addr=self.work.addr, + ) def is_inactive(self) -> bool: if not self.work.has_buffer() and \ @@ -334,23 +304,6 @@ def _parse_first_request(self, data: memoryview) -> bool: self.work._conn = output return False - def _encryption_enabled(self) -> bool: - return self.flags.keyfile is not None and \ - self.flags.certfile is not None - - def _optionally_wrap_socket( - self, conn: socket.socket, - ) -> Union[ssl.SSLSocket, socket.socket]: - """Attempts to wrap accepted client connection using provided certificates. - - Shutdown and closes client connection upon error. - """ - if self._encryption_enabled(): - assert self.flags.keyfile and self.flags.certfile - # TODO(abhinavsingh): Insecure TLS versions must not be accepted by default - conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) - return conn - def _connection_inactive_for(self) -> float: return time.time() - self.last_activity diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 7029a4333c..f0ccf11361 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -37,9 +37,9 @@ from ...common.types import Readables, Writables, Descriptors from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE -from ...common.constants import COMMA, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CERT_FILE +from ...common.constants import COMMA, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS -from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT +from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH from ...common.utils import text_ from ...common.pki import gen_public_key, gen_csr, sign_csr @@ -52,15 +52,6 @@ logger = logging.getLogger(__name__) -flags.add_argument( - '--server-recvbuf-size', - type=int, - default=DEFAULT_SERVER_RECVBUF_SIZE, - help='Default: ' + str(int(DEFAULT_SERVER_RECVBUF_SIZE / 1024)) + - ' KB. Maximum amount of data received from the ' - 'server in a single recv() operation.', -) - flags.add_argument( '--disable-http-proxy', action='store_true', @@ -116,14 +107,6 @@ 'HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file', ) -flags.add_argument( - '--cert-file', - type=str, - default=DEFAULT_CERT_FILE, - help='Default: None. Server certificate to enable end-to-end TLS encryption with clients. ' - 'If used, must also pass --key-file.', -) - flags.add_argument( '--auth-plugin', type=str, diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index 5f881a68f7..d4ad4effd9 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) HTTP_RESPONSE = okResponse(content=b'HTTP route response') -HTTPS_RESPONSE = okResponse(content=b'HTTP route response') +HTTPS_RESPONSE = okResponse(content=b'HTTPS route response') class WebServerPlugin(HttpWebServerBasePlugin): diff --git a/tests/common/test_pki.py b/tests/common/test_pki.py index 2bbebe06bb..dfe794bee7 100644 --- a/tests/common/test_pki.py +++ b/tests/common/test_pki.py @@ -123,7 +123,7 @@ def test_sign_csr(self) -> None: def _gen_public_private_key(self) -> Tuple[str, str, str]: key_path, nopass_key_path = self._gen_private_key() crt_path = os.path.join(self._tempdir, 'test_gen_public.crt') - pki.gen_public_key(crt_path, key_path, 'password', '/CN=example.com') + pki.gen_public_key(crt_path, key_path, 'password', '/CN=localhost') return (key_path, nopass_key_path, crt_path) def _gen_private_key(self) -> Tuple[str, str]: diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index a6f0ed4ed9..6acee8ee71 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -22,6 +22,13 @@ from proxy.common.constants import IS_WINDOWS +def _https_server_flags() -> str: + return ' '.join(( + '--key-file', 'https-key.pem', + '--cert-file', 'https-signed-cert.pem', + )) + + def _tls_interception_flags(ca_cert_suffix: str = '') -> str: return ' '.join(( '--ca-cert-file', 'ca-cert%s.pem' % ca_cert_suffix, @@ -36,6 +43,12 @@ def _tls_interception_flags(ca_cert_suffix: str = '') -> str: ('--threaded'), ) +PROXY_PY_HTTPS = ( + ('--threadless ' + _https_server_flags()), + ('--threadless --local-executor 0 ' + _https_server_flags()), + ('--threaded ' + _https_server_flags()), +) + PROXY_PY_FLAGS_TLS_INTERCEPTION = ( ('--threadless ' + _tls_interception_flags()), ('--threadless --local-executor 0 ' + _tls_interception_flags()), @@ -73,6 +86,16 @@ def _tls_interception_flags(ca_cert_suffix: str = '') -> str: ) +@pytest.fixture(scope='session', autouse=True) # type: ignore[misc] +def _gen_https_certificates(request: Any) -> None: + check_output([ + 'make', 'https-certificates', + ]) + check_output([ + 'make', 'sign-https-certificates', + ]) + + @pytest.fixture(scope='session', autouse=True) # type: ignore[misc] def _gen_ca_certificates(request: Any) -> None: check_output([ @@ -111,6 +134,7 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]: '--port', '0', '--port-file', str(port_file), '--enable-web-server', + '--plugin', 'proxy.plugin.WebServerPlugin', '--num-acceptors', '3', '--num-workers', '3', '--ca-cert-dir', str(ca_cert_dir), @@ -149,6 +173,24 @@ def test_integration(proxy_py_subprocess: int) -> None: check_output([str(shell_script_test), str(proxy_py_subprocess)]) +@pytest.mark.smoke # type: ignore[misc] +@pytest.mark.parametrize( + 'proxy_py_subprocess', + PROXY_PY_HTTPS, + indirect=True, +) # type: ignore[misc] +@pytest.mark.skipif( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', +) # type: ignore[misc] +def test_https_integration(proxy_py_subprocess: int) -> None: + """An acceptance test for HTTPS web and proxy server using ``curl`` through proxy.py.""" + this_test_module = Path(__file__) + shell_script_test = this_test_module.with_suffix('.sh') + # "1" means use-https scheme for requests to instance + check_output([str(shell_script_test), str(proxy_py_subprocess), '1']) + + @pytest.mark.smoke # type: ignore[misc] @pytest.mark.parametrize( 'proxy_py_subprocess', diff --git a/tests/integration/test_integration.sh b/tests/integration/test_integration.sh index 77c6cc8eba..06cafd2dc2 100755 --- a/tests/integration/test_integration.sh +++ b/tests/integration/test_integration.sh @@ -23,7 +23,20 @@ if [[ -z "$PROXY_PY_PORT" ]]; then exit 1 fi -PROXY_URL="127.0.0.1:$PROXY_PY_PORT" +PROXY_URL="http://localhost:$PROXY_PY_PORT" +TEST_URL="$PROXY_URL/http-route-example" +CURL_EXTRA_FLAGS="" +USE_HTTPS=$2 +if [[ ! -z "$USE_HTTPS" ]]; then + PROXY_URL="https://localhost:$PROXY_PY_PORT" + CURL_EXTRA_FLAGS=" -k --proxy-insecure " + # For https instances we don't use internal https web server + # See https://github.com/abhinavsingh/proxy.py/issues/994 + TEST_URL="http://google.com" + USE_HTTPS=true +else + USE_HTTPS=false +fi # Wait for server to come up WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '" @@ -37,12 +50,9 @@ while true; do done # Wait for http proxy and web server to start +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL" while true; do - curl -v \ - --max-time 1 \ - --connect-timeout 1 \ - -x $PROXY_URL \ - http://$PROXY_URL/ 2>/dev/null + RESPONSE=$($CMD 2> /dev/null) if [[ $? == 0 ]]; then break fi @@ -80,22 +90,27 @@ Disallow: /deny EOM echo "[Test HTTP Request via Proxy]" -CMD="curl -v -x $PROXY_URL http://httpbin.org/robots.txt" +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt" RESPONSE=$($CMD 2> /dev/null) verify_response "$RESPONSE" "$ROBOTS_RESPONSE" VERIFIED1=$? echo "[Test HTTPS Request via Proxy]" -CMD="curl -v -x $PROXY_URL https://httpbin.org/robots.txt" +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt" RESPONSE=$($CMD 2> /dev/null) verify_response "$RESPONSE" "$ROBOTS_RESPONSE" VERIFIED2=$? -echo "[Test Internal Web Server via Proxy]" -curl -v \ - -x $PROXY_URL \ - http://$PROXY_URL/ -VERIFIED3=$? +if $USE_HTTPS; then + VERIFIED3=0 +else + echo "[Test Internal Web Server via Proxy]" + curl -v \ + $CURL_EXTRA_FLAGS \ + -x $PROXY_URL \ + "$PROXY_URL" + VERIFIED3=$? +fi SHASUM=sha256sum if [ "$(uname)" = "Darwin" ]; @@ -107,6 +122,7 @@ echo "[Test Download File Hash Verifies 1]" touch downloaded.hash echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc -" > downloaded.hash curl -vL \ + $CURL_EXTRA_FLAGS \ -o downloaded.whl \ -x $PROXY_URL \ https://files.pythonhosted.org/packages/88/78/e642316313b1cd6396e4b85471a316e003eff968f29773e95ea191ea1d08/proxy.py-2.4.0rc4-py3-none-any.whl#sha256=3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc @@ -118,6 +134,7 @@ echo "[Test Download File Hash Verifies 2]" touch downloaded.hash echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 -" > downloaded.hash curl -vL \ + $CURL_EXTRA_FLAGS \ -o downloaded.whl \ -x $PROXY_URL \ https://files.pythonhosted.org/packages/20/9a/e5d9ec41927401e41aea8af6d16e78b5e612bca4699d417f646a9610a076/Jinja2-3.0.3-py3-none-any.whl#sha256=077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 From 552fb998df4e366e5c4d052c2713c0dc808f28b5 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Sun, 16 Jan 2022 18:34:33 +0530 Subject: [PATCH 05/15] [Middleware] `HttpClientConnection` preparation (#995) * Turn usual suspects to warnings, not error * Add `HttpClientConnection` skeleton * Fix doc build * Update references in http tests * Make `work` core agnostic to work object construction by adding an abstract static method to `Work` interface called `create` * Make mypy happy * Fix tests broken due to change in how work objects are now constructed * Doc ko bhi happy karo --- docs/conf.py | 2 ++ examples/ssl_echo_server.py | 6 +++++- examples/tcp_echo_server.py | 6 +++++- proxy/core/acceptor/acceptor.py | 4 +++- proxy/core/base/tcp_tunnel.py | 4 ++++ proxy/core/connection/pool.py | 8 ++++++-- proxy/core/ssh/handler.py | 2 +- proxy/core/ssh/listener.py | 2 +- proxy/core/work/threaded.py | 10 ++++++---- proxy/core/work/threadless.py | 3 +-- proxy/core/work/work.py | 8 ++++++++ proxy/http/__init__.py | 2 ++ proxy/http/connection.py | 20 +++++++++++++++++++ proxy/http/handler.py | 18 ++++++++--------- proxy/http/inspector/inspect_traffic.py | 9 ++++++--- proxy/http/inspector/transformer.py | 11 ++++++---- proxy/http/plugin.py | 6 +++--- proxy/http/proxy/plugin.py | 4 ++-- proxy/http/proxy/server.py | 4 ++-- proxy/http/server/plugin.py | 4 ++-- proxy/http/websocket/plugin.py | 11 ++++++---- tests/core/test_acceptor.py | 7 +++---- tests/core/test_conn_pool.py | 15 +++++++++++--- .../exceptions/test_http_proxy_auth_failed.py | 5 ++--- tests/http/proxy/test_http_proxy.py | 5 ++--- .../proxy/test_http_proxy_tls_interception.py | 6 +++--- tests/http/test_protocol_handler.py | 13 ++++++------ tests/http/web/test_web_server.py | 19 +++++++++--------- tests/plugin/test_http_proxy_plugins.py | 5 ++--- ...ttp_proxy_plugins_with_tls_interception.py | 6 +++--- 30 files changed, 144 insertions(+), 81 deletions(-) create mode 100644 proxy/http/connection.py diff --git a/docs/conf.py b/docs/conf.py index adabd032c0..350dd216e1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -308,12 +308,14 @@ (_py_class_role, 'unittest.result.TestResult'), (_py_class_role, 'UUID'), (_py_class_role, 'UpstreamConnectionPool'), + (_py_class_role, 'HttpClientConnection'), (_py_class_role, 'Url'), (_py_class_role, 'WebsocketFrame'), (_py_class_role, 'Work'), (_py_class_role, 'proxy.core.acceptor.work.Work'), (_py_class_role, 'connection.Connection'), (_py_class_role, 'EventQueue'), + (_py_class_role, 'T'), (_py_obj_role, 'proxy.core.work.threadless.T'), (_py_obj_role, 'proxy.core.work.work.T'), (_py_obj_role, 'proxy.core.base.tcp_server.T'), diff --git a/examples/ssl_echo_server.py b/examples/ssl_echo_server.py index 8103e01051..c2b25a136f 100644 --- a/examples/ssl_echo_server.py +++ b/examples/ssl_echo_server.py @@ -9,7 +9,7 @@ :license: BSD, see LICENSE for more details. """ import time -from typing import Optional +from typing import Any, Optional from proxy import Proxy from proxy.core.base import BaseTcpServerHandler @@ -20,6 +20,10 @@ class EchoSSLServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Wraps client socket during initialization.""" + @staticmethod + def create(**kwargs: Any) -> TcpClientConnection: + return TcpClientConnection(**kwargs) + def initialize(self) -> None: # Acceptors don't perform TLS handshake. Perform the same # here using wrap_socket() utility. diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index 309ac79e21..22030ce676 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -9,7 +9,7 @@ :license: BSD, see LICENSE for more details. """ import time -from typing import Optional +from typing import Any, Optional from proxy import Proxy from proxy.core.base import BaseTcpServerHandler @@ -19,6 +19,10 @@ class EchoServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Sets client socket to non-blocking during initialization.""" + @staticmethod + def create(**kwargs: Any) -> TcpClientConnection: + return TcpClientConnection(**kwargs) + def initialize(self) -> None: self.work.connection.setblocking(False) diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index f578e0af31..d83c5e00ae 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -224,7 +224,8 @@ def _work(self, conn: socket.socket, addr: Optional[Tuple[str, int]]) -> None: ), ) thread.start() - logger.debug( + # TODO: Move me into target method + logger.debug( # pragma: no cover 'Dispatched work#{0}.{1}.{2} to worker#{3}'.format( conn.fileno(), self.idd, self._total, index, ), @@ -237,6 +238,7 @@ def _work(self, conn: socket.socket, addr: Optional[Tuple[str, int]]) -> None: event_queue=self.event_queue, publisher_id=self.__class__.__name__, ) + # TODO: Move me into target method logger.debug( # pragma: no cover 'Started work#{0}.{1}.{2} in thread#{3}'.format( conn.fileno(), self.idd, self._total, thread.ident, diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index 20f0bbeff3..50320aacc2 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -47,6 +47,10 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: def handle_data(self, data: memoryview) -> Optional[bool]: pass # pragma: no cover + @staticmethod + def create(**kwargs: Any) -> TcpClientConnection: + return TcpClientConnection(**kwargs) + def initialize(self) -> None: self.work.connection.setblocking(False) diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index 399aa5923d..379875e7a4 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -16,7 +16,7 @@ import logging import selectors -from typing import TYPE_CHECKING, Set, Dict, Tuple +from typing import TYPE_CHECKING, Any, Set, Dict, Tuple from ...common.flag import flags from ...common.types import Readables, SelectableEvents, Writables @@ -77,6 +77,10 @@ def __init__(self) -> None: self.connections: Dict[int, TcpServerConnection] = {} self.pools: Dict[Tuple[str, int], Set[TcpServerConnection]] = {} + @staticmethod + def create(**kwargs: Any) -> TcpServerConnection: + return TcpServerConnection(**kwargs) + def acquire(self, addr: Tuple[str, int]) -> Tuple[bool, TcpServerConnection]: """Returns a reusable connection from the pool. @@ -152,7 +156,7 @@ def add(self, addr: Tuple[str, int]) -> TcpServerConnection: NOTE: You must not use the returned connection, instead use `acquire`. """ - new_conn = TcpServerConnection(addr[0], addr[1]) + new_conn = self.create(host=addr[0], port=addr[1]) new_conn.connect() self._add(new_conn) logger.debug( diff --git a/proxy/core/ssh/handler.py b/proxy/core/ssh/handler.py index fdf6f3aa65..1c021f8c86 100644 --- a/proxy/core/ssh/handler.py +++ b/proxy/core/ssh/handler.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Tuple -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover try: from paramiko.channel import Channel except ImportError: diff --git a/proxy/core/ssh/listener.py b/proxy/core/ssh/listener.py index 47c9b41419..352b60d467 100644 --- a/proxy/core/ssh/listener.py +++ b/proxy/core/ssh/listener.py @@ -16,7 +16,7 @@ try: from paramiko import SSHClient, AutoAddPolicy from paramiko.transport import Transport - if TYPE_CHECKING: + if TYPE_CHECKING: # pragma: no cover from paramiko.channel import Channel except ImportError: pass diff --git a/proxy/core/work/threaded.py b/proxy/core/work/threaded.py index 4e583a0ded..7366f8a2c4 100644 --- a/proxy/core/work/threaded.py +++ b/proxy/core/work/threaded.py @@ -12,25 +12,27 @@ import argparse import threading -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple, TypeVar -from ..connection import TcpClientConnection from ..event import EventQueue, eventNames if TYPE_CHECKING: # pragma: no cover from .work import Work +T = TypeVar('T') + +# TODO: Add generic T def start_threaded_work( flags: argparse.Namespace, conn: socket.socket, addr: Optional[Tuple[str, int]], event_queue: Optional[EventQueue] = None, publisher_id: Optional[str] = None, -) -> Tuple['Work[TcpClientConnection]', threading.Thread]: +) -> Tuple['Work[T]', threading.Thread]: """Utility method to start a work in a new thread.""" work = flags.work_klass( - TcpClientConnection(conn, addr), + flags.work_klass.create(conn=conn, addr=addr), flags=flags, event_queue=event_queue, upstream_conn_pool=None, diff --git a/proxy/core/work/threadless.py b/proxy/core/work/threadless.py index bf50507310..5ff7713766 100644 --- a/proxy/core/work/threadless.py +++ b/proxy/core/work/threadless.py @@ -25,7 +25,6 @@ from ...common.constants import DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT from ...common.constants import DEFAULT_WAIT_FOR_TASKS_TIMEOUT -from ..connection import TcpClientConnection from ..event import eventNames if TYPE_CHECKING: # pragma: no cover @@ -138,7 +137,7 @@ def work_on_tcp_conn( ) uid = '%s-%s-%s' % (self.iid, self._total, fileno) self.works[fileno] = self.flags.work_klass( - TcpClientConnection( + self.flags.work_klass.create( conn=conn, addr=addr, ), diff --git a/proxy/core/work/work.py b/proxy/core/work/work.py index 5c94d2bf53..7747d26906 100644 --- a/proxy/core/work/work.py +++ b/proxy/core/work/work.py @@ -47,6 +47,14 @@ def __init__( self.work = work self.upstream_conn_pool = upstream_conn_pool + @staticmethod + @abstractmethod + def create(**kwargs: Any) -> T: + """Implementations are responsible for creation of work objects + from incoming args. This helps keep work core agnostic to + creation of externally defined work class objects.""" + raise NotImplementedError() + @abstractmethod async def get_events(self) -> SelectableEvents: """Return sockets and events (read or write) that we are interested in.""" diff --git a/proxy/http/__init__.py b/proxy/http/__init__.py index b918c3ecf9..9c800b5d0a 100644 --- a/proxy/http/__init__.py +++ b/proxy/http/__init__.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ from .handler import HttpProtocolHandler +from .connection import HttpClientConnection from .plugin import HttpProtocolHandlerPlugin from .codes import httpStatusCodes from .methods import httpMethods @@ -17,6 +18,7 @@ __all__ = [ 'HttpProtocolHandler', + 'HttpClientConnection', 'HttpProtocolHandlerPlugin', 'httpStatusCodes', 'httpMethods', diff --git a/proxy/http/connection.py b/proxy/http/connection.py new file mode 100644 index 0000000000..d31d34036e --- /dev/null +++ b/proxy/http/connection.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable +""" +from ..core.connection import TcpClientConnection + + +class HttpClientConnection(TcpClientConnection): + pass diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 1d80b4e984..7b4a306362 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -19,10 +19,10 @@ from typing import Tuple, List, Type, Optional, Any from ..core.base import BaseTcpServerHandler -from ..core.connection import TcpClientConnection from ..common.types import Readables, SelectableEvents, Writables from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT +from .connection import HttpClientConnection from .exception import HttpProtocolException from .plugin import HttpProtocolHandlerPlugin from .responses import BAD_REQUEST_RESPONSE_PKT @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) -class HttpProtocolHandler(BaseTcpServerHandler[TcpClientConnection]): +class HttpProtocolHandler(BaseTcpServerHandler[HttpClientConnection]): """HTTP, HTTPS, HTTP2, WebSockets protocol handler. Accepts `Client` connection and delegates to HttpProtocolHandlerPlugin. @@ -56,14 +56,14 @@ def __init__(self, *args: Any, **kwargs: Any): # overrides Work class definitions. ## + @staticmethod + def create(**kwargs: Any) -> HttpClientConnection: + return HttpClientConnection(**kwargs) + def initialize(self) -> None: super().initialize() - # Update client connection reference if connection was wrapped - # This is here in `handler` and not `tcp_server` because - # `tcp_server` is agnostic to constructing TcpClientConnection - # objects. if self._encryption_enabled(): - self.work = TcpClientConnection( + self.work = HttpClientConnection( conn=self.work.connection, addr=self.work.addr, ) @@ -204,12 +204,12 @@ async def handle_writables(self, writables: Writables) -> bool: if teardown: return True except BrokenPipeError: - logger.error( + logger.warning( 'BrokenPipeError when flushing buffer for client', ) return True except OSError: - logger.error('OSError when flushing buffer to client') + logger.warning('OSError when flushing buffer to client') return True return False diff --git a/proxy/http/inspector/inspect_traffic.py b/proxy/http/inspector/inspect_traffic.py index 456626d97a..60225f5cf3 100644 --- a/proxy/http/inspector/inspect_traffic.py +++ b/proxy/http/inspector/inspect_traffic.py @@ -9,13 +9,16 @@ :license: BSD, see LICENSE for more details. """ import json -from typing import List, Dict, Any +from typing import TYPE_CHECKING, List, Dict, Any from ...common.utils import bytes_ from ...core.event import EventSubscriber -from ...core.connection import TcpClientConnection + from ..websocket import WebsocketFrame, WebSocketTransportBasePlugin +if TYPE_CHECKING: # pragma: no cover + from ..connection import HttpClientConnection + class InspectTrafficPlugin(WebSocketTransportBasePlugin): """Websocket API for inspect_traffic.ts frontend plugin.""" @@ -58,7 +61,7 @@ def handle_message(self, message: Dict[str, Any]) -> None: raise NotImplementedError() @staticmethod - def callback(client: TcpClientConnection, event: Dict[str, Any]) -> None: + def callback(client: 'HttpClientConnection', event: Dict[str, Any]) -> None: event['push'] = 'inspect_traffic' client.queue( memoryview( diff --git a/proxy/http/inspector/transformer.py b/proxy/http/inspector/transformer.py index da7e1d613e..ea9276dd19 100644 --- a/proxy/http/inspector/transformer.py +++ b/proxy/http/inspector/transformer.py @@ -10,15 +10,18 @@ """ import json import time -from typing import Any, Dict +from typing import TYPE_CHECKING, Any, Dict -from ..websocket import WebsocketFrame from ...common.constants import PROXY_PY_START_TIME, DEFAULT_DEVTOOLS_DOC_URL from ...common.constants import DEFAULT_DEVTOOLS_FRAME_ID, DEFAULT_DEVTOOLS_LOADER_ID from ...common.utils import bytes_ -from ...core.connection import TcpClientConnection from ...core.event import eventNames +from ..websocket import WebsocketFrame + +if TYPE_CHECKING: # pragma: no cover + from ..connection import HttpClientConnection + class CoreEventsToDevtoolsProtocol: """Open in Chrome @@ -30,7 +33,7 @@ class CoreEventsToDevtoolsProtocol: @staticmethod def transformer( - client: TcpClientConnection, + client: 'HttpClientConnection', event: Dict[str, Any], ) -> None: event_name = event['event_name'] diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py index fe44999444..3b587a1712 100644 --- a/proxy/http/plugin.py +++ b/proxy/http/plugin.py @@ -15,9 +15,9 @@ from typing import List, Union, Optional, TYPE_CHECKING from ..core.event import EventQueue -from ..core.connection import TcpClientConnection from .parser import HttpParser +from .connection import HttpClientConnection from .descriptors import DescriptorsHandlerMixin from .mixins import TlsInterceptionPropertyMixin @@ -55,7 +55,7 @@ def __init__( self, uid: str, flags: argparse.Namespace, - client: TcpClientConnection, + client: HttpClientConnection, request: HttpParser, event_queue: Optional[EventQueue] = None, upstream_conn_pool: Optional['UpstreamConnectionPool'] = None, @@ -63,7 +63,7 @@ def __init__( super().__init__(uid, flags, client, event_queue, upstream_conn_pool) self.uid: str = uid self.flags: argparse.Namespace = flags - self.client: TcpClientConnection = client + self.client: HttpClientConnection = client self.request: HttpParser = request self.event_queue = event_queue self.upstream_conn_pool = upstream_conn_pool diff --git a/proxy/http/proxy/plugin.py b/proxy/http/proxy/plugin.py index 2f893d2e59..9f6f114564 100644 --- a/proxy/http/proxy/plugin.py +++ b/proxy/http/proxy/plugin.py @@ -17,9 +17,9 @@ from ..parser import HttpParser from ..descriptors import DescriptorsHandlerMixin +from ..connection import HttpClientConnection from ...core.event import EventQueue -from ...core.connection import TcpClientConnection if TYPE_CHECKING: # pragma: no cover from ...core.connection import UpstreamConnectionPool @@ -38,7 +38,7 @@ def __init__( self, uid: str, flags: argparse.Namespace, - client: TcpClientConnection, + client: HttpClientConnection, event_queue: EventQueue, upstream_conn_pool: Optional['UpstreamConnectionPool'] = None, ) -> None: diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index f0ccf11361..3008898c86 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -196,7 +196,7 @@ async def write_to_descriptors(self, w: Writables) -> bool: ) return False except BrokenPipeError: - logger.error( + logger.warning( 'BrokenPipeError when flushing buffer for server', ) return self._close_and_release() @@ -840,7 +840,7 @@ def wrap_client(self) -> bool: ) do_close = True except BrokenPipeError: - logger.error( + logger.warning( 'BrokenPipeError when wrapping client for upstream: {0}'.format( self.upstream.addr[0], ), diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index dde91a9afa..069b0dff9d 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -16,8 +16,8 @@ from ..websocket import WebsocketFrame from ..parser import HttpParser from ..descriptors import DescriptorsHandlerMixin +from ..connection import HttpClientConnection -from ...core.connection import TcpClientConnection from ...core.event import EventQueue if TYPE_CHECKING: # pragma: no cover @@ -31,7 +31,7 @@ def __init__( self, uid: str, flags: argparse.Namespace, - client: TcpClientConnection, + client: HttpClientConnection, event_queue: EventQueue, upstream_conn_pool: Optional['UpstreamConnectionPool'] = None, ): diff --git a/proxy/http/websocket/plugin.py b/proxy/http/websocket/plugin.py index f6881632a9..f7f274859c 100644 --- a/proxy/http/websocket/plugin.py +++ b/proxy/http/websocket/plugin.py @@ -11,13 +11,16 @@ import argparse import json from abc import ABC, abstractmethod -from typing import List, Dict, Any +from typing import TYPE_CHECKING, List, Dict, Any from ...common.utils import bytes_ -from . import WebsocketFrame -from ...core.connection import TcpClientConnection from ...core.event import EventQueue +from . import WebsocketFrame + +if TYPE_CHECKING: # pragma: no cover + from ..connection import HttpClientConnection + class WebSocketTransportBasePlugin(ABC): """Abstract class for plugins extending dashboard websocket API.""" @@ -25,7 +28,7 @@ class WebSocketTransportBasePlugin(ABC): def __init__( self, flags: argparse.Namespace, - client: TcpClientConnection, + client: 'HttpClientConnection', event_queue: EventQueue, ) -> None: self.flags = flags diff --git a/tests/core/test_acceptor.py b/tests/core/test_acceptor.py index b18f640fa7..376186c539 100644 --- a/tests/core/test_acceptor.py +++ b/tests/core/test_acceptor.py @@ -24,9 +24,10 @@ class TestAcceptor(unittest.TestCase): def setUp(self) -> None: self.acceptor_id = 1 self.pipe = multiprocessing.Pipe() + self.work_klass = mock.MagicMock() self.flags = FlagParser.initialize( threaded=True, - work_klass=mock.MagicMock(), + work_klass=self.work_klass, local_executor=0, ) self.acceptor = Acceptor( @@ -63,7 +64,6 @@ def test_continues_when_no_events( sock.accept.assert_not_called() self.flags.work_klass.assert_not_called() - @mock.patch('proxy.core.work.threaded.TcpClientConnection') @mock.patch('threading.Thread') @mock.patch('selectors.DefaultSelector') @mock.patch('socket.fromfd') @@ -74,7 +74,6 @@ def test_accepts_client_from_server_socket( mock_fromfd: mock.Mock, mock_selector: mock.Mock, mock_thread: mock.Mock, - mock_client: mock.Mock, ) -> None: fileno = 10 conn = mock.MagicMock() @@ -99,7 +98,7 @@ def test_accepts_client_from_server_socket( type=socket.SOCK_STREAM, ) self.flags.work_klass.assert_called_with( - mock_client.return_value, + self.work_klass.create.return_value, flags=self.flags, event_queue=None, upstream_conn_pool=None, diff --git a/tests/core/test_conn_pool.py b/tests/core/test_conn_pool.py index 6547a0de81..3918372101 100644 --- a/tests/core/test_conn_pool.py +++ b/tests/core/test_conn_pool.py @@ -33,7 +33,10 @@ def test_acquire_and_retain_and_reacquire(self, mock_tcp_server_connection: mock mock_conn.closed = False # Acquire created, conn = pool.acquire(addr) - mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1]) + mock_tcp_server_connection.assert_called_once_with( + host=addr[0], + port=addr[1], + ) mock_conn.mark_inuse.assert_called_once() mock_conn.reset.assert_not_called() self.assertTrue(created) @@ -66,7 +69,10 @@ def test_closed_connections_are_removed_on_release( # Acquire created, conn = pool.acquire(addr) self.assertTrue(created) - mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1]) + mock_tcp_server_connection.assert_called_once_with( + host=addr[0], + port=addr[1], + ) self.assertEqual(conn, mock_conn) self.assertEqual(len(pool.pools[addr]), 1) self.assertTrue(conn in pool.pools[addr]) @@ -91,7 +97,10 @@ async def test_get_events(self, mocker: MockerFixture) -> None: mock_conn = mock_tcp_server_connection.return_value addr = mock_conn.addr pool.add(addr) - mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1]) + mock_tcp_server_connection.assert_called_once_with( + host=addr[0], + port=addr[1], + ) mock_conn.connect.assert_called_once() events = await pool.get_events() print(events) diff --git a/tests/http/exceptions/test_http_proxy_auth_failed.py b/tests/http/exceptions/test_http_proxy_auth_failed.py index e0695bcdb0..5d74aecb7a 100644 --- a/tests/http/exceptions/test_http_proxy_auth_failed.py +++ b/tests/http/exceptions/test_http_proxy_auth_failed.py @@ -14,11 +14,10 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpHeaders +from proxy.http import HttpProtocolHandler, httpHeaders, HttpClientConnection from proxy.common.flag import FlagParser from proxy.common.utils import build_http_request from proxy.http.responses import PROXY_AUTH_FAILED_RESPONSE_PKT -from proxy.core.connection import TcpClientConnection from ...test_assertions import Assertions @@ -39,7 +38,7 @@ def _setUp(self, mocker: MockerFixture) -> None: ) self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/http/proxy/test_http_proxy.py b/tests/http/proxy/test_http_proxy.py index 54789ff8a9..68f6bfd999 100644 --- a/tests/http/proxy/test_http_proxy.py +++ b/tests/http/proxy/test_http_proxy.py @@ -14,12 +14,11 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler +from proxy.http import HttpProtocolHandler, HttpClientConnection from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.common.utils import build_http_request from proxy.http.exception import HttpProtocolException -from proxy.core.connection import TcpClientConnection from proxy.common.constants import DEFAULT_HTTP_PORT @@ -43,7 +42,7 @@ def _setUp(self, mocker: MockerFixture) -> None: } self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py index 4acff4d8e7..b13c748d41 100644 --- a/tests/http/proxy/test_http_proxy_tls_interception.py +++ b/tests/http/proxy/test_http_proxy_tls_interception.py @@ -19,12 +19,12 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpMethods +from proxy.http import HttpProtocolHandler, httpMethods, HttpClientConnection from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.common.utils import bytes_, build_http_request from proxy.http.responses import PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT -from proxy.core.connection import TcpClientConnection, TcpServerConnection +from proxy.core.connection import TcpServerConnection from proxy.common.constants import DEFAULT_CA_FILE from ...test_assertions import Assertions @@ -88,7 +88,7 @@ def mock_connection() -> Any: } self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py index 6faf99f34b..ade2101a7c 100644 --- a/tests/http/test_protocol_handler.py +++ b/tests/http/test_protocol_handler.py @@ -17,7 +17,7 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpHeaders +from proxy.http import HttpProtocolHandler, httpHeaders, HttpClientConnection from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates @@ -28,7 +28,6 @@ BAD_GATEWAY_RESPONSE_PKT, PROXY_AUTH_FAILED_RESPONSE_PKT, PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, ) -from proxy.core.connection import TcpClientConnection from proxy.common.constants import ( CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, ) @@ -68,7 +67,7 @@ def _setUp(self, mocker: MockerFixture) -> None: ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() @@ -101,7 +100,7 @@ async def test_proxy_authentication_failed(self) -> None: bytes_(PLUGIN_PROXY_AUTH), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags, + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() self._conn.recv.return_value = CRLF.join([ @@ -138,7 +137,7 @@ def _setUp(self, mocker: MockerFixture) -> None: ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() @@ -301,7 +300,7 @@ async def test_authenticated_proxy_http_get(self) -> None: ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags, + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() assert self.http_server_port is not None @@ -349,7 +348,7 @@ async def test_authenticated_proxy_http_tunnel(self) -> None: ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags, + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() diff --git a/tests/http/web/test_web_server.py b/tests/http/web/test_web_server.py index bae92dd918..e5bbabb39b 100644 --- a/tests/http/web/test_web_server.py +++ b/tests/http/web/test_web_server.py @@ -18,13 +18,12 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler +from proxy.http import HttpProtocolHandler, HttpClientConnection from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates from proxy.common.utils import bytes_, build_http_request, build_http_response from proxy.common.plugins import Plugins from proxy.http.responses import NOT_FOUND_RESPONSE_PKT -from proxy.core.connection import TcpClientConnection from proxy.common.constants import ( CRLF, PROXY_PY_DIR, PLUGIN_PAC_FILE, PLUGIN_HTTP_PROXY, PLUGIN_WEB_SERVER, ) @@ -48,7 +47,7 @@ def test_on_client_connection_called_on_teardown(mocker: MockerFixture) -> None: _conn = mock_fromfd.return_value _addr = ('127.0.0.1', 54382) protocol_handler = HttpProtocolHandler( - TcpClientConnection(_conn, _addr), + HttpClientConnection(_conn, _addr), flags=flags, ) protocol_handler.initialize() @@ -81,7 +80,7 @@ def mock_selector_for_client_read(self: Any) -> None: # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} # self._conn = mock_fromfd.return_value # self.protocol_handler = HttpProtocolHandler( -# TcpClientConnection(self._conn, self._addr), +# HttpClientConnection(self._conn, self._addr), # flags=flags, # ) # self.protocol_handler.initialize() @@ -101,7 +100,7 @@ def mock_selector_for_client_read(self: Any) -> None: # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} # self._conn = mock_fromfd.return_value # self.protocol_handler = HttpProtocolHandler( -# TcpClientConnection(self._conn, self._addr), +# HttpClientConnection(self._conn, self._addr), # flags=flags, # ) # self.protocol_handler.initialize() @@ -121,7 +120,7 @@ def mock_selector_for_client_read(self: Any) -> None: # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} # self._conn = mock_fromfd.return_value # self.protocol_handler = HttpProtocolHandler( -# TcpClientConnection(self._conn, self._addr), +# HttpClientConnection(self._conn, self._addr), # flags=flags, # ) # self.protocol_handler.initialize() @@ -162,7 +161,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: bytes_(PLUGIN_PAC_FILE), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() @@ -221,7 +220,7 @@ def _setUp(self, mocker: MockerFixture) -> None: bytes_(PLUGIN_WEB_SERVER), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() @@ -328,7 +327,7 @@ def _setUp(self, mocker: MockerFixture) -> None: bytes_(PLUGIN_WEB_SERVER), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() @@ -353,7 +352,7 @@ async def test_default_web_server_returns_404(self) -> None: bytes_(PLUGIN_WEB_SERVER), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py index 3550c3295e..742b7fd742 100644 --- a/tests/plugin/test_http_proxy_plugins.py +++ b/tests/plugin/test_http_proxy_plugins.py @@ -20,13 +20,12 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpStatusCodes +from proxy.http import HttpProtocolHandler, httpStatusCodes, HttpClientConnection from proxy.plugin import ProposedRestApiPlugin, RedirectToCustomServerPlugin from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes from proxy.common.utils import bytes_, build_http_request, build_http_response -from proxy.core.connection import TcpClientConnection from proxy.common.constants import DEFAULT_HTTP_PORT, PROXY_AGENT_HEADER_VALUE from .utils import get_plugin_by_test_name from ..test_assertions import Assertions @@ -64,7 +63,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: } self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index 55d09b62a6..a985f0c20f 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -18,7 +18,7 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpMethods +from proxy.http import HttpProtocolHandler, httpMethods, HttpClientConnection from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes @@ -26,7 +26,7 @@ from proxy.http.responses import ( PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, okResponse, ) -from proxy.core.connection import TcpClientConnection, TcpServerConnection +from proxy.core.connection import TcpServerConnection from .utils import get_plugin_by_test_name from ..test_assertions import Assertions @@ -71,7 +71,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: self._conn = mocker.MagicMock(spec=socket.socket) self.mock_fromfd.return_value = self._conn self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=self.flags, + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() From 21499ad1cbb8bc8b0c0f9ca68413529b09d8cd0f Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Tue, 18 Jan 2022 00:11:48 +0530 Subject: [PATCH 06/15] Bail out early for non-HTTP but HTTP looking protocols (#972) * Add support in `Url` to parse all types of schemes * . * Guard handler against http looking protocol but not web or proxy requests * Fix condition for web server protocol detection * doc happy * Update flags and type check imports only --- README.md | 99 ++++++++++++++++----------- proxy/common/flag.py | 4 +- proxy/core/acceptor/pool.py | 11 +-- proxy/http/handler.py | 3 + proxy/http/parser/parser.py | 9 ++- proxy/http/protocols.py | 3 +- proxy/http/url.py | 42 ++++++++---- tests/http/parser/test_http_parser.py | 43 +++++++++++- tests/http/test_url.py | 9 +++ 9 files changed, 155 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index ec7f71713f..3bf8ce98d8 100644 --- a/README.md +++ b/README.md @@ -2206,23 +2206,26 @@ To run standalone benchmark for `proxy.py`, use the following command from repo ```console ❯ proxy -h -usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] - [--threaded] [--num-workers NUM_WORKERS] - [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG] - [--hostname HOSTNAME] [--port PORT] [--port-file PORT_FILE] - [--unix-socket-path UNIX_SOCKET_PATH] - [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL] - [--log-file LOG_FILE] [--log-format LOG_FORMAT] - [--open-file-limit OPEN_FILE_LIMIT] +usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] + [--tunnel-username TUNNEL_USERNAME] + [--tunnel-ssh-key TUNNEL_SSH_KEY] + [--tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE] + [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--enable-events] + [--threadless] [--threaded] [--num-workers NUM_WORKERS] + [--backlog BACKLOG] [--hostname HOSTNAME] [--port PORT] + [--port-file PORT_FILE] [--unix-socket-path UNIX_SOCKET_PATH] + [--local-executor LOCAL_EXECUTOR] [--num-acceptors NUM_ACCEPTORS] + [--version] [--log-level LOG_LEVEL] [--log-file LOG_FILE] + [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT] [--plugins PLUGINS [PLUGINS ...]] [--enable-dashboard] - [--work-klass WORK_KLASS] [--pid-file PID_FILE] - [--enable-proxy-protocol] - [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE] - [--timeout TIMEOUT] [--server-recvbuf-size SERVER_RECVBUF_SIZE] - [--disable-http-proxy] [--disable-headers DISABLE_HEADERS] - [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR] - [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE] - [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--cert-file CERT_FILE] + [--enable-ssh-tunnel] [--work-klass WORK_KLASS] + [--pid-file PID_FILE] [--enable-conn-pool] [--key-file KEY_FILE] + [--cert-file CERT_FILE] [--client-recvbuf-size CLIENT_RECVBUF_SIZE] + [--server-recvbuf-size SERVER_RECVBUF_SIZE] [--timeout TIMEOUT] + [--enable-proxy-protocol] [--disable-http-proxy] + [--disable-headers DISABLE_HEADERS] [--ca-key-file CA_KEY_FILE] + [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE] + [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--auth-plugin AUTH_PLUGIN] [--basic-auth BASIC_AUTH] [--cache-dir CACHE_DIR] [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] @@ -2235,15 +2238,28 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.4.0rc6.dev13+ga9b8034.d20220104 +proxy.py v2.4.0rc7.dev12+gd234339.d20220116 options: -h, --help show this help message and exit + --tunnel-hostname TUNNEL_HOSTNAME + Default: None. Remote hostname or IP address to which + SSH tunnel will be established. + --tunnel-port TUNNEL_PORT + Default: 22. SSH port of the remote host. + --tunnel-username TUNNEL_USERNAME + Default: None. Username to use for establishing SSH + tunnel. + --tunnel-ssh-key TUNNEL_SSH_KEY + Default: None. Private key path in pem format + --tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE + Default: None. Private key passphrase + --tunnel-remote-port TUNNEL_REMOTE_PORT + Default: 8899. Remote port which will be forwarded + locally for proxy. --enable-events Default: False. Enables core to dispatch lifecycle events. Plugins can be used to subscribe for core events. - --enable-conn-pool Default: False. (WIP) Enable upstream connection - pooling. --threadless Default: True. Enabled by default on Python 3.8+ (mac, linux). When disabled a new thread is spawned to handle each client connection. @@ -2252,14 +2268,6 @@ options: handle each client connection. --num-workers NUM_WORKERS Defaults to number of CPU cores. - --local-executor LOCAL_EXECUTOR - Default: 1. Enabled by default. Use 0 to disable. When - enabled acceptors will make use of local (same - process) executor instead of distributing load across - remote (other process) executors. Enable this option - to achieve CPU affinity between acceptors and - executors, instead of using underlying OS kernel - scheduling algorithm. --backlog BACKLOG Default: 100. Maximum number of pending connections to proxy server --hostname HOSTNAME Default: 127.0.0.1. Server IP address. @@ -2270,6 +2278,14 @@ options: --unix-socket-path UNIX_SOCKET_PATH Default: None. Unix socket path to use. When provided --host and --port flags are ignored + --local-executor LOCAL_EXECUTOR + Default: 1. Enabled by default. Use 0 to disable. When + enabled acceptors will make use of local (same + process) executor instead of distributing load across + remote (other process) executors. Enable this option + to achieve CPU affinity between acceptors and + executors, instead of using underlying OS kernel + scheduling algorithm. --num-acceptors NUM_ACCEPTORS Defaults to number of CPU cores. --version, -v Prints proxy.py version. @@ -2288,25 +2304,32 @@ options: Comma separated plugins. You may use --plugins flag multiple times. --enable-dashboard Default: False. Enables proxy.py dashboard. + --enable-ssh-tunnel Default: False. Enable SSH tunnel. --work-klass WORK_KLASS Default: proxy.http.HttpProtocolHandler. Work klass to use for work execution. --pid-file PID_FILE Default: None. Save "parent" process ID to a file. - --enable-proxy-protocol - Default: False. If used, will enable proxy protocol. - Only version 1 is currently supported. - --client-recvbuf-size CLIENT_RECVBUF_SIZE - Default: 128 KB. Maximum amount of data received from - the client in a single recv() operation. + --enable-conn-pool Default: False. (WIP) Enable upstream connection + pooling. --key-file KEY_FILE Default: None. Server key file to enable end-to-end TLS encryption with clients. If used, must also pass --cert-file. - --timeout TIMEOUT Default: 10.0. Number of seconds after which an - inactive connection must be dropped. Inactivity is - defined by no data sent or received by the client. + --cert-file CERT_FILE + Default: None. Server certificate to enable end-to-end + TLS encryption with clients. If used, must also pass + --key-file. + --client-recvbuf-size CLIENT_RECVBUF_SIZE + Default: 128 KB. Maximum amount of data received from + the client in a single recv() operation. --server-recvbuf-size SERVER_RECVBUF_SIZE Default: 128 KB. Maximum amount of data received from the server in a single recv() operation. + --timeout TIMEOUT Default: 10.0. Number of seconds after which an + inactive connection must be dropped. Inactivity is + defined by no data sent or received by the client. + --enable-proxy-protocol + Default: False. If used, will enable proxy protocol. + Only version 1 is currently supported. --disable-http-proxy Default: False. Whether to disable proxy.HttpProxyPlugin. --disable-headers DISABLE_HEADERS @@ -2333,10 +2356,6 @@ options: Default: None. CA signing key to use for dynamic generation of HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file - --cert-file CERT_FILE - Default: None. Server certificate to enable end-to-end - TLS encryption with clients. If used, must also pass - --key-file. --auth-plugin AUTH_PLUGIN Default: proxy.http.proxy.AuthPlugin. Auth plugin to use instead of default basic auth plugin. diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 63bef59818..9244ad38fe 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -307,8 +307,8 @@ def initialize( # See https://github.com/abhinavsingh/proxy.py/pull/714 description # to understand rationale behind the following logic. # - # --num-workers flag or option was found. We will use - # the same value for num_acceptors when --num-acceptors flag + # Num workers flag or option was found. We will use + # the same value for num_acceptors when num acceptors flag # is absent. if num_workers != DEFAULT_NUM_WORKERS and num_acceptors == DEFAULT_NUM_ACCEPTORS: args.num_acceptors = args.num_workers diff --git a/proxy/core/acceptor/pool.py b/proxy/core/acceptor/pool.py index 745d672e38..0e17cd106a 100644 --- a/proxy/core/acceptor/pool.py +++ b/proxy/core/acceptor/pool.py @@ -21,16 +21,17 @@ from multiprocessing import connection from multiprocessing.reduction import send_handle -from typing import Any, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional from .listener import Listener from .acceptor import Acceptor -from ..event import EventQueue - from ...common.flag import flags from ...common.constants import DEFAULT_NUM_ACCEPTORS +if TYPE_CHECKING: # pragma: no cover + from ..event import EventQueue + logger = logging.getLogger(__name__) @@ -69,7 +70,7 @@ def __init__( executor_queues: List[connection.Connection], executor_pids: List[int], executor_locks: List['multiprocessing.synchronize.Lock'], - event_queue: Optional[EventQueue] = None, + event_queue: Optional['EventQueue'] = None, ) -> None: self.flags = flags # File descriptor to use for accepting new work @@ -79,7 +80,7 @@ def __init__( self.executor_pids: List[int] = executor_pids self.executor_locks: List['multiprocessing.synchronize.Lock'] = executor_locks # Eventing core queue - self.event_queue: Optional[EventQueue] = event_queue + self.event_queue: Optional['EventQueue'] = event_queue # Acceptor process instances self.acceptors: List[Acceptor] = [] # Fd queues used to share file descriptor with acceptor processes diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 7b4a306362..c14d3d8263 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -22,6 +22,7 @@ from ..common.types import Readables, SelectableEvents, Writables from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT +from .protocols import httpProtocols from .connection import HttpClientConnection from .exception import HttpProtocolException from .plugin import HttpProtocolHandlerPlugin @@ -260,6 +261,8 @@ def _initialize_plugin( def _discover_plugin_klass(self, protocol: int) -> Optional[Type['HttpProtocolHandlerPlugin']]: """Discovers and return matching HTTP handler plugin matching protocol.""" + if protocol == httpProtocols.UNKNOWN: + return None if b'HttpProtocolHandlerPlugin' in self.flags.plugins: for klass in self.flags.plugins[b'HttpProtocolHandlerPlugin']: k: Type['HttpProtocolHandlerPlugin'] = klass diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index ecb774ef13..daeed72979 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -14,7 +14,7 @@ """ from typing import TypeVar, Optional, Dict, Type, Tuple, List -from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, DEFAULT_ENABLE_PROXY_PROTOCOL +from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, DEFAULT_ENABLE_PROXY_PROTOCOL, HTTP_1_0 from ...common.constants import HTTP_1_1, SLASH, CRLF from ...common.constants import WHITESPACE, DEFAULT_HTTP_PORT from ...common.utils import build_http_request, build_http_response, text_ @@ -157,7 +157,12 @@ def set_url(self, url: bytes) -> None: @property def http_handler_protocol(self) -> int: """Returns `HttpProtocols` that this request belongs to.""" - return httpProtocols.HTTP_PROXY if self.host is not None else httpProtocols.WEB_SERVER + if self.version in (HTTP_1_1, HTTP_1_0) and self._url is not None: + if self.host is not None: + return httpProtocols.HTTP_PROXY + if self._url.hostname is None: + return httpProtocols.WEB_SERVER + return httpProtocols.UNKNOWN @property def is_complete(self) -> bool: diff --git a/proxy/http/protocols.py b/proxy/http/protocols.py index 976a41852b..49485720c3 100644 --- a/proxy/http/protocols.py +++ b/proxy/http/protocols.py @@ -18,6 +18,7 @@ HttpProtocols = NamedTuple( 'HttpProtocols', [ + ('UNKNOWN', int), # Web server handling HTTP/1.0, HTTP/1.1, HTTP/2, HTTP/3 # over plain Text or encrypted connection with clients ('WEB_SERVER', int), @@ -30,4 +31,4 @@ ], ) -httpProtocols = HttpProtocols(1, 2, 3) +httpProtocols = HttpProtocols(1, 2, 3, 4) diff --git a/proxy/http/url.py b/proxy/http/url.py index fc06412b0f..f282b2bd3a 100644 --- a/proxy/http/url.py +++ b/proxy/http/url.py @@ -15,7 +15,7 @@ """ from typing import Optional, Tuple -from ..common.constants import COLON, SLASH, HTTP_URL_PREFIX, HTTPS_URL_PREFIX, AT +from ..common.constants import COLON, SLASH, AT from ..common.utils import text_ @@ -68,29 +68,41 @@ def from_bytes(cls, raw: bytes) -> 'Url': For a HTTPS connect tunnel, url is like ``httpbin.org:443`` For a HTTP proxy request, url is like ``http://httpbin.org/get`` + proxy.py internally never expects a https scheme in the request line. + But `Url` class provides support for parsing any scheme present in the URLs. + e.g. ftp, icap etc. + + If a url with no scheme is parsed, e.g. ``//host/abc.js``, then scheme + defaults to `http`. + Further: 1) URL may contain unicode characters 2) URL may contain IPv4 and IPv6 format addresses instead of domain names - - We use heuristics based approach for our URL parser. """ # SLASH == 47, check if URL starts with single slash but not double slash - is_single_slash = raw[0] == 47 - is_double_slash = is_single_slash and len(raw) >= 2 and raw[1] == 47 - if is_single_slash and not is_double_slash: + starts_with_single_slash = raw[0] == 47 + starts_with_double_slash = starts_with_single_slash and \ + len(raw) >= 2 and \ + raw[1] == 47 + if starts_with_single_slash and \ + not starts_with_double_slash: return cls(remainder=raw) - is_http = raw.startswith(HTTP_URL_PREFIX) - is_https = raw.startswith(HTTPS_URL_PREFIX) - if is_http or is_https or is_double_slash: - rest = raw[len(b'https://'):] \ - if is_https \ - else raw[len(b'http://'):] \ - if is_http \ - else raw[len(SLASH + SLASH):] + scheme = None + rest = None + if not starts_with_double_slash: + # Find scheme + parts = raw.split(b'://', 1) + if len(parts) == 2: + scheme = parts[0] + rest = parts[1] + else: + rest = raw[len(SLASH + SLASH):] + if scheme is not None or starts_with_double_slash: + assert rest is not None parts = rest.split(SLASH, 1) username, password, host, port = Url._parse(parts[0]) return cls( - scheme=b'https' if is_https else b'http', + scheme=scheme if not starts_with_double_slash else b'http', username=username, password=password, hostname=host, diff --git a/tests/http/parser/test_http_parser.py b/tests/http/parser/test_http_parser.py index 69f7e6f9cb..740c683711 100644 --- a/tests/http/parser/test_http_parser.py +++ b/tests/http/parser/test_http_parser.py @@ -678,9 +678,7 @@ def test_is_http_1_1_keep_alive(self) -> None: ) self.assertTrue(self.parser.is_http_1_1_keep_alive) - def test_is_http_1_1_keep_alive_with_non_close_connection_header( - self, - ) -> None: + def test_is_http_1_1_keep_alive_with_non_close_connection_header(self) -> None: self.parser.parse( build_http_request( httpMethods.GET, b'/', @@ -811,3 +809,42 @@ def test_is_safe_against_malicious_requests(self) -> None: b'//198.98.53.25:1389/TomcatBypass/Command/Base64d2dldCA0Ni4xNjEuNTIuMzcvRXhwbG9pd' + b'C5zaDsgY2htb2QgK3ggRXhwbG9pdC5zaDsgLi9FeHBsb2l0LnNoOw==}', ) + + def test_parses_icap_protocol(self) -> None: + # Ref https://datatracker.ietf.org/doc/html/rfc3507 + self.parser.parse( + b'REQMOD icap://icap-server.net/server?arg=87 ICAP/1.0\r\n' + + b'Host: icap-server.net\r\n' + + b'Encapsulated: req-hdr=0, req-body=154' + + b'\r\n\r\n' + + b'POST /origin-resource/form.pl HTTP/1.1\r\n' + + b'Host: www.origin-server.com\r\n' + + b'Accept: text/html, text/plain\r\n' + + b'Accept-Encoding: compress\r\n' + + b'Cache-Control: no-cache\r\n' + + b'\r\n' + + b'1e\r\n' + + b'I am posting this information.\r\n' + + b'0\r\n' + + b'\r\n', + ) + self.assertEqual(self.parser.method, b'REQMOD') + assert self.parser._url is not None + self.assertEqual(self.parser._url.scheme, b'icap') + + def test_cannot_parse_sip_protocol(self) -> None: + # Will fail to parse because of invalid host and port in the request line + # Our Url parser expects an integer port. + with self.assertRaises(ValueError): + self.parser.parse( + b'OPTIONS sip:nm SIP/2.0\r\n' + + b'Via: SIP/2.0/TCP nm;branch=foo\r\n' + + b'From: ;tag=root\r\nTo: \r\n' + + b'Call-ID: 50000\r\n' + + b'CSeq: 42 OPTIONS\r\n' + + b'Max-Forwards: 70\r\n' + + b'Content-Length: 0\r\n' + + b'Contact: \r\n' + + b'Accept: application/sdp\r\n' + + b'\r\n', + ) diff --git a/tests/http/test_url.py b/tests/http/test_url.py index 0cfb8c667a..958dc098bb 100644 --- a/tests/http/test_url.py +++ b/tests/http/test_url.py @@ -143,3 +143,12 @@ def test_no_scheme_suffix(self) -> None: self.assertEqual(url.remainder, b'/server?arg=87') self.assertEqual(url.username, None) self.assertEqual(url.password, None) + + def test_any_scheme_suffix(self) -> None: + url = Url.from_bytes(b'icap://example-server.net/server?arg=87') + self.assertEqual(url.scheme, b'icap') + self.assertEqual(url.hostname, b'example-server.net') + self.assertEqual(url.port, None) + self.assertEqual(url.remainder, b'/server?arg=87') + self.assertEqual(url.username, None) + self.assertEqual(url.password, None) From 2cba03eae0bd555751a008924b4f36caf6ff6380 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 14:55:55 +0530 Subject: [PATCH 07/15] npm: bump eslint-plugin-import from 2.25.3 to 2.25.4 in /dashboard (#1005) Bumps [eslint-plugin-import](https://github.com/import-js/eslint-plugin-import) from 2.25.3 to 2.25.4. - [Release notes](https://github.com/import-js/eslint-plugin-import/releases) - [Changelog](https://github.com/import-js/eslint-plugin-import/blob/main/CHANGELOG.md) - [Commits](https://github.com/import-js/eslint-plugin-import/compare/v2.25.3...v2.25.4) --- updated-dependencies: - dependency-name: eslint-plugin-import dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dashboard/package-lock.json | 61 ++++++++++++------------------------- dashboard/package.json | 2 +- 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 7380b91e55..2b1a63cd7c 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -17,7 +17,7 @@ "chrome-devtools-frontend": "^1.0.956881", "eslint": "^6.8.0", "eslint-config-standard": "^14.1.1", - "eslint-plugin-import": "^2.25.3", + "eslint-plugin-import": "^2.25.4", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^5.0.0", @@ -1209,23 +1209,22 @@ } }, "node_modules/eslint-module-utils": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.1.tgz", - "integrity": "sha512-fjoetBXQZq2tSTWZ9yWVl2KuFrTZZH3V+9iD1V1RfpDgxzJR+mPd/KZmMiA8gbPqdBzpNiEHOuT7IYEWxrH0zQ==", + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.2.tgz", + "integrity": "sha512-zquepFnWCY2ISMFwD/DqzaM++H+7PDzOpUvotJWm/y1BAFt5R4oeULgdrTejKqLkz7MA/tgstsUMNYc7wNdTrg==", "dev": true, "dependencies": { "debug": "^3.2.7", - "find-up": "^2.1.0", - "pkg-dir": "^2.0.0" + "find-up": "^2.1.0" }, "engines": { "node": ">=4" } }, "node_modules/eslint-plugin-import": { - "version": "2.25.3", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.3.tgz", - "integrity": "sha512-RzAVbby+72IB3iOEL8clzPLzL3wpDrlwjsTBAQXgyp5SeTqqY+0bFubwuo+y/HLhNZcXV4XqTBO4LGsfyHIDXg==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.4.tgz", + "integrity": "sha512-/KJBASVFxpu0xg1kIBn9AUa8hQVnszpwgE7Ld0lKAlx7Ie87yzEzCgSkekt+le/YVhiaosO4Y14GDAOc41nfxA==", "dev": true, "dependencies": { "array-includes": "^3.1.4", @@ -1233,14 +1232,14 @@ "debug": "^2.6.9", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.6", - "eslint-module-utils": "^2.7.1", + "eslint-module-utils": "^2.7.2", "has": "^1.0.3", "is-core-module": "^2.8.0", "is-glob": "^4.0.3", "minimatch": "^3.0.4", "object.values": "^1.1.5", "resolve": "^1.20.0", - "tsconfig-paths": "^3.11.0" + "tsconfig-paths": "^3.12.0" }, "engines": { "node": ">=4" @@ -3894,18 +3893,6 @@ "node": ">=0.10.0" } }, - "node_modules/pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", - "dev": true, - "dependencies": { - "find-up": "^2.1.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/pn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", @@ -6407,20 +6394,19 @@ } }, "eslint-module-utils": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.1.tgz", - "integrity": "sha512-fjoetBXQZq2tSTWZ9yWVl2KuFrTZZH3V+9iD1V1RfpDgxzJR+mPd/KZmMiA8gbPqdBzpNiEHOuT7IYEWxrH0zQ==", + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.2.tgz", + "integrity": "sha512-zquepFnWCY2ISMFwD/DqzaM++H+7PDzOpUvotJWm/y1BAFt5R4oeULgdrTejKqLkz7MA/tgstsUMNYc7wNdTrg==", "dev": true, "requires": { "debug": "^3.2.7", - "find-up": "^2.1.0", - "pkg-dir": "^2.0.0" + "find-up": "^2.1.0" } }, "eslint-plugin-import": { - "version": "2.25.3", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.3.tgz", - "integrity": "sha512-RzAVbby+72IB3iOEL8clzPLzL3wpDrlwjsTBAQXgyp5SeTqqY+0bFubwuo+y/HLhNZcXV4XqTBO4LGsfyHIDXg==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.4.tgz", + "integrity": "sha512-/KJBASVFxpu0xg1kIBn9AUa8hQVnszpwgE7Ld0lKAlx7Ie87yzEzCgSkekt+le/YVhiaosO4Y14GDAOc41nfxA==", "dev": true, "requires": { "array-includes": "^3.1.4", @@ -6428,14 +6414,14 @@ "debug": "^2.6.9", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.6", - "eslint-module-utils": "^2.7.1", + "eslint-module-utils": "^2.7.2", "has": "^1.0.3", "is-core-module": "^2.8.0", "is-glob": "^4.0.3", "minimatch": "^3.0.4", "object.values": "^1.1.5", "resolve": "^1.20.0", - "tsconfig-paths": "^3.11.0" + "tsconfig-paths": "^3.12.0" }, "dependencies": { "debug": { @@ -8158,15 +8144,6 @@ "pinkie": "^2.0.0" } }, - "pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", - "dev": true, - "requires": { - "find-up": "^2.1.0" - } - }, "pn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", diff --git a/dashboard/package.json b/dashboard/package.json index 6a83e60bc6..67f5fbb255 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -33,7 +33,7 @@ "chrome-devtools-frontend": "^1.0.956881", "eslint": "^6.8.0", "eslint-config-standard": "^14.1.1", - "eslint-plugin-import": "^2.25.3", + "eslint-plugin-import": "^2.25.4", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^5.0.0", From 7206a5d00f05744daf9d1ad52e8dbc3d1097626f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jan 2022 09:36:42 +0530 Subject: [PATCH 08/15] npm: bump ws from 8.4.0 to 8.4.2 in /dashboard (#1007) Bumps [ws](https://github.com/websockets/ws) from 8.4.0 to 8.4.2. - [Release notes](https://github.com/websockets/ws/releases) - [Commits](https://github.com/websockets/ws/compare/8.4.0...8.4.2) --- updated-dependencies: - dependency-name: ws dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dashboard/package-lock.json | 14 +++++++------- dashboard/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 2b1a63cd7c..bbbd23364d 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -34,7 +34,7 @@ "rollup-plugin-typescript": "^1.0.1", "ts-node": "^7.0.1", "typescript": "^4.5.4", - "ws": "^8.4.0" + "ws": "^8.4.2" } }, "node_modules/@babel/code-frame": { @@ -5051,9 +5051,9 @@ } }, "node_modules/ws": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.0.tgz", - "integrity": "sha512-IHVsKe2pjajSUIl4KYMQOdlyliovpEPquKkqbwswulszzI7r0SfQrxnXdWAEqOlDCLrVSJzo+O1hAwdog2sKSQ==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.2.tgz", + "integrity": "sha512-Kbk4Nxyq7/ZWqr/tarI9yIt/+iNNFOjBXEWgTb4ydaNHBNGgvf2QHbS9fdfsndfjFlFwEd4Al+mw83YkaD10ZA==", "dev": true, "engines": { "node": ">=10.0.0" @@ -9044,9 +9044,9 @@ } }, "ws": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.0.tgz", - "integrity": "sha512-IHVsKe2pjajSUIl4KYMQOdlyliovpEPquKkqbwswulszzI7r0SfQrxnXdWAEqOlDCLrVSJzo+O1hAwdog2sKSQ==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.2.tgz", + "integrity": "sha512-Kbk4Nxyq7/ZWqr/tarI9yIt/+iNNFOjBXEWgTb4ydaNHBNGgvf2QHbS9fdfsndfjFlFwEd4Al+mw83YkaD10ZA==", "dev": true, "requires": {} }, diff --git a/dashboard/package.json b/dashboard/package.json index 67f5fbb255..6596dbc00f 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -50,6 +50,6 @@ "rollup-plugin-typescript": "^1.0.1", "ts-node": "^7.0.1", "typescript": "^4.5.4", - "ws": "^8.4.0" + "ws": "^8.4.2" } } From f7b945800551e6bc981c365045cbe9007877f44c Mon Sep 17 00:00:00 2001 From: Sowmya Sudha Singh <83529764+sowmya-jaxl@users.noreply.github.com> Date: Wed, 19 Jan 2022 15:19:48 +0530 Subject: [PATCH 09/15] Fix broken `--local-executor` logic for windows ever since it was made default (#1008) Co-authored-by: sowmyasudhasingh --- proxy/core/acceptor/acceptor.py | 7 ++++--- proxy/core/work/threadless.py | 2 +- proxy/proxy.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index d83c5e00ae..8fa34afdf8 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -148,7 +148,8 @@ def run_once(self) -> None: if locked: self.lock.release() for work in works: - if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR): + if self.flags.threadless and \ + self.flags.local_executor: assert self._local_work_queue self._local_work_queue.put(work) else: @@ -171,7 +172,7 @@ def run(self) -> None: type=socket.SOCK_STREAM, ) try: - if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR): + if self.flags.threadless and self.flags.local_executor: self._start_local() self.selector.register(self.sock, selectors.EVENT_READ) while not self.running.is_set(): @@ -180,7 +181,7 @@ def run(self) -> None: pass finally: self.selector.unregister(self.sock) - if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR): + if self.flags.threadless and self.flags.local_executor: self._stop_local() self.sock.close() logger.debug('Acceptor#%d shutdown', self.idd) diff --git a/proxy/core/work/threadless.py b/proxy/core/work/threadless.py index 5ff7713766..e5c69a39c6 100644 --- a/proxy/core/work/threadless.py +++ b/proxy/core/work/threadless.py @@ -429,7 +429,7 @@ def run(self) -> None: data=wqfileno, ) assert self.loop - # logger.debug('Working on {0} works'.format(len(self.works))) + logger.debug('Working on {0} works'.format(len(self.works))) self.loop.create_task(self._run_forever()) self.loop.run_forever() except KeyboardInterrupt: diff --git a/proxy/proxy.py b/proxy/proxy.py index 9aed833f69..e2e2af1525 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -25,7 +25,7 @@ from .common.utils import bytes_ from .common.flag import FlagParser, flags -from .common.constants import DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_LOCAL_EXECUTOR, DEFAULT_LOG_FILE +from .common.constants import DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_LOG_FILE from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PLUGINS, DEFAULT_VERSION from .common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_WORK_KLASS, DEFAULT_PID_FILE from .common.constants import DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, IS_WINDOWS @@ -242,7 +242,7 @@ def shutdown(self) -> None: @property def remote_executors_enabled(self) -> bool: return self.flags.threadless and \ - not (self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR)) + not self.flags.local_executor def _write_pid_file(self) -> None: if self.flags.pid_file: From d046cea71c0b2918ea14ea16235d8b3ffbab287a Mon Sep 17 00:00:00 2001 From: Sowmya Sudha Singh <83529764+sowmya-jaxl@users.noreply.github.com> Date: Wed, 19 Jan 2022 18:51:27 +0530 Subject: [PATCH 10/15] [Windows] `--threaded` mode integration tests works locally but fails on GHA (#1009) * Enable remote threadless and threaded integration test for windows * Run only threaded on windows * Use powershell for execution of integration script on Windows * Update test_integration.py * Update test_integration.py Co-authored-by: sowmyasudhasingh Co-authored-by: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> --- tests/integration/test_integration.py | 103 +++++++++++++++++--------- 1 file changed, 66 insertions(+), 37 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 6acee8ee71..e8fd9b7ce5 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -16,12 +16,17 @@ import subprocess from pathlib import Path -from typing import Any, Generator -from subprocess import Popen, check_output +from typing import Any, Generator, List +from subprocess import Popen, check_output as _check_output from proxy.common.constants import IS_WINDOWS +def check_output(args: List[Any]) -> bytes: + args = args if not IS_WINDOWS else ['powershell'] + args + return _check_output(args) + + def _https_server_flags() -> str: return ' '.join(( '--key-file', 'https-key.pem', @@ -37,56 +42,80 @@ def _tls_interception_flags(ca_cert_suffix: str = '') -> str: )) -PROXY_PY_FLAGS_INTEGRATION = ( - ('--threadless'), - ('--threadless --local-executor 0'), +_PROXY_PY_FLAGS_INTEGRATION = [ ('--threaded'), -) - -PROXY_PY_HTTPS = ( - ('--threadless ' + _https_server_flags()), - ('--threadless --local-executor 0 ' + _https_server_flags()), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_INTEGRATION += [ + ('--threadless --local-executor 0'), + ('--threadless'), + ] +PROXY_PY_FLAGS_INTEGRATION = tuple(_PROXY_PY_FLAGS_INTEGRATION) + +_PROXY_PY_HTTPS = [ ('--threaded ' + _https_server_flags()), -) - -PROXY_PY_FLAGS_TLS_INTERCEPTION = ( - ('--threadless ' + _tls_interception_flags()), - ('--threadless --local-executor 0 ' + _tls_interception_flags()), +] +if not IS_WINDOWS: + _PROXY_PY_HTTPS += [ + ('--threadless --local-executor 0 ' + _https_server_flags()), + ('--threadless ' + _https_server_flags()), + ] +PROXY_PY_HTTPS = tuple(_PROXY_PY_HTTPS) + +_PROXY_PY_FLAGS_TLS_INTERCEPTION = [ ('--threaded ' + _tls_interception_flags()), -) - -PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = ( - ( - '--threadless --plugin proxy.plugin.ModifyChunkResponsePlugin ' + - _tls_interception_flags('-chunk') - ), - ( - '--threadless --local-executor 0 --plugin proxy.plugin.ModifyChunkResponsePlugin ' + - _tls_interception_flags('-chunk') - ), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_TLS_INTERCEPTION += [ + ('--threadless --local-executor 0 ' + _tls_interception_flags()), + ('--threadless ' + _tls_interception_flags()), + ] +PROXY_PY_FLAGS_TLS_INTERCEPTION = tuple(_PROXY_PY_FLAGS_TLS_INTERCEPTION) + +_PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = [ ( '--threaded --plugin proxy.plugin.ModifyChunkResponsePlugin ' + _tls_interception_flags('-chunk') ), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN += [ + ( + '--threadless --local-executor 0 --plugin proxy.plugin.ModifyChunkResponsePlugin ' + + _tls_interception_flags('-chunk') + ), + ( + '--threadless --plugin proxy.plugin.ModifyChunkResponsePlugin ' + + _tls_interception_flags('-chunk') + ), + ] +PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = tuple( + _PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN, ) -PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = ( - ( - '--threadless --plugin proxy.plugin.ModifyPostDataPlugin ' + - _tls_interception_flags('-post') - ), - ( - '--threadless --local-executor 0 --plugin proxy.plugin.ModifyPostDataPlugin ' + - _tls_interception_flags('-post') - ), +_PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = [ ( '--threaded --plugin proxy.plugin.ModifyPostDataPlugin ' + _tls_interception_flags('-post') ), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN += [ + ( + '--threadless --local-executor 0 --plugin proxy.plugin.ModifyPostDataPlugin ' + + _tls_interception_flags('-post') + ), + ( + '--threadless --plugin proxy.plugin.ModifyPostDataPlugin ' + + _tls_interception_flags('-post') + ), + ] +PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = tuple( + _PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN, ) -@pytest.fixture(scope='session', autouse=True) # type: ignore[misc] +@pytest.fixture(scope='session', autouse=not IS_WINDOWS) # type: ignore[misc] def _gen_https_certificates(request: Any) -> None: check_output([ 'make', 'https-certificates', @@ -96,7 +125,7 @@ def _gen_https_certificates(request: Any) -> None: ]) -@pytest.fixture(scope='session', autouse=True) # type: ignore[misc] +@pytest.fixture(scope='session', autouse=not IS_WINDOWS) # type: ignore[misc] def _gen_ca_certificates(request: Any) -> None: check_output([ 'make', 'ca-certificates', From efd1cac3ea23583006fb0a4ce7252e6c4664b5f9 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Wed, 19 Jan 2022 19:20:19 +0530 Subject: [PATCH 11/15] Restrict request handling to `DEFAULT_ALLOWED_URL_SCHEMES` (#1002) * Raise `HttpProtocolException` if request line scheme do not match `DEFAULT_ALLOWED_URL_SCHEMES` * ignore WPS329 * Fix tests * Pin to 4.3.2 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Test coverage for exception handling * type ignore Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/requirements.in | 2 +- proxy/common/constants.py | 1 + proxy/core/event/dispatcher.py | 2 +- proxy/http/handler.py | 8 +++++-- proxy/http/parser/parser.py | 23 ++++++++++++++------ proxy/http/url.py | 12 ++++++++--- tests/http/parser/test_http_parser.py | 1 + tests/http/test_protocol_handler.py | 30 ++++++++++++++++++++++++++- tests/http/test_url.py | 10 ++++++++- 9 files changed, 74 insertions(+), 15 deletions(-) diff --git a/docs/requirements.in b/docs/requirements.in index 24f2e03abd..b79321c965 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -1,6 +1,6 @@ myst-parser[linkify] >= 0.15.2 setuptools-scm >= 6.3.2 -Sphinx >= 4.3.0 +Sphinx == 4.3.2 furo >= 2021.11.15 sphinxcontrib-apidoc >= 0.3.0 sphinxcontrib-towncrier >= 0.2.0a0 diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 37ef452f54..04645af08f 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -94,6 +94,7 @@ def _env_threadless_compliant() -> bool: DEFAULT_EVENTS_QUEUE = None DEFAULT_ENABLE_STATIC_SERVER = False DEFAULT_ENABLE_WEB_SERVER = False +DEFAULT_ALLOWED_URL_SCHEMES = [HTTP_PROTO, HTTPS_PROTO] DEFAULT_IPV4_HOSTNAME = ipaddress.IPv4Address('127.0.0.1') DEFAULT_IPV6_HOSTNAME = ipaddress.IPv6Address('::1') DEFAULT_KEY_FILE = None diff --git a/proxy/core/event/dispatcher.py b/proxy/core/event/dispatcher.py index d26340d2ea..49408fdca6 100644 --- a/proxy/core/event/dispatcher.py +++ b/proxy/core/event/dispatcher.py @@ -61,7 +61,7 @@ def handle_event(self, ev: Dict[str, Any]) -> None: }) elif ev['event_name'] == eventNames.UNSUBSCRIBE: # send ack - print('unsubscription request ack sent') + logger.info('unsubscription request ack sent') self.subscribers[ev['event_payload']['sub_id']].send({ 'event_name': eventNames.UNSUBSCRIBED, }) diff --git a/proxy/http/handler.py b/proxy/http/handler.py index c14d3d8263..3d0b042941 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -277,10 +277,14 @@ def _parse_first_request(self, data: memoryview) -> bool: # memoryview compliant try: self.request.parse(data.tobytes()) - except Exception: + except HttpProtocolException as e: # noqa: WPS329 + self.work.queue(BAD_REQUEST_RESPONSE_PKT) + raise e + except Exception as e: + self.work.queue(BAD_REQUEST_RESPONSE_PKT) raise HttpProtocolException( 'Error when parsing request: %r' % data.tobytes(), - ) + ) from e if not self.request.is_complete: return False # Discover which HTTP handler plugin is capable of diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index daeed72979..6adbd558d8 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -149,9 +149,11 @@ def del_headers(self, headers: List[bytes]) -> None: for key in headers: self.del_header(key.lower()) - def set_url(self, url: bytes) -> None: + def set_url(self, url: bytes, allowed_url_schemes: Optional[List[bytes]] = None) -> None: """Given a request line, parses it and sets line attributes a.k.a. host, port, path.""" - self._url = Url.from_bytes(url) + self._url = Url.from_bytes( + url, allowed_url_schemes=allowed_url_schemes, + ) self._set_line_attributes() @property @@ -204,7 +206,7 @@ def body_expected(self) -> bool: """Returns true if content or chunked response is expected.""" return self._content_expected or self._is_chunked_encoded - def parse(self, raw: bytes) -> None: + def parse(self, raw: bytes, allowed_url_schemes: Optional[List[bytes]] = None) -> None: """Parses HTTP request out of raw bytes. Check for `HttpParser.state` after `parse` has successfully returned.""" @@ -217,7 +219,10 @@ def parse(self, raw: bytes) -> None: if self.state >= httpParserStates.HEADERS_COMPLETE: more, raw = self._process_body(raw) elif self.state == httpParserStates.INITIALIZED: - more, raw = self._process_line(raw) + more, raw = self._process_line( + raw, + allowed_url_schemes=allowed_url_schemes, + ) else: more, raw = self._process_headers(raw) # When server sends a response line without any header or body e.g. @@ -345,7 +350,11 @@ def _process_headers(self, raw: bytes) -> Tuple[bool, bytes]: break return len(raw) > 0, raw - def _process_line(self, raw: bytes) -> Tuple[bool, bytes]: + def _process_line( + self, + raw: bytes, + allowed_url_schemes: Optional[List[bytes]] = None, + ) -> Tuple[bool, bytes]: while True: parts = raw.split(CRLF, 1) if len(parts) == 1: @@ -363,7 +372,9 @@ def _process_line(self, raw: bytes) -> Tuple[bool, bytes]: self.method = parts[0] if self.method == httpMethods.CONNECT: self._is_https_tunnel = True - self.set_url(parts[1]) + self.set_url( + parts[1], allowed_url_schemes=allowed_url_schemes, + ) self.version = parts[2] self.state = httpParserStates.LINE_RCVD break diff --git a/proxy/http/url.py b/proxy/http/url.py index f282b2bd3a..9cbdb4ae10 100644 --- a/proxy/http/url.py +++ b/proxy/http/url.py @@ -13,11 +13,13 @@ http url """ -from typing import Optional, Tuple +from typing import List, Optional, Tuple -from ..common.constants import COLON, SLASH, AT +from ..common.constants import COLON, DEFAULT_ALLOWED_URL_SCHEMES, SLASH, AT from ..common.utils import text_ +from .exception import HttpProtocolException + class Url: """``urllib.urlparse`` doesn't work for proxy.py, so we wrote a simple URL. @@ -59,7 +61,7 @@ def __str__(self) -> str: return url @classmethod - def from_bytes(cls, raw: bytes) -> 'Url': + def from_bytes(cls, raw: bytes, allowed_url_schemes: Optional[List[bytes]] = None) -> 'Url': """A URL within proxy.py core can have several styles, because proxy.py supports both proxy and web server use cases. @@ -95,6 +97,10 @@ def from_bytes(cls, raw: bytes) -> 'Url': if len(parts) == 2: scheme = parts[0] rest = parts[1] + if scheme not in (allowed_url_schemes or DEFAULT_ALLOWED_URL_SCHEMES): + raise HttpProtocolException( + 'Invalid scheme received in the request line %r' % raw, + ) else: rest = raw[len(SLASH + SLASH):] if scheme is not None or starts_with_double_slash: diff --git a/tests/http/parser/test_http_parser.py b/tests/http/parser/test_http_parser.py index 740c683711..4ef092e919 100644 --- a/tests/http/parser/test_http_parser.py +++ b/tests/http/parser/test_http_parser.py @@ -827,6 +827,7 @@ def test_parses_icap_protocol(self) -> None: b'I am posting this information.\r\n' + b'0\r\n' + b'\r\n', + allowed_url_schemes=[b'icap'], ) self.assertEqual(self.parser.method, b'REQMOD') assert self.parser._url is not None diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py index ade2101a7c..707725ab6f 100644 --- a/tests/http/test_protocol_handler.py +++ b/tests/http/test_protocol_handler.py @@ -26,7 +26,7 @@ from proxy.common.version import __version__ from proxy.http.responses import ( BAD_GATEWAY_RESPONSE_PKT, PROXY_AUTH_FAILED_RESPONSE_PKT, - PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, BAD_REQUEST_RESPONSE_PKT, ) from proxy.common.constants import ( CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, @@ -114,6 +114,34 @@ async def test_proxy_authentication_failed(self) -> None: PROXY_AUTH_FAILED_RESPONSE_PKT, ) + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_bails_out_for_unknown_schemes(self) -> None: + mock_selector_for_client_read(self) + self._conn.recv.return_value = CRLF.join([ + b'REQMOD icap://icap-server.net/server?arg=87 ICAP/1.0', + b'Host: icap-server.net', + CRLF, + ]) + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.work.buffer[0], + BAD_REQUEST_RESPONSE_PKT, + ) + + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_bails_out_for_sip_request_lines(self) -> None: + mock_selector_for_client_read(self) + self._conn.recv.return_value = CRLF.join([ + b'OPTIONS sip:nm SIP/2.0', + b'Accept: application/sdp', + CRLF, + ]) + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.work.buffer[0], + BAD_REQUEST_RESPONSE_PKT, + ) + class TestHttpProtocolHandler(Assertions): diff --git a/tests/http/test_url.py b/tests/http/test_url.py index 958dc098bb..d246ca77f8 100644 --- a/tests/http/test_url.py +++ b/tests/http/test_url.py @@ -11,6 +11,7 @@ import unittest from proxy.http import Url +from proxy.http.exception import HttpProtocolException class TestUrl(unittest.TestCase): @@ -145,10 +146,17 @@ def test_no_scheme_suffix(self) -> None: self.assertEqual(url.password, None) def test_any_scheme_suffix(self) -> None: - url = Url.from_bytes(b'icap://example-server.net/server?arg=87') + url = Url.from_bytes( + b'icap://example-server.net/server?arg=87', + allowed_url_schemes=[b'icap'], + ) self.assertEqual(url.scheme, b'icap') self.assertEqual(url.hostname, b'example-server.net') self.assertEqual(url.port, None) self.assertEqual(url.remainder, b'/server?arg=87') self.assertEqual(url.username, None) self.assertEqual(url.password, None) + + def test_assert_raises_for_unknown_schemes(self) -> None: + with self.assertRaises(HttpProtocolException): + Url.from_bytes(b'icap://example-server.net/server?arg=87') From b40b91f289e40b622a9b70d5e93638111287a5b9 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Thu, 20 Jan 2022 09:34:05 +0530 Subject: [PATCH 12/15] [Doc] Threadless Remote vs Local Execution Mode (#1011) --- .gitignore | 2 ++ Makefile | 6 ++++++ README.md | 43 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 410b222657..ae8c07d098 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ proxy.py.iml .vscode/* !.vscode/settings.json +*.dot *.pyc *.egg-info *.csr @@ -31,4 +32,5 @@ htmlcov dist build +pyreverse.png profile.svg diff --git a/Makefile b/Makefile index 03232dc2d0..35f35b888c 100644 --- a/Makefile +++ b/Makefile @@ -169,6 +169,12 @@ lib-speedscope: --open-file-limit 65536 \ --log-file /dev/null +lib-pyreverse: + rm -f proxy.proxy.Proxy.dot pyreverse.png + pyreverse -ASmy -c proxy.proxy.Proxy proxy + dot -Tpng proxy.proxy.Proxy.dot > pyreverse.png + open pyreverse.png + devtools: pushd dashboard && npm run devtools && popd diff --git a/README.md b/README.md index 3bf8ce98d8..3b24f3bafd 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ - [Stable vs Develop](#stable-vs-develop) - [Release Schedule](#release-schedule) - [Threads vs Threadless](#threads-vs-threadless) + - [Threadless Remote vs Local Execution Mode](#threadless-remote-vs-local-execution-mode) - [SyntaxError: invalid syntax](#syntaxerror-invalid-syntax) - [Unable to load plugins](#unable-to-load-plugins) - [Unable to connect with proxy.py from remote host](#unable-to-connect-with-proxypy-from-remote-host) @@ -115,6 +116,9 @@ - [High level architecture](#high-level-architecture) - [Everything is a plugin](#everything-is-a-plugin) - [Internal Documentation](#internal-documentation) + - [Read The Doc](#read-the-doc) + - [pydoc](#pydoc) + - [pyreverse](#pyreverse) - [Development Guide](#development-guide) - [Setup Local Environment](#setup-local-environment) - [Setup Git Hooks](#setup-git-hooks) @@ -132,10 +136,8 @@ - Fast & Scalable - Scale up by using all available cores on the system - - Use `--num-acceptors` flag to control number of cores - Threadless executions using asyncio - - Use `--threaded` for synchronous thread based execution mode - Made to handle `tens-of-thousands` connections / sec @@ -186,6 +188,8 @@ [200] 100000 responses ``` + Consult [Threads vs Threadless](#threads-vs-threadless) and [Threadless Remote vs Local Execution Mode](#threadless-remote-vs-local-execution-mode) to control number of CPU cores utilized. + See [Benchmark](https://github.com/abhinavsingh/proxy.py/tree/develop/benchmark#readme) for more details and for how to run benchmarks locally. - Lightweight @@ -1689,11 +1693,24 @@ optional arguments: ## Internal Documentation -Code is well documented. You have a few options to browse the internal class hierarchy and documentation: +### Read The Doc + +- Visit [proxypy.readthedocs.io](https://proxypy.readthedocs.io/) +- Build locally using: + +`make lib-doc` + +### pydoc + +Code is well documented. Grab the source code and run: + +`pydoc3 proxy` + +### pyreverse + +Generate class level hierarchy UML diagrams for in-depth analysis: -1. Visit [proxypy.readthedocs.io](https://proxypy.readthedocs.io/) -2. Build and open docs locally using `make lib-doc` -2. Use `pydoc3` locally using `pydoc3 proxy` +`make lib-pyreverse` # Run Dashboard @@ -1893,6 +1910,20 @@ For `windows` and `Python < 3.8`, you can still try out threadless mode by start If threadless works for you, consider sending a PR by editing `_env_threadless_compliant` method in the `proxy/common/constants.py` file. +## Threadless Remote vs Local execution mode + +Original threadless implementation used `remote` execution mode. This is also depicted under [High level architecture](#high-level-architecture) as ASCII art. + +Under `remote` execution mode, acceptors delegate incoming client connection processing to a remote worker process. By default, acceptors delegate connections in round-robin fashion. Worker processing the request may or may not be running on the same CPU core as the acceptor. This architecture scales well for high throughput, but results in spawning two process per CPU core. + +Example, if there are N-CPUs on the machine, by default, N acceptors and N worker processes are started. You can tune number of processes using `--num-acceptors` and `--num-workers` flag. You might want more workers than acceptors or vice versa depending upon your use case. + +In v2.4.x, `local` execution mode was added, mainly to reduce number of processes spawned by default. This model serves well for day-to-day single user use cases and for developer testing scenarios. Under `local` execution mode, acceptors delegate client connections to a companion thread, instead of a remote process. `local` execution mode ensure CPU affinity, unlike in the `remote` mode where acceptor and worker might be running on different CPU cores. + +`--local-executor 1` was made default in v2.4.x series. Under `local` execution mode, `--num-workers` flag has no effect, as no remote workers are started. + +To use `remote` execution mode, use `--local-executor 0` flag. Then use `--num-workers` to tune number of worker processes. + ## SyntaxError: invalid syntax `proxy.py` is strictly typed and uses Python `typing` annotations. Example: From 7beef281c60cf4751a26466f4ef8fc4a979b579b Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Thu, 20 Jan 2022 11:45:29 +0530 Subject: [PATCH 13/15] [Coverage] For newly added components (#1014) * Add newly added code cov * Fix spelling --- examples/ssl_echo_server.py | 2 +- examples/tcp_echo_server.py | 2 +- proxy/core/base/tcp_tunnel.py | 2 +- proxy/core/connection/pool.py | 2 +- proxy/core/ssh/listener.py | 2 +- proxy/http/handler.py | 14 ++++--- tests/http/parser/test_http_parser.py | 5 +++ tests/test_main.py | 56 +++++++++++++++++++++++---- 8 files changed, 67 insertions(+), 18 deletions(-) diff --git a/examples/ssl_echo_server.py b/examples/ssl_echo_server.py index c2b25a136f..56a7d63ae1 100644 --- a/examples/ssl_echo_server.py +++ b/examples/ssl_echo_server.py @@ -21,7 +21,7 @@ class EchoSSLServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Wraps client socket during initialization.""" @staticmethod - def create(**kwargs: Any) -> TcpClientConnection: + def create(**kwargs: Any) -> TcpClientConnection: # pragma: no cover return TcpClientConnection(**kwargs) def initialize(self) -> None: diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index 22030ce676..5dddd639db 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -20,7 +20,7 @@ class EchoServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Sets client socket to non-blocking during initialization.""" @staticmethod - def create(**kwargs: Any) -> TcpClientConnection: + def create(**kwargs: Any) -> TcpClientConnection: # pragma: no cover return TcpClientConnection(**kwargs) def initialize(self) -> None: diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index 50320aacc2..ea63c27935 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -48,7 +48,7 @@ def handle_data(self, data: memoryview) -> Optional[bool]: pass # pragma: no cover @staticmethod - def create(**kwargs: Any) -> TcpClientConnection: + def create(**kwargs: Any) -> TcpClientConnection: # pragma: no cover return TcpClientConnection(**kwargs) def initialize(self) -> None: diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index 379875e7a4..29e00bd6bc 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -78,7 +78,7 @@ def __init__(self) -> None: self.pools: Dict[Tuple[str, int], Set[TcpServerConnection]] = {} @staticmethod - def create(**kwargs: Any) -> TcpServerConnection: + def create(**kwargs: Any) -> TcpServerConnection: # pragma: no cover return TcpServerConnection(**kwargs) def acquire(self, addr: Tuple[str, int]) -> Tuple[bool, TcpServerConnection]: diff --git a/proxy/core/ssh/listener.py b/proxy/core/ssh/listener.py index 352b60d467..e1bc565b96 100644 --- a/proxy/core/ssh/listener.py +++ b/proxy/core/ssh/listener.py @@ -18,7 +18,7 @@ from paramiko.transport import Transport if TYPE_CHECKING: # pragma: no cover from paramiko.channel import Channel -except ImportError: +except ImportError: # pragma: no cover pass from ...common.flag import flags diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 3d0b042941..8b2952f850 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -58,7 +58,7 @@ def __init__(self, *args: Any, **kwargs: Any): ## @staticmethod - def create(**kwargs: Any) -> HttpClientConnection: + def create(**kwargs: Any) -> HttpClientConnection: # pragma: no cover return HttpClientConnection(**kwargs) def initialize(self) -> None: @@ -205,12 +205,14 @@ async def handle_writables(self, writables: Writables) -> bool: if teardown: return True except BrokenPipeError: - logger.warning( + logger.warning( # pragma: no cover 'BrokenPipeError when flushing buffer for client', ) return True except OSError: - logger.warning('OSError when flushing buffer to client') + logger.warning( # pragma: no cover + 'OSError when flushing buffer to client', + ) return True return False @@ -261,8 +263,6 @@ def _initialize_plugin( def _discover_plugin_klass(self, protocol: int) -> Optional[Type['HttpProtocolHandlerPlugin']]: """Discovers and return matching HTTP handler plugin matching protocol.""" - if protocol == httpProtocols.UNKNOWN: - return None if b'HttpProtocolHandlerPlugin' in self.flags.plugins: for klass in self.flags.plugins[b'HttpProtocolHandlerPlugin']: k: Type['HttpProtocolHandlerPlugin'] = klass @@ -287,6 +287,10 @@ def _parse_first_request(self, data: memoryview) -> bool: ) from e if not self.request.is_complete: return False + # Bail out if http protocol is unknown + if self.request.http_handler_protocol == httpProtocols.UNKNOWN: + self.work.queue(BAD_REQUEST_RESPONSE_PKT) + return True # Discover which HTTP handler plugin is capable of # handling the current incoming request klass = self._discover_plugin_klass( diff --git a/tests/http/parser/test_http_parser.py b/tests/http/parser/test_http_parser.py index 4ef092e919..562574d4e9 100644 --- a/tests/http/parser/test_http_parser.py +++ b/tests/http/parser/test_http_parser.py @@ -17,6 +17,7 @@ ) from proxy.http.exception import HttpProtocolException from proxy.http.responses import okResponse +from proxy.http.protocols import httpProtocols from proxy.common.constants import CRLF, HTTP_1_0 @@ -832,6 +833,10 @@ def test_parses_icap_protocol(self) -> None: self.assertEqual(self.parser.method, b'REQMOD') assert self.parser._url is not None self.assertEqual(self.parser._url.scheme, b'icap') + self.assertEqual( + self.parser.http_handler_protocol, + httpProtocols.UNKNOWN, + ) def test_cannot_parse_sip_protocol(self) -> None: # Will fail to parse because of invalid host and port in the request line diff --git a/tests/test_main.py b/tests/test_main.py index 6b465c47f9..070b134bd5 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -308,11 +308,51 @@ def test_enable_devtools( mock_acceptor_pool.return_value.setup.assert_called_once() mock_listener.return_value.setup.assert_called_once() - # def test_pac_file(self) -> None: - # pass - - # def test_imports_plugin(self) -> None: - # pass - - # def test_cannot_enable_https_proxy_and_tls_interception_mutually(self) -> None: - # pass + @mock.patch('time.sleep') + @mock.patch('proxy.common.plugins.Plugins.load') + @mock.patch('proxy.common.flag.FlagParser.parse_args') + @mock.patch('proxy.proxy.EventManager') + @mock.patch('proxy.proxy.AcceptorPool') + @mock.patch('proxy.proxy.ThreadlessPool') + @mock.patch('proxy.proxy.Listener') + @mock.patch('proxy.proxy.SshHttpProtocolHandler') + @mock.patch('proxy.proxy.SshTunnelListener') + def test_enable_ssh_tunnel( + self, + mock_ssh_tunnel_listener: mock.Mock, + mock_ssh_http_proto_handler: mock.Mock, + mock_listener: mock.Mock, + mock_executor_pool: mock.Mock, + mock_acceptor_pool: mock.Mock, + mock_event_manager: mock.Mock, + mock_parse_args: mock.Mock, + mock_load_plugins: mock.Mock, + mock_sleep: mock.Mock, + ) -> None: + mock_sleep.side_effect = KeyboardInterrupt() + mock_args = mock_parse_args.return_value + self.mock_default_args(mock_args) + mock_args.enable_ssh_tunnel = True + mock_args.local_executor = 0 + main(enable_ssh_tunnel=True, local_executor=0) + mock_load_plugins.assert_called() + self.assertEqual( + mock_load_plugins.call_args_list[0][0][0], [ + bytes_(PLUGIN_HTTP_PROXY), + ], + ) + mock_parse_args.assert_called_once() + mock_event_manager.assert_not_called() + if _env_threadless_compliant(): + mock_executor_pool.assert_called_once() + mock_executor_pool.return_value.setup.assert_called_once() + mock_acceptor_pool.assert_called_once() + mock_acceptor_pool.return_value.setup.assert_called_once() + mock_listener.return_value.setup.assert_called_once() + mock_ssh_http_proto_handler.assert_called_once() + mock_ssh_tunnel_listener.assert_called_once() + mock_ssh_tunnel_listener.return_value.setup.assert_called_once() + mock_ssh_tunnel_listener.return_value.start_port_forward.assert_called_once() + mock_ssh_tunnel_listener.return_value.shutdown.assert_called_once() + # shutdown will internally call stop port forward + mock_ssh_tunnel_listener.return_value.stop_port_forward.assert_not_called() From 77b69e66601ce804740717d03cdc4ffe95da40c8 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Thu, 20 Jan 2022 11:46:27 +0530 Subject: [PATCH 14/15] [Devtools] Build as part of GHA workflow (#1015) * Fix devtools build * Build devtools as part of GHA workflows --- .github/workflows/test-library.yml | 5 +++++ Makefile | 4 ++-- dashboard/src/core/devtools.ts | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 785df5a80c..ef6f014860 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -703,6 +703,11 @@ jobs: cd dashboard npm run build cd .. + - name: Build Devtools + run: | + cd dashboard + npm run devtools + cd .. developer: runs-on: ${{ matrix.os }}-latest diff --git a/Makefile b/Makefile index 35f35b888c..51f12a4412 100644 --- a/Makefile +++ b/Makefile @@ -176,10 +176,10 @@ lib-pyreverse: open pyreverse.png devtools: - pushd dashboard && npm run devtools && popd + pushd dashboard && npm install && npm run devtools && popd dashboard: - pushd dashboard && npm run build && popd + pushd dashboard && npm install && npm run build && popd dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi diff --git a/dashboard/src/core/devtools.ts b/dashboard/src/core/devtools.ts index 77fc789451..fe933f9ab7 100644 --- a/dashboard/src/core/devtools.ts +++ b/dashboard/src/core/devtools.ts @@ -31,7 +31,7 @@ function setUpDevTools () { } const chromeDevTools = path.dirname( - require.resolve('chrome-devtools-frontend/front_end/inspector.json') + require.resolve('chrome-devtools-frontend/front_end/visibility.gni') ) console.log('Destination folder: ' + destinationFolderPath) From 54e74a911f382656fc49cb57def0c8f29c1b7582 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Thu, 20 Jan 2022 15:34:54 +0530 Subject: [PATCH 15/15] [isort] Lib modules (#1016) * isort `proxy.py` main class * isort init and main * isort common * pre-commit fix * isort dashboard and testing * isort plugins * isort core * Only sort top level http py files * isort http exception and websocket * Remove proxy auth plugin from proxy package exports and force discover `PLUGIN_PROXY_AUTH` flags * isort parser and web server * no setattr * isort all * Enable pre-commit isort hook * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 16 ++++----- benchmark/_blacksheep.py | 1 - benchmark/_proxy.py | 1 + benchmark/_starlette.py | 5 ++- benchmark/_tornado.py | 2 +- docs/_ext/spelling_stub_ext.py | 2 +- docs/conf.py | 2 +- examples/web_scraper.py | 2 +- proxy/__init__.py | 3 +- proxy/__main__.py | 1 + proxy/common/_scm_version.pyi | 1 + proxy/common/_version.py | 4 ++- proxy/common/backports.py | 3 +- proxy/common/constants.py | 8 ++--- proxy/common/flag.py | 32 ++++++++++------- proxy/common/logger.py | 4 +-- proxy/common/pki.py | 9 +++-- proxy/common/plugins.py | 10 +++--- proxy/common/types.py | 3 +- proxy/common/utils.py | 10 +++--- proxy/common/version.py | 2 +- proxy/core/acceptor/__init__.py | 5 +-- proxy/core/acceptor/acceptor.py | 9 ++--- proxy/core/acceptor/listener.py | 7 ++-- proxy/core/acceptor/pool.py | 8 ++--- proxy/core/base/__init__.py | 1 + proxy/core/base/tcp_server.py | 15 ++++---- proxy/core/base/tcp_tunnel.py | 7 ++-- proxy/core/base/tcp_upstream.py | 4 +-- proxy/core/connection/__init__.py | 7 ++-- proxy/core/connection/client.py | 5 ++- proxy/core/connection/connection.py | 5 ++- proxy/core/connection/pool.py | 8 ++--- proxy/core/connection/server.py | 8 ++--- proxy/core/event/__init__.py | 5 +-- proxy/core/event/dispatcher.py | 7 ++-- proxy/core/event/manager.py | 3 +- proxy/core/event/names.py | 1 + proxy/core/event/queue.py | 7 ++-- proxy/core/event/subscriber.py | 7 ++-- proxy/core/ssh/__init__.py | 1 + proxy/core/ssh/handler.py | 2 +- proxy/core/ssh/listener.py | 5 +-- proxy/core/work/__init__.py | 7 ++-- proxy/core/work/delegate.py | 3 +- proxy/core/work/local.py | 12 +++---- proxy/core/work/pool.py | 7 ++-- proxy/core/work/remote.py | 4 +-- proxy/core/work/threaded.py | 4 +-- proxy/core/work/threadless.py | 25 +++++++------ proxy/core/work/work.py | 10 +++--- proxy/dashboard/__init__.py | 1 + proxy/dashboard/dashboard.py | 7 ++-- proxy/http/__init__.py | 11 +++--- proxy/http/descriptors.py | 1 + proxy/http/exception/__init__.py | 3 +- proxy/http/exception/base.py | 3 +- proxy/http/exception/http_request_rejected.py | 4 +-- proxy/http/exception/proxy_auth_failed.py | 2 +- proxy/http/exception/proxy_conn_failed.py | 2 +- proxy/http/handler.py | 18 +++++----- proxy/http/inspector/devtools.py | 16 +++++---- proxy/http/inspector/inspect_traffic.py | 6 ++-- proxy/http/inspector/transformer.py | 10 +++--- proxy/http/parser/__init__.py | 7 ++-- proxy/http/parser/chunk.py | 2 +- proxy/http/parser/parser.py | 22 ++++++------ proxy/http/parser/protocol.py | 4 +-- proxy/http/parser/tls/__init__.py | 1 + proxy/http/parser/tls/certificate.py | 2 +- proxy/http/parser/tls/finished.py | 2 +- proxy/http/parser/tls/handshake.py | 14 +++++--- proxy/http/parser/tls/hello.py | 4 +-- proxy/http/parser/tls/key_exchange.py | 2 +- proxy/http/parser/tls/tls.py | 6 ++-- proxy/http/parser/tls/types.py | 1 + proxy/http/plugin.py | 9 +++-- proxy/http/proxy/__init__.py | 3 +- proxy/http/proxy/auth.py | 8 ++--- proxy/http/proxy/plugin.py | 8 ++--- proxy/http/proxy/server.py | 35 +++++++++---------- proxy/http/responses.py | 5 ++- proxy/http/server/__init__.py | 3 +- proxy/http/server/pac_plugin.py | 8 ++--- proxy/http/server/plugin.py | 9 +++-- proxy/http/server/protocols.py | 1 + proxy/http/server/web.py | 28 +++++++-------- proxy/http/url.py | 7 ++-- proxy/http/websocket/__init__.py | 1 + proxy/http/websocket/client.py | 17 ++++----- proxy/http/websocket/frame.py | 5 ++- proxy/http/websocket/plugin.py | 8 ++--- proxy/http/websocket/transport.py | 11 +++--- proxy/plugin/__init__.py | 19 +++++----- proxy/plugin/cache/__init__.py | 1 + proxy/plugin/cache/base.py | 7 ++-- proxy/plugin/cache/cache_responses.py | 2 +- proxy/plugin/cache/store/disk.py | 8 ++--- proxy/plugin/cloudflare_dns.py | 6 ++-- proxy/plugin/custom_dns_resolver.py | 3 +- proxy/plugin/filter_by_client_ip.py | 5 ++- proxy/plugin/filter_by_upstream.py | 7 ++-- proxy/plugin/filter_by_url_regex.py | 12 +++---- proxy/plugin/man_in_the_middle.py | 2 +- proxy/plugin/mock_rest_api.py | 7 ++-- proxy/plugin/modify_chunk_response.py | 2 +- proxy/plugin/modify_post_data.py | 5 ++- proxy/plugin/program_name.py | 6 ++-- proxy/plugin/proxy_pool.py | 16 ++++----- proxy/plugin/redirect_to_custom_server.py | 2 +- proxy/plugin/reverse_proxy.py | 12 +++---- proxy/plugin/shortlink.py | 7 ++-- proxy/plugin/web_server_route.py | 3 +- proxy/proxy.py | 22 ++++++------ proxy/testing/__init__.py | 1 + proxy/testing/test_case.py | 9 ++--- .../exceptions/test_http_proxy_auth_failed.py | 2 +- tests/http/parser/test_http_parser.py | 2 +- .../proxy/test_http_proxy_tls_interception.py | 2 +- tests/http/test_protocol_handler.py | 6 ++-- tests/integration/test_integration.py | 9 ++--- tests/plugin/test_http_proxy_plugins.py | 4 ++- ...ttp_proxy_plugins_with_tls_interception.py | 2 +- tests/test_main.py | 19 +++++----- 124 files changed, 428 insertions(+), 419 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6243648ef..489b01b0b2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,12 @@ repos: args: - --py36-plus -# - repo: https://github.com/timothycrosley/isort.git -# rev: 5.10.0 -# hooks: -# - id: isort -# args: -# - --honor-noqa +- repo: https://github.com/timothycrosley/isort.git + rev: 5.10.0 + hooks: + - id: isort + args: + - --honor-noqa - repo: https://github.com/Lucas-C/pre-commit-hooks.git rev: v1.1.7 @@ -24,8 +24,7 @@ repos: helper/proxy\.pac| Makefile| proxy/common/pki\.py| - README\.md| - .+\.(plist|pbxproj) + README\.md $ - repo: https://github.com/pre-commit/pre-commit-hooks.git @@ -36,7 +35,6 @@ repos: exclude: | (?x) ^ - \.github/workflows/codeql-analysis\.yml| dashboard/src/core/plugins/inspect_traffic\.json $ - id: check-merge-conflict diff --git a/benchmark/_blacksheep.py b/benchmark/_blacksheep.py index cd966b4465..99f7e40e2b 100644 --- a/benchmark/_blacksheep.py +++ b/benchmark/_blacksheep.py @@ -9,7 +9,6 @@ :license: BSD, see LICENSE for more details. """ import uvicorn - from blacksheep.server import Application from blacksheep.server.responses import text diff --git a/benchmark/_proxy.py b/benchmark/_proxy.py index 838918ee5b..4a5ec0cf8d 100644 --- a/benchmark/_proxy.py +++ b/benchmark/_proxy.py @@ -10,6 +10,7 @@ """ import time import ipaddress + import proxy diff --git a/benchmark/_starlette.py b/benchmark/_starlette.py index 0e27e023cb..977b00d8c2 100644 --- a/benchmark/_starlette.py +++ b/benchmark/_starlette.py @@ -9,10 +9,9 @@ :license: BSD, see LICENSE for more details. """ import uvicorn - -from starlette.applications import Starlette -from starlette.responses import Response from starlette.routing import Route +from starlette.responses import Response +from starlette.applications import Starlette async def homepage(request): # type: ignore[no-untyped-def] diff --git a/benchmark/_tornado.py b/benchmark/_tornado.py index 3af22abb60..95e02fe616 100644 --- a/benchmark/_tornado.py +++ b/benchmark/_tornado.py @@ -8,8 +8,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import tornado.ioloop import tornado.web +import tornado.ioloop # pylint: disable=W0223 diff --git a/docs/_ext/spelling_stub_ext.py b/docs/_ext/spelling_stub_ext.py index c8989dc149..502888c42f 100644 --- a/docs/_ext/spelling_stub_ext.py +++ b/docs/_ext/spelling_stub_ext.py @@ -2,9 +2,9 @@ from typing import List +from sphinx.util.nodes import nodes from sphinx.application import Sphinx from sphinx.util.docutils import SphinxDirective -from sphinx.util.nodes import nodes class SpellingNoOpDirective(SphinxDirective): diff --git a/docs/conf.py b/docs/conf.py index 350dd216e1..05d968a832 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,8 +4,8 @@ """Configuration for the Sphinx documentation generator.""" import sys -from functools import partial from pathlib import Path +from functools import partial from setuptools_scm import get_version diff --git a/examples/web_scraper.py b/examples/web_scraper.py index 0dce2bd38b..daf31d612f 100644 --- a/examples/web_scraper.py +++ b/examples/web_scraper.py @@ -11,8 +11,8 @@ import time from proxy import Proxy -from proxy.common.types import Readables, Writables, SelectableEvents from proxy.core.work import Work +from proxy.common.types import Readables, Writables, SelectableEvents from proxy.core.connection import TcpClientConnection diff --git a/proxy/__init__.py b/proxy/__init__.py index f08c2afce2..08da6e2aa0 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -8,9 +8,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .proxy import entry_point, main, Proxy, sleep_loop +from .proxy import Proxy, main, sleep_loop, entry_point from .testing import TestCase + __all__ = [ # PyPi package entry_point. See # https://github.com/abhinavsingh/proxy.py#from-command-line-when-installed-using-pip diff --git a/proxy/__main__.py b/proxy/__main__.py index d04d8529d7..844283ec42 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -10,5 +10,6 @@ """ from .proxy import entry_point + if __name__ == '__main__': entry_point() diff --git a/proxy/common/_scm_version.pyi b/proxy/common/_scm_version.pyi index 4b96455246..504112bd07 100644 --- a/proxy/common/_scm_version.pyi +++ b/proxy/common/_scm_version.pyi @@ -2,5 +2,6 @@ # autogenerated on build and absent on mypy checks time from typing import Tuple, Union + version: str version_tuple: Tuple[Union[int, str], ...] diff --git a/proxy/common/_version.py b/proxy/common/_version.py index 21031c649a..2d4fce9a28 100644 --- a/proxy/common/_version.py +++ b/proxy/common/_version.py @@ -12,9 +12,11 @@ """ from typing import Tuple, Union + try: # pylint: disable=unused-import - from ._scm_version import version as __version__, version_tuple as _ver_tup # noqa: WPS433, WPS436 + from ._scm_version import version as __version__ # noqa: WPS433, WPS436 + from ._scm_version import version_tuple as _ver_tup # noqa: WPS433, WPS436 except ImportError: # pragma: no cover from pkg_resources import get_distribution as _get_dist # noqa: WPS433 __version__ = _get_dist('proxy.py').version # noqa: WPS440 diff --git a/proxy/common/backports.py b/proxy/common/backports.py index 87dbbdf958..770ed9fcc1 100644 --- a/proxy/common/backports.py +++ b/proxy/common/backports.py @@ -10,9 +10,8 @@ """ import time import threading - -from typing import Any, Deque from queue import Empty +from typing import Any, Deque from collections import deque diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 04645af08f..e21586a7db 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -11,16 +11,16 @@ import os import sys import time -import secrets import pathlib +import secrets import platform -import sysconfig import ipaddress - +import sysconfig from typing import Any, List from .version import __version__ + SYS_PLATFORM = platform.system() IS_WINDOWS = SYS_PLATFORM == 'Windows' @@ -150,9 +150,9 @@ def _env_threadless_compliant() -> bool: 'HttpWebServerBasePlugin', 'WebSocketTransportBasePlugin', ] -PLUGIN_PROXY_AUTH = 'proxy.http.proxy.AuthPlugin' PLUGIN_DASHBOARD = 'proxy.dashboard.ProxyDashboard' PLUGIN_HTTP_PROXY = 'proxy.http.proxy.HttpProxyPlugin' +PLUGIN_PROXY_AUTH = 'proxy.http.proxy.auth.AuthPlugin' PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin' PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin' PLUGIN_DEVTOOLS_PROTOCOL = 'proxy.http.inspector.devtools.DevtoolsProtocolPlugin' diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 9244ad38fe..edd07b7ce6 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -16,20 +16,22 @@ import ipaddress import collections import multiprocessing +from typing import Any, List, Optional, cast -from typing import Optional, List, Any, cast - -from .plugins import Plugins from .types import IpAddress from .utils import bytes_, is_py2, is_threadless, set_open_file_limit -from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_ACCEPTORS, DEFAULT_NUM_WORKERS -from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE -from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, DEFAULT_MIN_COMPRESSION_LIMIT -from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE -from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH, IS_WINDOWS, PLUGIN_WEBSOCKET_TRANSPORT from .logger import Logger - +from .plugins import Plugins from .version import __version__ +from .constants import ( + COMMA, IS_WINDOWS, PLUGIN_PAC_FILE, PLUGIN_DASHBOARD, PLUGIN_HTTP_PROXY, + PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, DEFAULT_NUM_WORKERS, + DEFAULT_NUM_ACCEPTORS, PLUGIN_INSPECT_TRAFFIC, DEFAULT_DISABLE_HEADERS, + PY2_DEPRECATION_MESSAGE, DEFAULT_DEVTOOLS_WS_PATH, + PLUGIN_DEVTOOLS_PROTOCOL, PLUGIN_WEBSOCKET_TRANSPORT, + DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_MIN_COMPRESSION_LIMIT, +) + __homepage__ = 'https://github.com/abhinavsingh/proxy.py' @@ -96,13 +98,17 @@ def initialize( print(PY2_DEPRECATION_MESSAGE) sys.exit(1) + # Dirty hack to always discover --basic-auth flag + # defined by proxy auth plugin. + in_args = input_args + ['--plugin', PLUGIN_PROXY_AUTH] + # Discover flags from requested plugin. # This will also surface external plugin flags # under --help. - Plugins.discover(input_args) + Plugins.discover(in_args) # Parse flags - args = flags.parse_args(input_args) + args = flags.parse_args(in_args) # Print version and exit if args.version: @@ -135,9 +141,11 @@ def initialize( if isinstance(work_klass, str) \ else work_klass + # TODO: Plugin flag initialization logic must be moved within plugins. + # # Generate auth_code required for basic authentication if enabled auth_code = None - basic_auth = opts.get('basic_auth', args.basic_auth) + basic_auth = opts.get('basic_auth', getattr(args, 'basic_auth', None)) # Destroy passed credentials via flags or options args.basic_auth = None if 'basic_auth' in opts: diff --git a/proxy/common/logger.py b/proxy/common/logger.py index dbd2e8aa51..74421d47b5 100644 --- a/proxy/common/logger.py +++ b/proxy/common/logger.py @@ -9,10 +9,10 @@ :license: BSD, see LICENSE for more details. """ import logging +from typing import Any, Optional -from typing import Optional, Any +from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT -from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL SINGLE_CHAR_TO_LEVEL = { 'D': 'DEBUG', diff --git a/proxy/common/pki.py b/proxy/common/pki.py index 0ccf695481..f8189c0cb9 100644 --- a/proxy/common/pki.py +++ b/proxy/common/pki.py @@ -14,19 +14,18 @@ """ import os import sys -import uuid import time +import uuid import logging -import tempfile import argparse +import tempfile import contextlib import subprocess - -from typing import List, Generator, Optional, Tuple +from typing import List, Tuple, Optional, Generator from .utils import bytes_ -from .constants import COMMA from .version import __version__ +from .constants import COMMA logger = logging.getLogger(__name__) diff --git a/proxy/common/plugins.py b/proxy/common/plugins.py index 2349e6ec9c..c919154ccd 100644 --- a/proxy/common/plugins.py +++ b/proxy/common/plugins.py @@ -9,15 +9,15 @@ :license: BSD, see LICENSE for more details. """ import os -import logging import inspect -import itertools +import logging import importlib +import itertools +from typing import Any, Dict, List, Tuple, Union, Optional -from typing import Any, List, Dict, Optional, Tuple, Union +from .utils import text_, bytes_ +from .constants import DOT, COMMA, DEFAULT_ABC_PLUGINS -from .utils import bytes_, text_ -from .constants import DOT, DEFAULT_ABC_PLUGINS, COMMA logger = logging.getLogger(__name__) diff --git a/proxy/common/types.py b/proxy/common/types.py index 79bfc62cc4..bc230c9b4e 100644 --- a/proxy/common/types.py +++ b/proxy/common/types.py @@ -10,8 +10,7 @@ """ import queue import ipaddress - -from typing import TYPE_CHECKING, Dict, Any, List, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union if TYPE_CHECKING: # pragma: no cover diff --git a/proxy/common/utils.py b/proxy/common/utils.py index e7373ac24f..9dc11ef1bd 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -12,22 +12,22 @@ utils """ -import sys import ssl +import sys import socket import logging import functools import ipaddress import contextlib - from types import TracebackType -from typing import Optional, Dict, Any, List, Tuple, Type, Callable +from typing import Any, Dict, List, Type, Tuple, Callable, Optional from .constants import ( - HTTP_1_1, COLON, WHITESPACE, CRLF, - DEFAULT_TIMEOUT, DEFAULT_THREADLESS, IS_WINDOWS, + CRLF, COLON, HTTP_1_1, IS_WINDOWS, WHITESPACE, DEFAULT_TIMEOUT, + DEFAULT_THREADLESS, ) + if not IS_WINDOWS: # pragma: no cover import resource diff --git a/proxy/common/version.py b/proxy/common/version.py index 6940317403..530829c5aa 100644 --- a/proxy/common/version.py +++ b/proxy/common/version.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from ._version import __version__, VERSION # noqa: WPS436 +from ._version import VERSION, __version__ # noqa: WPS436 __all__ = '__version__', 'VERSION' diff --git a/proxy/core/acceptor/__init__.py b/proxy/core/acceptor/__init__.py index a7cd9b1bce..e5e3f606c6 100644 --- a/proxy/core/acceptor/__init__.py +++ b/proxy/core/acceptor/__init__.py @@ -12,9 +12,10 @@ pre """ -from .listener import Listener -from .acceptor import Acceptor from .pool import AcceptorPool +from .acceptor import Acceptor +from .listener import Listener + __all__ = [ 'Listener', diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index 8fa34afdf8..9572c1b5b4 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -19,20 +19,17 @@ import threading import multiprocessing import multiprocessing.synchronize - +from typing import List, Tuple, Optional from multiprocessing import connection from multiprocessing.reduction import recv_handle -from typing import List, Optional, Tuple - +from ..work import LocalExecutor, start_threaded_work, delegate_work_to_pool +from ..event import EventQueue from ...common.flag import flags from ...common.logger import Logger from ...common.backports import NonBlockingQueue from ...common.constants import DEFAULT_LOCAL_EXECUTOR -from ..event import EventQueue - -from ..work import LocalExecutor, delegate_work_to_pool, start_threaded_work logger = logging.getLogger(__name__) diff --git a/proxy/core/acceptor/listener.py b/proxy/core/acceptor/listener.py index d55962a933..0caef04d9a 100644 --- a/proxy/core/acceptor/listener.py +++ b/proxy/core/acceptor/listener.py @@ -16,11 +16,12 @@ import socket import logging import argparse - -from typing import Optional, Any +from typing import Any, Optional from ...common.flag import flags -from ...common.constants import DEFAULT_BACKLOG, DEFAULT_IPV4_HOSTNAME, DEFAULT_PORT, DEFAULT_PORT_FILE +from ...common.constants import ( + DEFAULT_PORT, DEFAULT_BACKLOG, DEFAULT_PORT_FILE, DEFAULT_IPV4_HOSTNAME, +) flags.add_argument( diff --git a/proxy/core/acceptor/pool.py b/proxy/core/acceptor/pool.py index 0e17cd106a..99299cf4fa 100644 --- a/proxy/core/acceptor/pool.py +++ b/proxy/core/acceptor/pool.py @@ -17,18 +17,16 @@ import logging import argparse import multiprocessing - +from typing import TYPE_CHECKING, Any, List, Optional from multiprocessing import connection from multiprocessing.reduction import send_handle -from typing import TYPE_CHECKING, Any, List, Optional - -from .listener import Listener from .acceptor import Acceptor - +from .listener import Listener from ...common.flag import flags from ...common.constants import DEFAULT_NUM_ACCEPTORS + if TYPE_CHECKING: # pragma: no cover from ..event import EventQueue diff --git a/proxy/core/base/__init__.py b/proxy/core/base/__init__.py index bc86952b7c..5ce5a827d9 100644 --- a/proxy/core/base/__init__.py +++ b/proxy/core/base/__init__.py @@ -12,6 +12,7 @@ from .tcp_tunnel import BaseTcpTunnelHandler from .tcp_upstream import TcpUpstreamConnectionHandler + __all__ = [ 'BaseTcpServerHandler', 'BaseTcpTunnelHandler', diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index 66804505b7..107e1ac9dc 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -16,18 +16,19 @@ import socket import logging import selectors - from abc import abstractmethod -from typing import Any, Optional, TypeVar, Union +from typing import Any, Union, TypeVar, Optional +from ...core.work import Work from ...common.flag import flags +from ...common.types import Readables, Writables, SelectableEvents from ...common.utils import wrap_socket -from ...common.types import Readables, SelectableEvents, Writables -from ...common.constants import DEFAULT_CERT_FILE, DEFAULT_CLIENT_RECVBUF_SIZE -from ...common.constants import DEFAULT_KEY_FILE, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_TIMEOUT - -from ...core.work import Work from ...core.connection import TcpClientConnection +from ...common.constants import ( + DEFAULT_TIMEOUT, DEFAULT_KEY_FILE, DEFAULT_CERT_FILE, + DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_SERVER_RECVBUF_SIZE, +) + logger = logging.getLogger(__name__) diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index ea63c27935..48444384a6 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -10,16 +10,15 @@ """ import logging import selectors - from abc import abstractmethod from typing import Any, Optional +from .tcp_server import BaseTcpServerHandler +from ..connection import TcpClientConnection, TcpServerConnection from ...http.parser import HttpParser, httpParserTypes -from ...common.types import Readables, SelectableEvents, Writables +from ...common.types import Readables, Writables, SelectableEvents from ...common.utils import text_ -from ..connection import TcpServerConnection, TcpClientConnection -from .tcp_server import BaseTcpServerHandler logger = logging.getLogger(__name__) diff --git a/proxy/core/base/tcp_upstream.py b/proxy/core/base/tcp_upstream.py index 402a22e426..564f9a5072 100644 --- a/proxy/core/base/tcp_upstream.py +++ b/proxy/core/base/tcp_upstream.py @@ -10,13 +10,13 @@ """ import ssl import logging - from abc import ABC, abstractmethod -from typing import Optional, Any +from typing import Any, Optional from ...common.types import Readables, Writables, Descriptors from ...core.connection import TcpServerConnection + logger = logging.getLogger(__name__) diff --git a/proxy/core/connection/__init__.py b/proxy/core/connection/__init__.py index 58d100a81b..3457f1dbed 100644 --- a/proxy/core/connection/__init__.py +++ b/proxy/core/connection/__init__.py @@ -13,11 +13,12 @@ reusability Submodules """ -from .connection import TcpConnection, TcpConnectionUninitializedException -from .client import TcpClientConnection -from .server import TcpServerConnection from .pool import UpstreamConnectionPool from .types import tcpConnectionTypes +from .client import TcpClientConnection +from .server import TcpServerConnection +from .connection import TcpConnection, TcpConnectionUninitializedException + __all__ = [ 'TcpConnection', diff --git a/proxy/core/connection/client.py b/proxy/core/connection/client.py index 966221e17b..0ba63885ef 100644 --- a/proxy/core/connection/client.py +++ b/proxy/core/connection/client.py @@ -10,11 +10,10 @@ """ import ssl import socket +from typing import Tuple, Union, Optional -from typing import Union, Tuple, Optional - -from .connection import TcpConnection, TcpConnectionUninitializedException from .types import tcpConnectionTypes +from .connection import TcpConnection, TcpConnectionUninitializedException class TcpClientConnection(TcpConnection): diff --git a/proxy/core/connection/connection.py b/proxy/core/connection/connection.py index 84c6c9de74..9aa0d77d46 100644 --- a/proxy/core/connection/connection.py +++ b/proxy/core/connection/connection.py @@ -11,13 +11,12 @@ import ssl import socket import logging - from abc import ABC, abstractmethod -from typing import Optional, Union, List +from typing import List, Union, Optional +from .types import tcpConnectionTypes from ...common.constants import DEFAULT_BUFFER_SIZE, DEFAULT_MAX_SEND_SIZE -from .types import tcpConnectionTypes logger = logging.getLogger(__name__) diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index 29e00bd6bc..86c54a328e 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -15,15 +15,13 @@ import socket import logging import selectors - from typing import TYPE_CHECKING, Any, Set, Dict, Tuple -from ...common.flag import flags -from ...common.types import Readables, SelectableEvents, Writables - from ..work import Work - from .server import TcpServerConnection +from ...common.flag import flags +from ...common.types import Readables, Writables, SelectableEvents + logger = logging.getLogger(__name__) diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py index 109c238e30..48690f96bf 100644 --- a/proxy/core/connection/server.py +++ b/proxy/core/connection/server.py @@ -10,13 +10,11 @@ """ import ssl import socket +from typing import Tuple, Union, Optional -from typing import Optional, Union, Tuple - -from ...common.utils import new_socket_connection - -from .connection import TcpConnection, TcpConnectionUninitializedException from .types import tcpConnectionTypes +from .connection import TcpConnection, TcpConnectionUninitializedException +from ...common.utils import new_socket_connection class TcpServerConnection(TcpConnection): diff --git a/proxy/core/event/__init__.py b/proxy/core/event/__init__.py index 17e1074e6e..05736f7b5d 100644 --- a/proxy/core/event/__init__.py +++ b/proxy/core/event/__init__.py @@ -8,11 +8,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .queue import EventQueue from .names import EventNames, eventNames +from .queue import EventQueue +from .manager import EventManager from .dispatcher import EventDispatcher from .subscriber import EventSubscriber -from .manager import EventManager + __all__ = [ 'eventNames', diff --git a/proxy/core/event/dispatcher.py b/proxy/core/event/dispatcher.py index 49408fdca6..7cc978f154 100644 --- a/proxy/core/event/dispatcher.py +++ b/proxy/core/event/dispatcher.py @@ -11,13 +11,12 @@ import queue import logging import threading - +from typing import Any, Dict, List from multiprocessing import connection -from typing import Dict, Any, List - -from .queue import EventQueue from .names import eventNames +from .queue import EventQueue + logger = logging.getLogger(__name__) diff --git a/proxy/core/event/manager.py b/proxy/core/event/manager.py index 3923c6c12e..6a100f904a 100644 --- a/proxy/core/event/manager.py +++ b/proxy/core/event/manager.py @@ -15,15 +15,14 @@ import logging import threading import multiprocessing - from typing import Any, Optional from .queue import EventQueue from .dispatcher import EventDispatcher - from ...common.flag import flags from ...common.constants import DEFAULT_ENABLE_EVENTS + logger = logging.getLogger(__name__) diff --git a/proxy/core/event/names.py b/proxy/core/event/names.py index 5040c880a8..369724aac4 100644 --- a/proxy/core/event/names.py +++ b/proxy/core/event/names.py @@ -10,6 +10,7 @@ """ from typing import NamedTuple + # Name of the events that eventing framework supports. # # Ideally this must be configurable via command line or diff --git a/proxy/core/event/queue.py b/proxy/core/event/queue.py index f86ba09b9a..878fe9bf45 100644 --- a/proxy/core/event/queue.py +++ b/proxy/core/event/queue.py @@ -11,14 +11,11 @@ import os import time import threading - +from typing import Any, Dict, Optional from multiprocessing import connection -from typing import Dict, Optional, Any - -from ...common.types import DictQueueType - from .names import eventNames +from ...common.types import DictQueueType class EventQueue: diff --git a/proxy/core/event/subscriber.py b/proxy/core/event/subscriber.py index 14567616ea..ea478dc46c 100644 --- a/proxy/core/event/subscriber.py +++ b/proxy/core/event/subscriber.py @@ -13,13 +13,12 @@ import logging import threading import multiprocessing - +from typing import Any, Dict, Callable, Optional from multiprocessing import connection -from typing import Dict, Optional, Any, Callable - -from .queue import EventQueue from .names import eventNames +from .queue import EventQueue + logger = logging.getLogger(__name__) diff --git a/proxy/core/ssh/__init__.py b/proxy/core/ssh/__init__.py index 2150e1557c..9d9d605de4 100644 --- a/proxy/core/ssh/__init__.py +++ b/proxy/core/ssh/__init__.py @@ -15,6 +15,7 @@ from .handler import SshHttpProtocolHandler from .listener import SshTunnelListener + __all__ = [ 'SshHttpProtocolHandler', 'SshTunnelListener', diff --git a/proxy/core/ssh/handler.py b/proxy/core/ssh/handler.py index 1c021f8c86..12557ed923 100644 --- a/proxy/core/ssh/handler.py +++ b/proxy/core/ssh/handler.py @@ -9,9 +9,9 @@ :license: BSD, see LICENSE for more details. """ import argparse - from typing import TYPE_CHECKING, Tuple + if TYPE_CHECKING: # pragma: no cover try: from paramiko.channel import Channel diff --git a/proxy/core/ssh/listener.py b/proxy/core/ssh/listener.py index e1bc565b96..1b17120e63 100644 --- a/proxy/core/ssh/listener.py +++ b/proxy/core/ssh/listener.py @@ -8,10 +8,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import argparse import logging +import argparse +from typing import TYPE_CHECKING, Any, Set, Tuple, Callable, Optional -from typing import TYPE_CHECKING, Any, Callable, Optional, Set, Tuple try: from paramiko import SSHClient, AutoAddPolicy @@ -23,6 +23,7 @@ from ...common.flag import flags + logger = logging.getLogger(__name__) diff --git a/proxy/core/work/__init__.py b/proxy/core/work/__init__.py index 403978ff17..a89add7841 100644 --- a/proxy/core/work/__init__.py +++ b/proxy/core/work/__init__.py @@ -12,13 +12,14 @@ pre """ +from .pool import ThreadlessPool from .work import Work -from .threadless import Threadless -from .remote import RemoteExecutor from .local import LocalExecutor -from .pool import ThreadlessPool +from .remote import RemoteExecutor from .delegate import delegate_work_to_pool from .threaded import start_threaded_work +from .threadless import Threadless + __all__ = [ 'Work', diff --git a/proxy/core/work/delegate.py b/proxy/core/work/delegate.py index 5a176e8992..76207d9635 100644 --- a/proxy/core/work/delegate.py +++ b/proxy/core/work/delegate.py @@ -8,9 +8,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Tuple, Optional from multiprocessing.reduction import send_handle + if TYPE_CHECKING: # pragma: no cover import socket import multiprocessing diff --git a/proxy/core/work/local.py b/proxy/core/work/local.py index 95c2c118c2..e081b2da9c 100644 --- a/proxy/core/work/local.py +++ b/proxy/core/work/local.py @@ -9,16 +9,16 @@ :license: BSD, see LICENSE for more details. """ import queue -import logging import asyncio +import logging import contextlib - -from typing import Optional -from typing import Any - -from ...common.backports import NonBlockingQueue # noqa: W0611, F401 pylint: disable=unused-import +from typing import Any, Optional from .threadless import Threadless +from ...common.backports import ( # noqa: W0611, F401 pylint: disable=unused-import + NonBlockingQueue, +) + logger = logging.getLogger(__name__) diff --git a/proxy/core/work/pool.py b/proxy/core/work/pool.py index 0a28ff1b50..96c43a7228 100644 --- a/proxy/core/work/pool.py +++ b/proxy/core/work/pool.py @@ -11,14 +11,13 @@ import logging import argparse import multiprocessing - +from typing import TYPE_CHECKING, Any, List, Optional from multiprocessing import connection -from typing import TYPE_CHECKING, Any, Optional, List from .remote import RemoteExecutor - from ...common.flag import flags -from ...common.constants import DEFAULT_NUM_WORKERS, DEFAULT_THREADLESS +from ...common.constants import DEFAULT_THREADLESS, DEFAULT_NUM_WORKERS + if TYPE_CHECKING: # pragma: no cover from ..event import EventQueue diff --git a/proxy/core/work/remote.py b/proxy/core/work/remote.py index bc16e83e40..6dd91ef053 100644 --- a/proxy/core/work/remote.py +++ b/proxy/core/work/remote.py @@ -10,13 +10,13 @@ """ import asyncio import logging - -from typing import Optional, Any +from typing import Any, Optional from multiprocessing import connection from multiprocessing.reduction import recv_handle from .threadless import Threadless + logger = logging.getLogger(__name__) diff --git a/proxy/core/work/threaded.py b/proxy/core/work/threaded.py index 7366f8a2c4..6f2569a001 100644 --- a/proxy/core/work/threaded.py +++ b/proxy/core/work/threaded.py @@ -11,11 +11,11 @@ import socket import argparse import threading - -from typing import TYPE_CHECKING, Optional, Tuple, TypeVar +from typing import TYPE_CHECKING, Tuple, TypeVar, Optional from ..event import EventQueue, eventNames + if TYPE_CHECKING: # pragma: no cover from .work import Work diff --git a/proxy/core/work/threadless.py b/proxy/core/work/threadless.py index e5c69a39c6..57eca4dcae 100644 --- a/proxy/core/work/threadless.py +++ b/proxy/core/work/threadless.py @@ -11,27 +11,30 @@ import os import ssl import socket -import logging import asyncio +import logging import argparse import selectors import multiprocessing +from abc import ABC, abstractmethod +from typing import ( + TYPE_CHECKING, Set, Dict, List, Tuple, Union, Generic, TypeVar, Optional, +) -from abc import abstractmethod, ABC -from typing import TYPE_CHECKING, Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union - +from ..event import eventNames +from ...common.types import Readables, Writables, SelectableEvents from ...common.logger import Logger -from ...common.types import Readables, SelectableEvents, Writables -from ...common.constants import DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT -from ...common.constants import DEFAULT_WAIT_FOR_TASKS_TIMEOUT +from ...common.constants import ( + DEFAULT_WAIT_FOR_TASKS_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT, + DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, +) -from ..event import eventNames if TYPE_CHECKING: # pragma: no cover from typing import Any - from ..event import EventQueue from .work import Work + from ..event import EventQueue T = TypeVar('T') @@ -92,7 +95,9 @@ def __init__( self._total: int = 0 # When put at the top, causes circular import error # since integrated ssh tunnel was introduced. - from ..connection import UpstreamConnectionPool # pylint: disable=C0415 + from ..connection import ( # pylint: disable=C0415 + UpstreamConnectionPool, + ) self._upstream_conn_pool: Optional['UpstreamConnectionPool'] = None self._upstream_conn_filenos: Set[int] = set() if self.flags.enable_conn_pool: diff --git a/proxy/core/work/work.py b/proxy/core/work/work.py index 7747d26906..08560103de 100644 --- a/proxy/core/work/work.py +++ b/proxy/core/work/work.py @@ -13,13 +13,13 @@ acceptor """ import argparse - -from uuid import uuid4 from abc import ABC, abstractmethod -from typing import Optional, Dict, Any, TypeVar, Generic, TYPE_CHECKING +from uuid import uuid4 +from typing import TYPE_CHECKING, Any, Dict, Generic, TypeVar, Optional + +from ..event import EventQueue, eventNames +from ...common.types import Readables, Writables, SelectableEvents -from ..event import eventNames, EventQueue -from ...common.types import Readables, SelectableEvents, Writables if TYPE_CHECKING: # pragma: no cover from ..connection import UpstreamConnectionPool diff --git a/proxy/dashboard/__init__.py b/proxy/dashboard/__init__.py index 96a79affef..b59e877a77 100644 --- a/proxy/dashboard/__init__.py +++ b/proxy/dashboard/__init__.py @@ -10,6 +10,7 @@ """ from .dashboard import ProxyDashboard + __all__ = [ 'ProxyDashboard', ] diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py index 6e719d2306..411c153b60 100644 --- a/proxy/dashboard/dashboard.py +++ b/proxy/dashboard/dashboard.py @@ -12,9 +12,12 @@ import logging from typing import List, Tuple -from ..http.responses import permanentRedirectResponse from ..http.parser import HttpParser -from ..http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes +from ..http.server import ( + HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes, +) +from ..http.responses import permanentRedirectResponse + logger = logging.getLogger(__name__) diff --git a/proxy/http/__init__.py b/proxy/http/__init__.py index 9c800b5d0a..37826426ab 100644 --- a/proxy/http/__init__.py +++ b/proxy/http/__init__.py @@ -8,13 +8,14 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .handler import HttpProtocolHandler -from .connection import HttpClientConnection -from .plugin import HttpProtocolHandlerPlugin +from .url import Url from .codes import httpStatusCodes -from .methods import httpMethods +from .plugin import HttpProtocolHandlerPlugin +from .handler import HttpProtocolHandler from .headers import httpHeaders -from .url import Url +from .methods import httpMethods +from .connection import HttpClientConnection + __all__ = [ 'HttpProtocolHandler', diff --git a/proxy/http/descriptors.py b/proxy/http/descriptors.py index bae5ea3857..102629bd21 100644 --- a/proxy/http/descriptors.py +++ b/proxy/http/descriptors.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ from typing import Any + from ..common.types import Readables, Writables, Descriptors diff --git a/proxy/http/exception/__init__.py b/proxy/http/exception/__init__.py index 513d2bd510..68776e923b 100644 --- a/proxy/http/exception/__init__.py +++ b/proxy/http/exception/__init__.py @@ -9,9 +9,10 @@ :license: BSD, see LICENSE for more details. """ from .base import HttpProtocolException -from .http_request_rejected import HttpRequestRejected from .proxy_auth_failed import ProxyAuthenticationFailed from .proxy_conn_failed import ProxyConnectionFailed +from .http_request_rejected import HttpRequestRejected + __all__ = [ 'HttpProtocolException', diff --git a/proxy/http/exception/base.py b/proxy/http/exception/base.py index e01be4abac..bd1233bae6 100644 --- a/proxy/http/exception/base.py +++ b/proxy/http/exception/base.py @@ -12,7 +12,8 @@ http """ -from typing import Any, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional + if TYPE_CHECKING: # pragma: no cover from ..parser import HttpParser diff --git a/proxy/http/exception/http_request_rejected.py b/proxy/http/exception/http_request_rejected.py index 7c6f8d48be..2b2e7a13b1 100644 --- a/proxy/http/exception/http_request_rejected.py +++ b/proxy/http/exception/http_request_rejected.py @@ -8,12 +8,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Any, Optional, Dict, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, Optional from .base import HttpProtocolException - from ...common.utils import build_http_response + if TYPE_CHECKING: # pragma: no cover from ..parser import HttpParser diff --git a/proxy/http/exception/proxy_auth_failed.py b/proxy/http/exception/proxy_auth_failed.py index 4fbe62b4e6..afb2e4048e 100644 --- a/proxy/http/exception/proxy_auth_failed.py +++ b/proxy/http/exception/proxy_auth_failed.py @@ -16,9 +16,9 @@ from typing import TYPE_CHECKING, Any from .base import HttpProtocolException - from ..responses import PROXY_AUTH_FAILED_RESPONSE_PKT + if TYPE_CHECKING: # pragma: no cover from ..parser import HttpParser diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py index 7aa709235c..2001b33605 100644 --- a/proxy/http/exception/proxy_conn_failed.py +++ b/proxy/http/exception/proxy_conn_failed.py @@ -15,9 +15,9 @@ from typing import TYPE_CHECKING, Any from .base import HttpProtocolException - from ..responses import BAD_GATEWAY_RESPONSE_PKT + if TYPE_CHECKING: # pragma: no cover from ..parser import HttpParser diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 8b2952f850..04fd507b33 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -15,19 +15,17 @@ import asyncio import logging import selectors +from typing import Any, List, Type, Tuple, Optional -from typing import Tuple, List, Type, Optional, Any - -from ..core.base import BaseTcpServerHandler -from ..common.types import Readables, SelectableEvents, Writables -from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT - -from .protocols import httpProtocols -from .connection import HttpClientConnection -from .exception import HttpProtocolException +from .parser import HttpParser, httpParserTypes, httpParserStates from .plugin import HttpProtocolHandlerPlugin +from .exception import HttpProtocolException +from .protocols import httpProtocols from .responses import BAD_REQUEST_RESPONSE_PKT -from .parser import HttpParser, httpParserStates, httpParserTypes +from ..core.base import BaseTcpServerHandler +from .connection import HttpClientConnection +from ..common.types import Readables, Writables, SelectableEvents +from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT logger = logging.getLogger(__name__) diff --git a/proxy/http/inspector/devtools.py b/proxy/http/inspector/devtools.py index 5b16571bf7..b5058764df 100644 --- a/proxy/http/inspector/devtools.py +++ b/proxy/http/inspector/devtools.py @@ -14,18 +14,20 @@ """ import json import logging -from typing import List, Tuple, Any, Dict +from typing import Any, Dict, List, Tuple -from .transformer import CoreEventsToDevtoolsProtocol from ..parser import HttpParser -from ..websocket import WebsocketFrame, websocketOpcodes from ..server import HttpWebServerBasePlugin, httpProtocolTypes - -from ...common.utils import bytes_, text_ +from ..websocket import WebsocketFrame, websocketOpcodes +from .transformer import CoreEventsToDevtoolsProtocol from ...core.event import EventSubscriber from ...common.flag import flags -from ...common.constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DEVTOOLS_DOC_URL -from ...common.constants import DEFAULT_ENABLE_DEVTOOLS +from ...common.utils import text_, bytes_ +from ...common.constants import ( + DEFAULT_ENABLE_DEVTOOLS, DEFAULT_DEVTOOLS_DOC_URL, + DEFAULT_DEVTOOLS_WS_PATH, +) + logger = logging.getLogger(__name__) diff --git a/proxy/http/inspector/inspect_traffic.py b/proxy/http/inspector/inspect_traffic.py index 60225f5cf3..8b5b01da9a 100644 --- a/proxy/http/inspector/inspect_traffic.py +++ b/proxy/http/inspector/inspect_traffic.py @@ -9,12 +9,12 @@ :license: BSD, see LICENSE for more details. """ import json -from typing import TYPE_CHECKING, List, Dict, Any +from typing import TYPE_CHECKING, Any, Dict, List -from ...common.utils import bytes_ +from ..websocket import WebsocketFrame, WebSocketTransportBasePlugin from ...core.event import EventSubscriber +from ...common.utils import bytes_ -from ..websocket import WebsocketFrame, WebSocketTransportBasePlugin if TYPE_CHECKING: # pragma: no cover from ..connection import HttpClientConnection diff --git a/proxy/http/inspector/transformer.py b/proxy/http/inspector/transformer.py index ea9276dd19..d04909e1ef 100644 --- a/proxy/http/inspector/transformer.py +++ b/proxy/http/inspector/transformer.py @@ -12,12 +12,14 @@ import time from typing import TYPE_CHECKING, Any, Dict -from ...common.constants import PROXY_PY_START_TIME, DEFAULT_DEVTOOLS_DOC_URL -from ...common.constants import DEFAULT_DEVTOOLS_FRAME_ID, DEFAULT_DEVTOOLS_LOADER_ID -from ...common.utils import bytes_ +from ..websocket import WebsocketFrame from ...core.event import eventNames +from ...common.utils import bytes_ +from ...common.constants import ( + PROXY_PY_START_TIME, DEFAULT_DEVTOOLS_DOC_URL, DEFAULT_DEVTOOLS_FRAME_ID, + DEFAULT_DEVTOOLS_LOADER_ID, +) -from ..websocket import WebsocketFrame if TYPE_CHECKING: # pragma: no cover from ..connection import HttpClientConnection diff --git a/proxy/http/parser/__init__.py b/proxy/http/parser/__init__.py index be633d4873..d197666522 100644 --- a/proxy/http/parser/__init__.py +++ b/proxy/http/parser/__init__.py @@ -13,10 +13,11 @@ http Submodules """ -from .parser import HttpParser from .chunk import ChunkParser, chunkParserStates -from .types import httpParserStates, httpParserTypes -from .protocol import ProxyProtocol, PROXY_PROTOCOL_V2_SIGNATURE +from .types import httpParserTypes, httpParserStates +from .parser import HttpParser +from .protocol import PROXY_PROTOCOL_V2_SIGNATURE, ProxyProtocol + __all__ = [ 'HttpParser', diff --git a/proxy/http/parser/chunk.py b/proxy/http/parser/chunk.py index 2d976c4449..691117926d 100644 --- a/proxy/http/parser/chunk.py +++ b/proxy/http/parser/chunk.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import NamedTuple, Tuple, List, Optional +from typing import List, Tuple, Optional, NamedTuple from ...common.utils import bytes_, find_http_line from ...common.constants import CRLF, DEFAULT_BUFFER_SIZE diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index 6adbd558d8..6573cabaf6 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -12,22 +12,22 @@ http """ -from typing import TypeVar, Optional, Dict, Type, Tuple, List - -from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, DEFAULT_ENABLE_PROXY_PROTOCOL, HTTP_1_0 -from ...common.constants import HTTP_1_1, SLASH, CRLF -from ...common.constants import WHITESPACE, DEFAULT_HTTP_PORT -from ...common.utils import build_http_request, build_http_response, text_ -from ...common.flag import flags +from typing import Dict, List, Type, Tuple, TypeVar, Optional from ..url import Url +from .chunk import ChunkParser, chunkParserStates +from .types import httpParserTypes, httpParserStates from ..methods import httpMethods -from ..protocols import httpProtocols +from .protocol import ProxyProtocol from ..exception import HttpProtocolException +from ..protocols import httpProtocols +from ...common.flag import flags +from ...common.utils import text_, build_http_request, build_http_response +from ...common.constants import ( + CRLF, COLON, SLASH, HTTP_1_0, HTTP_1_1, WHITESPACE, DEFAULT_HTTP_PORT, + DEFAULT_DISABLE_HEADERS, DEFAULT_ENABLE_PROXY_PROTOCOL, +) -from .protocol import ProxyProtocol -from .chunk import ChunkParser, chunkParserStates -from .types import httpParserTypes, httpParserStates flags.add_argument( '--enable-proxy-protocol', diff --git a/proxy/http/parser/protocol.py b/proxy/http/parser/protocol.py index c44192f0e5..2fc19df484 100644 --- a/proxy/http/parser/protocol.py +++ b/proxy/http/parser/protocol.py @@ -8,12 +8,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Tuple +from typing import Tuple, Optional from ..exception import HttpProtocolException - from ...common.constants import WHITESPACE + PROXY_PROTOCOL_V2_SIGNATURE = b'\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A' diff --git a/proxy/http/parser/tls/__init__.py b/proxy/http/parser/tls/__init__.py index d32e35bcd0..1ad8fb2c89 100644 --- a/proxy/http/parser/tls/__init__.py +++ b/proxy/http/parser/tls/__init__.py @@ -11,6 +11,7 @@ from .tls import TlsParser from .types import tlsContentType, tlsHandshakeType + __all__ = [ 'TlsParser', 'tlsContentType', diff --git a/proxy/http/parser/tls/certificate.py b/proxy/http/parser/tls/certificate.py index 1004210af7..f71e495c79 100644 --- a/proxy/http/parser/tls/certificate.py +++ b/proxy/http/parser/tls/certificate.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Tuple +from typing import Tuple, Optional class TlsCertificate: diff --git a/proxy/http/parser/tls/finished.py b/proxy/http/parser/tls/finished.py index 4cc7273372..df9db0625d 100644 --- a/proxy/http/parser/tls/finished.py +++ b/proxy/http/parser/tls/finished.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Tuple +from typing import Tuple, Optional class TlsFinished: diff --git a/proxy/http/parser/tls/handshake.py b/proxy/http/parser/tls/handshake.py index d859ceb589..7a03e2471c 100644 --- a/proxy/http/parser/tls/handshake.py +++ b/proxy/http/parser/tls/handshake.py @@ -10,14 +10,18 @@ """ import struct import logging +from typing import Tuple, Optional -from typing import Optional, Tuple - +from .hello import ( + TlsClientHello, TlsServerHello, TlsHelloRequest, TlsServerHelloDone, +) from .types import tlsHandshakeType -from .hello import TlsHelloRequest, TlsClientHello, TlsServerHello, TlsServerHelloDone -from .certificate import TlsCertificate, TlsCertificateRequest, TlsCertificateVerify -from .key_exchange import TlsClientKeyExchange, TlsServerKeyExchange from .finished import TlsFinished +from .certificate import ( + TlsCertificate, TlsCertificateVerify, TlsCertificateRequest, +) +from .key_exchange import TlsClientKeyExchange, TlsServerKeyExchange + logger = logging.getLogger(__name__) diff --git a/proxy/http/parser/tls/hello.py b/proxy/http/parser/tls/hello.py index 80287109d0..f60c724b22 100644 --- a/proxy/http/parser/tls/hello.py +++ b/proxy/http/parser/tls/hello.py @@ -11,11 +11,11 @@ import os import struct import logging - -from typing import Optional, Tuple +from typing import Tuple, Optional from .pretty import pretty_hexlify + logger = logging.getLogger(__name__) diff --git a/proxy/http/parser/tls/key_exchange.py b/proxy/http/parser/tls/key_exchange.py index ce56562b19..cb0059ed47 100644 --- a/proxy/http/parser/tls/key_exchange.py +++ b/proxy/http/parser/tls/key_exchange.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Tuple +from typing import Tuple, Optional class TlsServerKeyExchange: diff --git a/proxy/http/parser/tls/tls.py b/proxy/http/parser/tls/tls.py index 634c5b93ed..9e5aa89eb5 100644 --- a/proxy/http/parser/tls/tls.py +++ b/proxy/http/parser/tls/tls.py @@ -10,12 +10,12 @@ """ import struct import logging - -from typing import Optional, Tuple +from typing import Tuple, Optional from .types import tlsContentType -from .certificate import TlsCertificate from .handshake import TlsHandshake +from .certificate import TlsCertificate + logger = logging.getLogger(__name__) diff --git a/proxy/http/parser/tls/types.py b/proxy/http/parser/tls/types.py index 9dd05df321..640cffe282 100644 --- a/proxy/http/parser/tls/types.py +++ b/proxy/http/parser/tls/types.py @@ -10,6 +10,7 @@ """ from typing import NamedTuple + TlsContentType = NamedTuple( 'TlsContentType', [ ('CHANGE_CIPHER_SPEC', int), diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py index 3b587a1712..9eaa097779 100644 --- a/proxy/http/plugin.py +++ b/proxy/http/plugin.py @@ -10,16 +10,15 @@ """ import socket import argparse - from abc import ABC, abstractmethod -from typing import List, Union, Optional, TYPE_CHECKING - -from ..core.event import EventQueue +from typing import TYPE_CHECKING, List, Union, Optional +from .mixins import TlsInterceptionPropertyMixin from .parser import HttpParser from .connection import HttpClientConnection +from ..core.event import EventQueue from .descriptors import DescriptorsHandlerMixin -from .mixins import TlsInterceptionPropertyMixin + if TYPE_CHECKING: # pragma: no cover from ..core.connection import UpstreamConnectionPool diff --git a/proxy/http/proxy/__init__.py b/proxy/http/proxy/__init__.py index 4e18002c0d..a794ba078c 100644 --- a/proxy/http/proxy/__init__.py +++ b/proxy/http/proxy/__init__.py @@ -10,10 +10,9 @@ """ from .plugin import HttpProxyBasePlugin from .server import HttpProxyPlugin -from .auth import AuthPlugin + __all__ = [ 'HttpProxyBasePlugin', 'HttpProxyPlugin', - 'AuthPlugin', ] diff --git a/proxy/http/proxy/auth.py b/proxy/http/proxy/auth.py index a309d194c7..f2632d7957 100644 --- a/proxy/http/proxy/auth.py +++ b/proxy/http/proxy/auth.py @@ -15,14 +15,12 @@ """ from typing import Optional +from ...http import httpHeaders from ..exception import ProxyAuthenticationFailed - +from ...http.proxy import HttpProxyBasePlugin from ...common.flag import flags -from ...common.constants import DEFAULT_BASIC_AUTH - -from ...http import httpHeaders from ...http.parser import HttpParser -from ...http.proxy import HttpProxyBasePlugin +from ...common.constants import DEFAULT_BASIC_AUTH flags.add_argument( diff --git a/proxy/http/proxy/plugin.py b/proxy/http/proxy/plugin.py index 9f6f114564..ed951e313f 100644 --- a/proxy/http/proxy/plugin.py +++ b/proxy/http/proxy/plugin.py @@ -9,17 +9,15 @@ :license: BSD, see LICENSE for more details. """ import argparse - from abc import ABC -from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, Tuple, Optional from ..mixins import TlsInterceptionPropertyMixin - from ..parser import HttpParser -from ..descriptors import DescriptorsHandlerMixin from ..connection import HttpClientConnection - from ...core.event import EventQueue +from ..descriptors import DescriptorsHandlerMixin + if TYPE_CHECKING: # pragma: no cover from ...core.connection import UpstreamConnectionPool diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 3008898c86..10932fde9e 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -21,33 +21,32 @@ import logging import threading import subprocess - -from typing import Optional, List, Union, Dict, cast, Any +from typing import Any, Dict, List, Union, Optional, cast from .plugin import HttpProxyBasePlugin - +from ..parser import HttpParser, httpParserTypes, httpParserStates +from ..plugin import HttpProtocolHandlerPlugin from ..headers import httpHeaders from ..methods import httpMethods -from ..protocols import httpProtocols -from ..plugin import HttpProtocolHandlerPlugin from ..exception import HttpProtocolException, ProxyConnectionFailed -from ..parser import HttpParser, httpParserStates, httpParserTypes +from ..protocols import httpProtocols from ..responses import PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT - +from ...common.pki import gen_csr, sign_csr, gen_public_key +from ...core.event import eventNames +from ...common.flag import flags from ...common.types import Readables, Writables, Descriptors -from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE -from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE -from ...common.constants import COMMA, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT -from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS -from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT -from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH from ...common.utils import text_ -from ...common.pki import gen_public_key, gen_csr, sign_csr +from ...core.connection import ( + TcpServerConnection, TcpConnectionUninitializedException, +) +from ...common.constants import ( + COMMA, DEFAULT_CA_FILE, PLUGIN_PROXY_AUTH, DEFAULT_CA_CERT_DIR, + DEFAULT_CA_KEY_FILE, DEFAULT_CA_CERT_FILE, DEFAULT_DISABLE_HEADERS, + PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HTTP_PROXY, + DEFAULT_CA_SIGNING_KEY_FILE, DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT, + DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT, +) -from ...core.event import eventNames -from ...core.connection import TcpServerConnection -from ...core.connection import TcpConnectionUninitializedException -from ...common.flag import flags logger = logging.getLogger(__name__) diff --git a/proxy/http/responses.py b/proxy/http/responses.py index 3eaca1e606..c1e8a17395 100644 --- a/proxy/http/responses.py +++ b/proxy/http/responses.py @@ -11,11 +11,10 @@ import gzip from typing import Any, Dict, Optional +from .codes import httpStatusCodes from ..common.flag import flags from ..common.utils import build_http_response -from ..common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY - -from .codes import httpStatusCodes +from ..common.constants import PROXY_AGENT_HEADER_KEY, PROXY_AGENT_HEADER_VALUE PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview( diff --git a/proxy/http/server/__init__.py b/proxy/http/server/__init__.py index 059c2cc128..dfbaa02c85 100644 --- a/proxy/http/server/__init__.py +++ b/proxy/http/server/__init__.py @@ -9,9 +9,10 @@ :license: BSD, see LICENSE for more details. """ from .web import HttpWebServerPlugin -from .pac_plugin import HttpWebServerPacFilePlugin from .plugin import HttpWebServerBasePlugin from .protocols import httpProtocolTypes +from .pac_plugin import HttpWebServerPacFilePlugin + __all__ = [ 'HttpWebServerPlugin', diff --git a/proxy/http/server/pac_plugin.py b/proxy/http/server/pac_plugin.py index 8cc0a4d0f0..a52f213dd7 100644 --- a/proxy/http/server/pac_plugin.py +++ b/proxy/http/server/pac_plugin.py @@ -12,16 +12,14 @@ pac """ -from typing import List, Tuple, Optional, Any +from typing import Any, List, Tuple, Optional from .plugin import HttpWebServerBasePlugin -from .protocols import httpProtocolTypes - from ..parser import HttpParser +from .protocols import httpProtocolTypes from ..responses import okResponse - -from ...common.utils import bytes_, text_ from ...common.flag import flags +from ...common.utils import text_, bytes_ from ...common.constants import DEFAULT_PAC_FILE, DEFAULT_PAC_FILE_URL_PATH diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 069b0dff9d..62ebd3e7dc 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -9,16 +9,15 @@ :license: BSD, see LICENSE for more details. """ import argparse - from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Optional -from ..websocket import WebsocketFrame from ..parser import HttpParser -from ..descriptors import DescriptorsHandlerMixin +from ..websocket import WebsocketFrame from ..connection import HttpClientConnection - from ...core.event import EventQueue +from ..descriptors import DescriptorsHandlerMixin + if TYPE_CHECKING: # pragma: no cover from ...core.connection import UpstreamConnectionPool diff --git a/proxy/http/server/protocols.py b/proxy/http/server/protocols.py index a561cb936f..84b5d8ac77 100644 --- a/proxy/http/server/protocols.py +++ b/proxy/http/server/protocols.py @@ -15,6 +15,7 @@ """ from typing import NamedTuple + HttpProtocolTypes = NamedTuple( 'HttpProtocolTypes', [ ('HTTP', int), diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index 5035217138..efc734cddf 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -13,25 +13,25 @@ import socket import logging import mimetypes +from typing import Any, Dict, List, Tuple, Union, Pattern, Optional -from typing import List, Optional, Dict, Tuple, Union, Any, Pattern - -from ...common.constants import DEFAULT_STATIC_SERVER_DIR -from ...common.constants import DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_ENABLE_WEB_SERVER -from ...common.constants import DEFAULT_MIN_COMPRESSION_LIMIT, DEFAULT_WEB_ACCESS_LOG_FORMAT -from ...common.utils import bytes_, text_, build_websocket_handshake_response -from ...common.types import Readables, Writables, Descriptors -from ...common.flag import flags - -from ..exception import HttpProtocolException -from ..plugin import HttpProtocolHandlerPlugin -from ..websocket import WebsocketFrame, websocketOpcodes +from .plugin import HttpWebServerBasePlugin from ..parser import HttpParser, httpParserTypes +from ..plugin import HttpProtocolHandlerPlugin +from .protocols import httpProtocolTypes +from ..exception import HttpProtocolException from ..protocols import httpProtocols from ..responses import NOT_FOUND_RESPONSE_PKT, okResponse +from ..websocket import WebsocketFrame, websocketOpcodes +from ...common.flag import flags +from ...common.types import Readables, Writables, Descriptors +from ...common.utils import text_, bytes_, build_websocket_handshake_response +from ...common.constants import ( + DEFAULT_ENABLE_WEB_SERVER, DEFAULT_STATIC_SERVER_DIR, + DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_MIN_COMPRESSION_LIMIT, + DEFAULT_WEB_ACCESS_LOG_FORMAT, +) -from .plugin import HttpWebServerBasePlugin -from .protocols import httpProtocolTypes logger = logging.getLogger(__name__) diff --git a/proxy/http/url.py b/proxy/http/url.py index 9cbdb4ae10..c799efaa1a 100644 --- a/proxy/http/url.py +++ b/proxy/http/url.py @@ -13,12 +13,11 @@ http url """ -from typing import List, Optional, Tuple - -from ..common.constants import COLON, DEFAULT_ALLOWED_URL_SCHEMES, SLASH, AT -from ..common.utils import text_ +from typing import List, Tuple, Optional from .exception import HttpProtocolException +from ..common.utils import text_ +from ..common.constants import AT, COLON, SLASH, DEFAULT_ALLOWED_URL_SCHEMES class Url: diff --git a/proxy/http/websocket/__init__.py b/proxy/http/websocket/__init__.py index 5017387395..a787535cac 100644 --- a/proxy/http/websocket/__init__.py +++ b/proxy/http/websocket/__init__.py @@ -19,6 +19,7 @@ from .client import WebsocketClient from .plugin import WebSocketTransportBasePlugin + __all__ = [ 'websocketOpcodes', 'WebsocketFrame', diff --git a/proxy/http/websocket/client.py b/proxy/http/websocket/client.py index 498df469bb..f398b5d827 100644 --- a/proxy/http/websocket/client.py +++ b/proxy/http/websocket/client.py @@ -13,16 +13,17 @@ import socket import secrets import selectors - -from typing import Optional, Union, Callable +from typing import Union, Callable, Optional from .frame import WebsocketFrame - -from ..parser import httpParserTypes, HttpParser - -from ...common.constants import DEFAULT_BUFFER_SIZE, DEFAULT_SELECTOR_SELECT_TIMEOUT -from ...common.utils import new_socket_connection, build_websocket_handshake_request, text_ -from ...core.connection import tcpConnectionTypes, TcpConnection +from ..parser import HttpParser, httpParserTypes +from ...common.utils import ( + text_, new_socket_connection, build_websocket_handshake_request, +) +from ...core.connection import TcpConnection, tcpConnectionTypes +from ...common.constants import ( + DEFAULT_BUFFER_SIZE, DEFAULT_SELECTOR_SELECT_TIMEOUT, +) class WebsocketClient(TcpConnection): diff --git a/proxy/http/websocket/frame.py b/proxy/http/websocket/frame.py index e17490591f..7cccbcb067 100644 --- a/proxy/http/websocket/frame.py +++ b/proxy/http/websocket/frame.py @@ -19,10 +19,9 @@ import base64 import struct import hashlib -import secrets import logging - -from typing import TypeVar, Type, Optional, NamedTuple +import secrets +from typing import Type, TypeVar, Optional, NamedTuple WebsocketOpcodes = NamedTuple( diff --git a/proxy/http/websocket/plugin.py b/proxy/http/websocket/plugin.py index f7f274859c..be544773a7 100644 --- a/proxy/http/websocket/plugin.py +++ b/proxy/http/websocket/plugin.py @@ -8,15 +8,15 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import argparse import json +import argparse from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, List, Dict, Any +from typing import TYPE_CHECKING, Any, Dict, List -from ...common.utils import bytes_ +from . import WebsocketFrame from ...core.event import EventQueue +from ...common.utils import bytes_ -from . import WebsocketFrame if TYPE_CHECKING: # pragma: no cover from ..connection import HttpClientConnection diff --git a/proxy/http/websocket/transport.py b/proxy/http/websocket/transport.py index b18a8064e0..463195ef5c 100644 --- a/proxy/http/websocket/transport.py +++ b/proxy/http/websocket/transport.py @@ -10,15 +10,14 @@ """ import json import logging -from typing import List, Tuple, Any, Dict - -from ...common.utils import bytes_ - -from ..server import httpProtocolTypes, HttpWebServerBasePlugin -from ..parser import HttpParser +from typing import Any, Dict, List, Tuple from .frame import WebsocketFrame from .plugin import WebSocketTransportBasePlugin +from ..parser import HttpParser +from ..server import HttpWebServerBasePlugin, httpProtocolTypes +from ...common.utils import bytes_ + logger = logging.getLogger(__name__) diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index 578bc3bcc4..e76a253468 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -13,21 +13,22 @@ Cloudflare """ from .cache import CacheResponsesPlugin, BaseCacheResponsesPlugin -from .filter_by_upstream import FilterByUpstreamHostPlugin -from .man_in_the_middle import ManInTheMiddlePlugin +from .shortlink import ShortLinkPlugin +from .proxy_pool import ProxyPoolPlugin +from .program_name import ProgramNamePlugin from .mock_rest_api import ProposedRestApiPlugin +from .reverse_proxy import ReverseProxyPlugin +from .cloudflare_dns import CloudflareDnsResolverPlugin from .modify_post_data import ModifyPostDataPlugin -from .redirect_to_custom_server import RedirectToCustomServerPlugin -from .shortlink import ShortLinkPlugin from .web_server_route import WebServerPlugin -from .reverse_proxy import ReverseProxyPlugin -from .proxy_pool import ProxyPoolPlugin +from .man_in_the_middle import ManInTheMiddlePlugin +from .filter_by_upstream import FilterByUpstreamHostPlugin +from .custom_dns_resolver import CustomDnsResolverPlugin from .filter_by_client_ip import FilterByClientIpPlugin from .filter_by_url_regex import FilterByURLRegexPlugin from .modify_chunk_response import ModifyChunkResponsePlugin -from .custom_dns_resolver import CustomDnsResolverPlugin -from .cloudflare_dns import CloudflareDnsResolverPlugin -from .program_name import ProgramNamePlugin +from .redirect_to_custom_server import RedirectToCustomServerPlugin + __all__ = [ 'CacheResponsesPlugin', diff --git a/proxy/plugin/cache/__init__.py b/proxy/plugin/cache/__init__.py index f3bfb84b2c..ce6cfe4291 100644 --- a/proxy/plugin/cache/__init__.py +++ b/proxy/plugin/cache/__init__.py @@ -11,6 +11,7 @@ from .base import BaseCacheResponsesPlugin from .cache_responses import CacheResponsesPlugin + __all__ = [ 'BaseCacheResponsesPlugin', 'CacheResponsesPlugin', diff --git a/proxy/plugin/cache/base.py b/proxy/plugin/cache/base.py index 4451adf1a7..278f4eed2b 100644 --- a/proxy/plugin/cache/base.py +++ b/proxy/plugin/cache/base.py @@ -9,11 +9,12 @@ :license: BSD, see LICENSE for more details. """ import logging -from typing import Optional, Any +from typing import Any, Optional -from ...http.parser import HttpParser -from ...http.proxy import HttpProxyBasePlugin from .store.base import CacheStore +from ...http.proxy import HttpProxyBasePlugin +from ...http.parser import HttpParser + logger = logging.getLogger(__name__) diff --git a/proxy/plugin/cache/cache_responses.py b/proxy/plugin/cache/cache_responses.py index 200a60cb39..e9ccd126d6 100644 --- a/proxy/plugin/cache/cache_responses.py +++ b/proxy/plugin/cache/cache_responses.py @@ -11,8 +11,8 @@ import multiprocessing from typing import Any -from .store.disk import OnDiskCacheStore from .base import BaseCacheResponsesPlugin +from .store.disk import OnDiskCacheStore class CacheResponsesPlugin(BaseCacheResponsesPlugin): diff --git a/proxy/plugin/cache/store/disk.py b/proxy/plugin/cache/store/disk.py index d4e6b9dc39..8ea54b8c01 100644 --- a/proxy/plugin/cache/store/disk.py +++ b/proxy/plugin/cache/store/disk.py @@ -8,16 +8,16 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import logging import os +import logging import tempfile -from typing import Optional, BinaryIO +from typing import BinaryIO, Optional +from .base import CacheStore from ....common.flag import flags -from ....common.utils import text_ from ....http.parser import HttpParser +from ....common.utils import text_ -from .base import CacheStore logger = logging.getLogger(__name__) diff --git a/proxy/plugin/cloudflare_dns.py b/proxy/plugin/cloudflare_dns.py index 781cb8d489..9c459b56ab 100644 --- a/proxy/plugin/cloudflare_dns.py +++ b/proxy/plugin/cloudflare_dns.py @@ -16,15 +16,17 @@ """ import logging + try: import httpx except ImportError: # pragma: no cover pass -from typing import Optional, Tuple +from typing import Tuple, Optional -from ..common.flag import flags from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags + logger = logging.getLogger(__name__) diff --git a/proxy/plugin/custom_dns_resolver.py b/proxy/plugin/custom_dns_resolver.py index f40a93e8ed..5ee7aded91 100644 --- a/proxy/plugin/custom_dns_resolver.py +++ b/proxy/plugin/custom_dns_resolver.py @@ -13,8 +13,7 @@ dns """ import socket - -from typing import Optional, Tuple +from typing import Tuple, Optional from ..http.proxy import HttpProxyBasePlugin diff --git a/proxy/plugin/filter_by_client_ip.py b/proxy/plugin/filter_by_client_ip.py index fba981012d..2f19904284 100644 --- a/proxy/plugin/filter_by_client_ip.py +++ b/proxy/plugin/filter_by_client_ip.py @@ -14,11 +14,10 @@ """ from typing import Optional -from ..common.flag import flags - from ..http import httpStatusCodes -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags +from ..http.parser import HttpParser from ..http.exception import HttpRequestRejected diff --git a/proxy/plugin/filter_by_upstream.py b/proxy/plugin/filter_by_upstream.py index a257fdf286..39f70859f4 100644 --- a/proxy/plugin/filter_by_upstream.py +++ b/proxy/plugin/filter_by_upstream.py @@ -10,12 +10,11 @@ """ from typing import Optional -from ..common.utils import text_ -from ..common.flag import flags - from ..http import httpStatusCodes -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags +from ..http.parser import HttpParser +from ..common.utils import text_ from ..http.exception import HttpRequestRejected diff --git a/proxy/plugin/filter_by_url_regex.py b/proxy/plugin/filter_by_url_regex.py index 3d12ad342c..587c6f63b7 100644 --- a/proxy/plugin/filter_by_url_regex.py +++ b/proxy/plugin/filter_by_url_regex.py @@ -12,20 +12,18 @@ url """ +import re import json import logging - -from typing import Optional, List, Dict, Any - -from ..common.flag import flags -from ..common.utils import text_ +from typing import Any, Dict, List, Optional from ..http import httpStatusCodes -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags +from ..http.parser import HttpParser +from ..common.utils import text_ from ..http.exception import HttpRequestRejected -import re logger = logging.getLogger(__name__) diff --git a/proxy/plugin/man_in_the_middle.py b/proxy/plugin/man_in_the_middle.py index d571f25a23..1ceb3a21b4 100644 --- a/proxy/plugin/man_in_the_middle.py +++ b/proxy/plugin/man_in_the_middle.py @@ -10,8 +10,8 @@ """ from typing import Optional -from ..http.responses import okResponse from ..http.proxy import HttpProxyBasePlugin +from ..http.responses import okResponse class ManInTheMiddlePlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/mock_rest_api.py b/proxy/plugin/mock_rest_api.py index 56cac4cb27..2588ce0592 100644 --- a/proxy/plugin/mock_rest_api.py +++ b/proxy/plugin/mock_rest_api.py @@ -15,11 +15,10 @@ import json from typing import Optional -from ..common.utils import bytes_, text_ - -from ..http.responses import okResponse, NOT_FOUND_RESPONSE_PKT -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser +from ..common.utils import text_, bytes_ +from ..http.responses import NOT_FOUND_RESPONSE_PKT, okResponse class ProposedRestApiPlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/modify_chunk_response.py b/proxy/plugin/modify_chunk_response.py index a0838da58d..16171e1f11 100644 --- a/proxy/plugin/modify_chunk_response.py +++ b/proxy/plugin/modify_chunk_response.py @@ -10,8 +10,8 @@ """ from typing import Any, Optional -from ..http.parser import HttpParser, httpParserTypes from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser, httpParserTypes class ModifyChunkResponsePlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/modify_post_data.py b/proxy/plugin/modify_post_data.py index 21be9d358d..d4b5ba6174 100644 --- a/proxy/plugin/modify_post_data.py +++ b/proxy/plugin/modify_post_data.py @@ -10,11 +10,10 @@ """ from typing import Optional -from ..common.utils import bytes_ - from ..http import httpMethods -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser +from ..common.utils import bytes_ class ModifyPostDataPlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/program_name.py b/proxy/plugin/program_name.py index b8e205b4df..0aaadcb817 100644 --- a/proxy/plugin/program_name.py +++ b/proxy/plugin/program_name.py @@ -10,15 +10,13 @@ """ import os import subprocess - from typing import Any, Dict, Optional +from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser from ..common.utils import text_ from ..common.constants import IS_WINDOWS -from ..http.parser import HttpParser -from ..http.proxy import HttpProxyBasePlugin - class ProgramNamePlugin(HttpProxyBasePlugin): """Tries to identify the application connecting to the diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py index 41ffe5b711..6e04034721 100644 --- a/proxy/plugin/proxy_pool.py +++ b/proxy/plugin/proxy_pool.py @@ -12,19 +12,19 @@ import random import logging import ipaddress +from typing import Any, Dict, List, Optional -from typing import Dict, List, Optional, Any - +from ..http import Url, httpHeaders, httpMethods +from ..core.base import TcpUpstreamConnectionHandler +from ..http.proxy import HttpProxyBasePlugin from ..common.flag import flags -from ..common.utils import text_, bytes_ -from ..common.constants import COLON, LOCAL_INTERFACE_HOSTNAMES, ANY_INTERFACE_HOSTNAMES - -from ..http import Url, httpMethods, httpHeaders from ..http.parser import HttpParser +from ..common.utils import text_, bytes_ from ..http.exception import HttpProtocolException -from ..http.proxy import HttpProxyBasePlugin +from ..common.constants import ( + COLON, ANY_INTERFACE_HOSTNAMES, LOCAL_INTERFACE_HOSTNAMES, +) -from ..core.base import TcpUpstreamConnectionHandler logger = logging.getLogger(__name__) diff --git a/proxy/plugin/redirect_to_custom_server.py b/proxy/plugin/redirect_to_custom_server.py index 82f4b0bfbf..d1c893bd9b 100644 --- a/proxy/plugin/redirect_to_custom_server.py +++ b/proxy/plugin/redirect_to_custom_server.py @@ -8,8 +8,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from urllib import parse as urlparse from typing import Optional +from urllib import parse as urlparse from ..http.proxy import HttpProxyBasePlugin from ..http.parser import HttpParser diff --git a/proxy/plugin/reverse_proxy.py b/proxy/plugin/reverse_proxy.py index 26ebbb6aae..e21442e7e7 100644 --- a/proxy/plugin/reverse_proxy.py +++ b/proxy/plugin/reverse_proxy.py @@ -10,18 +10,16 @@ """ import random import logging - -from typing import List, Tuple, Any, Dict, Optional - -from ..common.utils import text_ -from ..common.constants import DEFAULT_HTTPS_PORT, DEFAULT_HTTP_PORT +from typing import Any, Dict, List, Tuple, Optional from ..http import Url -from ..http.exception import HttpProtocolException +from ..core.base import TcpUpstreamConnectionHandler from ..http.parser import HttpParser from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes +from ..common.utils import text_ +from ..http.exception import HttpProtocolException +from ..common.constants import DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT -from ..core.base import TcpUpstreamConnectionHandler logger = logging.getLogger(__name__) diff --git a/proxy/plugin/shortlink.py b/proxy/plugin/shortlink.py index 11b7a7d823..0f5840ba08 100644 --- a/proxy/plugin/shortlink.py +++ b/proxy/plugin/shortlink.py @@ -14,11 +14,10 @@ """ from typing import Optional -from ..common.constants import DOT, SLASH - -from ..http.responses import NOT_FOUND_RESPONSE_PKT, seeOthersResponse -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser +from ..http.responses import NOT_FOUND_RESPONSE_PKT, seeOthersResponse +from ..common.constants import DOT, SLASH class ShortLinkPlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index d4ad4effd9..205a8f9bf2 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -11,9 +11,10 @@ import logging from typing import List, Tuple -from ..http.responses import okResponse from ..http.parser import HttpParser from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes +from ..http.responses import okResponse + logger = logging.getLogger(__name__) diff --git a/proxy/proxy.py b/proxy/proxy.py index e2e2af1525..8f04b2af62 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -13,22 +13,20 @@ import time import signal import logging +from typing import Any, List, Optional -from typing import List, Optional, Any - -from proxy.core.ssh.listener import SshTunnelListener - +from .core.ssh import SshTunnelListener, SshHttpProtocolHandler from .core.work import ThreadlessPool from .core.event import EventManager -from .core.ssh import SshHttpProtocolHandler -from .core.acceptor import AcceptorPool, Listener - -from .common.utils import bytes_ from .common.flag import FlagParser, flags -from .common.constants import DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_LOG_FILE -from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PLUGINS, DEFAULT_VERSION -from .common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_WORK_KLASS, DEFAULT_PID_FILE -from .common.constants import DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, IS_WINDOWS +from .common.utils import bytes_ +from .core.acceptor import Listener, AcceptorPool +from .common.constants import ( + IS_WINDOWS, DEFAULT_PLUGINS, DEFAULT_VERSION, DEFAULT_LOG_FILE, + DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT, + DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_ENABLE_DASHBOARD, + DEFAULT_ENABLE_SSH_TUNNEL, +) logger = logging.getLogger(__name__) diff --git a/proxy/testing/__init__.py b/proxy/testing/__init__.py index e841545b30..f0b7ffe5af 100644 --- a/proxy/testing/__init__.py +++ b/proxy/testing/__init__.py @@ -10,6 +10,7 @@ """ from .test_case import TestCase + __all__ = [ 'TestCase', ] diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index f6d75ce0de..18c75216ce 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -8,15 +8,16 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import contextlib import time +import contextlib +from typing import Any, List, Optional, Generator + import unittest -from typing import Optional, List, Generator, Any from ..proxy import Proxy -from ..common.constants import DEFAULT_TIMEOUT -from ..common.utils import new_socket_connection from ..plugin import CacheResponsesPlugin +from ..common.utils import new_socket_connection +from ..common.constants import DEFAULT_TIMEOUT class TestCase(unittest.TestCase): diff --git a/tests/http/exceptions/test_http_proxy_auth_failed.py b/tests/http/exceptions/test_http_proxy_auth_failed.py index 5d74aecb7a..9dfbe2142e 100644 --- a/tests/http/exceptions/test_http_proxy_auth_failed.py +++ b/tests/http/exceptions/test_http_proxy_auth_failed.py @@ -14,7 +14,7 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpHeaders, HttpClientConnection +from proxy.http import HttpProtocolHandler, HttpClientConnection, httpHeaders from proxy.common.flag import FlagParser from proxy.common.utils import build_http_request from proxy.http.responses import PROXY_AUTH_FAILED_RESPONSE_PKT diff --git a/tests/http/parser/test_http_parser.py b/tests/http/parser/test_http_parser.py index 562574d4e9..68510415ea 100644 --- a/tests/http/parser/test_http_parser.py +++ b/tests/http/parser/test_http_parser.py @@ -16,8 +16,8 @@ bytes_, find_http_line, build_http_header, build_http_request, ) from proxy.http.exception import HttpProtocolException -from proxy.http.responses import okResponse from proxy.http.protocols import httpProtocols +from proxy.http.responses import okResponse from proxy.common.constants import CRLF, HTTP_1_0 diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py index b13c748d41..e7e166f372 100644 --- a/tests/http/proxy/test_http_proxy_tls_interception.py +++ b/tests/http/proxy/test_http_proxy_tls_interception.py @@ -19,7 +19,7 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpMethods, HttpClientConnection +from proxy.http import HttpProtocolHandler, HttpClientConnection, httpMethods from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.common.utils import bytes_, build_http_request diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py index 707725ab6f..a2425fd723 100644 --- a/tests/http/test_protocol_handler.py +++ b/tests/http/test_protocol_handler.py @@ -17,7 +17,7 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpHeaders, HttpClientConnection +from proxy.http import HttpProtocolHandler, HttpClientConnection, httpHeaders from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates @@ -25,8 +25,8 @@ from proxy.common.plugins import Plugins from proxy.common.version import __version__ from proxy.http.responses import ( - BAD_GATEWAY_RESPONSE_PKT, PROXY_AUTH_FAILED_RESPONSE_PKT, - PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, BAD_REQUEST_RESPONSE_PKT, + BAD_GATEWAY_RESPONSE_PKT, BAD_REQUEST_RESPONSE_PKT, + PROXY_AUTH_FAILED_RESPONSE_PKT, PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, ) from proxy.common.constants import ( CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index e8fd9b7ce5..93a88b731d 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -11,13 +11,14 @@ Test the simplest proxy use scenario for smoke. """ import time -import pytest import tempfile import subprocess - +from typing import Any, List, Generator from pathlib import Path -from typing import Any, Generator, List -from subprocess import Popen, check_output as _check_output +from subprocess import Popen +from subprocess import check_output as _check_output + +import pytest from proxy.common.constants import IS_WINDOWS diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py index 742b7fd742..5b9e30d12f 100644 --- a/tests/plugin/test_http_proxy_plugins.py +++ b/tests/plugin/test_http_proxy_plugins.py @@ -20,7 +20,9 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpStatusCodes, HttpClientConnection +from proxy.http import ( + HttpProtocolHandler, HttpClientConnection, httpStatusCodes, +) from proxy.plugin import ProposedRestApiPlugin, RedirectToCustomServerPlugin from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index a985f0c20f..77345a41d5 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -18,7 +18,7 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpMethods, HttpClientConnection +from proxy.http import HttpProtocolHandler, HttpClientConnection, httpMethods from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes diff --git a/tests/test_main.py b/tests/test_main.py index 070b134bd5..f3b7a85ba0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -17,20 +17,21 @@ from proxy.proxy import main, entry_point from proxy.common.utils import bytes_ from proxy.common.constants import ( # noqa: WPS450 - DEFAULT_CA_CERT_DIR, DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_PORT, DEFAULT_PLUGINS, DEFAULT_TIMEOUT, DEFAULT_KEY_FILE, + DEFAULT_PORT, DEFAULT_PLUGINS, DEFAULT_TIMEOUT, DEFAULT_KEY_FILE, DEFAULT_LOG_FILE, DEFAULT_PAC_FILE, DEFAULT_PID_FILE, PLUGIN_DASHBOARD, DEFAULT_CERT_FILE, DEFAULT_LOG_LEVEL, DEFAULT_PORT_FILE, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, DEFAULT_BASIC_AUTH, DEFAULT_LOG_FORMAT, DEFAULT_THREADLESS, DEFAULT_WORK_KLASS, - DEFAULT_CA_KEY_FILE, DEFAULT_NUM_WORKERS, DEFAULT_CA_CERT_FILE, - DEFAULT_ENABLE_EVENTS, DEFAULT_IPV6_HOSTNAME, DEFAULT_NUM_ACCEPTORS, - DEFAULT_LOCAL_EXECUTOR, PLUGIN_INSPECT_TRAFFIC, DEFAULT_ENABLE_DEVTOOLS, - DEFAULT_OPEN_FILE_LIMIT, DEFAULT_DEVTOOLS_WS_PATH, + DEFAULT_CA_CERT_DIR, DEFAULT_CA_KEY_FILE, DEFAULT_NUM_WORKERS, + DEFAULT_CA_CERT_FILE, DEFAULT_ENABLE_EVENTS, DEFAULT_IPV6_HOSTNAME, + DEFAULT_NUM_ACCEPTORS, DEFAULT_LOCAL_EXECUTOR, PLUGIN_INSPECT_TRAFFIC, + DEFAULT_ENABLE_DEVTOOLS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_ENABLE_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, - DEFAULT_ENABLE_WEB_SERVER, DEFAULT_DISABLE_HTTP_PROXY, - PLUGIN_WEBSOCKET_TRANSPORT, DEFAULT_CA_SIGNING_KEY_FILE, - DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_SERVER_RECVBUF_SIZE, - DEFAULT_ENABLE_STATIC_SERVER, _env_threadless_compliant, + DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_ENABLE_WEB_SERVER, + DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_WEBSOCKET_TRANSPORT, + DEFAULT_CA_SIGNING_KEY_FILE, DEFAULT_CLIENT_RECVBUF_SIZE, + DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_ENABLE_STATIC_SERVER, + _env_threadless_compliant, )