From 3c1f537c292a801936e2184fd5838f8deb21c981 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 13:08:26 +0530 Subject: [PATCH 1/4] `min_compression_length` consistency, it was used as `min_compression_limit` at a few places --- proxy/common/constants.py | 2 +- proxy/common/flag.py | 10 +++++----- proxy/http/responses.py | 6 +++--- proxy/http/server/web.py | 10 ++++++---- proxy/plugin/web_server_route.py | 2 +- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 86be7438c4..90d7243624 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -125,7 +125,7 @@ def _env_threadless_compliant() -> bool: DEFAULT_PORT = 8899 DEFAULT_SERVER_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE DEFAULT_STATIC_SERVER_DIR = os.path.join(PROXY_PY_DIR, "public") -DEFAULT_MIN_COMPRESSION_LIMIT = 20 # In bytes +DEFAULT_MIN_COMPRESSION_LENGTH = 20 # In bytes DEFAULT_THREADLESS = _env_threadless_compliant() DEFAULT_LOCAL_EXECUTOR = True DEFAULT_TIMEOUT = 10.0 diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 35849d2dc6..f74acc0ddd 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -30,7 +30,7 @@ PLUGIN_REVERSE_PROXY, DEFAULT_NUM_ACCEPTORS, PLUGIN_INSPECT_TRAFFIC, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE, DEFAULT_DEVTOOLS_WS_PATH, PLUGIN_DEVTOOLS_PROTOCOL, PLUGIN_WEBSOCKET_TRANSPORT, - DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_MIN_COMPRESSION_LIMIT, + DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_MIN_COMPRESSION_LENGTH, ) @@ -335,13 +335,13 @@ def initialize( args.static_server_dir, ), ) - args.min_compression_limit = cast( + args.min_compression_length = cast( bool, opts.get( - 'min_compression_limit', + 'min_compression_length', getattr( - args, 'min_compression_limit', - DEFAULT_MIN_COMPRESSION_LIMIT, + args, 'min_compression_length', + DEFAULT_MIN_COMPRESSION_LENGTH, ), ), ) diff --git a/proxy/http/responses.py b/proxy/http/responses.py index c1e8a17395..e49c281381 100644 --- a/proxy/http/responses.py +++ b/proxy/http/responses.py @@ -12,9 +12,8 @@ from typing import Any, Dict, Optional from .codes import httpStatusCodes -from ..common.flag import flags from ..common.utils import build_http_response -from ..common.constants import PROXY_AGENT_HEADER_KEY, PROXY_AGENT_HEADER_VALUE +from ..common.constants import DEFAULT_MIN_COMPRESSION_LENGTH, PROXY_AGENT_HEADER_KEY, PROXY_AGENT_HEADER_VALUE PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview( @@ -98,10 +97,11 @@ def okResponse( content: Optional[bytes] = None, headers: Optional[Dict[bytes, bytes]] = None, compress: bool = True, + min_compression_length: int = DEFAULT_MIN_COMPRESSION_LENGTH, **kwargs: Any, ) -> memoryview: do_compress: bool = False - if flags.args and compress and content and len(content) > flags.args.min_compression_limit: + if compress and content and len(content) > min_compression_length: do_compress = True if not headers: headers = {} diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index 34ab4d3fe4..bc2cc1e7f4 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -29,7 +29,7 @@ from ...common.constants import ( DEFAULT_ENABLE_WEB_SERVER, DEFAULT_STATIC_SERVER_DIR, DEFAULT_ENABLE_REVERSE_PROXY, DEFAULT_ENABLE_STATIC_SERVER, - DEFAULT_MIN_COMPRESSION_LIMIT, DEFAULT_WEB_ACCESS_LOG_FORMAT, + DEFAULT_MIN_COMPRESSION_LENGTH, DEFAULT_WEB_ACCESS_LOG_FORMAT, ) @@ -65,8 +65,8 @@ flags.add_argument( '--min-compression-length', type=int, - default=DEFAULT_MIN_COMPRESSION_LIMIT, - help='Default: ' + str(DEFAULT_MIN_COMPRESSION_LIMIT) + ' bytes. ' + + default=DEFAULT_MIN_COMPRESSION_LENGTH, + help='Default: ' + str(DEFAULT_MIN_COMPRESSION_LENGTH) + ' bytes. ' + 'Sets the minimum length of a response that will be compressed (gzipped).', ) @@ -125,7 +125,7 @@ def encryption_enabled(self) -> bool: self.flags.certfile is not None @staticmethod - def read_and_build_static_file_response(path: str) -> memoryview: + def read_and_build_static_file_response(path: str, min_compression_length: int) -> memoryview: try: with open(path, 'rb') as f: content = f.read() @@ -139,6 +139,7 @@ def read_and_build_static_file_response(path: str) -> memoryview: return okResponse( content=content, headers=headers, + min_compression_length=min_compression_length, # TODO: Should we really close or take advantage of keep-alive? conn_close=True, ) @@ -311,5 +312,6 @@ def _try_static_or_404(self, path: bytes) -> None: self.client.queue( self.read_and_build_static_file_response( self.flags.static_server_dir + path, + self.flags.min_compression_length, ), ) diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index 205a8f9bf2..a0d8dfbf8a 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -34,6 +34,6 @@ def routes(self) -> List[Tuple[int, str]]: def handle_request(self, request: HttpParser) -> None: if request.path == b'/http-route-example': - self.client.queue(HTTP_RESPONSE) + self.client.queue(okResponse()) elif request.path == b'/https-route-example': self.client.queue(HTTPS_RESPONSE) From c45c554883c9fe84904a8fec48f0f94477367700 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 07:40:02 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- proxy/http/responses.py | 5 ++++- proxy/http/server/web.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/proxy/http/responses.py b/proxy/http/responses.py index e49c281381..8a7ed5be7d 100644 --- a/proxy/http/responses.py +++ b/proxy/http/responses.py @@ -13,7 +13,10 @@ from .codes import httpStatusCodes from ..common.utils import build_http_response -from ..common.constants import DEFAULT_MIN_COMPRESSION_LENGTH, PROXY_AGENT_HEADER_KEY, PROXY_AGENT_HEADER_VALUE +from ..common.constants import ( + PROXY_AGENT_HEADER_KEY, PROXY_AGENT_HEADER_VALUE, + DEFAULT_MIN_COMPRESSION_LENGTH, +) PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview( diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index bc2cc1e7f4..d8e21f3324 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -29,7 +29,7 @@ from ...common.constants import ( DEFAULT_ENABLE_WEB_SERVER, DEFAULT_STATIC_SERVER_DIR, DEFAULT_ENABLE_REVERSE_PROXY, DEFAULT_ENABLE_STATIC_SERVER, - DEFAULT_MIN_COMPRESSION_LENGTH, DEFAULT_WEB_ACCESS_LOG_FORMAT, + DEFAULT_WEB_ACCESS_LOG_FORMAT, DEFAULT_MIN_COMPRESSION_LENGTH, ) From 2b30df65e5fb03a55325c60d4e2c046930e79427 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 13:10:15 +0530 Subject: [PATCH 3/4] revert back web server route --- proxy/plugin/web_server_route.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index a0d8dfbf8a..205a8f9bf2 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -34,6 +34,6 @@ def routes(self) -> List[Tuple[int, str]]: def handle_request(self, request: HttpParser) -> None: if request.path == b'/http-route-example': - self.client.queue(okResponse()) + self.client.queue(HTTP_RESPONSE) elif request.path == b'/https-route-example': self.client.queue(HTTPS_RESPONSE) From 51b9f5370e3d485e5527373f8d8b9f093f90d0bd Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 13:51:15 +0530 Subject: [PATCH 4/4] Move `serve_static_file` as a staticmethod within web plugin base --- proxy/dashboard/dashboard.py | 7 +++---- proxy/http/server/plugin.py | 25 +++++++++++++++++++++++++ proxy/http/server/web.py | 29 +++-------------------------- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py index 411c153b60..b5548ea524 100644 --- a/proxy/dashboard/dashboard.py +++ b/proxy/dashboard/dashboard.py @@ -13,9 +13,7 @@ from typing import List, Tuple from ..http.parser import HttpParser -from ..http.server import ( - HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes, -) +from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes from ..http.responses import permanentRedirectResponse @@ -46,11 +44,12 @@ def routes(self) -> List[Tuple[int, str]]: def handle_request(self, request: HttpParser) -> None: if request.path == b'/dashboard/': self.client.queue( - HttpWebServerPlugin.read_and_build_static_file_response( + self.serve_static_file( os.path.join( self.flags.static_server_dir, 'dashboard', 'proxy.html', ), + self.flags.min_compression_length, ), ) elif request.path in ( diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 7870620033..0115558c18 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -9,14 +9,17 @@ :license: BSD, see LICENSE for more details. """ import argparse +import mimetypes from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Optional from ..parser import HttpParser +from ..responses import NOT_FOUND_RESPONSE_PKT, okResponse from ..websocket import WebsocketFrame from ..connection import HttpClientConnection from ...core.event import EventQueue from ..descriptors import DescriptorsHandlerMixin +from ...common.utils import bytes_ if TYPE_CHECKING: # pragma: no cover @@ -40,6 +43,28 @@ def __init__( self.event_queue = event_queue self.upstream_conn_pool = upstream_conn_pool + @staticmethod + def serve_static_file(path: str, min_compression_length: int) -> memoryview: + try: + with open(path, 'rb') as f: + content = f.read() + content_type = mimetypes.guess_type(path)[0] + if content_type is None: + content_type = 'text/plain' + headers = { + b'Content-Type': bytes_(content_type), + b'Cache-Control': b'max-age=86400', + } + return okResponse( + content=content, + headers=headers, + min_compression_length=min_compression_length, + # TODO: Should we really close or take advantage of keep-alive? + conn_close=True, + ) + except FileNotFoundError: + return NOT_FOUND_RESPONSE_PKT + def name(self) -> str: """A unique name for your plugin. diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index d8e21f3324..04b97b4f50 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -12,7 +12,6 @@ import time import socket import logging -import mimetypes from typing import Any, Dict, List, Tuple, Union, Pattern, Optional from .plugin import HttpWebServerBasePlugin @@ -21,11 +20,11 @@ from .protocols import httpProtocolTypes from ..exception import HttpProtocolException from ..protocols import httpProtocols -from ..responses import NOT_FOUND_RESPONSE_PKT, okResponse +from ..responses import NOT_FOUND_RESPONSE_PKT from ..websocket import WebsocketFrame, websocketOpcodes from ...common.flag import flags from ...common.types import Readables, Writables, Descriptors -from ...common.utils import text_, bytes_, build_websocket_handshake_response +from ...common.utils import text_, build_websocket_handshake_response from ...common.constants import ( DEFAULT_ENABLE_WEB_SERVER, DEFAULT_STATIC_SERVER_DIR, DEFAULT_ENABLE_REVERSE_PROXY, DEFAULT_ENABLE_STATIC_SERVER, @@ -124,28 +123,6 @@ def encryption_enabled(self) -> bool: return self.flags.keyfile is not None and \ self.flags.certfile is not None - @staticmethod - def read_and_build_static_file_response(path: str, min_compression_length: int) -> memoryview: - try: - with open(path, 'rb') as f: - content = f.read() - content_type = mimetypes.guess_type(path)[0] - if content_type is None: - content_type = 'text/plain' - headers = { - b'Content-Type': bytes_(content_type), - b'Cache-Control': b'max-age=86400', - } - return okResponse( - content=content, - headers=headers, - min_compression_length=min_compression_length, - # TODO: Should we really close or take advantage of keep-alive? - conn_close=True, - ) - except FileNotFoundError: - return NOT_FOUND_RESPONSE_PKT - def switch_to_websocket(self) -> None: self.client.queue( memoryview( @@ -310,7 +287,7 @@ def _try_route(self, path: bytes) -> bool: def _try_static_or_404(self, path: bytes) -> None: path = text_(path).split('?', 1)[0] self.client.queue( - self.read_and_build_static_file_response( + HttpWebServerBasePlugin.serve_static_file( self.flags.static_server_dir + path, self.flags.min_compression_length, ),