diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 86be7438c4..90d7243624 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -125,7 +125,7 @@ def _env_threadless_compliant() -> bool: DEFAULT_PORT = 8899 DEFAULT_SERVER_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE DEFAULT_STATIC_SERVER_DIR = os.path.join(PROXY_PY_DIR, "public") -DEFAULT_MIN_COMPRESSION_LIMIT = 20 # In bytes +DEFAULT_MIN_COMPRESSION_LENGTH = 20 # In bytes DEFAULT_THREADLESS = _env_threadless_compliant() DEFAULT_LOCAL_EXECUTOR = True DEFAULT_TIMEOUT = 10.0 diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 35849d2dc6..f74acc0ddd 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -30,7 +30,7 @@ PLUGIN_REVERSE_PROXY, DEFAULT_NUM_ACCEPTORS, PLUGIN_INSPECT_TRAFFIC, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE, DEFAULT_DEVTOOLS_WS_PATH, PLUGIN_DEVTOOLS_PROTOCOL, PLUGIN_WEBSOCKET_TRANSPORT, - DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_MIN_COMPRESSION_LIMIT, + DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_MIN_COMPRESSION_LENGTH, ) @@ -335,13 +335,13 @@ def initialize( args.static_server_dir, ), ) - args.min_compression_limit = cast( + args.min_compression_length = cast( bool, opts.get( - 'min_compression_limit', + 'min_compression_length', getattr( - args, 'min_compression_limit', - DEFAULT_MIN_COMPRESSION_LIMIT, + args, 'min_compression_length', + DEFAULT_MIN_COMPRESSION_LENGTH, ), ), ) diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py index 411c153b60..b5548ea524 100644 --- a/proxy/dashboard/dashboard.py +++ b/proxy/dashboard/dashboard.py @@ -13,9 +13,7 @@ from typing import List, Tuple from ..http.parser import HttpParser -from ..http.server import ( - HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes, -) +from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes from ..http.responses import permanentRedirectResponse @@ -46,11 +44,12 @@ def routes(self) -> List[Tuple[int, str]]: def handle_request(self, request: HttpParser) -> None: if request.path == b'/dashboard/': self.client.queue( - HttpWebServerPlugin.read_and_build_static_file_response( + self.serve_static_file( os.path.join( self.flags.static_server_dir, 'dashboard', 'proxy.html', ), + self.flags.min_compression_length, ), ) elif request.path in ( diff --git a/proxy/http/responses.py b/proxy/http/responses.py index c1e8a17395..8a7ed5be7d 100644 --- a/proxy/http/responses.py +++ b/proxy/http/responses.py @@ -12,9 +12,11 @@ from typing import Any, Dict, Optional from .codes import httpStatusCodes -from ..common.flag import flags from ..common.utils import build_http_response -from ..common.constants import PROXY_AGENT_HEADER_KEY, PROXY_AGENT_HEADER_VALUE +from ..common.constants import ( + PROXY_AGENT_HEADER_KEY, PROXY_AGENT_HEADER_VALUE, + DEFAULT_MIN_COMPRESSION_LENGTH, +) PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview( @@ -98,10 +100,11 @@ def okResponse( content: Optional[bytes] = None, headers: Optional[Dict[bytes, bytes]] = None, compress: bool = True, + min_compression_length: int = DEFAULT_MIN_COMPRESSION_LENGTH, **kwargs: Any, ) -> memoryview: do_compress: bool = False - if flags.args and compress and content and len(content) > flags.args.min_compression_limit: + if compress and content and len(content) > min_compression_length: do_compress = True if not headers: headers = {} diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 7870620033..0115558c18 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -9,14 +9,17 @@ :license: BSD, see LICENSE for more details. """ import argparse +import mimetypes from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Optional from ..parser import HttpParser +from ..responses import NOT_FOUND_RESPONSE_PKT, okResponse from ..websocket import WebsocketFrame from ..connection import HttpClientConnection from ...core.event import EventQueue from ..descriptors import DescriptorsHandlerMixin +from ...common.utils import bytes_ if TYPE_CHECKING: # pragma: no cover @@ -40,6 +43,28 @@ def __init__( self.event_queue = event_queue self.upstream_conn_pool = upstream_conn_pool + @staticmethod + def serve_static_file(path: str, min_compression_length: int) -> memoryview: + try: + with open(path, 'rb') as f: + content = f.read() + content_type = mimetypes.guess_type(path)[0] + if content_type is None: + content_type = 'text/plain' + headers = { + b'Content-Type': bytes_(content_type), + b'Cache-Control': b'max-age=86400', + } + return okResponse( + content=content, + headers=headers, + min_compression_length=min_compression_length, + # TODO: Should we really close or take advantage of keep-alive? + conn_close=True, + ) + except FileNotFoundError: + return NOT_FOUND_RESPONSE_PKT + def name(self) -> str: """A unique name for your plugin. diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index 34ab4d3fe4..04b97b4f50 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -12,7 +12,6 @@ import time import socket import logging -import mimetypes from typing import Any, Dict, List, Tuple, Union, Pattern, Optional from .plugin import HttpWebServerBasePlugin @@ -21,15 +20,15 @@ from .protocols import httpProtocolTypes from ..exception import HttpProtocolException from ..protocols import httpProtocols -from ..responses import NOT_FOUND_RESPONSE_PKT, okResponse +from ..responses import NOT_FOUND_RESPONSE_PKT from ..websocket import WebsocketFrame, websocketOpcodes from ...common.flag import flags from ...common.types import Readables, Writables, Descriptors -from ...common.utils import text_, bytes_, build_websocket_handshake_response +from ...common.utils import text_, build_websocket_handshake_response from ...common.constants import ( DEFAULT_ENABLE_WEB_SERVER, DEFAULT_STATIC_SERVER_DIR, DEFAULT_ENABLE_REVERSE_PROXY, DEFAULT_ENABLE_STATIC_SERVER, - DEFAULT_MIN_COMPRESSION_LIMIT, DEFAULT_WEB_ACCESS_LOG_FORMAT, + DEFAULT_WEB_ACCESS_LOG_FORMAT, DEFAULT_MIN_COMPRESSION_LENGTH, ) @@ -65,8 +64,8 @@ flags.add_argument( '--min-compression-length', type=int, - default=DEFAULT_MIN_COMPRESSION_LIMIT, - help='Default: ' + str(DEFAULT_MIN_COMPRESSION_LIMIT) + ' bytes. ' + + default=DEFAULT_MIN_COMPRESSION_LENGTH, + help='Default: ' + str(DEFAULT_MIN_COMPRESSION_LENGTH) + ' bytes. ' + 'Sets the minimum length of a response that will be compressed (gzipped).', ) @@ -124,27 +123,6 @@ def encryption_enabled(self) -> bool: return self.flags.keyfile is not None and \ self.flags.certfile is not None - @staticmethod - def read_and_build_static_file_response(path: str) -> memoryview: - try: - with open(path, 'rb') as f: - content = f.read() - content_type = mimetypes.guess_type(path)[0] - if content_type is None: - content_type = 'text/plain' - headers = { - b'Content-Type': bytes_(content_type), - b'Cache-Control': b'max-age=86400', - } - return okResponse( - content=content, - headers=headers, - # TODO: Should we really close or take advantage of keep-alive? - conn_close=True, - ) - except FileNotFoundError: - return NOT_FOUND_RESPONSE_PKT - def switch_to_websocket(self) -> None: self.client.queue( memoryview( @@ -309,7 +287,8 @@ def _try_route(self, path: bytes) -> bool: def _try_static_or_404(self, path: bytes) -> None: path = text_(path).split('?', 1)[0] self.client.queue( - self.read_and_build_static_file_response( + HttpWebServerBasePlugin.serve_static_file( self.flags.static_server_dir + path, + self.flags.min_compression_length, ), )