From 4f8854b731b117a6994b68b2c69d733681d9c6c7 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 16:49:28 +0530 Subject: [PATCH 1/6] Add plugin tests, responses notebook and enhancements to `build_http_packet` --- README.md | 2 +- proxy/common/utils.py | 23 +- proxy/plugin/modify_post_data.py | 16 +- proxy/plugin/web_server_route.py | 4 + tests/plugin/test_http_proxy_plugins.py | 154 +++++++++++++ tests/plugin/utils.py | 8 +- tutorial/requests.ipynb | 273 ++++++++++++++++++++++++ tutorial/responses.ipynb | 6 +- 8 files changed, 470 insertions(+), 16 deletions(-) create mode 100644 tutorial/requests.ipynb diff --git a/README.md b/README.md index 234c915be6..e38d107c43 100644 --- a/README.md +++ b/README.md @@ -2297,7 +2297,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.0rc8.dev17+g59a4335.d20220123 +proxy.py v2.4.0rc9.dev8+gea0253d.d20220126 options: -h, --help show this help message and exit diff --git a/proxy/common/utils.py b/proxy/common/utils.py index cb2cb1dbd9..d157775d7f 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -26,7 +26,7 @@ from .types import HostPort from .constants import ( CRLF, COLON, HTTP_1_1, IS_WINDOWS, WHITESPACE, DEFAULT_TIMEOUT, - DEFAULT_THREADLESS, + DEFAULT_THREADLESS, PROXY_AGENT_HEADER_VALUE, ) @@ -84,14 +84,30 @@ def bytes_(s: Any, encoding: str = 'utf-8', errors: str = 'strict') -> Any: def build_http_request( method: bytes, url: bytes, protocol_version: bytes = HTTP_1_1, + content_type: Optional[bytes] = None, headers: Optional[Dict[bytes, bytes]] = None, body: Optional[bytes] = None, conn_close: bool = False, + no_ua: bool = False, ) -> bytes: """Build and returns a HTTP request packet.""" + headers = headers or {} + if content_type is not None: + headers[b'Content-Type'] = content_type + has_transfer_encoding = False + has_user_agent = False + for k, _ in headers.items(): + if k.lower() == b'transfer-encoding': + has_transfer_encoding = True + elif k.lower() == b'user-agent': + has_user_agent = True + if body and not has_transfer_encoding: + headers[b'Content-Length'] = bytes_(len(body)) + if not has_user_agent and not no_ua: + headers[b'User-Agent'] = PROXY_AGENT_HEADER_VALUE return build_http_pkt( [method, url, protocol_version], - headers or {}, + headers, body, conn_close, ) @@ -109,8 +125,7 @@ def build_http_response( line = [protocol_version, bytes_(status_code)] if reason: line.append(reason) - if headers is None: - headers = {} + headers = headers or {} has_transfer_encoding = False for k, _ in headers.items(): if k.lower() == b'transfer-encoding': diff --git a/proxy/plugin/modify_post_data.py b/proxy/plugin/modify_post_data.py index d4b5ba6174..83df16c921 100644 --- a/proxy/plugin/modify_post_data.py +++ b/proxy/plugin/modify_post_data.py @@ -12,7 +12,7 @@ from ..http import httpMethods from ..http.proxy import HttpProxyBasePlugin -from ..http.parser import HttpParser +from ..http.parser import HttpParser, ChunkParser from ..common.utils import bytes_ @@ -30,14 +30,20 @@ def handle_client_request( self, request: HttpParser, ) -> Optional[HttpParser]: if request.method == httpMethods.POST: - request.body = ModifyPostDataPlugin.MODIFIED_BODY - # Update Content-Length header only when request is NOT chunked - # encoded + # If request data is compressed, compress the body too + body = ModifyPostDataPlugin.MODIFIED_BODY + # If the request is of type chunked encoding + # add post data as chunk if not request.is_chunked_encoded: + body = ChunkParser.to_chunks( + ModifyPostDataPlugin.MODIFIED_BODY, + ) + else: request.add_header( b'Content-Length', - bytes_(len(request.body)), + bytes_(len(body)), ) + request.body = body # Enforce content-type json if request.has_header(b'Content-Type'): request.del_header(b'Content-Type') diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index 205a8f9bf2..77a09af1b1 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -14,6 +14,7 @@ from ..http.parser import HttpParser from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes from ..http.responses import okResponse +from ..http.websocket.frame import WebsocketFrame logger = logging.getLogger(__name__) @@ -37,3 +38,6 @@ def handle_request(self, request: HttpParser) -> None: self.client.queue(HTTP_RESPONSE) elif request.path == b'/https-route-example': self.client.queue(HTTPS_RESPONSE) + + def on_websocket_message(self, frame: WebsocketFrame) -> None: + self.client.queue(memoryview(WebsocketFrame.text(frame.data or b''))) diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py index 5b9e30d12f..8157edc2ca 100644 --- a/tests/plugin/test_http_proxy_plugins.py +++ b/tests/plugin/test_http_proxy_plugins.py @@ -28,6 +28,9 @@ from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes from proxy.common.utils import bytes_, build_http_request, build_http_response +from proxy.http.responses import ( + NOT_FOUND_RESPONSE_PKT, PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, +) from proxy.common.constants import DEFAULT_HTTP_PORT, PROXY_AGENT_HEADER_VALUE from .utils import get_plugin_by_test_name from ..test_assertions import Assertions @@ -215,6 +218,40 @@ async def test_redirect_to_custom_server_plugin(self) -> None: ), ) + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_redirect_to_custom_server_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_redirect_to_custom_server_plugin_skips_https(self) -> None: + request = build_http_request( + b'CONNECT', b'jaxl.com:443', + headers={ + b'Host': b'jaxl.com:443', + }, + ) + self._conn.recv.return_value = request + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() + self.mock_server_conn.assert_called_with('jaxl.com', 443) + self.assertEqual( + self.protocol_handler.work.buffer[0].tobytes(), + PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + ) + @pytest.mark.asyncio # type: ignore[misc] @pytest.mark.parametrize( "_setUp", @@ -386,3 +423,120 @@ async def test_filter_by_url_regex_plugin(self) -> None: conn_close=True, ), ) + + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_shortlink_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_shortlink_plugin(self) -> None: + request = build_http_request( + b'GET', b'http://t/', + headers={ + b'Host': b't', + }, + ) + self._conn.recv.return_value = request + + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.work.buffer[0].tobytes(), + build_http_response( + status_code=httpStatusCodes.SEE_OTHER, + reason=b'See Other', + headers={ + b'Location': b'http://twitter.com/', + }, + conn_close=True, + ), + ) + + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_shortlink_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_shortlink_plugin_unknown(self) -> None: + request = build_http_request( + b'GET', b'http://unknown/', + headers={ + b'Host': b'unknown', + }, + ) + self._conn.recv.return_value = request + + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.work.buffer[0].tobytes(), + NOT_FOUND_RESPONSE_PKT, + ) + + @pytest.mark.asyncio # type: ignore[misc] + @pytest.mark.parametrize( + "_setUp", + ( + ('test_shortlink_plugin'), + ), + indirect=True, + ) # type: ignore[misc] + async def test_shortlink_plugin_external(self) -> None: + request = build_http_request( + b'GET', b'http://jaxl.com/', + headers={ + b'Host': b'jaxl.com', + }, + ) + self._conn.recv.return_value = request + + self.mock_selector.return_value.select.side_effect = [ + [( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + )], + ] + await self.protocol_handler._run_once() + self.mock_server_conn.assert_called_once_with('jaxl.com', 80) + self.mock_server_conn.return_value.queue.assert_called_with( + build_http_request( + b'GET', b'/', + headers={ + b'Host': b'jaxl.com', + b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, + }, + ), + ) + self.assertFalse(self.protocol_handler.work.has_buffer()) diff --git a/tests/plugin/utils.py b/tests/plugin/utils.py index 7e6335057d..400e874d53 100644 --- a/tests/plugin/utils.py +++ b/tests/plugin/utils.py @@ -11,9 +11,9 @@ from typing import Type from proxy.plugin import ( - CacheResponsesPlugin, ManInTheMiddlePlugin, ModifyPostDataPlugin, - ProposedRestApiPlugin, FilterByURLRegexPlugin, FilterByUpstreamHostPlugin, - RedirectToCustomServerPlugin, + ShortLinkPlugin, CacheResponsesPlugin, ManInTheMiddlePlugin, + ModifyPostDataPlugin, ProposedRestApiPlugin, FilterByURLRegexPlugin, + FilterByUpstreamHostPlugin, RedirectToCustomServerPlugin, ) from proxy.http.proxy import HttpProxyBasePlugin @@ -34,4 +34,6 @@ def get_plugin_by_test_name(test_name: str) -> Type[HttpProxyBasePlugin]: plugin = ManInTheMiddlePlugin elif test_name == 'test_filter_by_url_regex_plugin': plugin = FilterByURLRegexPlugin + elif test_name == 'test_shortlink_plugin': + plugin = ShortLinkPlugin return plugin diff --git a/tutorial/requests.ipynb b/tutorial/requests.ipynb new file mode 100644 index 0000000000..fb2cc36393 --- /dev/null +++ b/tutorial/requests.ipynb @@ -0,0 +1,273 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Http Requests\n", + "\n", + "## Usage\n", + "\n", + "To construct a HTTP request packet you have a variety of facilities available.\n", + "\n", + "Previously we saw how to parse HTTP responses using `HttpParser`. We also saw how `HttpParser` class is capable of parsing various type of HTTP protocols. Remember the _take away_ from that tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\n\\r\\n'\n" + ] + } + ], + "source": [ + "from proxy.http.parser import HttpParser, httpParserTypes\n", + "from proxy.http import httpMethods\n", + "from proxy.common.utils import HTTP_1_1\n", + "\n", + "request = HttpParser(httpParserTypes.REQUEST_PARSER)\n", + "request.path, request.method, request.version = b'/', httpMethods.GET, HTTP_1_1\n", + "request.add_header(b'Host', b'jaxl.com')\n", + "\n", + "print(request.build())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But, this is a painful way to construct request packets. Hence, other high level abstractions are available.\n", + "\n", + "Example, following one liner will give us the same request packet." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\n\\r\\n'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from proxy.common.utils import build_http_request\n", + "\n", + "build_http_request(\n", + " method=httpMethods.GET,\n", + " url=b'/',\n", + " headers={b'Host': b'jaxl.com'},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`build_http_request` ensures a `User-Agent` header. You can provide your own too:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nUser-Agent: my app v1\\r\\n\\r\\n'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "build_http_request(\n", + " method=httpMethods.GET,\n", + " url=b'/',\n", + " headers={\n", + " b'Host': b'jaxl.com',\n", + " b'User-Agent': b'my app v1'\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Or, if you don't want a `User-Agent` header at all" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\n\\r\\n'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "build_http_request(\n", + " method=httpMethods.GET,\n", + " url=b'/',\n", + " headers={b'Host': b'jaxl.com'},\n", + " no_ua=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To add a connection close header, simply pass `conn_close=True`" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\nConnection: close\\r\\n\\r\\n'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "build_http_request(\n", + " method=httpMethods.GET,\n", + " url=b'/',\n", + " headers={b'Host': b'jaxl.com'},\n", + " conn_close=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For `POST` requests, provide the `body` attribute" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'POST / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nContent-Type: application/x-www-form-urlencoded\\r\\nContent-Length: 21\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\nConnection: close\\r\\n\\r\\nkey=value&hello=world'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "build_http_request(\n", + " method=httpMethods.POST,\n", + " url=b'/',\n", + " headers={b'Host': b'jaxl.com'},\n", + " body=b'key=value&hello=world',\n", + " content_type=b'application/x-www-form-urlencoded',\n", + " conn_close=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For chunked data, simply include a `Transfer-Encoding` header. This will omit the `Content-Length` header then:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'POST / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nTransfer-Encoding: chunked\\r\\nContent-Type: application/x-www-form-urlencoded\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\nConnection: close\\r\\n\\r\\n5\\r\\nkey=v\\r\\n5\\r\\nalue&\\r\\n5\\r\\nhello\\r\\n5\\r\\n=worl\\r\\n1\\r\\nd\\r\\n0\\r\\n\\r\\n'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from proxy.http.parser import ChunkParser\n", + "\n", + "build_http_request(\n", + " method=httpMethods.POST,\n", + " url=b'/',\n", + " headers={\n", + " b'Host': b'jaxl.com',\n", + " b'Transfer-Encoding': b'chunked',\n", + " },\n", + " body=ChunkParser.to_chunks(b'key=value&hello=world', chunk_size=5),\n", + " content_type=b'application/x-www-form-urlencoded',\n", + " conn_close=True,\n", + ")" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "da9d6927d62b2b95bde149eedfbd5367cb7f465aad65a736f49c99ee3db39df7" + }, + "kernelspec": { + "display_name": "Python 3.10.0 64-bit ('venv310': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorial/responses.ipynb b/tutorial/responses.ipynb index 11a7a6c5f1..0ff7378461 100644 --- a/tutorial/responses.ipynb +++ b/tutorial/responses.ipynb @@ -8,9 +8,9 @@ "\n", "## Usage\n", "\n", - "To construct a response packet you have a variety of facilities available. We previously experienced how to parse HTTP responses using `HttpParser`. Of-course, we can also construct a response packet using `HttpParser` class.\n", + "To construct a response packet you have a variety of facilities available.\n", "\n", - "Let's construct a HTTP response packet:" + "Previously we saw how to parse HTTP responses using `HttpParser`. Of-course, we can also construct a response packet using `HttpParser` class." ] }, { @@ -42,7 +42,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "But it is a little painful to construct responses like above. Hence, other high level abstractions are available.\n", + "But, this is a painful way to construct responses. Hence, other high level abstractions are available.\n", "\n", "Example, following one liner will give us the same response packet." ] From 4830877238ec03b228c338f00722434933c46463 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 16:51:00 +0530 Subject: [PATCH 2/6] Add js code snip for ws example --- proxy/plugin/web_server_route.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index 77a09af1b1..3139b0e648 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -40,4 +40,11 @@ def handle_request(self, request: HttpParser) -> None: self.client.queue(HTTPS_RESPONSE) def on_websocket_message(self, frame: WebsocketFrame) -> None: + """Open chrome devtools and try using following commands: + + ws = new WebSocket("ws://localhost:8899/ws-route-example") + ws.onmessage = function(m) { console.log(m); } + ws.send('hello') + + """ self.client.queue(memoryview(WebsocketFrame.text(frame.data or b''))) From c6d69fe9bc9ff5fbe3cd398107961b53bcfbdbd4 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 19:23:46 +0530 Subject: [PATCH 3/6] Fix tests --- proxy/http/parser/parser.py | 1 + proxy/plugin/modify_post_data.py | 2 +- .../http/proxy/test_http_proxy_tls_interception.py | 1 + tests/plugin/test_http_proxy_plugins.py | 13 ++++++++++++- ...test_http_proxy_plugins_with_tls_interception.py | 6 +++++- tutorial/responses.ipynb | 2 +- 6 files changed, 21 insertions(+), 4 deletions(-) diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index 7d29f76e40..4877667229 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -270,6 +270,7 @@ def build(self, disable_headers: Optional[List[bytes]] = None, for_proxy: bool = k.lower() not in disable_headers }, body=body, + no_ua=True, ) def build_response(self) -> bytes: diff --git a/proxy/plugin/modify_post_data.py b/proxy/plugin/modify_post_data.py index 83df16c921..a290ed59f0 100644 --- a/proxy/plugin/modify_post_data.py +++ b/proxy/plugin/modify_post_data.py @@ -34,7 +34,7 @@ def handle_client_request( body = ModifyPostDataPlugin.MODIFIED_BODY # If the request is of type chunked encoding # add post data as chunk - if not request.is_chunked_encoded: + if request.is_chunked_encoded: body = ChunkParser.to_chunks( ModifyPostDataPlugin.MODIFIED_BODY, ) diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py index 800cfb82f9..b3734ca23f 100644 --- a/tests/http/proxy/test_http_proxy_tls_interception.py +++ b/tests/http/proxy/test_http_proxy_tls_interception.py @@ -119,6 +119,7 @@ def mock_connection() -> Any: connect_request = build_http_request( httpMethods.CONNECT, bytes_(netloc), headers=headers, + no_ua=True, ) self._conn.recv.return_value = connect_request get_request = build_http_request( diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py index 482db973f4..1a04c288dd 100644 --- a/tests/plugin/test_http_proxy_plugins.py +++ b/tests/plugin/test_http_proxy_plugins.py @@ -93,6 +93,7 @@ async def test_modify_post_data_plugin(self) -> None: b'Content-Length': bytes_(len(original)), }, body=original, + no_ua=True, ) self.mock_selector.return_value.select.side_effect = [ [( @@ -120,6 +121,7 @@ async def test_modify_post_data_plugin(self) -> None: b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, }, body=modified, + no_ua=True, ), ) @@ -189,6 +191,7 @@ async def test_redirect_to_custom_server_plugin(self) -> None: headers={ b'Host': b'example.org', }, + no_ua=True, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ @@ -215,6 +218,7 @@ async def test_redirect_to_custom_server_plugin(self) -> None: b'Host': upstream.netloc, b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, }, + no_ua=True, ), ) @@ -305,6 +309,7 @@ async def test_man_in_the_middle_plugin(self) -> None: headers={ b'Host': b'super.secure', }, + no_ua=True, ) self._conn.recv.return_value = request @@ -363,8 +368,12 @@ def closed() -> bool: b'Host': b'super.secure', b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, }, + no_ua=True, ) - server.queue.assert_called_once_with(queued_request) + server.queue.assert_called_once() + print(server.queue.call_args_list[0][0][0].tobytes()) + print(queued_request) + self.assertEqual(server.queue.call_args_list[0][0][0], queued_request) # Server write await self.protocol_handler._run_once() @@ -514,6 +523,7 @@ async def test_shortlink_plugin_external(self) -> None: headers={ b'Host': b'jaxl.com', }, + no_ua=True, ) self._conn.recv.return_value = request @@ -537,6 +547,7 @@ async def test_shortlink_plugin_external(self) -> None: b'Host': b'jaxl.com', b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, }, + no_ua=True, ), ) self.assertFalse(self.protocol_handler.work.has_buffer()) diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index d48cccd335..64dd9440f6 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -152,7 +152,9 @@ def send(raw: bytes) -> int: self._conn.send.side_effect = send self._conn.recv.return_value = build_http_request( - httpMethods.CONNECT, b'uni.corn:443', + httpMethods.CONNECT, + b'uni.corn:443', + no_ua=True, ) @pytest.mark.asyncio # type: ignore[misc] @@ -193,6 +195,7 @@ async def test_modify_post_data_plugin(self) -> None: b'Content-Type': b'application/x-www-form-urlencoded', }, body=original, + no_ua=True, ) await self.protocol_handler._run_once() self.server.queue.assert_called_once() @@ -242,6 +245,7 @@ async def test_man_in_the_middle_plugin(self) -> None: headers={ b'Host': b'uni.corn', }, + no_ua=True, ) self.client_ssl_connection.recv.return_value = request diff --git a/tutorial/responses.ipynb b/tutorial/responses.ipynb index 82c133d813..f74f2eb0ab 100644 --- a/tutorial/responses.ipynb +++ b/tutorial/responses.ipynb @@ -49,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "metadata": {}, "outputs": [ { From 92c4635270450b7d8144e8a842bb8c640292c1a5 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 19:25:57 +0530 Subject: [PATCH 4/6] ignore all ipynb from codespell --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa09dfadbc..7812dc555d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -127,7 +127,7 @@ repos: - id: codespell exclude: > (?x)^( - tutorial/responses.ipynb| + tutorial/*.ipynb| tests/http/test_responses\.py| ^.+\.min\.js$ )$ From fa2f41c23ead9287079f0bb609c48aa9c01700fc Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 19:27:56 +0530 Subject: [PATCH 5/6] ignore all ipynb from codespell --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7812dc555d..7b7eb48b51 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -127,7 +127,7 @@ repos: - id: codespell exclude: > (?x)^( - tutorial/*.ipynb| + ^.+\.ipynb$| tests/http/test_responses\.py| ^.+\.min\.js$ )$ From 03c58e8eba29ecee348aa09dfe24b9ab57fc6c96 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 19:41:23 +0530 Subject: [PATCH 6/6] Fix tests and doc spell --- proxy/plugin/__init__.py | 2 ++ proxy/plugin/web_server_route.py | 13 ++++++++++--- tests/http/parser/test_http_parser.py | 5 ++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index e76a253468..f75dcbbfc1 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -11,6 +11,8 @@ .. spelling:: Cloudflare + ws + onmessage """ from .cache import CacheResponsesPlugin, BaseCacheResponsesPlugin from .shortlink import ShortLinkPlugin diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index 3139b0e648..3b927cd55d 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + ws + onmessage """ import logging from typing import List, Tuple @@ -42,9 +47,11 @@ def handle_request(self, request: HttpParser) -> None: def on_websocket_message(self, frame: WebsocketFrame) -> None: """Open chrome devtools and try using following commands: - ws = new WebSocket("ws://localhost:8899/ws-route-example") - ws.onmessage = function(m) { console.log(m); } - ws.send('hello') + Example: + + ws = new WebSocket("ws://localhost:8899/ws-route-example") + ws.onmessage = function(m) { console.log(m); } + ws.send('hello') """ self.client.queue(memoryview(WebsocketFrame.text(frame.data or b''))) diff --git a/tests/http/parser/test_http_parser.py b/tests/http/parser/test_http_parser.py index 95c052dec4..427553c310 100644 --- a/tests/http/parser/test_http_parser.py +++ b/tests/http/parser/test_http_parser.py @@ -124,7 +124,7 @@ def test_valid_ipv6_in_request_line(self) -> None: def test_build_request(self) -> None: self.assertEqual( build_http_request( - b'GET', b'http://localhost:12345', b'HTTP/1.1', + b'GET', b'http://localhost:12345', b'HTTP/1.1', no_ua=True, ), CRLF.join([ b'GET http://localhost:12345 HTTP/1.1', @@ -135,6 +135,7 @@ def test_build_request(self) -> None: build_http_request( b'GET', b'http://localhost:12345', b'HTTP/1.1', headers={b'key': b'value'}, + no_ua=True, ), CRLF.join([ b'GET http://localhost:12345 HTTP/1.1', @@ -147,10 +148,12 @@ def test_build_request(self) -> None: b'GET', b'http://localhost:12345', b'HTTP/1.1', headers={b'key': b'value'}, body=b'Hello from py', + no_ua=True, ), CRLF.join([ b'GET http://localhost:12345 HTTP/1.1', b'key: value', + b'Content-Length: 13', CRLF, ]) + b'Hello from py', )