[Jupyter] Request creation notebook

.pre-commit-config.yaml

@@ -127,7 +127,7 @@ repos:
   - id: codespell
     exclude: >
       (?x)^(
-        tutorial/responses.ipynb|
+        ^.+\.ipynb$|
         tests/http/test_responses\.py|
         ^.+\.min\.js$
       )$

README.md

@@ -2297,7 +2297,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--filtered-client-ips FILTERED_CLIENT_IPS]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
 
-proxy.py v2.4.0rc8.dev17+g59a4335.d20220123
+proxy.py v2.4.0rc9.dev8+gea0253d.d20220126
 
 options:
   -h, --help            show this help message and exit

proxy/common/utils.py

@@ -26,7 +26,7 @@
 from .types import HostPort
 from .constants import (
     CRLF, COLON, HTTP_1_1, IS_WINDOWS, WHITESPACE, DEFAULT_TIMEOUT,
-    DEFAULT_THREADLESS,
+    DEFAULT_THREADLESS, PROXY_AGENT_HEADER_VALUE,
 )
 
 
@@ -84,14 +84,30 @@ def bytes_(s: Any, encoding: str = 'utf-8', errors: str = 'strict') -> Any:
 def build_http_request(
     method: bytes, url: bytes,
     protocol_version: bytes = HTTP_1_1,
+    content_type: Optional[bytes] = None,
     headers: Optional[Dict[bytes, bytes]] = None,
     body: Optional[bytes] = None,
     conn_close: bool = False,
+    no_ua: bool = False,
 ) -> bytes:
     """Build and returns a HTTP request packet."""
+    headers = headers or {}
+    if content_type is not None:
+        headers[b'Content-Type'] = content_type
+    has_transfer_encoding = False
+    has_user_agent = False
+    for k, _ in headers.items():
+        if k.lower() == b'transfer-encoding':
+            has_transfer_encoding = True
+        elif k.lower() == b'user-agent':
+            has_user_agent = True
+    if body and not has_transfer_encoding:
+        headers[b'Content-Length'] = bytes_(len(body))
+    if not has_user_agent and not no_ua:
+        headers[b'User-Agent'] = PROXY_AGENT_HEADER_VALUE
     return build_http_pkt(
         [method, url, protocol_version],
-        headers or {},
+        headers,
         body,
         conn_close,
     )
@@ -109,8 +125,7 @@ def build_http_response(
     line = [protocol_version, bytes_(status_code)]
     if reason:
         line.append(reason)
-    if headers is None:
-        headers = {}
+    headers = headers or {}
     has_transfer_encoding = False
     for k, _ in headers.items():
         if k.lower() == b'transfer-encoding':

proxy/http/parser/parser.py

@@ -270,6 +270,7 @@ def build(self, disable_headers: Optional[List[bytes]] = None, for_proxy: bool =
                 k.lower() not in disable_headers
             },
             body=body,
+            no_ua=True,
         )
 
     def build_response(self) -> bytes:

proxy/plugin/__init__.py

@@ -11,6 +11,8 @@
     .. spelling::
 
        Cloudflare
+       ws
+       onmessage
 """
 from .cache import CacheResponsesPlugin, BaseCacheResponsesPlugin
 from .shortlink import ShortLinkPlugin

proxy/plugin/modify_post_data.py

@@ -12,7 +12,7 @@
 
 from ..http import httpMethods
 from ..http.proxy import HttpProxyBasePlugin
-from ..http.parser import HttpParser
+from ..http.parser import HttpParser, ChunkParser
 from ..common.utils import bytes_
 
 
@@ -30,14 +30,20 @@ def handle_client_request(
             self, request: HttpParser,
     ) -> Optional[HttpParser]:
         if request.method == httpMethods.POST:
-            request.body = ModifyPostDataPlugin.MODIFIED_BODY
-            # Update Content-Length header only when request is NOT chunked
-            # encoded
-            if not request.is_chunked_encoded:
+            # If request data is compressed, compress the body too
+            body = ModifyPostDataPlugin.MODIFIED_BODY
+            # If the request is of type chunked encoding
+            # add post data as chunk
+            if request.is_chunked_encoded:
+                body = ChunkParser.to_chunks(
+                    ModifyPostDataPlugin.MODIFIED_BODY,
+                )
+            else:
                 request.add_header(
                     b'Content-Length',
-                    bytes_(len(request.body)),
+                    bytes_(len(body)),
                 )
+            request.body = body
             # Enforce content-type json
             if request.has_header(b'Content-Type'):
                 request.del_header(b'Content-Type')

proxy/plugin/web_server_route.py

@@ -7,13 +7,19 @@
 
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
+
+    .. spelling::
+
+       ws
+       onmessage
 """
 import logging
 from typing import List, Tuple
 
 from ..http.parser import HttpParser
 from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes
 from ..http.responses import okResponse
+from ..http.websocket.frame import WebsocketFrame
 
 
 logger = logging.getLogger(__name__)
@@ -37,3 +43,15 @@ def handle_request(self, request: HttpParser) -> None:
             self.client.queue(HTTP_RESPONSE)
         elif request.path == b'/https-route-example':
             self.client.queue(HTTPS_RESPONSE)
+
+    def on_websocket_message(self, frame: WebsocketFrame) -> None:
+        """Open chrome devtools and try using following commands:
+
+        Example:
+
+            ws = new WebSocket("ws://localhost:8899/ws-route-example")
+            ws.onmessage = function(m) { console.log(m); }
+            ws.send('hello')
+
+        """
+        self.client.queue(memoryview(WebsocketFrame.text(frame.data or b'')))

tests/http/parser/test_http_parser.py

@@ -124,7 +124,7 @@ def test_valid_ipv6_in_request_line(self) -> None:
     def test_build_request(self) -> None:
         self.assertEqual(
             build_http_request(
-                b'GET', b'http://localhost:12345', b'HTTP/1.1',
+                b'GET', b'http://localhost:12345', b'HTTP/1.1', no_ua=True,
             ),
             CRLF.join([
                 b'GET http://localhost:12345 HTTP/1.1',
@@ -135,6 +135,7 @@ def test_build_request(self) -> None:
             build_http_request(
                 b'GET', b'http://localhost:12345', b'HTTP/1.1',
                 headers={b'key': b'value'},
+                no_ua=True,
             ),
             CRLF.join([
                 b'GET http://localhost:12345 HTTP/1.1',
@@ -147,10 +148,12 @@ def test_build_request(self) -> None:
                 b'GET', b'http://localhost:12345', b'HTTP/1.1',
                 headers={b'key': b'value'},
                 body=b'Hello from py',
+                no_ua=True,
             ),
             CRLF.join([
                 b'GET http://localhost:12345 HTTP/1.1',
                 b'key: value',
+                b'Content-Length: 13',
                 CRLF,
             ]) + b'Hello from py',
         )

tests/http/proxy/test_http_proxy_tls_interception.py

@@ -119,6 +119,7 @@ def mock_connection() -> Any:
         connect_request = build_http_request(
             httpMethods.CONNECT, bytes_(netloc),
             headers=headers,
+            no_ua=True,
         )
         self._conn.recv.return_value = connect_request
         get_request = build_http_request(