From ea0253d59aa0d32023a0633e804f84a1560787b2 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 14:57:09 +0530 Subject: [PATCH 1/4] Add a response generation jupyter notebook --- tests/http/test_responses.py | 109 ++++++++++++ tutorial/responses.ipynb | 328 +++++++++++++++++++++++++++++++++++ 2 files changed, 437 insertions(+) create mode 100644 tests/http/test_responses.py create mode 100644 tutorial/responses.ipynb diff --git a/tests/http/test_responses.py b/tests/http/test_responses.py new file mode 100644 index 0000000000..1ef9ee0214 --- /dev/null +++ b/tests/http/test_responses.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import gzip + +import unittest + +from proxy.http.parser import ChunkParser +from proxy.http.responses import okResponse +from proxy.common.constants import CRLF + + +class TestResponses(unittest.TestCase): + + def test_basic(self) -> None: + self.assertEqual( + okResponse(), + b'HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n', + ) + self.assertEqual( + okResponse( + headers={ + b'X-Custom-Header': b'my value', + }, + ), + b'HTTP/1.1 200 OK\r\nX-Custom-Header: my value\r\nContent-Length: 0\r\n\r\n', + ) + self.assertEqual( + okResponse( + content=b'Hello World', + headers={ + b'X-Custom-Header': b'my value', + }, + ), + b'HTTP/1.1 200 OK\r\nX-Custom-Header: my value\r\nContent-Length: 11\r\n\r\nHello World', + ) + + def test_compression(self) -> None: + content = b'H' * 21 + self.assertEqual( + gzip.decompress( + okResponse( + content=content, + headers={ + b'X-Custom-Header': b'my value', + }, + ).tobytes().split(CRLF + CRLF, maxsplit=1)[-1], + ), + content, + ) + self.assertEqual( + okResponse( + content=content, + headers={ + b'Host': b'jaxl.com', + }, + min_compression_length=len(content), + ), + b'HTTP/1.1 200 OK\r\nHost: jaxl.com\r\nContent-Length: 21\r\n\r\nHHHHHHHHHHHHHHHHHHHHH', + ) + + def test_close_header(self) -> None: + self.assertEqual( + okResponse( + content=b'Hello World', + headers={ + b'Host': b'jaxl.com', + }, + conn_close=True, + ), + b'HTTP/1.1 200 OK\r\nHost: jaxl.com\r\nContent-Length: 11\r\nConnection: close\r\n\r\nHello World' + ) + + def test_chunked_without_compression(self) -> None: + chunks = ChunkParser.to_chunks(b'Hello World', chunk_size=5) + self.assertEqual( + okResponse( + content=chunks, + headers={ + b'Transfer-Encoding': b'chunked', + }, + # Avoid compressing chunks for demo purposes here + # Ideally you should omit this flag and send + # compressed chunks. + min_compression_length=len(chunks), + ), + b'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nHello\r\n5\r\n Worl\r\n1\r\nd\r\n0\r\n\r\n', + ) + + def test_chunked_with_compression(self) -> None: + chunks = ChunkParser.to_chunks(b'Hello World', chunk_size=5) + self.assertEqual( + gzip.decompress( + okResponse( + content=chunks, + headers={ + b'Transfer-Encoding': b'chunked', + }, + ).tobytes().split(CRLF + CRLF, maxsplit=1)[-1], + ), + chunks, + ) diff --git a/tutorial/responses.ipynb b/tutorial/responses.ipynb new file mode 100644 index 0000000000..11a7a6c5f1 --- /dev/null +++ b/tutorial/responses.ipynb @@ -0,0 +1,328 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Http Response\n", + "\n", + "## Usage\n", + "\n", + "To construct a response packet you have a variety of facilities available. We previously experienced how to parse HTTP responses using `HttpParser`. Of-course, we can also construct a response packet using `HttpParser` class.\n", + "\n", + "Let's construct a HTTP response packet:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nContent-Length: 0\\r\\n\\r\\n'\n" + ] + } + ], + "source": [ + "from proxy.http.parser import HttpParser, httpParserTypes\n", + "from proxy.common.constants import HTTP_1_1\n", + "\n", + "response = HttpParser(httpParserTypes.RESPONSE_PARSER)\n", + "response.code = b'200'\n", + "response.reason = b'OK'\n", + "response.version = HTTP_1_1\n", + "\n", + "print(response.build_response())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But it is a little painful to construct responses like above. Hence, other high level abstractions are available.\n", + "\n", + "Example, following one liner will give us the same response packet." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nContent-Length: 0\\r\\n\\r\\n'\n" + ] + } + ], + "source": [ + "from proxy.http.responses import okResponse\n", + "\n", + "print(okResponse().tobytes())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice how `okResponse` will always add a `Content-Length` header for you.\n", + "\n", + "You can also customize other headers" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nX-Custom-Header: my value\\r\\nContent-Length: 0\\r\\n\\r\\n'\n" + ] + } + ], + "source": [ + "response = okResponse(\n", + " headers={\n", + " b'X-Custom-Header': b'my value',\n", + " },\n", + ")\n", + "\n", + "print(response.tobytes())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's add some content to our response packet" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nX-Custom-Header: my value\\r\\nContent-Length: 11\\r\\n\\r\\nHello World'\n" + ] + } + ], + "source": [ + "response = okResponse(\n", + " content=b'Hello World',\n", + " headers={\n", + " b'X-Custom-Header': b'my value',\n", + " },\n", + ")\n", + "\n", + "print(response.tobytes())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note, how `okResponse` automatically added a `Content-Length` header for us.\n", + "\n", + "Depending upon `--min-compression-length` flag, `okResponse` will also perform compression for content.\n", + "\n", + "Example, default value for min compression length is 20." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nX-Custom-Header: my value\\r\\nContent-Encoding: gzip\\r\\nContent-Length: 23\\r\\n\\r\\n\\x1f\\x8b\\x08\\x00F\\x0e\\xf1a\\x02\\xff\\xf3\\xf0\\xc0\\x02\\x00h\\x81?s\\x15\\x00\\x00\\x00'\n" + ] + } + ], + "source": [ + "response = okResponse(\n", + " content=b'H' * 21,\n", + " headers={\n", + " b'X-Custom-Header': b'my value',\n", + " },\n", + ")\n", + "\n", + "print(response.tobytes())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can pass a custom value for `min_compression_length` kwarg to `okResponse`." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nHost: jaxl.com\\r\\nContent-Length: 21\\r\\n\\r\\nHHHHHHHHHHHHHHHHHHHHH'\n" + ] + } + ], + "source": [ + "response = okResponse(\n", + " content=b'H' * 21,\n", + " headers={\n", + " b'Host': b'jaxl.com',\n", + " },\n", + " min_compression_length=21,\n", + ")\n", + "\n", + "print(response.tobytes())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Internally, `okResponse` uses `build_http_response` and hence you can also pass any argument also accepted by `build_http_response`. Example, it supports a `conn_close` argument which will add a `Connection: close` header. Simply, pass `conn_close=True`." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nHost: jaxl.com\\r\\nContent-Length: 11\\r\\nConnection: close\\r\\n\\r\\nHello World'\n" + ] + } + ], + "source": [ + "response = okResponse(\n", + " content=b'Hello World',\n", + " headers={\n", + " b'Host': b'jaxl.com',\n", + " },\n", + " conn_close=True,\n", + ")\n", + "\n", + "print(response.tobytes())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chunked Encoding\n", + "\n", + "You can also send chunked encoded responses." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nTransfer-Encoding: chunked\\r\\n\\r\\n5\\r\\nHello\\r\\n5\\r\\n Worl\\r\\n1\\r\\nd\\r\\n0\\r\\n\\r\\n'\n" + ] + } + ], + "source": [ + "from proxy.http.parser import ChunkParser\n", + "\n", + "chunks = ChunkParser.to_chunks(b'Hello World', chunk_size=5)\n", + "response = okResponse(\n", + " content=chunks,\n", + " headers={\n", + " b'Transfer-Encoding': b'chunked',\n", + " },\n", + " # Avoid compressing chunks for demo purposes here\n", + " # Ideally you should omit this flag and send\n", + " # compressed chunks.\n", + " min_compression_length=len(chunks),\n", + ")\n", + "\n", + "print(response.tobytes())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If we omit the `min_compression_length` flag" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'HTTP/1.1 200 OK\\r\\nTransfer-Encoding: chunked\\r\\nContent-Encoding: gzip\\r\\n\\r\\n\\x1f\\x8b\\x08\\x00\\xd3\\n\\xf1a\\x02\\xff3\\xe5\\xe5\\xf2H\\xcd\\xc9\\xc9\\xe7\\xe52\\xe5\\xe5R\\x08\\xcf/\\xca\\xe1\\xe52\\xe4\\xe5J\\xe1\\xe52\\xe0\\xe5\\xe2\\xe5\\x02\\x00\\x90S\\xbb/\\x1f\\x00\\x00\\x00'\n" + ] + } + ], + "source": [ + "response = okResponse(\n", + " content=chunks,\n", + " headers={\n", + " b'Transfer-Encoding': b'chunked',\n", + " },\n", + ")\n", + "\n", + "print(response.tobytes())" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "da9d6927d62b2b95bde149eedfbd5367cb7f465aad65a736f49c99ee3db39df7" + }, + "kernelspec": { + "display_name": "Python 3.10.0 64-bit ('venv310': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} From af155d655e500ebbcead2da7bf0a57e8e071abca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 09:28:52 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/http/test_responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/http/test_responses.py b/tests/http/test_responses.py index 1ef9ee0214..f03c8a30f9 100644 --- a/tests/http/test_responses.py +++ b/tests/http/test_responses.py @@ -75,7 +75,7 @@ def test_close_header(self) -> None: }, conn_close=True, ), - b'HTTP/1.1 200 OK\r\nHost: jaxl.com\r\nContent-Length: 11\r\nConnection: close\r\n\r\nHello World' + b'HTTP/1.1 200 OK\r\nHost: jaxl.com\r\nContent-Length: 11\r\nConnection: close\r\n\r\nHello World', ) def test_chunked_without_compression(self) -> None: From 32daf4f0541f7bb484ce77c43f398cc6fcd9654c Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 15:09:04 +0530 Subject: [PATCH 3/4] Make codespell happy --- tests/http/test_responses.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/http/test_responses.py b/tests/http/test_responses.py index 1ef9ee0214..7b414fae7e 100644 --- a/tests/http/test_responses.py +++ b/tests/http/test_responses.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + nd """ import gzip From 3e34862dbec3be68331fee8f283ef398a556a0ee Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 26 Jan 2022 15:18:11 +0530 Subject: [PATCH 4/4] precommit codespell exclude --- .pre-commit-config.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 21b2f9943f..aa09dfadbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -125,8 +125,12 @@ repos: rev: v2.1.0 hooks: - id: codespell - exclude: >- - ^.+\.min\.js$ + exclude: > + (?x)^( + tutorial/responses.ipynb| + tests/http/test_responses\.py| + ^.+\.min\.js$ + )$ - repo: https://github.com/adrienverge/yamllint.git rev: v1.26.2