diff --git a/.github/chronographer.yml b/.github/chronographer.yml new file mode 100644 index 0000000000..24168796fd --- /dev/null +++ b/.github/chronographer.yml @@ -0,0 +1,11 @@ +--- +enforce_name: + suffix: .md +exclude: + bots: + - dependabot-preview + - dependabot + - patchback + humans: + - pyup-bot +... diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 5d416fd3be..58e93a8bfb 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -820,7 +820,7 @@ jobs: - name: >- Publish a GitHub Release for ${{ needs.pre-setup.outputs.git-tag }} - uses: ncipollo/release-action@v1.8.10 + uses: ncipollo/release-action@v1.9.0 with: allowUpdates: false artifactErrorsFailBuild: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db86982576..44c5dc6a96 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -176,6 +176,7 @@ repos: - --show-error-context - --strict - --strict-optional + - benchmark/ - examples/ - proxy/ - tests/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..e46ec51f00 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,31 @@ + +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + + + +## v2.x + +- No longer ~~a single file module~~. +- Added support for threadless execution. +- Added dashboard app. +- Added support for unit testing. + +## v1.x + +- `Python3` only. + - Deprecated support for ~~Python 2.x~~. +- Added support multi core accept. +- Added plugin support. + +## v0.x + +- Single file. +- Single threaded server. + +For detailed changelog refer to release PRs or commit history. diff --git a/Makefile b/Makefile index d54ab9242f..aa6e4b5757 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ endif .PHONY: all https-certificates sign-https-certificates ca-certificates .PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest .PHONY: lib-release-test lib-release lib-profile lib-doc -.PHONY: lib-dep lib-flake8 lib-mypy +.PHONY: lib-dep lib-flake8 lib-mypy lib-speedscope .PHONY: container container-run container-release container-build container-buildx .PHONY: devtools dashboard dashboard-clean @@ -138,9 +138,26 @@ lib-profile: -o profile.svg \ -t -F -s -- \ python -m proxy \ + --hostname 127.0.0.1 \ + --num-acceptors 1 \ + --num-workers 1 \ + --enable-web-server \ + --plugin proxy.plugin.WebServerPlugin \ + --local-executor \ + --backlog 65536 \ + --open-file-limit 65536 \ + --log-file /dev/null + +lib-speedscope: + ulimit -n 65536 && \ + sudo py-spy record \ + -o profile.speedscope.json \ + -f speedscope \ + -t -F -s -- \ + python -m proxy \ + --hostname 127.0.0.1 \ --num-acceptors 1 \ --num-workers 1 \ - --disable-http-proxy \ --enable-web-server \ --plugin proxy.plugin.WebServerPlugin \ --local-executor \ diff --git a/README.md b/README.md index e9ba838e9f..c0c0cc0990 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,10 @@ - [Sending a Pull Request](#sending-a-pull-request) - [Benchmarks](#benchmarks) - [Flags](#flags) -- [Changelog](#changelog) - - [v2.x](#v2x) - - [v1.x](#v1x) - - [v0.x](#v0x) +- [Changelog](https://proxypy.rtfd.io/en/latest/changelog) + - [v2.x](https://proxypy.rtfd.io/en/latest/changelog#v2x) + - [v1.x](https://proxypy.rtfd.io/en/latest/changelog#v1x) + - [v0.x](https://proxypy.rtfd.io/en/latest/changelog#v0x) [//]: # (DO-NOT-REMOVE-docs-badges-END) @@ -128,57 +128,55 @@ ```console # On Macbook Pro 2019 / 2.4 GHz 8-Core Intel Core i9 / 32 GB RAM ❯ ./helper/benchmark.sh - CONCURRENCY: 100 workers, TOTAL REQUESTS: 100000 req, QPS: 8000 req/sec, TIMEOUT: 1 sec + CONCURRENCY: 100 workers, TOTAL REQUESTS: 100000 req Summary: - Total: 3.1217 secs - Slowest: 0.0499 secs - Fastest: 0.0004 secs - Average: 0.0030 secs - Requests/sec: 32033.7261 + Success rate: 1.0000 + Total: 2.5489 secs + Slowest: 0.0443 secs + Fastest: 0.0006 secs + Average: 0.0025 secs + Requests/sec: 39232.6572 - Total data: 1900000 bytes - Size/request: 19 bytes + Total data: 1.81 MiB + Size/request: 19 B + Size/sec: 727.95 KiB Response time histogram: - 0.000 [1] | - 0.005 [92268] |■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ - 0.010 [7264] |■■■ - 0.015 [318] | - 0.020 [102] | - 0.025 [32] | - 0.030 [6] | - 0.035 [4] | - 0.040 [1] | - 0.045 [2] | - 0.050 [2] | - + 0.001 [5006] |■■■■■ + 0.001 [19740] |■■■■■■■■■■■■■■■■■■■■■ + 0.002 [29701] |■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ + 0.002 [21278] |■■■■■■■■■■■■■■■■■■■■■■ + 0.003 [15376] |■■■■■■■■■■■■■■■■ + 0.004 [6644] |■■■■■■■ + 0.004 [1609] |■ + 0.005 [434] | + 0.006 [83] | + 0.006 [29] | + 0.007 [100] | Latency distribution: - 10% in 0.0017 secs - 25% in 0.0020 secs - 50% in 0.0025 secs - 75% in 0.0036 secs - 90% in 0.0050 secs - 95% in 0.0060 secs - 99% in 0.0087 secs + 10% in 0.0014 secs + 25% in 0.0018 secs + 50% in 0.0023 secs + 75% in 0.0030 secs + 90% in 0.0036 secs + 95% in 0.0040 secs + 99% in 0.0047 secs Details (average, fastest, slowest): - DNS+dialup: 0.0000 secs, 0.0004 secs, 0.0499 secs - DNS-lookup: 0.0000 secs, 0.0000 secs, 0.0000 secs - req write: 0.0000 secs, 0.0000 secs, 0.0020 secs - resp wait: 0.0030 secs, 0.0004 secs, 0.0462 secs - resp read: 0.0000 secs, 0.0000 secs, 0.0027 secs + DNS+dialup: 0.0025 secs, 0.0015 secs, 0.0030 secs + DNS-lookup: 0.0000 secs, 0.0000 secs, 0.0001 secs Status code distribution: - [200] 100000 responses + [200] 100000 responses ``` - PS: `proxy.py` and benchmark tools are running on the same machine during the above load test. - Checkout the repo and try it for yourself. See [Benchmarks](#benchmarks) for more details. + - See [Benchmark](https://github.com/abhinavsingh/proxy.py/tree/develop/benchmark#readme) for more details and how to run them locally. - Lightweight - - Uses only `~5-20MB` RAM + - Uses `~5-20 MB` RAM + - Compressed containers size is `~18.04 MB` - No external dependency other than standard Python library - Programmable - Customize proxy behavior using [Proxy Server Plugins](#http-proxy-plugins). Example: @@ -1627,21 +1625,11 @@ optional arguments: ## Internal Documentation -Code is well documented. Browse through internal class hierarchy and documentation using `pydoc3` - -```console -❯ pydoc3 proxy - -PACKAGE CONTENTS - __main__ - common (package) - core (package) - http (package) - main +Code is well documented. You have a few options to browse the internal class hierarchy and documentation: -FILE - /Users/abhinav/Dev/proxy.py/proxy/__init__.py -``` +1. Visit [proxypy.readthedocs.io](https://proxypy.readthedocs.io/) +2. Build and open docs locally using `make lib-doc` +2. Use `pydoc3` locally using `pydoc3 proxy` # Run Dashboard @@ -2028,7 +2016,9 @@ for list of tests. # Benchmarks -Simply run the following command from repo root to start benchmark +See [Benchmark](https://github.com/abhinavsingh/proxy.py/tree/develop/benchmark) directory on how to run benchmark comparisons with other OSS web servers. + +To run standalone benchmark for `proxy.py`, use the following command from repo root: ```console ❯ ./helper/benchmark.sh @@ -2216,30 +2206,3 @@ options: Proxy.py not working? Report at: https://github.com/abhinavsingh/proxy.py/issues/new ``` - -# Changelog - -## v2.4.0 - -- No longer support `Python 3.6` due to `asyncio.run` usage in the core. - -## v2.x - -- No longer ~~a single file module~~. -- Added support for threadless execution. -- Added dashboard app. -- Added support for unit testing. - -## v1.x - -- `Python3` only. - - Deprecated support for ~~Python 2.x~~. -- Added support multi core accept. -- Added plugin support. - -## v0.x - -- Single file. -- Single threaded server. - -For detailed changelog refer to release PRs or commit history. diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000000..e22da05bd4 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,36 @@ +# Benchmark + +# Table of Contents +- [TL;DR](#tldr) +- [Usage](#usage) +- [Results](#results) + +## TL;DR + +NOTE: On Macbook Pro 2019 / 2.4 GHz 8-Core Intel Core i9 / 32 GB RAM + +| Server | Throughput (request/sec) | Num Workers | Runner | +| ------ | ------------ | ------------------------| ------ | +| `blacksheep` | 46,564 | 10 | uvicorn | +| `starlette` | 44,102 | 10 | uvicorn | +| `proxy.py` | 39,232 | 10 | - | +| `aiohttp` | 6,615 | 1 | - | +| `tornado` | 3,301 | 1 | - | + +- On a single core, `proxy.py` yields `~9449 req/sec` throughput. +- Try it using `--num-acceptors=1` + +## Usage + +```console +❯ git clone https://github.com/abhinavsingh/proxy.py.git +❯ cd proxy.py +❯ pip install -r benchmark/requirements.txt +❯ ./benchmark/compare.sh > /tmp/compare.log 2>&1 +``` + +## Results + +```console +❯ cat /tmp/compare.log +``` diff --git a/tests/benchmark/__init__.py b/benchmark/__init__.py similarity index 100% rename from tests/benchmark/__init__.py rename to benchmark/__init__.py diff --git a/benchmark/aiohttp/__init__.py b/benchmark/aiohttp/__init__.py new file mode 100644 index 0000000000..232621f0b5 --- /dev/null +++ b/benchmark/aiohttp/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" diff --git a/benchmark/aiohttp/server.py b/benchmark/aiohttp/server.py new file mode 100644 index 0000000000..9c9075b91a --- /dev/null +++ b/benchmark/aiohttp/server.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from aiohttp import web + + +async def handle(request: web.Request) -> web.StreamResponse: + return web.Response(body=b'HTTP route response') + + +app = web.Application() + +app.add_routes( + [ + web.get('/http-route-example', handle), + ], +) + +web.run_app(app, host='127.0.0.1', port=8080) diff --git a/benchmark/blacksheep/__init__.py b/benchmark/blacksheep/__init__.py new file mode 100644 index 0000000000..232621f0b5 --- /dev/null +++ b/benchmark/blacksheep/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" diff --git a/benchmark/blacksheep/server.py b/benchmark/blacksheep/server.py new file mode 100644 index 0000000000..cd966b4465 --- /dev/null +++ b/benchmark/blacksheep/server.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import uvicorn + +from blacksheep.server import Application +from blacksheep.server.responses import text + + +app = Application() + + +@app.route('/http-route-example') +async def home(request): # type: ignore[no-untyped-def] + return text('HTTP route response') + +if __name__ == '__main__': + uvicorn.run('server:app', port=9000, workers=10, log_level='warning') diff --git a/benchmark/compare.sh b/benchmark/compare.sh new file mode 100755 index 0000000000..c6444e28d7 --- /dev/null +++ b/benchmark/compare.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# proxy.py +# ~~~~~~~~ +# ⚡⚡⚡ Fast, Lightweight, Programmable, TLS interception capable +# proxy server for Application debugging, testing and development. +# +# :copyright: (c) 2013-present by Abhinav Singh and contributors. +# :license: BSD, see LICENSE for more details. +# +usage() { + echo "Usage: ./benchmark/compare.sh" + echo "You must run this script from proxy.py repo root." +} + +DIRNAME=$(dirname "$0") +if [ "$DIRNAME" != "./benchmark" ]; then + usage + exit 1 +fi + +BASENAME=$(basename "$0") +if [ "$BASENAME" != "compare.sh" ]; then + usage + exit 1 +fi + +PWD=$(pwd) +if [ $(basename $PWD) != "proxy.py" ]; then + usage + exit 1 +fi + +TIMEOUT=1sec +CONCURRENCY=100 +DURATION=1m +TOTAL_REQUESTS=100000 +OPEN_FILE_LIMIT=65536 +BACKLOG=OPEN_FILE_LIMIT + +SERVER_HOST=127.0.0.1 + +AIOHTTP_PORT=8080 +TORNADO_PORT=8888 +STARLETTE_PORT=8890 +PROXYPY_PORT=8899 +BLACKSHEEP_PORT=9000 + +ulimit -n $OPEN_FILE_LIMIT + +echo "CONCURRENCY: $CONCURRENCY workers, DURATION: $DURATION, TIMEOUT: $TIMEOUT" + +run_benchmark() { + oha \ + --no-tui \ + --latency-correction \ + -z $DURATION \ + -c $CONCURRENCY \ + -t $TIMEOUT \ + http://127.0.0.1:$1/http-route-example +} + +benchmark_lib() { + python ./benchmark/$1/server.py > /dev/null 2>&1 & + local SERVER_PID=$! + echo "Server (pid:$SERVER_PID) running" + sleep 1 + run_benchmark $2 + kill -15 $SERVER_PID + sleep 1 + kill -0 $SERVER_PID > /dev/null 2>&1 + local RUNNING=$? + if [ "$RUNNING" == "1" ]; then + echo "Server gracefully shutdown" + fi +} + +benchmark_proxy_py() { + python -m proxy \ + --hostname 127.0.0.1 \ + --port $1 \ + --backlog 65536 \ + --open-file-limit 65536 \ + --enable-web-server \ + --plugin proxy.plugin.WebServerPlugin \ + --disable-http-proxy \ + --num-acceptors 10 \ + --local-executor \ + --log-file /dev/null > /dev/null 2>&1 & + local SERVER_PID=$! + echo "Server (pid:$SERVER_PID) running" + sleep 1 + run_benchmark $1 + kill -15 $SERVER_PID + sleep 1 + kill -0 $SERVER_PID > /dev/null 2>&1 + local RUNNING=$? + if [ "$RUNNING" == "1" ]; then + echo "Server gracefully shutdown" + fi +} + +benchmark_asgi() { + uvicorn \ + --port $1 \ + --backlog 65536 \ + $2 > /dev/null 2>&1 & + local SERVER_PID=$! + echo "Server (pid:$SERVER_PID) running" + sleep 1 + run_benchmark $1 + kill -15 $SERVER_PID + sleep 1 + kill -0 $SERVER_PID > /dev/null 2>&1 + local RUNNING=$? + if [ "$RUNNING" == "1" ]; then + echo "Server gracefully shutdown" + fi +} + +# echo "=============================" +# echo "Benchmarking Proxy.Py" +# benchmark_proxy_py $PROXYPY_PORT +# echo "=============================" + +# echo "=============================" +# echo "Benchmarking Blacksheep" +# benchmark_lib blacksheep $BLACKSHEEP_PORT +# echo "=============================" + +# echo "=============================" +# echo "Benchmarking Starlette" +# benchmark_lib starlette $STARLETTE_PORT +# echo "=============================" + +# echo "=============================" +# echo "Benchmarking AIOHTTP" +# benchmark_lib aiohttp $AIOHTTP_PORT +# echo "=============================" + +# echo "=============================" +# echo "Benchmarking Tornado" +# benchmark_lib tornado $TORNADO_PORT +# echo "=============================" diff --git a/benchmark/requirements.txt b/benchmark/requirements.txt new file mode 100644 index 0000000000..1241908f99 --- /dev/null +++ b/benchmark/requirements.txt @@ -0,0 +1,5 @@ +aiohttp==3.8.1 +blacksheep==1.2.1 +starlette==0.17.1 +tornado==6.1 +uvicorn==0.15.0 diff --git a/benchmark/starlette/__init__.py b/benchmark/starlette/__init__.py new file mode 100644 index 0000000000..232621f0b5 --- /dev/null +++ b/benchmark/starlette/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" diff --git a/benchmark/starlette/server.py b/benchmark/starlette/server.py new file mode 100644 index 0000000000..0e27e023cb --- /dev/null +++ b/benchmark/starlette/server.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import uvicorn + +from starlette.applications import Starlette +from starlette.responses import Response +from starlette.routing import Route + + +async def homepage(request): # type: ignore[no-untyped-def] + return Response('HTTP route response', media_type='text/plain') + + +app = Starlette( + debug=True, routes=[ + Route('/http-route-example', homepage), + ], +) + + +if __name__ == '__main__': + uvicorn.run('server:app', port=8890, workers=10, log_level='warning') diff --git a/benchmark/tornado/__init__.py b/benchmark/tornado/__init__.py new file mode 100644 index 0000000000..232621f0b5 --- /dev/null +++ b/benchmark/tornado/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" diff --git a/benchmark/tornado/server.py b/benchmark/tornado/server.py new file mode 100644 index 0000000000..3af22abb60 --- /dev/null +++ b/benchmark/tornado/server.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import tornado.ioloop +import tornado.web + + +# pylint: disable=W0223 +class MainHandler(tornado.web.RequestHandler): # type: ignore[misc] + def get(self) -> None: + self.write('HTTP route response') + + +def make_app() -> tornado.web.Application: + return tornado.web.Application([ + (r'/http-route-example', MainHandler), + ]) + + +if __name__ == '__main__': + app = make_app() + app.listen(8888, address='127.0.0.1') + tornado.ioloop.IOLoop.current().start() diff --git a/check.py b/check.py index 038758a30f..af3b905bcb 100644 --- a/check.py +++ b/check.py @@ -36,6 +36,7 @@ list(REPO_ROOT.glob('*.py')) + list((REPO_ROOT / 'proxy').rglob('*.py')) + list((REPO_ROOT / 'examples').rglob('*.py')) + + list((REPO_ROOT / 'benchmark').rglob('*.py')) + list((REPO_ROOT / 'tests').rglob('*.py')) ) diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index a4a79a68e4..a49bff5d25 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -20,16 +20,16 @@ "eslint-plugin-import": "^2.25.3", "eslint-plugin-node": "^10.0.0", "eslint-plugin-promise": "^4.2.1", - "eslint-plugin-standard": "^4.1.0", + "eslint-plugin-standard": "^5.0.0", "http-server": "^0.12.3", "jasmine": "^3.6.3", "jasmine-ts": "^0.3.0", "jquery": "^3.5.1", - "js-cookie": "^2.2.1", + "js-cookie": "^3.0.1", "jsdom": "^15.2.1", "ncp": "^2.0.0", "rollup": "^1.32.1", - "rollup-plugin-copy": "^3.3.0", + "rollup-plugin-copy": "^3.4.0", "rollup-plugin-javascript-obfuscator": "^1.0.4", "rollup-plugin-typescript": "^1.0.1", "ts-node": "^7.0.1", @@ -1364,9 +1364,10 @@ } }, "node_modules/eslint-plugin-standard": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz", - "integrity": "sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-5.0.0.tgz", + "integrity": "sha512-eSIXPc9wBM4BrniMzJRBm2uoVuXz2EPa+NXPk2+itrVt+r5SbKFERx/IgrK/HmfjddyKVz2f+j+7gBRvu19xLg==", + "deprecated": "standard 16.0.0 and eslint-config-standard 16.0.0 no longer require the eslint-plugin-standard package. You can remove it from your dependencies with 'npm rm eslint-plugin-standard'. More info here: https://github.com/standard/standard/issues/1316", "dev": true, "funding": [ { @@ -3246,10 +3247,13 @@ "dev": true }, "node_modules/js-cookie": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz", - "integrity": "sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==", - "dev": true + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.1.tgz", + "integrity": "sha512-+0rgsUXZu4ncpPxRL+lNEptWMOWl9etvPHc/koSRp6MPwpRYAhmk0dUG00J4bxVV3r9uUzfo24wW0knS07SKSw==", + "dev": true, + "engines": { + "node": ">=12" + } }, "node_modules/js-string-escape": { "version": "1.0.1", @@ -4362,9 +4366,9 @@ } }, "node_modules/rollup-plugin-copy": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/rollup-plugin-copy/-/rollup-plugin-copy-3.3.0.tgz", - "integrity": "sha512-euDjCUSBXZa06nqnwCNADbkAcYDfzwowfZQkto9K/TFhiH+QG7I4PUsEMwM9tDgomGWJc//z7KLW8t+tZwxADA==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/rollup-plugin-copy/-/rollup-plugin-copy-3.4.0.tgz", + "integrity": "sha512-rGUmYYsYsceRJRqLVlE9FivJMxJ7X6jDlP79fmFkL8sJs7VVMSVyA2yfyL+PGyO/vJs4A87hwhgVfz61njI+uQ==", "dev": true, "dependencies": { "@types/fs-extra": "^8.0.1", @@ -6712,9 +6716,9 @@ "dev": true }, "eslint-plugin-standard": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz", - "integrity": "sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-5.0.0.tgz", + "integrity": "sha512-eSIXPc9wBM4BrniMzJRBm2uoVuXz2EPa+NXPk2+itrVt+r5SbKFERx/IgrK/HmfjddyKVz2f+j+7gBRvu19xLg==", "dev": true, "requires": {} }, @@ -7840,9 +7844,9 @@ "dev": true }, "js-cookie": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz", - "integrity": "sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.1.tgz", + "integrity": "sha512-+0rgsUXZu4ncpPxRL+lNEptWMOWl9etvPHc/koSRp6MPwpRYAhmk0dUG00J4bxVV3r9uUzfo24wW0knS07SKSw==", "dev": true }, "js-string-escape": { @@ -8705,9 +8709,9 @@ } }, "rollup-plugin-copy": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/rollup-plugin-copy/-/rollup-plugin-copy-3.3.0.tgz", - "integrity": "sha512-euDjCUSBXZa06nqnwCNADbkAcYDfzwowfZQkto9K/TFhiH+QG7I4PUsEMwM9tDgomGWJc//z7KLW8t+tZwxADA==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/rollup-plugin-copy/-/rollup-plugin-copy-3.4.0.tgz", + "integrity": "sha512-rGUmYYsYsceRJRqLVlE9FivJMxJ7X6jDlP79fmFkL8sJs7VVMSVyA2yfyL+PGyO/vJs4A87hwhgVfz61njI+uQ==", "dev": true, "requires": { "@types/fs-extra": "^8.0.1", diff --git a/dashboard/package.json b/dashboard/package.json index 295ebd2963..ab87ca18a3 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -36,16 +36,16 @@ "eslint-plugin-import": "^2.25.3", "eslint-plugin-node": "^10.0.0", "eslint-plugin-promise": "^4.2.1", - "eslint-plugin-standard": "^4.1.0", + "eslint-plugin-standard": "^5.0.0", "http-server": "^0.12.3", "jasmine": "^3.6.3", "jasmine-ts": "^0.3.0", "jquery": "^3.5.1", - "js-cookie": "^2.2.1", + "js-cookie": "^3.0.1", "jsdom": "^15.2.1", "ncp": "^2.0.0", "rollup": "^1.32.1", - "rollup-plugin-copy": "^3.3.0", + "rollup-plugin-copy": "^3.4.0", "rollup-plugin-javascript-obfuscator": "^1.0.4", "rollup-plugin-typescript": "^1.0.1", "ts-node": "^7.0.1", diff --git a/docs/changelog-fragments.d/.CHANGELOG-TEMPLATE.md.j2 b/docs/changelog-fragments.d/.CHANGELOG-TEMPLATE.md.j2 new file mode 100644 index 0000000000..b231e9b214 --- /dev/null +++ b/docs/changelog-fragments.d/.CHANGELOG-TEMPLATE.md.j2 @@ -0,0 +1,31 @@ + +{% for section, _ in sections.items() %} +{% set title_prefix = underlines[0] %} +{% if section %} +{{ title_prefix }} {{ section }} +{% set title_prefix = underlines[1] %} +{% endif %} + +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section] %} +{{ title_prefix }} {{ definitions[category]['name'] }} +{% if definitions[category]['showcontent'] %} + +{% for text, values in sections[section][category].items() %} +* {{ text }} + ({{ values|join(',\n ') }}) +{% endfor -%} + +{% else %} +* {{ sections[section][category]['']|join(', ') }} +{% endif %} + +{% if sections[section][category]|length == 0 %} +No significant changes. +{% endif %} +{% endfor %} + +{% else %} +No significant changes. +{% endif %} +{% endfor %} diff --git a/docs/changelog-fragments.d/.gitignore b/docs/changelog-fragments.d/.gitignore new file mode 100644 index 0000000000..46978d2256 --- /dev/null +++ b/docs/changelog-fragments.d/.gitignore @@ -0,0 +1,25 @@ +* +!.CHANGELOG-TEMPLATE.md.j2 +!.gitignore +!README.md +!*.bugfix +!*.bugfix.md +!*.bugfix.*.md +!*.breaking +!*.breaking.md +!*.breaking.*.md +!*.deprecation +!*.deprecation.md +!*.deprecation.*.md +!*.doc +!*.doc.md +!*.doc.*.md +!*.feature +!*.feature.md +!*.feature.*.md +!*.internal +!*.internal.md +!*.internal.*.md +!*.misc +!*.misc.md +!*.misc.*.md diff --git a/docs/changelog-fragments.d/823.misc.md b/docs/changelog-fragments.d/823.misc.md new file mode 100644 index 0000000000..d4652c91fa --- /dev/null +++ b/docs/changelog-fragments.d/823.misc.md @@ -0,0 +1,4 @@ +Added changelog fragment management infrastructure using [Towncrier] +-- by {user}`webknjaz` + +[Towncrier]: https://github.com/twisted/towncrier diff --git a/docs/changelog-fragments.d/README.md b/docs/changelog-fragments.d/README.md new file mode 100644 index 0000000000..c94ab340cf --- /dev/null +++ b/docs/changelog-fragments.d/README.md @@ -0,0 +1,76 @@ +# Adding change notes with your PRs + +It is very important to maintain a log for news of how +updating to the new version of the software will affect +end-users. This is why we enforce collection of the change +fragment files in pull requests as per [Towncrier philosophy]. + +The idea is that when somebody makes a change, they must record +the bits that would affect end-users only including information +that would be useful to them. Then, when the maintainers publish +a new release, they'll automatically use these records to compose +a change log for the respective version. It is important to +understand that including unnecessary low-level implementation +related details generates noise that is not particularly useful +to the end-users most of the time. And so such details should be +recorded in the Git history rather than a changelog. + +# Alright! So how do I add a news fragment? + +To submit a change note about your PR, add a text file into the +`docs/changelog-fragments.d/` folder. It should contain an +explanation of what applying this PR will change in the way +end-users interact with the project. One sentence is usually +enough but feel free to add as many details as you feel necessary +for the users to understand what it means. + +**Use the past tense** for the text in your fragment because, +combined with others, it will be a part of the "news digest" +telling the readers **what changed** in a specific version of +the library *since the previous version*. You should also use +[MyST Markdown] syntax for highlighting code (inline or block), +linking parts of the docs or external sites. +At the end, sign your change note by adding ```-- by +{user}`github-username``` (replace `github-username` with +your own!). + +Finally, name your file following the convention that Towncrier +understands: it should start with the number of an issue or a +PR followed by a dot, then add a patch type, like `feature`, +`bugfix`, `doc`, `misc` etc., and add `.md` as a suffix. If you +need to add more than one fragment, you may add an optional +sequence number (delimited with another period) between the type +and the suffix. + +# Examples for changelog entries adding to your Pull Requests + +File `docs/changelog-fragments.d/112.doc.md`: + +```md +Added a `{user}` role to Sphinx config -- by {user}`webknjaz` +``` + +File `docs/changelog-fragments.d/105.feature.md`: + +```md +Added the support for keyboard-authentication method +-- by {user}`Qalthos` +``` + +File `docs/changelog-fragments.d/57.bugfix.md`: + +```md +Fixed flaky SEGFAULTs in `pylibsshext.channel.Channel.exec_command()` +calls -- by {user}`ganeshrn` +``` + +```{tip} +See `pyproject.toml` for all available categories +(`tool.towncrier.type`). +``` + + +[MyST Markdown]: +https://myst-parser.rtfd.io/en/latest/syntax/syntax.html +[Towncrier philosophy]: +https://towncrier.rtfd.io/en/actual-freaking-docs/#philosophy diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000000..975d25a5eb --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,6 @@ +```{spelling} +Changelog +``` + +```{include} ../CHANGELOG.md +``` diff --git a/docs/conf.py b/docs/conf.py index 7391c7911f..7900e48be6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -118,6 +118,13 @@ # Usually you set "language" from the command line for these cases. language = 'en' +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [ + 'changelog-fragments.d/**', # Towncrier-managed change notes +] + # -- Options for HTML output ------------------------------------------------- diff --git a/docs/contributing/guidelines.md b/docs/contributing/guidelines.md index c12c38338c..4caa0c3d3c 100644 --- a/docs/contributing/guidelines.md +++ b/docs/contributing/guidelines.md @@ -3,6 +3,7 @@ de facto Pre reStructuredText +Towncrier ``` ```{include} ../../CONTRIBUTING.md @@ -32,3 +33,6 @@ the docs ` to learn more on how to use it. [MyST parser]: https://pypi.org/project/myst-parser/ [Read The Docs]: https://readthedocs.org [Sphinx]: https://www.sphinx-doc.org + +```{include} ../changelog-fragments.d/README.md +``` diff --git a/docs/index.md b/docs/index.md index aefb3dc7c6..2a72e170bf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,6 +28,7 @@ Scalable ```{toctree} :hidden: +changelog Glossary ``` diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 5562179d58..7c61e32077 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1,13 +1,27 @@ IPv Nginx Pluggable -scm +Submodules +Subpackages Threadless -threadless -youtube -socio -sexualized +WebSocket +Websocket +acceptor +acceptors +eventing +faq +html +http https +iterable +readables +scm +sexualized +socio +tcp +teardown +threadless +websocket +writables www -html -faq +youtube diff --git a/examples/README.md b/examples/README.md index 90455dc4f4..39e0426cfe 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,7 +6,6 @@ Looking for `proxy.py` plugin examples? Check [proxy/plugin](https://github.com Table of Contents ================= -* [Generic Work Acceptor and Executor](#generic-work-acceptor-and-executor) * [WebSocket Client](#websocket-client) * [TCP Echo Server](#tcp-echo-server) * [TCP Echo Client](#tcp-echo-client) @@ -14,17 +13,7 @@ Table of Contents * [SSL Echo Client](#ssl-echo-client) * [PubSub Eventing](#pubsub-eventing) * [Https Connect Tunnel](#https-connect-tunnel) - -## Generic Work Acceptor and Executor - -1. Makes use of `proxy.core.AcceptorPool` and `proxy.core.Work` -2. Demonstrates how to perform generic work using `proxy.py` core. - -Start `web_scraper.py` as: - -```console -❯ PYTHONPATH=. python examples/web_scraper.py -``` +* [Generic Work Acceptor and Executor](#generic-work-acceptor-and-executor) ## WebSocket Client @@ -148,3 +137,14 @@ Send https requests via tunnel as: ``` ❯ curl -x localhost:12345 https://httpbin.org/get ``` + +## Generic Work Acceptor and Executor + +1. Makes use of `proxy.core.AcceptorPool` and `proxy.core.Work` +2. Demonstrates how to perform generic work using `proxy.py` core. + +Start `web_scraper.py` as: + +```console +❯ PYTHONPATH=. python examples/web_scraper.py +``` diff --git a/examples/https_connect_tunnel.py b/examples/https_connect_tunnel.py index e60edaeda0..b138b7bdce 100644 --- a/examples/https_connect_tunnel.py +++ b/examples/https_connect_tunnel.py @@ -50,7 +50,7 @@ def handle_data(self, data: memoryview) -> Optional[bool]: self.request.parse(data) # Drop the request if not a CONNECT request - if not self.request.is_https_tunnel(): + if not self.request.is_https_tunnel: self.work.queue( HttpsConnectTunnelHandler.PROXY_TUNNEL_UNSUPPORTED_SCHEME, ) diff --git a/helper/benchmark.sh b/helper/benchmark.sh index aaa62321f0..ca0cc43c7d 100755 --- a/helper/benchmark.sh +++ b/helper/benchmark.sh @@ -31,8 +31,7 @@ if [ $(basename $PWD) != "proxy.py" ]; then exit 1 fi -TIMEOUT=1 -QPS=8000 +TIMEOUT=1sec CONCURRENCY=100 TOTAL_REQUESTS=100000 OPEN_FILE_LIMIT=65536 @@ -50,13 +49,18 @@ ADDR=$(lsof -Pan -p $PID -i | grep -v COMMAND | awk '{ print $9 }') PRE_RUN_OPEN_FILES=$(./helper/monitor_open_files.sh) -echo "CONCURRENCY: $CONCURRENCY workers, TOTAL REQUESTS: $TOTAL_REQUESTS req, QPS: $QPS req/sec, TIMEOUT: $TIMEOUT sec" -hey \ +run_benchmark() { + echo "CONCURRENCY: $CONCURRENCY workers, TOTAL REQUESTS: $TOTAL_REQUESTS req" + oha \ + --no-tui \ + --latency-correction \ -n $TOTAL_REQUESTS \ -c $CONCURRENCY \ - -q $QPS \ -t $TIMEOUT \ http://$ADDR/http-route-example +} + +run_benchmark POST_RUN_OPEN_FILES=$(./helper/monitor_open_files.sh) diff --git a/proxy/__init__.py b/proxy/__init__.py index 94b6d7203d..a2e0fa77ad 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -7,12 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - eventing - Submodules - Subpackages """ from .proxy import entry_point, main, Proxy from .testing import TestCase diff --git a/proxy/common/__init__.py b/proxy/common/__init__.py index 02bacd6ce5..232621f0b5 100644 --- a/proxy/common/__init__.py +++ b/proxy/common/__init__.py @@ -7,8 +7,4 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - Submodules """ diff --git a/proxy/common/backports.py b/proxy/common/backports.py index b6f5dfd927..9aec16df4c 100644 --- a/proxy/common/backports.py +++ b/proxy/common/backports.py @@ -38,8 +38,7 @@ def randint(self): two-element tuple with the last computed property value and the last time it was updated in seconds since the epoch. - The default time-to-live (TTL) is 300 seconds (5 minutes). Set the TTL to - zero for the cached value to never expire. + The default time-to-live (TTL) is 0 seconds i.e. cached value will never expire. To expire a cached property value manually just do:: del instance._cached_properties[] @@ -58,7 +57,7 @@ def randint(self): Arndt """ - def __init__(self, ttl: float = 300.0): + def __init__(self, ttl: float = 0): self.ttl = ttl def __call__(self, fget: Any, doc: Any = None) -> 'cached_property': diff --git a/proxy/common/constants.py b/proxy/common/constants.py index c941676f0d..b2e4f6d71f 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -111,6 +111,7 @@ def _env_threadless_compliant() -> bool: # 25 milliseconds to keep the loops hot # Will consume ~0.3-0.6% CPU when idle. DEFAULT_SELECTOR_SELECT_TIMEOUT = 25 / 1000 +DEFAULT_WAIT_FOR_TASKS_TIMEOUT = 1 / 1000 DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT = 1 # in seconds DEFAULT_DEVTOOLS_DOC_URL = 'http://proxy' diff --git a/proxy/common/utils.py b/proxy/common/utils.py index 673b73d1ba..f76c2faaa6 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -11,9 +11,6 @@ .. spelling:: utils - websocket - Websocket - WebSocket """ import sys import ssl diff --git a/proxy/core/acceptor/__init__.py b/proxy/core/acceptor/__init__.py index 577e2022f5..bd39723594 100644 --- a/proxy/core/acceptor/__init__.py +++ b/proxy/core/acceptor/__init__.py @@ -10,10 +10,7 @@ .. spelling:: - acceptor - acceptors pre - Submodules """ from .acceptor import Acceptor from .pool import AcceptorPool diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index 05cfa09c26..0304769a1c 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -10,7 +10,6 @@ .. spelling:: - acceptor pre """ import socket @@ -75,6 +74,7 @@ def __init__( fd_queue: connection.Connection, flags: argparse.Namespace, lock: multiprocessing.synchronize.Lock, + # semaphore: multiprocessing.synchronize.Semaphore, executor_queues: List[connection.Connection], executor_pids: List[int], executor_locks: List[multiprocessing.synchronize.Lock], @@ -88,6 +88,7 @@ def __init__( self.idd = idd # Mutex used for synchronization with acceptors self.lock = lock + # self.semaphore = semaphore # Queue over which server socket fd is received on start-up self.fd_queue: connection.Connection = fd_queue # Available executors @@ -106,36 +107,54 @@ def __init__( self._local: Optional[LocalExecutor] = None self._lthread: Optional[threading.Thread] = None - def accept(self, events: List[Tuple[selectors.SelectorKey, int]]) -> None: + def accept( + self, + events: List[Tuple[selectors.SelectorKey, int]], + ) -> List[Tuple[socket.socket, Optional[Tuple[str, int]]]]: + works = [] for _, mask in events: if mask & selectors.EVENT_READ: if self.sock is not None: - conn, addr = self.sock.accept() - logging.debug( - 'Accepting new work#{0}'.format(conn.fileno()), - ) - work = (conn, addr or None) - if self.flags.local_executor: - assert self._local_work_queue - self._local_work_queue.put(work) - else: - self._work(*work) + try: + conn, addr = self.sock.accept() + logging.debug( + 'Accepting new work#{0}'.format(conn.fileno()), + ) + works.append((conn, addr or None)) + except BlockingIOError: + # logger.info('blocking io error') + pass + return works def run_once(self) -> None: if self.selector is not None: events = self.selector.select(timeout=1) if len(events) == 0: return - locked = False + # locked = False + # try: + # if self.lock.acquire(block=False): + # locked = True + # self.semaphore.release() + # finally: + # if locked: + # self.lock.release() + locked, works = False, [] try: + # if not self.semaphore.acquire(False, None): + # return if self.lock.acquire(block=False): locked = True - self.accept(events) - except BlockingIOError: - pass + works = self.accept(events) finally: if locked: self.lock.release() + for work in works: + if self.flags.local_executor: + assert self._local_work_queue + self._local_work_queue.put(work) + else: + self._work(*work) def run(self) -> None: Logger.setup( diff --git a/proxy/core/acceptor/executors.py b/proxy/core/acceptor/executors.py index 065e78bded..ec3a29ace7 100644 --- a/proxy/core/acceptor/executors.py +++ b/proxy/core/acceptor/executors.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - acceptor """ import socket import logging diff --git a/proxy/core/acceptor/listener.py b/proxy/core/acceptor/listener.py index bef4b4461f..ac3b211ff5 100644 --- a/proxy/core/acceptor/listener.py +++ b/proxy/core/acceptor/listener.py @@ -108,6 +108,7 @@ def _listen_server_port(self) -> None: self._socket = socket.socket(self.flags.family, socket.SOCK_STREAM) self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # self._socket.setsockopt(socket.SOL_TCP, socket.TCP_FASTOPEN, 5) self._socket.bind((str(self.flags.hostname), self.flags.port)) self._socket.listen(self.flags.backlog) self._socket.setblocking(False) diff --git a/proxy/core/acceptor/local.py b/proxy/core/acceptor/local.py index bb4909815f..95c2c118c2 100644 --- a/proxy/core/acceptor/local.py +++ b/proxy/core/acceptor/local.py @@ -7,11 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - acceptor - teardown """ import queue import logging @@ -38,7 +33,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: @property def loop(self) -> Optional[asyncio.AbstractEventLoop]: if self._loop is None: - self._loop = asyncio.new_event_loop() + self._loop = asyncio.get_event_loop_policy().new_event_loop() return self._loop def work_queue_fileno(self) -> Optional[int]: diff --git a/proxy/core/acceptor/pool.py b/proxy/core/acceptor/pool.py index dd110d3285..90e7c77560 100644 --- a/proxy/core/acceptor/pool.py +++ b/proxy/core/acceptor/pool.py @@ -86,6 +86,7 @@ def __init__( self.fd_queues: List[connection.Connection] = [] # Internals self.lock = multiprocessing.Lock() + # self.semaphore = multiprocessing.Semaphore(0) def __enter__(self) -> 'AcceptorPool': self.setup() @@ -125,6 +126,7 @@ def _start(self) -> None: fd_queue=work_queue[1], flags=self.flags, lock=self.lock, + # semaphore=self.semaphore, event_queue=self.event_queue, executor_queues=self.executor_queues, executor_pids=self.executor_pids, diff --git a/proxy/core/acceptor/remote.py b/proxy/core/acceptor/remote.py index 76f8877d21..8fe4ed453a 100644 --- a/proxy/core/acceptor/remote.py +++ b/proxy/core/acceptor/remote.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - acceptor """ import asyncio import logging diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py index cc9292c2e6..6df3dde1ef 100644 --- a/proxy/core/acceptor/threadless.py +++ b/proxy/core/acceptor/threadless.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - acceptor """ import os import ssl @@ -27,6 +23,7 @@ from ...common.logger import Logger from ...common.types import Readables, Writables from ...common.constants import DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT +from ...common.constants import DEFAULT_WAIT_FOR_TASKS_TIMEOUT from ..connection import TcpClientConnection from ..event import eventNames, EventQueue @@ -85,7 +82,7 @@ def __init__( # fileno, mask Dict[int, int], ] = {} - self.wait_timeout: float = DEFAULT_SELECTOR_SELECT_TIMEOUT + self.wait_timeout: float = DEFAULT_WAIT_FOR_TASKS_TIMEOUT self.cleanup_inactive_timeout: float = DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT @property @@ -121,13 +118,18 @@ def work_on_tcp_conn( addr: Optional[Tuple[str, int]] = None, conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None, ) -> None: + conn = conn or socket.fromfd( + fileno, family=socket.AF_INET if self.flags.hostname.version == 4 else socket.AF_INET6, + type=socket.SOCK_STREAM, + ) self.works[fileno] = self.flags.work_klass( TcpClientConnection( - conn=conn or self._fromfd(fileno), + conn=conn, addr=addr, ), flags=self.flags, event_queue=self.event_queue, + uid=fileno, ) self.works[fileno].publish_event( event_name=eventNames.WORK_STARTED, @@ -143,57 +145,86 @@ def work_on_tcp_conn( ) self._cleanup(fileno) - async def _selected_events(self) -> Tuple[ - Dict[int, Tuple[Readables, Writables]], - bool, - ]: - """For each work, collects events they are interested in. - Calls select for events of interest. """ + async def _update_work_events(self, work_id: int) -> None: assert self.selector is not None - for work_id in self.works: - worker_events = await self.works[work_id].get_events() - # NOTE: Current assumption is that multiple works will not - # be interested in the same fd. Descriptors of interests - # returned by work must be unique. - # - # TODO: Ideally we must diff and unregister socks not - # returned of interest within this _select_events call - # but exists in registered_socks_by_work_ids - for fileno in worker_events: - if work_id not in self.registered_events_by_work_ids: - self.registered_events_by_work_ids[work_id] = {} - mask = worker_events[fileno] - if fileno in self.registered_events_by_work_ids[work_id]: - oldmask = self.registered_events_by_work_ids[work_id][fileno] - if mask != oldmask: - self.selector.modify( - fileno, events=mask, - data=work_id, - ) - self.registered_events_by_work_ids[work_id][fileno] = mask - logger.debug( - 'fd#{0} modified for mask#{1} by work#{2}'.format( - fileno, mask, work_id, - ), - ) - else: - # Can throw ValueError: Invalid file descriptor: -1 - # - # A guard within Work classes may not help here due to - # asynchronous nature. Hence, threadless will handle - # ValueError exceptions raised by selector.register - # for invalid fd. - self.selector.register( + worker_events = await self.works[work_id].get_events() + # NOTE: Current assumption is that multiple works will not + # be interested in the same fd. Descriptors of interests + # returned by work must be unique. + # + # TODO: Ideally we must diff and unregister socks not + # returned of interest within current _select_events call + # but exists in the registered_socks_by_work_ids registry. + for fileno in worker_events: + if work_id not in self.registered_events_by_work_ids: + self.registered_events_by_work_ids[work_id] = {} + mask = worker_events[fileno] + if fileno in self.registered_events_by_work_ids[work_id]: + oldmask = self.registered_events_by_work_ids[work_id][fileno] + if mask != oldmask: + self.selector.modify( fileno, events=mask, data=work_id, ) self.registered_events_by_work_ids[work_id][fileno] = mask - logger.debug( - 'fd#{0} registered for mask#{1} by work#{2}'.format( - fileno, mask, work_id, - ), - ) - selected = self.selector.select( + # logger.debug( + # 'fd#{0} modified for mask#{1} by work#{2}'.format( + # fileno, mask, work_id, + # ), + # ) + # else: + # logger.info( + # 'fd#{0} by work#{1} not modified'.format(fileno, work_id)) + else: + # Can throw ValueError: Invalid file descriptor: -1 + # + # A guard within Work classes may not help here due to + # asynchronous nature. Hence, threadless will handle + # ValueError exceptions raised by selector.register + # for invalid fd. + self.selector.register( + fileno, events=mask, + data=work_id, + ) + self.registered_events_by_work_ids[work_id][fileno] = mask + # logger.debug( + # 'fd#{0} registered for mask#{1} by work#{2}'.format( + # fileno, mask, work_id, + # ), + # ) + + async def _update_selector(self) -> None: + assert self.selector is not None + unfinished_work_ids = set() + for task in self.unfinished: + unfinished_work_ids.add(task._work_id) # type: ignore + for work_id in self.works: + # We don't want to invoke work objects which haven't + # yet finished their previous task + if work_id in unfinished_work_ids: + continue + await self._update_work_events(work_id) + + async def _selected_events(self) -> Tuple[ + Dict[int, Tuple[Readables, Writables]], + bool, + ]: + """For each work, collects events that they are interested in. + Calls select for events of interest. + + Returns a 2-tuple containing a dictionary and boolean. + Dictionary keys are work IDs and values are 2-tuple + containing ready readables & writables. + + Returned boolean value indicates whether there is + a newly accepted work waiting to be received and + queued for processing. This is only applicable when + :class:`~proxy.core.acceptor.threadless.Threadless.work_queue_fileno` + returns a valid fd. + """ + assert self.selector is not None + await self._update_selector() + events = self.selector.select( timeout=DEFAULT_SELECTOR_SELECT_TIMEOUT, ) # Keys are work_id and values are 2-tuple indicating @@ -203,9 +234,11 @@ async def _selected_events(self) -> Tuple[ new_work_available = False wqfileno = self.work_queue_fileno() if wqfileno is None: + # When ``work_queue_fileno`` returns None, + # always return True for the boolean value. new_work_available = True - for key, mask in selected: - if wqfileno is not None and key.fileobj == wqfileno: + for key, mask in events: + if not new_work_available and wqfileno is not None and key.fileobj == wqfileno: assert mask & selectors.EVENT_READ new_work_available = True continue @@ -217,22 +250,13 @@ async def _selected_events(self) -> Tuple[ work_by_ids[key.data][1].append(key.fileobj) return (work_by_ids, new_work_available) - async def _wait_for_tasks(self) -> None: + async def _wait_for_tasks(self) -> Set['asyncio.Task[bool]']: finished, self.unfinished = await asyncio.wait( self.unfinished, timeout=self.wait_timeout, return_when=asyncio.FIRST_COMPLETED, ) - for task in finished: - if task.result(): - self._cleanup(task._work_id) # type: ignore - # self.cleanup(int(task.get_name())) - - def _fromfd(self, fileno: int) -> socket.socket: - return socket.fromfd( - fileno, family=socket.AF_INET if self.flags.hostname.version == 4 else socket.AF_INET6, - type=socket.SOCK_STREAM, - ) + return finished # noqa: WPS331 def _cleanup_inactive(self) -> None: inactive_works: List[int] = [] @@ -292,7 +316,19 @@ async def _run_once(self) -> bool: # Invoke Threadless.handle_events self.unfinished.update(self._create_tasks(work_by_ids)) # logger.debug('Executing {0} works'.format(len(self.unfinished))) - await self._wait_for_tasks() + # Cleanup finished tasks + for task in await self._wait_for_tasks(): + # Checking for result can raise exception e.g. + # CancelledError, InvalidStateError or an exception + # from underlying task e.g. TimeoutError. + teardown = False + work_id = task._work_id # type: ignore + try: + teardown = task.result() + finally: + if teardown: + self._cleanup(work_id) + # self.cleanup(int(task.get_name())) # logger.debug( # 'Done executing works, {0} pending, {1} registered'.format( # len(self.unfinished), len(self.registered_events_by_work_ids), @@ -306,8 +342,10 @@ async def _run_forever(self) -> None: while True: if await self._run_once(): break - # Check for inactive and shutdown signal only second - if (tick * DEFAULT_SELECTOR_SELECT_TIMEOUT) > self.cleanup_inactive_timeout: + # Check for inactive and shutdown signal + elapsed = tick * \ + (DEFAULT_SELECTOR_SELECT_TIMEOUT + self.wait_timeout) + if elapsed >= self.cleanup_inactive_timeout: self._cleanup_inactive() if self.running.is_set(): break diff --git a/proxy/core/base/__init__.py b/proxy/core/base/__init__.py index 8a307776d0..bc86952b7c 100644 --- a/proxy/core/base/__init__.py +++ b/proxy/core/base/__init__.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - Submodules """ from .tcp_server import BaseTcpServerHandler from .tcp_tunnel import BaseTcpTunnelHandler diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index 3c17ce8e31..61abb59dc2 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - tcp """ import logging import selectors diff --git a/proxy/core/base/tcp_upstream.py b/proxy/core/base/tcp_upstream.py index 1de1e4a910..1fa342fcac 100644 --- a/proxy/core/base/tcp_upstream.py +++ b/proxy/core/base/tcp_upstream.py @@ -28,10 +28,6 @@ class TcpUpstreamConnectionHandler(ABC): Then, directly use ``self.upstream`` object within your class. See :class:`~proxy.plugin.proxy_pool.ProxyPoolPlugin` for example usage. - - .. spelling:: - - tcp """ def __init__(self, *args: Any, **kwargs: Any) -> None: diff --git a/proxy/core/connection/connection.py b/proxy/core/connection/connection.py index 69036755aa..84c6c9de74 100644 --- a/proxy/core/connection/connection.py +++ b/proxy/core/connection/connection.py @@ -75,7 +75,7 @@ def close(self) -> bool: return self.closed def has_buffer(self) -> bool: - return self._num_buffer > 0 + return self._num_buffer != 0 def queue(self, mv: memoryview) -> None: self.buffer.append(mv) diff --git a/proxy/core/connection/types.py b/proxy/core/connection/types.py index 663c0ae4d4..44522c81b8 100644 --- a/proxy/core/connection/types.py +++ b/proxy/core/connection/types.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - iterable """ from typing import NamedTuple diff --git a/proxy/core/event/__init__.py b/proxy/core/event/__init__.py index 08b6f5be49..17e1074e6e 100644 --- a/proxy/core/event/__init__.py +++ b/proxy/core/event/__init__.py @@ -7,12 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - eventing - iterable - Submodules """ from .queue import EventQueue from .names import EventNames, eventNames diff --git a/proxy/core/event/names.py b/proxy/core/event/names.py index 9a58926c6c..5040c880a8 100644 --- a/proxy/core/event/names.py +++ b/proxy/core/event/names.py @@ -7,11 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - eventing - iterable """ from typing import NamedTuple diff --git a/proxy/dashboard/__init__.py b/proxy/dashboard/__init__.py index 61f5ec0232..0f5d329522 100644 --- a/proxy/dashboard/__init__.py +++ b/proxy/dashboard/__init__.py @@ -7,12 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - Submodules - websocket - Websocket """ from .dashboard import ProxyDashboard from .inspect_traffic import InspectTrafficPlugin diff --git a/proxy/dashboard/inspect_traffic.py b/proxy/dashboard/inspect_traffic.py index 15edc0d687..33fa212091 100644 --- a/proxy/dashboard/inspect_traffic.py +++ b/proxy/dashboard/inspect_traffic.py @@ -7,11 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - websocket - Websocket """ import json from typing import List, Dict, Any diff --git a/proxy/http/__init__.py b/proxy/http/__init__.py index b1d6877d15..f8d2e4fa4f 100644 --- a/proxy/http/__init__.py +++ b/proxy/http/__init__.py @@ -7,12 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http - Subpackages - Submodules """ from .handler import HttpProtocolHandler from .plugin import HttpProtocolHandlerPlugin diff --git a/proxy/http/exception/__init__.py b/proxy/http/exception/__init__.py index bb0bf7b1e8..513d2bd510 100644 --- a/proxy/http/exception/__init__.py +++ b/proxy/http/exception/__init__.py @@ -7,11 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http - Submodules """ from .base import HttpProtocolException from .http_request_rejected import HttpRequestRejected diff --git a/proxy/http/exception/http_request_rejected.py b/proxy/http/exception/http_request_rejected.py index d17571bd26..a0fa810fc1 100644 --- a/proxy/http/exception/http_request_rejected.py +++ b/proxy/http/exception/http_request_rejected.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http """ from typing import Optional, Dict diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py index 86d13cc880..cfba1d26dd 100644 --- a/proxy/http/exception/proxy_conn_failed.py +++ b/proxy/http/exception/proxy_conn_failed.py @@ -11,7 +11,6 @@ .. spelling:: conn - http """ from .base import HttpProtocolException diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 91c96624ca..6c158e2d22 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http """ import ssl import time @@ -117,6 +113,7 @@ def is_inactive(self) -> bool: def shutdown(self) -> None: try: # Flush pending buffer in threaded mode only. + # # For threadless mode, BaseTcpServerHandler implements # the must_flush_before_shutdown logic automagically. if self.selector and self.work.has_buffer(): @@ -139,6 +136,15 @@ def shutdown(self) -> None: except OSError: pass finally: + # Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending readable data + # could lead to an immediate reset being sent. + # + # "A host MAY implement a 'half-duplex' TCP close sequence, so that an application + # that has called CLOSE cannot continue to read data from the connection. + # If such a host issues a CLOSE call while received data is still pending in TCP, + # or if new data is received after CLOSE is called, its TCP SHOULD send a RST to + # show that data was lost." + # self.work.connection.close() logger.debug('Client connection closed') super().shutdown() @@ -189,19 +195,12 @@ async def handle_events( return False def handle_data(self, data: memoryview) -> Optional[bool]: + """Handles incoming data from client.""" if data is None: logger.debug('Client closed connection, tearing down...') self.work.closed = True return True - try: - # HttpProtocolHandlerPlugin.on_client_data - # Can raise HttpProtocolException to tear down the connection - for plugin in self.plugins.values(): - optional_data = plugin.on_client_data(data) - if optional_data is None: - break - data = optional_data # Don't parse incoming data any further after 1st request has completed. # # This specially does happen for pipeline requests. @@ -209,8 +208,9 @@ def handle_data(self, data: memoryview) -> Optional[bool]: # Plugins can utilize on_client_data for such cases and # apply custom logic to handle request data sent after 1st # valid request. - if data and self.request.state != httpParserStates.COMPLETE: + if self.request.state != httpParserStates.COMPLETE: # Parse http request + # # TODO(abhinavsingh): Remove .tobytes after parser is # memoryview compliant self.request.parse(data.tobytes()) @@ -228,6 +228,14 @@ def handle_data(self, data: memoryview) -> Optional[bool]: plugin_.client._conn = upgraded_sock elif isinstance(upgraded_sock, bool) and upgraded_sock is True: return True + else: + # HttpProtocolHandlerPlugin.on_client_data + # Can raise HttpProtocolException to tear down the connection + for plugin in self.plugins.values(): + optional_data = plugin.on_client_data(data) + if optional_data is None: + break + data = optional_data except HttpProtocolException as e: logger.debug('HttpProtocolException raised') response: Optional[memoryview] = e.response(self.request) diff --git a/proxy/http/inspector/devtools.py b/proxy/http/inspector/devtools.py index ee2a4d8e67..5b16571bf7 100644 --- a/proxy/http/inspector/devtools.py +++ b/proxy/http/inspector/devtools.py @@ -11,7 +11,6 @@ .. spelling:: devtools - http """ import json import logging diff --git a/proxy/http/inspector/transformer.py b/proxy/http/inspector/transformer.py index 97fec409b3..da7e1d613e 100644 --- a/proxy/http/inspector/transformer.py +++ b/proxy/http/inspector/transformer.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http """ import json import time diff --git a/proxy/http/parser/chunk.py b/proxy/http/parser/chunk.py index fa60d437df..2d976c4449 100644 --- a/proxy/http/parser/chunk.py +++ b/proxy/http/parser/chunk.py @@ -7,11 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http - iterable """ from typing import NamedTuple, Tuple, List, Optional diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index cff81abd1a..3554cf2bfe 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -15,7 +15,7 @@ from typing import TypeVar, Optional, Dict, Type, Tuple, List from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, DEFAULT_ENABLE_PROXY_PROTOCOL -from ...common.constants import HTTP_1_1, HTTP_1_0, SLASH, CRLF +from ...common.constants import HTTP_1_1, SLASH, CRLF from ...common.constants import WHITESPACE, DEFAULT_HTTP_PORT from ...common.utils import build_http_request, build_http_response, find_http_line, text_ from ...common.flag import flags @@ -83,6 +83,10 @@ def __init__( self.chunk: Optional[ChunkParser] = None # Internal request line as a url structure self._url: Optional[Url] = None + # Deduced states from the packet + self._is_chunked_encoded: bool = False + self._content_expected: bool = False + self._is_https_tunnel: bool = False @classmethod def request( @@ -113,9 +117,13 @@ def has_header(self, key: bytes) -> bool: """Returns true if header key was found in payload.""" return key.lower() in self.headers - def add_header(self, key: bytes, value: bytes) -> None: - """Add/Update a header to internal data structure.""" - self.headers[key.lower()] = (key, value) + def add_header(self, key: bytes, value: bytes) -> bytes: + """Add/Update a header to internal data structure. + + Returns key with which passed (key, value) tuple is available.""" + k = key.lower() + self.headers[k] = (key, value) + return k def add_headers(self, headers: List[Tuple[bytes, bytes]]) -> None: """Add/Update multiple headers to internal data structure""" @@ -143,6 +151,7 @@ def has_host(self) -> bool: NOTE: Host field WILL be None for incoming local WebServer requests.""" return self.host is not None + @property def is_http_1_1_keep_alive(self) -> bool: """Returns true for HTTP/1.1 keep-alive connections.""" return self.version == HTTP_1_1 and \ @@ -151,28 +160,32 @@ def is_http_1_1_keep_alive(self) -> bool: self.header(b'Connection').lower() == b'keep-alive' ) + @property def is_connection_upgrade(self) -> bool: """Returns true for websocket upgrade requests.""" return self.version == HTTP_1_1 and \ self.has_header(b'Connection') and \ self.has_header(b'Upgrade') + @property def is_https_tunnel(self) -> bool: """Returns true for HTTPS CONNECT tunnel request.""" - return self.method == httpMethods.CONNECT + return self._is_https_tunnel + @property def is_chunked_encoded(self) -> bool: """Returns true if transfer-encoding chunked is used.""" - return b'transfer-encoding' in self.headers and \ - self.headers[b'transfer-encoding'][1].lower() == b'chunked' + return self._is_chunked_encoded + @property def content_expected(self) -> bool: """Returns true if content-length is present and not 0.""" - return b'content-length' in self.headers and int(self.header(b'content-length')) > 0 + return self._content_expected + @property def body_expected(self) -> bool: """Returns true if content or chunked response is expected.""" - return self.content_expected() or self.is_chunked_encoded() + return self.content_expected or self.is_chunked_encoded def parse(self, raw: bytes) -> None: """Parses HTTP request out of raw bytes. @@ -186,6 +199,18 @@ def parse(self, raw: bytes) -> None: more, raw = self._process_body(raw) \ if self.state >= httpParserStates.HEADERS_COMPLETE else \ self._process_line_and_headers(raw) + # When server sends a response line without any header or body e.g. + # HTTP/1.1 200 Connection established\r\n\r\n + if self.state == httpParserStates.LINE_RCVD and \ + raw == CRLF and \ + self.type == httpParserTypes.RESPONSE_PARSER: + self.state = httpParserStates.COMPLETE + # Mark request as complete if headers received and no incoming + # body indication received. + elif self.state == httpParserStates.HEADERS_COMPLETE and \ + not self.body_expected and \ + raw == b'': + self.state = httpParserStates.COMPLETE self.buffer = raw def build(self, disable_headers: Optional[List[bytes]] = None, for_proxy: bool = False) -> bytes: @@ -204,7 +229,7 @@ def build(self, disable_headers: Optional[List[bytes]] = None, for_proxy: bool = COLON + str(self.port).encode() + path - ) if not self.is_https_tunnel() else (self.host + COLON + str(self.port).encode()) + ) if not self.is_https_tunnel else (self.host + COLON + str(self.port).encode()) return build_http_request( self.method, path, self.version, headers={} if not self.headers else { @@ -238,7 +263,7 @@ def _process_body(self, raw: bytes) -> Tuple[bool, bytes]: # the latter MUST be ignored. # # TL;DR -- Give transfer-encoding header preference over content-length. - if self.is_chunked_encoded(): + if self.is_chunked_encoded: if not self.chunk: self.chunk = ChunkParser() raw = self.chunk.parse(raw) @@ -246,7 +271,7 @@ def _process_body(self, raw: bytes) -> Tuple[bool, bytes]: self.body = self.chunk.body self.state = httpParserStates.COMPLETE more = False - elif b'content-length' in self.headers: + elif self.content_expected: self.state = httpParserStates.RCVING_BODY if self.body is None: self.body = b'' @@ -258,48 +283,52 @@ def _process_body(self, raw: bytes) -> Tuple[bool, bytes]: self.state = httpParserStates.COMPLETE more, raw = len(raw) > 0, raw[total_size - received_size:] else: - # HTTP/1.0 scenario only - assert self.version == HTTP_1_0 self.state = httpParserStates.RCVING_BODY # Received a packet without content-length header # and no transfer-encoding specified. # + # This can happen for both HTTP/1.0 and HTTP/1.1 scenarios. + # Currently, we consume the remaining buffer as body. + # # Ref https://github.com/abhinavsingh/proxy.py/issues/398 + # # See TestHttpParser.test_issue_398 scenario self.body = raw more, raw = False, b'' return more, raw def _process_line_and_headers(self, raw: bytes) -> Tuple[bool, bytes]: - """Returns False when no CRLF could be found in received bytes.""" - line, raw = find_http_line(raw) - if line is None: - return False, raw + """Returns False when no CRLF could be found in received bytes. + + TODO: We should not return until parser reaches headers complete + state or when there is no more data left to parse. + + TODO: For protection against Slowloris attack, we must parse the + request line and headers only after receiving end of header marker. + This will also help make the parser even more stateless. + """ + while True: + line, raw = find_http_line(raw) + if line is None: + return False, raw - if self.state == httpParserStates.INITIALIZED: - self._process_line(line) if self.state == httpParserStates.INITIALIZED: - return len(raw) > 0, raw - elif self.state in (httpParserStates.LINE_RCVD, httpParserStates.RCVING_HEADERS): - if self.state == httpParserStates.LINE_RCVD: - # LINE_RCVD state is equivalent to RCVING_HEADERS - self.state = httpParserStates.RCVING_HEADERS - if line.strip() == b'': # Blank line received. - self.state = httpParserStates.HEADERS_COMPLETE - else: - self._process_header(line) - - # When server sends a response line without any header or body e.g. - # HTTP/1.1 200 Connection established\r\n\r\n - if self.state == httpParserStates.LINE_RCVD and \ - self.type == httpParserTypes.RESPONSE_PARSER and \ - raw == CRLF: - self.state = httpParserStates.COMPLETE - elif self.state == httpParserStates.HEADERS_COMPLETE and \ - not self.body_expected() and \ - raw == b'': - self.state = httpParserStates.COMPLETE + self._process_line(line) + if self.state == httpParserStates.INITIALIZED: + # return len(raw) > 0, raw + continue + elif self.state in (httpParserStates.LINE_RCVD, httpParserStates.RCVING_HEADERS): + if self.state == httpParserStates.LINE_RCVD: + self.state = httpParserStates.RCVING_HEADERS + if line == b'' or line.strip() == b'': # Blank line received. + self.state = httpParserStates.HEADERS_COMPLETE + else: + self._process_header(line) + # If raw length is now zero, bail out + # If we have received all headers, bail out + if raw == b'' or self.state == httpParserStates.HEADERS_COMPLETE: + break return len(raw) > 0, raw def _process_line(self, raw: bytes) -> None: @@ -310,9 +339,11 @@ def _process_line(self, raw: bytes) -> None: self.protocol.parse(raw) else: # Ref: https://datatracker.ietf.org/doc/html/rfc2616#section-5.1 - line = raw.split(WHITESPACE) + line = raw.split(WHITESPACE, 2) if len(line) == 3: self.method = line[0].upper() + if self.method == httpMethods.CONNECT: + self._is_https_tunnel = True self.set_url(line[1]) self.version = line[2] self.state = httpParserStates.LINE_RCVD @@ -324,26 +355,37 @@ def _process_line(self, raw: bytes) -> None: # but we should solve circular import problem first. raise ValueError('Invalid request line') else: - line = raw.split(WHITESPACE) + line = raw.split(WHITESPACE, 2) self.version = line[0] self.code = line[1] - self.reason = WHITESPACE.join(line[2:]) + # Our own WebServerPlugin example currently doesn't send any reason + if len(line) == 3: + self.reason = line[2] self.state = httpParserStates.LINE_RCVD def _process_header(self, raw: bytes) -> None: - parts = raw.split(COLON) - key = parts[0].strip() - value = COLON.join(parts[1:]).strip() - self.add_headers([(key, value)]) + parts = raw.split(COLON, 1) + key, value = ( + parts[0].strip(), + b'' if len(parts) == 1 else parts[1].strip(), + ) + k = self.add_header(key, value) + # b'content-length' in self.headers and int(self.header(b'content-length')) > 0 + if k == b'content-length' and int(value) > 0: + self._content_expected = True + # return b'transfer-encoding' in self.headers and \ + # self.headers[b'transfer-encoding'][1].lower() == b'chunked' + elif k == b'transfer-encoding' and value.lower() == b'chunked': + self._is_chunked_encoded = True def _get_body_or_chunks(self) -> Optional[bytes]: return ChunkParser.to_chunks(self.body) \ - if self.body and self.is_chunked_encoded() else \ + if self.body and self.is_chunked_encoded else \ self.body def _set_line_attributes(self) -> None: if self.type == httpParserTypes.REQUEST_PARSER: - if self.is_https_tunnel() and self._url: + if self.is_https_tunnel and self._url: self.host = self._url.hostname self.port = 443 if self._url.port is None else self._url.port elif self._url: diff --git a/proxy/http/parser/protocol.py b/proxy/http/parser/protocol.py index 84982d1022..d749fdc08a 100644 --- a/proxy/http/parser/protocol.py +++ b/proxy/http/parser/protocol.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http """ from typing import Optional, Tuple diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py index d5510b5c2b..ceac20661a 100644 --- a/proxy/http/plugin.py +++ b/proxy/http/plugin.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http """ import socket import argparse @@ -94,6 +90,7 @@ async def read_from_descriptors(self, r: Readables) -> bool: @abstractmethod def on_client_data(self, raw: memoryview) -> Optional[memoryview]: + """Called only after original request has been completely received.""" return raw # pragma: no cover @abstractmethod diff --git a/proxy/http/proxy/__init__.py b/proxy/http/proxy/__init__.py index 69568bd96f..4e18002c0d 100644 --- a/proxy/http/proxy/__init__.py +++ b/proxy/http/proxy/__init__.py @@ -7,11 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http - Submodules """ from .plugin import HttpProxyBasePlugin from .server import HttpProxyPlugin diff --git a/proxy/http/proxy/plugin.py b/proxy/http/proxy/plugin.py index 81392d1d63..7d223038d5 100644 --- a/proxy/http/proxy/plugin.py +++ b/proxy/http/proxy/plugin.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http """ import argparse @@ -121,6 +117,8 @@ def handle_client_data( Essentially, if you return None from within before_upstream_connection, be prepared to handle_client_data and not handle_client_request. + Only called after initial request from client has been received. + Raise HttpRequestRejected to tear down the connection Return None to drop the connection """ diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index c9f01d1639..7a7622bd12 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -307,7 +307,7 @@ async def read_from_descriptors(self, r: Readables) -> bool: # parse incoming response packet # only for non-https requests and when # tls interception is enabled - if not self.request.is_https_tunnel(): + if not self.request.is_https_tunnel: # See https://github.com/abhinavsingh/proxy.py/issues/127 for why # currently response parsing is disabled when TLS interception is enabled. # @@ -425,7 +425,7 @@ def on_client_connection_close(self) -> None: def access_log(self, log_attrs: Dict[str, Any]) -> None: access_log_format = DEFAULT_HTTPS_ACCESS_LOG_FORMAT - if not self.request.is_https_tunnel(): + if not self.request.is_https_tunnel: access_log_format = DEFAULT_HTTP_ACCESS_LOG_FORMAT logger.info(access_log_format.format_map(log_attrs)) @@ -435,7 +435,7 @@ def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]: # However, this must also be accompanied by resetting both request # and response objects. # - # if not self.request.is_https_tunnel() and \ + # if not self.request.is_https_tunnel and \ # self.response.state == httpParserStates.COMPLETE: # self.access_log() return chunk @@ -466,11 +466,11 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: # We also handle pipeline scenario for https proxy # requests is TLS interception is enabled. if self.request.state == httpParserStates.COMPLETE and ( - not self.request.is_https_tunnel() or + not self.request.is_https_tunnel or self.tls_interception_enabled() ): if self.pipeline_request is not None and \ - self.pipeline_request.is_connection_upgrade(): + self.pipeline_request.is_connection_upgrade: # Previous pipelined request was a WebSocket # upgrade request. Incoming client data now # must be treated as WebSocket protocol packets. @@ -503,7 +503,7 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: self.pipeline_request.build(), ), ) - if not self.pipeline_request.is_connection_upgrade(): + if not self.pipeline_request.is_connection_upgrade: self.pipeline_request = None # For scenarios where we cannot peek into the data, # simply queue for upstream server. @@ -549,7 +549,7 @@ def on_request_complete(self) -> Union[socket.socket, bool]: # For https requests, respond back with tunnel established response. # Optionally, setup interceptor if TLS interception is enabled. if self.upstream: - if self.request.is_https_tunnel(): + if self.request.is_https_tunnel: self.client.queue( HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, ) @@ -903,14 +903,13 @@ def wrap_client(self) -> bool: def emit_request_complete(self) -> None: if not self.flags.enable_events: return - assert self.request.port self.event_queue.publish( request_id=self.uid.hex, event_name=eventNames.REQUEST_COMPLETE, event_payload={ 'url': text_(self.request.path) - if self.request.is_https_tunnel() + if self.request.is_https_tunnel else 'http://%s:%d%s' % (text_(self.request.host), self.request.port, text_(self.request.path)), 'method': text_(self.request.method), 'headers': {text_(k): text_(v[1]) for k, v in self.request.headers.items()}, @@ -924,7 +923,6 @@ def emit_request_complete(self) -> None: def emit_response_events(self, chunk_size: int) -> None: if not self.flags.enable_events: return - if self.response.state == httpParserStates.COMPLETE: self.emit_response_complete() elif self.response.state == httpParserStates.RCVING_BODY: @@ -935,7 +933,6 @@ def emit_response_events(self, chunk_size: int) -> None: def emit_response_headers_complete(self) -> None: if not self.flags.enable_events: return - self.event_queue.publish( request_id=self.uid.hex, event_name=eventNames.RESPONSE_HEADERS_COMPLETE, @@ -948,7 +945,6 @@ def emit_response_headers_complete(self) -> None: def emit_response_chunk_received(self, chunk_size: int) -> None: if not self.flags.enable_events: return - self.event_queue.publish( request_id=self.uid.hex, event_name=eventNames.RESPONSE_CHUNK_RECEIVED, @@ -962,7 +958,6 @@ def emit_response_chunk_received(self, chunk_size: int) -> None: def emit_response_complete(self) -> None: if not self.flags.enable_events: return - self.event_queue.publish( request_id=self.uid.hex, event_name=eventNames.RESPONSE_COMPLETE, diff --git a/proxy/http/server/__init__.py b/proxy/http/server/__init__.py index f100ddaf31..059c2cc128 100644 --- a/proxy/http/server/__init__.py +++ b/proxy/http/server/__init__.py @@ -7,11 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http - Submodules """ from .web import HttpWebServerPlugin from .pac_plugin import HttpWebServerPacFilePlugin diff --git a/proxy/http/server/pac_plugin.py b/proxy/http/server/pac_plugin.py index 68aad02cc8..f6858cf452 100644 --- a/proxy/http/server/pac_plugin.py +++ b/proxy/http/server/pac_plugin.py @@ -10,7 +10,6 @@ .. spelling:: - http pac """ import gzip diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index c9a8fe2da0..26807b41ac 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http """ import argparse diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index 129c088ffb..eacee97024 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http """ import re import gzip @@ -25,7 +21,6 @@ from ...common.constants import DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_ENABLE_WEB_SERVER from ...common.constants import DEFAULT_MIN_COMPRESSION_LIMIT, DEFAULT_WEB_ACCESS_LOG_FORMAT from ...common.utils import bytes_, text_, build_http_response, build_websocket_handshake_response -from ...common.backports import cached_property from ...common.types import Readables, Writables from ...common.flag import flags @@ -114,27 +109,28 @@ def __init__( self.route: Optional[HttpWebServerBasePlugin] = None self.plugins: Dict[str, HttpWebServerBasePlugin] = {} - if b'HttpWebServerBasePlugin' in self.flags.plugins: - for klass in self.flags.plugins[b'HttpWebServerBasePlugin']: - instance: HttpWebServerBasePlugin = klass( - self.uid, - self.flags, - self.client, - self.event_queue, - ) - self.plugins[instance.name()] = instance - - @cached_property(ttl=0) - def routes(self) -> Dict[int, Dict[Pattern[str], HttpWebServerBasePlugin]]: - r: Dict[int, Dict[Pattern[str], HttpWebServerBasePlugin]] = { + self.routes: Dict[ + int, Dict[Pattern[str], HttpWebServerBasePlugin], + ] = { httpProtocolTypes.HTTP: {}, httpProtocolTypes.HTTPS: {}, httpProtocolTypes.WEBSOCKET: {}, } - for name in self.plugins: - for (protocol, route) in self.plugins[name].routes(): - r[protocol][re.compile(route)] = self.plugins[name] - return r + if b'HttpWebServerBasePlugin' in self.flags.plugins: + self._initialize_web_plugins() + + def _initialize_web_plugins(self) -> None: + for klass in self.flags.plugins[b'HttpWebServerBasePlugin']: + instance: HttpWebServerBasePlugin = klass( + self.uid, + self.flags, + self.client, + self.event_queue, + ) + self.plugins[instance.name()] = instance + for (protocol, route) in instance.routes(): + pattern = re.compile(route) + self.routes[protocol][pattern] = self.plugins[instance.name()] def encryption_enabled(self) -> bool: return self.flags.keyfile is not None and \ @@ -223,7 +219,7 @@ def on_request_complete(self) -> Union[socket.socket, bool]: break # No-route found, try static serving if enabled if self.flags.enable_static_server: - path = text_(path).split('?')[0] + path = text_(path).split('?', 1)[0] self.client.queue( self.read_and_build_static_file_response( self.flags.static_server_dir + path, @@ -279,7 +275,7 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: # If 1st valid request was completed and it's a HTTP/1.1 keep-alive # And only if we have a route, parse pipeline requests if self.request.state == httpParserStates.COMPLETE and \ - self.request.is_http_1_1_keep_alive() and \ + self.request.is_http_1_1_keep_alive and \ self.route is not None: if self.pipeline_request is None: self.pipeline_request = HttpParser( @@ -290,7 +286,7 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: self.pipeline_request.parse(raw.tobytes()) if self.pipeline_request.state == httpParserStates.COMPLETE: self.route.handle_request(self.pipeline_request) - if not self.pipeline_request.is_http_1_1_keep_alive(): + if not self.pipeline_request.is_http_1_1_keep_alive: logger.error( 'Pipelined request is not keep-alive, will tear down request...', ) diff --git a/proxy/http/url.py b/proxy/http/url.py index 2d50743a71..177e3823ca 100644 --- a/proxy/http/url.py +++ b/proxy/http/url.py @@ -73,14 +73,14 @@ def from_bytes(cls, raw: bytes) -> 'Url': rest = raw[len(b'https://'):] \ if is_https \ else raw[len(b'http://'):] - parts = rest.split(SLASH) + parts = rest.split(SLASH, 1) host, port = Url.parse_host_and_port(parts[0]) return cls( scheme=b'https' if is_https else b'http', hostname=host, port=port, remainder=None if len(parts) == 1 else ( - SLASH + SLASH.join(parts[1:]) + SLASH + parts[1] ), ) host, port = Url.parse_host_and_port(raw) @@ -105,6 +105,8 @@ def parse_host_and_port(raw: bytes) -> Tuple[bytes, Optional[int]]: host, port = raw, None # patch up invalid ipv6 scenario rhost = host.decode('utf-8') - if COLON.decode('utf-8') in rhost and rhost[0] != '[' and rhost[-1] != ']': + if COLON.decode('utf-8') in rhost and \ + rhost[0] != '[' and \ + rhost[-1] != ']': host = b'[' + host + b']' return host, port diff --git a/proxy/http/websocket/client.py b/proxy/http/websocket/client.py index f4e25573bb..018866d2a0 100644 --- a/proxy/http/websocket/client.py +++ b/proxy/http/websocket/client.py @@ -7,11 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - http - websocket """ import ssl import base64 diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index 1581c64933..cd6b3a205f 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -11,7 +11,6 @@ .. spelling:: Cloudflare - Submodules """ from .cache import CacheResponsesPlugin, BaseCacheResponsesPlugin from .filter_by_upstream import FilterByUpstreamHostPlugin diff --git a/proxy/plugin/modify_chunk_response.py b/proxy/plugin/modify_chunk_response.py index ccfba8cb5c..f1ace20a04 100644 --- a/proxy/plugin/modify_chunk_response.py +++ b/proxy/plugin/modify_chunk_response.py @@ -37,7 +37,7 @@ def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: if self.response.state == httpParserStates.COMPLETE: # Avoid setting a body for responses where a body is not expected. # Otherwise, example curl will report warnings. - if self.response.body_expected(): + if self.response.body_expected: self.response.body = b'\n'.join(self.DEFAULT_CHUNKS) + b'\n' self.client.queue(memoryview(self.response.build_response())) return memoryview(b'') diff --git a/proxy/plugin/modify_post_data.py b/proxy/plugin/modify_post_data.py index 97d696261a..21be9d358d 100644 --- a/proxy/plugin/modify_post_data.py +++ b/proxy/plugin/modify_post_data.py @@ -34,7 +34,7 @@ def handle_client_request( request.body = ModifyPostDataPlugin.MODIFIED_BODY # Update Content-Length header only when request is NOT chunked # encoded - if not request.is_chunked_encoded(): + if not request.is_chunked_encoded: request.add_header( b'Content-Length', bytes_(len(request.body)), diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py index 7ae7372c5f..c39c6af01d 100644 --- a/proxy/plugin/proxy_pool.py +++ b/proxy/plugin/proxy_pool.py @@ -10,10 +10,12 @@ """ import random import logging +import ipaddress from typing import Dict, List, Optional, Any from ..common.flag import flags +from ..common.utils import text_ from ..http import Url, httpMethods from ..http.parser import HttpParser @@ -78,16 +80,23 @@ def before_upstream_connection( ) -> Optional[HttpParser]: """Avoids establishing the default connection to upstream server by returning None. + + TODO(abhinavsingh): Ideally connection to upstream proxy endpoints + must be bootstrapped within it's own re-usable and garbage collected pool, + to avoid establishing a new upstream proxy connection for each client request. + + See :class:`~proxy.core.connection.pool.ConnectionPool` which is a work + in progress for SSL cache handling. """ - # TODO(abhinavsingh): Ideally connection to upstream proxy endpoints - # must be bootstrapped within it's own re-usable and gc'd pool, to avoid establishing - # a fresh upstream proxy connection for each client request. - # - # See :class:`~proxy.core.connection.pool.ConnectionPool` which is a work - # in progress for SSL cache handling. - # - # Implement your own logic here e.g. round-robin, least connection etc. - endpoint = random.choice(self.flags.proxy_pool)[0].split(':') + # We don't want to send private IP requests to remote proxies + try: + if ipaddress.ip_address(text_(request.host)).is_private: + return request + except ValueError: + pass + # Choose a random proxy from the pool + # TODO: Implement your own logic here e.g. round-robin, least connection etc. + endpoint = random.choice(self.flags.proxy_pool)[0].split(':', 1) if endpoint[0] == 'localhost' and endpoint[1] == '8899': return request logger.debug('Using endpoint: {0}:{1}'.format(*endpoint)) @@ -142,7 +151,7 @@ def handle_client_request( assert url.hostname host, port = url.hostname.decode('utf-8'), url.port port = port if port else ( - 443 if request.is_https_tunnel() else 80 + 443 if request.is_https_tunnel else 80 ) path = None if not request.path else request.path.decode() self.request_host_port_path_method = [ diff --git a/proxy/plugin/redirect_to_custom_server.py b/proxy/plugin/redirect_to_custom_server.py index d04d7ca089..82f4b0bfbf 100644 --- a/proxy/plugin/redirect_to_custom_server.py +++ b/proxy/plugin/redirect_to_custom_server.py @@ -24,7 +24,7 @@ def before_upstream_connection( self, request: HttpParser, ) -> Optional[HttpParser]: # Redirect all non-https requests to inbuilt WebServer. - if not request.is_https_tunnel(): + if not request.is_https_tunnel: request.set_url(self.UPSTREAM_SERVER) # Update Host header too, otherwise upstream can reject our request if request.has_header(b'Host'): diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index 52f85deb48..bfbe4bd345 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -18,6 +18,18 @@ from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes +HTTP_RESPONSE = memoryview( + build_http_response( + httpStatusCodes.OK, body=b'HTTP route response', + ), +) + +HTTPS_RESPONSE = memoryview( + build_http_response( + httpStatusCodes.OK, body=b'HTTPS route response', + ), +) + logger = logging.getLogger(__name__) @@ -33,21 +45,9 @@ def routes(self) -> List[Tuple[int, str]]: def handle_request(self, request: HttpParser) -> None: if request.path == b'/http-route-example': - self.client.queue( - memoryview( - build_http_response( - httpStatusCodes.OK, body=b'HTTP route response', - ), - ), - ) + self.client.queue(HTTP_RESPONSE) elif request.path == b'/https-route-example': - self.client.queue( - memoryview( - build_http_response( - httpStatusCodes.OK, body=b'HTTPS route response', - ), - ), - ) + self.client.queue(HTTPS_RESPONSE) def on_websocket_open(self) -> None: logger.info('Websocket open') diff --git a/proxy/proxy.py b/proxy/proxy.py index 0f2346f935..4142691a16 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - eventing """ import os import sys diff --git a/pyproject.toml b/pyproject.toml index 6ac5005a09..7f9f7173cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,3 +11,50 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "proxy/common/_scm_version.py" + +[tool.towncrier] + directory = "docs/changelog-fragments.d/" + filename = "CHANGELOG.md" + issue_format = "{{issue}}`{issue}`" + start_string = "\n\n" + template = "docs/changelog-fragments.d/.CHANGELOG-TEMPLATE.md.j2" + title_format = "## [{version}] - {project_date}" + underlines = ["##", "###", "####", "#####"] + + [[tool.towncrier.section]] + path = "" + + [[tool.towncrier.type]] + directory = "bugfix" + name = "Bugfixes" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Features" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations (removal in next major release)" + showcontent = true + + [[tool.towncrier.type]] + directory = "breaking" + name = "Backward incompatible changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "doc" + name = "Documentation" + showcontent = true + + [[tool.towncrier.type]] + directory = "misc" + name = "Miscellaneous" + showcontent = true + + [[tool.towncrier.type]] + directory = "internal" + name = "Contributor-facing changes" + showcontent = true diff --git a/pytest.ini b/pytest.ini index bc8ff5be0f..adf8bedc7e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -74,6 +74,7 @@ norecursedirs = build dist docs + benchmark examples proxy.egg-info .cache diff --git a/requirements-release.txt b/requirements-release.txt index ae95ad7991..a51979cfab 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1,2 +1,2 @@ setuptools-scm == 6.3.2 -twine==3.5.0 +twine==3.6.0 diff --git a/requirements-testing.txt b/requirements-testing.txt index 5a8b8a0331..1398fc3615 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -13,5 +13,5 @@ py-spy==0.3.10 codecov==2.1.12 tox==3.24.4 mccabe==0.6.1 -pylint==2.11.1 +pylint==2.12.1 rope==0.22.0 diff --git a/requirements-tunnel.txt b/requirements-tunnel.txt index b247379dba..b9e24a5aab 100644 --- a/requirements-tunnel.txt +++ b/requirements-tunnel.txt @@ -1,2 +1,2 @@ paramiko==2.8.0 -types-paramiko==2.8.1 +types-paramiko==2.8.2 diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py index 8d97a8fa6b..2d1cc883c9 100644 --- a/tests/http/test_http_parser.py +++ b/tests/http/test_http_parser.py @@ -56,21 +56,25 @@ def test_issue_398(self) -> None: def test_urlparse(self) -> None: self.parser.parse(b'CONNECT httpbin.org:443 HTTP/1.1\r\n') - self.assertTrue(self.parser.is_https_tunnel()) + self.assertTrue(self.parser.is_https_tunnel) + self.assertFalse(self.parser.is_connection_upgrade) + self.assertTrue(self.parser.is_http_1_1_keep_alive) + self.assertFalse(self.parser.content_expected) + self.assertFalse(self.parser.body_expected) self.assertEqual(self.parser.host, b'httpbin.org') self.assertEqual(self.parser.port, 443) self.assertNotEqual(self.parser.state, httpParserStates.COMPLETE) def test_urlparse_on_invalid_connect_request(self) -> None: self.parser.parse(b'CONNECT / HTTP/1.0\r\n\r\n') - self.assertTrue(self.parser.is_https_tunnel()) + self.assertTrue(self.parser.is_https_tunnel) self.assertEqual(self.parser.host, None) self.assertEqual(self.parser.port, 443) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_unicode_character_domain_connect(self) -> None: self.parser.parse(bytes_('CONNECT ççç.org:443 HTTP/1.1\r\n')) - self.assertTrue(self.parser.is_https_tunnel()) + self.assertTrue(self.parser.is_https_tunnel) self.assertEqual(self.parser.host, bytes_('ççç.org')) self.assertEqual(self.parser.port, 443) @@ -78,7 +82,7 @@ def test_invalid_ipv6_in_request_line(self) -> None: self.parser.parse( bytes_('CONNECT 2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF:443 HTTP/1.1\r\n'), ) - self.assertTrue(self.parser.is_https_tunnel()) + self.assertTrue(self.parser.is_https_tunnel) self.assertEqual( self.parser.host, bytes_( '[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]', @@ -92,7 +96,7 @@ def test_valid_ipv6_in_request_line(self) -> None: 'CONNECT [2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:443 HTTP/1.1\r\n', ), ) - self.assertTrue(self.parser.is_https_tunnel()) + self.assertTrue(self.parser.is_https_tunnel) self.assertEqual( self.parser.host, bytes_( '[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]', @@ -663,7 +667,7 @@ def test_is_http_1_1_keep_alive(self) -> None: httpMethods.GET, b'/', ), ) - self.assertTrue(self.parser.is_http_1_1_keep_alive()) + self.assertTrue(self.parser.is_http_1_1_keep_alive) def test_is_http_1_1_keep_alive_with_non_close_connection_header( self, @@ -676,7 +680,7 @@ def test_is_http_1_1_keep_alive_with_non_close_connection_header( }, ), ) - self.assertTrue(self.parser.is_http_1_1_keep_alive()) + self.assertTrue(self.parser.is_http_1_1_keep_alive) def test_is_not_http_1_1_keep_alive_with_close_header(self) -> None: self.parser.parse( @@ -687,7 +691,7 @@ def test_is_not_http_1_1_keep_alive_with_close_header(self) -> None: }, ), ) - self.assertFalse(self.parser.is_http_1_1_keep_alive()) + self.assertFalse(self.parser.is_http_1_1_keep_alive) def test_is_not_http_1_1_keep_alive_for_http_1_0(self) -> None: self.parser.parse( @@ -695,7 +699,7 @@ def test_is_not_http_1_1_keep_alive_for_http_1_0(self) -> None: httpMethods.GET, b'/', protocol_version=b'HTTP/1.0', ), ) - self.assertFalse(self.parser.is_http_1_1_keep_alive()) + self.assertFalse(self.parser.is_http_1_1_keep_alive) def test_paramiko_doc(self) -> None: response = b'HTTP/1.1 304 Not Modified\r\nDate: Tue, 03 Dec 2019 02:31:55 GMT\r\nConnection: keep-alive' \ diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py index 1fcfa71720..e10ec0fcd3 100644 --- a/tests/http/test_http_proxy_tls_interception.py +++ b/tests/http/test_http_proxy_tls_interception.py @@ -145,9 +145,8 @@ async def asyncReturnBool(val: bool) -> bool: # Assert our mocked plugins invocations self.plugin.return_value.get_descriptors.assert_called() self.plugin.return_value.write_to_descriptors.assert_called_with([]) - self.plugin.return_value.on_client_data.assert_called_with( - connect_request, - ) + # on_client_data is only called after initial request has completed + self.plugin.return_value.on_client_data.assert_not_called() self.plugin.return_value.on_request_complete.assert_called() self.plugin.return_value.read_from_descriptors.assert_called_with([ self._conn.fileno(), diff --git a/tests/test_main.py b/tests/test_main.py index 28874e822f..5927823fbb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - eventing """ import os import tempfile diff --git a/tox.ini b/tox.ini index daa25f9bb3..7e01c70ce4 100644 --- a/tox.ini +++ b/tox.ini @@ -153,6 +153,41 @@ skip_install = {[testenv:build-docs]skip_install} usedevelop = {[testenv:build-docs]usedevelop} +[testenv:make-changelog] +basepython = python3 +depends = + check-changelog +description = + Generate a changelog from fragments using Towncrier. Getting an + unreleased changelog preview does not require extra arguments. + When invoking to update the changelog, pass the desired version as an + argument after `--`. For example, `tox -e {envname} -- 1.3.2`. +commands = + {envpython} -m \ + towncrier.build \ + --version \ + {posargs:'[UNRELEASED DRAFT]' --draft} +deps = + towncrier == 21.3.0 +isolated_build = true +skip_install = true + + +[testenv:check-changelog] +basepython = {[testenv:make-changelog]basepython} +description = + Check Towncrier change notes +commands = + {envpython} -m \ + towncrier.check \ + --compare-with origin/main \ + {posargs:} +deps = + {[testenv:make-changelog]deps} +isolated_build = {[testenv:make-changelog]isolated_build} +skip_install = {[testenv:make-changelog]skip_install} + + [testenv:cleanup-dists] description = Wipe the the `{env:PEP517_OUT_DIR}{/}` folder @@ -231,5 +266,6 @@ deps = pytest-mock >= 3.6.1 -r docs/requirements.in -r requirements-tunnel.txt + -r benchmark/requirements.txt isolated_build = true skip_install = true