diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 785df5a80c..ef6f014860 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -703,6 +703,11 @@ jobs: cd dashboard npm run build cd .. + - name: Build Devtools + run: | + cd dashboard + npm run devtools + cd .. developer: runs-on: ${{ matrix.os }}-latest diff --git a/.gitignore b/.gitignore index 410b222657..ae8c07d098 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ proxy.py.iml .vscode/* !.vscode/settings.json +*.dot *.pyc *.egg-info *.csr @@ -31,4 +32,5 @@ htmlcov dist build +pyreverse.png profile.svg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6243648ef..489b01b0b2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,12 @@ repos: args: - --py36-plus -# - repo: https://github.com/timothycrosley/isort.git -# rev: 5.10.0 -# hooks: -# - id: isort -# args: -# - --honor-noqa +- repo: https://github.com/timothycrosley/isort.git + rev: 5.10.0 + hooks: + - id: isort + args: + - --honor-noqa - repo: https://github.com/Lucas-C/pre-commit-hooks.git rev: v1.1.7 @@ -24,8 +24,7 @@ repos: helper/proxy\.pac| Makefile| proxy/common/pki\.py| - README\.md| - .+\.(plist|pbxproj) + README\.md $ - repo: https://github.com/pre-commit/pre-commit-hooks.git @@ -36,7 +35,6 @@ repos: exclude: | (?x) ^ - \.github/workflows/codeql-analysis\.yml| dashboard/src/core/plugins/inspect_traffic\.json $ - id: check-merge-conflict diff --git a/Makefile b/Makefile index 793edfad9c..51f12a4412 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ sign-https-certificates: python -m proxy.common.pki sign_csr \ --csr-path $(HTTPS_CSR_FILE_PATH) \ --crt-path $(HTTPS_SIGNED_CERT_FILE_PATH) \ - --hostname example.com \ + --hostname localhost \ --private-key-path $(CA_KEY_FILE_PATH) \ --public-key-path $(CA_CERT_FILE_PATH) @@ -169,11 +169,17 @@ lib-speedscope: --open-file-limit 65536 \ --log-file /dev/null +lib-pyreverse: + rm -f proxy.proxy.Proxy.dot pyreverse.png + pyreverse -ASmy -c proxy.proxy.Proxy proxy + dot -Tpng proxy.proxy.Proxy.dot > pyreverse.png + open pyreverse.png + devtools: - pushd dashboard && npm run devtools && popd + pushd dashboard && npm install && npm run devtools && popd dashboard: - pushd dashboard && npm run build && popd + pushd dashboard && npm install && npm run build && popd dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi diff --git a/README.md b/README.md index 6e0e5b2c1a..3b24f3bafd 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ - [Stable vs Develop](#stable-vs-develop) - [Release Schedule](#release-schedule) - [Threads vs Threadless](#threads-vs-threadless) + - [Threadless Remote vs Local Execution Mode](#threadless-remote-vs-local-execution-mode) - [SyntaxError: invalid syntax](#syntaxerror-invalid-syntax) - [Unable to load plugins](#unable-to-load-plugins) - [Unable to connect with proxy.py from remote host](#unable-to-connect-with-proxypy-from-remote-host) @@ -115,6 +116,9 @@ - [High level architecture](#high-level-architecture) - [Everything is a plugin](#everything-is-a-plugin) - [Internal Documentation](#internal-documentation) + - [Read The Doc](#read-the-doc) + - [pydoc](#pydoc) + - [pyreverse](#pyreverse) - [Development Guide](#development-guide) - [Setup Local Environment](#setup-local-environment) - [Setup Git Hooks](#setup-git-hooks) @@ -132,10 +136,8 @@ - Fast & Scalable - Scale up by using all available cores on the system - - Use `--num-acceptors` flag to control number of cores - Threadless executions using asyncio - - Use `--threaded` for synchronous thread based execution mode - Made to handle `tens-of-thousands` connections / sec @@ -186,6 +188,8 @@ [200] 100000 responses ``` + Consult [Threads vs Threadless](#threads-vs-threadless) and [Threadless Remote vs Local Execution Mode](#threadless-remote-vs-local-execution-mode) to control number of CPU cores utilized. + See [Benchmark](https://github.com/abhinavsingh/proxy.py/tree/develop/benchmark#readme) for more details and for how to run benchmarks locally. - Lightweight @@ -1272,8 +1276,15 @@ Start `proxy.py` as: --tunnel-username username \ --tunnel-hostname ip.address.or.domain.name \ --tunnel-port 22 \ - --tunnel-remote-host 127.0.0.1 - --tunnel-remote-port 8899 + --tunnel-remote-port 8899 \ + --tunnel-ssh-key /path/to/ssh/private.key \ + --tunnel-ssh-key-passphrase XXXXX +...[redacted]... [I] listener.setup:97 - Listening on 127.0.0.1:8899 +...[redacted]... [I] pool.setup:106 - Started 16 acceptors in threadless (local) mode +...[redacted]... [I] transport._log:1873 - Connected (version 2.0, client OpenSSH_7.6p1) +...[redacted]... [I] transport._log:1873 - Authentication (publickey) successful! +...[redacted]... [I] listener.setup:116 - SSH connection established to ip.address.or.domain.name:22... +...[redacted]... [I] listener.start_port_forward:91 - :8899 forwarding successful... ``` Make a HTTP proxy request on `remote` server and @@ -1312,6 +1323,13 @@ access_log:328 - remote:52067 - GET httpbin.org:80 FIREWALL (allow tcp/22) +Not planned. + +If you have a valid use case, kindly open an issue. You are always welcome to send +contributions via pull-requests to add this functionality :) + +> To proxy local requests remotely, make use of [Proxy Pool Plugin](#proxypoolplugin). + # Embed proxy.py ## Blocking Mode @@ -1326,19 +1344,7 @@ if __name__ == '__main__': proxy.main() ``` -Customize startup flags by passing list of input arguments: - -```python -import proxy - -if __name__ == '__main__': - proxy.main([ - '--hostname', '::1', - '--port', '8899' - ]) -``` - -or, customize startup flags by passing them as kwargs: +Customize startup flags by passing them as kwargs: ```python import ipaddress @@ -1353,8 +1359,10 @@ if __name__ == '__main__': Note that: -1. Calling `main` is simply equivalent to starting `proxy.py` from command line. -2. `main` will block until `proxy.py` shuts down. +1. `main` is equivalent to starting `proxy.py` from command line. +2. `main` does not accept any `args` (only `kwargs`). +3. `main` will automatically consume any available `sys.argv` as `args`. +3. `main` will block until `proxy.py` shuts down. ## Non-blocking Mode @@ -1365,20 +1373,21 @@ by using `Proxy` context manager: Example: import proxy if __name__ == '__main__': - with proxy.Proxy([]) as p: - # ... your logic here ... + with proxy.Proxy() as p: + # Uncomment the line below and + # implement your app your logic here + proxy.sleep_loop() ``` Note that: -1. `Proxy` is similar to `main`, except `Proxy` does not block. -2. Internally `Proxy` is a context manager. -3. It will start `proxy.py` when called and will shut it down - once the scope ends. -4. Just like `main`, startup flags with `Proxy` - can be customized by either passing flags as list of - input arguments e.g. `Proxy(['--port', '8899'])` or +1. `Proxy` is similar to `main`, except `Proxy` will not block. +2. Internally, `Proxy` is a context manager which will start + `proxy.py` when called and will shut it down once the scope ends. +3. Unlike `main`, startup flags with `Proxy` can also be customized + by using `args` and `kwargs`. e.g. `Proxy(['--port', '8899'])` or by using passing flags as kwargs e.g. `Proxy(port=8899)`. +4. Unlike `main`, `Proxy` will not inspect `sys.argv`. ## Ephemeral Port @@ -1390,8 +1399,9 @@ In embedded mode, you can access this port. Example: import proxy if __name__ == '__main__': - with proxy.Proxy([]) as p: + with proxy.Proxy() as p: print(p.flags.port) + proxy.sleep_loop() ``` `flags.port` will give you access to the random port allocated by the kernel. @@ -1412,9 +1422,7 @@ Example, load a single plugin using `--plugins` flag: import proxy if __name__ == '__main__': - proxy.main([ - '--plugins', 'proxy.plugin.CacheResponsesPlugin', - ]) + proxy.main(plugins=['proxy.plugin.CacheResponsesPlugin']) ``` For simplicity, you can also pass the list of plugins as a keyword argument to `proxy.main` or the `Proxy` constructor. @@ -1426,7 +1434,7 @@ import proxy from proxy.plugin import FilterByUpstreamHostPlugin if __name__ == '__main__': - proxy.main([], plugins=[ + proxy.main(plugins=[ b'proxy.plugin.CacheResponsesPlugin', FilterByUpstreamHostPlugin, ]) @@ -1436,8 +1444,7 @@ if __name__ == '__main__': ## `proxy.TestCase` -To setup and tear down `proxy.py` for your Python `unittest` classes, -simply use `proxy.TestCase` instead of `unittest.TestCase`. +To setup and tear down `proxy.py` for your Python `unittest` classes, simply use `proxy.TestCase` instead of `unittest.TestCase`. Example: ```python @@ -1686,11 +1693,24 @@ optional arguments: ## Internal Documentation -Code is well documented. You have a few options to browse the internal class hierarchy and documentation: +### Read The Doc + +- Visit [proxypy.readthedocs.io](https://proxypy.readthedocs.io/) +- Build locally using: + +`make lib-doc` + +### pydoc + +Code is well documented. Grab the source code and run: + +`pydoc3 proxy` + +### pyreverse -1. Visit [proxypy.readthedocs.io](https://proxypy.readthedocs.io/) -2. Build and open docs locally using `make lib-doc` -2. Use `pydoc3` locally using `pydoc3 proxy` +Generate class level hierarchy UML diagrams for in-depth analysis: + +`make lib-pyreverse` # Run Dashboard @@ -1890,6 +1910,20 @@ For `windows` and `Python < 3.8`, you can still try out threadless mode by start If threadless works for you, consider sending a PR by editing `_env_threadless_compliant` method in the `proxy/common/constants.py` file. +## Threadless Remote vs Local execution mode + +Original threadless implementation used `remote` execution mode. This is also depicted under [High level architecture](#high-level-architecture) as ASCII art. + +Under `remote` execution mode, acceptors delegate incoming client connection processing to a remote worker process. By default, acceptors delegate connections in round-robin fashion. Worker processing the request may or may not be running on the same CPU core as the acceptor. This architecture scales well for high throughput, but results in spawning two process per CPU core. + +Example, if there are N-CPUs on the machine, by default, N acceptors and N worker processes are started. You can tune number of processes using `--num-acceptors` and `--num-workers` flag. You might want more workers than acceptors or vice versa depending upon your use case. + +In v2.4.x, `local` execution mode was added, mainly to reduce number of processes spawned by default. This model serves well for day-to-day single user use cases and for developer testing scenarios. Under `local` execution mode, acceptors delegate client connections to a companion thread, instead of a remote process. `local` execution mode ensure CPU affinity, unlike in the `remote` mode where acceptor and worker might be running on different CPU cores. + +`--local-executor 1` was made default in v2.4.x series. Under `local` execution mode, `--num-workers` flag has no effect, as no remote workers are started. + +To use `remote` execution mode, use `--local-executor 0` flag. Then use `--num-workers` to tune number of worker processes. + ## SyntaxError: invalid syntax `proxy.py` is strictly typed and uses Python `typing` annotations. Example: @@ -2203,23 +2237,26 @@ To run standalone benchmark for `proxy.py`, use the following command from repo ```console ❯ proxy -h -usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] - [--threaded] [--num-workers NUM_WORKERS] - [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG] - [--hostname HOSTNAME] [--port PORT] [--port-file PORT_FILE] - [--unix-socket-path UNIX_SOCKET_PATH] - [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL] - [--log-file LOG_FILE] [--log-format LOG_FORMAT] - [--open-file-limit OPEN_FILE_LIMIT] +usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] + [--tunnel-username TUNNEL_USERNAME] + [--tunnel-ssh-key TUNNEL_SSH_KEY] + [--tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE] + [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--enable-events] + [--threadless] [--threaded] [--num-workers NUM_WORKERS] + [--backlog BACKLOG] [--hostname HOSTNAME] [--port PORT] + [--port-file PORT_FILE] [--unix-socket-path UNIX_SOCKET_PATH] + [--local-executor LOCAL_EXECUTOR] [--num-acceptors NUM_ACCEPTORS] + [--version] [--log-level LOG_LEVEL] [--log-file LOG_FILE] + [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT] [--plugins PLUGINS [PLUGINS ...]] [--enable-dashboard] - [--work-klass WORK_KLASS] [--pid-file PID_FILE] - [--enable-proxy-protocol] - [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE] - [--timeout TIMEOUT] [--server-recvbuf-size SERVER_RECVBUF_SIZE] - [--disable-http-proxy] [--disable-headers DISABLE_HEADERS] - [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR] - [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE] - [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--cert-file CERT_FILE] + [--enable-ssh-tunnel] [--work-klass WORK_KLASS] + [--pid-file PID_FILE] [--enable-conn-pool] [--key-file KEY_FILE] + [--cert-file CERT_FILE] [--client-recvbuf-size CLIENT_RECVBUF_SIZE] + [--server-recvbuf-size SERVER_RECVBUF_SIZE] [--timeout TIMEOUT] + [--enable-proxy-protocol] [--disable-http-proxy] + [--disable-headers DISABLE_HEADERS] [--ca-key-file CA_KEY_FILE] + [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE] + [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--auth-plugin AUTH_PLUGIN] [--basic-auth BASIC_AUTH] [--cache-dir CACHE_DIR] [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] @@ -2232,15 +2269,28 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.4.0rc6.dev13+ga9b8034.d20220104 +proxy.py v2.4.0rc7.dev12+gd234339.d20220116 options: -h, --help show this help message and exit + --tunnel-hostname TUNNEL_HOSTNAME + Default: None. Remote hostname or IP address to which + SSH tunnel will be established. + --tunnel-port TUNNEL_PORT + Default: 22. SSH port of the remote host. + --tunnel-username TUNNEL_USERNAME + Default: None. Username to use for establishing SSH + tunnel. + --tunnel-ssh-key TUNNEL_SSH_KEY + Default: None. Private key path in pem format + --tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE + Default: None. Private key passphrase + --tunnel-remote-port TUNNEL_REMOTE_PORT + Default: 8899. Remote port which will be forwarded + locally for proxy. --enable-events Default: False. Enables core to dispatch lifecycle events. Plugins can be used to subscribe for core events. - --enable-conn-pool Default: False. (WIP) Enable upstream connection - pooling. --threadless Default: True. Enabled by default on Python 3.8+ (mac, linux). When disabled a new thread is spawned to handle each client connection. @@ -2249,14 +2299,6 @@ options: handle each client connection. --num-workers NUM_WORKERS Defaults to number of CPU cores. - --local-executor LOCAL_EXECUTOR - Default: 1. Enabled by default. Use 0 to disable. When - enabled acceptors will make use of local (same - process) executor instead of distributing load across - remote (other process) executors. Enable this option - to achieve CPU affinity between acceptors and - executors, instead of using underlying OS kernel - scheduling algorithm. --backlog BACKLOG Default: 100. Maximum number of pending connections to proxy server --hostname HOSTNAME Default: 127.0.0.1. Server IP address. @@ -2267,6 +2309,14 @@ options: --unix-socket-path UNIX_SOCKET_PATH Default: None. Unix socket path to use. When provided --host and --port flags are ignored + --local-executor LOCAL_EXECUTOR + Default: 1. Enabled by default. Use 0 to disable. When + enabled acceptors will make use of local (same + process) executor instead of distributing load across + remote (other process) executors. Enable this option + to achieve CPU affinity between acceptors and + executors, instead of using underlying OS kernel + scheduling algorithm. --num-acceptors NUM_ACCEPTORS Defaults to number of CPU cores. --version, -v Prints proxy.py version. @@ -2285,25 +2335,32 @@ options: Comma separated plugins. You may use --plugins flag multiple times. --enable-dashboard Default: False. Enables proxy.py dashboard. + --enable-ssh-tunnel Default: False. Enable SSH tunnel. --work-klass WORK_KLASS Default: proxy.http.HttpProtocolHandler. Work klass to use for work execution. --pid-file PID_FILE Default: None. Save "parent" process ID to a file. - --enable-proxy-protocol - Default: False. If used, will enable proxy protocol. - Only version 1 is currently supported. - --client-recvbuf-size CLIENT_RECVBUF_SIZE - Default: 128 KB. Maximum amount of data received from - the client in a single recv() operation. + --enable-conn-pool Default: False. (WIP) Enable upstream connection + pooling. --key-file KEY_FILE Default: None. Server key file to enable end-to-end TLS encryption with clients. If used, must also pass --cert-file. - --timeout TIMEOUT Default: 10.0. Number of seconds after which an - inactive connection must be dropped. Inactivity is - defined by no data sent or received by the client. + --cert-file CERT_FILE + Default: None. Server certificate to enable end-to-end + TLS encryption with clients. If used, must also pass + --key-file. + --client-recvbuf-size CLIENT_RECVBUF_SIZE + Default: 128 KB. Maximum amount of data received from + the client in a single recv() operation. --server-recvbuf-size SERVER_RECVBUF_SIZE Default: 128 KB. Maximum amount of data received from the server in a single recv() operation. + --timeout TIMEOUT Default: 10.0. Number of seconds after which an + inactive connection must be dropped. Inactivity is + defined by no data sent or received by the client. + --enable-proxy-protocol + Default: False. If used, will enable proxy protocol. + Only version 1 is currently supported. --disable-http-proxy Default: False. Whether to disable proxy.HttpProxyPlugin. --disable-headers DISABLE_HEADERS @@ -2330,10 +2387,6 @@ options: Default: None. CA signing key to use for dynamic generation of HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file - --cert-file CERT_FILE - Default: None. Server certificate to enable end-to-end - TLS encryption with clients. If used, must also pass - --key-file. --auth-plugin AUTH_PLUGIN Default: proxy.http.proxy.AuthPlugin. Auth plugin to use instead of default basic auth plugin. diff --git a/benchmark/_blacksheep.py b/benchmark/_blacksheep.py index cd966b4465..99f7e40e2b 100644 --- a/benchmark/_blacksheep.py +++ b/benchmark/_blacksheep.py @@ -9,7 +9,6 @@ :license: BSD, see LICENSE for more details. """ import uvicorn - from blacksheep.server import Application from blacksheep.server.responses import text diff --git a/benchmark/_proxy.py b/benchmark/_proxy.py index 838918ee5b..4a5ec0cf8d 100644 --- a/benchmark/_proxy.py +++ b/benchmark/_proxy.py @@ -10,6 +10,7 @@ """ import time import ipaddress + import proxy diff --git a/benchmark/_starlette.py b/benchmark/_starlette.py index 0e27e023cb..977b00d8c2 100644 --- a/benchmark/_starlette.py +++ b/benchmark/_starlette.py @@ -9,10 +9,9 @@ :license: BSD, see LICENSE for more details. """ import uvicorn - -from starlette.applications import Starlette -from starlette.responses import Response from starlette.routing import Route +from starlette.responses import Response +from starlette.applications import Starlette async def homepage(request): # type: ignore[no-untyped-def] diff --git a/benchmark/_tornado.py b/benchmark/_tornado.py index 3af22abb60..95e02fe616 100644 --- a/benchmark/_tornado.py +++ b/benchmark/_tornado.py @@ -8,8 +8,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import tornado.ioloop import tornado.web +import tornado.ioloop # pylint: disable=W0223 diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 7380b91e55..bbbd23364d 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -17,7 +17,7 @@ "chrome-devtools-frontend": "^1.0.956881", "eslint": "^6.8.0", "eslint-config-standard": "^14.1.1", - "eslint-plugin-import": "^2.25.3", + "eslint-plugin-import": "^2.25.4", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^5.0.0", @@ -34,7 +34,7 @@ "rollup-plugin-typescript": "^1.0.1", "ts-node": "^7.0.1", "typescript": "^4.5.4", - "ws": "^8.4.0" + "ws": "^8.4.2" } }, "node_modules/@babel/code-frame": { @@ -1209,23 +1209,22 @@ } }, "node_modules/eslint-module-utils": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.1.tgz", - "integrity": "sha512-fjoetBXQZq2tSTWZ9yWVl2KuFrTZZH3V+9iD1V1RfpDgxzJR+mPd/KZmMiA8gbPqdBzpNiEHOuT7IYEWxrH0zQ==", + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.2.tgz", + "integrity": "sha512-zquepFnWCY2ISMFwD/DqzaM++H+7PDzOpUvotJWm/y1BAFt5R4oeULgdrTejKqLkz7MA/tgstsUMNYc7wNdTrg==", "dev": true, "dependencies": { "debug": "^3.2.7", - "find-up": "^2.1.0", - "pkg-dir": "^2.0.0" + "find-up": "^2.1.0" }, "engines": { "node": ">=4" } }, "node_modules/eslint-plugin-import": { - "version": "2.25.3", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.3.tgz", - "integrity": "sha512-RzAVbby+72IB3iOEL8clzPLzL3wpDrlwjsTBAQXgyp5SeTqqY+0bFubwuo+y/HLhNZcXV4XqTBO4LGsfyHIDXg==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.4.tgz", + "integrity": "sha512-/KJBASVFxpu0xg1kIBn9AUa8hQVnszpwgE7Ld0lKAlx7Ie87yzEzCgSkekt+le/YVhiaosO4Y14GDAOc41nfxA==", "dev": true, "dependencies": { "array-includes": "^3.1.4", @@ -1233,14 +1232,14 @@ "debug": "^2.6.9", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.6", - "eslint-module-utils": "^2.7.1", + "eslint-module-utils": "^2.7.2", "has": "^1.0.3", "is-core-module": "^2.8.0", "is-glob": "^4.0.3", "minimatch": "^3.0.4", "object.values": "^1.1.5", "resolve": "^1.20.0", - "tsconfig-paths": "^3.11.0" + "tsconfig-paths": "^3.12.0" }, "engines": { "node": ">=4" @@ -3894,18 +3893,6 @@ "node": ">=0.10.0" } }, - "node_modules/pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", - "dev": true, - "dependencies": { - "find-up": "^2.1.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/pn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", @@ -5064,9 +5051,9 @@ } }, "node_modules/ws": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.0.tgz", - "integrity": "sha512-IHVsKe2pjajSUIl4KYMQOdlyliovpEPquKkqbwswulszzI7r0SfQrxnXdWAEqOlDCLrVSJzo+O1hAwdog2sKSQ==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.2.tgz", + "integrity": "sha512-Kbk4Nxyq7/ZWqr/tarI9yIt/+iNNFOjBXEWgTb4ydaNHBNGgvf2QHbS9fdfsndfjFlFwEd4Al+mw83YkaD10ZA==", "dev": true, "engines": { "node": ">=10.0.0" @@ -6407,20 +6394,19 @@ } }, "eslint-module-utils": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.1.tgz", - "integrity": "sha512-fjoetBXQZq2tSTWZ9yWVl2KuFrTZZH3V+9iD1V1RfpDgxzJR+mPd/KZmMiA8gbPqdBzpNiEHOuT7IYEWxrH0zQ==", + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.2.tgz", + "integrity": "sha512-zquepFnWCY2ISMFwD/DqzaM++H+7PDzOpUvotJWm/y1BAFt5R4oeULgdrTejKqLkz7MA/tgstsUMNYc7wNdTrg==", "dev": true, "requires": { "debug": "^3.2.7", - "find-up": "^2.1.0", - "pkg-dir": "^2.0.0" + "find-up": "^2.1.0" } }, "eslint-plugin-import": { - "version": "2.25.3", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.3.tgz", - "integrity": "sha512-RzAVbby+72IB3iOEL8clzPLzL3wpDrlwjsTBAQXgyp5SeTqqY+0bFubwuo+y/HLhNZcXV4XqTBO4LGsfyHIDXg==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.4.tgz", + "integrity": "sha512-/KJBASVFxpu0xg1kIBn9AUa8hQVnszpwgE7Ld0lKAlx7Ie87yzEzCgSkekt+le/YVhiaosO4Y14GDAOc41nfxA==", "dev": true, "requires": { "array-includes": "^3.1.4", @@ -6428,14 +6414,14 @@ "debug": "^2.6.9", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.6", - "eslint-module-utils": "^2.7.1", + "eslint-module-utils": "^2.7.2", "has": "^1.0.3", "is-core-module": "^2.8.0", "is-glob": "^4.0.3", "minimatch": "^3.0.4", "object.values": "^1.1.5", "resolve": "^1.20.0", - "tsconfig-paths": "^3.11.0" + "tsconfig-paths": "^3.12.0" }, "dependencies": { "debug": { @@ -8158,15 +8144,6 @@ "pinkie": "^2.0.0" } }, - "pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", - "dev": true, - "requires": { - "find-up": "^2.1.0" - } - }, "pn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", @@ -9067,9 +9044,9 @@ } }, "ws": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.0.tgz", - "integrity": "sha512-IHVsKe2pjajSUIl4KYMQOdlyliovpEPquKkqbwswulszzI7r0SfQrxnXdWAEqOlDCLrVSJzo+O1hAwdog2sKSQ==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.2.tgz", + "integrity": "sha512-Kbk4Nxyq7/ZWqr/tarI9yIt/+iNNFOjBXEWgTb4ydaNHBNGgvf2QHbS9fdfsndfjFlFwEd4Al+mw83YkaD10ZA==", "dev": true, "requires": {} }, diff --git a/dashboard/package.json b/dashboard/package.json index 6a83e60bc6..6596dbc00f 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -33,7 +33,7 @@ "chrome-devtools-frontend": "^1.0.956881", "eslint": "^6.8.0", "eslint-config-standard": "^14.1.1", - "eslint-plugin-import": "^2.25.3", + "eslint-plugin-import": "^2.25.4", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^5.0.0", @@ -50,6 +50,6 @@ "rollup-plugin-typescript": "^1.0.1", "ts-node": "^7.0.1", "typescript": "^4.5.4", - "ws": "^8.4.0" + "ws": "^8.4.2" } } diff --git a/dashboard/src/core/devtools.ts b/dashboard/src/core/devtools.ts index 77fc789451..fe933f9ab7 100644 --- a/dashboard/src/core/devtools.ts +++ b/dashboard/src/core/devtools.ts @@ -31,7 +31,7 @@ function setUpDevTools () { } const chromeDevTools = path.dirname( - require.resolve('chrome-devtools-frontend/front_end/inspector.json') + require.resolve('chrome-devtools-frontend/front_end/visibility.gni') ) console.log('Destination folder: ' + destinationFolderPath) diff --git a/docs/_ext/spelling_stub_ext.py b/docs/_ext/spelling_stub_ext.py index c8989dc149..502888c42f 100644 --- a/docs/_ext/spelling_stub_ext.py +++ b/docs/_ext/spelling_stub_ext.py @@ -2,9 +2,9 @@ from typing import List +from sphinx.util.nodes import nodes from sphinx.application import Sphinx from sphinx.util.docutils import SphinxDirective -from sphinx.util.nodes import nodes class SpellingNoOpDirective(SphinxDirective): diff --git a/docs/conf.py b/docs/conf.py index 8543ee8014..05d968a832 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,8 +4,8 @@ """Configuration for the Sphinx documentation generator.""" import sys -from functools import partial from pathlib import Path +from functools import partial from setuptools_scm import get_version @@ -284,6 +284,7 @@ (_py_class_role, '_asyncio.Task'), (_py_class_role, 'asyncio.events.AbstractEventLoop'), (_py_class_role, 'CacheStore'), + (_py_class_role, 'Channel'), (_py_class_role, 'HttpParser'), (_py_class_role, 'HttpProtocolHandlerPlugin'), (_py_class_role, 'HttpProxyBasePlugin'), @@ -307,12 +308,15 @@ (_py_class_role, 'unittest.result.TestResult'), (_py_class_role, 'UUID'), (_py_class_role, 'UpstreamConnectionPool'), + (_py_class_role, 'HttpClientConnection'), (_py_class_role, 'Url'), (_py_class_role, 'WebsocketFrame'), (_py_class_role, 'Work'), (_py_class_role, 'proxy.core.acceptor.work.Work'), (_py_class_role, 'connection.Connection'), (_py_class_role, 'EventQueue'), + (_py_class_role, 'T'), (_py_obj_role, 'proxy.core.work.threadless.T'), (_py_obj_role, 'proxy.core.work.work.T'), + (_py_obj_role, 'proxy.core.base.tcp_server.T'), ] diff --git a/docs/requirements.in b/docs/requirements.in index 24f2e03abd..b79321c965 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -1,6 +1,6 @@ myst-parser[linkify] >= 0.15.2 setuptools-scm >= 6.3.2 -Sphinx >= 4.3.0 +Sphinx == 4.3.2 furo >= 2021.11.15 sphinxcontrib-apidoc >= 0.3.0 sphinxcontrib-towncrier >= 0.2.0a0 diff --git a/examples/ssl_echo_server.py b/examples/ssl_echo_server.py index 433af3878d..56a7d63ae1 100644 --- a/examples/ssl_echo_server.py +++ b/examples/ssl_echo_server.py @@ -9,7 +9,7 @@ :license: BSD, see LICENSE for more details. """ import time -from typing import Optional +from typing import Any, Optional from proxy import Proxy from proxy.core.base import BaseTcpServerHandler @@ -17,9 +17,13 @@ from proxy.core.connection import TcpClientConnection -class EchoSSLServerHandler(BaseTcpServerHandler): +class EchoSSLServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Wraps client socket during initialization.""" + @staticmethod + def create(**kwargs: Any) -> TcpClientConnection: # pragma: no cover + return TcpClientConnection(**kwargs) + def initialize(self) -> None: # Acceptors don't perform TLS handshake. Perform the same # here using wrap_socket() utility. diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index cd4924150f..5dddd639db 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -9,15 +9,20 @@ :license: BSD, see LICENSE for more details. """ import time -from typing import Optional +from typing import Any, Optional from proxy import Proxy from proxy.core.base import BaseTcpServerHandler +from proxy.core.connection import TcpClientConnection -class EchoServerHandler(BaseTcpServerHandler): +class EchoServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Sets client socket to non-blocking during initialization.""" + @staticmethod + def create(**kwargs: Any) -> TcpClientConnection: # pragma: no cover + return TcpClientConnection(**kwargs) + def initialize(self) -> None: self.work.connection.setblocking(False) diff --git a/examples/web_scraper.py b/examples/web_scraper.py index 0dce2bd38b..daf31d612f 100644 --- a/examples/web_scraper.py +++ b/examples/web_scraper.py @@ -11,8 +11,8 @@ import time from proxy import Proxy -from proxy.common.types import Readables, Writables, SelectableEvents from proxy.core.work import Work +from proxy.common.types import Readables, Writables, SelectableEvents from proxy.core.connection import TcpClientConnection diff --git a/proxy/__init__.py b/proxy/__init__.py index a2e0fa77ad..08da6e2aa0 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -8,9 +8,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .proxy import entry_point, main, Proxy +from .proxy import Proxy, main, sleep_loop, entry_point from .testing import TestCase + __all__ = [ # PyPi package entry_point. See # https://github.com/abhinavsingh/proxy.py#from-command-line-when-installed-using-pip @@ -22,4 +23,6 @@ # https://github.com/abhinavsingh/proxy.py#unit-testing-with-proxypy 'TestCase', 'Proxy', + # Utility exposed for demos + 'sleep_loop', ] diff --git a/proxy/__main__.py b/proxy/__main__.py index d04d8529d7..844283ec42 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -10,5 +10,6 @@ """ from .proxy import entry_point + if __name__ == '__main__': entry_point() diff --git a/proxy/common/_scm_version.pyi b/proxy/common/_scm_version.pyi index 4b96455246..504112bd07 100644 --- a/proxy/common/_scm_version.pyi +++ b/proxy/common/_scm_version.pyi @@ -2,5 +2,6 @@ # autogenerated on build and absent on mypy checks time from typing import Tuple, Union + version: str version_tuple: Tuple[Union[int, str], ...] diff --git a/proxy/common/_version.py b/proxy/common/_version.py index 21031c649a..2d4fce9a28 100644 --- a/proxy/common/_version.py +++ b/proxy/common/_version.py @@ -12,9 +12,11 @@ """ from typing import Tuple, Union + try: # pylint: disable=unused-import - from ._scm_version import version as __version__, version_tuple as _ver_tup # noqa: WPS433, WPS436 + from ._scm_version import version as __version__ # noqa: WPS433, WPS436 + from ._scm_version import version_tuple as _ver_tup # noqa: WPS433, WPS436 except ImportError: # pragma: no cover from pkg_resources import get_distribution as _get_dist # noqa: WPS433 __version__ = _get_dist('proxy.py').version # noqa: WPS440 diff --git a/proxy/common/backports.py b/proxy/common/backports.py index 87dbbdf958..770ed9fcc1 100644 --- a/proxy/common/backports.py +++ b/proxy/common/backports.py @@ -10,9 +10,8 @@ """ import time import threading - -from typing import Any, Deque from queue import Empty +from typing import Any, Deque from collections import deque diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 1da2f5e30a..e21586a7db 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -11,16 +11,16 @@ import os import sys import time -import secrets import pathlib +import secrets import platform -import sysconfig import ipaddress - +import sysconfig from typing import Any, List from .version import __version__ + SYS_PLATFORM = platform.system() IS_WINDOWS = SYS_PLATFORM == 'Windows' @@ -88,11 +88,13 @@ def _env_threadless_compliant() -> bool: DEFAULT_DISABLE_HEADERS: List[bytes] = [] DEFAULT_DISABLE_HTTP_PROXY = False DEFAULT_ENABLE_DASHBOARD = False +DEFAULT_ENABLE_SSH_TUNNEL = False DEFAULT_ENABLE_DEVTOOLS = False DEFAULT_ENABLE_EVENTS = False DEFAULT_EVENTS_QUEUE = None DEFAULT_ENABLE_STATIC_SERVER = False DEFAULT_ENABLE_WEB_SERVER = False +DEFAULT_ALLOWED_URL_SCHEMES = [HTTP_PROTO, HTTPS_PROTO] DEFAULT_IPV4_HOSTNAME = ipaddress.IPv4Address('127.0.0.1') DEFAULT_IPV6_HOSTNAME = ipaddress.IPv6Address('::1') DEFAULT_KEY_FILE = None @@ -148,9 +150,9 @@ def _env_threadless_compliant() -> bool: 'HttpWebServerBasePlugin', 'WebSocketTransportBasePlugin', ] -PLUGIN_PROXY_AUTH = 'proxy.http.proxy.AuthPlugin' PLUGIN_DASHBOARD = 'proxy.dashboard.ProxyDashboard' PLUGIN_HTTP_PROXY = 'proxy.http.proxy.HttpProxyPlugin' +PLUGIN_PROXY_AUTH = 'proxy.http.proxy.auth.AuthPlugin' PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin' PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin' PLUGIN_DEVTOOLS_PROTOCOL = 'proxy.http.inspector.devtools.DevtoolsProtocolPlugin' diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 63bef59818..edd07b7ce6 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -16,20 +16,22 @@ import ipaddress import collections import multiprocessing +from typing import Any, List, Optional, cast -from typing import Optional, List, Any, cast - -from .plugins import Plugins from .types import IpAddress from .utils import bytes_, is_py2, is_threadless, set_open_file_limit -from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_ACCEPTORS, DEFAULT_NUM_WORKERS -from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE -from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, DEFAULT_MIN_COMPRESSION_LIMIT -from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE -from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH, IS_WINDOWS, PLUGIN_WEBSOCKET_TRANSPORT from .logger import Logger - +from .plugins import Plugins from .version import __version__ +from .constants import ( + COMMA, IS_WINDOWS, PLUGIN_PAC_FILE, PLUGIN_DASHBOARD, PLUGIN_HTTP_PROXY, + PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, DEFAULT_NUM_WORKERS, + DEFAULT_NUM_ACCEPTORS, PLUGIN_INSPECT_TRAFFIC, DEFAULT_DISABLE_HEADERS, + PY2_DEPRECATION_MESSAGE, DEFAULT_DEVTOOLS_WS_PATH, + PLUGIN_DEVTOOLS_PROTOCOL, PLUGIN_WEBSOCKET_TRANSPORT, + DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_MIN_COMPRESSION_LIMIT, +) + __homepage__ = 'https://github.com/abhinavsingh/proxy.py' @@ -96,13 +98,17 @@ def initialize( print(PY2_DEPRECATION_MESSAGE) sys.exit(1) + # Dirty hack to always discover --basic-auth flag + # defined by proxy auth plugin. + in_args = input_args + ['--plugin', PLUGIN_PROXY_AUTH] + # Discover flags from requested plugin. # This will also surface external plugin flags # under --help. - Plugins.discover(input_args) + Plugins.discover(in_args) # Parse flags - args = flags.parse_args(input_args) + args = flags.parse_args(in_args) # Print version and exit if args.version: @@ -135,9 +141,11 @@ def initialize( if isinstance(work_klass, str) \ else work_klass + # TODO: Plugin flag initialization logic must be moved within plugins. + # # Generate auth_code required for basic authentication if enabled auth_code = None - basic_auth = opts.get('basic_auth', args.basic_auth) + basic_auth = opts.get('basic_auth', getattr(args, 'basic_auth', None)) # Destroy passed credentials via flags or options args.basic_auth = None if 'basic_auth' in opts: @@ -307,8 +315,8 @@ def initialize( # See https://github.com/abhinavsingh/proxy.py/pull/714 description # to understand rationale behind the following logic. # - # --num-workers flag or option was found. We will use - # the same value for num_acceptors when --num-acceptors flag + # Num workers flag or option was found. We will use + # the same value for num_acceptors when num acceptors flag # is absent. if num_workers != DEFAULT_NUM_WORKERS and num_acceptors == DEFAULT_NUM_ACCEPTORS: args.num_acceptors = args.num_workers diff --git a/proxy/common/logger.py b/proxy/common/logger.py index dbd2e8aa51..74421d47b5 100644 --- a/proxy/common/logger.py +++ b/proxy/common/logger.py @@ -9,10 +9,10 @@ :license: BSD, see LICENSE for more details. """ import logging +from typing import Any, Optional -from typing import Optional, Any +from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT -from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL SINGLE_CHAR_TO_LEVEL = { 'D': 'DEBUG', diff --git a/proxy/common/pki.py b/proxy/common/pki.py index 93aab09e46..f8189c0cb9 100644 --- a/proxy/common/pki.py +++ b/proxy/common/pki.py @@ -14,19 +14,18 @@ """ import os import sys -import uuid import time +import uuid import logging -import tempfile import argparse +import tempfile import contextlib import subprocess - -from typing import List, Generator, Optional, Tuple +from typing import List, Tuple, Optional, Generator from .utils import bytes_ -from .constants import COMMA from .version import __version__ +from .constants import COMMA logger = logging.getLogger(__name__) @@ -268,8 +267,8 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: parser.add_argument( '--subject', type=str, - default='/CN=example.com', - help='Subject to use for public key generation. Default: /CN=example.com', + default='/CN=localhost', + help='Subject to use for public key generation. Default: /CN=localhost', ) parser.add_argument( '--csr-path', diff --git a/proxy/common/plugins.py b/proxy/common/plugins.py index 2349e6ec9c..c919154ccd 100644 --- a/proxy/common/plugins.py +++ b/proxy/common/plugins.py @@ -9,15 +9,15 @@ :license: BSD, see LICENSE for more details. """ import os -import logging import inspect -import itertools +import logging import importlib +import itertools +from typing import Any, Dict, List, Tuple, Union, Optional -from typing import Any, List, Dict, Optional, Tuple, Union +from .utils import text_, bytes_ +from .constants import DOT, COMMA, DEFAULT_ABC_PLUGINS -from .utils import bytes_, text_ -from .constants import DOT, DEFAULT_ABC_PLUGINS, COMMA logger = logging.getLogger(__name__) diff --git a/proxy/common/types.py b/proxy/common/types.py index 79bfc62cc4..bc230c9b4e 100644 --- a/proxy/common/types.py +++ b/proxy/common/types.py @@ -10,8 +10,7 @@ """ import queue import ipaddress - -from typing import TYPE_CHECKING, Dict, Any, List, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union if TYPE_CHECKING: # pragma: no cover diff --git a/proxy/common/utils.py b/proxy/common/utils.py index e7373ac24f..9dc11ef1bd 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -12,22 +12,22 @@ utils """ -import sys import ssl +import sys import socket import logging import functools import ipaddress import contextlib - from types import TracebackType -from typing import Optional, Dict, Any, List, Tuple, Type, Callable +from typing import Any, Dict, List, Type, Tuple, Callable, Optional from .constants import ( - HTTP_1_1, COLON, WHITESPACE, CRLF, - DEFAULT_TIMEOUT, DEFAULT_THREADLESS, IS_WINDOWS, + CRLF, COLON, HTTP_1_1, IS_WINDOWS, WHITESPACE, DEFAULT_TIMEOUT, + DEFAULT_THREADLESS, ) + if not IS_WINDOWS: # pragma: no cover import resource diff --git a/proxy/common/version.py b/proxy/common/version.py index 6940317403..530829c5aa 100644 --- a/proxy/common/version.py +++ b/proxy/common/version.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from ._version import __version__, VERSION # noqa: WPS436 +from ._version import VERSION, __version__ # noqa: WPS436 __all__ = '__version__', 'VERSION' diff --git a/proxy/core/acceptor/__init__.py b/proxy/core/acceptor/__init__.py index a7cd9b1bce..e5e3f606c6 100644 --- a/proxy/core/acceptor/__init__.py +++ b/proxy/core/acceptor/__init__.py @@ -12,9 +12,10 @@ pre """ -from .listener import Listener -from .acceptor import Acceptor from .pool import AcceptorPool +from .acceptor import Acceptor +from .listener import Listener + __all__ = [ 'Listener', diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index f578e0af31..9572c1b5b4 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -19,20 +19,17 @@ import threading import multiprocessing import multiprocessing.synchronize - +from typing import List, Tuple, Optional from multiprocessing import connection from multiprocessing.reduction import recv_handle -from typing import List, Optional, Tuple - +from ..work import LocalExecutor, start_threaded_work, delegate_work_to_pool +from ..event import EventQueue from ...common.flag import flags from ...common.logger import Logger from ...common.backports import NonBlockingQueue from ...common.constants import DEFAULT_LOCAL_EXECUTOR -from ..event import EventQueue - -from ..work import LocalExecutor, delegate_work_to_pool, start_threaded_work logger = logging.getLogger(__name__) @@ -148,7 +145,8 @@ def run_once(self) -> None: if locked: self.lock.release() for work in works: - if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR): + if self.flags.threadless and \ + self.flags.local_executor: assert self._local_work_queue self._local_work_queue.put(work) else: @@ -171,7 +169,7 @@ def run(self) -> None: type=socket.SOCK_STREAM, ) try: - if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR): + if self.flags.threadless and self.flags.local_executor: self._start_local() self.selector.register(self.sock, selectors.EVENT_READ) while not self.running.is_set(): @@ -180,7 +178,7 @@ def run(self) -> None: pass finally: self.selector.unregister(self.sock) - if self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR): + if self.flags.threadless and self.flags.local_executor: self._stop_local() self.sock.close() logger.debug('Acceptor#%d shutdown', self.idd) @@ -224,7 +222,8 @@ def _work(self, conn: socket.socket, addr: Optional[Tuple[str, int]]) -> None: ), ) thread.start() - logger.debug( + # TODO: Move me into target method + logger.debug( # pragma: no cover 'Dispatched work#{0}.{1}.{2} to worker#{3}'.format( conn.fileno(), self.idd, self._total, index, ), @@ -237,6 +236,7 @@ def _work(self, conn: socket.socket, addr: Optional[Tuple[str, int]]) -> None: event_queue=self.event_queue, publisher_id=self.__class__.__name__, ) + # TODO: Move me into target method logger.debug( # pragma: no cover 'Started work#{0}.{1}.{2} in thread#{3}'.format( conn.fileno(), self.idd, self._total, thread.ident, diff --git a/proxy/core/acceptor/listener.py b/proxy/core/acceptor/listener.py index d55962a933..0caef04d9a 100644 --- a/proxy/core/acceptor/listener.py +++ b/proxy/core/acceptor/listener.py @@ -16,11 +16,12 @@ import socket import logging import argparse - -from typing import Optional, Any +from typing import Any, Optional from ...common.flag import flags -from ...common.constants import DEFAULT_BACKLOG, DEFAULT_IPV4_HOSTNAME, DEFAULT_PORT, DEFAULT_PORT_FILE +from ...common.constants import ( + DEFAULT_PORT, DEFAULT_BACKLOG, DEFAULT_PORT_FILE, DEFAULT_IPV4_HOSTNAME, +) flags.add_argument( diff --git a/proxy/core/acceptor/pool.py b/proxy/core/acceptor/pool.py index 745d672e38..99299cf4fa 100644 --- a/proxy/core/acceptor/pool.py +++ b/proxy/core/acceptor/pool.py @@ -17,20 +17,19 @@ import logging import argparse import multiprocessing - +from typing import TYPE_CHECKING, Any, List, Optional from multiprocessing import connection from multiprocessing.reduction import send_handle -from typing import Any, List, Optional - -from .listener import Listener from .acceptor import Acceptor - -from ..event import EventQueue - +from .listener import Listener from ...common.flag import flags from ...common.constants import DEFAULT_NUM_ACCEPTORS + +if TYPE_CHECKING: # pragma: no cover + from ..event import EventQueue + logger = logging.getLogger(__name__) @@ -69,7 +68,7 @@ def __init__( executor_queues: List[connection.Connection], executor_pids: List[int], executor_locks: List['multiprocessing.synchronize.Lock'], - event_queue: Optional[EventQueue] = None, + event_queue: Optional['EventQueue'] = None, ) -> None: self.flags = flags # File descriptor to use for accepting new work @@ -79,7 +78,7 @@ def __init__( self.executor_pids: List[int] = executor_pids self.executor_locks: List['multiprocessing.synchronize.Lock'] = executor_locks # Eventing core queue - self.event_queue: Optional[EventQueue] = event_queue + self.event_queue: Optional['EventQueue'] = event_queue # Acceptor process instances self.acceptors: List[Acceptor] = [] # Fd queues used to share file descriptor with acceptor processes diff --git a/proxy/core/base/__init__.py b/proxy/core/base/__init__.py index bc86952b7c..5ce5a827d9 100644 --- a/proxy/core/base/__init__.py +++ b/proxy/core/base/__init__.py @@ -12,6 +12,7 @@ from .tcp_tunnel import BaseTcpTunnelHandler from .tcp_upstream import TcpUpstreamConnectionHandler + __all__ = [ 'BaseTcpServerHandler', 'BaseTcpTunnelHandler', diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index edc5361510..107e1ac9dc 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -12,20 +12,76 @@ tcp """ +import ssl +import socket import logging import selectors - from abc import abstractmethod -from typing import Any, Optional +from typing import Any, Union, TypeVar, Optional from ...core.work import Work +from ...common.flag import flags +from ...common.types import Readables, Writables, SelectableEvents +from ...common.utils import wrap_socket from ...core.connection import TcpClientConnection -from ...common.types import Readables, SelectableEvents, Writables +from ...common.constants import ( + DEFAULT_TIMEOUT, DEFAULT_KEY_FILE, DEFAULT_CERT_FILE, + DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_SERVER_RECVBUF_SIZE, +) + logger = logging.getLogger(__name__) -class BaseTcpServerHandler(Work[TcpClientConnection]): +flags.add_argument( + '--key-file', + type=str, + default=DEFAULT_KEY_FILE, + help='Default: None. Server key file to enable end-to-end TLS encryption with clients. ' + 'If used, must also pass --cert-file.', +) + +flags.add_argument( + '--cert-file', + type=str, + default=DEFAULT_CERT_FILE, + help='Default: None. Server certificate to enable end-to-end TLS encryption with clients. ' + 'If used, must also pass --key-file.', +) + +flags.add_argument( + '--client-recvbuf-size', + type=int, + default=DEFAULT_CLIENT_RECVBUF_SIZE, + help='Default: ' + str(int(DEFAULT_CLIENT_RECVBUF_SIZE / 1024)) + + ' KB. Maximum amount of data received from the ' + 'client in a single recv() operation.', +) + +flags.add_argument( + '--server-recvbuf-size', + type=int, + default=DEFAULT_SERVER_RECVBUF_SIZE, + help='Default: ' + str(int(DEFAULT_SERVER_RECVBUF_SIZE / 1024)) + + ' KB. Maximum amount of data received from the ' + 'server in a single recv() operation.', +) + +flags.add_argument( + '--timeout', + type=int, + default=DEFAULT_TIMEOUT, + help='Default: ' + str(DEFAULT_TIMEOUT) + + '. Number of seconds after which ' + 'an inactive connection must be dropped. Inactivity is defined by no ' + 'data sent or received by the client.', +) + + +T = TypeVar('T', bound=TcpClientConnection) + + +class BaseTcpServerHandler(Work[T]): """BaseTcpServerHandler implements Work interface. BaseTcpServerHandler lifecycle is controlled by Threadless core @@ -56,6 +112,14 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.work.address, ) + def initialize(self) -> None: + """Optionally upgrades connection to HTTPS, + sets ``conn`` in non-blocking mode and initializes + HTTP protocol plugins.""" + conn = self._optionally_wrap_socket(self.work.connection) + conn.setblocking(False) + logger.debug('Handling connection %s' % self.work.address) + @abstractmethod def handle_data(self, data: memoryview) -> Optional[bool]: """Optionally return True to close client connection.""" @@ -139,3 +203,21 @@ async def handle_readables(self, readables: Readables) -> bool: else: teardown = True return teardown + + def _encryption_enabled(self) -> bool: + return self.flags.keyfile is not None and \ + self.flags.certfile is not None + + def _optionally_wrap_socket( + self, conn: socket.socket, + ) -> Union[ssl.SSLSocket, socket.socket]: + """Attempts to wrap accepted client connection using provided certificates. + + Shutdown and closes client connection upon error. + """ + if self._encryption_enabled(): + assert self.flags.keyfile and self.flags.certfile + # TODO(abhinavsingh): Insecure TLS versions must not be accepted by default + conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) + self.work._conn = conn + return conn diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index fb230ec0e7..48444384a6 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -10,21 +10,20 @@ """ import logging import selectors - from abc import abstractmethod from typing import Any, Optional +from .tcp_server import BaseTcpServerHandler +from ..connection import TcpClientConnection, TcpServerConnection from ...http.parser import HttpParser, httpParserTypes -from ...common.types import Readables, SelectableEvents, Writables +from ...common.types import Readables, Writables, SelectableEvents from ...common.utils import text_ -from ..connection import TcpServerConnection -from .tcp_server import BaseTcpServerHandler logger = logging.getLogger(__name__) -class BaseTcpTunnelHandler(BaseTcpServerHandler): +class BaseTcpTunnelHandler(BaseTcpServerHandler[TcpClientConnection]): """BaseTcpTunnelHandler build on-top of BaseTcpServerHandler work class. On-top of BaseTcpServerHandler implementation, @@ -47,6 +46,10 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: def handle_data(self, data: memoryview) -> Optional[bool]: pass # pragma: no cover + @staticmethod + def create(**kwargs: Any) -> TcpClientConnection: # pragma: no cover + return TcpClientConnection(**kwargs) + def initialize(self) -> None: self.work.connection.setblocking(False) diff --git a/proxy/core/base/tcp_upstream.py b/proxy/core/base/tcp_upstream.py index 402a22e426..564f9a5072 100644 --- a/proxy/core/base/tcp_upstream.py +++ b/proxy/core/base/tcp_upstream.py @@ -10,13 +10,13 @@ """ import ssl import logging - from abc import ABC, abstractmethod -from typing import Optional, Any +from typing import Any, Optional from ...common.types import Readables, Writables, Descriptors from ...core.connection import TcpServerConnection + logger = logging.getLogger(__name__) diff --git a/proxy/core/connection/__init__.py b/proxy/core/connection/__init__.py index 58d100a81b..3457f1dbed 100644 --- a/proxy/core/connection/__init__.py +++ b/proxy/core/connection/__init__.py @@ -13,11 +13,12 @@ reusability Submodules """ -from .connection import TcpConnection, TcpConnectionUninitializedException -from .client import TcpClientConnection -from .server import TcpServerConnection from .pool import UpstreamConnectionPool from .types import tcpConnectionTypes +from .client import TcpClientConnection +from .server import TcpServerConnection +from .connection import TcpConnection, TcpConnectionUninitializedException + __all__ = [ 'TcpConnection', diff --git a/proxy/core/connection/client.py b/proxy/core/connection/client.py index 966221e17b..0ba63885ef 100644 --- a/proxy/core/connection/client.py +++ b/proxy/core/connection/client.py @@ -10,11 +10,10 @@ """ import ssl import socket +from typing import Tuple, Union, Optional -from typing import Union, Tuple, Optional - -from .connection import TcpConnection, TcpConnectionUninitializedException from .types import tcpConnectionTypes +from .connection import TcpConnection, TcpConnectionUninitializedException class TcpClientConnection(TcpConnection): diff --git a/proxy/core/connection/connection.py b/proxy/core/connection/connection.py index 84c6c9de74..9aa0d77d46 100644 --- a/proxy/core/connection/connection.py +++ b/proxy/core/connection/connection.py @@ -11,13 +11,12 @@ import ssl import socket import logging - from abc import ABC, abstractmethod -from typing import Optional, Union, List +from typing import List, Union, Optional +from .types import tcpConnectionTypes from ...common.constants import DEFAULT_BUFFER_SIZE, DEFAULT_MAX_SEND_SIZE -from .types import tcpConnectionTypes logger = logging.getLogger(__name__) diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index 399aa5923d..86c54a328e 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -15,15 +15,13 @@ import socket import logging import selectors - -from typing import TYPE_CHECKING, Set, Dict, Tuple - -from ...common.flag import flags -from ...common.types import Readables, SelectableEvents, Writables +from typing import TYPE_CHECKING, Any, Set, Dict, Tuple from ..work import Work - from .server import TcpServerConnection +from ...common.flag import flags +from ...common.types import Readables, Writables, SelectableEvents + logger = logging.getLogger(__name__) @@ -77,6 +75,10 @@ def __init__(self) -> None: self.connections: Dict[int, TcpServerConnection] = {} self.pools: Dict[Tuple[str, int], Set[TcpServerConnection]] = {} + @staticmethod + def create(**kwargs: Any) -> TcpServerConnection: # pragma: no cover + return TcpServerConnection(**kwargs) + def acquire(self, addr: Tuple[str, int]) -> Tuple[bool, TcpServerConnection]: """Returns a reusable connection from the pool. @@ -152,7 +154,7 @@ def add(self, addr: Tuple[str, int]) -> TcpServerConnection: NOTE: You must not use the returned connection, instead use `acquire`. """ - new_conn = TcpServerConnection(addr[0], addr[1]) + new_conn = self.create(host=addr[0], port=addr[1]) new_conn.connect() self._add(new_conn) logger.debug( diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py index 109c238e30..48690f96bf 100644 --- a/proxy/core/connection/server.py +++ b/proxy/core/connection/server.py @@ -10,13 +10,11 @@ """ import ssl import socket +from typing import Tuple, Union, Optional -from typing import Optional, Union, Tuple - -from ...common.utils import new_socket_connection - -from .connection import TcpConnection, TcpConnectionUninitializedException from .types import tcpConnectionTypes +from .connection import TcpConnection, TcpConnectionUninitializedException +from ...common.utils import new_socket_connection class TcpServerConnection(TcpConnection): diff --git a/proxy/core/event/__init__.py b/proxy/core/event/__init__.py index 17e1074e6e..05736f7b5d 100644 --- a/proxy/core/event/__init__.py +++ b/proxy/core/event/__init__.py @@ -8,11 +8,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .queue import EventQueue from .names import EventNames, eventNames +from .queue import EventQueue +from .manager import EventManager from .dispatcher import EventDispatcher from .subscriber import EventSubscriber -from .manager import EventManager + __all__ = [ 'eventNames', diff --git a/proxy/core/event/dispatcher.py b/proxy/core/event/dispatcher.py index d26340d2ea..7cc978f154 100644 --- a/proxy/core/event/dispatcher.py +++ b/proxy/core/event/dispatcher.py @@ -11,13 +11,12 @@ import queue import logging import threading - +from typing import Any, Dict, List from multiprocessing import connection -from typing import Dict, Any, List - -from .queue import EventQueue from .names import eventNames +from .queue import EventQueue + logger = logging.getLogger(__name__) @@ -61,7 +60,7 @@ def handle_event(self, ev: Dict[str, Any]) -> None: }) elif ev['event_name'] == eventNames.UNSUBSCRIBE: # send ack - print('unsubscription request ack sent') + logger.info('unsubscription request ack sent') self.subscribers[ev['event_payload']['sub_id']].send({ 'event_name': eventNames.UNSUBSCRIBED, }) diff --git a/proxy/core/event/manager.py b/proxy/core/event/manager.py index 3923c6c12e..6a100f904a 100644 --- a/proxy/core/event/manager.py +++ b/proxy/core/event/manager.py @@ -15,15 +15,14 @@ import logging import threading import multiprocessing - from typing import Any, Optional from .queue import EventQueue from .dispatcher import EventDispatcher - from ...common.flag import flags from ...common.constants import DEFAULT_ENABLE_EVENTS + logger = logging.getLogger(__name__) diff --git a/proxy/core/event/names.py b/proxy/core/event/names.py index 5040c880a8..369724aac4 100644 --- a/proxy/core/event/names.py +++ b/proxy/core/event/names.py @@ -10,6 +10,7 @@ """ from typing import NamedTuple + # Name of the events that eventing framework supports. # # Ideally this must be configurable via command line or diff --git a/proxy/core/event/queue.py b/proxy/core/event/queue.py index f86ba09b9a..878fe9bf45 100644 --- a/proxy/core/event/queue.py +++ b/proxy/core/event/queue.py @@ -11,14 +11,11 @@ import os import time import threading - +from typing import Any, Dict, Optional from multiprocessing import connection -from typing import Dict, Optional, Any - -from ...common.types import DictQueueType - from .names import eventNames +from ...common.types import DictQueueType class EventQueue: diff --git a/proxy/core/event/subscriber.py b/proxy/core/event/subscriber.py index 14567616ea..ea478dc46c 100644 --- a/proxy/core/event/subscriber.py +++ b/proxy/core/event/subscriber.py @@ -13,13 +13,12 @@ import logging import threading import multiprocessing - +from typing import Any, Dict, Callable, Optional from multiprocessing import connection -from typing import Dict, Optional, Any, Callable - -from .queue import EventQueue from .names import eventNames +from .queue import EventQueue + logger = logging.getLogger(__name__) diff --git a/proxy/core/ssh/__init__.py b/proxy/core/ssh/__init__.py index e37310801c..9d9d605de4 100644 --- a/proxy/core/ssh/__init__.py +++ b/proxy/core/ssh/__init__.py @@ -12,10 +12,11 @@ Submodules """ -from .client import SshClient -from .tunnel import Tunnel +from .handler import SshHttpProtocolHandler +from .listener import SshTunnelListener + __all__ = [ - 'SshClient', - 'Tunnel', + 'SshHttpProtocolHandler', + 'SshTunnelListener', ] diff --git a/proxy/core/ssh/client.py b/proxy/core/ssh/client.py deleted file mode 100644 index 4657b1a3c1..0000000000 --- a/proxy/core/ssh/client.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on - Network monitoring, controls & Application development, testing, debugging. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -import socket -import ssl -from typing import Union - -from ..connection import TcpClientConnection - - -class SshClient(TcpClientConnection): - """Overrides TcpClientConnection. - - This is necessary because paramiko ``fileno()`` can be used for polling - but not for send / recv. - """ - - @property - def connection(self) -> Union[ssl.SSLSocket, socket.socket]: - # Dummy return to comply with - return socket.socket() diff --git a/proxy/core/ssh/handler.py b/proxy/core/ssh/handler.py new file mode 100644 index 0000000000..12557ed923 --- /dev/null +++ b/proxy/core/ssh/handler.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import argparse +from typing import TYPE_CHECKING, Tuple + + +if TYPE_CHECKING: # pragma: no cover + try: + from paramiko.channel import Channel + except ImportError: + pass + + +class SshHttpProtocolHandler: + """Handles incoming connections over forwarded SSH transport.""" + + def __init__(self, flags: argparse.Namespace) -> None: + self.flags = flags + + def on_connection( + self, + chan: 'Channel', + origin: Tuple[str, int], + server: Tuple[str, int], + ) -> None: + pass diff --git a/proxy/core/ssh/listener.py b/proxy/core/ssh/listener.py new file mode 100644 index 0000000000..1b17120e63 --- /dev/null +++ b/proxy/core/ssh/listener.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import logging +import argparse +from typing import TYPE_CHECKING, Any, Set, Tuple, Callable, Optional + + +try: + from paramiko import SSHClient, AutoAddPolicy + from paramiko.transport import Transport + if TYPE_CHECKING: # pragma: no cover + from paramiko.channel import Channel +except ImportError: # pragma: no cover + pass + +from ...common.flag import flags + + +logger = logging.getLogger(__name__) + + +flags.add_argument( + '--tunnel-hostname', + type=str, + default=None, + help='Default: None. Remote hostname or IP address to which SSH tunnel will be established.', +) + +flags.add_argument( + '--tunnel-port', + type=int, + default=22, + help='Default: 22. SSH port of the remote host.', +) + +flags.add_argument( + '--tunnel-username', + type=str, + default=None, + help='Default: None. Username to use for establishing SSH tunnel.', +) + +flags.add_argument( + '--tunnel-ssh-key', + type=str, + default=None, + help='Default: None. Private key path in pem format', +) + +flags.add_argument( + '--tunnel-ssh-key-passphrase', + type=str, + default=None, + help='Default: None. Private key passphrase', +) + +flags.add_argument( + '--tunnel-remote-port', + type=int, + default=8899, + help='Default: 8899. Remote port which will be forwarded locally for proxy.', +) + + +class SshTunnelListener: + """Connects over SSH and forwards a remote port to local host. + + Incoming connections are delegated to provided callback.""" + + def __init__( + self, + flags: argparse.Namespace, + on_connection_callback: Callable[['Channel', Tuple[str, int], Tuple[str, int]], None], + ) -> None: + self.flags = flags + self.on_connection_callback = on_connection_callback + self.ssh: Optional[SSHClient] = None + self.transport: Optional[Transport] = None + self.forwarded: Set[Tuple[str, int]] = set() + + def start_port_forward(self, remote_addr: Tuple[str, int]) -> None: + assert self.transport is not None + self.transport.request_port_forward( + *remote_addr, + handler=self.on_connection_callback, + ) + self.forwarded.add(remote_addr) + logger.info('%s:%d forwarding successful...' % remote_addr) + + def stop_port_forward(self, remote_addr: Tuple[str, int]) -> None: + assert self.transport is not None + self.transport.cancel_port_forward(*remote_addr) + self.forwarded.remove(remote_addr) + + def __enter__(self) -> 'SshTunnelListener': + self.setup() + return self + + def __exit__(self, *args: Any) -> None: + self.shutdown() + + def setup(self) -> None: + self.ssh = SSHClient() + self.ssh.load_system_host_keys() + self.ssh.set_missing_host_key_policy(AutoAddPolicy()) + self.ssh.connect( + hostname=self.flags.tunnel_hostname, + port=self.flags.tunnel_port, + username=self.flags.tunnel_username, + key_filename=self.flags.tunnel_ssh_key, + passphrase=self.flags.tunnel_ssh_key_passphrase, + ) + logger.info( + 'SSH connection established to %s:%d...' % ( + self.flags.tunnel_hostname, + self.flags.tunnel_port, + ), + ) + self.transport = self.ssh.get_transport() + + def shutdown(self) -> None: + for remote_addr in list(self.forwarded): + self.stop_port_forward(remote_addr) + self.forwarded.clear() + if self.transport is not None: + self.transport.close() + if self.ssh is not None: + self.ssh.close() diff --git a/proxy/core/ssh/tunnel.py b/proxy/core/ssh/tunnel.py deleted file mode 100644 index 4a899543ae..0000000000 --- a/proxy/core/ssh/tunnel.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on - Network monitoring, controls & Application development, testing, debugging. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -import logging -import paramiko - -from typing import Optional, Tuple, Callable - -logger = logging.getLogger(__name__) - - -class Tunnel: - """Establishes a tunnel between local (machine where Tunnel is running) and remote host. - Once a tunnel has been established, remote host can route HTTP(s) traffic to - ``localhost`` over tunnel. - """ - - def __init__( - self, - ssh_username: str, - remote_addr: Tuple[str, int], - private_pem_key: str, - remote_proxy_port: int, - conn_handler: Callable[[paramiko.channel.Channel], None], - ) -> None: - self.remote_addr = remote_addr - self.ssh_username = ssh_username - self.private_pem_key = private_pem_key - self.remote_proxy_port = remote_proxy_port - self.conn_handler = conn_handler - - def run(self) -> None: - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - ssh.set_missing_host_key_policy(paramiko.WarningPolicy()) - try: - ssh.connect( - hostname=self.remote_addr[0], - port=self.remote_addr[1], - username=self.ssh_username, - key_filename=self.private_pem_key, - ) - logger.info('SSH connection established...') - transport: Optional[paramiko.transport.Transport] = ssh.get_transport( - ) - assert transport is not None - transport.request_port_forward('', self.remote_proxy_port) - logger.info('Tunnel port forward setup successful...') - while True: - conn: Optional[paramiko.channel.Channel] = transport.accept( - timeout=1, - ) - assert conn is not None - e = transport.get_exception() - if e: - raise e - if conn is None: - continue - self.conn_handler(conn) - except KeyboardInterrupt: - pass - finally: - ssh.close() diff --git a/proxy/core/work/__init__.py b/proxy/core/work/__init__.py index 403978ff17..a89add7841 100644 --- a/proxy/core/work/__init__.py +++ b/proxy/core/work/__init__.py @@ -12,13 +12,14 @@ pre """ +from .pool import ThreadlessPool from .work import Work -from .threadless import Threadless -from .remote import RemoteExecutor from .local import LocalExecutor -from .pool import ThreadlessPool +from .remote import RemoteExecutor from .delegate import delegate_work_to_pool from .threaded import start_threaded_work +from .threadless import Threadless + __all__ = [ 'Work', diff --git a/proxy/core/work/delegate.py b/proxy/core/work/delegate.py index 5a176e8992..76207d9635 100644 --- a/proxy/core/work/delegate.py +++ b/proxy/core/work/delegate.py @@ -8,9 +8,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Tuple, Optional from multiprocessing.reduction import send_handle + if TYPE_CHECKING: # pragma: no cover import socket import multiprocessing diff --git a/proxy/core/work/local.py b/proxy/core/work/local.py index 95c2c118c2..e081b2da9c 100644 --- a/proxy/core/work/local.py +++ b/proxy/core/work/local.py @@ -9,16 +9,16 @@ :license: BSD, see LICENSE for more details. """ import queue -import logging import asyncio +import logging import contextlib - -from typing import Optional -from typing import Any - -from ...common.backports import NonBlockingQueue # noqa: W0611, F401 pylint: disable=unused-import +from typing import Any, Optional from .threadless import Threadless +from ...common.backports import ( # noqa: W0611, F401 pylint: disable=unused-import + NonBlockingQueue, +) + logger = logging.getLogger(__name__) diff --git a/proxy/core/work/pool.py b/proxy/core/work/pool.py index 0a28ff1b50..96c43a7228 100644 --- a/proxy/core/work/pool.py +++ b/proxy/core/work/pool.py @@ -11,14 +11,13 @@ import logging import argparse import multiprocessing - +from typing import TYPE_CHECKING, Any, List, Optional from multiprocessing import connection -from typing import TYPE_CHECKING, Any, Optional, List from .remote import RemoteExecutor - from ...common.flag import flags -from ...common.constants import DEFAULT_NUM_WORKERS, DEFAULT_THREADLESS +from ...common.constants import DEFAULT_THREADLESS, DEFAULT_NUM_WORKERS + if TYPE_CHECKING: # pragma: no cover from ..event import EventQueue diff --git a/proxy/core/work/remote.py b/proxy/core/work/remote.py index bc16e83e40..6dd91ef053 100644 --- a/proxy/core/work/remote.py +++ b/proxy/core/work/remote.py @@ -10,13 +10,13 @@ """ import asyncio import logging - -from typing import Optional, Any +from typing import Any, Optional from multiprocessing import connection from multiprocessing.reduction import recv_handle from .threadless import Threadless + logger = logging.getLogger(__name__) diff --git a/proxy/core/work/threaded.py b/proxy/core/work/threaded.py index 4e583a0ded..6f2569a001 100644 --- a/proxy/core/work/threaded.py +++ b/proxy/core/work/threaded.py @@ -11,26 +11,28 @@ import socket import argparse import threading +from typing import TYPE_CHECKING, Tuple, TypeVar, Optional -from typing import TYPE_CHECKING, Optional, Tuple - -from ..connection import TcpClientConnection from ..event import EventQueue, eventNames + if TYPE_CHECKING: # pragma: no cover from .work import Work +T = TypeVar('T') + +# TODO: Add generic T def start_threaded_work( flags: argparse.Namespace, conn: socket.socket, addr: Optional[Tuple[str, int]], event_queue: Optional[EventQueue] = None, publisher_id: Optional[str] = None, -) -> Tuple['Work[TcpClientConnection]', threading.Thread]: +) -> Tuple['Work[T]', threading.Thread]: """Utility method to start a work in a new thread.""" work = flags.work_klass( - TcpClientConnection(conn, addr), + flags.work_klass.create(conn=conn, addr=addr), flags=flags, event_queue=event_queue, upstream_conn_pool=None, diff --git a/proxy/core/work/threadless.py b/proxy/core/work/threadless.py index d713e94440..57eca4dcae 100644 --- a/proxy/core/work/threadless.py +++ b/proxy/core/work/threadless.py @@ -11,28 +11,30 @@ import os import ssl import socket -import logging import asyncio +import logging import argparse import selectors import multiprocessing +from abc import ABC, abstractmethod +from typing import ( + TYPE_CHECKING, Set, Dict, List, Tuple, Union, Generic, TypeVar, Optional, +) -from abc import abstractmethod, ABC -from typing import TYPE_CHECKING, Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union - +from ..event import eventNames +from ...common.types import Readables, Writables, SelectableEvents from ...common.logger import Logger -from ...common.types import Readables, SelectableEvents, Writables -from ...common.constants import DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT -from ...common.constants import DEFAULT_WAIT_FOR_TASKS_TIMEOUT +from ...common.constants import ( + DEFAULT_WAIT_FOR_TASKS_TIMEOUT, DEFAULT_SELECTOR_SELECT_TIMEOUT, + DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT, +) -from ..connection import TcpClientConnection, UpstreamConnectionPool -from ..event import eventNames if TYPE_CHECKING: # pragma: no cover from typing import Any - from ..event import EventQueue from .work import Work + from ..event import EventQueue T = TypeVar('T') @@ -91,7 +93,12 @@ def __init__( self.wait_timeout: float = DEFAULT_WAIT_FOR_TASKS_TIMEOUT self.cleanup_inactive_timeout: float = DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT self._total: int = 0 - self._upstream_conn_pool: Optional[UpstreamConnectionPool] = None + # When put at the top, causes circular import error + # since integrated ssh tunnel was introduced. + from ..connection import ( # pylint: disable=C0415 + UpstreamConnectionPool, + ) + self._upstream_conn_pool: Optional['UpstreamConnectionPool'] = None self._upstream_conn_filenos: Set[int] = set() if self.flags.enable_conn_pool: self._upstream_conn_pool = UpstreamConnectionPool() @@ -135,7 +142,7 @@ def work_on_tcp_conn( ) uid = '%s-%s-%s' % (self.iid, self._total, fileno) self.works[fileno] = self.flags.work_klass( - TcpClientConnection( + self.flags.work_klass.create( conn=conn, addr=addr, ), @@ -427,7 +434,7 @@ def run(self) -> None: data=wqfileno, ) assert self.loop - # logger.debug('Working on {0} works'.format(len(self.works))) + logger.debug('Working on {0} works'.format(len(self.works))) self.loop.create_task(self._run_forever()) self.loop.run_forever() except KeyboardInterrupt: diff --git a/proxy/core/work/work.py b/proxy/core/work/work.py index 5c94d2bf53..08560103de 100644 --- a/proxy/core/work/work.py +++ b/proxy/core/work/work.py @@ -13,13 +13,13 @@ acceptor """ import argparse - -from uuid import uuid4 from abc import ABC, abstractmethod -from typing import Optional, Dict, Any, TypeVar, Generic, TYPE_CHECKING +from uuid import uuid4 +from typing import TYPE_CHECKING, Any, Dict, Generic, TypeVar, Optional + +from ..event import EventQueue, eventNames +from ...common.types import Readables, Writables, SelectableEvents -from ..event import eventNames, EventQueue -from ...common.types import Readables, SelectableEvents, Writables if TYPE_CHECKING: # pragma: no cover from ..connection import UpstreamConnectionPool @@ -47,6 +47,14 @@ def __init__( self.work = work self.upstream_conn_pool = upstream_conn_pool + @staticmethod + @abstractmethod + def create(**kwargs: Any) -> T: + """Implementations are responsible for creation of work objects + from incoming args. This helps keep work core agnostic to + creation of externally defined work class objects.""" + raise NotImplementedError() + @abstractmethod async def get_events(self) -> SelectableEvents: """Return sockets and events (read or write) that we are interested in.""" diff --git a/proxy/dashboard/__init__.py b/proxy/dashboard/__init__.py index 96a79affef..b59e877a77 100644 --- a/proxy/dashboard/__init__.py +++ b/proxy/dashboard/__init__.py @@ -10,6 +10,7 @@ """ from .dashboard import ProxyDashboard + __all__ = [ 'ProxyDashboard', ] diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py index 6e719d2306..411c153b60 100644 --- a/proxy/dashboard/dashboard.py +++ b/proxy/dashboard/dashboard.py @@ -12,9 +12,12 @@ import logging from typing import List, Tuple -from ..http.responses import permanentRedirectResponse from ..http.parser import HttpParser -from ..http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes +from ..http.server import ( + HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes, +) +from ..http.responses import permanentRedirectResponse + logger = logging.getLogger(__name__) diff --git a/proxy/http/__init__.py b/proxy/http/__init__.py index b918c3ecf9..37826426ab 100644 --- a/proxy/http/__init__.py +++ b/proxy/http/__init__.py @@ -8,15 +8,18 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .handler import HttpProtocolHandler -from .plugin import HttpProtocolHandlerPlugin +from .url import Url from .codes import httpStatusCodes -from .methods import httpMethods +from .plugin import HttpProtocolHandlerPlugin +from .handler import HttpProtocolHandler from .headers import httpHeaders -from .url import Url +from .methods import httpMethods +from .connection import HttpClientConnection + __all__ = [ 'HttpProtocolHandler', + 'HttpClientConnection', 'HttpProtocolHandlerPlugin', 'httpStatusCodes', 'httpMethods', diff --git a/proxy/http/connection.py b/proxy/http/connection.py new file mode 100644 index 0000000000..d31d34036e --- /dev/null +++ b/proxy/http/connection.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable +""" +from ..core.connection import TcpClientConnection + + +class HttpClientConnection(TcpClientConnection): + pass diff --git a/proxy/http/descriptors.py b/proxy/http/descriptors.py index bae5ea3857..102629bd21 100644 --- a/proxy/http/descriptors.py +++ b/proxy/http/descriptors.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ from typing import Any + from ..common.types import Readables, Writables, Descriptors diff --git a/proxy/http/exception/__init__.py b/proxy/http/exception/__init__.py index 513d2bd510..68776e923b 100644 --- a/proxy/http/exception/__init__.py +++ b/proxy/http/exception/__init__.py @@ -9,9 +9,10 @@ :license: BSD, see LICENSE for more details. """ from .base import HttpProtocolException -from .http_request_rejected import HttpRequestRejected from .proxy_auth_failed import ProxyAuthenticationFailed from .proxy_conn_failed import ProxyConnectionFailed +from .http_request_rejected import HttpRequestRejected + __all__ = [ 'HttpProtocolException', diff --git a/proxy/http/exception/base.py b/proxy/http/exception/base.py index e01be4abac..bd1233bae6 100644 --- a/proxy/http/exception/base.py +++ b/proxy/http/exception/base.py @@ -12,7 +12,8 @@ http """ -from typing import Any, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional + if TYPE_CHECKING: # pragma: no cover from ..parser import HttpParser diff --git a/proxy/http/exception/http_request_rejected.py b/proxy/http/exception/http_request_rejected.py index 7c6f8d48be..2b2e7a13b1 100644 --- a/proxy/http/exception/http_request_rejected.py +++ b/proxy/http/exception/http_request_rejected.py @@ -8,12 +8,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Any, Optional, Dict, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, Optional from .base import HttpProtocolException - from ...common.utils import build_http_response + if TYPE_CHECKING: # pragma: no cover from ..parser import HttpParser diff --git a/proxy/http/exception/proxy_auth_failed.py b/proxy/http/exception/proxy_auth_failed.py index 4fbe62b4e6..afb2e4048e 100644 --- a/proxy/http/exception/proxy_auth_failed.py +++ b/proxy/http/exception/proxy_auth_failed.py @@ -16,9 +16,9 @@ from typing import TYPE_CHECKING, Any from .base import HttpProtocolException - from ..responses import PROXY_AUTH_FAILED_RESPONSE_PKT + if TYPE_CHECKING: # pragma: no cover from ..parser import HttpParser diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py index 7aa709235c..2001b33605 100644 --- a/proxy/http/exception/proxy_conn_failed.py +++ b/proxy/http/exception/proxy_conn_failed.py @@ -15,9 +15,9 @@ from typing import TYPE_CHECKING, Any from .base import HttpProtocolException - from ..responses import BAD_GATEWAY_RESPONSE_PKT + if TYPE_CHECKING: # pragma: no cover from ..parser import HttpParser diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 7966174dbb..04fd507b33 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -15,53 +15,23 @@ import asyncio import logging import selectors +from typing import Any, List, Type, Tuple, Optional -from typing import Tuple, List, Type, Union, Optional, Any - -from ..common.flag import flags -from ..common.utils import wrap_socket -from ..core.base import BaseTcpServerHandler -from ..core.connection import TcpClientConnection -from ..common.types import Readables, SelectableEvents, Writables -from ..common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_KEY_FILE -from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT, DEFAULT_TIMEOUT - -from .exception import HttpProtocolException +from .parser import HttpParser, httpParserTypes, httpParserStates from .plugin import HttpProtocolHandlerPlugin +from .exception import HttpProtocolException +from .protocols import httpProtocols from .responses import BAD_REQUEST_RESPONSE_PKT -from .parser import HttpParser, httpParserStates, httpParserTypes +from ..core.base import BaseTcpServerHandler +from .connection import HttpClientConnection +from ..common.types import Readables, Writables, SelectableEvents +from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT logger = logging.getLogger(__name__) -flags.add_argument( - '--client-recvbuf-size', - type=int, - default=DEFAULT_CLIENT_RECVBUF_SIZE, - help='Default: ' + str(int(DEFAULT_CLIENT_RECVBUF_SIZE / 1024)) + - ' KB. Maximum amount of data received from the ' - 'client in a single recv() operation.', -) -flags.add_argument( - '--key-file', - type=str, - default=DEFAULT_KEY_FILE, - help='Default: None. Server key file to enable end-to-end TLS encryption with clients. ' - 'If used, must also pass --cert-file.', -) -flags.add_argument( - '--timeout', - type=int, - default=DEFAULT_TIMEOUT, - help='Default: ' + str(DEFAULT_TIMEOUT) + - '. Number of seconds after which ' - 'an inactive connection must be dropped. Inactivity is defined by no ' - 'data sent or received by the client.', -) - - -class HttpProtocolHandler(BaseTcpServerHandler): +class HttpProtocolHandler(BaseTcpServerHandler[HttpClientConnection]): """HTTP, HTTPS, HTTP2, WebSockets protocol handler. Accepts `Client` connection and delegates to HttpProtocolHandlerPlugin. @@ -85,18 +55,17 @@ def __init__(self, *args: Any, **kwargs: Any): # overrides Work class definitions. ## + @staticmethod + def create(**kwargs: Any) -> HttpClientConnection: # pragma: no cover + return HttpClientConnection(**kwargs) + def initialize(self) -> None: - """Optionally upgrades connection to HTTPS, - sets ``conn`` in non-blocking mode and initializes - HTTP protocol plugins. - """ - conn = self._optionally_wrap_socket(self.work.connection) - conn.setblocking(False) - # Update client connection reference if connection was wrapped + super().initialize() if self._encryption_enabled(): - self.work = TcpClientConnection(conn=conn, addr=self.work.addr) - # self._initialize_plugins() - logger.debug('Handling connection %s' % self.work.address) + self.work = HttpClientConnection( + conn=self.work.connection, + addr=self.work.addr, + ) def is_inactive(self) -> bool: if not self.work.has_buffer() and \ @@ -234,12 +203,14 @@ async def handle_writables(self, writables: Writables) -> bool: if teardown: return True except BrokenPipeError: - logger.error( + logger.warning( # pragma: no cover 'BrokenPipeError when flushing buffer for client', ) return True except OSError: - logger.error('OSError when flushing buffer to client') + logger.warning( # pragma: no cover + 'OSError when flushing buffer to client', + ) return True return False @@ -304,12 +275,20 @@ def _parse_first_request(self, data: memoryview) -> bool: # memoryview compliant try: self.request.parse(data.tobytes()) - except Exception: + except HttpProtocolException as e: # noqa: WPS329 + self.work.queue(BAD_REQUEST_RESPONSE_PKT) + raise e + except Exception as e: + self.work.queue(BAD_REQUEST_RESPONSE_PKT) raise HttpProtocolException( 'Error when parsing request: %r' % data.tobytes(), - ) + ) from e if not self.request.is_complete: return False + # Bail out if http protocol is unknown + if self.request.http_handler_protocol == httpProtocols.UNKNOWN: + self.work.queue(BAD_REQUEST_RESPONSE_PKT) + return True # Discover which HTTP handler plugin is capable of # handling the current incoming request klass = self._discover_plugin_klass( @@ -334,23 +313,6 @@ def _parse_first_request(self, data: memoryview) -> bool: self.work._conn = output return False - def _encryption_enabled(self) -> bool: - return self.flags.keyfile is not None and \ - self.flags.certfile is not None - - def _optionally_wrap_socket( - self, conn: socket.socket, - ) -> Union[ssl.SSLSocket, socket.socket]: - """Attempts to wrap accepted client connection using provided certificates. - - Shutdown and closes client connection upon error. - """ - if self._encryption_enabled(): - assert self.flags.keyfile and self.flags.certfile - # TODO(abhinavsingh): Insecure TLS versions must not be accepted by default - conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) - return conn - def _connection_inactive_for(self) -> float: return time.time() - self.last_activity diff --git a/proxy/http/inspector/devtools.py b/proxy/http/inspector/devtools.py index 5b16571bf7..b5058764df 100644 --- a/proxy/http/inspector/devtools.py +++ b/proxy/http/inspector/devtools.py @@ -14,18 +14,20 @@ """ import json import logging -from typing import List, Tuple, Any, Dict +from typing import Any, Dict, List, Tuple -from .transformer import CoreEventsToDevtoolsProtocol from ..parser import HttpParser -from ..websocket import WebsocketFrame, websocketOpcodes from ..server import HttpWebServerBasePlugin, httpProtocolTypes - -from ...common.utils import bytes_, text_ +from ..websocket import WebsocketFrame, websocketOpcodes +from .transformer import CoreEventsToDevtoolsProtocol from ...core.event import EventSubscriber from ...common.flag import flags -from ...common.constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DEVTOOLS_DOC_URL -from ...common.constants import DEFAULT_ENABLE_DEVTOOLS +from ...common.utils import text_, bytes_ +from ...common.constants import ( + DEFAULT_ENABLE_DEVTOOLS, DEFAULT_DEVTOOLS_DOC_URL, + DEFAULT_DEVTOOLS_WS_PATH, +) + logger = logging.getLogger(__name__) diff --git a/proxy/http/inspector/inspect_traffic.py b/proxy/http/inspector/inspect_traffic.py index 456626d97a..8b5b01da9a 100644 --- a/proxy/http/inspector/inspect_traffic.py +++ b/proxy/http/inspector/inspect_traffic.py @@ -9,12 +9,15 @@ :license: BSD, see LICENSE for more details. """ import json -from typing import List, Dict, Any +from typing import TYPE_CHECKING, Any, Dict, List -from ...common.utils import bytes_ -from ...core.event import EventSubscriber -from ...core.connection import TcpClientConnection from ..websocket import WebsocketFrame, WebSocketTransportBasePlugin +from ...core.event import EventSubscriber +from ...common.utils import bytes_ + + +if TYPE_CHECKING: # pragma: no cover + from ..connection import HttpClientConnection class InspectTrafficPlugin(WebSocketTransportBasePlugin): @@ -58,7 +61,7 @@ def handle_message(self, message: Dict[str, Any]) -> None: raise NotImplementedError() @staticmethod - def callback(client: TcpClientConnection, event: Dict[str, Any]) -> None: + def callback(client: 'HttpClientConnection', event: Dict[str, Any]) -> None: event['push'] = 'inspect_traffic' client.queue( memoryview( diff --git a/proxy/http/inspector/transformer.py b/proxy/http/inspector/transformer.py index da7e1d613e..d04909e1ef 100644 --- a/proxy/http/inspector/transformer.py +++ b/proxy/http/inspector/transformer.py @@ -10,14 +10,19 @@ """ import json import time -from typing import Any, Dict +from typing import TYPE_CHECKING, Any, Dict from ..websocket import WebsocketFrame -from ...common.constants import PROXY_PY_START_TIME, DEFAULT_DEVTOOLS_DOC_URL -from ...common.constants import DEFAULT_DEVTOOLS_FRAME_ID, DEFAULT_DEVTOOLS_LOADER_ID -from ...common.utils import bytes_ -from ...core.connection import TcpClientConnection from ...core.event import eventNames +from ...common.utils import bytes_ +from ...common.constants import ( + PROXY_PY_START_TIME, DEFAULT_DEVTOOLS_DOC_URL, DEFAULT_DEVTOOLS_FRAME_ID, + DEFAULT_DEVTOOLS_LOADER_ID, +) + + +if TYPE_CHECKING: # pragma: no cover + from ..connection import HttpClientConnection class CoreEventsToDevtoolsProtocol: @@ -30,7 +35,7 @@ class CoreEventsToDevtoolsProtocol: @staticmethod def transformer( - client: TcpClientConnection, + client: 'HttpClientConnection', event: Dict[str, Any], ) -> None: event_name = event['event_name'] diff --git a/proxy/http/parser/__init__.py b/proxy/http/parser/__init__.py index be633d4873..d197666522 100644 --- a/proxy/http/parser/__init__.py +++ b/proxy/http/parser/__init__.py @@ -13,10 +13,11 @@ http Submodules """ -from .parser import HttpParser from .chunk import ChunkParser, chunkParserStates -from .types import httpParserStates, httpParserTypes -from .protocol import ProxyProtocol, PROXY_PROTOCOL_V2_SIGNATURE +from .types import httpParserTypes, httpParserStates +from .parser import HttpParser +from .protocol import PROXY_PROTOCOL_V2_SIGNATURE, ProxyProtocol + __all__ = [ 'HttpParser', diff --git a/proxy/http/parser/chunk.py b/proxy/http/parser/chunk.py index 2d976c4449..691117926d 100644 --- a/proxy/http/parser/chunk.py +++ b/proxy/http/parser/chunk.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import NamedTuple, Tuple, List, Optional +from typing import List, Tuple, Optional, NamedTuple from ...common.utils import bytes_, find_http_line from ...common.constants import CRLF, DEFAULT_BUFFER_SIZE diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index ecb774ef13..6573cabaf6 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -12,22 +12,22 @@ http """ -from typing import TypeVar, Optional, Dict, Type, Tuple, List - -from ...common.constants import DEFAULT_DISABLE_HEADERS, COLON, DEFAULT_ENABLE_PROXY_PROTOCOL -from ...common.constants import HTTP_1_1, SLASH, CRLF -from ...common.constants import WHITESPACE, DEFAULT_HTTP_PORT -from ...common.utils import build_http_request, build_http_response, text_ -from ...common.flag import flags +from typing import Dict, List, Type, Tuple, TypeVar, Optional from ..url import Url +from .chunk import ChunkParser, chunkParserStates +from .types import httpParserTypes, httpParserStates from ..methods import httpMethods -from ..protocols import httpProtocols +from .protocol import ProxyProtocol from ..exception import HttpProtocolException +from ..protocols import httpProtocols +from ...common.flag import flags +from ...common.utils import text_, build_http_request, build_http_response +from ...common.constants import ( + CRLF, COLON, SLASH, HTTP_1_0, HTTP_1_1, WHITESPACE, DEFAULT_HTTP_PORT, + DEFAULT_DISABLE_HEADERS, DEFAULT_ENABLE_PROXY_PROTOCOL, +) -from .protocol import ProxyProtocol -from .chunk import ChunkParser, chunkParserStates -from .types import httpParserTypes, httpParserStates flags.add_argument( '--enable-proxy-protocol', @@ -149,15 +149,22 @@ def del_headers(self, headers: List[bytes]) -> None: for key in headers: self.del_header(key.lower()) - def set_url(self, url: bytes) -> None: + def set_url(self, url: bytes, allowed_url_schemes: Optional[List[bytes]] = None) -> None: """Given a request line, parses it and sets line attributes a.k.a. host, port, path.""" - self._url = Url.from_bytes(url) + self._url = Url.from_bytes( + url, allowed_url_schemes=allowed_url_schemes, + ) self._set_line_attributes() @property def http_handler_protocol(self) -> int: """Returns `HttpProtocols` that this request belongs to.""" - return httpProtocols.HTTP_PROXY if self.host is not None else httpProtocols.WEB_SERVER + if self.version in (HTTP_1_1, HTTP_1_0) and self._url is not None: + if self.host is not None: + return httpProtocols.HTTP_PROXY + if self._url.hostname is None: + return httpProtocols.WEB_SERVER + return httpProtocols.UNKNOWN @property def is_complete(self) -> bool: @@ -199,7 +206,7 @@ def body_expected(self) -> bool: """Returns true if content or chunked response is expected.""" return self._content_expected or self._is_chunked_encoded - def parse(self, raw: bytes) -> None: + def parse(self, raw: bytes, allowed_url_schemes: Optional[List[bytes]] = None) -> None: """Parses HTTP request out of raw bytes. Check for `HttpParser.state` after `parse` has successfully returned.""" @@ -212,7 +219,10 @@ def parse(self, raw: bytes) -> None: if self.state >= httpParserStates.HEADERS_COMPLETE: more, raw = self._process_body(raw) elif self.state == httpParserStates.INITIALIZED: - more, raw = self._process_line(raw) + more, raw = self._process_line( + raw, + allowed_url_schemes=allowed_url_schemes, + ) else: more, raw = self._process_headers(raw) # When server sends a response line without any header or body e.g. @@ -340,7 +350,11 @@ def _process_headers(self, raw: bytes) -> Tuple[bool, bytes]: break return len(raw) > 0, raw - def _process_line(self, raw: bytes) -> Tuple[bool, bytes]: + def _process_line( + self, + raw: bytes, + allowed_url_schemes: Optional[List[bytes]] = None, + ) -> Tuple[bool, bytes]: while True: parts = raw.split(CRLF, 1) if len(parts) == 1: @@ -358,7 +372,9 @@ def _process_line(self, raw: bytes) -> Tuple[bool, bytes]: self.method = parts[0] if self.method == httpMethods.CONNECT: self._is_https_tunnel = True - self.set_url(parts[1]) + self.set_url( + parts[1], allowed_url_schemes=allowed_url_schemes, + ) self.version = parts[2] self.state = httpParserStates.LINE_RCVD break diff --git a/proxy/http/parser/protocol.py b/proxy/http/parser/protocol.py index c44192f0e5..2fc19df484 100644 --- a/proxy/http/parser/protocol.py +++ b/proxy/http/parser/protocol.py @@ -8,12 +8,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Tuple +from typing import Tuple, Optional from ..exception import HttpProtocolException - from ...common.constants import WHITESPACE + PROXY_PROTOCOL_V2_SIGNATURE = b'\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A' diff --git a/proxy/http/parser/tls/__init__.py b/proxy/http/parser/tls/__init__.py index d32e35bcd0..1ad8fb2c89 100644 --- a/proxy/http/parser/tls/__init__.py +++ b/proxy/http/parser/tls/__init__.py @@ -11,6 +11,7 @@ from .tls import TlsParser from .types import tlsContentType, tlsHandshakeType + __all__ = [ 'TlsParser', 'tlsContentType', diff --git a/proxy/http/parser/tls/certificate.py b/proxy/http/parser/tls/certificate.py index 1004210af7..f71e495c79 100644 --- a/proxy/http/parser/tls/certificate.py +++ b/proxy/http/parser/tls/certificate.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Tuple +from typing import Tuple, Optional class TlsCertificate: diff --git a/proxy/http/parser/tls/finished.py b/proxy/http/parser/tls/finished.py index 4cc7273372..df9db0625d 100644 --- a/proxy/http/parser/tls/finished.py +++ b/proxy/http/parser/tls/finished.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Tuple +from typing import Tuple, Optional class TlsFinished: diff --git a/proxy/http/parser/tls/handshake.py b/proxy/http/parser/tls/handshake.py index d859ceb589..7a03e2471c 100644 --- a/proxy/http/parser/tls/handshake.py +++ b/proxy/http/parser/tls/handshake.py @@ -10,14 +10,18 @@ """ import struct import logging +from typing import Tuple, Optional -from typing import Optional, Tuple - +from .hello import ( + TlsClientHello, TlsServerHello, TlsHelloRequest, TlsServerHelloDone, +) from .types import tlsHandshakeType -from .hello import TlsHelloRequest, TlsClientHello, TlsServerHello, TlsServerHelloDone -from .certificate import TlsCertificate, TlsCertificateRequest, TlsCertificateVerify -from .key_exchange import TlsClientKeyExchange, TlsServerKeyExchange from .finished import TlsFinished +from .certificate import ( + TlsCertificate, TlsCertificateVerify, TlsCertificateRequest, +) +from .key_exchange import TlsClientKeyExchange, TlsServerKeyExchange + logger = logging.getLogger(__name__) diff --git a/proxy/http/parser/tls/hello.py b/proxy/http/parser/tls/hello.py index 80287109d0..f60c724b22 100644 --- a/proxy/http/parser/tls/hello.py +++ b/proxy/http/parser/tls/hello.py @@ -11,11 +11,11 @@ import os import struct import logging - -from typing import Optional, Tuple +from typing import Tuple, Optional from .pretty import pretty_hexlify + logger = logging.getLogger(__name__) diff --git a/proxy/http/parser/tls/key_exchange.py b/proxy/http/parser/tls/key_exchange.py index ce56562b19..cb0059ed47 100644 --- a/proxy/http/parser/tls/key_exchange.py +++ b/proxy/http/parser/tls/key_exchange.py @@ -8,7 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import Optional, Tuple +from typing import Tuple, Optional class TlsServerKeyExchange: diff --git a/proxy/http/parser/tls/tls.py b/proxy/http/parser/tls/tls.py index 634c5b93ed..9e5aa89eb5 100644 --- a/proxy/http/parser/tls/tls.py +++ b/proxy/http/parser/tls/tls.py @@ -10,12 +10,12 @@ """ import struct import logging - -from typing import Optional, Tuple +from typing import Tuple, Optional from .types import tlsContentType -from .certificate import TlsCertificate from .handshake import TlsHandshake +from .certificate import TlsCertificate + logger = logging.getLogger(__name__) diff --git a/proxy/http/parser/tls/types.py b/proxy/http/parser/tls/types.py index 9dd05df321..640cffe282 100644 --- a/proxy/http/parser/tls/types.py +++ b/proxy/http/parser/tls/types.py @@ -10,6 +10,7 @@ """ from typing import NamedTuple + TlsContentType = NamedTuple( 'TlsContentType', [ ('CHANGE_CIPHER_SPEC', int), diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py index fe44999444..9eaa097779 100644 --- a/proxy/http/plugin.py +++ b/proxy/http/plugin.py @@ -10,16 +10,15 @@ """ import socket import argparse - from abc import ABC, abstractmethod -from typing import List, Union, Optional, TYPE_CHECKING - -from ..core.event import EventQueue -from ..core.connection import TcpClientConnection +from typing import TYPE_CHECKING, List, Union, Optional +from .mixins import TlsInterceptionPropertyMixin from .parser import HttpParser +from .connection import HttpClientConnection +from ..core.event import EventQueue from .descriptors import DescriptorsHandlerMixin -from .mixins import TlsInterceptionPropertyMixin + if TYPE_CHECKING: # pragma: no cover from ..core.connection import UpstreamConnectionPool @@ -55,7 +54,7 @@ def __init__( self, uid: str, flags: argparse.Namespace, - client: TcpClientConnection, + client: HttpClientConnection, request: HttpParser, event_queue: Optional[EventQueue] = None, upstream_conn_pool: Optional['UpstreamConnectionPool'] = None, @@ -63,7 +62,7 @@ def __init__( super().__init__(uid, flags, client, event_queue, upstream_conn_pool) self.uid: str = uid self.flags: argparse.Namespace = flags - self.client: TcpClientConnection = client + self.client: HttpClientConnection = client self.request: HttpParser = request self.event_queue = event_queue self.upstream_conn_pool = upstream_conn_pool diff --git a/proxy/http/protocols.py b/proxy/http/protocols.py index 976a41852b..49485720c3 100644 --- a/proxy/http/protocols.py +++ b/proxy/http/protocols.py @@ -18,6 +18,7 @@ HttpProtocols = NamedTuple( 'HttpProtocols', [ + ('UNKNOWN', int), # Web server handling HTTP/1.0, HTTP/1.1, HTTP/2, HTTP/3 # over plain Text or encrypted connection with clients ('WEB_SERVER', int), @@ -30,4 +31,4 @@ ], ) -httpProtocols = HttpProtocols(1, 2, 3) +httpProtocols = HttpProtocols(1, 2, 3, 4) diff --git a/proxy/http/proxy/__init__.py b/proxy/http/proxy/__init__.py index 4e18002c0d..a794ba078c 100644 --- a/proxy/http/proxy/__init__.py +++ b/proxy/http/proxy/__init__.py @@ -10,10 +10,9 @@ """ from .plugin import HttpProxyBasePlugin from .server import HttpProxyPlugin -from .auth import AuthPlugin + __all__ = [ 'HttpProxyBasePlugin', 'HttpProxyPlugin', - 'AuthPlugin', ] diff --git a/proxy/http/proxy/auth.py b/proxy/http/proxy/auth.py index a309d194c7..f2632d7957 100644 --- a/proxy/http/proxy/auth.py +++ b/proxy/http/proxy/auth.py @@ -15,14 +15,12 @@ """ from typing import Optional +from ...http import httpHeaders from ..exception import ProxyAuthenticationFailed - +from ...http.proxy import HttpProxyBasePlugin from ...common.flag import flags -from ...common.constants import DEFAULT_BASIC_AUTH - -from ...http import httpHeaders from ...http.parser import HttpParser -from ...http.proxy import HttpProxyBasePlugin +from ...common.constants import DEFAULT_BASIC_AUTH flags.add_argument( diff --git a/proxy/http/proxy/plugin.py b/proxy/http/proxy/plugin.py index 2f893d2e59..ed951e313f 100644 --- a/proxy/http/proxy/plugin.py +++ b/proxy/http/proxy/plugin.py @@ -9,17 +9,15 @@ :license: BSD, see LICENSE for more details. """ import argparse - from abc import ABC -from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, Tuple, Optional from ..mixins import TlsInterceptionPropertyMixin - from ..parser import HttpParser +from ..connection import HttpClientConnection +from ...core.event import EventQueue from ..descriptors import DescriptorsHandlerMixin -from ...core.event import EventQueue -from ...core.connection import TcpClientConnection if TYPE_CHECKING: # pragma: no cover from ...core.connection import UpstreamConnectionPool @@ -38,7 +36,7 @@ def __init__( self, uid: str, flags: argparse.Namespace, - client: TcpClientConnection, + client: HttpClientConnection, event_queue: EventQueue, upstream_conn_pool: Optional['UpstreamConnectionPool'] = None, ) -> None: diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 7029a4333c..10932fde9e 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -21,46 +21,36 @@ import logging import threading import subprocess - -from typing import Optional, List, Union, Dict, cast, Any +from typing import Any, Dict, List, Union, Optional, cast from .plugin import HttpProxyBasePlugin - +from ..parser import HttpParser, httpParserTypes, httpParserStates +from ..plugin import HttpProtocolHandlerPlugin from ..headers import httpHeaders from ..methods import httpMethods -from ..protocols import httpProtocols -from ..plugin import HttpProtocolHandlerPlugin from ..exception import HttpProtocolException, ProxyConnectionFailed -from ..parser import HttpParser, httpParserStates, httpParserTypes +from ..protocols import httpProtocols from ..responses import PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT - +from ...common.pki import gen_csr, sign_csr, gen_public_key +from ...core.event import eventNames +from ...common.flag import flags from ...common.types import Readables, Writables, Descriptors -from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE -from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE -from ...common.constants import COMMA, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CERT_FILE -from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS -from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT -from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH from ...common.utils import text_ -from ...common.pki import gen_public_key, gen_csr, sign_csr +from ...core.connection import ( + TcpServerConnection, TcpConnectionUninitializedException, +) +from ...common.constants import ( + COMMA, DEFAULT_CA_FILE, PLUGIN_PROXY_AUTH, DEFAULT_CA_CERT_DIR, + DEFAULT_CA_KEY_FILE, DEFAULT_CA_CERT_FILE, DEFAULT_DISABLE_HEADERS, + PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HTTP_PROXY, + DEFAULT_CA_SIGNING_KEY_FILE, DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT, + DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT, +) -from ...core.event import eventNames -from ...core.connection import TcpServerConnection -from ...core.connection import TcpConnectionUninitializedException -from ...common.flag import flags logger = logging.getLogger(__name__) -flags.add_argument( - '--server-recvbuf-size', - type=int, - default=DEFAULT_SERVER_RECVBUF_SIZE, - help='Default: ' + str(int(DEFAULT_SERVER_RECVBUF_SIZE / 1024)) + - ' KB. Maximum amount of data received from the ' - 'server in a single recv() operation.', -) - flags.add_argument( '--disable-http-proxy', action='store_true', @@ -116,14 +106,6 @@ 'HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file', ) -flags.add_argument( - '--cert-file', - type=str, - default=DEFAULT_CERT_FILE, - help='Default: None. Server certificate to enable end-to-end TLS encryption with clients. ' - 'If used, must also pass --key-file.', -) - flags.add_argument( '--auth-plugin', type=str, @@ -213,7 +195,7 @@ async def write_to_descriptors(self, w: Writables) -> bool: ) return False except BrokenPipeError: - logger.error( + logger.warning( 'BrokenPipeError when flushing buffer for server', ) return self._close_and_release() @@ -857,7 +839,7 @@ def wrap_client(self) -> bool: ) do_close = True except BrokenPipeError: - logger.error( + logger.warning( 'BrokenPipeError when wrapping client for upstream: {0}'.format( self.upstream.addr[0], ), diff --git a/proxy/http/responses.py b/proxy/http/responses.py index 3eaca1e606..c1e8a17395 100644 --- a/proxy/http/responses.py +++ b/proxy/http/responses.py @@ -11,11 +11,10 @@ import gzip from typing import Any, Dict, Optional +from .codes import httpStatusCodes from ..common.flag import flags from ..common.utils import build_http_response -from ..common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY - -from .codes import httpStatusCodes +from ..common.constants import PROXY_AGENT_HEADER_KEY, PROXY_AGENT_HEADER_VALUE PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = memoryview( diff --git a/proxy/http/server/__init__.py b/proxy/http/server/__init__.py index 059c2cc128..dfbaa02c85 100644 --- a/proxy/http/server/__init__.py +++ b/proxy/http/server/__init__.py @@ -9,9 +9,10 @@ :license: BSD, see LICENSE for more details. """ from .web import HttpWebServerPlugin -from .pac_plugin import HttpWebServerPacFilePlugin from .plugin import HttpWebServerBasePlugin from .protocols import httpProtocolTypes +from .pac_plugin import HttpWebServerPacFilePlugin + __all__ = [ 'HttpWebServerPlugin', diff --git a/proxy/http/server/pac_plugin.py b/proxy/http/server/pac_plugin.py index 8cc0a4d0f0..a52f213dd7 100644 --- a/proxy/http/server/pac_plugin.py +++ b/proxy/http/server/pac_plugin.py @@ -12,16 +12,14 @@ pac """ -from typing import List, Tuple, Optional, Any +from typing import Any, List, Tuple, Optional from .plugin import HttpWebServerBasePlugin -from .protocols import httpProtocolTypes - from ..parser import HttpParser +from .protocols import httpProtocolTypes from ..responses import okResponse - -from ...common.utils import bytes_, text_ from ...common.flag import flags +from ...common.utils import text_, bytes_ from ...common.constants import DEFAULT_PAC_FILE, DEFAULT_PAC_FILE_URL_PATH diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index dde91a9afa..62ebd3e7dc 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -9,16 +9,15 @@ :license: BSD, see LICENSE for more details. """ import argparse - from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Optional -from ..websocket import WebsocketFrame from ..parser import HttpParser +from ..websocket import WebsocketFrame +from ..connection import HttpClientConnection +from ...core.event import EventQueue from ..descriptors import DescriptorsHandlerMixin -from ...core.connection import TcpClientConnection -from ...core.event import EventQueue if TYPE_CHECKING: # pragma: no cover from ...core.connection import UpstreamConnectionPool @@ -31,7 +30,7 @@ def __init__( self, uid: str, flags: argparse.Namespace, - client: TcpClientConnection, + client: HttpClientConnection, event_queue: EventQueue, upstream_conn_pool: Optional['UpstreamConnectionPool'] = None, ): diff --git a/proxy/http/server/protocols.py b/proxy/http/server/protocols.py index a561cb936f..84b5d8ac77 100644 --- a/proxy/http/server/protocols.py +++ b/proxy/http/server/protocols.py @@ -15,6 +15,7 @@ """ from typing import NamedTuple + HttpProtocolTypes = NamedTuple( 'HttpProtocolTypes', [ ('HTTP', int), diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index 5035217138..efc734cddf 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -13,25 +13,25 @@ import socket import logging import mimetypes +from typing import Any, Dict, List, Tuple, Union, Pattern, Optional -from typing import List, Optional, Dict, Tuple, Union, Any, Pattern - -from ...common.constants import DEFAULT_STATIC_SERVER_DIR -from ...common.constants import DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_ENABLE_WEB_SERVER -from ...common.constants import DEFAULT_MIN_COMPRESSION_LIMIT, DEFAULT_WEB_ACCESS_LOG_FORMAT -from ...common.utils import bytes_, text_, build_websocket_handshake_response -from ...common.types import Readables, Writables, Descriptors -from ...common.flag import flags - -from ..exception import HttpProtocolException -from ..plugin import HttpProtocolHandlerPlugin -from ..websocket import WebsocketFrame, websocketOpcodes +from .plugin import HttpWebServerBasePlugin from ..parser import HttpParser, httpParserTypes +from ..plugin import HttpProtocolHandlerPlugin +from .protocols import httpProtocolTypes +from ..exception import HttpProtocolException from ..protocols import httpProtocols from ..responses import NOT_FOUND_RESPONSE_PKT, okResponse +from ..websocket import WebsocketFrame, websocketOpcodes +from ...common.flag import flags +from ...common.types import Readables, Writables, Descriptors +from ...common.utils import text_, bytes_, build_websocket_handshake_response +from ...common.constants import ( + DEFAULT_ENABLE_WEB_SERVER, DEFAULT_STATIC_SERVER_DIR, + DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_MIN_COMPRESSION_LIMIT, + DEFAULT_WEB_ACCESS_LOG_FORMAT, +) -from .plugin import HttpWebServerBasePlugin -from .protocols import httpProtocolTypes logger = logging.getLogger(__name__) diff --git a/proxy/http/url.py b/proxy/http/url.py index fc06412b0f..c799efaa1a 100644 --- a/proxy/http/url.py +++ b/proxy/http/url.py @@ -13,10 +13,11 @@ http url """ -from typing import Optional, Tuple +from typing import List, Tuple, Optional -from ..common.constants import COLON, SLASH, HTTP_URL_PREFIX, HTTPS_URL_PREFIX, AT +from .exception import HttpProtocolException from ..common.utils import text_ +from ..common.constants import AT, COLON, SLASH, DEFAULT_ALLOWED_URL_SCHEMES class Url: @@ -59,7 +60,7 @@ def __str__(self) -> str: return url @classmethod - def from_bytes(cls, raw: bytes) -> 'Url': + def from_bytes(cls, raw: bytes, allowed_url_schemes: Optional[List[bytes]] = None) -> 'Url': """A URL within proxy.py core can have several styles, because proxy.py supports both proxy and web server use cases. @@ -68,29 +69,45 @@ def from_bytes(cls, raw: bytes) -> 'Url': For a HTTPS connect tunnel, url is like ``httpbin.org:443`` For a HTTP proxy request, url is like ``http://httpbin.org/get`` + proxy.py internally never expects a https scheme in the request line. + But `Url` class provides support for parsing any scheme present in the URLs. + e.g. ftp, icap etc. + + If a url with no scheme is parsed, e.g. ``//host/abc.js``, then scheme + defaults to `http`. + Further: 1) URL may contain unicode characters 2) URL may contain IPv4 and IPv6 format addresses instead of domain names - - We use heuristics based approach for our URL parser. """ # SLASH == 47, check if URL starts with single slash but not double slash - is_single_slash = raw[0] == 47 - is_double_slash = is_single_slash and len(raw) >= 2 and raw[1] == 47 - if is_single_slash and not is_double_slash: + starts_with_single_slash = raw[0] == 47 + starts_with_double_slash = starts_with_single_slash and \ + len(raw) >= 2 and \ + raw[1] == 47 + if starts_with_single_slash and \ + not starts_with_double_slash: return cls(remainder=raw) - is_http = raw.startswith(HTTP_URL_PREFIX) - is_https = raw.startswith(HTTPS_URL_PREFIX) - if is_http or is_https or is_double_slash: - rest = raw[len(b'https://'):] \ - if is_https \ - else raw[len(b'http://'):] \ - if is_http \ - else raw[len(SLASH + SLASH):] + scheme = None + rest = None + if not starts_with_double_slash: + # Find scheme + parts = raw.split(b'://', 1) + if len(parts) == 2: + scheme = parts[0] + rest = parts[1] + if scheme not in (allowed_url_schemes or DEFAULT_ALLOWED_URL_SCHEMES): + raise HttpProtocolException( + 'Invalid scheme received in the request line %r' % raw, + ) + else: + rest = raw[len(SLASH + SLASH):] + if scheme is not None or starts_with_double_slash: + assert rest is not None parts = rest.split(SLASH, 1) username, password, host, port = Url._parse(parts[0]) return cls( - scheme=b'https' if is_https else b'http', + scheme=scheme if not starts_with_double_slash else b'http', username=username, password=password, hostname=host, diff --git a/proxy/http/websocket/__init__.py b/proxy/http/websocket/__init__.py index 5017387395..a787535cac 100644 --- a/proxy/http/websocket/__init__.py +++ b/proxy/http/websocket/__init__.py @@ -19,6 +19,7 @@ from .client import WebsocketClient from .plugin import WebSocketTransportBasePlugin + __all__ = [ 'websocketOpcodes', 'WebsocketFrame', diff --git a/proxy/http/websocket/client.py b/proxy/http/websocket/client.py index 498df469bb..f398b5d827 100644 --- a/proxy/http/websocket/client.py +++ b/proxy/http/websocket/client.py @@ -13,16 +13,17 @@ import socket import secrets import selectors - -from typing import Optional, Union, Callable +from typing import Union, Callable, Optional from .frame import WebsocketFrame - -from ..parser import httpParserTypes, HttpParser - -from ...common.constants import DEFAULT_BUFFER_SIZE, DEFAULT_SELECTOR_SELECT_TIMEOUT -from ...common.utils import new_socket_connection, build_websocket_handshake_request, text_ -from ...core.connection import tcpConnectionTypes, TcpConnection +from ..parser import HttpParser, httpParserTypes +from ...common.utils import ( + text_, new_socket_connection, build_websocket_handshake_request, +) +from ...core.connection import TcpConnection, tcpConnectionTypes +from ...common.constants import ( + DEFAULT_BUFFER_SIZE, DEFAULT_SELECTOR_SELECT_TIMEOUT, +) class WebsocketClient(TcpConnection): diff --git a/proxy/http/websocket/frame.py b/proxy/http/websocket/frame.py index e17490591f..7cccbcb067 100644 --- a/proxy/http/websocket/frame.py +++ b/proxy/http/websocket/frame.py @@ -19,10 +19,9 @@ import base64 import struct import hashlib -import secrets import logging - -from typing import TypeVar, Type, Optional, NamedTuple +import secrets +from typing import Type, TypeVar, Optional, NamedTuple WebsocketOpcodes = NamedTuple( diff --git a/proxy/http/websocket/plugin.py b/proxy/http/websocket/plugin.py index f6881632a9..be544773a7 100644 --- a/proxy/http/websocket/plugin.py +++ b/proxy/http/websocket/plugin.py @@ -8,15 +8,18 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import argparse import json +import argparse from abc import ABC, abstractmethod -from typing import List, Dict, Any +from typing import TYPE_CHECKING, Any, Dict, List -from ...common.utils import bytes_ from . import WebsocketFrame -from ...core.connection import TcpClientConnection from ...core.event import EventQueue +from ...common.utils import bytes_ + + +if TYPE_CHECKING: # pragma: no cover + from ..connection import HttpClientConnection class WebSocketTransportBasePlugin(ABC): @@ -25,7 +28,7 @@ class WebSocketTransportBasePlugin(ABC): def __init__( self, flags: argparse.Namespace, - client: TcpClientConnection, + client: 'HttpClientConnection', event_queue: EventQueue, ) -> None: self.flags = flags diff --git a/proxy/http/websocket/transport.py b/proxy/http/websocket/transport.py index b18a8064e0..463195ef5c 100644 --- a/proxy/http/websocket/transport.py +++ b/proxy/http/websocket/transport.py @@ -10,15 +10,14 @@ """ import json import logging -from typing import List, Tuple, Any, Dict - -from ...common.utils import bytes_ - -from ..server import httpProtocolTypes, HttpWebServerBasePlugin -from ..parser import HttpParser +from typing import Any, Dict, List, Tuple from .frame import WebsocketFrame from .plugin import WebSocketTransportBasePlugin +from ..parser import HttpParser +from ..server import HttpWebServerBasePlugin, httpProtocolTypes +from ...common.utils import bytes_ + logger = logging.getLogger(__name__) diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index 578bc3bcc4..e76a253468 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -13,21 +13,22 @@ Cloudflare """ from .cache import CacheResponsesPlugin, BaseCacheResponsesPlugin -from .filter_by_upstream import FilterByUpstreamHostPlugin -from .man_in_the_middle import ManInTheMiddlePlugin +from .shortlink import ShortLinkPlugin +from .proxy_pool import ProxyPoolPlugin +from .program_name import ProgramNamePlugin from .mock_rest_api import ProposedRestApiPlugin +from .reverse_proxy import ReverseProxyPlugin +from .cloudflare_dns import CloudflareDnsResolverPlugin from .modify_post_data import ModifyPostDataPlugin -from .redirect_to_custom_server import RedirectToCustomServerPlugin -from .shortlink import ShortLinkPlugin from .web_server_route import WebServerPlugin -from .reverse_proxy import ReverseProxyPlugin -from .proxy_pool import ProxyPoolPlugin +from .man_in_the_middle import ManInTheMiddlePlugin +from .filter_by_upstream import FilterByUpstreamHostPlugin +from .custom_dns_resolver import CustomDnsResolverPlugin from .filter_by_client_ip import FilterByClientIpPlugin from .filter_by_url_regex import FilterByURLRegexPlugin from .modify_chunk_response import ModifyChunkResponsePlugin -from .custom_dns_resolver import CustomDnsResolverPlugin -from .cloudflare_dns import CloudflareDnsResolverPlugin -from .program_name import ProgramNamePlugin +from .redirect_to_custom_server import RedirectToCustomServerPlugin + __all__ = [ 'CacheResponsesPlugin', diff --git a/proxy/plugin/cache/__init__.py b/proxy/plugin/cache/__init__.py index f3bfb84b2c..ce6cfe4291 100644 --- a/proxy/plugin/cache/__init__.py +++ b/proxy/plugin/cache/__init__.py @@ -11,6 +11,7 @@ from .base import BaseCacheResponsesPlugin from .cache_responses import CacheResponsesPlugin + __all__ = [ 'BaseCacheResponsesPlugin', 'CacheResponsesPlugin', diff --git a/proxy/plugin/cache/base.py b/proxy/plugin/cache/base.py index 4451adf1a7..278f4eed2b 100644 --- a/proxy/plugin/cache/base.py +++ b/proxy/plugin/cache/base.py @@ -9,11 +9,12 @@ :license: BSD, see LICENSE for more details. """ import logging -from typing import Optional, Any +from typing import Any, Optional -from ...http.parser import HttpParser -from ...http.proxy import HttpProxyBasePlugin from .store.base import CacheStore +from ...http.proxy import HttpProxyBasePlugin +from ...http.parser import HttpParser + logger = logging.getLogger(__name__) diff --git a/proxy/plugin/cache/cache_responses.py b/proxy/plugin/cache/cache_responses.py index 200a60cb39..e9ccd126d6 100644 --- a/proxy/plugin/cache/cache_responses.py +++ b/proxy/plugin/cache/cache_responses.py @@ -11,8 +11,8 @@ import multiprocessing from typing import Any -from .store.disk import OnDiskCacheStore from .base import BaseCacheResponsesPlugin +from .store.disk import OnDiskCacheStore class CacheResponsesPlugin(BaseCacheResponsesPlugin): diff --git a/proxy/plugin/cache/store/disk.py b/proxy/plugin/cache/store/disk.py index d4e6b9dc39..8ea54b8c01 100644 --- a/proxy/plugin/cache/store/disk.py +++ b/proxy/plugin/cache/store/disk.py @@ -8,16 +8,16 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import logging import os +import logging import tempfile -from typing import Optional, BinaryIO +from typing import BinaryIO, Optional +from .base import CacheStore from ....common.flag import flags -from ....common.utils import text_ from ....http.parser import HttpParser +from ....common.utils import text_ -from .base import CacheStore logger = logging.getLogger(__name__) diff --git a/proxy/plugin/cloudflare_dns.py b/proxy/plugin/cloudflare_dns.py index 781cb8d489..9c459b56ab 100644 --- a/proxy/plugin/cloudflare_dns.py +++ b/proxy/plugin/cloudflare_dns.py @@ -16,15 +16,17 @@ """ import logging + try: import httpx except ImportError: # pragma: no cover pass -from typing import Optional, Tuple +from typing import Tuple, Optional -from ..common.flag import flags from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags + logger = logging.getLogger(__name__) diff --git a/proxy/plugin/custom_dns_resolver.py b/proxy/plugin/custom_dns_resolver.py index f40a93e8ed..5ee7aded91 100644 --- a/proxy/plugin/custom_dns_resolver.py +++ b/proxy/plugin/custom_dns_resolver.py @@ -13,8 +13,7 @@ dns """ import socket - -from typing import Optional, Tuple +from typing import Tuple, Optional from ..http.proxy import HttpProxyBasePlugin diff --git a/proxy/plugin/filter_by_client_ip.py b/proxy/plugin/filter_by_client_ip.py index fba981012d..2f19904284 100644 --- a/proxy/plugin/filter_by_client_ip.py +++ b/proxy/plugin/filter_by_client_ip.py @@ -14,11 +14,10 @@ """ from typing import Optional -from ..common.flag import flags - from ..http import httpStatusCodes -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags +from ..http.parser import HttpParser from ..http.exception import HttpRequestRejected diff --git a/proxy/plugin/filter_by_upstream.py b/proxy/plugin/filter_by_upstream.py index a257fdf286..39f70859f4 100644 --- a/proxy/plugin/filter_by_upstream.py +++ b/proxy/plugin/filter_by_upstream.py @@ -10,12 +10,11 @@ """ from typing import Optional -from ..common.utils import text_ -from ..common.flag import flags - from ..http import httpStatusCodes -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags +from ..http.parser import HttpParser +from ..common.utils import text_ from ..http.exception import HttpRequestRejected diff --git a/proxy/plugin/filter_by_url_regex.py b/proxy/plugin/filter_by_url_regex.py index 3d12ad342c..587c6f63b7 100644 --- a/proxy/plugin/filter_by_url_regex.py +++ b/proxy/plugin/filter_by_url_regex.py @@ -12,20 +12,18 @@ url """ +import re import json import logging - -from typing import Optional, List, Dict, Any - -from ..common.flag import flags -from ..common.utils import text_ +from typing import Any, Dict, List, Optional from ..http import httpStatusCodes -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..common.flag import flags +from ..http.parser import HttpParser +from ..common.utils import text_ from ..http.exception import HttpRequestRejected -import re logger = logging.getLogger(__name__) diff --git a/proxy/plugin/man_in_the_middle.py b/proxy/plugin/man_in_the_middle.py index d571f25a23..1ceb3a21b4 100644 --- a/proxy/plugin/man_in_the_middle.py +++ b/proxy/plugin/man_in_the_middle.py @@ -10,8 +10,8 @@ """ from typing import Optional -from ..http.responses import okResponse from ..http.proxy import HttpProxyBasePlugin +from ..http.responses import okResponse class ManInTheMiddlePlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/mock_rest_api.py b/proxy/plugin/mock_rest_api.py index 56cac4cb27..2588ce0592 100644 --- a/proxy/plugin/mock_rest_api.py +++ b/proxy/plugin/mock_rest_api.py @@ -15,11 +15,10 @@ import json from typing import Optional -from ..common.utils import bytes_, text_ - -from ..http.responses import okResponse, NOT_FOUND_RESPONSE_PKT -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser +from ..common.utils import text_, bytes_ +from ..http.responses import NOT_FOUND_RESPONSE_PKT, okResponse class ProposedRestApiPlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/modify_chunk_response.py b/proxy/plugin/modify_chunk_response.py index a0838da58d..16171e1f11 100644 --- a/proxy/plugin/modify_chunk_response.py +++ b/proxy/plugin/modify_chunk_response.py @@ -10,8 +10,8 @@ """ from typing import Any, Optional -from ..http.parser import HttpParser, httpParserTypes from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser, httpParserTypes class ModifyChunkResponsePlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/modify_post_data.py b/proxy/plugin/modify_post_data.py index 21be9d358d..d4b5ba6174 100644 --- a/proxy/plugin/modify_post_data.py +++ b/proxy/plugin/modify_post_data.py @@ -10,11 +10,10 @@ """ from typing import Optional -from ..common.utils import bytes_ - from ..http import httpMethods -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser +from ..common.utils import bytes_ class ModifyPostDataPlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/program_name.py b/proxy/plugin/program_name.py index b8e205b4df..0aaadcb817 100644 --- a/proxy/plugin/program_name.py +++ b/proxy/plugin/program_name.py @@ -10,15 +10,13 @@ """ import os import subprocess - from typing import Any, Dict, Optional +from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser from ..common.utils import text_ from ..common.constants import IS_WINDOWS -from ..http.parser import HttpParser -from ..http.proxy import HttpProxyBasePlugin - class ProgramNamePlugin(HttpProxyBasePlugin): """Tries to identify the application connecting to the diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py index 41ffe5b711..6e04034721 100644 --- a/proxy/plugin/proxy_pool.py +++ b/proxy/plugin/proxy_pool.py @@ -12,19 +12,19 @@ import random import logging import ipaddress +from typing import Any, Dict, List, Optional -from typing import Dict, List, Optional, Any - +from ..http import Url, httpHeaders, httpMethods +from ..core.base import TcpUpstreamConnectionHandler +from ..http.proxy import HttpProxyBasePlugin from ..common.flag import flags -from ..common.utils import text_, bytes_ -from ..common.constants import COLON, LOCAL_INTERFACE_HOSTNAMES, ANY_INTERFACE_HOSTNAMES - -from ..http import Url, httpMethods, httpHeaders from ..http.parser import HttpParser +from ..common.utils import text_, bytes_ from ..http.exception import HttpProtocolException -from ..http.proxy import HttpProxyBasePlugin +from ..common.constants import ( + COLON, ANY_INTERFACE_HOSTNAMES, LOCAL_INTERFACE_HOSTNAMES, +) -from ..core.base import TcpUpstreamConnectionHandler logger = logging.getLogger(__name__) diff --git a/proxy/plugin/redirect_to_custom_server.py b/proxy/plugin/redirect_to_custom_server.py index 82f4b0bfbf..d1c893bd9b 100644 --- a/proxy/plugin/redirect_to_custom_server.py +++ b/proxy/plugin/redirect_to_custom_server.py @@ -8,8 +8,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from urllib import parse as urlparse from typing import Optional +from urllib import parse as urlparse from ..http.proxy import HttpProxyBasePlugin from ..http.parser import HttpParser diff --git a/proxy/plugin/reverse_proxy.py b/proxy/plugin/reverse_proxy.py index 26ebbb6aae..e21442e7e7 100644 --- a/proxy/plugin/reverse_proxy.py +++ b/proxy/plugin/reverse_proxy.py @@ -10,18 +10,16 @@ """ import random import logging - -from typing import List, Tuple, Any, Dict, Optional - -from ..common.utils import text_ -from ..common.constants import DEFAULT_HTTPS_PORT, DEFAULT_HTTP_PORT +from typing import Any, Dict, List, Tuple, Optional from ..http import Url -from ..http.exception import HttpProtocolException +from ..core.base import TcpUpstreamConnectionHandler from ..http.parser import HttpParser from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes +from ..common.utils import text_ +from ..http.exception import HttpProtocolException +from ..common.constants import DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT -from ..core.base import TcpUpstreamConnectionHandler logger = logging.getLogger(__name__) diff --git a/proxy/plugin/shortlink.py b/proxy/plugin/shortlink.py index 11b7a7d823..0f5840ba08 100644 --- a/proxy/plugin/shortlink.py +++ b/proxy/plugin/shortlink.py @@ -14,11 +14,10 @@ """ from typing import Optional -from ..common.constants import DOT, SLASH - -from ..http.responses import NOT_FOUND_RESPONSE_PKT, seeOthersResponse -from ..http.parser import HttpParser from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser +from ..http.responses import NOT_FOUND_RESPONSE_PKT, seeOthersResponse +from ..common.constants import DOT, SLASH class ShortLinkPlugin(HttpProxyBasePlugin): diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index 5f881a68f7..205a8f9bf2 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -11,14 +11,15 @@ import logging from typing import List, Tuple -from ..http.responses import okResponse from ..http.parser import HttpParser from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes +from ..http.responses import okResponse + logger = logging.getLogger(__name__) HTTP_RESPONSE = okResponse(content=b'HTTP route response') -HTTPS_RESPONSE = okResponse(content=b'HTTP route response') +HTTPS_RESPONSE = okResponse(content=b'HTTPS route response') class WebServerPlugin(HttpWebServerBasePlugin): diff --git a/proxy/proxy.py b/proxy/proxy.py index b37e80a812..8f04b2af62 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -13,18 +13,20 @@ import time import signal import logging +from typing import Any, List, Optional -from typing import List, Optional, Any - +from .core.ssh import SshTunnelListener, SshHttpProtocolHandler from .core.work import ThreadlessPool from .core.event import EventManager -from .core.acceptor import AcceptorPool, Listener - -from .common.utils import bytes_ from .common.flag import FlagParser, flags -from .common.constants import DEFAULT_LOCAL_EXECUTOR, DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, IS_WINDOWS -from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PLUGINS, DEFAULT_VERSION -from .common.constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_WORK_KLASS, DEFAULT_PID_FILE +from .common.utils import bytes_ +from .core.acceptor import Listener, AcceptorPool +from .common.constants import ( + IS_WINDOWS, DEFAULT_PLUGINS, DEFAULT_VERSION, DEFAULT_LOG_FILE, + DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT, + DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_ENABLE_DASHBOARD, + DEFAULT_ENABLE_SSH_TUNNEL, +) logger = logging.getLogger(__name__) @@ -96,6 +98,13 @@ help='Default: False. Enables proxy.py dashboard.', ) +flags.add_argument( + '--enable-ssh-tunnel', + action='store_true', + default=DEFAULT_ENABLE_SSH_TUNNEL, + help='Default: False. Enable SSH tunnel.', +) + flags.add_argument( '--work-klass', type=str, @@ -135,6 +144,8 @@ def __init__(self, input_args: Optional[List[str]] = None, **opts: Any) -> None: self.executors: Optional[ThreadlessPool] = None self.acceptors: Optional[AcceptorPool] = None self.event_manager: Optional[EventManager] = None + self.ssh_http_protocol_handler: Optional[SshHttpProtocolHandler] = None + self.ssh_tunnel_listener: Optional[SshTunnelListener] = None def __enter__(self) -> 'Proxy': self.setup() @@ -193,10 +204,26 @@ def setup(self) -> None: event_queue=event_queue, ) self.acceptors.setup() + # Start SSH tunnel acceptor if enabled + if self.flags.enable_ssh_tunnel: + self.ssh_http_protocol_handler = SshHttpProtocolHandler( + flags=self.flags, + ) + self.ssh_tunnel_listener = SshTunnelListener( + flags=self.flags, + on_connection_callback=self.ssh_http_protocol_handler.on_connection, + ) + self.ssh_tunnel_listener.setup() + self.ssh_tunnel_listener.start_port_forward( + ('', self.flags.tunnel_remote_port), + ) # TODO: May be close listener fd as we don't need it now self._register_signals() def shutdown(self) -> None: + if self.flags.enable_ssh_tunnel: + assert self.ssh_tunnel_listener is not None + self.ssh_tunnel_listener.shutdown() assert self.acceptors self.acceptors.shutdown() if self.remote_executors_enabled: @@ -213,7 +240,7 @@ def shutdown(self) -> None: @property def remote_executors_enabled(self) -> bool: return self.flags.threadless and \ - not (self.flags.local_executor == int(DEFAULT_LOCAL_EXECUTOR)) + not self.flags.local_executor def _write_pid_file(self) -> None: if self.flags.pid_file: diff --git a/proxy/testing/__init__.py b/proxy/testing/__init__.py index e841545b30..f0b7ffe5af 100644 --- a/proxy/testing/__init__.py +++ b/proxy/testing/__init__.py @@ -10,6 +10,7 @@ """ from .test_case import TestCase + __all__ = [ 'TestCase', ] diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index f6d75ce0de..18c75216ce 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -8,15 +8,16 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import contextlib import time +import contextlib +from typing import Any, List, Optional, Generator + import unittest -from typing import Optional, List, Generator, Any from ..proxy import Proxy -from ..common.constants import DEFAULT_TIMEOUT -from ..common.utils import new_socket_connection from ..plugin import CacheResponsesPlugin +from ..common.utils import new_socket_connection +from ..common.constants import DEFAULT_TIMEOUT class TestCase(unittest.TestCase): diff --git a/tests/common/test_pki.py b/tests/common/test_pki.py index 2bbebe06bb..dfe794bee7 100644 --- a/tests/common/test_pki.py +++ b/tests/common/test_pki.py @@ -123,7 +123,7 @@ def test_sign_csr(self) -> None: def _gen_public_private_key(self) -> Tuple[str, str, str]: key_path, nopass_key_path = self._gen_private_key() crt_path = os.path.join(self._tempdir, 'test_gen_public.crt') - pki.gen_public_key(crt_path, key_path, 'password', '/CN=example.com') + pki.gen_public_key(crt_path, key_path, 'password', '/CN=localhost') return (key_path, nopass_key_path, crt_path) def _gen_private_key(self) -> Tuple[str, str]: diff --git a/tests/core/test_acceptor.py b/tests/core/test_acceptor.py index b18f640fa7..376186c539 100644 --- a/tests/core/test_acceptor.py +++ b/tests/core/test_acceptor.py @@ -24,9 +24,10 @@ class TestAcceptor(unittest.TestCase): def setUp(self) -> None: self.acceptor_id = 1 self.pipe = multiprocessing.Pipe() + self.work_klass = mock.MagicMock() self.flags = FlagParser.initialize( threaded=True, - work_klass=mock.MagicMock(), + work_klass=self.work_klass, local_executor=0, ) self.acceptor = Acceptor( @@ -63,7 +64,6 @@ def test_continues_when_no_events( sock.accept.assert_not_called() self.flags.work_klass.assert_not_called() - @mock.patch('proxy.core.work.threaded.TcpClientConnection') @mock.patch('threading.Thread') @mock.patch('selectors.DefaultSelector') @mock.patch('socket.fromfd') @@ -74,7 +74,6 @@ def test_accepts_client_from_server_socket( mock_fromfd: mock.Mock, mock_selector: mock.Mock, mock_thread: mock.Mock, - mock_client: mock.Mock, ) -> None: fileno = 10 conn = mock.MagicMock() @@ -99,7 +98,7 @@ def test_accepts_client_from_server_socket( type=socket.SOCK_STREAM, ) self.flags.work_klass.assert_called_with( - mock_client.return_value, + self.work_klass.create.return_value, flags=self.flags, event_queue=None, upstream_conn_pool=None, diff --git a/tests/core/test_conn_pool.py b/tests/core/test_conn_pool.py index 6547a0de81..3918372101 100644 --- a/tests/core/test_conn_pool.py +++ b/tests/core/test_conn_pool.py @@ -33,7 +33,10 @@ def test_acquire_and_retain_and_reacquire(self, mock_tcp_server_connection: mock mock_conn.closed = False # Acquire created, conn = pool.acquire(addr) - mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1]) + mock_tcp_server_connection.assert_called_once_with( + host=addr[0], + port=addr[1], + ) mock_conn.mark_inuse.assert_called_once() mock_conn.reset.assert_not_called() self.assertTrue(created) @@ -66,7 +69,10 @@ def test_closed_connections_are_removed_on_release( # Acquire created, conn = pool.acquire(addr) self.assertTrue(created) - mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1]) + mock_tcp_server_connection.assert_called_once_with( + host=addr[0], + port=addr[1], + ) self.assertEqual(conn, mock_conn) self.assertEqual(len(pool.pools[addr]), 1) self.assertTrue(conn in pool.pools[addr]) @@ -91,7 +97,10 @@ async def test_get_events(self, mocker: MockerFixture) -> None: mock_conn = mock_tcp_server_connection.return_value addr = mock_conn.addr pool.add(addr) - mock_tcp_server_connection.assert_called_once_with(addr[0], addr[1]) + mock_tcp_server_connection.assert_called_once_with( + host=addr[0], + port=addr[1], + ) mock_conn.connect.assert_called_once() events = await pool.get_events() print(events) diff --git a/tests/http/exceptions/test_http_proxy_auth_failed.py b/tests/http/exceptions/test_http_proxy_auth_failed.py index e0695bcdb0..9dfbe2142e 100644 --- a/tests/http/exceptions/test_http_proxy_auth_failed.py +++ b/tests/http/exceptions/test_http_proxy_auth_failed.py @@ -14,11 +14,10 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpHeaders +from proxy.http import HttpProtocolHandler, HttpClientConnection, httpHeaders from proxy.common.flag import FlagParser from proxy.common.utils import build_http_request from proxy.http.responses import PROXY_AUTH_FAILED_RESPONSE_PKT -from proxy.core.connection import TcpClientConnection from ...test_assertions import Assertions @@ -39,7 +38,7 @@ def _setUp(self, mocker: MockerFixture) -> None: ) self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/http/parser/test_http_parser.py b/tests/http/parser/test_http_parser.py index 69f7e6f9cb..68510415ea 100644 --- a/tests/http/parser/test_http_parser.py +++ b/tests/http/parser/test_http_parser.py @@ -16,6 +16,7 @@ bytes_, find_http_line, build_http_header, build_http_request, ) from proxy.http.exception import HttpProtocolException +from proxy.http.protocols import httpProtocols from proxy.http.responses import okResponse from proxy.common.constants import CRLF, HTTP_1_0 @@ -678,9 +679,7 @@ def test_is_http_1_1_keep_alive(self) -> None: ) self.assertTrue(self.parser.is_http_1_1_keep_alive) - def test_is_http_1_1_keep_alive_with_non_close_connection_header( - self, - ) -> None: + def test_is_http_1_1_keep_alive_with_non_close_connection_header(self) -> None: self.parser.parse( build_http_request( httpMethods.GET, b'/', @@ -811,3 +810,47 @@ def test_is_safe_against_malicious_requests(self) -> None: b'//198.98.53.25:1389/TomcatBypass/Command/Base64d2dldCA0Ni4xNjEuNTIuMzcvRXhwbG9pd' + b'C5zaDsgY2htb2QgK3ggRXhwbG9pdC5zaDsgLi9FeHBsb2l0LnNoOw==}', ) + + def test_parses_icap_protocol(self) -> None: + # Ref https://datatracker.ietf.org/doc/html/rfc3507 + self.parser.parse( + b'REQMOD icap://icap-server.net/server?arg=87 ICAP/1.0\r\n' + + b'Host: icap-server.net\r\n' + + b'Encapsulated: req-hdr=0, req-body=154' + + b'\r\n\r\n' + + b'POST /origin-resource/form.pl HTTP/1.1\r\n' + + b'Host: www.origin-server.com\r\n' + + b'Accept: text/html, text/plain\r\n' + + b'Accept-Encoding: compress\r\n' + + b'Cache-Control: no-cache\r\n' + + b'\r\n' + + b'1e\r\n' + + b'I am posting this information.\r\n' + + b'0\r\n' + + b'\r\n', + allowed_url_schemes=[b'icap'], + ) + self.assertEqual(self.parser.method, b'REQMOD') + assert self.parser._url is not None + self.assertEqual(self.parser._url.scheme, b'icap') + self.assertEqual( + self.parser.http_handler_protocol, + httpProtocols.UNKNOWN, + ) + + def test_cannot_parse_sip_protocol(self) -> None: + # Will fail to parse because of invalid host and port in the request line + # Our Url parser expects an integer port. + with self.assertRaises(ValueError): + self.parser.parse( + b'OPTIONS sip:nm SIP/2.0\r\n' + + b'Via: SIP/2.0/TCP nm;branch=foo\r\n' + + b'From: ;tag=root\r\nTo: \r\n' + + b'Call-ID: 50000\r\n' + + b'CSeq: 42 OPTIONS\r\n' + + b'Max-Forwards: 70\r\n' + + b'Content-Length: 0\r\n' + + b'Contact: \r\n' + + b'Accept: application/sdp\r\n' + + b'\r\n', + ) diff --git a/tests/http/proxy/test_http_proxy.py b/tests/http/proxy/test_http_proxy.py index 54789ff8a9..68f6bfd999 100644 --- a/tests/http/proxy/test_http_proxy.py +++ b/tests/http/proxy/test_http_proxy.py @@ -14,12 +14,11 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler +from proxy.http import HttpProtocolHandler, HttpClientConnection from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.common.utils import build_http_request from proxy.http.exception import HttpProtocolException -from proxy.core.connection import TcpClientConnection from proxy.common.constants import DEFAULT_HTTP_PORT @@ -43,7 +42,7 @@ def _setUp(self, mocker: MockerFixture) -> None: } self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py index 4acff4d8e7..e7e166f372 100644 --- a/tests/http/proxy/test_http_proxy_tls_interception.py +++ b/tests/http/proxy/test_http_proxy_tls_interception.py @@ -19,12 +19,12 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpMethods +from proxy.http import HttpProtocolHandler, HttpClientConnection, httpMethods from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.common.utils import bytes_, build_http_request from proxy.http.responses import PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT -from proxy.core.connection import TcpClientConnection, TcpServerConnection +from proxy.core.connection import TcpServerConnection from proxy.common.constants import DEFAULT_CA_FILE from ...test_assertions import Assertions @@ -88,7 +88,7 @@ def mock_connection() -> Any: } self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py index 6faf99f34b..a2425fd723 100644 --- a/tests/http/test_protocol_handler.py +++ b/tests/http/test_protocol_handler.py @@ -17,7 +17,7 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpHeaders +from proxy.http import HttpProtocolHandler, HttpClientConnection, httpHeaders from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates @@ -25,10 +25,9 @@ from proxy.common.plugins import Plugins from proxy.common.version import __version__ from proxy.http.responses import ( - BAD_GATEWAY_RESPONSE_PKT, PROXY_AUTH_FAILED_RESPONSE_PKT, - PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, + BAD_GATEWAY_RESPONSE_PKT, BAD_REQUEST_RESPONSE_PKT, + PROXY_AUTH_FAILED_RESPONSE_PKT, PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, ) -from proxy.core.connection import TcpClientConnection from proxy.common.constants import ( CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, ) @@ -68,7 +67,7 @@ def _setUp(self, mocker: MockerFixture) -> None: ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() @@ -101,7 +100,7 @@ async def test_proxy_authentication_failed(self) -> None: bytes_(PLUGIN_PROXY_AUTH), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags, + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() self._conn.recv.return_value = CRLF.join([ @@ -115,6 +114,34 @@ async def test_proxy_authentication_failed(self) -> None: PROXY_AUTH_FAILED_RESPONSE_PKT, ) + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_bails_out_for_unknown_schemes(self) -> None: + mock_selector_for_client_read(self) + self._conn.recv.return_value = CRLF.join([ + b'REQMOD icap://icap-server.net/server?arg=87 ICAP/1.0', + b'Host: icap-server.net', + CRLF, + ]) + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.work.buffer[0], + BAD_REQUEST_RESPONSE_PKT, + ) + + @pytest.mark.asyncio # type: ignore[misc] + async def test_proxy_bails_out_for_sip_request_lines(self) -> None: + mock_selector_for_client_read(self) + self._conn.recv.return_value = CRLF.join([ + b'OPTIONS sip:nm SIP/2.0', + b'Accept: application/sdp', + CRLF, + ]) + await self.protocol_handler._run_once() + self.assertEqual( + self.protocol_handler.work.buffer[0], + BAD_REQUEST_RESPONSE_PKT, + ) + class TestHttpProtocolHandler(Assertions): @@ -138,7 +165,7 @@ def _setUp(self, mocker: MockerFixture) -> None: ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() @@ -301,7 +328,7 @@ async def test_authenticated_proxy_http_get(self) -> None: ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags, + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() assert self.http_server_port is not None @@ -349,7 +376,7 @@ async def test_authenticated_proxy_http_tunnel(self) -> None: ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=flags, + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() diff --git a/tests/http/test_url.py b/tests/http/test_url.py index 0cfb8c667a..d246ca77f8 100644 --- a/tests/http/test_url.py +++ b/tests/http/test_url.py @@ -11,6 +11,7 @@ import unittest from proxy.http import Url +from proxy.http.exception import HttpProtocolException class TestUrl(unittest.TestCase): @@ -143,3 +144,19 @@ def test_no_scheme_suffix(self) -> None: self.assertEqual(url.remainder, b'/server?arg=87') self.assertEqual(url.username, None) self.assertEqual(url.password, None) + + def test_any_scheme_suffix(self) -> None: + url = Url.from_bytes( + b'icap://example-server.net/server?arg=87', + allowed_url_schemes=[b'icap'], + ) + self.assertEqual(url.scheme, b'icap') + self.assertEqual(url.hostname, b'example-server.net') + self.assertEqual(url.port, None) + self.assertEqual(url.remainder, b'/server?arg=87') + self.assertEqual(url.username, None) + self.assertEqual(url.password, None) + + def test_assert_raises_for_unknown_schemes(self) -> None: + with self.assertRaises(HttpProtocolException): + Url.from_bytes(b'icap://example-server.net/server?arg=87') diff --git a/tests/http/web/test_web_server.py b/tests/http/web/test_web_server.py index bae92dd918..e5bbabb39b 100644 --- a/tests/http/web/test_web_server.py +++ b/tests/http/web/test_web_server.py @@ -18,13 +18,12 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler +from proxy.http import HttpProtocolHandler, HttpClientConnection from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates from proxy.common.utils import bytes_, build_http_request, build_http_response from proxy.common.plugins import Plugins from proxy.http.responses import NOT_FOUND_RESPONSE_PKT -from proxy.core.connection import TcpClientConnection from proxy.common.constants import ( CRLF, PROXY_PY_DIR, PLUGIN_PAC_FILE, PLUGIN_HTTP_PROXY, PLUGIN_WEB_SERVER, ) @@ -48,7 +47,7 @@ def test_on_client_connection_called_on_teardown(mocker: MockerFixture) -> None: _conn = mock_fromfd.return_value _addr = ('127.0.0.1', 54382) protocol_handler = HttpProtocolHandler( - TcpClientConnection(_conn, _addr), + HttpClientConnection(_conn, _addr), flags=flags, ) protocol_handler.initialize() @@ -81,7 +80,7 @@ def mock_selector_for_client_read(self: Any) -> None: # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} # self._conn = mock_fromfd.return_value # self.protocol_handler = HttpProtocolHandler( -# TcpClientConnection(self._conn, self._addr), +# HttpClientConnection(self._conn, self._addr), # flags=flags, # ) # self.protocol_handler.initialize() @@ -101,7 +100,7 @@ def mock_selector_for_client_read(self: Any) -> None: # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} # self._conn = mock_fromfd.return_value # self.protocol_handler = HttpProtocolHandler( -# TcpClientConnection(self._conn, self._addr), +# HttpClientConnection(self._conn, self._addr), # flags=flags, # ) # self.protocol_handler.initialize() @@ -121,7 +120,7 @@ def mock_selector_for_client_read(self: Any) -> None: # flags.plugins = {b'HttpProtocolHandlerPlugin': [plugin]} # self._conn = mock_fromfd.return_value # self.protocol_handler = HttpProtocolHandler( -# TcpClientConnection(self._conn, self._addr), +# HttpClientConnection(self._conn, self._addr), # flags=flags, # ) # self.protocol_handler.initialize() @@ -162,7 +161,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: bytes_(PLUGIN_PAC_FILE), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() @@ -221,7 +220,7 @@ def _setUp(self, mocker: MockerFixture) -> None: bytes_(PLUGIN_WEB_SERVER), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() @@ -328,7 +327,7 @@ def _setUp(self, mocker: MockerFixture) -> None: bytes_(PLUGIN_WEB_SERVER), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() @@ -353,7 +352,7 @@ async def test_default_web_server_returns_404(self) -> None: bytes_(PLUGIN_WEB_SERVER), ]) self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=flags, ) self.protocol_handler.initialize() diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index a6f0ed4ed9..93a88b731d 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -11,17 +11,30 @@ Test the simplest proxy use scenario for smoke. """ import time -import pytest import tempfile import subprocess - +from typing import Any, List, Generator from pathlib import Path -from typing import Any, Generator -from subprocess import Popen, check_output +from subprocess import Popen +from subprocess import check_output as _check_output + +import pytest from proxy.common.constants import IS_WINDOWS +def check_output(args: List[Any]) -> bytes: + args = args if not IS_WINDOWS else ['powershell'] + args + return _check_output(args) + + +def _https_server_flags() -> str: + return ' '.join(( + '--key-file', 'https-key.pem', + '--cert-file', 'https-signed-cert.pem', + )) + + def _tls_interception_flags(ca_cert_suffix: str = '') -> str: return ' '.join(( '--ca-cert-file', 'ca-cert%s.pem' % ca_cert_suffix, @@ -30,50 +43,90 @@ def _tls_interception_flags(ca_cert_suffix: str = '') -> str: )) -PROXY_PY_FLAGS_INTEGRATION = ( - ('--threadless'), - ('--threadless --local-executor 0'), +_PROXY_PY_FLAGS_INTEGRATION = [ ('--threaded'), -) +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_INTEGRATION += [ + ('--threadless --local-executor 0'), + ('--threadless'), + ] +PROXY_PY_FLAGS_INTEGRATION = tuple(_PROXY_PY_FLAGS_INTEGRATION) -PROXY_PY_FLAGS_TLS_INTERCEPTION = ( - ('--threadless ' + _tls_interception_flags()), - ('--threadless --local-executor 0 ' + _tls_interception_flags()), +_PROXY_PY_HTTPS = [ + ('--threaded ' + _https_server_flags()), +] +if not IS_WINDOWS: + _PROXY_PY_HTTPS += [ + ('--threadless --local-executor 0 ' + _https_server_flags()), + ('--threadless ' + _https_server_flags()), + ] +PROXY_PY_HTTPS = tuple(_PROXY_PY_HTTPS) + +_PROXY_PY_FLAGS_TLS_INTERCEPTION = [ ('--threaded ' + _tls_interception_flags()), -) +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_TLS_INTERCEPTION += [ + ('--threadless --local-executor 0 ' + _tls_interception_flags()), + ('--threadless ' + _tls_interception_flags()), + ] +PROXY_PY_FLAGS_TLS_INTERCEPTION = tuple(_PROXY_PY_FLAGS_TLS_INTERCEPTION) -PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = ( - ( - '--threadless --plugin proxy.plugin.ModifyChunkResponsePlugin ' + - _tls_interception_flags('-chunk') - ), - ( - '--threadless --local-executor 0 --plugin proxy.plugin.ModifyChunkResponsePlugin ' + - _tls_interception_flags('-chunk') - ), +_PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = [ ( '--threaded --plugin proxy.plugin.ModifyChunkResponsePlugin ' + _tls_interception_flags('-chunk') ), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN += [ + ( + '--threadless --local-executor 0 --plugin proxy.plugin.ModifyChunkResponsePlugin ' + + _tls_interception_flags('-chunk') + ), + ( + '--threadless --plugin proxy.plugin.ModifyChunkResponsePlugin ' + + _tls_interception_flags('-chunk') + ), + ] +PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = tuple( + _PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN, ) -PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = ( - ( - '--threadless --plugin proxy.plugin.ModifyPostDataPlugin ' + - _tls_interception_flags('-post') - ), - ( - '--threadless --local-executor 0 --plugin proxy.plugin.ModifyPostDataPlugin ' + - _tls_interception_flags('-post') - ), +_PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = [ ( '--threaded --plugin proxy.plugin.ModifyPostDataPlugin ' + _tls_interception_flags('-post') ), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN += [ + ( + '--threadless --local-executor 0 --plugin proxy.plugin.ModifyPostDataPlugin ' + + _tls_interception_flags('-post') + ), + ( + '--threadless --plugin proxy.plugin.ModifyPostDataPlugin ' + + _tls_interception_flags('-post') + ), + ] +PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = tuple( + _PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN, ) -@pytest.fixture(scope='session', autouse=True) # type: ignore[misc] +@pytest.fixture(scope='session', autouse=not IS_WINDOWS) # type: ignore[misc] +def _gen_https_certificates(request: Any) -> None: + check_output([ + 'make', 'https-certificates', + ]) + check_output([ + 'make', 'sign-https-certificates', + ]) + + +@pytest.fixture(scope='session', autouse=not IS_WINDOWS) # type: ignore[misc] def _gen_ca_certificates(request: Any) -> None: check_output([ 'make', 'ca-certificates', @@ -111,6 +164,7 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]: '--port', '0', '--port-file', str(port_file), '--enable-web-server', + '--plugin', 'proxy.plugin.WebServerPlugin', '--num-acceptors', '3', '--num-workers', '3', '--ca-cert-dir', str(ca_cert_dir), @@ -149,6 +203,24 @@ def test_integration(proxy_py_subprocess: int) -> None: check_output([str(shell_script_test), str(proxy_py_subprocess)]) +@pytest.mark.smoke # type: ignore[misc] +@pytest.mark.parametrize( + 'proxy_py_subprocess', + PROXY_PY_HTTPS, + indirect=True, +) # type: ignore[misc] +@pytest.mark.skipif( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', +) # type: ignore[misc] +def test_https_integration(proxy_py_subprocess: int) -> None: + """An acceptance test for HTTPS web and proxy server using ``curl`` through proxy.py.""" + this_test_module = Path(__file__) + shell_script_test = this_test_module.with_suffix('.sh') + # "1" means use-https scheme for requests to instance + check_output([str(shell_script_test), str(proxy_py_subprocess), '1']) + + @pytest.mark.smoke # type: ignore[misc] @pytest.mark.parametrize( 'proxy_py_subprocess', diff --git a/tests/integration/test_integration.sh b/tests/integration/test_integration.sh index 77c6cc8eba..06cafd2dc2 100755 --- a/tests/integration/test_integration.sh +++ b/tests/integration/test_integration.sh @@ -23,7 +23,20 @@ if [[ -z "$PROXY_PY_PORT" ]]; then exit 1 fi -PROXY_URL="127.0.0.1:$PROXY_PY_PORT" +PROXY_URL="http://localhost:$PROXY_PY_PORT" +TEST_URL="$PROXY_URL/http-route-example" +CURL_EXTRA_FLAGS="" +USE_HTTPS=$2 +if [[ ! -z "$USE_HTTPS" ]]; then + PROXY_URL="https://localhost:$PROXY_PY_PORT" + CURL_EXTRA_FLAGS=" -k --proxy-insecure " + # For https instances we don't use internal https web server + # See https://github.com/abhinavsingh/proxy.py/issues/994 + TEST_URL="http://google.com" + USE_HTTPS=true +else + USE_HTTPS=false +fi # Wait for server to come up WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '" @@ -37,12 +50,9 @@ while true; do done # Wait for http proxy and web server to start +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL" while true; do - curl -v \ - --max-time 1 \ - --connect-timeout 1 \ - -x $PROXY_URL \ - http://$PROXY_URL/ 2>/dev/null + RESPONSE=$($CMD 2> /dev/null) if [[ $? == 0 ]]; then break fi @@ -80,22 +90,27 @@ Disallow: /deny EOM echo "[Test HTTP Request via Proxy]" -CMD="curl -v -x $PROXY_URL http://httpbin.org/robots.txt" +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt" RESPONSE=$($CMD 2> /dev/null) verify_response "$RESPONSE" "$ROBOTS_RESPONSE" VERIFIED1=$? echo "[Test HTTPS Request via Proxy]" -CMD="curl -v -x $PROXY_URL https://httpbin.org/robots.txt" +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt" RESPONSE=$($CMD 2> /dev/null) verify_response "$RESPONSE" "$ROBOTS_RESPONSE" VERIFIED2=$? -echo "[Test Internal Web Server via Proxy]" -curl -v \ - -x $PROXY_URL \ - http://$PROXY_URL/ -VERIFIED3=$? +if $USE_HTTPS; then + VERIFIED3=0 +else + echo "[Test Internal Web Server via Proxy]" + curl -v \ + $CURL_EXTRA_FLAGS \ + -x $PROXY_URL \ + "$PROXY_URL" + VERIFIED3=$? +fi SHASUM=sha256sum if [ "$(uname)" = "Darwin" ]; @@ -107,6 +122,7 @@ echo "[Test Download File Hash Verifies 1]" touch downloaded.hash echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc -" > downloaded.hash curl -vL \ + $CURL_EXTRA_FLAGS \ -o downloaded.whl \ -x $PROXY_URL \ https://files.pythonhosted.org/packages/88/78/e642316313b1cd6396e4b85471a316e003eff968f29773e95ea191ea1d08/proxy.py-2.4.0rc4-py3-none-any.whl#sha256=3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc @@ -118,6 +134,7 @@ echo "[Test Download File Hash Verifies 2]" touch downloaded.hash echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 -" > downloaded.hash curl -vL \ + $CURL_EXTRA_FLAGS \ -o downloaded.whl \ -x $PROXY_URL \ https://files.pythonhosted.org/packages/20/9a/e5d9ec41927401e41aea8af6d16e78b5e612bca4699d417f646a9610a076/Jinja2-3.0.3-py3-none-any.whl#sha256=077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py index 3550c3295e..5b9e30d12f 100644 --- a/tests/plugin/test_http_proxy_plugins.py +++ b/tests/plugin/test_http_proxy_plugins.py @@ -20,13 +20,14 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpStatusCodes +from proxy.http import ( + HttpProtocolHandler, HttpClientConnection, httpStatusCodes, +) from proxy.plugin import ProposedRestApiPlugin, RedirectToCustomServerPlugin from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes from proxy.common.utils import bytes_, build_http_request, build_http_response -from proxy.core.connection import TcpClientConnection from proxy.common.constants import DEFAULT_HTTP_PORT, PROXY_AGENT_HEADER_VALUE from .utils import get_plugin_by_test_name from ..test_assertions import Assertions @@ -64,7 +65,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: } self._conn = self.mock_fromfd.return_value self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index 55d09b62a6..77345a41d5 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -18,7 +18,7 @@ from pytest_mock import MockerFixture -from proxy.http import HttpProtocolHandler, httpMethods +from proxy.http import HttpProtocolHandler, HttpClientConnection, httpMethods from proxy.http.proxy import HttpProxyPlugin from proxy.common.flag import FlagParser from proxy.http.parser import HttpParser, httpParserTypes @@ -26,7 +26,7 @@ from proxy.http.responses import ( PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, okResponse, ) -from proxy.core.connection import TcpClientConnection, TcpServerConnection +from proxy.core.connection import TcpServerConnection from .utils import get_plugin_by_test_name from ..test_assertions import Assertions @@ -71,7 +71,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: self._conn = mocker.MagicMock(spec=socket.socket) self.mock_fromfd.return_value = self._conn self.protocol_handler = HttpProtocolHandler( - TcpClientConnection(self._conn, self._addr), flags=self.flags, + HttpClientConnection(self._conn, self._addr), flags=self.flags, ) self.protocol_handler.initialize() diff --git a/tests/test_main.py b/tests/test_main.py index 8ac149fb53..f3b7a85ba0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -17,20 +17,21 @@ from proxy.proxy import main, entry_point from proxy.common.utils import bytes_ from proxy.common.constants import ( # noqa: WPS450 - DEFAULT_CA_CERT_DIR, DEFAULT_PORT, DEFAULT_PLUGINS, DEFAULT_TIMEOUT, DEFAULT_KEY_FILE, + DEFAULT_PORT, DEFAULT_PLUGINS, DEFAULT_TIMEOUT, DEFAULT_KEY_FILE, DEFAULT_LOG_FILE, DEFAULT_PAC_FILE, DEFAULT_PID_FILE, PLUGIN_DASHBOARD, DEFAULT_CERT_FILE, DEFAULT_LOG_LEVEL, DEFAULT_PORT_FILE, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, DEFAULT_BASIC_AUTH, DEFAULT_LOG_FORMAT, DEFAULT_THREADLESS, DEFAULT_WORK_KLASS, - DEFAULT_CA_KEY_FILE, DEFAULT_NUM_WORKERS, DEFAULT_CA_CERT_FILE, - DEFAULT_ENABLE_EVENTS, DEFAULT_IPV6_HOSTNAME, DEFAULT_NUM_ACCEPTORS, - DEFAULT_LOCAL_EXECUTOR, PLUGIN_INSPECT_TRAFFIC, DEFAULT_ENABLE_DEVTOOLS, - DEFAULT_OPEN_FILE_LIMIT, DEFAULT_DEVTOOLS_WS_PATH, + DEFAULT_CA_CERT_DIR, DEFAULT_CA_KEY_FILE, DEFAULT_NUM_WORKERS, + DEFAULT_CA_CERT_FILE, DEFAULT_ENABLE_EVENTS, DEFAULT_IPV6_HOSTNAME, + DEFAULT_NUM_ACCEPTORS, DEFAULT_LOCAL_EXECUTOR, PLUGIN_INSPECT_TRAFFIC, + DEFAULT_ENABLE_DEVTOOLS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_ENABLE_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, - DEFAULT_ENABLE_WEB_SERVER, DEFAULT_DISABLE_HTTP_PROXY, - PLUGIN_WEBSOCKET_TRANSPORT, DEFAULT_CA_SIGNING_KEY_FILE, - DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_SERVER_RECVBUF_SIZE, - DEFAULT_ENABLE_STATIC_SERVER, _env_threadless_compliant, + DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_ENABLE_WEB_SERVER, + DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_WEBSOCKET_TRANSPORT, + DEFAULT_CA_SIGNING_KEY_FILE, DEFAULT_CLIENT_RECVBUF_SIZE, + DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_ENABLE_STATIC_SERVER, + _env_threadless_compliant, ) @@ -75,6 +76,7 @@ def mock_default_args(mock_args: mock.Mock) -> None: mock_args.work_klass = DEFAULT_WORK_KLASS mock_args.local_executor = int(DEFAULT_LOCAL_EXECUTOR) mock_args.port_file = DEFAULT_PORT_FILE + mock_args.enable_ssh_tunnel = DEFAULT_ENABLE_SSH_TUNNEL @mock.patch('os.remove') @mock.patch('os.path.exists') @@ -103,6 +105,7 @@ def test_entry_point( mock_initialize.return_value.enable_events = False mock_initialize.return_value.pid_file = pid_file mock_initialize.return_value.port_file = None + mock_initialize.return_value.enable_ssh_tunnel = False entry_point() mock_event_manager.assert_not_called() mock_listener.assert_called_once_with( @@ -151,6 +154,7 @@ def test_main_with_no_flags( mock_initialize.return_value.local_executor = 0 mock_initialize.return_value.enable_events = False mock_initialize.return_value.port_file = None + mock_initialize.return_value.enable_ssh_tunnel = False main() mock_event_manager.assert_not_called() mock_listener.assert_called_once_with( @@ -192,6 +196,7 @@ def test_enable_events( mock_initialize.return_value.local_executor = 0 mock_initialize.return_value.enable_events = True mock_initialize.return_value.port_file = None + mock_initialize.return_value.enable_ssh_tunnel = False main() mock_event_manager.assert_called_once() mock_event_manager.return_value.setup.assert_called_once() @@ -304,11 +309,51 @@ def test_enable_devtools( mock_acceptor_pool.return_value.setup.assert_called_once() mock_listener.return_value.setup.assert_called_once() - # def test_pac_file(self) -> None: - # pass - - # def test_imports_plugin(self) -> None: - # pass - - # def test_cannot_enable_https_proxy_and_tls_interception_mutually(self) -> None: - # pass + @mock.patch('time.sleep') + @mock.patch('proxy.common.plugins.Plugins.load') + @mock.patch('proxy.common.flag.FlagParser.parse_args') + @mock.patch('proxy.proxy.EventManager') + @mock.patch('proxy.proxy.AcceptorPool') + @mock.patch('proxy.proxy.ThreadlessPool') + @mock.patch('proxy.proxy.Listener') + @mock.patch('proxy.proxy.SshHttpProtocolHandler') + @mock.patch('proxy.proxy.SshTunnelListener') + def test_enable_ssh_tunnel( + self, + mock_ssh_tunnel_listener: mock.Mock, + mock_ssh_http_proto_handler: mock.Mock, + mock_listener: mock.Mock, + mock_executor_pool: mock.Mock, + mock_acceptor_pool: mock.Mock, + mock_event_manager: mock.Mock, + mock_parse_args: mock.Mock, + mock_load_plugins: mock.Mock, + mock_sleep: mock.Mock, + ) -> None: + mock_sleep.side_effect = KeyboardInterrupt() + mock_args = mock_parse_args.return_value + self.mock_default_args(mock_args) + mock_args.enable_ssh_tunnel = True + mock_args.local_executor = 0 + main(enable_ssh_tunnel=True, local_executor=0) + mock_load_plugins.assert_called() + self.assertEqual( + mock_load_plugins.call_args_list[0][0][0], [ + bytes_(PLUGIN_HTTP_PROXY), + ], + ) + mock_parse_args.assert_called_once() + mock_event_manager.assert_not_called() + if _env_threadless_compliant(): + mock_executor_pool.assert_called_once() + mock_executor_pool.return_value.setup.assert_called_once() + mock_acceptor_pool.assert_called_once() + mock_acceptor_pool.return_value.setup.assert_called_once() + mock_listener.return_value.setup.assert_called_once() + mock_ssh_http_proto_handler.assert_called_once() + mock_ssh_tunnel_listener.assert_called_once() + mock_ssh_tunnel_listener.return_value.setup.assert_called_once() + mock_ssh_tunnel_listener.return_value.start_port_forward.assert_called_once() + mock_ssh_tunnel_listener.return_value.shutdown.assert_called_once() + # shutdown will internally call stop port forward + mock_ssh_tunnel_listener.return_value.stop_port_forward.assert_not_called()