From f84ecc121ca9d3d1269d7e2182356364dcaf4c9d Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 6 Jan 2020 20:32:55 -0800 Subject: [PATCH 1/3] Add py.typed marker --- MANIFEST.in | 1 + README.md | 13 +++++++++++++ proxy/py.typed | 1 + setup.py | 10 ++++++---- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 proxy/py.typed diff --git a/MANIFEST.in b/MANIFEST.in index 7b741275f0..e58675772d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include LICENSE include README.md include requirements.txt +recursive-exclude * __pycache__ diff --git a/README.md b/README.md index 08841ae868..83fceed35d 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Table of Contents * [API Usage](#api-usage) * [CLI Usage](#cli-usage) * [Frequently Asked Questions](#frequently-asked-questions) + * [Threads vs Threadless](#threads-vs-threadless) * [SyntaxError: invalid syntax](#syntaxerror-invalid-syntax) * [Unable to load plugins](#unable-to-load-plugins) * [Unable to connect with proxy.py from remote host](#unable-to-connect-with-proxypy-from-remote-host) @@ -1293,6 +1294,18 @@ FILE Frequently Asked Questions ========================== +## Threads vs Threadless + +Pre v2.x, `proxy.py` used to spawn new threads for handling +client requests. + +Starting v2.x, `proxy.py` added support for threadless execution of +client requests using `asyncio`. + +In future, threadless execution will be the default mode. +Till then if you are interested in trying it out, +start `proxy.py` with `--threadless` flag. + ## SyntaxError: invalid syntax Make sure you are using `Python 3`. Verify the version before running `proxy.py`: diff --git a/proxy/py.typed b/proxy/py.typed new file mode 100644 index 0000000000..d5b3fa90fa --- /dev/null +++ b/proxy/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. The proxy package uses inline types. diff --git a/setup.py b/setup.py index 5e7b9d98e0..8199cc28bb 100644 --- a/setup.py +++ b/setup.py @@ -10,9 +10,10 @@ """ from setuptools import setup, find_packages -VERSION = (2, 0, 0) +VERSION = (2, 1, 0) __version__ = '.'.join(map(str, VERSION[0:3])) -__description__ = '⚡⚡⚡ Fast, Lightweight, Programmable Proxy Server in a single Python file.' +__description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server + focused on Network monitoring, controls & Application development, testing, debugging.''' __author__ = 'Abhinav Singh' __author_email__ = 'mailsforabhinav@gmail.com' __homepage__ = 'https://github.com/abhinavsingh/proxy.py' @@ -31,8 +32,9 @@ download_url=__download_url__, license=__license__, python_requires='!=2.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', - zip_safe=True, - packages=find_packages(exclude=["tests", "tests.*"]), + zip_safe=False, + packages=find_packages(exclude=['tests', 'tests.*']), + package_data={'proxy': ['py.typed']}, install_requires=open('requirements.txt', 'r').read().strip().split(), entry_points={ 'console_scripts': [ From 07917cac57021eb31207651b8933acf8abb32657 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 6 Jan 2020 21:01:48 -0800 Subject: [PATCH 2/3] Add version-check.py --- MANIFEST.in | 1 - Makefile | 12 ++-- setup.py | 155 ++++++++++++++++++++++++----------------------- version-check.py | 22 +++++++ 4 files changed, 108 insertions(+), 82 deletions(-) create mode 100644 version-check.py diff --git a/MANIFEST.in b/MANIFEST.in index e58675772d..7b741275f0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ include LICENSE include README.md include requirements.txt -recursive-exclude * __pycache__ diff --git a/Makefile b/Makefile index d47a5543c4..17152b4e8f 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,12 @@ CA_CERT_FILE_PATH := ca-cert.pem CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem .PHONY: all https-certificates ca-certificates autopep8 devtools -.PHONY: lib-clean lib-test lib-package lib-release-test lib-release lib-coverage lib-lint lib-profile +.PHONY: lib-version lib-clean lib-test lib-package lib-coverage lib-lint +.PHONY: lib-release-test lib-release lib-profile .PHONY: container container-run container-release .PHONY: dashboard dashboard-clean -all: lib-clean lib-test +all: lib-test devtools: pushd dashboard && npm run devtools && popd @@ -56,6 +57,9 @@ ca-certificates: python -m proxy.common.pki remove_passphrase \ --private-key-path $(CA_SIGNING_KEY_FILE_PATH) +lib-version: + python version-check.py + lib-clean: find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + @@ -72,10 +76,10 @@ lib-lint: flake8 --ignore=W504 --max-line-length=127 --max-complexity=19 proxy/ tests/ setup.py mypy --strict --ignore-missing-imports proxy/ tests/ setup.py -lib-test: lib-lint +lib-test: lib-clean lib-version lib-lint pytest -v tests/ -lib-package: lib-clean +lib-package: lib-clean lib-version python setup.py sdist lib-release-test: lib-package diff --git a/setup.py b/setup.py index 8199cc28bb..8ff5ec1f2f 100644 --- a/setup.py +++ b/setup.py @@ -20,81 +20,82 @@ __download_url__ = '%s/archive/master.zip' % __homepage__ __license__ = 'BSD' -setup( - name='proxy.py', - version=__version__, - author=__author__, - author_email=__author_email__, - url=__homepage__, - description=__description__, - long_description=open('README.md', 'r', encoding='utf-8').read().strip(), - long_description_content_type='text/markdown', - download_url=__download_url__, - license=__license__, - python_requires='!=2.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', - zip_safe=False, - packages=find_packages(exclude=['tests', 'tests.*']), - package_data={'proxy': ['py.typed']}, - install_requires=open('requirements.txt', 'r').read().strip().split(), - entry_points={ - 'console_scripts': [ - 'proxy = proxy:entry_point' - ] - }, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Environment :: No Input/Output (Daemon)', - 'Environment :: Web Environment', - 'Environment :: MacOS X', - 'Environment :: Plugins', - 'Environment :: Win32 (MS Windows)', - 'Framework :: Robot Framework', - 'Framework :: Robot Framework :: Library', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: System Administrators', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Operating System :: MacOS', - 'Operating System :: MacOS :: MacOS 9', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX', - 'Operating System :: POSIX :: Linux', - 'Operating System :: Unix', - 'Operating System :: Microsoft', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: Microsoft :: Windows :: Windows 10', - 'Operating System :: Android', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: Implementation', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Topic :: Internet', - 'Topic :: Internet :: Proxy Servers', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Internet :: WWW/HTTP :: Browsers', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries', - 'Topic :: Internet :: WWW/HTTP :: HTTP Servers', - 'Topic :: Scientific/Engineering :: Information Analysis', - 'Topic :: Software Development :: Debuggers', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: System :: Monitoring', - 'Topic :: System :: Networking', - 'Topic :: System :: Networking :: Firewalls', - 'Topic :: System :: Networking :: Monitoring', - 'Topic :: Utilities', - 'Typing :: Typed', - ], - keywords=( - 'http, proxy, http proxy server, proxy server, http server,' - 'http web server, proxy framework, web framework, Python3' +if __name__ == '__main__': + setup( + name='proxy.py', + version=__version__, + author=__author__, + author_email=__author_email__, + url=__homepage__, + description=__description__, + long_description=open('README.md', 'r', encoding='utf-8').read().strip(), + long_description_content_type='text/markdown', + download_url=__download_url__, + license=__license__, + python_requires='!=2.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + zip_safe=False, + packages=find_packages(exclude=['tests', 'tests.*']), + package_data={'proxy': ['py.typed']}, + install_requires=open('requirements.txt', 'r').read().strip().split(), + entry_points={ + 'console_scripts': [ + 'proxy = proxy:entry_point' + ] + }, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Environment :: No Input/Output (Daemon)', + 'Environment :: Web Environment', + 'Environment :: MacOS X', + 'Environment :: Plugins', + 'Environment :: Win32 (MS Windows)', + 'Framework :: Robot Framework', + 'Framework :: Robot Framework :: Library', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: End Users/Desktop', + 'Intended Audience :: System Administrators', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: MacOS', + 'Operating System :: MacOS :: MacOS 9', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX', + 'Operating System :: POSIX :: Linux', + 'Operating System :: Unix', + 'Operating System :: Microsoft', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: Microsoft :: Windows :: Windows 10', + 'Operating System :: Android', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: Implementation', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Topic :: Internet', + 'Topic :: Internet :: Proxy Servers', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Internet :: WWW/HTTP :: Browsers', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries', + 'Topic :: Internet :: WWW/HTTP :: HTTP Servers', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Topic :: Software Development :: Debuggers', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: System :: Monitoring', + 'Topic :: System :: Networking', + 'Topic :: System :: Networking :: Firewalls', + 'Topic :: System :: Networking :: Monitoring', + 'Topic :: Utilities', + 'Typing :: Typed', + ], + keywords=( + 'http, proxy, http proxy server, proxy server, http server,' + 'http web server, proxy framework, web framework, Python3' + ) ) -) diff --git a/version-check.py b/version-check.py new file mode 100644 index 0000000000..6732a6e964 --- /dev/null +++ b/version-check.py @@ -0,0 +1,22 @@ +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Programmable, TLS interception capable + proxy server for Application debugging, testing and development. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import sys +from proxy.common.version import __version__ as lib_version +from setup import __version__ as pkg_version + +# This script ensures our versions never run out of sync. +# +# 1. setup.py doesn't import proxy and hence they both use +# their own respective __version__ +# 2. TODO: Version is also hardcoded in homebrew stable package +# installer file, but it only needs to match with other +# versions if git branch is master +if lib_version != pkg_version: + sys.exit(1) From 388599957feedfe73156e96668ea46f0eef98856 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 6 Jan 2020 21:22:37 -0800 Subject: [PATCH 3/3] Remove deprecated assertDictContainsSubset usage --- tests/http/test_http_parser.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py index ee13621125..6867749c3b 100644 --- a/tests/http/test_http_parser.py +++ b/tests/http/test_http_parser.py @@ -134,8 +134,7 @@ def test_get_full_parse(self) -> None: self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) - self.assertDictContainsSubset( - {b'host': (b'Host', b'example.com')}, self.parser.headers) + self.assertEqual(self.parser.headers[b'host'], (b'Host', b'example.com')) self.parser.del_headers([b'host']) self.parser.add_headers([(b'Host', b'example.com')]) self.assertEqual( @@ -189,8 +188,7 @@ def test_get_partial_parse1(self) -> None: self.parser.parse(CRLF * 2) self.assertEqual(self.parser.total_size, len(pkt) + (3 * len(CRLF)) + len(host_hdr)) - self.assertDictContainsSubset( - {b'host': (b'Host', b'localhost:8080')}, self.parser.headers) + self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080')) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_get_partial_parse2(self) -> None: @@ -207,8 +205,7 @@ def test_get_partial_parse2(self) -> None: self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) self.parser.parse(b'localhost:8080' + CRLF) - self.assertDictContainsSubset( - {b'host': (b'Host', b'localhost:8080')}, self.parser.headers) + self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080')) self.assertEqual(self.parser.buffer, b'') self.assertEqual( self.parser.state, @@ -216,8 +213,8 @@ def test_get_partial_parse2(self) -> None: self.parser.parse(b'Content-Type: text/plain' + CRLF) self.assertEqual(self.parser.buffer, b'') - self.assertDictContainsSubset( - {b'content-type': (b'Content-Type', b'text/plain')}, self.parser.headers) + self.assertEqual( + self.parser.headers[b'content-type'], (b'Content-Type', b'text/plain')) self.assertEqual( self.parser.state, httpParserStates.RCVING_HEADERS) @@ -239,10 +236,10 @@ def test_post_full_parse(self) -> None: self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') - self.assertDictContainsSubset( - {b'content-type': (b'Content-Type', b'application/x-www-form-urlencoded')}, self.parser.headers) - self.assertDictContainsSubset( - {b'content-length': (b'Content-Length', b'7')}, self.parser.headers) + self.assertEqual(self.parser.headers[b'content-type'], + (b'Content-Type', b'application/x-www-form-urlencoded')) + self.assertEqual(self.parser.headers[b'content-length'], + (b'Content-Length', b'7')) self.assertEqual(self.parser.body, b'a=b&c=d') self.assertEqual(self.parser.buffer, b'') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) @@ -376,8 +373,8 @@ def test_response_parse(self) -> None: b'\n' + b'301 Moved\n

301 Moved

\nThe document has moved\n' + b'here.\r\n\r\n') - self.assertDictContainsSubset( - {b'content-length': (b'Content-Length', b'219')}, self.parser.headers) + self.assertEqual(self.parser.headers[b'content-length'], + (b'Content-Length', b'219')) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_response_partial_parse(self) -> None: @@ -394,8 +391,8 @@ def test_response_partial_parse(self) -> None: b'X-XSS-Protection: 1; mode=block\r\n', b'X-Frame-Options: SAMEORIGIN\r\n' ])) - self.assertDictContainsSubset( - {b'x-frame-options': (b'X-Frame-Options', b'SAMEORIGIN')}, self.parser.headers) + self.assertEqual(self.parser.headers[b'x-frame-options'], + (b'X-Frame-Options', b'SAMEORIGIN')) self.assertEqual( self.parser.state, httpParserStates.RCVING_HEADERS)