From c2d856327b8de1813a7dc10d344c68719ad361b2 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Tue, 18 Jan 2022 00:46:41 +0530 Subject: [PATCH 1/6] Fixes #1000, #1001 --- proxy/common/constants.py | 2 +- proxy/core/acceptor/acceptor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 37ef452f54..a828ab0cf4 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -122,7 +122,7 @@ def _env_threadless_compliant() -> bool: DEFAULT_STATIC_SERVER_DIR = os.path.join(PROXY_PY_DIR, "public") DEFAULT_MIN_COMPRESSION_LIMIT = 20 # In bytes DEFAULT_THREADLESS = _env_threadless_compliant() -DEFAULT_LOCAL_EXECUTOR = True +DEFAULT_LOCAL_EXECUTOR = not IS_WINDOWS DEFAULT_TIMEOUT = 10.0 DEFAULT_VERSION = False DEFAULT_HTTP_PORT = 80 diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index d83c5e00ae..af6ef73478 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -42,7 +42,7 @@ type=int, default=int(DEFAULT_LOCAL_EXECUTOR), help='Default: ' + ('1' if DEFAULT_LOCAL_EXECUTOR else '0') + '. ' + - 'Enabled by default. Use 0 to disable. When enabled acceptors ' + + 'Enabled by default (except on Windows). Use 0 to disable. When enabled acceptors ' + 'will make use of local (same process) executor instead of distributing load across ' + 'remote (other process) executors. Enable this option to achieve CPU affinity between ' + 'acceptors and executors, instead of using underlying OS kernel scheduling algorithm.', From dc572fbbc7df3e61f0a785138a382ead8a31cd34 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Tue, 18 Jan 2022 00:47:06 +0530 Subject: [PATCH 2/6] Update readme flags --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3bf8ce98d8..5c6f30978f 100644 --- a/README.md +++ b/README.md @@ -2238,7 +2238,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.4.0rc7.dev12+gd234339.d20220116 +proxy.py v2.4.0rc7.dev15+gc4ea34e options: -h, --help show this help message and exit @@ -2279,13 +2279,13 @@ options: Default: None. Unix socket path to use. When provided --host and --port flags are ignored --local-executor LOCAL_EXECUTOR - Default: 1. Enabled by default. Use 0 to disable. When - enabled acceptors will make use of local (same - process) executor instead of distributing load across - remote (other process) executors. Enable this option - to achieve CPU affinity between acceptors and - executors, instead of using underlying OS kernel - scheduling algorithm. + Default: 1. Enabled by default (except on Windows). + Use 0 to disable. When enabled acceptors will make use + of local (same process) executor instead of + distributing load across remote (other process) + executors. Enable this option to achieve CPU affinity + between acceptors and executors, instead of using + underlying OS kernel scheduling algorithm. --num-acceptors NUM_ACCEPTORS Defaults to number of CPU cores. --version, -v Prints proxy.py version. From e017336a6adece8c9c370651a640f47d3e4728b1 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Tue, 18 Jan 2022 00:54:29 +0530 Subject: [PATCH 3/6] Enable integration tests for windows, only for the mode that works on windows --- tests/integration/test_integration.py | 110 +++++++++++++------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 6acee8ee71..02d4cf8ec9 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -37,52 +37,76 @@ def _tls_interception_flags(ca_cert_suffix: str = '') -> str: )) -PROXY_PY_FLAGS_INTEGRATION = ( - ('--threadless'), +_PROXY_PY_FLAGS_INTEGRATION = [ ('--threadless --local-executor 0'), - ('--threaded'), -) - -PROXY_PY_HTTPS = ( - ('--threadless ' + _https_server_flags()), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_INTEGRATION += [ + ('--threadless'), + ('--threaded'), + ] +PROXY_PY_FLAGS_INTEGRATION = tuple(_PROXY_PY_FLAGS_INTEGRATION) + +_PROXY_PY_HTTPS = [ ('--threadless --local-executor 0 ' + _https_server_flags()), - ('--threaded ' + _https_server_flags()), -) - -PROXY_PY_FLAGS_TLS_INTERCEPTION = ( - ('--threadless ' + _tls_interception_flags()), +] +if not IS_WINDOWS: + _PROXY_PY_HTTPS += [ + ('--threadless ' + _https_server_flags()), + ('--threaded ' + _https_server_flags()), + ] +PROXY_PY_HTTPS = tuple(_PROXY_PY_HTTPS) + +_PROXY_PY_FLAGS_TLS_INTERCEPTION = [ ('--threadless --local-executor 0 ' + _tls_interception_flags()), - ('--threaded ' + _tls_interception_flags()), -) - -PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = ( - ( - '--threadless --plugin proxy.plugin.ModifyChunkResponsePlugin ' + - _tls_interception_flags('-chunk') - ), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_TLS_INTERCEPTION += [ + ('--threadless ' + _tls_interception_flags()), + ('--threaded ' + _tls_interception_flags()), + ] +PROXY_PY_FLAGS_TLS_INTERCEPTION = tuple(_PROXY_PY_FLAGS_TLS_INTERCEPTION) + +_PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = [ ( '--threadless --local-executor 0 --plugin proxy.plugin.ModifyChunkResponsePlugin ' + _tls_interception_flags('-chunk') ), - ( - '--threaded --plugin proxy.plugin.ModifyChunkResponsePlugin ' + - _tls_interception_flags('-chunk') - ), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN += [ + ( + '--threadless --plugin proxy.plugin.ModifyChunkResponsePlugin ' + + _tls_interception_flags('-chunk') + ), + ( + '--threaded --plugin proxy.plugin.ModifyChunkResponsePlugin ' + + _tls_interception_flags('-chunk') + ), + ] +PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = tuple( + _PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN, ) -PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = ( - ( - '--threadless --plugin proxy.plugin.ModifyPostDataPlugin ' + - _tls_interception_flags('-post') - ), +_PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = [ ( '--threadless --local-executor 0 --plugin proxy.plugin.ModifyPostDataPlugin ' + _tls_interception_flags('-post') ), - ( - '--threaded --plugin proxy.plugin.ModifyPostDataPlugin ' + - _tls_interception_flags('-post') - ), +] +if not IS_WINDOWS: + _PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN += [ + ( + '--threaded --plugin proxy.plugin.ModifyPostDataPlugin ' + + _tls_interception_flags('-post') + ), + ( + '--threadless --plugin proxy.plugin.ModifyPostDataPlugin ' + + _tls_interception_flags('-post') + ), + ] +PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = tuple( + _PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN, ) @@ -162,10 +186,6 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]: PROXY_PY_FLAGS_INTEGRATION, indirect=True, ) # type: ignore[misc] -@pytest.mark.skipif( - IS_WINDOWS, - reason='OSError: [WinError 193] %1 is not a valid Win32 application', -) # type: ignore[misc] def test_integration(proxy_py_subprocess: int) -> None: """An acceptance test using ``curl`` through proxy.py.""" this_test_module = Path(__file__) @@ -179,10 +199,6 @@ def test_integration(proxy_py_subprocess: int) -> None: PROXY_PY_HTTPS, indirect=True, ) # type: ignore[misc] -@pytest.mark.skipif( - IS_WINDOWS, - reason='OSError: [WinError 193] %1 is not a valid Win32 application', -) # type: ignore[misc] def test_https_integration(proxy_py_subprocess: int) -> None: """An acceptance test for HTTPS web and proxy server using ``curl`` through proxy.py.""" this_test_module = Path(__file__) @@ -197,10 +213,6 @@ def test_https_integration(proxy_py_subprocess: int) -> None: PROXY_PY_FLAGS_TLS_INTERCEPTION, indirect=True, ) # type: ignore[misc] -@pytest.mark.skipif( - IS_WINDOWS, - reason='OSError: [WinError 193] %1 is not a valid Win32 application', -) # type: ignore[misc] def test_integration_with_interception_flags(proxy_py_subprocess: int) -> None: """An acceptance test for TLS interception using ``curl`` through proxy.py.""" shell_script_test = Path(__file__).parent / 'test_interception.sh' @@ -213,10 +225,6 @@ def test_integration_with_interception_flags(proxy_py_subprocess: int) -> None: PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN, indirect=True, ) # type: ignore[misc] -@pytest.mark.skipif( - IS_WINDOWS, - reason='OSError: [WinError 193] %1 is not a valid Win32 application', -) # type: ignore[misc] def test_modify_chunk_response_integration(proxy_py_subprocess: int) -> None: """An acceptance test for :py:class:`~proxy.plugin.ModifyChunkResponsePlugin` interception using ``curl`` through proxy.py.""" @@ -230,10 +238,6 @@ def test_modify_chunk_response_integration(proxy_py_subprocess: int) -> None: PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN, indirect=True, ) # type: ignore[misc] -@pytest.mark.skipif( - IS_WINDOWS, - reason='OSError: [WinError 193] %1 is not a valid Win32 application', -) # type: ignore[misc] def test_modify_post_response_integration(proxy_py_subprocess: int) -> None: """An acceptance test for :py:class:`~proxy.plugin.ModifyPostDataPlugin` interception using ``curl`` through proxy.py.""" From d701f7d2f40d7ee5ca8826af3ceb68c8a1187573 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Tue, 18 Jan 2022 01:04:54 +0530 Subject: [PATCH 4/6] Revisit later to enable integration on windows, someone needs to try manually first --- tests/integration/test_integration.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 02d4cf8ec9..6a76ad29ed 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -186,6 +186,10 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]: PROXY_PY_FLAGS_INTEGRATION, indirect=True, ) # type: ignore[misc] +@pytest.mark.skipif( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', +) # type: ignore[misc] def test_integration(proxy_py_subprocess: int) -> None: """An acceptance test using ``curl`` through proxy.py.""" this_test_module = Path(__file__) @@ -199,6 +203,10 @@ def test_integration(proxy_py_subprocess: int) -> None: PROXY_PY_HTTPS, indirect=True, ) # type: ignore[misc] +@pytest.mark.skipif( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', +) # type: ignore[misc] def test_https_integration(proxy_py_subprocess: int) -> None: """An acceptance test for HTTPS web and proxy server using ``curl`` through proxy.py.""" this_test_module = Path(__file__) @@ -213,6 +221,10 @@ def test_https_integration(proxy_py_subprocess: int) -> None: PROXY_PY_FLAGS_TLS_INTERCEPTION, indirect=True, ) # type: ignore[misc] +@pytest.mark.skipif( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', +) # type: ignore[misc] def test_integration_with_interception_flags(proxy_py_subprocess: int) -> None: """An acceptance test for TLS interception using ``curl`` through proxy.py.""" shell_script_test = Path(__file__).parent / 'test_interception.sh' @@ -225,6 +237,10 @@ def test_integration_with_interception_flags(proxy_py_subprocess: int) -> None: PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN, indirect=True, ) # type: ignore[misc] +@pytest.mark.skipif( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', +) # type: ignore[misc] def test_modify_chunk_response_integration(proxy_py_subprocess: int) -> None: """An acceptance test for :py:class:`~proxy.plugin.ModifyChunkResponsePlugin` interception using ``curl`` through proxy.py.""" @@ -238,6 +254,10 @@ def test_modify_chunk_response_integration(proxy_py_subprocess: int) -> None: PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN, indirect=True, ) # type: ignore[misc] +@pytest.mark.skipif( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', +) # type: ignore[misc] def test_modify_post_response_integration(proxy_py_subprocess: int) -> None: """An acceptance test for :py:class:`~proxy.plugin.ModifyPostDataPlugin` interception using ``curl`` through proxy.py.""" From 6ea0d1005569d239b6e0b2a712e97d7b702cb988 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Tue, 18 Jan 2022 01:06:01 +0530 Subject: [PATCH 5/6] Disable `spellcheck-docs` until sphinx bug is resolved --- tox.ini | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tox.ini b/tox.ini index 83bc337213..e325c8dd87 100644 --- a/tox.ini +++ b/tox.ini @@ -123,33 +123,33 @@ skip_install = {[testenv:build-docs]skip_install} usedevelop = {[testenv:build-docs]usedevelop} -[testenv:spellcheck-docs] -allowlist_externals = - {[testenv:build-docs]allowlist_externals} -basepython = {[testenv:build-docs]basepython} -commands_pre = {[testenv:build-docs]commands_pre} -commands = - # Retrieve possibly missing commits: - -git fetch --unshallow - -git fetch --tags +; [testenv:spellcheck-docs] +; allowlist_externals = +; {[testenv:build-docs]allowlist_externals} +; basepython = {[testenv:build-docs]basepython} +; commands_pre = {[testenv:build-docs]commands_pre} +; commands = +; # Retrieve possibly missing commits: +; -git fetch --unshallow +; -git fetch --tags - # Spellcheck docs site: - python -m sphinx \ - -j auto \ - -a -n -W \ - --keep-going \ - -b spelling --color \ - -d "{toxworkdir}/docs_doctree" \ - . "{toxworkdir}/docs_out" -changedir = {[testenv:build-docs]changedir} -deps = - sphinxcontrib-spelling >= 7.2.0 - -r{toxinidir}/docs/requirements.in -description = Spellcheck The Docs -isolated_build = {[testenv:build-docs]isolated_build} -passenv = {[testenv:build-docs]passenv} -skip_install = {[testenv:build-docs]skip_install} -usedevelop = {[testenv:build-docs]usedevelop} +; # Spellcheck docs site: +; python -m sphinx \ +; -j auto \ +; -a -n -W \ +; --keep-going \ +; -b spelling --color \ +; -d "{toxworkdir}/docs_doctree" \ +; . "{toxworkdir}/docs_out" +; changedir = {[testenv:build-docs]changedir} +; deps = +; sphinxcontrib-spelling >= 7.2.0 +; -r{toxinidir}/docs/requirements.in +; description = Spellcheck The Docs +; isolated_build = {[testenv:build-docs]isolated_build} +; passenv = {[testenv:build-docs]passenv} +; skip_install = {[testenv:build-docs]skip_install} +; usedevelop = {[testenv:build-docs]usedevelop} [testenv:make-changelog] From e3a825f2b120cc6185b8f488eec1d077f3227688 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Tue, 18 Jan 2022 01:12:16 +0530 Subject: [PATCH 6/6] Keep it broken --- tox.ini | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tox.ini b/tox.ini index e325c8dd87..83bc337213 100644 --- a/tox.ini +++ b/tox.ini @@ -123,33 +123,33 @@ skip_install = {[testenv:build-docs]skip_install} usedevelop = {[testenv:build-docs]usedevelop} -; [testenv:spellcheck-docs] -; allowlist_externals = -; {[testenv:build-docs]allowlist_externals} -; basepython = {[testenv:build-docs]basepython} -; commands_pre = {[testenv:build-docs]commands_pre} -; commands = -; # Retrieve possibly missing commits: -; -git fetch --unshallow -; -git fetch --tags +[testenv:spellcheck-docs] +allowlist_externals = + {[testenv:build-docs]allowlist_externals} +basepython = {[testenv:build-docs]basepython} +commands_pre = {[testenv:build-docs]commands_pre} +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags -; # Spellcheck docs site: -; python -m sphinx \ -; -j auto \ -; -a -n -W \ -; --keep-going \ -; -b spelling --color \ -; -d "{toxworkdir}/docs_doctree" \ -; . "{toxworkdir}/docs_out" -; changedir = {[testenv:build-docs]changedir} -; deps = -; sphinxcontrib-spelling >= 7.2.0 -; -r{toxinidir}/docs/requirements.in -; description = Spellcheck The Docs -; isolated_build = {[testenv:build-docs]isolated_build} -; passenv = {[testenv:build-docs]passenv} -; skip_install = {[testenv:build-docs]skip_install} -; usedevelop = {[testenv:build-docs]usedevelop} + # Spellcheck docs site: + python -m sphinx \ + -j auto \ + -a -n -W \ + --keep-going \ + -b spelling --color \ + -d "{toxworkdir}/docs_doctree" \ + . "{toxworkdir}/docs_out" +changedir = {[testenv:build-docs]changedir} +deps = + sphinxcontrib-spelling >= 7.2.0 + -r{toxinidir}/docs/requirements.in +description = Spellcheck The Docs +isolated_build = {[testenv:build-docs]isolated_build} +passenv = {[testenv:build-docs]passenv} +skip_install = {[testenv:build-docs]skip_install} +usedevelop = {[testenv:build-docs]usedevelop} [testenv:make-changelog]