diff --git a/Makefile b/Makefile index 793edfad9c..03232dc2d0 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ sign-https-certificates: python -m proxy.common.pki sign_csr \ --csr-path $(HTTPS_CSR_FILE_PATH) \ --crt-path $(HTTPS_SIGNED_CERT_FILE_PATH) \ - --hostname example.com \ + --hostname localhost \ --private-key-path $(CA_KEY_FILE_PATH) \ --public-key-path $(CA_CERT_FILE_PATH) diff --git a/docs/conf.py b/docs/conf.py index 8daf9fd725..adabd032c0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -316,4 +316,5 @@ (_py_class_role, 'EventQueue'), (_py_obj_role, 'proxy.core.work.threadless.T'), (_py_obj_role, 'proxy.core.work.work.T'), + (_py_obj_role, 'proxy.core.base.tcp_server.T'), ] diff --git a/examples/ssl_echo_server.py b/examples/ssl_echo_server.py index 433af3878d..8103e01051 100644 --- a/examples/ssl_echo_server.py +++ b/examples/ssl_echo_server.py @@ -17,7 +17,7 @@ from proxy.core.connection import TcpClientConnection -class EchoSSLServerHandler(BaseTcpServerHandler): +class EchoSSLServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Wraps client socket during initialization.""" def initialize(self) -> None: diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index cd4924150f..309ac79e21 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -13,9 +13,10 @@ from proxy import Proxy from proxy.core.base import BaseTcpServerHandler +from proxy.core.connection import TcpClientConnection -class EchoServerHandler(BaseTcpServerHandler): +class EchoServerHandler(BaseTcpServerHandler[TcpClientConnection]): """Sets client socket to non-blocking during initialization.""" def initialize(self) -> None: diff --git a/proxy/common/pki.py b/proxy/common/pki.py index 93aab09e46..0ccf695481 100644 --- a/proxy/common/pki.py +++ b/proxy/common/pki.py @@ -268,8 +268,8 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: parser.add_argument( '--subject', type=str, - default='/CN=example.com', - help='Subject to use for public key generation. Default: /CN=example.com', + default='/CN=localhost', + help='Subject to use for public key generation. Default: /CN=localhost', ) parser.add_argument( '--csr-path', diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index edc5361510..66804505b7 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -12,20 +12,75 @@ tcp """ +import ssl +import socket import logging import selectors from abc import abstractmethod -from typing import Any, Optional +from typing import Any, Optional, TypeVar, Union + +from ...common.flag import flags +from ...common.utils import wrap_socket +from ...common.types import Readables, SelectableEvents, Writables +from ...common.constants import DEFAULT_CERT_FILE, DEFAULT_CLIENT_RECVBUF_SIZE +from ...common.constants import DEFAULT_KEY_FILE, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_TIMEOUT from ...core.work import Work from ...core.connection import TcpClientConnection -from ...common.types import Readables, SelectableEvents, Writables logger = logging.getLogger(__name__) -class BaseTcpServerHandler(Work[TcpClientConnection]): +flags.add_argument( + '--key-file', + type=str, + default=DEFAULT_KEY_FILE, + help='Default: None. Server key file to enable end-to-end TLS encryption with clients. ' + 'If used, must also pass --cert-file.', +) + +flags.add_argument( + '--cert-file', + type=str, + default=DEFAULT_CERT_FILE, + help='Default: None. Server certificate to enable end-to-end TLS encryption with clients. ' + 'If used, must also pass --key-file.', +) + +flags.add_argument( + '--client-recvbuf-size', + type=int, + default=DEFAULT_CLIENT_RECVBUF_SIZE, + help='Default: ' + str(int(DEFAULT_CLIENT_RECVBUF_SIZE / 1024)) + + ' KB. Maximum amount of data received from the ' + 'client in a single recv() operation.', +) + +flags.add_argument( + '--server-recvbuf-size', + type=int, + default=DEFAULT_SERVER_RECVBUF_SIZE, + help='Default: ' + str(int(DEFAULT_SERVER_RECVBUF_SIZE / 1024)) + + ' KB. Maximum amount of data received from the ' + 'server in a single recv() operation.', +) + +flags.add_argument( + '--timeout', + type=int, + default=DEFAULT_TIMEOUT, + help='Default: ' + str(DEFAULT_TIMEOUT) + + '. Number of seconds after which ' + 'an inactive connection must be dropped. Inactivity is defined by no ' + 'data sent or received by the client.', +) + + +T = TypeVar('T', bound=TcpClientConnection) + + +class BaseTcpServerHandler(Work[T]): """BaseTcpServerHandler implements Work interface. BaseTcpServerHandler lifecycle is controlled by Threadless core @@ -56,6 +111,14 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.work.address, ) + def initialize(self) -> None: + """Optionally upgrades connection to HTTPS, + sets ``conn`` in non-blocking mode and initializes + HTTP protocol plugins.""" + conn = self._optionally_wrap_socket(self.work.connection) + conn.setblocking(False) + logger.debug('Handling connection %s' % self.work.address) + @abstractmethod def handle_data(self, data: memoryview) -> Optional[bool]: """Optionally return True to close client connection.""" @@ -139,3 +202,21 @@ async def handle_readables(self, readables: Readables) -> bool: else: teardown = True return teardown + + def _encryption_enabled(self) -> bool: + return self.flags.keyfile is not None and \ + self.flags.certfile is not None + + def _optionally_wrap_socket( + self, conn: socket.socket, + ) -> Union[ssl.SSLSocket, socket.socket]: + """Attempts to wrap accepted client connection using provided certificates. + + Shutdown and closes client connection upon error. + """ + if self._encryption_enabled(): + assert self.flags.keyfile and self.flags.certfile + # TODO(abhinavsingh): Insecure TLS versions must not be accepted by default + conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) + self.work._conn = conn + return conn diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index fb230ec0e7..20f0bbeff3 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -18,13 +18,13 @@ from ...common.types import Readables, SelectableEvents, Writables from ...common.utils import text_ -from ..connection import TcpServerConnection +from ..connection import TcpServerConnection, TcpClientConnection from .tcp_server import BaseTcpServerHandler logger = logging.getLogger(__name__) -class BaseTcpTunnelHandler(BaseTcpServerHandler): +class BaseTcpTunnelHandler(BaseTcpServerHandler[TcpClientConnection]): """BaseTcpTunnelHandler build on-top of BaseTcpServerHandler work class. On-top of BaseTcpServerHandler implementation, diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 7966174dbb..1d80b4e984 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -16,15 +16,12 @@ import logging import selectors -from typing import Tuple, List, Type, Union, Optional, Any +from typing import Tuple, List, Type, Optional, Any -from ..common.flag import flags -from ..common.utils import wrap_socket from ..core.base import BaseTcpServerHandler from ..core.connection import TcpClientConnection from ..common.types import Readables, SelectableEvents, Writables -from ..common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_KEY_FILE -from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT, DEFAULT_TIMEOUT +from ..common.constants import DEFAULT_SELECTOR_SELECT_TIMEOUT from .exception import HttpProtocolException from .plugin import HttpProtocolHandlerPlugin @@ -35,33 +32,7 @@ logger = logging.getLogger(__name__) -flags.add_argument( - '--client-recvbuf-size', - type=int, - default=DEFAULT_CLIENT_RECVBUF_SIZE, - help='Default: ' + str(int(DEFAULT_CLIENT_RECVBUF_SIZE / 1024)) + - ' KB. Maximum amount of data received from the ' - 'client in a single recv() operation.', -) -flags.add_argument( - '--key-file', - type=str, - default=DEFAULT_KEY_FILE, - help='Default: None. Server key file to enable end-to-end TLS encryption with clients. ' - 'If used, must also pass --cert-file.', -) -flags.add_argument( - '--timeout', - type=int, - default=DEFAULT_TIMEOUT, - help='Default: ' + str(DEFAULT_TIMEOUT) + - '. Number of seconds after which ' - 'an inactive connection must be dropped. Inactivity is defined by no ' - 'data sent or received by the client.', -) - - -class HttpProtocolHandler(BaseTcpServerHandler): +class HttpProtocolHandler(BaseTcpServerHandler[TcpClientConnection]): """HTTP, HTTPS, HTTP2, WebSockets protocol handler. Accepts `Client` connection and delegates to HttpProtocolHandlerPlugin. @@ -86,17 +57,16 @@ def __init__(self, *args: Any, **kwargs: Any): ## def initialize(self) -> None: - """Optionally upgrades connection to HTTPS, - sets ``conn`` in non-blocking mode and initializes - HTTP protocol plugins. - """ - conn = self._optionally_wrap_socket(self.work.connection) - conn.setblocking(False) + super().initialize() # Update client connection reference if connection was wrapped + # This is here in `handler` and not `tcp_server` because + # `tcp_server` is agnostic to constructing TcpClientConnection + # objects. if self._encryption_enabled(): - self.work = TcpClientConnection(conn=conn, addr=self.work.addr) - # self._initialize_plugins() - logger.debug('Handling connection %s' % self.work.address) + self.work = TcpClientConnection( + conn=self.work.connection, + addr=self.work.addr, + ) def is_inactive(self) -> bool: if not self.work.has_buffer() and \ @@ -334,23 +304,6 @@ def _parse_first_request(self, data: memoryview) -> bool: self.work._conn = output return False - def _encryption_enabled(self) -> bool: - return self.flags.keyfile is not None and \ - self.flags.certfile is not None - - def _optionally_wrap_socket( - self, conn: socket.socket, - ) -> Union[ssl.SSLSocket, socket.socket]: - """Attempts to wrap accepted client connection using provided certificates. - - Shutdown and closes client connection upon error. - """ - if self._encryption_enabled(): - assert self.flags.keyfile and self.flags.certfile - # TODO(abhinavsingh): Insecure TLS versions must not be accepted by default - conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) - return conn - def _connection_inactive_for(self) -> float: return time.time() - self.last_activity diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 7029a4333c..f0ccf11361 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -37,9 +37,9 @@ from ...common.types import Readables, Writables, Descriptors from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE -from ...common.constants import COMMA, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CERT_FILE +from ...common.constants import COMMA, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS -from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT +from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH from ...common.utils import text_ from ...common.pki import gen_public_key, gen_csr, sign_csr @@ -52,15 +52,6 @@ logger = logging.getLogger(__name__) -flags.add_argument( - '--server-recvbuf-size', - type=int, - default=DEFAULT_SERVER_RECVBUF_SIZE, - help='Default: ' + str(int(DEFAULT_SERVER_RECVBUF_SIZE / 1024)) + - ' KB. Maximum amount of data received from the ' - 'server in a single recv() operation.', -) - flags.add_argument( '--disable-http-proxy', action='store_true', @@ -116,14 +107,6 @@ 'HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file', ) -flags.add_argument( - '--cert-file', - type=str, - default=DEFAULT_CERT_FILE, - help='Default: None. Server certificate to enable end-to-end TLS encryption with clients. ' - 'If used, must also pass --key-file.', -) - flags.add_argument( '--auth-plugin', type=str, diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index 5f881a68f7..d4ad4effd9 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) HTTP_RESPONSE = okResponse(content=b'HTTP route response') -HTTPS_RESPONSE = okResponse(content=b'HTTP route response') +HTTPS_RESPONSE = okResponse(content=b'HTTPS route response') class WebServerPlugin(HttpWebServerBasePlugin): diff --git a/tests/common/test_pki.py b/tests/common/test_pki.py index 2bbebe06bb..dfe794bee7 100644 --- a/tests/common/test_pki.py +++ b/tests/common/test_pki.py @@ -123,7 +123,7 @@ def test_sign_csr(self) -> None: def _gen_public_private_key(self) -> Tuple[str, str, str]: key_path, nopass_key_path = self._gen_private_key() crt_path = os.path.join(self._tempdir, 'test_gen_public.crt') - pki.gen_public_key(crt_path, key_path, 'password', '/CN=example.com') + pki.gen_public_key(crt_path, key_path, 'password', '/CN=localhost') return (key_path, nopass_key_path, crt_path) def _gen_private_key(self) -> Tuple[str, str]: diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index a6f0ed4ed9..6acee8ee71 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -22,6 +22,13 @@ from proxy.common.constants import IS_WINDOWS +def _https_server_flags() -> str: + return ' '.join(( + '--key-file', 'https-key.pem', + '--cert-file', 'https-signed-cert.pem', + )) + + def _tls_interception_flags(ca_cert_suffix: str = '') -> str: return ' '.join(( '--ca-cert-file', 'ca-cert%s.pem' % ca_cert_suffix, @@ -36,6 +43,12 @@ def _tls_interception_flags(ca_cert_suffix: str = '') -> str: ('--threaded'), ) +PROXY_PY_HTTPS = ( + ('--threadless ' + _https_server_flags()), + ('--threadless --local-executor 0 ' + _https_server_flags()), + ('--threaded ' + _https_server_flags()), +) + PROXY_PY_FLAGS_TLS_INTERCEPTION = ( ('--threadless ' + _tls_interception_flags()), ('--threadless --local-executor 0 ' + _tls_interception_flags()), @@ -73,6 +86,16 @@ def _tls_interception_flags(ca_cert_suffix: str = '') -> str: ) +@pytest.fixture(scope='session', autouse=True) # type: ignore[misc] +def _gen_https_certificates(request: Any) -> None: + check_output([ + 'make', 'https-certificates', + ]) + check_output([ + 'make', 'sign-https-certificates', + ]) + + @pytest.fixture(scope='session', autouse=True) # type: ignore[misc] def _gen_ca_certificates(request: Any) -> None: check_output([ @@ -111,6 +134,7 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]: '--port', '0', '--port-file', str(port_file), '--enable-web-server', + '--plugin', 'proxy.plugin.WebServerPlugin', '--num-acceptors', '3', '--num-workers', '3', '--ca-cert-dir', str(ca_cert_dir), @@ -149,6 +173,24 @@ def test_integration(proxy_py_subprocess: int) -> None: check_output([str(shell_script_test), str(proxy_py_subprocess)]) +@pytest.mark.smoke # type: ignore[misc] +@pytest.mark.parametrize( + 'proxy_py_subprocess', + PROXY_PY_HTTPS, + indirect=True, +) # type: ignore[misc] +@pytest.mark.skipif( + IS_WINDOWS, + reason='OSError: [WinError 193] %1 is not a valid Win32 application', +) # type: ignore[misc] +def test_https_integration(proxy_py_subprocess: int) -> None: + """An acceptance test for HTTPS web and proxy server using ``curl`` through proxy.py.""" + this_test_module = Path(__file__) + shell_script_test = this_test_module.with_suffix('.sh') + # "1" means use-https scheme for requests to instance + check_output([str(shell_script_test), str(proxy_py_subprocess), '1']) + + @pytest.mark.smoke # type: ignore[misc] @pytest.mark.parametrize( 'proxy_py_subprocess', diff --git a/tests/integration/test_integration.sh b/tests/integration/test_integration.sh index 77c6cc8eba..06cafd2dc2 100755 --- a/tests/integration/test_integration.sh +++ b/tests/integration/test_integration.sh @@ -23,7 +23,20 @@ if [[ -z "$PROXY_PY_PORT" ]]; then exit 1 fi -PROXY_URL="127.0.0.1:$PROXY_PY_PORT" +PROXY_URL="http://localhost:$PROXY_PY_PORT" +TEST_URL="$PROXY_URL/http-route-example" +CURL_EXTRA_FLAGS="" +USE_HTTPS=$2 +if [[ ! -z "$USE_HTTPS" ]]; then + PROXY_URL="https://localhost:$PROXY_PY_PORT" + CURL_EXTRA_FLAGS=" -k --proxy-insecure " + # For https instances we don't use internal https web server + # See https://github.com/abhinavsingh/proxy.py/issues/994 + TEST_URL="http://google.com" + USE_HTTPS=true +else + USE_HTTPS=false +fi # Wait for server to come up WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '" @@ -37,12 +50,9 @@ while true; do done # Wait for http proxy and web server to start +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL" while true; do - curl -v \ - --max-time 1 \ - --connect-timeout 1 \ - -x $PROXY_URL \ - http://$PROXY_URL/ 2>/dev/null + RESPONSE=$($CMD 2> /dev/null) if [[ $? == 0 ]]; then break fi @@ -80,22 +90,27 @@ Disallow: /deny EOM echo "[Test HTTP Request via Proxy]" -CMD="curl -v -x $PROXY_URL http://httpbin.org/robots.txt" +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt" RESPONSE=$($CMD 2> /dev/null) verify_response "$RESPONSE" "$ROBOTS_RESPONSE" VERIFIED1=$? echo "[Test HTTPS Request via Proxy]" -CMD="curl -v -x $PROXY_URL https://httpbin.org/robots.txt" +CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt" RESPONSE=$($CMD 2> /dev/null) verify_response "$RESPONSE" "$ROBOTS_RESPONSE" VERIFIED2=$? -echo "[Test Internal Web Server via Proxy]" -curl -v \ - -x $PROXY_URL \ - http://$PROXY_URL/ -VERIFIED3=$? +if $USE_HTTPS; then + VERIFIED3=0 +else + echo "[Test Internal Web Server via Proxy]" + curl -v \ + $CURL_EXTRA_FLAGS \ + -x $PROXY_URL \ + "$PROXY_URL" + VERIFIED3=$? +fi SHASUM=sha256sum if [ "$(uname)" = "Darwin" ]; @@ -107,6 +122,7 @@ echo "[Test Download File Hash Verifies 1]" touch downloaded.hash echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc -" > downloaded.hash curl -vL \ + $CURL_EXTRA_FLAGS \ -o downloaded.whl \ -x $PROXY_URL \ https://files.pythonhosted.org/packages/88/78/e642316313b1cd6396e4b85471a316e003eff968f29773e95ea191ea1d08/proxy.py-2.4.0rc4-py3-none-any.whl#sha256=3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc @@ -118,6 +134,7 @@ echo "[Test Download File Hash Verifies 2]" touch downloaded.hash echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 -" > downloaded.hash curl -vL \ + $CURL_EXTRA_FLAGS \ -o downloaded.whl \ -x $PROXY_URL \ https://files.pythonhosted.org/packages/20/9a/e5d9ec41927401e41aea8af6d16e78b5e612bca4699d417f646a9610a076/Jinja2-3.0.3-py3-none-any.whl#sha256=077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8