diff --git a/Dockerfile b/Dockerfile index 60f88802d3..fb079fcb5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,9 @@ LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ COPY --from=builder /deps /usr/local +# Install openssl to enable TLS interception within container +RUN apk update && apk add openssl + EXPOSE 8899/tcp ENTRYPOINT [ "proxy" ] CMD [ "--hostname=0.0.0.0" ] diff --git a/README.md b/README.md index 1ce5e2163a..ed6f8b0434 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ Table of Contents * [Plugin Ordering](#plugin-ordering) * [End-to-End Encryption](#end-to-end-encryption) * [TLS Interception](#tls-interception) + * [TLS Interception With Docker](#tls-interception-with-docker) * [Proxy Over SSH Tunnel](#proxy-over-ssh-tunnel) * [Proxy Remote Requests Locally](#proxy-remote-requests-locally) * [Proxy Local Requests Remotely](#proxy-local-requests-remotely) @@ -862,9 +863,94 @@ cached file instead of plain text. Now use CA flags with other [plugin examples](#plugin-examples) to see them work with `https` traffic. +## TLS Interception With Docker + +Important notes about TLS Interception with Docker container: + +- Since `v2.2.0`, `proxy.py` docker container also ships with `openssl`. This allows `proxy.py` +to generate certificates on the fly for TLS Interception. + +- For security reasons, `proxy.py` docker container doesn't ship with CA certificates. + +Here is how to start a `proxy.py` docker container +with TLS Interception: + +1. Generate CA certificates on host computer + + ```bash + ❯ make ca-certificates + ``` + +2. Copy all generated certificates into a separate directory. We'll later mount this directory into our docker container + + ```bash + ❯ mkdir /tmp/ca-certificates + ❯ cp ca-cert.pem ca-key.pem ca-signing-key.pem /tmp/ca-certificates + ``` + +3. Start docker container + + ```bash + ❯ docker run -it --rm \ + -v /tmp/ca-certificates:/tmp/ca-certificates \ + -p 8899:8899 \ + abhinavsingh/proxy.py:latest \ + --hostname 0.0.0.0 \ + --plugins proxy.plugin.CacheResponsesPlugin \ + --ca-key-file /tmp/ca-certificates/ca-key.pem \ + --ca-cert-file /tmp/ca-certificates/ca-cert.pem \ + --ca-signing-key /tmp/ca-certificates/ca-signing-key.pem + ``` + + - `-v /tmp/ca-certificates:/tmp/ca-certificates` flag mounts our CA certificate directory in container environment + - `--plugins proxy.plugin.CacheResponsesPlugin` enables `CacheResponsesPlugin` so that we can inspect intercepted traffic + - `--ca-*` flags enable TLS Interception. + +4. From another terminal, try TLS Interception using `curl`. You can omit `--cacert` flag if CA certificate is already trusted by the system. + + ```bash + ❯ curl -v \ + --cacert ca-cert.pem \ + -x 127.0.0.1:8899 \ + https://httpbin.org/get + ``` + +5. Verify `issuer` field from response headers. + + ```bash + * Server certificate: + * subject: CN=httpbin.org; C=NA; ST=Unavailable; L=Unavailable; O=Unavailable; OU=Unavailable + * start date: Jun 17 09:26:57 2020 GMT + * expire date: Jun 17 09:26:57 2022 GMT + * subjectAltName: host "httpbin.org" matched cert's "httpbin.org" + * issuer: CN=example.com + * SSL certificate verify ok. + ``` + +6. Back on docker terminal, copy response dump path logs. + + ```bash + ...[redacted]... [I] access_log:338 - 172.17.0.1:56498 - CONNECT httpbin.org:443 - 1031 bytes - 1216.70 ms + ...[redacted]... [I] close:49 - Cached response at /tmp/httpbin.org-ae1a927d064e4ab386ea319eb38fe251.txt + ``` + +7. In another terminal, `cat` the response dump: + + ```bash + ❯ docker exec -it $(docker ps | grep proxy.py | awk '{ print $1 }') cat /tmp/httpbin.org-ae1a927d064e4ab386ea319eb38fe251.txt + HTTP/1.1 200 OK + ...[redacted]... + { + ...[redacted]..., + "url": "http://httpbin.org/get" + } + ``` + Proxy Over SSH Tunnel ===================== +**This is a WIP and may not work as documented** + Requires `paramiko` to work. See [requirements-tunnel.txt](https://github.com/abhinavsingh/proxy.py/blob/develop/requirements-tunnel.txt) ## Proxy Remote Requests Locally diff --git a/proxy/common/constants.py b/proxy/common/constants.py index fc9dac9372..1479523a71 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -10,6 +10,7 @@ """ import os import time +import pathlib import ipaddress from typing import List @@ -75,3 +76,13 @@ DEFAULT_VERSION = False DEFAULT_HTTP_PORT = 80 DEFAULT_MAX_SEND_SIZE = 16 * 1024 + +DEFAULT_DATA_DIRECTORY_PATH = os.path.join(str(pathlib.Path.home()), '.proxy') + +# Cor plugins enabled by default or via flags +PLUGIN_HTTP_PROXY = 'proxy.http.proxy.HttpProxyPlugin' +PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin' +PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin' +PLUGIN_DEVTOOLS_PROTOCOL = 'proxy.http.inspector.DevtoolsProtocolPlugin' +PLUGIN_DASHBOARD = 'proxy.dashboard.dashboard.ProxyDashboard' +PLUGIN_INSPECT_TRAFFIC = 'proxy.dashboard.inspect_traffic.InspectTrafficPlugin' diff --git a/proxy/common/flags.py b/proxy/common/flags.py index 5b74173c14..182c2c35c3 100644 --- a/proxy/common/flags.py +++ b/proxy/common/flags.py @@ -20,7 +20,6 @@ import multiprocessing import sys import inspect -import pathlib from typing import Optional, Union, Dict, List, TypeVar, Type, cast, Any, Tuple @@ -33,7 +32,9 @@ from .constants import DEFAULT_PAC_FILE_URL_PATH, DEFAULT_PAC_FILE, DEFAULT_PLUGINS, DEFAULT_PID_FILE, DEFAULT_PORT from .constants import DEFAULT_NUM_WORKERS, DEFAULT_VERSION, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_IPV6_HOSTNAME from .constants import DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_STATIC_SERVER_DIR -from .constants import DEFAULT_ENABLE_DASHBOARD, COMMA, DOT +from .constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_DATA_DIRECTORY_PATH, COMMA, DOT +from .constants import PLUGIN_HTTP_PROXY, PLUGIN_WEB_SERVER, PLUGIN_PAC_FILE +from .constants import PLUGIN_DEVTOOLS_PROTOCOL, PLUGIN_DASHBOARD, PLUGIN_INSPECT_TRAFFIC from .version import __version__ __homepage__ = 'https://github.com/abhinavsingh/proxy.py' @@ -49,9 +50,6 @@ class Flags: """Contains all input flags and inferred input parameters.""" - ROOT_DATA_DIR_NAME = '.proxy.py' - GENERATED_CERTS_DIR_NAME = 'certificates' - def __init__( self, auth_code: Optional[bytes] = DEFAULT_BASIC_AUTH, @@ -112,16 +110,25 @@ def __init__( self.devtools_ws_path: bytes = devtools_ws_path self.enable_events: bool = enable_events - self.proxy_py_data_dir = os.path.join( - str(pathlib.Path.home()), self.ROOT_DATA_DIR_NAME) + self.proxy_py_data_dir = DEFAULT_DATA_DIRECTORY_PATH os.makedirs(self.proxy_py_data_dir, exist_ok=True) self.ca_cert_dir: Optional[str] = ca_cert_dir if self.ca_cert_dir is None: self.ca_cert_dir = os.path.join( - self.proxy_py_data_dir, self.GENERATED_CERTS_DIR_NAME) + self.proxy_py_data_dir, 'certificates') os.makedirs(self.ca_cert_dir, exist_ok=True) + def tls_interception_enabled(self) -> bool: + return self.ca_key_file is not None and \ + self.ca_cert_dir is not None and \ + self.ca_signing_key_file is not None and \ + self.ca_cert_file is not None + + def encryption_enabled(self) -> bool: + return self.keyfile is not None and \ + self.certfile is not None + @classmethod def initialize( cls: Type[T], @@ -138,51 +145,59 @@ def initialize( 'A future version of pip will drop support for Python 2.7.') sys.exit(1) - args = Flags.init_parser().parse_args(input_args) + parser = Flags.init_parser() + args = parser.parse_args(input_args) + # Print version and exit if args.version: print(__version__) sys.exit(0) - if (args.cert_file and args.key_file) and \ - (args.ca_key_file and args.ca_cert_file and args.ca_signing_key_file): - print('You can either enable end-to-end encryption OR TLS interception,' - 'not both together.') - sys.exit(1) - - auth_code = None - if args.basic_auth: - auth_code = b'Basic %s' % base64.b64encode(bytes_(args.basic_auth)) - + # Setup logging module Flags.setup_logger(args.log_file, args.log_level, args.log_format) - Flags.set_open_file_limit(args.open_file_limit) - http_proxy_plugin = 'proxy.http.proxy.HttpProxyPlugin' - web_server_plugin = 'proxy.http.server.HttpWebServerPlugin' - pac_file_plugin = 'proxy.http.server.HttpWebServerPacFilePlugin' - devtools_protocol_plugin = 'proxy.http.inspector.DevtoolsProtocolPlugin' - dashboard_plugin = 'proxy.dashboard.dashboard.ProxyDashboard' - inspect_traffic_plugin = 'proxy.dashboard.inspect_traffic.InspectTrafficPlugin' + # Setup limits + Flags.set_open_file_limit(args.open_file_limit) default_plugins: List[Tuple[str, bool]] = [] if args.enable_dashboard: - default_plugins.append((web_server_plugin, True)) + default_plugins.append((PLUGIN_WEB_SERVER, True)) args.enable_static_server = True - default_plugins.append((dashboard_plugin, True)) - default_plugins.append((inspect_traffic_plugin, True)) + default_plugins.append((PLUGIN_DASHBOARD, True)) + default_plugins.append((PLUGIN_INSPECT_TRAFFIC, True)) args.enable_events = True args.enable_devtools = True if args.enable_devtools: - default_plugins.append((devtools_protocol_plugin, True)) - default_plugins.append((web_server_plugin, True)) + default_plugins.append((PLUGIN_DEVTOOLS_PROTOCOL, True)) + default_plugins.append((PLUGIN_WEB_SERVER, True)) if not args.disable_http_proxy: - default_plugins.append((http_proxy_plugin, True)) + default_plugins.append((PLUGIN_HTTP_PROXY, True)) if args.enable_web_server or \ args.pac_file is not None or \ args.enable_static_server: - default_plugins.append((web_server_plugin, True)) + default_plugins.append((PLUGIN_WEB_SERVER, True)) if args.pac_file is not None: - default_plugins.append((pac_file_plugin, True)) + default_plugins.append((PLUGIN_PAC_FILE, True)) + + plugins = Flags.load_plugins( + bytes_( + '%s,%s' % + (text_(COMMA).join(collections.OrderedDict(default_plugins).keys()), + opts.get('plugins', args.plugins)))) + + # proxy.py currently cannot serve over HTTPS and perform TLS interception + # at the same time. Check if user is trying to enable both feature + # at the same time. + if (args.cert_file and args.key_file) and \ + (args.ca_key_file and args.ca_cert_file and args.ca_signing_key_file): + print('You can either enable end-to-end encryption OR TLS interception,' + 'not both together.') + sys.exit(1) + + # Generate auth_code required for basic authentication if enabled + auth_code = None + if args.basic_auth: + auth_code = b'Basic %s' % base64.b64encode(bytes_(args.basic_auth)) return cls( auth_code=cast(Optional[bytes], opts.get('auth_code', auth_code)), @@ -258,23 +273,9 @@ def initialize( opts.get( 'enable_events', args.enable_events)), - plugins=Flags.load_plugins( - bytes_( - '%s,%s' % - (text_(COMMA).join(collections.OrderedDict(default_plugins).keys()), - opts.get('plugins', args.plugins)))), + plugins=plugins, pid_file=cast(Optional[str], opts.get('pid_file', args.pid_file))) - def tls_interception_enabled(self) -> bool: - return self.ca_key_file is not None and \ - self.ca_cert_dir is not None and \ - self.ca_signing_key_file is not None and \ - self.ca_cert_file is not None - - def encryption_enabled(self) -> bool: - return self.keyfile is not None and \ - self.certfile is not None - @staticmethod def init_parser() -> argparse.ArgumentParser: """Initializes and returns argument parser."""