diff --git a/.flake8 b/.flake8 index b18b1a582e..0999bbdc5d 100644 --- a/.flake8 +++ b/.flake8 @@ -148,7 +148,6 @@ extend-ignore = WPS420 # FIXME: pointless keyword like `pass` WPS421 # FIXME: call to `print()` WPS425 # FIXME: bool non-keyword arg - WPS427 # FIXME: unreachable code WPS428 # FIXME: pointless statement WPS430 # FIXME: nested func WPS431 # FIXME: nested class diff --git a/.gitignore b/.gitignore index 217ce78fd2..09bd936373 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,8 @@ proxy.py.iml *.crt *.key *.pem + +.venv* venv* cover diff --git a/.pylintrc b/.pylintrc index 1203c8dec9..25434cea23 100644 --- a/.pylintrc +++ b/.pylintrc @@ -127,7 +127,6 @@ disable=raw-checker-failed, too-many-return-statements, too-many-statements, unnecessary-pass, - unreachable, unused-argument, useless-return, useless-super-delegation, diff --git a/README.md b/README.md index e8743686c5..4446ff21be 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,12 @@ - [Cache Responses Plugin](#cacheresponsesplugin) - [Man-In-The-Middle Plugin](#maninthemiddleplugin) - [Proxy Pool Plugin](#proxypoolplugin) - - [FilterByClientIpPlugin](#filterbyclientipplugin) - - [ModifyChunkResponsePlugin](#modifychunkresponseplugin) - - [CloudflareDnsResolverPlugin](#cloudflarednsresolverplugin) - - [CustomDnsResolverPlugin](#customdnsresolverplugin) - - [CustomNetworkInterface](#customnetworkinterface) + - [Filter By Client IP Plugin](#filterbyclientipplugin) + - [Modify Chunk Response Plugin](#modifychunkresponseplugin) + - [Cloudflare DNS Resolver Plugin](#cloudflarednsresolverplugin) + - [Custom DNS Resolver Plugin](#customdnsresolverplugin) + - [Custom Network Interface](#customnetworkinterface) + - [Program Name Plugin](#programnameplugin) - [HTTP Web Server Plugins](#http-web-server-plugins) - [Reverse Proxy](#reverse-proxy) - [Web Server Route](#web-server-route) @@ -578,7 +579,7 @@ Verify mock API response using `curl -x localhost:8899 http://api.example.com/v1 Verify the same by inspecting `proxy.py` logs: ```console -2019-09-27 12:44:02,212 - INFO - pid:7077 - access_log:1210 - ::1:64792 - GET None:None/v1/users/ - None None - 0 byte +... [redacted] ... - access_log:1210 - ::1:64792 - GET None:None/v1/users/ - None None - 0 byte ``` Access log shows `None:None` as server `ip:port`. `None` simply means that @@ -618,8 +619,8 @@ Verify the same by inspecting the logs for `proxy.py`. Along with the proxy request log, you must also see a http web server request log. ``` -2019-09-24 19:09:33,602 - INFO - pid:49996 - access_log:1241 - ::1:49525 - GET / -2019-09-24 19:09:33,603 - INFO - pid:49995 - access_log:1157 - ::1:49524 - GET localhost:8899/ - 404 NOT FOUND - 70 bytes +... [redacted] ... - access_log:1241 - ::1:49525 - GET / +... [redacted] ... - access_log:1157 - ::1:49524 - GET localhost:8899/ - 404 NOT FOUND - 70 bytes ``` ### FilterByUpstreamHostPlugin @@ -650,10 +651,10 @@ Above `418 I'm a tea pot` is sent by our plugin. Verify the same by inspecting logs for `proxy.py`: ```console -2019-09-24 19:21:37,893 - ERROR - pid:50074 - handle_readables:1347 - HttpProtocolException type raised +... [redacted] ... - handle_readables:1347 - HttpProtocolException type raised Traceback (most recent call last): ... [redacted] ... -2019-09-24 19:21:37,897 - INFO - pid:50074 - access_log:1157 - ::1:49911 - GET None:None/ - None None - 0 bytes +... [redacted] ... - access_log:1157 - ::1:49911 - GET None:None/ - None None - 0 bytes ``` ### CacheResponsesPlugin @@ -887,6 +888,34 @@ for more details. PS: There is no plugin named, but [CustomDnsResolverPlugin](#customdnsresolverplugin) can be easily customized according to your needs. +### ProgramNamePlugin + +Attempts to resolve program `(application)` name for proxy requests originating from the local machine. +If identified, client IP in the access logs is replaced with program name. + +Start `proxy.py` as: + +```console +❯ proxy \ + --plugins proxy.plugin.ProgramNamePlugin +``` + +Make a request using `curl`: + +```console +❯ curl -v -x localhost:8899 https://httpbin.org/get +``` + +You must see log lines like this: + +```console +... [redacted] ... - [I] server.access_log:419 - curl:58096 - CONNECT httpbin.org:443 - 6010 bytes - 1824.62ms +``` + +Notice `curl` in-place of `::1` or `127.0.0.1` as client IP. + +[![WARNING](https://img.shields.io/static/v1?label=Compatibility&message=warning&color=red)](#programnameplugin) If `ProgramNamePlugin` does not work reliably on your operating system, kindly contribute by sending a pull request and/or open an issue. Thank you!!! + ## HTTP Web Server Plugins ### Reverse Proxy diff --git a/proxy/common/_compat.py b/proxy/common/_compat.py deleted file mode 100644 index c3ec75e411..0000000000 --- a/proxy/common/_compat.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on - Network monitoring, controls & Application development, testing, debugging. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. - - Compatibility code for using Proxy.py across various versions of Python. - - .. spelling:: - - compat - py -""" - -import platform - - -SYS_PLATFORM = platform.system() -IS_WINDOWS = SYS_PLATFORM == 'Windows' diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 7b5a947d99..80d12ac29d 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -13,14 +13,17 @@ import time import secrets import pathlib +import platform import sysconfig import ipaddress from typing import Any, List -from ._compat import IS_WINDOWS # noqa: WPS436 from .version import __version__ +SYS_PLATFORM = platform.system() +IS_WINDOWS = SYS_PLATFORM == 'Windows' + def _env_threadless_compliant() -> bool: """Returns true for Python 3.8+ across all platforms @@ -96,12 +99,13 @@ def _env_threadless_compliant() -> bool: DEFAULT_LOG_FILE = None DEFAULT_LOG_FORMAT = '%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(module)s.%(funcName)s:%(lineno)d - %(message)s' DEFAULT_LOG_LEVEL = 'INFO' -DEFAULT_WEB_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - {request_method} {request_path} - {connection_time_ms}ms' -DEFAULT_HTTP_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \ +DEFAULT_WEB_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' \ + '{request_method} {request_path} - {request_ua} - {connection_time_ms}ms' +DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \ '{request_method} {server_host}:{server_port}{request_path} - ' + \ '{response_code} {response_reason} - {response_bytes} bytes - ' + \ '{connection_time_ms}ms' -DEFAULT_HTTPS_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \ +DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \ '{request_method} {server_host}:{server_port} - ' + \ '{response_bytes} bytes - {connection_time_ms}ms' DEFAULT_NUM_ACCEPTORS = 0 diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 3c7f88b855..6161dbd0b6 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -19,7 +19,6 @@ from typing import Optional, List, Any, cast -from ._compat import IS_WINDOWS # noqa: WPS436 from .plugins import Plugins from .types import IpAddress from .utils import bytes_, is_py2, is_threadless, set_open_file_limit @@ -27,7 +26,7 @@ from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, DEFAULT_MIN_COMPRESSION_LIMIT from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE -from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH +from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH, IS_WINDOWS from .logger import Logger from .version import __version__ diff --git a/proxy/common/types.py b/proxy/common/types.py index a95a49e8d2..3d226829bd 100644 --- a/proxy/common/types.py +++ b/proxy/common/types.py @@ -22,6 +22,7 @@ else: from typing_extensions import Protocol + if TYPE_CHECKING: DictQueueType = queue.Queue[Dict[str, Any]] # pragma: no cover else: diff --git a/proxy/common/utils.py b/proxy/common/utils.py index 3a51dbf0f4..0bc2bc9929 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -23,8 +23,10 @@ from types import TracebackType from typing import Optional, Dict, Any, List, Tuple, Type, Callable -from ._compat import IS_WINDOWS # noqa: WPS436 -from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT, DEFAULT_THREADLESS +from .constants import ( + HTTP_1_1, COLON, WHITESPACE, CRLF, + DEFAULT_TIMEOUT, DEFAULT_THREADLESS, IS_WINDOWS, +) if not IS_WINDOWS: import resource diff --git a/proxy/core/event/dispatcher.py b/proxy/core/event/dispatcher.py index 1bcb40201d..d26340d2ea 100644 --- a/proxy/core/event/dispatcher.py +++ b/proxy/core/event/dispatcher.py @@ -83,11 +83,7 @@ def run(self) -> None: self.run_once() except queue.Empty: pass - except BrokenPipeError: - pass - except EOFError: - pass - except KeyboardInterrupt: + except (BrokenPipeError, EOFError, KeyboardInterrupt): pass except Exception as e: logger.exception('Dispatcher exception', exc_info=e) diff --git a/proxy/core/event/subscriber.py b/proxy/core/event/subscriber.py index c92803074f..14b9e8f1a5 100644 --- a/proxy/core/event/subscriber.py +++ b/proxy/core/event/subscriber.py @@ -103,9 +103,7 @@ def unsubscribe(self) -> None: return try: self.event_queue.unsubscribe(self.relay_sub_id) - except BrokenPipeError: - pass - except EOFError: + except (BrokenPipeError, EOFError): pass finally: # self.relay_sub_id = None diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 7c2c41083c..55604abf63 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -38,7 +38,7 @@ from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE from ...common.constants import COMMA, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CERT_FILE from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS -from ...common.constants import DEFAULT_HTTP_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_ACCESS_LOG_FORMAT +from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH from ...common.utils import text_ from ...common.pki import gen_public_key, gen_csr, sign_csr @@ -413,9 +413,9 @@ def on_client_connection_close(self) -> None: ) def access_log(self, log_attrs: Dict[str, Any]) -> None: - access_log_format = DEFAULT_HTTPS_ACCESS_LOG_FORMAT + access_log_format = DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT if not self.request.is_https_tunnel: - access_log_format = DEFAULT_HTTP_ACCESS_LOG_FORMAT + access_log_format = DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT logger.info(access_log_format.format_map(log_attrs)) def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]: diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index 89ed43c618..eda6f7c40b 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -272,10 +272,10 @@ def on_client_connection_close(self) -> None: 'request_method': text_(self.request.method), 'request_path': text_(self.request.path), 'request_bytes': self.request.total_size, - 'request_ua': self.request.header(b'user-agent') + 'request_ua': text_(self.request.header(b'user-agent')) if self.request.has_header(b'user-agent') else None, - 'request_version': self.request.version, + 'request_version': None if not self.request.version else text_(self.request.version), # Response # # TODO: Track and inject web server specific response attributes diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index cd6b3a205f..578bc3bcc4 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -27,6 +27,7 @@ from .modify_chunk_response import ModifyChunkResponsePlugin from .custom_dns_resolver import CustomDnsResolverPlugin from .cloudflare_dns import CloudflareDnsResolverPlugin +from .program_name import ProgramNamePlugin __all__ = [ 'CacheResponsesPlugin', @@ -45,4 +46,5 @@ 'FilterByURLRegexPlugin', 'CustomDnsResolverPlugin', 'CloudflareDnsResolverPlugin', + 'ProgramNamePlugin', ] diff --git a/proxy/plugin/filter_by_url_regex.py b/proxy/plugin/filter_by_url_regex.py index c9659e3732..3d12ad342c 100644 --- a/proxy/plugin/filter_by_url_regex.py +++ b/proxy/plugin/filter_by_url_regex.py @@ -72,8 +72,7 @@ def handle_client_request( request.path, ) # check URL against list - rule_number = 1 - for blocked_entry in self.filters: + for rule_number, blocked_entry in enumerate(self.filters, start=1): # if regex matches on URL if re.search(text_(blocked_entry['regex']), text_(url)): # log that the request has been filtered @@ -90,8 +89,4 @@ def handle_client_request( status_code=httpStatusCodes.NOT_FOUND, reason=b'Blocked', ) - # stop looping through filter list - break - # increment rule number - rule_number += 1 return request diff --git a/proxy/plugin/man_in_the_middle.py b/proxy/plugin/man_in_the_middle.py index a6d6220dfd..7975820ec0 100644 --- a/proxy/plugin/man_in_the_middle.py +++ b/proxy/plugin/man_in_the_middle.py @@ -15,5 +15,5 @@ class ManInTheMiddlePlugin(HttpProxyBasePlugin): """Modifies upstream server responses.""" - def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: + def handle_upstream_chunk(self, _chunk: memoryview) -> memoryview: return okResponse(content=b'Hello from man in the middle') diff --git a/proxy/plugin/program_name.py b/proxy/plugin/program_name.py new file mode 100644 index 0000000000..b8e205b4df --- /dev/null +++ b/proxy/plugin/program_name.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import os +import subprocess + +from typing import Any, Dict, Optional + +from ..common.utils import text_ +from ..common.constants import IS_WINDOWS + +from ..http.parser import HttpParser +from ..http.proxy import HttpProxyBasePlugin + + +class ProgramNamePlugin(HttpProxyBasePlugin): + """Tries to identify the application connecting to the + proxy instance. This is only possible when connection + itself originates from the same machine where the proxy + instance is running.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.program_name: Optional[str] = None + + def before_upstream_connection( + self, request: HttpParser, + ) -> Optional[HttpParser]: + if IS_WINDOWS: + raise NotImplementedError() + assert self.client.addr + if self.client.addr[0] in ('::1', '127.0.0.1'): + assert self.client.addr + port = self.client.addr[1] + ls = subprocess.Popen( + ('lsof', '-i', '-P', '-n'), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + output = subprocess.check_output( + ('grep', '{0}'.format(port)), + stdin=ls.stdout, + ) + port_programs = output.splitlines() + for program in port_programs: + parts = program.split() + if int(parts[1]) != os.getpid(): + self.program_name = text_(parts[0]) + break + except subprocess.CalledProcessError: + pass + finally: + ls.wait(timeout=1) + if self.program_name is None: + self.program_name = self.client.addr[0] + return request + + def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]: + context.update({'client_ip': self.program_name}) + return context diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py index fe5e95707c..5f881a68f7 100644 --- a/proxy/plugin/web_server_route.py +++ b/proxy/plugin/web_server_route.py @@ -13,7 +13,6 @@ from ..http.responses import okResponse from ..http.parser import HttpParser -from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes logger = logging.getLogger(__name__) @@ -37,12 +36,3 @@ def handle_request(self, request: HttpParser) -> None: self.client.queue(HTTP_RESPONSE) elif request.path == b'/https-route-example': self.client.queue(HTTPS_RESPONSE) - - def on_websocket_open(self) -> None: - logger.info('Websocket open') - - def on_websocket_message(self, frame: WebsocketFrame) -> None: - logger.info(frame.data) - - def on_client_connection_close(self) -> None: - logger.debug('Client connection close') diff --git a/tests/core/test_listener.py b/tests/core/test_listener.py index 5ca29e58f9..8d5706953b 100644 --- a/tests/core/test_listener.py +++ b/tests/core/test_listener.py @@ -18,7 +18,7 @@ import pytest from proxy.core.acceptor import Listener -from proxy.common._compat import IS_WINDOWS # noqa: WPS436 +from proxy.common.constants import IS_WINDOWS from proxy.common.flag import FlagParser diff --git a/tests/http/test_http2.py b/tests/http/test_http2.py index dc15740462..942e5edf8b 100644 --- a/tests/http/test_http2.py +++ b/tests/http/test_http2.py @@ -8,10 +8,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import pytest import httpx +import pytest -from proxy.common._compat import IS_WINDOWS # noqa: WPS436 +from proxy.common.constants import IS_WINDOWS from proxy import TestCase diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 9df24fa8d1..bf7df0b889 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -10,14 +10,14 @@ Test the simplest proxy use scenario for smoke. """ +import pytest + from pathlib import Path from subprocess import check_output, Popen from typing import Generator, Any -import pytest - +from proxy.common.constants import IS_WINDOWS from proxy.common.utils import get_available_port -from proxy.common._compat import IS_WINDOWS # noqa: WPS436 # FIXME: Ignore is necessary for as long as pytest hasn't figured out diff --git a/tests/test_set_open_file_limit.py b/tests/test_set_open_file_limit.py index c785b5adb6..f02e3c2215 100644 --- a/tests/test_set_open_file_limit.py +++ b/tests/test_set_open_file_limit.py @@ -13,7 +13,7 @@ import pytest -from proxy.common._compat import IS_WINDOWS # noqa: WPS436 +from proxy.common.constants import IS_WINDOWS from proxy.common.utils import set_open_file_limit if not IS_WINDOWS: diff --git a/tests/testing/test_embed.py b/tests/testing/test_embed.py index 1ab6f404e0..a1f485bf3e 100644 --- a/tests/testing/test_embed.py +++ b/tests/testing/test_embed.py @@ -15,8 +15,7 @@ import pytest from proxy import TestCase -from proxy.common._compat import IS_WINDOWS # noqa: WPS436 -from proxy.common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, PROXY_AGENT_HEADER_VALUE +from proxy.common.constants import DEFAULT_CLIENT_RECVBUF_SIZE, PROXY_AGENT_HEADER_VALUE, IS_WINDOWS from proxy.common.utils import socket_connection, build_http_request from proxy.http import httpMethods from proxy.http.responses import NOT_FOUND_RESPONSE_PKT