diff --git a/mitmproxy/ctx.py b/mitmproxy/ctx.py index adae8cf68c..7b5231e6ff 100644 --- a/mitmproxy/ctx.py +++ b/mitmproxy/ctx.py @@ -1,2 +1,4 @@ +import mitmproxy.master # noqa +import mitmproxy.log # noqa master = None # type: "mitmproxy.master.Master" log = None # type: "mitmproxy.log.Log" diff --git a/mitmproxy/http.py b/mitmproxy/http.py index c6b1753347..c09778fea7 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -52,9 +52,7 @@ def __init__( def get_state(self): state = super().get_state() - state.update( - is_replay=self.is_replay - ) + state["is_replay"] = self.is_replay return state def set_state(self, state): @@ -167,11 +165,12 @@ def __init__(self, client_conn, server_conn, live=None, mode="regular"): """ What mode was the proxy layer in when receiving this request? """ _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( + # mypy doesn't support update with kwargs + _stateobject_attributes.update(dict( request=HTTPRequest, response=HTTPResponse, mode=str - ) + )) def __repr__(self): s = " Union[str, bytes]: +def decode( + encoded: Optional[bytes], encoding: str, errors: str='strict' +) -> Optional[str]: """ Decode the given input object @@ -62,7 +66,7 @@ def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> U )) -def encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]: +def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optional[bytes]: """ Encode the given input object diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 506674d630..1040c6ce17 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -1,5 +1,5 @@ import re -from typing import Optional +from typing import Optional, Union # noqa from mitmproxy.utils import strutils from mitmproxy.net.http import encoding @@ -8,6 +8,8 @@ class MessageData(serializable.Serializable): + content = None # type: bytes + def __eq__(self, other): if isinstance(other, MessageData): return self.__dict__ == other.__dict__ @@ -31,6 +33,8 @@ def from_state(cls, state): class Message(serializable.Serializable): + data = None # type: MessageData + def __eq__(self, other): if isinstance(other, Message): return self.data == other.data @@ -159,6 +163,7 @@ def _get_content_type_charset(self) -> Optional[str]: ct = headers.parse_content_type(self.headers.get("content-type", "")) if ct: return ct[2].get("charset") + return None def _guess_encoding(self) -> str: enc = self._get_content_type_charset() diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 90a1f924c3..d23b500a7c 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -82,8 +82,8 @@ def make( cls, method: str, url: str, - content: AnyStr = b"", - headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]] = () + content: bytes = b"", + headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]] = () ): """ Simplified API for creating request objects. @@ -327,6 +327,15 @@ def pretty_url(self): return "%s:%d" % (self.pretty_host, self.port) return mitmproxy.net.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path) + def _get_query(self): + query = urllib.parse.urlparse(self.url).query + return tuple(mitmproxy.net.http.url.decode(query)) + + def _set_query(self, query_data): + query = mitmproxy.net.http.url.encode(query_data) + _, _, path, params, _, fragment = urllib.parse.urlparse(self.url) + self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) + @property def query(self) -> multidict.MultiDictView: """ @@ -337,19 +346,17 @@ def query(self) -> multidict.MultiDictView: self._set_query ) - def _get_query(self): - query = urllib.parse.urlparse(self.url).query - return tuple(mitmproxy.net.http.url.decode(query)) - - def _set_query(self, query_data): - query = mitmproxy.net.http.url.encode(query_data) - _, _, path, params, _, fragment = urllib.parse.urlparse(self.url) - self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) - @query.setter def query(self, value): self._set_query(value) + def _get_cookies(self): + h = self.headers.get_all("Cookie") + return tuple(cookies.parse_cookie_headers(h)) + + def _set_cookies(self, value): + self.headers["cookie"] = cookies.format_cookie_header(value) + @property def cookies(self) -> multidict.MultiDictView: """ @@ -362,13 +369,6 @@ def cookies(self) -> multidict.MultiDictView: self._set_cookies ) - def _get_cookies(self): - h = self.headers.get_all("Cookie") - return tuple(cookies.parse_cookie_headers(h)) - - def _set_cookies(self, value): - self.headers["cookie"] = cookies.format_cookie_header(value) - @cookies.setter def cookies(self, value): self._set_cookies(value) @@ -426,20 +426,6 @@ def constrain_encoding(self): ) ) - @property - def urlencoded_form(self): - """ - The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. - An empty multidict.MultiDictView if the content-type indicates non-form data - or the content could not be parsed. - - Starting with mitmproxy 1.0, key and value are strings. - """ - return multidict.MultiDictView( - self._get_urlencoded_form, - self._set_urlencoded_form - ) - def _get_urlencoded_form(self): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: @@ -457,24 +443,24 @@ def _set_urlencoded_form(self, form_data): self.headers["content-type"] = "application/x-www-form-urlencoded" self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode() - @urlencoded_form.setter - def urlencoded_form(self, value): - self._set_urlencoded_form(value) - @property - def multipart_form(self): + def urlencoded_form(self): """ - The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. + The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. An empty multidict.MultiDictView if the content-type indicates non-form data or the content could not be parsed. - Key and value are bytes. + Starting with mitmproxy 1.0, key and value are strings. """ return multidict.MultiDictView( - self._get_multipart_form, - self._set_multipart_form + self._get_urlencoded_form, + self._set_urlencoded_form ) + @urlencoded_form.setter + def urlencoded_form(self, value): + self._set_urlencoded_form(value) + def _get_multipart_form(self): is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower() if is_valid_content_type: @@ -487,6 +473,20 @@ def _get_multipart_form(self): def _set_multipart_form(self, value): raise NotImplementedError() + @property + def multipart_form(self): + """ + The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. + An empty multidict.MultiDictView if the content-type indicates non-form data + or the content could not be parsed. + + Key and value are bytes. + """ + return multidict.MultiDictView( + self._get_multipart_form, + self._set_multipart_form + ) + @multipart_form.setter def multipart_form(self, value): self._set_multipart_form(value) diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 53c9c1ca6a..cfef31954a 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -69,8 +69,8 @@ def __repr__(self): def make( cls, status_code: int=200, - content: AnyStr=b"", - headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]]=() + content: bytes=b"", + headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]]=() ): """ Simplified API for creating response objects. @@ -129,6 +129,17 @@ def reason(self): def reason(self, reason): self.data.reason = strutils.always_bytes(reason, "ISO-8859-1", "surrogateescape") + def _get_cookies(self): + h = self.headers.get_all("set-cookie") + return tuple(cookies.parse_set_cookie_headers(h)) + + def _set_cookies(self, value): + cookie_headers = [] + for k, v in value: + header = cookies.format_set_cookie_header([(k, v[0], v[1])]) + cookie_headers.append(header) + self.headers.set_all("set-cookie", cookie_headers) + @property def cookies(self) -> multidict.MultiDictView: """ @@ -146,17 +157,6 @@ def cookies(self) -> multidict.MultiDictView: self._set_cookies ) - def _get_cookies(self): - h = self.headers.get_all("set-cookie") - return tuple(cookies.parse_set_cookie_headers(h)) - - def _set_cookies(self, value): - cookie_headers = [] - for k, v in value: - header = cookies.format_set_cookie_header([(k, v[0], v[1])]) - cookie_headers.append(header) - self.headers.set_all("set-cookie", cookie_headers) - @cookies.setter def cookies(self, value): self._set_cookies(value) diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index 86ce976425..f2c8c4736c 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -1,4 +1,4 @@ -import urllib +import urllib.parse from typing import Sequence from typing import Tuple diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 5b84ac9302..703928032b 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -332,7 +332,7 @@ def __init__(self, **kwargs) -> None: Set supported SSL/TLS versions for client connections. SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. """, - choices=tcp.sslversion_choices.keys(), + choices=list(tcp.sslversion_choices.keys()), ) self.add_option( "ssl_version_server", str, "secure", @@ -340,7 +340,7 @@ def __init__(self, **kwargs) -> None: Set supported SSL/TLS versions for server connections. SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. """, - choices=tcp.sslversion_choices.keys(), + choices=list(tcp.sslversion_choices.keys()), ) self.add_option( "ssl_insecure", bool, False, diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 8417ebad7b..b809d89a56 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -37,11 +37,11 @@ class ProxyConfig: def __init__(self, options: moptions.Options) -> None: self.options = options - self.check_ignore = None - self.check_tcp = None - self.certstore = None - self.client_certs = None - self.openssl_verification_mode_server = None + self.check_ignore = None # type: HostMatcher + self.check_tcp = None # type: HostMatcher + self.certstore = None # type: certs.CertStore + self.client_certs = None # type: str + self.openssl_verification_mode_server = None # type: int self.configure(options, set(options.keys())) options.changed.connect(self.configure) diff --git a/mitmproxy/proxy/protocol/base.py b/mitmproxy/proxy/protocol/base.py index b10bb8f585..7c0f78ae5a 100644 --- a/mitmproxy/proxy/protocol/base.py +++ b/mitmproxy/proxy/protocol/base.py @@ -1,5 +1,7 @@ from mitmproxy import exceptions from mitmproxy import connections +from mitmproxy import controller # noqa +from mitmproxy.proxy import config # noqa class _LayerCodeCompletion: @@ -12,14 +14,10 @@ def __init__(self, **mixin_args): # pragma: no cover super().__init__(**mixin_args) if True: return - self.config = None - """@type: mitmproxy.proxy.ProxyConfig""" - self.client_conn = None - """@type: mitmproxy.connections.ClientConnection""" - self.server_conn = None - """@type: mitmproxy.connections.ServerConnection""" - self.channel = None - """@type: mitmproxy.controller.Channel""" + self.config = None # type: config.ProxyConfig + self.client_conn = None # type: connections.ClientConnection + self.server_conn = None # type: connections.ServerConnection + self.channel = None # type: controller.Channel self.ctx = None """@type: mitmproxy.proxy.protocol.Layer""" diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 166922340a..9f783bc323 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -3,10 +3,11 @@ from mitmproxy import exceptions from mitmproxy import connections +from mitmproxy import controller # noqa from mitmproxy import http from mitmproxy import log from mitmproxy import platform -from mitmproxy.proxy import ProxyConfig +from mitmproxy.proxy import config from mitmproxy.proxy import modes from mitmproxy.proxy import root_context from mitmproxy.net import tcp @@ -34,7 +35,7 @@ class ProxyServer(tcp.TCPServer): allow_reuse_address = True bound = True - def __init__(self, config: ProxyConfig): + def __init__(self, config: config.ProxyConfig) -> None: """ Raises ServerException if there's a startup problem. """ @@ -49,7 +50,7 @@ def __init__(self, config: ProxyConfig): raise exceptions.ServerException( 'Error starting proxy server: ' + repr(e) ) from e - self.channel = None + self.channel = None # type: controller.Channel def set_channel(self, channel): self.channel = channel @@ -67,8 +68,7 @@ def handle_client_connection(self, conn, client_address): class ConnectionHandler: def __init__(self, client_conn, client_address, config, channel): - self.config = config - """@type: mitmproxy.proxy.config.ProxyConfig""" + self.config = config # type: config.ProxyConfig self.client_conn = connections.ClientConnection( client_conn, client_address, diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 1415900171..a0deaec933 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,5 +1,6 @@ from typing import Any from typing import List +from typing import MutableMapping # noqa from mitmproxy.types import serializable @@ -19,7 +20,7 @@ class StateObject(serializable.Serializable): or StateObject instances themselves. """ - _stateobject_attributes = None + _stateobject_attributes = None # type: MutableMapping[str, Any] """ An attribute-name -> class-or-type dict containing all attributes that should be serialized. If the attribute is a class, it must implement the diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py index 067fbfe307..fe9f217bb8 100644 --- a/mitmproxy/tcp.py +++ b/mitmproxy/tcp.py @@ -41,9 +41,7 @@ def __init__(self, client_conn, server_conn, live=None): self.messages = [] # type: List[TCPMessage] _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - messages=List[TCPMessage] - ) + _stateobject_attributes["messages"] = List[TCPMessage] def __repr__(self): return "".format(len(self.messages)) diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 2946561599..1b90c2e50b 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -1,11 +1,11 @@ import re import codecs -from typing import AnyStr, Optional +from typing import AnyStr, Optional, cast def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]: if isinstance(str_or_bytes, bytes) or str_or_bytes is None: - return str_or_bytes + return cast(Optional[bytes], str_or_bytes) elif isinstance(str_or_bytes, str): return str_or_bytes.encode(*encode_args) else: @@ -18,7 +18,7 @@ def always_str(str_or_bytes: Optional[AnyStr], *decode_args) -> Optional[str]: str_or_bytes unmodified, if """ if isinstance(str_or_bytes, str) or str_or_bytes is None: - return str_or_bytes + return cast(Optional[str], str_or_bytes) elif isinstance(str_or_bytes, bytes): return str_or_bytes.decode(*decode_args) else: diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index bdd83ee6fc..3d58da68ea 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -1,7 +1,7 @@ import typing -def check_type(name: str, value: typing.Any, typeinfo: type) -> None: +def check_type(name: str, value: typing.Any, typeinfo: typing.Any) -> None: """ This function checks if the provided value is an instance of typeinfo and raises a TypeError otherwise. diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index 5d76aafc29..2efa7ad124 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -8,11 +8,13 @@ class WebSocketMessage(serializable.Serializable): - def __init__(self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None): + def __init__( + self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None + ) -> None: self.type = type self.from_client = from_client self.content = content - self.timestamp = timestamp or time.time() # type: int + self.timestamp = timestamp or int(time.time()) # type: int @classmethod def from_state(cls, state): @@ -62,7 +64,8 @@ def __init__(self, client_conn, server_conn, handshake_flow, live=None): self.handshake_flow = handshake_flow _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( + # mypy doesn't support update with kwargs + _stateobject_attributes.update(dict( messages=List[WebSocketMessage], close_sender=str, close_code=str, @@ -77,7 +80,7 @@ def __init__(self, client_conn, server_conn, handshake_flow, live=None): # Do not include handshake_flow, to prevent recursive serialization! # Since mitmproxy-console currently only displays HTTPFlows, # dumping the handshake_flow will include the WebSocketFlow too. - ) + )) @classmethod def from_state(cls, state): diff --git a/setup.py b/setup.py index 9ea69f67a6..ec28eded75 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ 'dev': [ "Flask>=0.10.1, <0.13", "flake8>=3.2.1, <3.4", - "mypy>=0.471, <0.502", + "mypy>=0.501, <0.502", "rstcheck>=2.2, <4.0", "tox>=2.3, <3", "pytest>=3, <3.1", diff --git a/tox.ini b/tox.ini index a9904e8760..a1ed53f7a3 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,8 @@ commands = mitmproxy/tools/dump.py \ mitmproxy/tools/web/ \ mitmproxy/contentviews/ + mypy --ignore-missing-imports \ + mitmproxy/master.py [testenv:individual_coverage] deps =