Make mypy succeed with imports on master.py

We get little benefit from our mypy QA checks at the moment, because we skip
imports. This patch is what's needed to make mypy succeed with imports on a
single file: master.py

It also updates mypy to the current version, and enables a QA check.

Mypy bugs I encountered:

dict.update with kwargs not supported:

python/mypy#1031

property setters and getters must be adjacent:

python/mypy#1465

mitmproxy/ctx.py

@@ -1,2 +1,4 @@
+import mitmproxy.master  # noqa
+import mitmproxy.log  # noqa
 master = None  # type: "mitmproxy.master.Master"
 log = None  # type: "mitmproxy.log.Log"

mitmproxy/http.py

@@ -52,9 +52,7 @@ def __init__(
 
     def get_state(self):
         state = super().get_state()
-        state.update(
-            is_replay=self.is_replay
-        )
+        state["is_replay"] = self.is_replay
         return state
 
     def set_state(self, state):
@@ -167,11 +165,12 @@ def __init__(self, client_conn, server_conn, live=None, mode="regular"):
         """ What mode was the proxy layer in when receiving this request? """
 
     _stateobject_attributes = flow.Flow._stateobject_attributes.copy()
-    _stateobject_attributes.update(
+    # mypy doesn't support update with kwargs
+    _stateobject_attributes.update(dict(
         request=HTTPRequest,
         response=HTTPResponse,
         mode=str
-    )
+    ))
 
     def __repr__(self):
         s = "<HTTPFlow"
@@ -223,8 +222,7 @@ def make_error_response(
         status_code=status_code,
         reason=reason,
         message=html.escape(message),
-    )
-    body = body.encode("utf8", "replace")
+    ).encode("utf8", "replace")
 
     if not headers:
         headers = http.Headers(

mitmproxy/net/http/encoding.py

@@ -10,18 +10,22 @@
 import zlib
 import brotli
 
-from typing import Union
+from typing import Union, Optional  # noqa
 
 
 # We have a shared single-element cache for encoding and decoding.
 # This is quite useful in practice, e.g.
 # flow.request.content = flow.request.content.replace(b"foo", b"bar")
 # does not require an .encode() call if content does not contain b"foo"
-CachedDecode = collections.namedtuple("CachedDecode", "encoded encoding errors decoded")
+CachedDecode = collections.namedtuple(
+    "CachedDecode", "encoded encoding errors decoded"
+)
 _cache = CachedDecode(None, None, None, None)
 
 
-def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:
+def decode(
+    encoded: Optional[bytes], encoding: str, errors: str='strict'
+) -> Optional[str]:
     """
     Decode the given input object
 
@@ -62,7 +66,7 @@ def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> U
         ))
 
 
-def encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:
+def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optional[bytes]:
     """
     Encode the given input object
 

mitmproxy/net/http/message.py

@@ -1,5 +1,5 @@
 import re
-from typing import Optional
+from typing import Optional, Union  # noqa
 
 from mitmproxy.utils import strutils
 from mitmproxy.net.http import encoding
@@ -8,6 +8,8 @@
 
 
 class MessageData(serializable.Serializable):
+    content = None  # type: bytes
+
     def __eq__(self, other):
         if isinstance(other, MessageData):
             return self.__dict__ == other.__dict__
@@ -31,6 +33,8 @@ def from_state(cls, state):
 
 
 class Message(serializable.Serializable):
+    data = None  # type: MessageData
+
     def __eq__(self, other):
         if isinstance(other, Message):
             return self.data == other.data
@@ -159,6 +163,7 @@ def _get_content_type_charset(self) -> Optional[str]:
         ct = headers.parse_content_type(self.headers.get("content-type", ""))
         if ct:
             return ct[2].get("charset")
+        return None
 
     def _guess_encoding(self) -> str:
         enc = self._get_content_type_charset()

mitmproxy/net/http/request.py

@@ -82,8 +82,8 @@ def make(
             cls,
             method: str,
             url: str,
-            content: AnyStr = b"",
-            headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]] = ()
+            content: bytes = b"",
+            headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]] = ()
     ):
         """
         Simplified API for creating request objects.
@@ -327,6 +327,15 @@ def pretty_url(self):
             return "%s:%d" % (self.pretty_host, self.port)
         return mitmproxy.net.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path)
 
+    def _get_query(self):
+        query = urllib.parse.urlparse(self.url).query
+        return tuple(mitmproxy.net.http.url.decode(query))
+
+    def _set_query(self, query_data):
+        query = mitmproxy.net.http.url.encode(query_data)
+        _, _, path, params, _, fragment = urllib.parse.urlparse(self.url)
+        self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
+
     @property
     def query(self) -> multidict.MultiDictView:
         """
@@ -337,19 +346,17 @@ def query(self) -> multidict.MultiDictView:
             self._set_query
         )
 
-    def _get_query(self):
-        query = urllib.parse.urlparse(self.url).query
-        return tuple(mitmproxy.net.http.url.decode(query))
-
-    def _set_query(self, query_data):
-        query = mitmproxy.net.http.url.encode(query_data)
-        _, _, path, params, _, fragment = urllib.parse.urlparse(self.url)
-        self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
-
     @query.setter
     def query(self, value):
         self._set_query(value)
 
+    def _get_cookies(self):
+        h = self.headers.get_all("Cookie")
+        return tuple(cookies.parse_cookie_headers(h))
+
+    def _set_cookies(self, value):
+        self.headers["cookie"] = cookies.format_cookie_header(value)
+
     @property
     def cookies(self) -> multidict.MultiDictView:
         """
@@ -362,13 +369,6 @@ def cookies(self) -> multidict.MultiDictView:
             self._set_cookies
         )
 
-    def _get_cookies(self):
-        h = self.headers.get_all("Cookie")
-        return tuple(cookies.parse_cookie_headers(h))
-
-    def _set_cookies(self, value):
-        self.headers["cookie"] = cookies.format_cookie_header(value)
-
     @cookies.setter
     def cookies(self, value):
         self._set_cookies(value)
@@ -426,20 +426,6 @@ def constrain_encoding(self):
                 )
             )
 
-    @property
-    def urlencoded_form(self):
-        """
-        The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
-        An empty multidict.MultiDictView if the content-type indicates non-form data
-        or the content could not be parsed.
-
-        Starting with mitmproxy 1.0, key and value are strings.
-        """
-        return multidict.MultiDictView(
-            self._get_urlencoded_form,
-            self._set_urlencoded_form
-        )
-
     def _get_urlencoded_form(self):
         is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
         if is_valid_content_type:
@@ -457,24 +443,24 @@ def _set_urlencoded_form(self, form_data):
         self.headers["content-type"] = "application/x-www-form-urlencoded"
         self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode()
 
-    @urlencoded_form.setter
-    def urlencoded_form(self, value):
-        self._set_urlencoded_form(value)
-
     @property
-    def multipart_form(self):
+    def urlencoded_form(self):
         """
-        The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
+        The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
         An empty multidict.MultiDictView if the content-type indicates non-form data
         or the content could not be parsed.
 
-        Key and value are bytes.
+        Starting with mitmproxy 1.0, key and value are strings.
         """
         return multidict.MultiDictView(
-            self._get_multipart_form,
-            self._set_multipart_form
+            self._get_urlencoded_form,
+            self._set_urlencoded_form
         )
 
+    @urlencoded_form.setter
+    def urlencoded_form(self, value):
+        self._set_urlencoded_form(value)
+
     def _get_multipart_form(self):
         is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower()
         if is_valid_content_type:
@@ -487,6 +473,20 @@ def _get_multipart_form(self):
     def _set_multipart_form(self, value):
         raise NotImplementedError()
 
+    @property
+    def multipart_form(self):
+        """
+        The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
+        An empty multidict.MultiDictView if the content-type indicates non-form data
+        or the content could not be parsed.
+
+        Key and value are bytes.
+        """
+        return multidict.MultiDictView(
+            self._get_multipart_form,
+            self._set_multipart_form
+        )
+
     @multipart_form.setter
     def multipart_form(self, value):
         self._set_multipart_form(value)

mitmproxy/net/http/response.py

@@ -69,8 +69,8 @@ def __repr__(self):
     def make(
             cls,
             status_code: int=200,
-            content: AnyStr=b"",
-            headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]]=()
+            content: bytes=b"",
+            headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]]=()
     ):
         """
         Simplified API for creating response objects.
@@ -129,6 +129,17 @@ def reason(self):
     def reason(self, reason):
         self.data.reason = strutils.always_bytes(reason, "ISO-8859-1", "surrogateescape")
 
+    def _get_cookies(self):
+        h = self.headers.get_all("set-cookie")
+        return tuple(cookies.parse_set_cookie_headers(h))
+
+    def _set_cookies(self, value):
+        cookie_headers = []
+        for k, v in value:
+            header = cookies.format_set_cookie_header([(k, v[0], v[1])])
+            cookie_headers.append(header)
+        self.headers.set_all("set-cookie", cookie_headers)
+
     @property
     def cookies(self) -> multidict.MultiDictView:
         """
@@ -146,17 +157,6 @@ def cookies(self) -> multidict.MultiDictView:
             self._set_cookies
         )
 
-    def _get_cookies(self):
-        h = self.headers.get_all("set-cookie")
-        return tuple(cookies.parse_set_cookie_headers(h))
-
-    def _set_cookies(self, value):
-        cookie_headers = []
-        for k, v in value:
-            header = cookies.format_set_cookie_header([(k, v[0], v[1])])
-            cookie_headers.append(header)
-        self.headers.set_all("set-cookie", cookie_headers)
-
     @cookies.setter
     def cookies(self, value):
         self._set_cookies(value)

mitmproxy/net/http/url.py

@@ -1,4 +1,4 @@
-import urllib
+import urllib.parse
 from typing import Sequence
 from typing import Tuple
 

mitmproxy/options.py

@@ -332,15 +332,15 @@ def __init__(self, **kwargs) -> None:
             Set supported SSL/TLS versions for client connections. SSLv2, SSLv3
             and 'all' are INSECURE. Defaults to secure, which is TLS1.0+.
             """,
-            choices=tcp.sslversion_choices.keys(),
+            choices=list(tcp.sslversion_choices.keys()),
         )
         self.add_option(
             "ssl_version_server", str, "secure",
             """
             Set supported SSL/TLS versions for server connections. SSLv2, SSLv3
             and 'all' are INSECURE. Defaults to secure, which is TLS1.0+.
             """,
-            choices=tcp.sslversion_choices.keys(),
+            choices=list(tcp.sslversion_choices.keys()),
         )
         self.add_option(
             "ssl_insecure", bool, False,

mitmproxy/proxy/config.py

@@ -37,11 +37,11 @@ class ProxyConfig:
     def __init__(self, options: moptions.Options) -> None:
         self.options = options
 
-        self.check_ignore = None
-        self.check_tcp = None
-        self.certstore = None
-        self.client_certs = None
-        self.openssl_verification_mode_server = None
+        self.check_ignore = None  # type: HostMatcher
+        self.check_tcp = None  # type: HostMatcher
+        self.certstore = None  # type: certs.CertStore
+        self.client_certs = None  # type: str
+        self.openssl_verification_mode_server = None  # type: int
         self.configure(options, set(options.keys()))
         options.changed.connect(self.configure)
 

mitmproxy/proxy/protocol/base.py

@@ -1,5 +1,7 @@
 from mitmproxy import exceptions
 from mitmproxy import connections
+from mitmproxy import controller  # noqa
+from mitmproxy.proxy import config  # noqa
 
 
 class _LayerCodeCompletion:
@@ -12,14 +14,10 @@ def __init__(self, **mixin_args):  # pragma: no cover
         super().__init__(**mixin_args)
         if True:
             return
-        self.config = None
-        """@type: mitmproxy.proxy.ProxyConfig"""
-        self.client_conn = None
-        """@type: mitmproxy.connections.ClientConnection"""
-        self.server_conn = None
-        """@type: mitmproxy.connections.ServerConnection"""
-        self.channel = None
-        """@type: mitmproxy.controller.Channel"""
+        self.config = None  # type: config.ProxyConfig
+        self.client_conn = None  # type: connections.ClientConnection
+        self.server_conn = None  # type: connections.ServerConnection
+        self.channel = None  # type: controller.Channel
         self.ctx = None
         """@type: mitmproxy.proxy.protocol.Layer"""