Merge pull request #1192 from cortesi/testsuite

WIP: Solidify pathod test suite

README.rst

@@ -3,19 +3,24 @@ mitmproxy
 
 |travis| |coveralls| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects, as well as their shared networking library, **netlib**.
+This repository contains the **mitmproxy** and **pathod** projects, as well as
+their shared networking library, **netlib**.
 
-``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console interface.
+``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
+interface.
 
 ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
-``pathoc`` and ``pathod`` are perverse HTTP client and server applications designed to let you craft almost any conceivable HTTP request, including ones that creatively violate the standards.
+``pathoc`` and ``pathod`` are perverse HTTP client and server applications
+designed to let you craft almost any conceivable HTTP request, including ones
+that creatively violate the standards.
 
 
 Documentation & Help
 --------------------
 
-Documentation, tutorials and precompiled binaries can be found on the mitmproxy and pathod websites.
+Documentation, tutorials and precompiled binaries can be found on the mitmproxy
+and pathod websites.
 
 |mitmproxy_site| |pathod_site|
 
@@ -39,8 +44,8 @@ Hacking
 -------
 
 To get started hacking on mitmproxy, make sure you have Python_ 2.7.x. with
-virtualenv_ installed (you can find installation instructions for virtualenv here_).
-Then do the following:
+virtualenv_ installed (you can find installation instructions for virtualenv
+here_). Then do the following:
 
 .. code-block:: text
 
@@ -49,10 +54,11 @@ Then do the following:
     ./dev.sh
 
 
-The *dev* script will create a virtualenv environment in a directory called "venv",
-and install all mandatory and optional dependencies into it.
-The primary mitmproxy components - mitmproxy, netlib and pathod - are installed as "editable",
-so any changes to the source in the repository will be reflected live in the virtualenv.
+The *dev* script will create a virtualenv environment in a directory called
+"venv", and install all mandatory and optional dependencies into it. The
+primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
+"editable", so any changes to the source in the repository will be reflected
+live in the virtualenv.
 
 To confirm that you're up and running, activate the virtualenv, and run the
 mitmproxy test suite:
@@ -63,9 +69,9 @@ mitmproxy test suite:
     py.test
 
 Note that the main executables for the project - ``mitmdump``, ``mitmproxy``,
-``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the virtualenv. After activating the
-virtualenv, they will be on your $PATH, and you can run them like any other
-command:
+``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the
+virtualenv. After activating the virtualenv, they will be on your $PATH, and
+you can run them like any other command:
 
 .. code-block:: text
 
@@ -92,9 +98,9 @@ suite. The project tries to maintain 100% test coverage.
 Documentation
 -------------
 
-The mitmproxy documentation is build using Sphinx_, which is installed automatically if you set up a development
-environment as described above.
-After installation, you can render the documentation like this:
+The mitmproxy documentation is build using Sphinx_, which is installed
+automatically if you set up a development environment as described above. After
+installation, you can render the documentation like this:
 
 .. code-block:: text
 

netlib/tcp.py

@@ -6,6 +6,7 @@
 import threading
 import time
 import traceback
+import contextlib
 
 import binascii
 from six.moves import range
@@ -577,6 +578,12 @@ def alpn_select_callback(conn_, options):
         return context
 
 
+@contextlib.contextmanager
+def _closer(client):
+    yield
+    client.close()
+
+
 class TCPClient(_Connection):
 
     def __init__(self, address, source_address=None):
@@ -708,6 +715,7 @@ def connect(self):
         self.connection = connection
         self.ip_address = Address(connection.getpeername())
         self._makefile()
+        return _closer(self)
 
     def settimeout(self, n):
         self.connection.settimeout(n)
@@ -833,6 +841,25 @@ def get_alpn_proto_negotiated(self):
             return b""
 
 
+class Counter:
+    def __init__(self):
+        self._count = 0
+        self._lock = threading.Lock()
+
+    @property
+    def count(self):
+        with self._lock:
+            return self._count
+
+    def __enter__(self):
+        with self._lock:
+            self._count += 1
+
+    def __exit__(self, *args):
+        with self._lock:
+            self._count -= 1
+
+
 class TCPServer(object):
     request_queue_size = 20
 
@@ -845,15 +872,17 @@ def __init__(self, address):
         self.socket.bind(self.address())
         self.address = Address.wrap(self.socket.getsockname())
         self.socket.listen(self.request_queue_size)
+        self.handler_counter = Counter()
 
     def connection_thread(self, connection, client_address):
-        client_address = Address(client_address)
-        try:
-            self.handle_client_connection(connection, client_address)
-        except:
-            self.handle_error(connection, client_address)
-        finally:
-            close_socket(connection)
+        with self.handler_counter:
+            client_address = Address(client_address)
+            try:
+                self.handle_client_connection(connection, client_address)
+            except:
+                self.handle_error(connection, client_address)
+            finally:
+                close_socket(connection)
 
     def serve_forever(self, poll_interval=0.1):
         self.__is_shut_down.clear()

pathod/pathoc.py

@@ -286,7 +286,7 @@ def connect(self, connect_to=None, showssl=False, fp=sys.stdout):
         if self.use_http2 and not self.ssl:
             raise NotImplementedError("HTTP2 without SSL is not supported.")
 
-        tcp.TCPClient.connect(self)
+        ret = tcp.TCPClient.connect(self)
 
         if connect_to:
             self.http_connect(connect_to)
@@ -324,6 +324,7 @@ def connect(self, connect_to=None, showssl=False, fp=sys.stdout):
 
         if self.timeout:
             self.settimeout(self.timeout)
+        return ret
 
     def stop(self):
         if self.ws_framereader:

pathod/pathod.py

@@ -353,6 +353,8 @@ def __init__(
             staticdir=self.staticdir
         )
 
+        self.loglock = threading.Lock()
+
     def check_policy(self, req, settings):
         """
             A policy check that verifies the request size is within limits.
@@ -403,8 +405,7 @@ def handle_client_connection(self, request, client_address):
 
     def add_log(self, d):
         if not self.noapi:
-            lock = threading.Lock()
-            with lock:
+            with self.loglock:
                 d["id"] = self.logid
                 self.log.insert(0, d)
                 if len(self.log) > self.LOGBUF:
@@ -413,17 +414,18 @@ def add_log(self, d):
             return d["id"]
 
     def clear_log(self):
-        lock = threading.Lock()
-        with lock:
+        with self.loglock:
             self.log = []
 
     def log_by_id(self, identifier):
-        for i in self.log:
-            if i["id"] == identifier:
-                return i
+        with self.loglock:
+            for i in self.log:
+                if i["id"] == identifier:
+                    return i
 
     def get_log(self):
-        return self.log
+        with self.loglock:
+            return self.log
 
 
 def main(args):  # pragma: no cover

pathod/protocols/websockets.py

@@ -18,7 +18,7 @@ def handle_websocket(self, logger):
                     frm = websockets.Frame.from_file(self.pathod_handler.rfile)
                 except NetlibException as e:
                     lg("Error reading websocket frame: %s" % e)
-                    break
+                    return None, None
                 ended = time.time()
                 lg(frm.human_readable())
             retlog = dict(

pathod/test.py

@@ -1,12 +1,14 @@
 from six.moves import cStringIO as StringIO
 import threading
+import time
+
 from six.moves import queue
 
-import requests
-import requests.packages.urllib3
 from . import pathod
 
-requests.packages.urllib3.disable_warnings()
+
+class TimeoutError(Exception):
+    pass
 
 
 class Daemon:
@@ -39,39 +41,51 @@ def p(self, spec):
         """
         return "%s/p/%s" % (self.urlbase, spec)
 
-    def info(self):
-        """
-            Return some basic info about the remote daemon.
-        """
-        resp = requests.get("%s/api/info" % self.urlbase, verify=False)
-        return resp.json()
-
     def text_log(self):
         return self.logfp.getvalue()
 
+    def wait_for_silence(self, timeout=5):
+        start = time.time()
+        while 1:
+            if time.time() - start >= timeout:
+                raise TimeoutError(
+                    "%s service threads still alive" %
+                    self.thread.server.handler_counter.count
+                )
+            if self.thread.server.handler_counter.count == 0:
+                return
+
+    def expect_log(self, n, timeout=5):
+        l = []
+        start = time.time()
+        while True:
+            l = self.log()
+            if time.time() - start >= timeout:
+                return None
+            if len(l) >= n:
+                break
+        return l
+
     def last_log(self):
         """
             Returns the last logged request, or None.
         """
-        l = self.log()
+        l = self.expect_log(1)
         if not l:
             return None
-        return l[0]
+        return l[-1]
 
     def log(self):
         """
             Return the log buffer as a list of dictionaries.
         """
-        resp = requests.get("%s/api/log" % self.urlbase, verify=False)
-        return resp.json()["log"]
+        return self.thread.server.get_log()
 
     def clear_log(self):
         """
             Clear the log.
         """
-        self.logfp.truncate(0)
-        resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False)
-        return resp.ok
+        return self.thread.server.clear_log()
 
     def shutdown(self):
         """
@@ -88,6 +102,7 @@ def __init__(self, iface, q, ssl, daemonargs):
         self.name = "PathodThread"
         self.iface, self.q, self.ssl = iface, q, ssl
         self.daemonargs = daemonargs
+        self.server = None
 
     def run(self):
         self.server = pathod.Pathod(

setup.cfg

@@ -11,8 +11,7 @@ addopts = --capture=no
 
 [coverage:run]
 branch = True
-include = *mitmproxy*, *netlib*, *pathod*
-omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
+omit = *contrib*, *tnetstring*, *platform*, *main.py
 
 [coverage:report]
 show_missing = True

test/pathod/test_app.py

@@ -11,11 +11,11 @@ def test_index(self):
 
     def test_about(self):
         r = self.getpath("/about")
-        assert r.ok
+        assert r.status_code == 200
 
     def test_download(self):
         r = self.getpath("/download")
-        assert r.ok
+        assert r.status_code == 200
 
     def test_docs(self):
         assert self.getpath("/docs/pathod").status_code == 200
@@ -27,7 +27,7 @@ def test_docs(self):
     def test_log(self):
         assert self.getpath("/log").status_code == 200
         assert self.get("200:da").status_code == 200
-        id = self.d.log()[0]["id"]
+        id = self.d.expect_log(1)[0]["id"]
         assert self.getpath("/log").status_code == 200
         assert self.getpath("/log/%s" % id).status_code == 200
         assert self.getpath("/log/9999999").status_code == 404