Adding an in-memory stub version of treq

setup.py

@@ -31,6 +31,7 @@
         "pyOpenSSL",
         "requests >= 2.1.0",
         "service_identity",
+        "six"
     ],
     package_data={"treq": ["_version"]},
     author="David Reid",

treq/client.py

@@ -86,9 +86,11 @@ def deliverBody(self, protocol):
 
 
 class HTTPClient(object):
-    def __init__(self, agent, cookiejar=None):
+    def __init__(self, agent, cookiejar=None,
+                 data_to_body_producer=IBodyProducer):
         self._agent = agent
         self._cookiejar = cookiejar or cookiejar_from_dict({})
+        self._data_to_body_producer = data_to_body_producer
 
     def get(self, url, **kwargs):
         return self.request('GET', url, **kwargs)
@@ -161,7 +163,7 @@ def request(self, method, url, **kwargs):
                 headers.setRawHeaders(
                     'content-type', ['application/x-www-form-urlencoded'])
                 data = urlencode(data, doseq=True)
-            bodyProducer = IBodyProducer(data)
+            bodyProducer = self._data_to_body_producer(data)
 
         cookies = kwargs.get('cookies', {})
 

treq/test/test_testing.py

@@ -0,0 +1,158 @@
+"""
+In-memory treq returns stubbed responses.
+"""
+from inspect import getmembers, isfunction
+
+from six import text_type, binary_type
+
+from twisted.web.client import ResponseFailed
+from twisted.web.error import SchemeNotSupported
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
+
+import treq
+
+from treq.test.util import TestCase
+from treq.testing import StubTreq
+
+
+class _StaticTestResource(Resource):
+    """Resource that always returns 418 "I'm a teapot"""
+    isLeaf = True
+
+    def render(self, request):
+        request.setResponseCode(418)
+        request.setHeader("x-teapot", "teapot!")
+        return "I'm a teapot"
+
+
+class _NonResponsiveTestResource(Resource):
+    """Resource that returns NOT_DONE_YET and never finishes the request"""
+    isLeaf = True
+
+    def render(self, request):
+        return NOT_DONE_YET
+
+
+class StubbingTests(TestCase):
+    """
+    Tests for :class:`StubTreq`.
+    """
+    def test_stubtreq_provides_all_functions_in_treq_all(self):
+        """
+        Every single function and attribute exposed by :obj:`treq.__all__` is
+        provided by :obj:`StubTreq`.
+        """
+        treq_things = [(name, obj) for name, obj in getmembers(treq)
+                       if name in treq.__all__]
+        stub = StubTreq(_StaticTestResource())
+
+        api_things = [(name, obj) for name, obj in treq_things
+                      if obj.__module__ == "treq.api"]
+        content_things = [(name, obj) for name, obj in treq_things
+                          if obj.__module__ == "treq.content"]
+
+        # sanity checks - this test should fail if treq exposes a new API
+        # without changes being made to StubTreq and this test.
+        msg = ("At the time this test was written, StubTreq only knew about "
+               "treq exposing functions from treq.api and treq.content.  If "
+               "this has changed, StubTreq will need to be updated, as will "
+               "this test.")
+        self.assertTrue(all(isfunction(obj) for name, obj in treq_things), msg)
+        self.assertEqual(set(treq_things), set(api_things + content_things),
+                         msg)
+
+        for name, obj in api_things:
+            self.assertTrue(
+                isfunction(getattr(stub, name, None)),
+                "StubTreq.{0} should be a function.".format(name))
+
+        for name, obj in content_things:
+            self.assertIs(
+                getattr(stub, name, None), obj,
+                "StubTreq.{0} should just expose treq.{0}".format(name))
+
+    def test_providing_resource_to_stub_treq(self):
+        """
+        The resource provided to StubTreq responds to every request no
+        matter what the URI or parameters or data.
+        """
+        verbs = ('GET', 'PUT', 'HEAD', 'PATCH', 'DELETE', 'POST')
+        urls = (
+            'http://supports-http.com',
+            'https://supports-https.com',
+            'http://this/has/a/path/and/invalid/domain/name'
+            'https://supports-https.com:8080',
+            'http://supports-http.com:8080',
+        )
+        params = (None, {}, {'page': [1]})
+        headers = (None, {}, {'x-random-header': ['value', 'value2']})
+        data = (None, "", 'some data', '{"some": "json"}')
+
+        stub = StubTreq(_StaticTestResource())
+
+        combos = (
+            (verb, {"url": url, "params": p, "headers": h, "data": d})
+            for verb in verbs
+            for url in urls
+            for p in params
+            for h in headers
+            for d in data
+        )
+        for combo in combos:
+            verb, kwargs = combo
+            deferreds = (stub.request(verb, **kwargs),
+                         getattr(stub, verb.lower())(**kwargs))
+            for d in deferreds:
+                resp = self.successResultOf(d)
+                self.assertEqual(418, resp.code)
+                self.assertEqual(['teapot!'],
+                                 resp.headers.getRawHeaders('x-teapot'))
+                self.assertEqual("" if verb == "HEAD" else "I'm a teapot",
+                                 self.successResultOf(stub.content(resp)))
+
+    def test_handles_invalid_schemes(self):
+        """
+        Invalid URLs errback with a :obj:`SchemeNotSupported` failure, and does
+        so even after a successful request.
+        """
+        stub = StubTreq(_StaticTestResource())
+        self.failureResultOf(stub.get(""), SchemeNotSupported)
+        self.successResultOf(stub.get("http://url.com"))
+        self.failureResultOf(stub.get(""), SchemeNotSupported)
+
+    def test_files_are_rejected(self):
+        """
+        StubTreq does not handle files yet - it should reject requests which
+        attempt to pass files.
+        """
+        stub = StubTreq(_StaticTestResource())
+        self.assertRaises(
+            AssertionError, stub.request,
+            'method', 'http://url', files='some file')
+
+    def test_passing_in_strange_data_is_rejected(self):
+        """
+        StubTreq rejects data that isn't list/dictionary/tuple/bytes/unicode.
+        """
+        stub = StubTreq(_StaticTestResource())
+        self.assertRaises(
+            AssertionError, stub.request, 'method', 'http://url',
+            data=object())
+        self.successResultOf(stub.request('method', 'http://url', data={}))
+        self.successResultOf(stub.request('method', 'http://url', data=[]))
+        self.successResultOf(stub.request('method', 'http://url', data=()))
+        self.successResultOf(
+            stub.request('method', 'http://url', data=binary_type("")))
+        self.successResultOf(
+            stub.request('method', 'http://url', data=text_type("")))
+
+    def test_handles_asynchronous_requests(self):
+        """
+        Handle a resource returning NOT_DONE_YET.
+        """
+        stub = StubTreq(_NonResponsiveTestResource())
+        d = stub.request('method', 'http://url', data="1234")
+        self.assertNoResult(d)
+        d.cancel()
+        self.failureResultOf(d, ResponseFailed)

treq/testing.py

@@ -0,0 +1,207 @@
+"""
+In-memory version of treq for testing.
+"""
+from functools import wraps
+
+from six import string_types
+
+from twisted.test.proto_helpers import StringTransport, MemoryReactor
+
+from twisted.internet.address import IPv4Address
+from twisted.internet.error import ConnectionDone
+from twisted.internet.defer import succeed
+from twisted.internet.interfaces import ISSLTransport
+
+from twisted.python.urlpath import URLPath
+
+from twisted.web.client import Agent
+from twisted.web.server import Site
+from twisted.web.iweb import IAgent, IBodyProducer
+
+from twisted.python.failure import Failure
+
+from zope.interface import directlyProvides, implementer
+
+import treq
+from treq.client import HTTPClient
+
+
+class AbortableStringTransport(StringTransport):
+    """
+    A :obj:`StringTransport` that supports ``abortConnection``.
+    """
+    def abortConnection(self):
+        """
+        Since all connection cessation is immediate in this in-memory
+        transport, just call ``loseConnection``.
+        """
+        self.loseConnection()
+
+
+@implementer(IAgent)
+class RequestTraversalAgent(object):
+    """
+    :obj:`IAgent` implementation that issues an in-memory request rather than
+    going out to a real network socket.
+    """
+
+    def __init__(self, rootResource):
+        """
+        :param rootResource: The twisted IResource at the root of the resource
+            tree.
+        """
+        self._memoryReactor = MemoryReactor()
+        self._realAgent = Agent(reactor=self._memoryReactor)
+        self._rootResource = rootResource
+
+    def request(self, method, uri, headers=None, bodyProducer=None):
+        """
+        Implement IAgent.request.
+        """
+        # We want to use Agent to parse the HTTP response, so let's ask it to
+        # make a request against our in-memory reactor.
+        response = self._realAgent.request(method, uri, headers, bodyProducer)
+
+        # If the request has already finished, just propagate the result.  In
+        # reality this would only happen in failure, but if the agent ever adds
+        # a local cache this might be a success.
+        already_called = []
+
+        def check_already_called(r):
+            already_called.append(r)
+            return r
+        response.addBoth(check_already_called)
+        if already_called:
+            return response
+
+        # That will try to establish an HTTP connection with the reactor's
+        # connectTCP method, and MemoryReactor will place Agent's factory into
+        # the tcpClients list.  Alternately, it will try to establish an HTTPS
+        # connection with the reactor's connectSSL method, and MemoryReactor
+        # will place it into the sslClients list.  We'll extract that.
+        scheme = URLPath.fromString(uri).scheme
+        if scheme == "https":
+            host, port, factory, context_factory, timeout, bindAddress = (
+                self._memoryReactor.sslClients[-1])
+        else:
+            host, port, factory, timeout, bindAddress = (
+                self._memoryReactor.tcpClients[-1])
+
+        # Then we need to convince that factory it's connected to something and
+        # it will give us a protocol for that connection.
+        protocol = factory.buildProtocol(None)
+
+        # We want to capture the output of that connection so we'll make an
+        # in-memory transport.
+        clientTransport = AbortableStringTransport()
+        if scheme == "https":
+            directlyProvides(clientTransport, ISSLTransport)
+
+        # When the protocol is connected to a transport, it ought to send the
+        # whole request because callers of this should not use an asynchronous
+        # bodyProducer.
+        protocol.makeConnection(clientTransport)
+
+        # Get the data from the request.
+        requestData = clientTransport.io.getvalue()
+
+        # Now time for the server to do its job.  Ask it to build an HTTP
+        # channel.
+        channel = Site(self._rootResource).buildProtocol(None)
+
+        # Connect the channel to another in-memory transport so we can collect
+        # the response.
+        serverTransport = AbortableStringTransport()
+        if scheme == "https":
+            directlyProvides(serverTransport, ISSLTransport)
+        serverTransport.hostAddr = IPv4Address('TCP', '127.0.0.1', port)
+        channel.makeConnection(serverTransport)
+
+        # Feed it the data that the Agent synthesized.
+        channel.dataReceived(requestData)
+
+        # Now we have the response data, let's give it back to the Agent.
+        protocol.dataReceived(serverTransport.io.getvalue())
+
+        def finish(r):
+            # By now the Agent should have all it needs to parse a response.
+            protocol.connectionLost(Failure(ConnectionDone()))
+            # Tell it that the connection is now complete so it can clean up.
+            channel.connectionLost(Failure(ConnectionDone()))
+            # Propogate the response.
+            return r
+
+        # Return the response in the accepted format (Deferred firing
+        # IResponse).  This should be synchronously fired, and if not, it's the
+        # system under test's problem.
+        return response.addBoth(finish)
+
+
+@implementer(IBodyProducer)
+class _SynchronousProducer(object):
+    """
+    A partial implementation of an :obj:`IBodyProducer` which produces its
+    entire payload immediately.  There is no way to access to an instance of
+    this object from :obj:`RequestTraversalAgent` or :obj:`StubTreq`, or even a
+    :obj:`Resource: passed to :obj:`StubTreq`.
+
+    This does not implement the :func:`IBodyProducer.stopProducing` method,
+    because that is very difficult to trigger.  (The request from
+    RequestTraversalAgent would have to be canceled while it is still in the
+    transmitting state), and the intent is to use RequestTraversalAgent to
+    make synchronous requests.
+    """
+
+    def __init__(self, body):
+        """
+        Create a synchronous producer with some bytes.
+        """
+        self.body = body
+        msg = ("StubTreq currently only supports url-encodable types, bytes, "
+               "or unicode as data.")
+        assert isinstance(body, string_types), msg
+        self.length = len(body)
+
+    def startProducing(self, consumer):
+        """
+        Immediately produce all data.
+        """
+        consumer.write(self.body)
+        return succeed(None)
+
+
+def _reject_files(f):
+    """
+    Decorator that rejects the 'files' keyword argument to the request
+    functions, because that is not handled by this yet.
+    """
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        if 'files' in kwargs:
+            raise AssertionError("StubTreq cannot handle files.")
+        return f(*args, **kwargs)
+    return wrapper
+
+
+class StubTreq(object):
+    """
+    A fake version of the treq module that can be used for testing that
+    provides all the function calls exposed in treq.__all__.
+
+    :ivar resource: A :obj:`Resource` object that provides the fake responses
+    """
+    def __init__(self, resource):
+        """
+        Construct a client, and pass through client methods and/or
+        treq.content functions.
+        """
+        _client = HTTPClient(agent=RequestTraversalAgent(resource),
+                             data_to_body_producer=_SynchronousProducer)
+        for function_name in treq.__all__:
+            function = getattr(_client, function_name, None)
+            if function is None:
+                function = getattr(treq, function_name)
+            else:
+                function = _reject_files(function)
+
+            setattr(self, function_name, function)