Merge branch 'develop' into feature/issue-862-index-listof-fields

vumi/components/message_formatters.py

@@ -0,0 +1,74 @@
+# -*- test-case-name: vumi.components.tests.test_message_formatters -*-
+
+from csv import writer
+
+from zope.interface import Interface, implements
+
+
+class IMessageFormatter(Interface):
+    """ Interface for writing messages to an HTTP request. """
+
+    def add_http_headers(request):
+        """
+        Add any needed HTTP headers to the request.
+
+        Often used to set the Content-Type header.
+        """
+
+    def write_row_header(request):
+        """
+        Write any header bytes that need to be written to the request before
+        messages.
+        """
+
+    def write_row(request, message):
+        """
+        Write a :class:`TransportUserMessage` to the request.
+        """
+
+
+class JsonFormatter(object):
+    """ Formatter for writing messages to requests as JSON. """
+
+    implements(IMessageFormatter)
+
+    def add_http_headers(self, request):
+        resp_headers = request.responseHeaders
+        resp_headers.addRawHeader(
+            'Content-Type', 'application/json; charset=utf-8')
+
+    def write_row_header(self, request):
+        pass
+
+    def write_row(self, request, message):
+        request.write(message.to_json())
+        request.write('\n')
+
+
+class CsvFormatter(object):
+    """ Formatter for writing messages to requests as CSV. """
+
+    implements(IMessageFormatter)
+
+    FIELDS = (
+        'message_id',
+        'to_addr',
+        'from_addr',
+        'in_reply_to',
+        'session_event',
+        'content',
+        'group',
+    )
+
+    def add_http_headers(self, request):
+        resp_headers = request.responseHeaders
+        resp_headers.addRawHeader(
+            'Content-Type', 'text/csv; charset=utf-8')
+
+    def write_row_header(self, request):
+        writer(request).writerow(self.FIELDS)
+
+    def write_row(self, request, message):
+        writer(request).writerow(list(
+            (message[key] or '').encode('utf-8')
+            for key in self.FIELDS))

vumi/components/message_store_resource.py

@@ -6,6 +6,7 @@
 from twisted.web.server import NOT_DONE_YET
 
 from vumi.components.message_store import MessageStore
+from vumi.components.message_formatters import JsonFormatter, CsvFormatter
 from vumi.config import (
     ConfigDict, ConfigText, ConfigServerEndpoint, ConfigInt,
     ServerEndpointFallback)
@@ -29,15 +30,15 @@ class MessageStoreProxyResource(Resource):
     isLeaf = True
     default_concurrency = 10
 
-    def __init__(self, message_store, batch_id):
+    def __init__(self, message_store, batch_id, formatter):
         Resource.__init__(self)
         self.message_store = message_store
         self.batch_id = batch_id
+        self.formatter = formatter
 
     def render_GET(self, request):
-        resp_headers = request.responseHeaders
-        resp_headers.addRawHeader(
-            'Content-Type', 'application/json; charset=utf-8')
+        self.formatter.add_http_headers(request)
+        self.formatter.write_row_header(request)
 
         if 'concurrency' in request.args:
             concurrency = int(request.args['concurrency'][0])
@@ -100,8 +101,7 @@ def handle_message(self, message_key, request):
         return d
 
     def write_message(self, message, request):
-        request.write(message.to_json())
-        request.write('\n')
+        self.formatter.write_row(request, message)
 
 
 class InboundResource(MessageStoreProxyResource):
@@ -124,19 +124,24 @@ def get_message(self, message_store, message_id):
 
 class BatchResource(Resource):
 
+    RESOURCES = {
+        'inbound.json': (InboundResource, JsonFormatter),
+        'outbound.json': (OutboundResource, JsonFormatter),
+        'inbound.csv': (InboundResource, CsvFormatter),
+        'outbound.csv': (OutboundResource, CsvFormatter),
+    }
+
     def __init__(self, message_store, batch_id):
         Resource.__init__(self)
         self.message_store = message_store
         self.batch_id = batch_id
 
     def getChild(self, path, request):
-        resource_class = {
-            'inbound.json': InboundResource,
-            'outbound.json': OutboundResource,
-        }.get(path)
-        if resource_class is None:
+        if path not in self.RESOURCES:
             return NoResource()
-        return resource_class(self.message_store, self.batch_id)
+        resource_class, message_formatter = self.RESOURCES.get(path)
+        return resource_class(
+            self.message_store, self.batch_id, message_formatter())
 
 
 class MessageStoreResource(Resource):

vumi/components/tests/test_message_formatters.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+
+from twisted.web.test.test_web import DummyRequest
+
+from vumi.components.message_formatters import (
+    IMessageFormatter, JsonFormatter, CsvFormatter)
+
+from vumi.tests.helpers import VumiTestCase, MessageHelper
+
+
+class TestJsonFormatter(VumiTestCase):
+    def setUp(self):
+        self.msg_helper = self.add_helper(MessageHelper())
+        self.request = DummyRequest([''])
+        self.formatter = JsonFormatter()
+
+    def test_implements_IMessageFormatter(self):
+        self.assertTrue(IMessageFormatter.providedBy(self.formatter))
+
+    def test_add_http_headers(self):
+        self.formatter.add_http_headers(self.request)
+        self.assertEqual(
+            self.request.responseHeaders.getRawHeaders('Content-Type'),
+            ['application/json; charset=utf-8'])
+
+    def test_write_row_header(self):
+        self.formatter.write_row_header(self.request)
+        self.assertEqual(self.request.written, [])
+
+    def test_write_row(self):
+        msg = self.msg_helper.make_inbound("foo")
+        self.formatter.write_row(self.request, msg)
+        self.assertEqual(self.request.written, [
+            msg.to_json(), "\n",
+        ])
+
+
+class TestCsvFormatter(VumiTestCase):
+    def setUp(self):
+        self.msg_helper = self.add_helper(MessageHelper())
+        self.request = DummyRequest([''])
+        self.formatter = CsvFormatter()
+
+    def test_implements_IMessageFormatter(self):
+        self.assertTrue(IMessageFormatter.providedBy(self.formatter))
+
+    def test_add_http_headers(self):
+        self.formatter.add_http_headers(self.request)
+        self.assertEqual(
+            self.request.responseHeaders.getRawHeaders('Content-Type'),
+            ['text/csv; charset=utf-8'])
+
+    def test_write_row_header(self):
+        self.formatter.write_row_header(self.request)
+        self.assertEqual(self.request.written, [
+            "message_id,to_addr,from_addr,in_reply_to,session_event,content,"
+            "group\r\n"
+        ])
+
+    def test_write_row(self):
+        msg = self.msg_helper.make_inbound("foo")
+        self.formatter.write_row(self.request, msg)
+        self.assertEqual(self.request.written, [
+            "%s,9292,+41791234567,,,foo,\r\n" % msg['message_id']
+        ])
+
+    def test_write_row_with_in_reply_to(self):
+        msg = self.msg_helper.make_inbound("foo", in_reply_to="msg-2")
+        self.formatter.write_row(self.request, msg)
+        self.assertEqual(self.request.written, [
+            "%s,9292,+41791234567,msg-2,,foo,\r\n" % msg['message_id']
+        ])
+
+    def test_write_row_with_session_event(self):
+        msg = self.msg_helper.make_inbound("foo", session_event="new")
+        self.formatter.write_row(self.request, msg)
+        self.assertEqual(self.request.written, [
+            "%s,9292,+41791234567,,new,foo,\r\n" % msg['message_id']
+        ])
+
+    def test_write_row_with_group(self):
+        msg = self.msg_helper.make_inbound("foo", group="#channel")
+        self.formatter.write_row(self.request, msg)
+        self.assertEqual(self.request.written, [
+            "%s,9292,+41791234567,,,foo,#channel\r\n" % msg['message_id']
+        ])
+
+    def test_write_row_with_unicode_content(self):
+        msg = self.msg_helper.make_inbound(u"føø", group="#channel")
+        self.formatter.write_row(self.request, msg)
+        self.assertEqual(self.request.written, [
+            u"%s,9292,+41791234567,,,føø,#channel\r\n".encode("utf-8")
+            % msg['message_id']
+        ])

vumi/components/tests/test_message_store_resource.py

@@ -81,6 +81,22 @@ def test_get_inbound(self):
             set([msg['message_id'] for msg in messages]),
             set([msg1['message_id'], msg2['message_id']]))
 
+    @inlineCallbacks
+    def test_get_inbound_csv(self):
+        batch_id = yield self.make_batch(('foo', 'bar'))
+        msg1 = yield self.make_inbound(batch_id, 'føø')
+        msg2 = yield self.make_inbound(batch_id, 'føø')
+        resp = yield self.make_request('GET', batch_id, 'inbound.csv')
+        rows = resp.delivered_body.split('\r\n')
+        header, rows = rows[0], rows[1:-1]
+        self.assertEqual(header, (
+            "message_id,to_addr,from_addr,in_reply_to,session_event,content,"
+            "group"))
+        self.assertEqual(sorted(rows), sorted([
+            "%s,9292,+41791234567,,,føø," % msg1['message_id'],
+            "%s,9292,+41791234567,,,føø," % msg2['message_id'],
+        ]))
+
     @inlineCallbacks
     def test_get_outbound(self):
         batch_id = yield self.make_batch(('foo', 'bar'))
@@ -93,6 +109,22 @@ def test_get_outbound(self):
             set([msg['message_id'] for msg in messages]),
             set([msg1['message_id'], msg2['message_id']]))
 
+    @inlineCallbacks
+    def test_get_outbound_csv(self):
+        batch_id = yield self.make_batch(('foo', 'bar'))
+        msg1 = yield self.make_outbound(batch_id, 'føø')
+        msg2 = yield self.make_outbound(batch_id, 'føø')
+        resp = yield self.make_request('GET', batch_id, 'outbound.csv')
+        rows = resp.delivered_body.split('\r\n')
+        header, rows = rows[0], rows[1:-1]
+        self.assertEqual(header, (
+            "message_id,to_addr,from_addr,in_reply_to,session_event,content,"
+            "group"))
+        self.assertEqual(sorted(rows), sorted([
+            "%s,+41791234567,9292,,,føø," % msg1['message_id'],
+            "%s,+41791234567,9292,,,føø," % msg2['message_id'],
+        ]))
+
     @inlineCallbacks
     def test_get_inbound_multiple_pages(self):
         self.store.DEFAULT_MAX_RESULTS = 1