Progress on exporting KV values.

go/apps/jsbox/definition.py

@@ -10,11 +10,17 @@ class ViewLogsAction(ConversationAction):
     redirect_to = 'jsbox_logs'
 
 
+class ExportAnswersAction(ConversationAction):
+    action_name = 'export_answers'
+    action_display_name = 'Export answers to CSV'
+    redirect_to = 'jsbox_answers'
+
+
 class ConversationDefinition(ConversationDefinitionBase):
     conversation_type = 'jsbox'
     conversation_display_name = 'Javascript App'
 
-    actions = (ViewLogsAction,)
+    actions = (ViewLogsAction, ExportAnswersAction)
 
     def configured_endpoints(self, config):
         # TODO: make jsbox apps define these explicitly and

go/apps/jsbox/kv.py

@@ -0,0 +1,51 @@
+# -*- test-case-name: go.apps.jsbox.tests.test_kv -*-
+# -*- coding: utf-8 -*-
+
+import json
+
+from twisted.internet.defer import returnValue
+
+from vumi.persist.redis_base import Manager
+
+
+class KeyValueManager(object):
+    """
+    Retrieves key value data for a jsbox application.
+    """
+
+    # this uses Manager.calls_manager so that it can be used from
+    # Django.
+
+    def __init__(self, redis):
+        self.redis = self.manager = redis
+
+    def _sandboxed_key(self, sandbox_id, key):
+        # TODO: refactor vumi.application.sandbox.RedisResource
+        #       to make this a static method (or something else
+        #       that allows sharing this logic).
+        return "#".join(["sandboxes", sandbox_id, key])
+
+    def _sub_manager_for_user_store(self, campaign_key, user_store):
+        if user_store:
+            user_key_prefix = "users.%s" % (user_store,)
+        else:
+            user_key_prefix = "users"
+        sub_store_key = self._sandboxed_key(campaign_key, user_key_prefix)
+        sub_redis = self.redis.sub_manager(sub_store_key)
+        # TODO: make key_separator an option on sub_manager
+        sub_redis._key_separator = "."
+        return sub_redis
+
+    @Manager.calls_manager
+    def answers(self, campaign_key, user_store=None):
+        sub_redis = self._sub_manager_for_user_store(campaign_key, user_store)
+        keys = yield sub_redis.keys()
+        items = {}
+        for key in keys:
+            raw_value = yield sub_redis.get(key)
+            try:
+                value = json.loads(raw_value)
+            except:
+                continue
+            items[key] = value
+        returnValue(items)

go/apps/jsbox/tests/test_kv.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+
+import json
+import logging
+import datetime
+import re
+
+from mock import Mock
+from twisted.trial.unittest import TestCase
+from twisted.internet.defer import inlineCallbacks
+
+from vumi.tests.utils import LogCatcher
+
+from go.apps.jsbox.kv import KeyValueManager
+from go.vumitools.tests.utils import GoPersistenceMixin
+
+
+class LogCheckerMixin(object):
+    """Mixing for test cases that want to check logs."""
+    def parse_iso_format(self, iso_string):
+        dt_string, _sep, micro_string = iso_string.partition(".")
+        dt = datetime.datetime.strptime(dt_string, "%Y-%m-%dT%H:%M:%S")
+        microsecond = int(micro_string or '0')
+        return dt.replace(microsecond=microsecond)
+
+    def check_logs(self, actual, expected, epsilon_dt=None):
+        if epsilon_dt is None:
+            epsilon_dt = datetime.timedelta(seconds=5)
+        zero_dt = datetime.timedelta(seconds=0)
+        now = datetime.datetime.utcnow()
+        log_re = re.compile(r"^\[(?P<dt>.*?), (?P<lvl>.*?)\] (?P<msg>.*)$")
+        actual = list(reversed(actual))
+        for msg, (expected_level, expected_msg) in zip(actual, expected):
+            match = log_re.match(msg)
+            self.assertTrue(match is not None,
+                            "Expected formatted log message but got %r"
+                            % (msg,))
+            self.assertEqual(match.group('msg'), expected_msg)
+            self.assertEqual(match.group('lvl'), expected_level)
+            dt = self.parse_iso_format(match.group('dt'))
+            self.assertTrue(zero_dt <= now - dt < epsilon_dt)
+        self.assertEqual(len(actual), len(expected))
+
+
+class TestTxKeyValueManager(TestCase, GoPersistenceMixin, LogCheckerMixin):
+    @inlineCallbacks
+    def setUp(self):
+        super(TestTxKeyValueManager, self).setUp()
+        yield self._persist_setUp()
+        self.parent_redis = yield self.get_redis_manager()
+        self.redis = self.parent_redis.sub_manager(
+            KeyValueManager.DEFAULT_SUB_STORE)
+
+    @inlineCallbacks
+    def tearDown(self):
+        yield super(TestTxKeyValueManager, self).tearDown()
+        yield self._persist_tearDown()
+
+    def kv_manager(self):
+        return KeyValueManager(self.parent_redis)
+
+    @inlineCallbacks
+    def test_add_log(self):
+        lm = self.log_manager()
+        yield lm.add_log("campaign-1", "conv-1", "Hello info!", logging.INFO)
+        logs = yield self.redis.lrange("campaign-1:conv-1", 0, -1)
+        self.check_logs(logs, [("INFO", "Hello info!")])
+
+    @inlineCallbacks
+    def test_add_log_trims(self):
+        lm = self.log_manager(max_logs=10)
+        for i in range(10):
+            yield lm.add_log("campaign-1", "conv-1", "%d" % i, logging.INFO)
+        logs = yield self.redis.lrange("campaign-1:conv-1", 0, -1)
+        self.check_logs(logs, [
+            ("INFO", "%d" % i) for i in range(10)
+        ])
+
+        yield lm.add_log("campaign-1", "conv-1", "10", logging.INFO)
+        logs = yield self.redis.lrange("campaign-1:conv-1", 0, -1)
+        self.check_logs(logs, [
+            ("INFO", "%d" % i) for i in range(1, 11)
+        ])
+
+    @inlineCallbacks
+    def test_get_logs(self):
+        lm = self.log_manager()
+        for i in range(3):
+            yield self.redis.lpush("campaign-1:conv-1", str(i))
+        logs = yield lm.get_logs("campaign-1", "conv-1")
+        self.assertEqual(logs, ['2', '1', '0'])
+
+
+class TestKeyValueManager(TestTxKeyValueManager):
+    sync_persistence = True

go/apps/jsbox/view_definition.py

@@ -1,9 +1,13 @@
+from StringIO import StringIO
+
 from go.conversation.view_definition import (
     ConversationViewDefinitionBase, ConversationTemplateView,
     EditConversationView)
 
 from go.apps.jsbox.forms import JsboxForm, JsboxAppConfigFormset
 from go.apps.jsbox.log import LogManager
+from go.apps.jsbox.kv import KeyValueManager
+from go.base.utils import UnicodeCSVWriter
 
 
 class JSBoxLogsView(ConversationTemplateView):
@@ -22,6 +26,52 @@ def get(self, request, conversation):
         })
 
 
+class JSBoxAnswersView(ConversationTemplateView):
+    view_name = 'jsbox_answers'
+    path_suffix = 'jsbox_answers/'
+
+    @staticmethod
+    def _answers_to_csv(answers):
+        io = StringIO()
+        writer = UnicodeCSVWriter(io)
+
+        fieldnames = set()
+        rows = []
+        for line in infile:
+            try:
+                data = json.loads(line)
+            except ValueError:
+                continue
+            if "key" not in data:
+                continue
+            if key_re.match(data["key"]) is None:
+                continue
+            row = flatten(json.loads(data["value"]))
+            rows.append(row)
+            fieldnames.update(row.keys())
+            row["key"] = data["key"]
+
+        fieldnames = ["key"] + sorted(fieldnames)
+        writer = csv.DictWriter(outfile, fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+
+    def get(self, request, conversation):
+        campaign_key = request.user_api.user_account_key
+        kv_manager = KeyValueManager(request.user_api.api.redis)
+        user_store = conversation.config.get("TODO")
+        answers = kv_manager.answers(campaign_key, user_store)
+
+
+
+        return self.render_to_response({
+            "conversation": conversation,
+            "answers": answers,
+        })
+
+
 class EditJSBoxView(EditConversationView):
     edit_forms = (
         ('jsbox', JsboxForm),