Merge branch 'develop' into feature/issue-1171-update-messagecost-for…

…ms-with-session-length

go/account/templates/account/account_summary_mail.txt

@@ -20,8 +20,8 @@ Totals per conversation type:
 {{type}}
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 {% for conversation in conversations %}{{conversation.name}}
-    Sent: {{conversation.count_sent_messages}} to {{conversation.count_outbound_uniques }} uniques.
-    Received: {{conversation.count_replies}} from {{conversation.count_inbound_uniques }} uniques.
+    Sent: {{conversation.count_outbound_messages}} to {{conversation.count_outbound_uniques }} uniques.
+    Received: {{conversation.count_inbound_messages}} from {{conversation.count_inbound_uniques }} uniques.
 
 {% endfor %}
 {% endfor %}

go/account/utils.py

@@ -22,11 +22,12 @@ def get_uniques(contact_store, contact_keys=None,
 def get_messages_count(conversations):
     totals = {}
     for conv in conversations:
-        totals.setdefault(conv.conversation_type, {})
-        totals[conv.conversation_type].setdefault('sent', 0)
-        totals[conv.conversation_type].setdefault('received', 0)
-        totals[conv.conversation_type]['sent'] += conv.count_sent_messages()
-        totals[conv.conversation_type]['received'] += conv.count_replies()
+        conv_type = conv.conversation_type
+        totals.setdefault(conv_type, {})
+        totals[conv_type].setdefault('sent', 0)
+        totals[conv_type].setdefault('received', 0)
+        totals[conv_type]['sent'] += conv.count_outbound_messages()
+        totals[conv_type]['received'] += conv.count_inbound_messages()
     return totals
 
 

go/apps/jsbox/message_store.py

@@ -105,7 +105,7 @@ def handle_count_replies(self, conversation, api, command):
             - ``reason``: Reason for the failure.
 
         """
-        count = yield conversation.count_replies()
+        count = yield conversation.count_inbound_messages()
         returnValue(self.reply(command, success=True, count=count))
 
     @conversation_owner
@@ -131,7 +131,7 @@ def handle_count_sent_messages(self, conversation, api, command):
             - ``reason``: Reason for the failure.
 
         """
-        count = yield conversation.count_sent_messages()
+        count = yield conversation.count_outbound_messages()
         returnValue(self.reply(command, success=True, count=count))
 
     @conversation_owner

go/billing/tests/helpers.py

@@ -48,6 +48,10 @@ def mk_message_cost(**fields):
     fields['message_cost'] = maybe_decimal(fields.get('message_cost', 0.0))
     fields['session_cost'] = maybe_decimal(fields.get('session_cost', 0.0))
     fields['storage_cost'] = maybe_decimal(fields.get('storage_cost', 0.0))
+    fields['session_unit_cost'] = maybe_decimal(
+        fields.get('session_unit_cost', 0.0))
+    fields['session_unit_time'] = maybe_decimal(
+        fields.get('session_unit_time', 0.0))
     fields['markup_percent'] = maybe_decimal(fields.get('markup_percent', 0.0))
     fields.setdefault('message_direction', MessageCost.DIRECTION_INBOUND)
 
@@ -62,6 +66,7 @@ def mk_transaction(account, tag_pool_name='pool1',
                    transaction_type=Transaction.TRANSACTION_TYPE_MESSAGE,
                    message_direction=MessageCost.DIRECTION_INBOUND,
                    message_cost=100, storage_cost=50, session_cost=10,
+                   session_unit_cost=10, session_unit_time=20,
                    markup_percent=10.0, credit_factor=0.25, credit_amount=28,
                    provider=None, status=Transaction.STATUS_COMPLETED,
                    created=None, **kwargs):
@@ -75,6 +80,8 @@ def mk_transaction(account, tag_pool_name='pool1',
         message_cost=maybe_decimal(message_cost),
         storage_cost=maybe_decimal(storage_cost),
         session_cost=maybe_decimal(session_cost),
+        session_unit_cost=maybe_decimal(session_unit_cost),
+        session_unit_time=maybe_decimal(session_unit_time),
         markup_percent=maybe_decimal(markup_percent),
         credit_factor=maybe_decimal(credit_factor),
         credit_amount=credit_amount,

go/billing/tests/test_django_utils.py

@@ -38,8 +38,8 @@ def test_to_json(self):
                 u"markup_percent": 10.0,
                 u"message_cost": 100.0,
                 u"storage_cost": 50.0,
-                u'session_unit_cost': 0.0,
-                u'session_unit_time': 0.0,
+                u'session_unit_cost': 10.0,
+                u'session_unit_time': 20.0,
                 u'session_length_credits': 0.0,
                 u"message_credits": float(get_message_credits(100.0, 10.0)),
                 u"storage_credits": float(get_storage_credits(50.0, 10.0)),

go/conversation/tasks.py

@@ -74,9 +74,8 @@ def row_for_outbound_message(message, mdb):
 def load_messages_in_chunks(conversation, direction='inbound',
                             include_sensitive=False, scrubber=None):
     """
-    Load the conversation's messages in chunks of `size`.
-    Uses `proxy.load_all_bunches()` lower down but allows skipping and/or
-    scrubbing of messages depending on `include_sensitive` and `scrubber`.
+    Load the conversation's messages one index page at a time, skipping and/or
+    scrubbing messages depending on `include_sensitive` and `scrubber`.
 
     :param Conversation conv:
         The conversation.
@@ -90,19 +89,23 @@ def load_messages_in_chunks(conversation, direction='inbound',
         modified on the fly.
     """
     if direction == 'inbound':
-        bunches = conversation.mdb.inbound_messages.load_all_bunches(
-            conversation.inbound_keys())
+        index_page = conversation.mdb.batch_inbound_keys_page(
+            conversation.batch.key)
+        get_msg = conversation.mdb.get_inbound_message
     elif direction == 'outbound':
-        bunches = conversation.mdb.outbound_messages.load_all_bunches(
-            conversation.outbound_keys())
+        index_page = conversation.mdb.batch_outbound_keys_page(
+            conversation.batch.key)
+        get_msg = conversation.mdb.get_outbound_message
     else:
         raise ValueError('Invalid value (%s) received for `direction`. '
                          'Only `inbound` and `outbound` are allowed.' %
                          (direction,))
 
-    for messages in bunches:
+    while index_page is not None:
+        messages = [get_msg(key) for key in index_page]
         yield conversation.filter_and_scrub_messages(
             messages, include_sensitive=include_sensitive, scrubber=scrubber)
+        index_page = index_page.next_page()
 
 
 def email_export(user_profile, conversation, io):

go/conversation/templates/conversation/dashboard.html

@@ -40,7 +40,7 @@
                         </ul>
                     </td>
                     <td class="status {{ conversation.get_status }}">{{ conversation.get_status }}</td>
-                    <td>{{ conversation.count_replies|add:conversation.count_sent_messages }} </td>
+                    <td>{{ conversation.count_inbound_messages|add:conversation.count_outbound_messages }} </td>
                     <!-- <td></td> -->
                     <td title="{{conversation.created_at }}">{{ conversation.created_at|naturalday }}</td>
                     <td>

go/conversation/tests.py

@@ -1023,9 +1023,10 @@ def test_export_conversation_messages_unsorted(self):
             email, 'messages-export.zip', 'messages-export.csv')
         reader = csv.DictReader(fp)
         message_ids = [row['message_id'] for row in reader]
-        self.assertEqual(
-            set(message_ids),
-            set(conv.inbound_keys() + conv.outbound_keys()))
+        all_keys = set()
+        all_keys.update(conv.mdb.batch_inbound_keys(conv.batch.key))
+        all_keys.update(conv.mdb.batch_outbound_keys(conv.batch.key))
+        self.assertEqual(set(message_ids), all_keys)
 
     def test_export_conversation_message_session_events(self):
         conv = self.create_conversation(reply_count=0)

go/conversation/view_definition.py

@@ -215,9 +215,10 @@ def get(self, request, conversation):
     def post(self, request, conversation):
         export_conversation_messages_unsorted.delay(
             request.user_api.user_account_key, conversation.key)
-        messages.info(request, 'Conversation messages CSV file export '
-                                'scheduled. CSV file should arrive in '
-                                'your mailbox shortly.')
+        messages.info(
+            request,
+            'Conversation messages CSV file export scheduled. CSV file should'
+            ' arrive in your mailbox shortly.')
         return self.redirect_to(
             'message_list', conversation_key=conversation.key)
 
@@ -248,14 +249,14 @@ def get(self, request, conversation):
         batch_id = conversation.batch.key
 
         # Paginator starts counting at 1 so 0 would also be invalid
-        inbound_message_paginator = Paginator(
-            PagedMessageCache(conversation.count_replies(),
-                lambda start, stop: conversation.received_messages_in_cache(
-                    start, stop)), 20)
-        outbound_message_paginator = Paginator(
-            PagedMessageCache(conversation.count_sent_messages(),
-                lambda start, stop: conversation.sent_messages_in_cache(
-                    start, stop)), 20)
+        inbound_message_paginator = Paginator(PagedMessageCache(
+            conversation.count_inbound_messages(),
+            lambda start, stop: conversation.received_messages_in_cache(
+                start, stop)), 20)
+        outbound_message_paginator = Paginator(PagedMessageCache(
+            conversation.count_outbound_messages(),
+            lambda start, stop: conversation.sent_messages_in_cache(
+                start, stop)), 20)
 
         tag_context = {
             'batch_id': batch_id,

go/vumitools/conversation/tests/test_utils.py

@@ -56,29 +56,17 @@ def add_channels_to_conversation(self, conv, *channel_tags):
         yield user_account.save()
 
     @inlineCallbacks
-    def test_count_replies(self):
+    def test_count_inbound_messages(self):
         # XXX: Does this test make sense at all?
         yield self.conv.start()
         yield self.msg_helper.add_inbound_to_conv(self.conv, 5)
-        self.assertEqual((yield self.conv.count_replies()), 5)
+        self.assertEqual((yield self.conv.count_inbound_messages()), 5)
 
     @inlineCallbacks
-    def test_inbound_keys(self):
-        messages = yield self.msg_helper.add_inbound_to_conv(self.conv, 5)
-        keys = set([message['message_id'] for message in messages])
-        self.assertEqual(set((yield self.conv.inbound_keys())), keys)
-
-    @inlineCallbacks
-    def test_count_sent_messages(self):
+    def test_count_outbound_messages(self):
         yield self.conv.start()
         yield self.msg_helper.add_outbound_to_conv(self.conv, 5)
-        self.assertEqual((yield self.conv.count_sent_messages()), 5)
-
-    @inlineCallbacks
-    def test_outbound_keys(self):
-        messages = yield self.msg_helper.add_outbound_to_conv(self.conv, 5)
-        keys = set([message['message_id'] for message in messages])
-        self.assertEqual(set((yield self.conv.outbound_keys())), keys)
+        self.assertEqual((yield self.conv.count_outbound_messages()), 5)
 
     @inlineCallbacks
     def test_count_inbound_uniques(self):
@@ -283,22 +271,6 @@ def test_get_progress_status(self):
             'delivery_report_pending': 1,
         })
 
-    @inlineCallbacks
-    def test_get_progress_percentage_acks(self):
-        yield self.conv.start()
-        self.assertEqual((yield self.conv.get_progress_percentage()), 0)
-        outbound = yield self.msg_helper.add_outbound_to_conv(self.conv, 5)
-        yield self.store_events(outbound, 'ack', count=4)
-        self.assertEqual((yield self.conv.get_progress_percentage()), 80)
-
-    @inlineCallbacks
-    def test_get_progress_percentage_nacks(self):
-        yield self.conv.start()
-        self.assertEqual((yield self.conv.get_progress_percentage()), 0)
-        outbound = yield self.msg_helper.add_outbound_to_conv(self.conv, 5)
-        yield self.store_events(outbound, 'nack', count=4)
-        self.assertEqual((yield self.conv.get_progress_percentage()), 80)
-
     @inlineCallbacks
     def test_get_opted_in_contact_bunches(self):
         contact_store = self.user_helper.user_api.contact_store
@@ -368,93 +340,6 @@ def test_get_outbound_throughput(self):
         self.assertEqual(
             (yield self.conv.get_outbound_throughput(sample_time=20)), 60)
 
-    @inlineCallbacks
-    def do_search(self, conv, direction, *args, **kwargs):
-        search_callback = {
-            'inbound': conv.find_inbound_messages_matching,
-            'outbound': conv.find_outbound_messages_matching,
-        }[direction]
-
-        results_callback = {
-            'inbound': conv.get_inbound_messages_for_token,
-            'outbound': conv.get_outbound_messages_for_token,
-        }[direction]
-
-        kwargs.update({'wait': True})
-        token = yield search_callback(*args, **kwargs)
-        messages = yield results_callback(token)
-        returnValue(messages)
-
-    @inlineCallbacks
-    def test_find_inbound_messages_matching(self):
-        yield self.conv.start()
-        yield self.msg_helper.add_inbound_to_conv(self.conv, count=11)
-        matching = yield self.do_search(self.conv, 'inbound', 'inbound')
-        self.assertEqual(len(matching), 11)
-        matching = yield self.do_search(self.conv, 'inbound', 'inbound 0')
-        self.assertEqual(len(matching), 1)
-        matching = yield self.do_search(self.conv, 'inbound', 'inbound 1')
-        self.assertEqual(len(matching), 2)
-        matching = yield self.do_search(self.conv, 'inbound', 'inbound 1$')
-        self.assertEqual(len(matching), 1)
-
-    @inlineCallbacks
-    def test_find_inbound_messages_matching_flags(self):
-        yield self.conv.start()
-        yield self.msg_helper.add_inbound_to_conv(self.conv, count=2)
-        matching = yield self.do_search(
-            self.conv, 'inbound', 'INBOUND', flags="i")
-        self.assertEqual(len(matching), 2)
-        matching = yield self.do_search(
-            self.conv, 'inbound', 'INBOUND', flags="")
-        self.assertEqual(len(matching), 0)
-
-    @inlineCallbacks
-    def test_find_inbound_messages_matching_flags_custom_key(self):
-        yield self.conv.start()
-        yield self.msg_helper.add_inbound_to_conv(self.conv, count=2)
-        matching = yield self.do_search(
-            self.conv, 'inbound', 'FROM', flags='i', key='msg.from_addr')
-        self.assertEqual(len(matching), 2)
-        matching = yield self.do_search(
-            self.conv, 'inbound', 'FROM', flags='', key='msg.from_addr')
-        self.assertEqual(len(matching), 0)
-
-    @inlineCallbacks
-    def test_find_outbound_messages_matching(self):
-        yield self.conv.start()
-        yield self.msg_helper.add_outbound_to_conv(self.conv, count=11)
-        matching = yield self.do_search(self.conv, 'outbound', 'outbound')
-        self.assertEqual(len(matching), 11)
-        matching = yield self.do_search(self.conv, 'outbound', 'outbound 0')
-        self.assertEqual(len(matching), 1)
-        matching = yield self.do_search(self.conv, 'outbound', 'outbound 1')
-        self.assertEqual(len(matching), 2)
-        matching = yield self.do_search(self.conv, 'outbound', 'outbound 1$')
-        self.assertEqual(len(matching), 1)
-
-    @inlineCallbacks
-    def test_find_outbound_messages_matching_flags(self):
-        yield self.conv.start()
-        yield self.msg_helper.add_outbound_to_conv(self.conv, count=2)
-        matching = yield self.do_search(
-            self.conv, 'outbound', 'OUTBOUND', flags='i')
-        self.assertEqual(len(matching), 2)
-        matching = yield self.do_search(
-            self.conv, 'outbound', 'OUTBOUND', flags='')
-        self.assertEqual(len(matching), 0)
-
-    @inlineCallbacks
-    def test_find_outbound_messages_matching_flags_custom_key(self):
-        yield self.conv.start()
-        yield self.msg_helper.add_outbound_to_conv(self.conv, count=2)
-        matching = yield self.do_search(
-            self.conv, 'outbound', 'TO', flags='i', key='msg.to_addr')
-        self.assertEqual(len(matching), 2)
-        matching = yield self.do_search(
-            self.conv, 'outbound', 'TO', flags='', key='msg.to_addr')
-        self.assertEqual(len(matching), 0)
-
     @inlineCallbacks
     def test_get_aggregate_keys(self):
         yield self.conv.start()