Remove unnecessary batches_with_timestamps indexes.

vumi/components/message_store.py

@@ -90,30 +90,26 @@ def load(cls, manager, key, result=None):
 
 
 class OutboundMessage(Model):
-    VERSION = 4
+    VERSION = 5
     MIGRATOR = OutboundMessageMigrator
 
     # key is message_id
     msg = VumiMessage(TransportUserMessage)
     batches = ManyToMany(Batch)
 
     # Extra fields for compound indexes
-    batches_with_timestamps = ListOf(Unicode(), index=True)
     batches_with_addresses = ListOf(Unicode(), index=True)
     batches_with_addresses_reverse = ListOf(Unicode(), index=True)
 
     def save(self):
         # We override this method to set our index fields before saving.
-        self.batches_with_timestamps = []
         self.batches_with_addresses = []
         self.batches_with_addresses_reverse = []
         timestamp = self.msg['timestamp']
         if not isinstance(timestamp, basestring):
             timestamp = format_vumi_date(timestamp)
         reverse_ts = to_reverse_timestamp(timestamp)
         for batch_id in self.batches.keys():
-            self.batches_with_timestamps.append(
-                u"%s$%s" % (batch_id, timestamp))
             self.batches_with_addresses.append(
                 u"%s$%s$%s" % (batch_id, timestamp, self.msg['to_addr']))
             self.batches_with_addresses_reverse.append(
@@ -153,30 +149,26 @@ def save(self):
 
 
 class InboundMessage(Model):
-    VERSION = 4
+    VERSION = 5
     MIGRATOR = InboundMessageMigrator
 
     # key is message_id
     msg = VumiMessage(TransportUserMessage)
     batches = ManyToMany(Batch)
 
     # Extra fields for compound indexes
-    batches_with_timestamps = ListOf(Unicode(), index=True)
     batches_with_addresses = ListOf(Unicode(), index=True)
     batches_with_addresses_reverse = ListOf(Unicode(), index=True)
 
     def save(self):
         # We override this method to set our index fields before saving.
-        self.batches_with_timestamps = []
         self.batches_with_addresses = []
         self.batches_with_addresses_reverse = []
         timestamp = self.msg['timestamp']
         if not isinstance(timestamp, basestring):
             timestamp = format_vumi_date(timestamp)
         reverse_ts = to_reverse_timestamp(timestamp)
         for batch_id in self.batches.keys():
-            self.batches_with_timestamps.append(
-                u"%s$%s" % (batch_id, timestamp))
             self.batches_with_addresses.append(
                 u"%s$%s$%s" % (batch_id, timestamp, self.msg['from_addr']))
             self.batches_with_addresses_reverse.append(

vumi/components/message_store_migrators.py

@@ -139,6 +139,31 @@ def reverse_from_4(self, mdata):
 
         return mdata
 
+    def migrate_from_4(self, mdata):
+        # We copy existing fields and indexes over except for the indexes we're
+        # removing.
+        mdata.set_value('$VERSION', 5)
+        self._copy_msg_field('msg', mdata)
+        mdata.copy_values('batches')
+        mdata.copy_indexes('batches_bin')
+        mdata.copy_indexes('batches_with_addresses_bin')
+        mdata.copy_indexes('batches_with_addresses_reverse_bin')
+
+        return mdata
+
+    def reverse_from_5(self, mdata):
+        # The only difference between v4 and v5 is an index that's computed at
+        # save time, so the reverse migration is identical to the forward
+        # migration except for the version we set.
+        mdata.set_value('$VERSION', 4)
+        self._copy_msg_field('msg', mdata)
+        mdata.copy_values('batches')
+        mdata.copy_indexes('batches_bin')
+        mdata.copy_indexes('batches_with_addresses_bin')
+        mdata.copy_indexes('batches_with_addresses_reverse_bin')
+
+        return mdata
+
 
 class InboundMessageMigrator(MessageMigratorBase):
     def migrate_from_unversioned(self, mdata):
@@ -206,3 +231,28 @@ def reverse_from_4(self, mdata):
         mdata.copy_indexes('batches_with_addresses_bin')
 
         return mdata
+
+    def migrate_from_4(self, mdata):
+        # We copy existing fields and indexes over except for the indexes we're
+        # removing.
+        mdata.set_value('$VERSION', 5)
+        self._copy_msg_field('msg', mdata)
+        mdata.copy_values('batches')
+        mdata.copy_indexes('batches_bin')
+        mdata.copy_indexes('batches_with_addresses_bin')
+        mdata.copy_indexes('batches_with_addresses_reverse_bin')
+
+        return mdata
+
+    def reverse_from_5(self, mdata):
+        # The only difference between v4 and v5 is an index that's computed at
+        # save time, so the reverse migration is identical to the forward
+        # migration except for the version we set.
+        mdata.set_value('$VERSION', 4)
+        self._copy_msg_field('msg', mdata)
+        mdata.copy_values('batches')
+        mdata.copy_indexes('batches_bin')
+        mdata.copy_indexes('batches_with_addresses_bin')
+        mdata.copy_indexes('batches_with_addresses_reverse_bin')
+
+        return mdata

vumi/components/tests/message_store_old_models.py

@@ -1,7 +1,10 @@
 """Previous versions of message store models."""
 
+from calendar import timegm
+from datetime import datetime
 
-from vumi.message import TransportUserMessage, TransportEvent, format_vumi_date
+from vumi.message import (
+    TransportUserMessage, TransportEvent, format_vumi_date, parse_vumi_date)
 from vumi.persist.model import Model
 from vumi.persist.fields import (
     VumiMessage, ForeignKey, ListOf, Dynamic, Tag, Unicode, ManyToMany)
@@ -187,3 +190,94 @@ def save(self):
         self.message_with_status = u"%s$%s$%s" % (
             self.message.key, timestamp, status)
         return super(EventV1, self).save()
+
+
+def to_reverse_timestamp(vumi_timestamp):
+    """
+    Turn a vumi_date-formatted string into a string that sorts in reverse order
+    and can be turned back into a timestamp later.
+
+    This is done by converting to a unix timestamp and subtracting it from
+    0xffffffffff (2**40 - 1) to get a number well outside the range
+    representable by the datetime module. The result is returned as a
+    hexadecimal string.
+    """
+    timestamp = timegm(parse_vumi_date(vumi_timestamp).timetuple())
+    return "%X" % (0xffffffffff - timestamp)
+
+
+def from_reverse_timestamp(reverse_timestamp):
+    """
+    Turn a reverse timestamp string (from `to_reverse_timestamp()`) into a
+    vumi_date-formatted string.
+    """
+    timestamp = 0xffffffffff - int(reverse_timestamp, 16)
+    return format_vumi_date(datetime.utcfromtimestamp(timestamp))
+
+
+class OutboundMessageV4(Model):
+    bucket = 'outboundmessage'
+
+    VERSION = 4
+    MIGRATOR = OutboundMessageMigrator
+
+    # key is message_id
+    msg = VumiMessage(TransportUserMessage)
+    batches = ManyToMany(BatchVNone)
+
+    # Extra fields for compound indexes
+    batches_with_timestamps = ListOf(Unicode(), index=True)
+    batches_with_addresses = ListOf(Unicode(), index=True)
+    batches_with_addresses_reverse = ListOf(Unicode(), index=True)
+
+    def save(self):
+        # We override this method to set our index fields before saving.
+        self.batches_with_timestamps = []
+        self.batches_with_addresses = []
+        self.batches_with_addresses_reverse = []
+        timestamp = self.msg['timestamp']
+        if not isinstance(timestamp, basestring):
+            timestamp = format_vumi_date(timestamp)
+        reverse_ts = to_reverse_timestamp(timestamp)
+        for batch_id in self.batches.keys():
+            self.batches_with_timestamps.append(
+                u"%s$%s" % (batch_id, timestamp))
+            self.batches_with_addresses.append(
+                u"%s$%s$%s" % (batch_id, timestamp, self.msg['to_addr']))
+            self.batches_with_addresses_reverse.append(
+                u"%s$%s$%s" % (batch_id, reverse_ts, self.msg['to_addr']))
+        return super(OutboundMessageV4, self).save()
+
+
+class InboundMessageV4(Model):
+    bucket = 'inboundmessage'
+
+    VERSION = 4
+    MIGRATOR = InboundMessageMigrator
+
+    # key is message_id
+    msg = VumiMessage(TransportUserMessage)
+    batches = ManyToMany(BatchVNone)
+
+    # Extra fields for compound indexes
+    batches_with_timestamps = ListOf(Unicode(), index=True)
+    batches_with_addresses = ListOf(Unicode(), index=True)
+    batches_with_addresses_reverse = ListOf(Unicode(), index=True)
+
+    def save(self):
+        # We override this method to set our index fields before saving.
+        self.batches_with_timestamps = []
+        self.batches_with_addresses = []
+        self.batches_with_addresses_reverse = []
+        timestamp = self.msg['timestamp']
+        if not isinstance(timestamp, basestring):
+            timestamp = format_vumi_date(timestamp)
+        reverse_ts = to_reverse_timestamp(timestamp)
+        for batch_id in self.batches.keys():
+            self.batches_with_timestamps.append(
+                u"%s$%s" % (batch_id, timestamp))
+            self.batches_with_addresses.append(
+                u"%s$%s$%s" % (batch_id, timestamp, self.msg['from_addr']))
+            self.batches_with_addresses_reverse.append(
+                u"%s$%s$%s" % (batch_id, reverse_ts, self.msg['from_addr']))
+        return super(InboundMessageV4, self).save()

vumi/components/tests/test_message_store.py

@@ -236,10 +236,6 @@ def test_add_outbound_message_to_multiple_batches(self):
         self.assertEqual(stored_msg._riak_object.get_indexes(), set([
             ('batches_bin', batch_id_1),
             ('batches_bin', batch_id_2),
-            ('batches_with_timestamps_bin',
-             "%s$%s" % (batch_id_1, timestamp)),
-            ('batches_with_timestamps_bin',
-             "%s$%s" % (batch_id_2, timestamp)),
             ('batches_with_addresses_bin',
              "%s$%s$%s" % (batch_id_1, timestamp, msg['to_addr'])),
             ('batches_with_addresses_bin',
@@ -533,10 +529,6 @@ def test_add_inbound_message_to_multiple_batches(self):
         self.assertEqual(stored_msg._riak_object.get_indexes(), set([
             ('batches_bin', batch_id_1),
             ('batches_bin', batch_id_2),
-            ('batches_with_timestamps_bin',
-             "%s$%s" % (batch_id_1, timestamp)),
-            ('batches_with_timestamps_bin',
-             "%s$%s" % (batch_id_2, timestamp)),
             ('batches_with_addresses_bin',
              "%s$%s$%s" % (batch_id_1, timestamp, msg['from_addr'])),
             ('batches_with_addresses_bin',