Implement batch_last_updated for postgres

Moved the test that was mongo specific into the base class that runs for
all storage engines and made it a bit more robust (previously it would
have passed with:
```
def batch_last_updated(self, data_sets):
  pass
```).

backdrop/core/storage/postgres.py

@@ -14,6 +14,7 @@
     create_find_record_query,
     create_update_record_query,
     create_delete_record_query,
+    create_batch_last_updated_query,
     CREATE_TABLE_SQL,
 )
 from .. import timeutils
@@ -63,8 +64,13 @@ def get_last_updated(self, data_set_id):
             return _parse_datetime_fields(record)['_updated_at']
 
     def batch_last_updated(self, data_sets):
-        # TODO - this requires quite a complex query
-        pass
+        collections = [collection.name for collection in data_sets]
+        with self.connection.cursor() as cursor:
+            cursor.execute(create_batch_last_updated_query(cursor.mogrify, collections))
+            results = cursor.fetchall()
+            timestamp_by_collection = {collection: max_timestamp for [collection, max_timestamp] in results}
+            for data_set in data_sets:
+                data_set._last_updated = timestamp_by_collection[data_set.name].replace(tzinfo=pytz.UTC)
 
     def empty_data_set(self, data_set_id):
         self.delete_data_set(data_set_id)

backdrop/core/storage/sql_query_factory.py

@@ -100,6 +100,16 @@ def create_delete_record_query(mogrify, data_set_id, record_id):
     )
 
 
+def create_batch_last_updated_query(mogrify, collections):
+    query_template = """
+        SELECT collection, max(timestamp) FROM mongo
+        WHERE collection in ({}) group by collection
+    """
+    return mogrify(
+        query_template.format(','.join(['%s' for collection in collections])),
+        collections
+    )
+
 def create_sql_query(mogrify, data_set_id, user_query):
     """
     Creates a sql query and a funtion which transforms the output into a list

tests/core/storage/test_mongo.py

@@ -34,25 +34,6 @@ def test_create_data_set(self):
 
         assert_that(indicies, has_key('_timestamp_-1'))
 
-    def test_batch_last_updated(self):
-        timestamp = time_as_utc(datetime.datetime.utcnow())
-        self.engine.create_data_set('some_data', 0)
-        self.engine.save_record('some_data', {
-            '_timestamp': timestamp,
-        })
-
-        data_set = DataSet(self.engine, {'name': 'some_data'})
-
-        self.engine.batch_last_updated([data_set])
-        last_updated = data_set.get_last_updated()
-
-        assert_that(last_updated.year, is_(timestamp.year))
-        assert_that(last_updated.month, is_(timestamp.month))
-        assert_that(last_updated.day, is_(timestamp.day))
-        assert_that(last_updated.hour, is_(timestamp.hour))
-        assert_that(last_updated.minute, is_(timestamp.minute))
-        assert_that(last_updated.second, is_(timestamp.second))
-
     def teardown(self):
         mongo_client = self.engine._mongo_client
         database_name = mongo_client.get_database().name

tests/core/storage/test_storage.py

@@ -5,6 +5,7 @@
     instance_of, has_entry, contains_inanyorder
 from nose.tools import assert_raises
 
+from backdrop.core.data_set import DataSet
 from backdrop.core.errors import DataSetCreationError
 from backdrop.core.query import Query
 from backdrop.core.records import add_period_keys
@@ -325,3 +326,36 @@ def test_basic_query_with_inclusive_time_limits(self):
             inclusive=True))
 
         assert_that(len(results), is_(3))
+
+    def test_batch_last_updated(self):
+        records = {
+            # timestamps in ascending order
+            'some_data': [
+                {'_id':'test_id_42', '_timestamp': d_tz(2018, 1, 1)},
+                {'_id':'test_id_43', '_timestamp': d_tz(2019, 1, 1)},
+                {'_id':'test_id_44', '_timestamp': d_tz(2020, 1, 1)},
+            ],
+            # timestamps in descending order
+            'some_other_data': [
+                {'_id':'test_id_45', '_timestamp': d_tz(2017, 1, 1)},
+                {'_id':'test_id_46', '_timestamp': d_tz(2016, 1, 1)},
+                {'_id':'test_id_47', '_timestamp': d_tz(2015, 1, 1)},
+            ]
+        }
+
+        for key, items in records.iteritems():
+            self.engine.create_data_set(key, 0)
+            for item in items:
+                self.engine.save_record(key, item)
+
+        some_data_set = DataSet(self.engine, {'name': 'some_data'})
+        some_other_data_set = DataSet(self.engine, {'name': 'some_other_data'})
+
+        self.engine.batch_last_updated([some_data_set, some_other_data_set])
+
+        some_data_set_last_updated = some_data_set.get_last_updated()
+        some_other_data_set_last_updated = some_other_data_set.get_last_updated()
+
+        assert_that(some_data_set_last_updated, is_(d_tz(2020, 1, 1, 0, 0, 0)))
+        assert_that(some_other_data_set_last_updated, is_(d_tz(2017, 1, 1, 0, 0, 0)))
+