Ignore data of specified types

For some data points we are only interested in seasonally adjusted
data. This allows us to ignore data points with specified types meaning
that the latest data will always be of the correct type.

backdrop/transformers/tasks/latest_transaction_explorer_values.py

@@ -5,12 +5,12 @@
 from .util import encode_id, group_by, is_latest_data
 
 REQUIRED_DATA_POINTS = [
-    "cost_per_transaction",
-    "digital_cost_per_transaction",
-    "digital_takeup",
-    "number_of_digital_transactions",
-    "number_of_transactions",
-    "total_cost",
+    {'name': "cost_per_transaction", 'ignore': 'quarterly'},
+    {'name': "digital_cost_per_transaction", 'ignore': None},
+    {'name': "digital_takeup", 'ignore': None},
+    {'name': "number_of_digital_transactions", 'ignore': 'quarterly'},
+    {'name': "number_of_transactions", 'ignore': 'quarterly'},
+    {'name': "total_cost", 'ignore': 'quarterly'},
 ]
 
 ADDITIONAL_FIELDS = [
@@ -30,9 +30,17 @@
 
 
 def _get_latest_data_point(data, data_point_name):
+    def _use_data_point(data_point, name, ignore):
+        has_data = (name in data_point and data_point[name])
+        should_not_be_ignored = (ignore != data_point['type'])
+        return has_data and should_not_be_ignored
+
+    name = data_point_name['name']
+    ignore = data_point_name['ignore']
+
     data.sort(key=lambda item: item['_timestamp'], reverse=True)
     for data_point in data:
-        if data_point_name in data_point and data_point[data_point_name]:
+        if _use_data_point(data_point, name, ignore):
             return data_point
     return None
 
@@ -50,7 +58,7 @@ def _get_stripped_down_data_for_data_point_name_only(
     those if present. If a REQUIRED_FIELD is not found we return None for
     this data_point.
     """
-    required_fields = REQUIRED_FIELDS + [data_point_name]
+    required_fields = REQUIRED_FIELDS + [data_point_name['name']]
     new_data = {}
     for field in required_fields:
         if field in latest_data_points:
@@ -63,7 +71,7 @@ def _get_stripped_down_data_for_data_point_name_only(
     new_data['dashboard_slug'] = dashboard_config['slug']
     new_data['_id'] = encode_id(
         new_data['dashboard_slug'],
-        data_point_name)
+        data_point_name['name'])
     return new_data
 
 

tests/transformers/fixtures/transactions_explorer_example_data.json

@@ -1,4 +1,24 @@
 [
+    {
+        "_day_start_at": "2012-01-01T00:00:00+00:00",
+        "_hour_start_at": "2012-01-01T00:00:00+00:00",
+        "_id": "MjAxMi0wMS0wMSAwMDowMDowMDIwMTMtMDEtMDEgMDA6MDA6MDBiaXMtYW5udWFsLXJldHVybnM=",
+        "_month_start_at": "2012-01-01T00:00:00+00:00",
+        "_quarter_start_at": "2012-01-01T00:00:00+00:00",
+        "_timestamp": "2012-12-12T00:00:00+00:00",
+        "_updated_at": "2014-03-19T10:44:32.287000+00:00",
+        "_week_start_at": "2013-12-26T00:00:00+00:00",
+        "cost_per_transaction": 2.63,
+        "digital_cost_per_transaction": 2.36,
+        "digital_takeup": 0.9756123825537215,
+        "end_at": "2013-01-01T00:00:00+00:00",
+        "number_of_digital_transactions": 2301214,
+        "number_of_transactions": 2358738,
+        "period": "year",
+        "service_id": "service-with-quarterly-data",
+        "total_cost": 6203480.9399999995,
+        "type": "quarterly"
+    },
     {
         "_day_start_at": "2013-04-01T00:00:00+00:00",
         "_hour_start_at": "2013-04-01T00:00:00+00:00",

tests/transformers/tasks/test_latest_transaction_explorer_values.py

@@ -22,6 +22,26 @@
     data = json.loads(f.read())
 
 data_to_post = [
+    {
+        "_id": encode_id('quarterly-nonsense', 'digital_cost_per_transaction'),
+        "_timestamp": "2012-12-12T00:00:00+00:00",
+        "digital_cost_per_transaction": 2.36,
+        "end_at": "2013-01-01T00:00:00+00:00",
+        "period": "year",
+        "service_id": "service-with-quarterly-data",
+        "dashboard_slug": "quarterly-nonsense",
+        "type": "quarterly"
+    },
+    {
+        "_id": encode_id('quarterly-nonsense', 'digital_takeup'),
+        "_timestamp": "2012-12-12T00:00:00+00:00",
+        "digital_takeup": 0.9756123825537215,
+        "end_at": "2013-01-01T00:00:00+00:00",
+        "period": "year",
+        "service_id": "service-with-quarterly-data",
+        "dashboard_slug": "quarterly-nonsense",
+        "type": "quarterly"
+    },
     {
         "_id": encode_id('sorn', 'cost_per_transaction'),
         "_timestamp": "2013-04-01T00:00:00+00:00",
@@ -138,6 +158,18 @@
     }
 ]
 
+sorn_dashboard_config = [{
+    'slug': 'sorn'
+}]
+
+bis_returns_dashboard_config = [{
+    'slug': 'bis-returns'
+}]
+
+quarterly_data_dashboard_config = [{
+    'slug': 'quarterly-nonsense'
+}]
+
 
 class ComputeTestCase(unittest.TestCase):
 
@@ -156,23 +188,16 @@ def test_compute(self, mock_dashboard_finder, mock_dataset):
         }
         mock_dataset.return_value = mockdata
 
-        sorn_dashboard_config = [{
-            'slug': 'sorn'
-        }]
-
-        bis_returns_dashboard_config = [{
-            'slug': 'bis-returns'
-        }]
-
         mock_dashboard_finder.side_effect = lambda x: {
             'bis-annual-returns': bis_returns_dashboard_config,
-            'sorn-innit': sorn_dashboard_config
+            'sorn-innit': sorn_dashboard_config,
+            'service-with-quarterly-data': quarterly_data_dashboard_config
         }.get(x, [])
         transformed_data = compute(data, {'output': {
             'data-group': 'transactions-explorer',
             'data-type': 'spreadsheet'}})
 
-        assert_that(len(transformed_data), is_(11))
+        assert_that(len(transformed_data), is_(13))
         assert_that(transformed_data, contains_inanyorder(*data_to_post))
 
     @patch("performanceplatform.client.DataSet.from_group_and_type")
@@ -193,17 +218,10 @@ def test_compute_when_no_new_data(
         }
         mock_dataset.return_value = mockdata
 
-        sorn_dashboard_config = [{
-            'slug': 'sorn'
-        }]
-
-        bis_returns_dashboard_config = [{
-            'slug': 'bis-returns'
-        }]
-
         mock_dashboard_finder.side_effect = lambda x: {
             'bis-annual-returns': bis_returns_dashboard_config,
-            'sorn-innit': sorn_dashboard_config
+            'sorn-innit': sorn_dashboard_config,
+            'service-with-quarterly-data': quarterly_data_dashboard_config
         }.get(x, [])
         transformed_data = compute(data, {'output': {
             'data-group': 'transactions-explorer',