Write transformed data out to data set

The data group of the data set written to will either be defined by the transform or will be inferred by the data group of the inputted data set.
alphagov · Dec 15, 2014 · 4702eb2 · 4702eb2
1 parent 25d2a46
commit 4702eb2
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 1 deletion.
diff --git a/backdrop/transformers/dispatch.py b/backdrop/transformers/dispatch.py
@@ -55,4 +55,13 @@ def run_transform(data_set_config, transform, earliest, latest):
 
     transformed_data = transform_function(data['data'], transform['options'])
 
-    logger.info(transformed_data)
+    output_group = transform['output'].get('data-group', data_set_config['data_group'])
+    output_type = transform['output']['data-type']
+
+    output_data_set = DataSet.from_group_and_type(
+        config.BACKDROP_URL,
+        output_group,
+        output_type,
+        token=data_set_config['token'],
+    )
+    output_data_set.post(transformed_data)
diff --git a/tests/transformers/test_dispatch.py b/tests/transformers/test_dispatch.py
@@ -33,6 +33,7 @@ def test_entrypoint(self, mock_app, mock_adminAPI):
     @patch('backdrop.transformers.dispatch.DataSet')
     @patch('backdrop.transformers.tasks.debug.logging')
     def test_run_transform(self, mock_logging_task, mock_data_set):
+        mock_logging_task.return_value = [{'new-data': 'point'}]
         data_set_instance = MagicMock()
         data_set_instance.get.return_value = {
             'data': [
@@ -44,6 +45,7 @@ def test_run_transform(self, mock_logging_task, mock_data_set):
         run_transform({
             'data_group': 'group',
             'data_type': 'type',
+            'token': 'foo',
         }, {
             'type': {
                 'function': 'backdrop.transformers.tasks.debug.logging',
@@ -52,8 +54,15 @@ def test_run_transform(self, mock_logging_task, mock_data_set):
                 'period': 'day',
             },
             'options': {},
+            'output': {
+                'data-group': 'other-group',
+                'data-type': 'other-type',
+            },
         }, 'earliest', 'latest')
 
+        mock_data_set.from_group_and_type.assert_any_call(
+            'http://backdrop/data', 'group', 'type',
+        )
         data_set_instance.get.assert_called_with(
             query_parameters={
                 'period': 'day',
@@ -66,3 +75,40 @@ def test_run_transform(self, mock_logging_task, mock_data_set):
             [{'data': 'point'}],
             {}
         )
+        mock_data_set.from_group_and_type.assert_any_call(
+            'http://backdrop/data', 'other-group', 'other-type', token='foo',
+        )
+        data_set_instance.post.assert_called_with([{'new-data': 'point'}])
+
+    @patch('backdrop.transformers.dispatch.DataSet')
+    @patch('backdrop.transformers.tasks.debug.logging')
+    def test_run_transform_no_output_group(self, mock_logging_task, mock_data_set):
+        mock_logging_task.return_value = [{'new-data': 'point'}]
+        data_set_instance = MagicMock()
+        data_set_instance.get.return_value = {
+            'data': [
+                {'data': 'point'},
+            ],
+        }
+        mock_data_set.from_group_and_type.return_value = data_set_instance
+
+        run_transform({
+            'data_group': 'group',
+            'data_type': 'type',
+            'token': 'foo',
+        }, {
+            'type': {
+                'function': 'backdrop.transformers.tasks.debug.logging',
+            },
+            'query-parameters': {
+                'period': 'day',
+            },
+            'options': {},
+            'output': {
+                'data-type': 'other-type',
+            },
+        }, 'earliest', 'latest')
+
+        mock_data_set.from_group_and_type.assert_any_call(
+            'http://backdrop/data', 'group', 'other-type', token='foo',
+        )