Skip to content
This repository has been archived by the owner on Mar 24, 2021. It is now read-only.

Commit

Permalink
Check value is latest against target dataset
Browse files Browse the repository at this point in the history 
not source - this is more reliable for things with multiple potential
sources like digital takeup.

These is_latest_data tests (the stuff around calls need to be moved to
test_utils though it is work keeping some component integration tests
for sanity.
  • Loading branch information
@jcbashdown
jcbashdown committed Jan 23, 2015
1 parent b42b75c commit 7e720ca
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 12 deletions.
11 changes: 10 additions & 1 deletion backdrop/transformers/tasks/latest_transaction_explorer_values.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,16 @@ def _get_data_points_for_each_tx_metric(data, transform, data_set_config):
for data_point_name in REQUIRED_DATA_POINTS:
latest_datum = _get_stripped_down_data_for_data_point_name_only(
dashboard_config, latest_data, data_point_name)
if is_latest_data(data_set_config, transform, latest_datum):
# we need to look at, for example,
# digital-takeup on the output data set - tx not the only source.
if is_latest_data({'data_group': transform['output']['data-group'],
'data_type': transform['output']['data-type']},
transform,
latest_datum,
additional_read_params={
'filter_by': 'dashboard_slug:{}'.format(
latest_datum['dashboard_slug'])
}):
yield latest_datum


Expand Down
9 changes: 7 additions & 2 deletions backdrop/transformers/tasks/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,10 @@ def _get_read_params(transform_params, latest_timestamp):
return read_params


def is_latest_data(data_set_config, transform, latest_datum):
def is_latest_data(data_set_config,
transform,
latest_datum,
additional_read_params={}):
"""
Read from backdrop to determine if new data is the latest.
"""
Expand All @@ -52,8 +55,10 @@ def is_latest_data(data_set_config, transform, latest_datum):
)

transform_params = transform.get('query_parameters', {})
read_params = _get_read_params(
generated_read_params = _get_read_params(
transform_params, latest_datum['_timestamp'])
read_params = dict(
generated_read_params.items() + additional_read_params.items())
existing_data = data_set.get(query_parameters=read_params)

if existing_data['data']:
Expand Down
44 changes: 35 additions & 9 deletions tests/transformers/tasks/test_latest_transaction_explorer_values.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from backdrop.transformers.tasks.util import(
encode_id)

from mock import patch, Mock
from mock import patch, Mock, call

import json
import os
Expand Down Expand Up @@ -178,11 +178,23 @@ def test_compute(self, mock_dashboard_finder, mock_dataset):
'bis-annual-returns': bis_returns_dashboard_config,
'sorn-innit': sorn_dashboard_config
}.get(x, [])
# we need to look at digital-takeup on target - tx not only source.
transformed_data = compute(data, {}, {
'data_group': 'transactions-explorer',
'data_type': 'spreadsheet'})

transformed_data = compute(data, {'output': {
'data-group': 'transactions-explorer',
'data-type': 'spreadsheet'}})

calls = [call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'})]
mockdata.get.has_calls(calls)
assert_that(transformed_data, contains_inanyorder(*data_to_post))
assert_that(len(transformed_data), is_(12))

Expand Down Expand Up @@ -216,9 +228,23 @@ def test_compute_when_no_new_data(
'bis-annual-returns': bis_returns_dashboard_config,
'sorn-innit': sorn_dashboard_config
}.get(x, [])
transformed_data = compute(data, {}, {
'data_group': 'transactions-explorer',
'data_type': 'spreadsheet'})
transformed_data = compute(data, {'output': {
'data-group': 'transactions-explorer',
'data-type': 'spreadsheet'}})

calls = [call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:bis-returns', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'}),
call(query_parameters={'filter_by': 'dashboard_slug:sorn', 'start_at': u'2013-04-01T00:00:00+00:00', 'sort_by': '_timestamp:descending'})]
mockdata.get.has_calls(calls)

assert_that(transformed_data, equal_to([]))
assert_that(len(transformed_data), is_(0))

0 comments on commit 7e720ca

Please sign in to comment.