From 2342fcc1150f93f2bb31074d7657439eedec28bf Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Fri, 6 Mar 2015 12:30:17 +0200 Subject: [PATCH] standardize filters --- commcare_export/cli.py | 1 + commcare_export/commcare_minilinq.py | 51 +++++++++------------------- commcare_export/excel_query.py | 11 +++--- commcare_export/writers.py | 3 +- tests/test_commcare_minilinq.py | 10 +++--- tests/test_excel_query.py | 12 +++---- 6 files changed, 30 insertions(+), 58 deletions(-) diff --git a/commcare_export/cli.py b/commcare_export/cli.py index e673ff81..08821388 100644 --- a/commcare_export/cli.py +++ b/commcare_export/cli.py @@ -56,6 +56,7 @@ def main(argv): if args.verbose: logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + logging.getLogger('alembic').setLevel(logging.WARN) else: logging.basicConfig(level=logging.WARN, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') diff --git a/commcare_export/commcare_minilinq.py b/commcare_export/commcare_minilinq.py index 0fd8a221..732f2d82 100644 --- a/commcare_export/commcare_minilinq.py +++ b/commcare_export/commcare_minilinq.py @@ -5,9 +5,16 @@ API directly. """ -import simplejson from commcare_export.env import DictEnv, CannotBind, CannotReplace +resource_since_params = { + 'form': ('received_on_start', 'received_on_end'), + 'case': ('server_date_modified_start', 'server_date_modified_end'), + 'device-log': ('date__gte', 'date__lte'), + 'user': None, + 'application': None, + 'web-user': None, +} class CommCareHqEnv(DictEnv): """ @@ -27,45 +34,19 @@ def api_data(self, resource, payload=None, include_referenced_items=None): payload = dict(payload or {}) # Do not mutate passed-in dicts params = {'limit': 1000} - # Currently the form resource endpoint and the case resource endpoint are completely different - if resource == 'form': - if self.since: - if 'filter' not in payload: payload['filter'] = {} - - payload['filter'] = { - "and": [ - payload.get("filter", {"match_all":{}}), - {'range': {'received_on': {'from': self.since.isoformat()}}}, - ] - } + if resource not in resource_since_params: + raise ValueError('I do not know how to access the API resource "%s"' % resource) - if payload: - params.update({'_search': simplejson.dumps(payload, separators=(',',':'))}) # compact encoding - - elif resource == 'case': + if (self.since or self.until) and resource_since_params[resource]: + since_param, until_param = resource_since_params[resource] if self.since: - payload['server_date_modified_start'] = self.since.isoformat() + payload[since_param] = self.since.isoformat() - if payload: - params.update(payload) - - elif resource == 'device-log': - if self.since: - payload['date__gte'] = self.since.isoformat() - if self.until: - payload['date__lte'] = self.until.isoformat() - - if payload: - params.update(payload) - - # these take no since argument - elif resource in ('user','application','web-user'): - if payload: - params.update(payload) + payload[until_param] = self.until.isoformat() - else: - raise ValueError('I do not know how to access the API resource "%s"' % resource) + if payload: + params.update(payload) if include_referenced_items: params.update([ ('%s__full' % referenced_item, 'true') for referenced_item in include_referenced_items]) diff --git a/commcare_export/excel_query.py b/commcare_export/excel_query.py index e05f23b6..f053f161 100644 --- a/commcare_export/excel_query.py +++ b/commcare_export/excel_query.py @@ -113,9 +113,9 @@ def compile_source(worksheet): For example, this spreadsheet - Data Source Filter Name Filter Value Include Referenced Items - ----------------------------- ------------ ------------------ -------------------------- - form[*].form.child_questions app_id cases + Data Source Filter Name Filter Value Include Referenced Items Flags + ----------------------------- ------------ ------------------ -------------------------- ---------------- + form[*].form.child_questions app_id cases include_archived xmlns.exact Should fetch from api/form?app_id=&xmlns.exact=&cases__full=true @@ -144,10 +144,7 @@ def compile_source(worksheet): if include_referenced_items: api_query_args.append(Literal(None)) # Pad the argument list if we have further args; keeps tests and user code more readable at the expense of this conditional else: - if data_source == 'form': - api_query_args.append(Literal( {'filter': {'and': [{'term': {filter_name: filter_value}} for filter_name, filter_value in filters]}})) - elif data_source == 'case': - api_query_args.append(Literal(dict(filters))) + api_query_args.append(Literal(dict(filters))) if include_referenced_items: api_query_args.append(Literal(include_referenced_items)) diff --git a/commcare_export/writers.py b/commcare_export/writers.py index c55f4a66..4212e65d 100644 --- a/commcare_export/writers.py +++ b/commcare_export/writers.py @@ -212,7 +212,7 @@ def best_type_for(self, val): # try to use VARCHAR when possible. # 3. But SQLite really barfs on altering columns. Luckily it does actually have real types, # so we count on other parts of this code to not bother running column alterations - if len(val) < self.MAX_VARCHAR_LEN: # FIXME: Is 255 an interesting cutoff? + if len(val) < self.MAX_VARCHAR_LEN: # FIXME: Is 255 an interesting cutoff? return self.sqlalchemy.Unicode( max(len(val), self.MIN_VARCHAR_LEN) ) else: return self.sqlalchemy.UnicodeText() @@ -313,4 +313,3 @@ def write_table(self, table): self.upsert(self.table(table_name), row_dict) if logger.getEffectiveLevel() == 'DEBUG': sys.stderr.write('\n') - diff --git a/tests/test_commcare_minilinq.py b/tests/test_commcare_minilinq.py index 5a6f7441..b652c6cc 100644 --- a/tests/test_commcare_minilinq.py +++ b/tests/test_commcare_minilinq.py @@ -1,6 +1,4 @@ import unittest -import simplejson -import urllib from itertools import * from jsonpath_rw import jsonpath @@ -26,11 +24,11 @@ def die(msg): raise Exception(msg) client = MockCommCareHqClient({ 'form': [ ( - {'limit': 1000, '_search': simplejson.dumps({"filter":"test1"}, separators=(',',':'))}, + {'limit': 1000, 'filter': 'test1'}, [1, 2, 3], ), ( - {'limit': 1000, '_search': simplejson.dumps({"filter":"test2"}, separators=(',', ':'))}, + {'limit': 1000, 'filter': 'test2'}, [ { 'x': [{ 'y': 1 }, {'y': 2}] }, { 'x': [{ 'y': 3 }, {'z': 4}] }, @@ -38,13 +36,13 @@ def die(msg): raise Exception(msg) ] ), ( - {'limit': 1000, '_search': simplejson.dumps({'filter':'laziness-test'}, separators=(',', ':'))}, + {'limit': 1000, 'filter': 'laziness-test'}, (i if i < 5 else die('Not lazy enough') for i in range(12)) ), ( {'limit': 1000, 'cases__full': 'true'}, [1, 2, 3, 4, 5] - ) + ), ], 'case': [ diff --git a/tests/test_excel_query.py b/tests/test_excel_query.py index 4487f598..50c9a2cf 100644 --- a/tests/test_excel_query.py +++ b/tests/test_excel_query.py @@ -79,13 +79,9 @@ def test_compile_sheet(self): source=Apply(Reference("api_data"), Literal("form"), Literal({ - 'filter': { - 'and': [ - {'term': { 'app_id': 'foobizzle' }}, - {'term': { 'type': 'intake' }} - ] - } - })))), + 'app_id': 'foobizzle', + 'type': 'intake', + }) ))), ('003_DataSourceAndEmitColumns.xlsx', Emit(table = 'Forms', @@ -109,7 +105,7 @@ def test_compile_sheet(self): source=Apply(Reference("api_data"), Literal("form"), Literal(None), - Literal(['foo', 'bar', 'bizzle'])))) + Literal(['foo', 'bar', 'bizzle'])))), ] for filename, minilinq in test_cases: