Skip to content

Commit

Permalink
standardize filters
Browse files Browse the repository at this point in the history
  • Loading branch information
@snopoke
snopoke committed Mar 6, 2015
1 parent d88d4a2 commit 2342fcc
Show file tree
Hide file tree
Showing 6 changed files with 30 additions and 58 deletions.
1 change: 1 addition & 0 deletions commcare_export/cli.py
View file
Expand Up @@ -56,6 +56,7 @@ def main(argv):
if args.verbose:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
logging.getLogger('alembic').setLevel(logging.WARN)
else:
logging.basicConfig(level=logging.WARN,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
Expand Down
51 changes: 16 additions & 35 deletions commcare_export/commcare_minilinq.py
View file
Expand Up @@ -5,9 +5,16 @@
API directly.
"""

import simplejson
from commcare_export.env import DictEnv, CannotBind, CannotReplace

resource_since_params = {
'form': ('received_on_start', 'received_on_end'),
'case': ('server_date_modified_start', 'server_date_modified_end'),
'device-log': ('date__gte', 'date__lte'),
'user': None,
'application': None,
'web-user': None,
}

class CommCareHqEnv(DictEnv):
"""
Expand All @@ -27,45 +34,19 @@ def api_data(self, resource, payload=None, include_referenced_items=None):
payload = dict(payload or {}) # Do not mutate passed-in dicts
params = {'limit': 1000}

# Currently the form resource endpoint and the case resource endpoint are completely different
if resource == 'form':
if self.since:
if 'filter' not in payload: payload['filter'] = {}

payload['filter'] = {
"and": [
payload.get("filter", {"match_all":{}}),
{'range': {'received_on': {'from': self.since.isoformat()}}},
]
}
if resource not in resource_since_params:
raise ValueError('I do not know how to access the API resource "%s"' % resource)

if payload:
params.update({'_search': simplejson.dumps(payload, separators=(',',':'))}) # compact encoding

elif resource == 'case':
if (self.since or self.until) and resource_since_params[resource]:
since_param, until_param = resource_since_params[resource]
if self.since:
payload['server_date_modified_start'] = self.since.isoformat()
payload[since_param] = self.since.isoformat()

if payload:
params.update(payload)

elif resource == 'device-log':
if self.since:
payload['date__gte'] = self.since.isoformat()

if self.until:
payload['date__lte'] = self.until.isoformat()

if payload:
params.update(payload)

# these take no since argument
elif resource in ('user','application','web-user'):
if payload:
params.update(payload)
payload[until_param] = self.until.isoformat()

else:
raise ValueError('I do not know how to access the API resource "%s"' % resource)
if payload:
params.update(payload)

if include_referenced_items:
params.update([ ('%s__full' % referenced_item, 'true') for referenced_item in include_referenced_items])
Expand Down
11 changes: 4 additions & 7 deletions commcare_export/excel_query.py
View file
Expand Up @@ -113,9 +113,9 @@ def compile_source(worksheet):
For example, this spreadsheet
Data Source Filter Name Filter Value Include Referenced Items
----------------------------- ------------ ------------------ --------------------------
form[*].form.child_questions app_id <app id> cases
Data Source Filter Name Filter Value Include Referenced Items Flags
----------------------------- ------------ ------------------ -------------------------- ----------------
form[*].form.child_questions app_id <app id> cases include_archived
xmlns.exact <some form xmlns>
Should fetch from api/form?app_id=<app id>&xmlns.exact=<some form xmlns>&cases__full=true
Expand Down Expand Up @@ -144,10 +144,7 @@ def compile_source(worksheet):
if include_referenced_items:
api_query_args.append(Literal(None)) # Pad the argument list if we have further args; keeps tests and user code more readable at the expense of this conditional
else:
if data_source == 'form':
api_query_args.append(Literal( {'filter': {'and': [{'term': {filter_name: filter_value}} for filter_name, filter_value in filters]}}))
elif data_source == 'case':
api_query_args.append(Literal(dict(filters)))
api_query_args.append(Literal(dict(filters)))

if include_referenced_items:
api_query_args.append(Literal(include_referenced_items))
Expand Down
3 changes: 1 addition & 2 deletions commcare_export/writers.py
View file
Expand Up @@ -212,7 +212,7 @@ def best_type_for(self, val):
# try to use VARCHAR when possible.
# 3. But SQLite really barfs on altering columns. Luckily it does actually have real types,
# so we count on other parts of this code to not bother running column alterations
if len(val) < self.MAX_VARCHAR_LEN: # FIXME: Is 255 an interesting cutoff?
if len(val) < self.MAX_VARCHAR_LEN: # FIXME: Is 255 an interesting cutoff?
return self.sqlalchemy.Unicode( max(len(val), self.MIN_VARCHAR_LEN) )
else:
return self.sqlalchemy.UnicodeText()
Expand Down Expand Up @@ -313,4 +313,3 @@ def write_table(self, table):
self.upsert(self.table(table_name), row_dict)

if logger.getEffectiveLevel() == 'DEBUG': sys.stderr.write('\n')

10 changes: 4 additions & 6 deletions tests/test_commcare_minilinq.py
View file
@@ -1,6 +1,4 @@
import unittest
import simplejson
import urllib
from itertools import *

from jsonpath_rw import jsonpath
Expand All @@ -26,25 +24,25 @@ def die(msg): raise Exception(msg)
client = MockCommCareHqClient({
'form': [
(
{'limit': 1000, '_search': simplejson.dumps({"filter":"test1"}, separators=(',',':'))},
{'limit': 1000, 'filter': 'test1'},
[1, 2, 3],
),
(
{'limit': 1000, '_search': simplejson.dumps({"filter":"test2"}, separators=(',', ':'))},
{'limit': 1000, 'filter': 'test2'},
[
{ 'x': [{ 'y': 1 }, {'y': 2}] },
{ 'x': [{ 'y': 3 }, {'z': 4}] },
{ 'x': [{ 'y': 5 }] }
]
),
(
{'limit': 1000, '_search': simplejson.dumps({'filter':'laziness-test'}, separators=(',', ':'))},
{'limit': 1000, 'filter': 'laziness-test'},
(i if i < 5 else die('Not lazy enough') for i in range(12))
),
(
{'limit': 1000, 'cases__full': 'true'},
[1, 2, 3, 4, 5]
)
),
],

'case': [
Expand Down
12 changes: 4 additions & 8 deletions tests/test_excel_query.py
View file
Expand Up @@ -79,13 +79,9 @@ def test_compile_sheet(self):
source=Apply(Reference("api_data"),
Literal("form"),
Literal({
'filter': {
'and': [
{'term': { 'app_id': 'foobizzle' }},
{'term': { 'type': 'intake' }}
]
}
})))),
'app_id': 'foobizzle',
'type': 'intake',
}) ))),

('003_DataSourceAndEmitColumns.xlsx',
Emit(table = 'Forms',
Expand All @@ -109,7 +105,7 @@ def test_compile_sheet(self):
source=Apply(Reference("api_data"),
Literal("form"),
Literal(None),
Literal(['foo', 'bar', 'bizzle']))))
Literal(['foo', 'bar', 'bizzle'])))),
]

for filename, minilinq in test_cases:
Expand Down

0 comments on commit 2342fcc

Please sign in to comment.