diff --git a/api/handlers/dataexplorerhandler.py b/api/handlers/dataexplorerhandler.py index 39b6b56f3..2cd8be03e 100644 --- a/api/handlers/dataexplorerhandler.py +++ b/api/handlers/dataexplorerhandler.py @@ -156,19 +156,22 @@ "subject.sex" : { "terms" : { "field" : "subject.sex.raw", - "size" : 15 + "size" : 15, + "missing": "null" } }, "session.tags" : { "terms" : { "field" : "subject.tags.raw", - "size" : 15 + "size" : 15, + "missing": "null" } }, "subject.code" : { "terms" : { "field" : "subject.code.raw", - "size" : 15 + "size" : 15, + "missing": "null" } }, "session.timestamp" : { @@ -213,13 +216,15 @@ "file.measurements" : { "terms" : { "field" : "file.measurements.raw", - "size" : 15 + "size" : 15, + "missing": "null" } }, "file.type" : { "terms" : { "field" : "file.type.raw", - "size" : 15 + "size" : 15, + "missing": "null" } } } @@ -305,7 +310,40 @@ def _parse_request(self, request_type='search'): for f in filters: if f.get('terms'): for k,v in f['terms'].iteritems(): - modified_filters.append({'terms': {k+'.raw': v}}) + if "null" in v: + if isinstance(v, list): + v.remove("null") + elif isinstance(v, str): + v = None + null_filter = { + 'bool': { + 'should': [ + { + 'bool': { + 'must': [ + { + 'bool':{ + 'must_not': [ + { + 'exists': {'field': k} + } + ] + } + } + ] + } + } + ] + } + } + if len(k.split('.')) > 1: + null_filter['bool']['should'][0]['bool']['must'].append({'exists': {'field': k.split('.')[0]}}) + if v: + null_filter['bool']['should'].append({'terms': {k+'.raw': v}}) + modified_filters.append(null_filter) + + else: + modified_filters.append({'terms': {k+'.raw': v}}) else: modified_filters.append(f) @@ -364,7 +402,8 @@ def aggregate_field_values(self): "results" : { "terms" : { "field" : field_name + ".raw", - "size" : 15 + "size" : 15, + "missing": "null" } } } diff --git a/test/unit_tests/python/test_dataexplorer.py b/test/unit_tests/python/test_dataexplorer.py index 363e55715..96bc3c939 100644 --- a/test/unit_tests/python/test_dataexplorer.py +++ b/test/unit_tests/python/test_dataexplorer.py @@ -140,6 +140,45 @@ def test_search(as_public, as_drone, es): assert r.ok assert r.json['results'] == results + # file search w/ search null filter + r = as_drone.post('/dataexplorer/search', json={'return_type': cont_type, 'all_data': True, 'filters': [ + {'terms': {filter_key: [filter_value, "null"]}}, + ]}) + es.search.assert_called_with( + body={ + '_source': deh.SOURCE[cont_type], + 'query': {'bool': { + 'filter': {'bool': {'must': [ + {'term': {'container_type': cont_type}}, + {'bool': + {'should': + [ + {'bool': + { + 'must': [ + { + 'bool': { + 'must_not': [ + {"exists": {"field":filter_key}} + ] + } + } + ] + } + }, + {'terms': {filter_key + '.raw': [filter_value]}} + ] + } + } + ]}} + }}, + 'size': 100}, + doc_type='flywheel', + index='data_explorer') + assert r.ok + assert r.json['results'] == results + + def test_get_facets(as_public, as_drone, es): # try to get facets w/o login @@ -302,7 +341,7 @@ def test_aggregate_field_values(as_public, as_drone, es): es.search.return_value = {'aggregations': {'results': result}} r = as_drone.post('/dataexplorer/search/fields/aggregate', json={'field_name': field_name}) es.search.assert_called_with( - body={'aggs': {'results': {'terms': {'field': field_name + '.raw', 'size': 15}}}, + body={'aggs': {'results': {'terms': {'field': field_name + '.raw', 'size': 15, 'missing': 'null'}}}, 'query': {'bool': {'filter': [{'term': {'permissions._id': None}}], 'must': {'match_all': {}}}}, 'size': 0}, doc_type='flywheel', @@ -313,7 +352,7 @@ def test_aggregate_field_values(as_public, as_drone, es): # get typeahead w/ search string for string|boolean field type r = as_drone.post('/dataexplorer/search/fields/aggregate', json={'field_name': field_name, 'search_string': search_str}) es.search.assert_called_with( - body={'aggs': {'results': {'terms': {'field': field_name + '.raw', 'size': 15}}}, + body={'aggs': {'results': {'terms': {'field': field_name + '.raw', 'size': 15, 'missing': 'null'}}}, 'query': {'bool': {'filter': [{'term': {'permissions._id': None}}], 'must': {'match': {'field': search_str}}}}, 'size': 0}, doc_type='flywheel',