From a3771f7b6042650b634f5b5328fe6d9eb7385fd5 Mon Sep 17 00:00:00 2001 From: Harsha Kethineni Date: Fri, 28 Jul 2017 13:33:31 -0500 Subject: [PATCH 1/4] null search filter --- api/handlers/dataexplorerhandler.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/api/handlers/dataexplorerhandler.py b/api/handlers/dataexplorerhandler.py index 39b6b56f3..2a54b3746 100644 --- a/api/handlers/dataexplorerhandler.py +++ b/api/handlers/dataexplorerhandler.py @@ -305,7 +305,34 @@ def _parse_request(self, request_type='search'): for f in filters: if f.get('terms'): for k,v in f['terms'].iteritems(): - modified_filters.append({'terms': {k+'.raw': v}}) + if "null" in v: + v.remove("null") + null_filter = { + 'bool': { + 'should': [ + { + 'bool': { + 'must': [ + { + 'bool': { + 'must_not': { + 'exists': {'field': k} + } + } + }, + {"exists":{"field":k.split('.')[0]}} + ] + } + } + ] + } + } + if v: + null_filter['bool']['should'].append({'terms': {k+'.raw': v}}) + modified_filters.append(null_filter) + + else: + modified_filters.append({'terms': {k+'.raw': v}}) else: modified_filters.append(f) From 2a1b09024e1615efc1912c17e52a348bd01dbb1e Mon Sep 17 00:00:00 2001 From: Harsha Kethineni Date: Fri, 28 Jul 2017 14:39:39 -0500 Subject: [PATCH 2/4] null search filter unit test --- api/handlers/dataexplorerhandler.py | 13 ++++--- test/unit_tests/python/test_dataexplorer.py | 40 +++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/api/handlers/dataexplorerhandler.py b/api/handlers/dataexplorerhandler.py index 2a54b3746..ea4685913 100644 --- a/api/handlers/dataexplorerhandler.py +++ b/api/handlers/dataexplorerhandler.py @@ -306,7 +306,10 @@ def _parse_request(self, request_type='search'): if f.get('terms'): for k,v in f['terms'].iteritems(): if "null" in v: - v.remove("null") + try: + v.remove("null") + except AttributeError: + v = None null_filter = { 'bool': { 'should': [ @@ -315,9 +318,11 @@ def _parse_request(self, request_type='search'): 'must': [ { 'bool': { - 'must_not': { - 'exists': {'field': k} - } + 'must_not': [ + { + 'exists': {'field': k} + } + ] } }, {"exists":{"field":k.split('.')[0]}} diff --git a/test/unit_tests/python/test_dataexplorer.py b/test/unit_tests/python/test_dataexplorer.py index 363e55715..60f94a6b1 100644 --- a/test/unit_tests/python/test_dataexplorer.py +++ b/test/unit_tests/python/test_dataexplorer.py @@ -140,6 +140,46 @@ def test_search(as_public, as_drone, es): assert r.ok assert r.json['results'] == results + # file search w/ search null filter + r = as_drone.post('/dataexplorer/search', json={'return_type': cont_type, 'all_data': True, 'filters': [ + {'terms': {filter_key: [filter_value, "null"]}}, + ]}) + es.search.assert_called_with( + body={ + '_source': deh.SOURCE[cont_type], + 'query': {'bool': { + 'filter': {'bool': {'must': [ + {'term': {'container_type': cont_type}}, + {'bool': + {'should': + [ + {'bool': + {'must': + [ + {'bool': + {'must_not': + [ + {"exists": {"field":filter_key}} + ] + } + }, + {'exists': {'field': filter_key.split('.')[0]}} + ] + } + }, + {'terms': {filter_key + '.raw': [filter_value]}} + ] + } + } + ]}} + }}, + 'size': 100}, + doc_type='flywheel', + index='data_explorer') + assert r.ok + assert r.json['results'] == results + + def test_get_facets(as_public, as_drone, es): # try to get facets w/o login From 9029f4095f7e8e690bcc4d891e776f82930c4f8d Mon Sep 17 00:00:00 2001 From: Harsha Kethineni Date: Fri, 28 Jul 2017 15:25:20 -0500 Subject: [PATCH 3/4] add missing filter, removed extra bools --- api/handlers/dataexplorerhandler.py | 35 ++++++++++----------- test/unit_tests/python/test_dataexplorer.py | 15 +++------ 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/api/handlers/dataexplorerhandler.py b/api/handlers/dataexplorerhandler.py index ea4685913..1133172d2 100644 --- a/api/handlers/dataexplorerhandler.py +++ b/api/handlers/dataexplorerhandler.py @@ -156,19 +156,22 @@ "subject.sex" : { "terms" : { "field" : "subject.sex.raw", - "size" : 15 + "size" : 15, + "missing": "null" } }, "session.tags" : { "terms" : { "field" : "subject.tags.raw", - "size" : 15 + "size" : 15, + "missing": "null" } }, "subject.code" : { "terms" : { "field" : "subject.code.raw", - "size" : 15 + "size" : 15, + "missing": "null" } }, "session.timestamp" : { @@ -213,13 +216,15 @@ "file.measurements" : { "terms" : { "field" : "file.measurements.raw", - "size" : 15 + "size" : 15, + "missing": "null" } }, "file.type" : { "terms" : { "field" : "file.type.raw", - "size" : 15 + "size" : 15, + "missing": "null" } } } @@ -306,26 +311,19 @@ def _parse_request(self, request_type='search'): if f.get('terms'): for k,v in f['terms'].iteritems(): if "null" in v: - try: + if isinstance(v, list): v.remove("null") - except AttributeError: + elif isinstance(v, str): v = None null_filter = { 'bool': { 'should': [ { 'bool': { - 'must': [ + 'must_not': [ { - 'bool': { - 'must_not': [ - { - 'exists': {'field': k} - } - ] - } - }, - {"exists":{"field":k.split('.')[0]}} + 'exists': {'field': k} + } ] } } @@ -396,7 +394,8 @@ def aggregate_field_values(self): "results" : { "terms" : { "field" : field_name + ".raw", - "size" : 15 + "size" : 15, + "missing": "null" } } } diff --git a/test/unit_tests/python/test_dataexplorer.py b/test/unit_tests/python/test_dataexplorer.py index 60f94a6b1..66834fd9d 100644 --- a/test/unit_tests/python/test_dataexplorer.py +++ b/test/unit_tests/python/test_dataexplorer.py @@ -154,16 +154,9 @@ def test_search(as_public, as_drone, es): {'should': [ {'bool': - {'must': + {'must_not': [ - {'bool': - {'must_not': - [ - {"exists": {"field":filter_key}} - ] - } - }, - {'exists': {'field': filter_key.split('.')[0]}} + {"exists": {"field":filter_key}} ] } }, @@ -342,7 +335,7 @@ def test_aggregate_field_values(as_public, as_drone, es): es.search.return_value = {'aggregations': {'results': result}} r = as_drone.post('/dataexplorer/search/fields/aggregate', json={'field_name': field_name}) es.search.assert_called_with( - body={'aggs': {'results': {'terms': {'field': field_name + '.raw', 'size': 15}}}, + body={'aggs': {'results': {'terms': {'field': field_name + '.raw', 'size': 15, 'missing': 'null'}}}, 'query': {'bool': {'filter': [{'term': {'permissions._id': None}}], 'must': {'match_all': {}}}}, 'size': 0}, doc_type='flywheel', @@ -353,7 +346,7 @@ def test_aggregate_field_values(as_public, as_drone, es): # get typeahead w/ search string for string|boolean field type r = as_drone.post('/dataexplorer/search/fields/aggregate', json={'field_name': field_name, 'search_string': search_str}) es.search.assert_called_with( - body={'aggs': {'results': {'terms': {'field': field_name + '.raw', 'size': 15}}}, + body={'aggs': {'results': {'terms': {'field': field_name + '.raw', 'size': 15, 'missing': 'null'}}}, 'query': {'bool': {'filter': [{'term': {'permissions._id': None}}], 'must': {'match': {'field': search_str}}}}, 'size': 0}, doc_type='flywheel', From 9f6ae934a65fcda30be3e6bc8b8d6b5378f07115 Mon Sep 17 00:00:00 2001 From: Harsha Kethineni Date: Mon, 31 Jul 2017 09:40:50 -0500 Subject: [PATCH 4/4] conditional addition of top level container filter --- api/handlers/dataexplorerhandler.py | 12 ++++++++++-- test/unit_tests/python/test_dataexplorer.py | 12 +++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/api/handlers/dataexplorerhandler.py b/api/handlers/dataexplorerhandler.py index 1133172d2..2cd8be03e 100644 --- a/api/handlers/dataexplorerhandler.py +++ b/api/handlers/dataexplorerhandler.py @@ -320,9 +320,15 @@ def _parse_request(self, request_type='search'): 'should': [ { 'bool': { - 'must_not': [ + 'must': [ { - 'exists': {'field': k} + 'bool':{ + 'must_not': [ + { + 'exists': {'field': k} + } + ] + } } ] } @@ -330,6 +336,8 @@ def _parse_request(self, request_type='search'): ] } } + if len(k.split('.')) > 1: + null_filter['bool']['should'][0]['bool']['must'].append({'exists': {'field': k.split('.')[0]}}) if v: null_filter['bool']['should'].append({'terms': {k+'.raw': v}}) modified_filters.append(null_filter) diff --git a/test/unit_tests/python/test_dataexplorer.py b/test/unit_tests/python/test_dataexplorer.py index 66834fd9d..96bc3c939 100644 --- a/test/unit_tests/python/test_dataexplorer.py +++ b/test/unit_tests/python/test_dataexplorer.py @@ -154,9 +154,15 @@ def test_search(as_public, as_drone, es): {'should': [ {'bool': - {'must_not': - [ - {"exists": {"field":filter_key}} + { + 'must': [ + { + 'bool': { + 'must_not': [ + {"exists": {"field":filter_key}} + ] + } + } ] } },