diff --git a/complaint_search/es_builders.py b/complaint_search/es_builders.py index 5dd32a62..abb152f0 100644 --- a/complaint_search/es_builders.py +++ b/complaint_search/es_builders.py @@ -94,7 +94,15 @@ def build(self): search = { "from": self.params.get("frm"), "size": self.params.get("size"), - "query": {"match_all": {}}, + "query": { + "query_string": { + "query": "*", + "fields": [ + self.params.get("field") + ], + "default_operator": "AND" + } + }, "highlight": { "fields": { self.params.get("field"): {} @@ -111,24 +119,29 @@ def build(self): # query if self.params.get("search_term"): - search["query"] = { - "match": { - self.params.get("field"): { - "query": self.params.get("search_term"), - "operator": "and" + if any(keyword in self.params.get("search_term") + for keyword in ("AND", "OR", "NOT", "TO")): + + # QueryString Query + search["query"] = { + "query_string": { + "query": self.params.get("search_term"), + "fields": [ + self.params.get("field") + ], + "default_operator": "AND" } } - } - else: - search["query"] = { - "query_string": { - "query": "*", - "fields": [ - self.params.get("field") - ], - "default_operator": "AND" + else: + # Match Query + search["query"] = { + "match": { + self.params.get("field"): { + "query": self.params.get("search_term"), + "operator": "and" + } + } } - } return search diff --git a/complaint_search/tests/expected_results/search_with_search_term__valid.json b/complaint_search/tests/expected_results/search_with_search_term_match__valid.json similarity index 100% rename from complaint_search/tests/expected_results/search_with_search_term__valid.json rename to complaint_search/tests/expected_results/search_with_search_term_match__valid.json diff --git a/complaint_search/tests/expected_results/search_with_search_term_qsq_and__valid.json b/complaint_search/tests/expected_results/search_with_search_term_qsq_and__valid.json new file mode 100644 index 00000000..8e0a89bb --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_search_term_qsq_and__valid.json @@ -0,0 +1,314 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "test AND term", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 0 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 0 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 0 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 0 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 0 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 0 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 0 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 0 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 0 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 0 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 0 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed.raw", + "size": 0 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 0 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 0 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 0 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_search_term_qsq_not__valid.json b/complaint_search/tests/expected_results/search_with_search_term_qsq_not__valid.json new file mode 100644 index 00000000..90f25579 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_search_term_qsq_not__valid.json @@ -0,0 +1,314 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "test NOT term", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 0 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 0 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 0 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 0 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 0 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 0 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 0 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 0 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 0 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 0 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 0 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed.raw", + "size": 0 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 0 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 0 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 0 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_search_term_qsq_or__valid.json b/complaint_search/tests/expected_results/search_with_search_term_qsq_or__valid.json new file mode 100644 index 00000000..3a3fe5b1 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_search_term_qsq_or__valid.json @@ -0,0 +1,314 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "test OR term", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 0 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 0 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 0 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 0 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 0 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 0 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 0 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 0 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 0 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 0 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 0 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed.raw", + "size": 0 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 0 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 0 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 0 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_search_term_qsq_to__valid.json b/complaint_search/tests/expected_results/search_with_search_term_qsq_to__valid.json new file mode 100644 index 00000000..3e1fa8e1 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_search_term_qsq_to__valid.json @@ -0,0 +1,314 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "term TO test", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 0 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 0 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 0 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 0 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 0 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 0 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 0 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 0 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 0 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 0 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 0 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed.raw", + "size": 0 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 0 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 0 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 0 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/test_es_interface.py b/complaint_search/tests/test_es_interface.py index 0b70dd6d..84667326 100644 --- a/complaint_search/tests/test_es_interface.py +++ b/complaint_search/tests/test_es_interface.py @@ -205,10 +205,10 @@ def test_search_with_sort__valid(self, mock_count, mock_search): @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') @mock.patch.object(Elasticsearch, 'count') - def test_search_with_search_term__valid(self, mock_count, mock_search): + def test_search_with_search_term_match__valid(self, mock_count, mock_search): mock_search.side_effect = self.MOCK_SEARCH_SIDE_EFFECT mock_count.return_value = self.MOCK_COUNT_RETURN_VALUE - body = self.load("search_with_search_term__valid") + body = self.load("search_with_search_term_match__valid") res = search(search_term="test_term") self.assertEqual(2, len(mock_search.call_args_list)) self.assertEqual(2, len(mock_search.call_args_list[0])) @@ -219,6 +219,74 @@ def test_search_with_search_term__valid(self, mock_count, mock_search): self.assertEqual(mock_search.call_args_list[0][1]['index'], 'INDEX') self.assertEqual(self.MOCK_SEARCH_RESULT, res) + @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") + @mock.patch.object(Elasticsearch, 'search') + @mock.patch.object(Elasticsearch, 'count') + def test_search_with_search_term_qsq_and__valid(self, mock_count, mock_search): + mock_search.side_effect = self.MOCK_SEARCH_SIDE_EFFECT + mock_count.return_value = self.MOCK_COUNT_RETURN_VALUE + body = self.load("search_with_search_term_qsq_and__valid") + res = search(search_term="test AND term") + self.assertEqual(2, len(mock_search.call_args_list)) + self.assertEqual(2, len(mock_search.call_args_list[0])) + self.assertEqual(0, len(mock_search.call_args_list[0][0])) + self.assertEqual(2, len(mock_search.call_args_list[0][1])) + act_body = mock_search.call_args_list[0][1]['body'] + self.assertDictEqual(mock_search.call_args_list[0][1]['body'], body) + self.assertEqual(mock_search.call_args_list[0][1]['index'], 'INDEX') + self.assertEqual(self.MOCK_SEARCH_RESULT, res) + + @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") + @mock.patch.object(Elasticsearch, 'search') + @mock.patch.object(Elasticsearch, 'count') + def test_search_with_search_term_qsq_or__valid(self, mock_count, mock_search): + mock_search.side_effect = self.MOCK_SEARCH_SIDE_EFFECT + mock_count.return_value = self.MOCK_COUNT_RETURN_VALUE + body = self.load("search_with_search_term_qsq_or__valid") + res = search(search_term="test OR term") + self.assertEqual(2, len(mock_search.call_args_list)) + self.assertEqual(2, len(mock_search.call_args_list[0])) + self.assertEqual(0, len(mock_search.call_args_list[0][0])) + self.assertEqual(2, len(mock_search.call_args_list[0][1])) + act_body = mock_search.call_args_list[0][1]['body'] + self.assertDictEqual(mock_search.call_args_list[0][1]['body'], body) + self.assertEqual(mock_search.call_args_list[0][1]['index'], 'INDEX') + self.assertEqual(self.MOCK_SEARCH_RESULT, res) + + @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") + @mock.patch.object(Elasticsearch, 'search') + @mock.patch.object(Elasticsearch, 'count') + def test_search_with_search_term_qsq_not__valid(self, mock_count, mock_search): + mock_search.side_effect = self.MOCK_SEARCH_SIDE_EFFECT + mock_count.return_value = self.MOCK_COUNT_RETURN_VALUE + body = self.load("search_with_search_term_qsq_not__valid") + res = search(search_term="test NOT term") + self.assertEqual(2, len(mock_search.call_args_list)) + self.assertEqual(2, len(mock_search.call_args_list[0])) + self.assertEqual(0, len(mock_search.call_args_list[0][0])) + self.assertEqual(2, len(mock_search.call_args_list[0][1])) + act_body = mock_search.call_args_list[0][1]['body'] + self.assertDictEqual(mock_search.call_args_list[0][1]['body'], body) + self.assertEqual(mock_search.call_args_list[0][1]['index'], 'INDEX') + self.assertEqual(self.MOCK_SEARCH_RESULT, res) + + @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") + @mock.patch.object(Elasticsearch, 'search') + @mock.patch.object(Elasticsearch, 'count') + def test_search_with_search_term_qsq_to__valid(self, mock_count, mock_search): + mock_search.side_effect = self.MOCK_SEARCH_SIDE_EFFECT + mock_count.return_value = self.MOCK_COUNT_RETURN_VALUE + body = self.load("search_with_search_term_qsq_to__valid") + res = search(search_term="term TO test") + self.assertEqual(2, len(mock_search.call_args_list)) + self.assertEqual(2, len(mock_search.call_args_list[0])) + self.assertEqual(0, len(mock_search.call_args_list[0][0])) + self.assertEqual(2, len(mock_search.call_args_list[0][1])) + act_body = mock_search.call_args_list[0][1]['body'] + self.assertDictEqual(mock_search.call_args_list[0][1]['body'], body) + self.assertEqual(mock_search.call_args_list[0][1]['index'], 'INDEX') + self.assertEqual(self.MOCK_SEARCH_RESULT, res) + @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') @mock.patch.object(Elasticsearch, 'count')