diff --git a/.gitignore b/.gitignore index a88947d7..9b861d5b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ htmlcov/ .env *.pyc -db.sqlite3 +db.sqlite3 \ No newline at end of file diff --git a/complaint_search/es_interface.py b/complaint_search/es_interface.py index 5e2347e4..0c28585f 100644 --- a/complaint_search/es_interface.py +++ b/complaint_search/es_interface.py @@ -15,6 +15,25 @@ _COMPLAINT_ES_INDEX = os.environ.get('COMPLAINT_ES_INDEX', 'complaint-index') _COMPLAINT_DOC_TYPE = os.environ.get('COMPLAINT_DOC_TYPE', 'complaint-doctype') +_OPTIONAL_FILTERS = ("product", "issue", "company", "state", "zip_code", "timely", + "company_response", "company_public_response", + "consumer_consent_provided", "submitted_via", "tag") + +_OPTIONAL_FILTERS_PARAM_TO_ES_MAP = { + "product": "product.raw", + "sub_product": "sub_product.raw", + "issue": "issue.raw", + "sub_issue": "sub_issue.raw", + "company_public_response": "company_public_response.raw", + "consumer_consent_provided": "consumer_consent_provided.raw" +} +_OPTIONAL_FILTERS_CHILD_MAP = { + "product": "sub_product", + "issue": "sub_issue" +} + +_OPTIONAL_FILTERS_STRING_TO_BOOL = ("consumer_disputed", "has_narratives") + def get_es(): global _ES_INSTANCE if _ES_INSTANCE is None: @@ -22,13 +41,106 @@ def get_es(): timeout=100) return _ES_INSTANCE -def _create_and_append_bool_should_clauses(es_field_name, value_list, - filter_list, with_subitems=False, es_subitem_field_name=None): + + +def _create_aggregation(**kwargs): + + Field = namedtuple('Field', 'name size has_subfield') + fields = [ + Field('has_narratives', 10, False), + Field('company', 10000, False), + Field('product', 10000, True), + Field('issue', 10000, True), + Field('state', 50, False), + Field('zip_code', 1000, False), + Field('timely', 10, False), + Field('company_response', 100, False), + Field('company_public_response', 100, False), + Field('consumer_disputed', 100, False), + Field('consumer_consent_provided', 100, False), + Field('tag', 100, False), + Field('submitted_via', 100, False) + ] + aggs = {} + + # Creating aggregation object for each field above + for field in fields: + field_aggs = { + "filter": { + "and": { + "filters": [ + + ] + } + } + } + + es_field_name = _OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(field.name, field.name) + es_subfield_name = None + if field.has_subfield: + es_subfield_name = _OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(_OPTIONAL_FILTERS_CHILD_MAP.get(field.name)) + field_aggs["aggs"] = { + field.name: { + "terms": { + "field": es_field_name, + "size": field.size + }, + "aggs": { + es_subfield_name: { + "terms": { + "field": es_subfield_name, + "size": field.size + } + } + } + } + } + else: + field_aggs["aggs"] = { + field.name: { + "terms": { + "field": es_field_name, + "size": field.size + } + } + } + + date_filter = { + "range": { + "date_received": { + + } + } + } + if "min_date" in kwargs: + date_filter["range"]["date_received"]["from"] = kwargs["min_date"] + if "max_date" in kwargs: + date_filter["range"]["date_received"]["to"] = kwargs["max_date"] + + field_aggs["filter"]["and"]["filters"].append(date_filter) + + for item in kwargs: + if item in _OPTIONAL_FILTERS and item != field.name: + clauses = _create_and_append_bool_should_clauses(_OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(item, item), + kwargs[item], field_aggs["filter"]["and"]["filters"], + with_subitems=item in _OPTIONAL_FILTERS_CHILD_MAP, + es_subitem_field_name=_OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(_OPTIONAL_FILTERS_CHILD_MAP.get(item))) + elif item in _OPTIONAL_FILTERS_STRING_TO_BOOL and item != field.name: + clauses = _create_and_append_bool_should_clauses(_OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(item, item), + [ 0 if cd.lower() == "no" else 1 for cd in kwargs[item] ], + field_aggs["filter"]["and"]["filters"]) + + aggs[field.name] = field_aggs + + return aggs + +def _create_bool_should_clauses(es_field_name, value_list, + with_subitems=False, es_subitem_field_name=None): if value_list: if not with_subitems: term_list = [ {"terms": {es_field_name: [value]}} for value in value_list ] - filter_list.append({"bool": {"should": term_list}}) + return {"bool": {"should": term_list}} else: item_dict = defaultdict(list) for v in value_list: @@ -53,7 +165,16 @@ def _create_and_append_bool_should_clauses(es_field_name, value_list, subitem_term = {"terms": {es_subitem_field_name: subitems}} f_list.append({"and": {"filters": [item_term, subitem_term]}}) - filter_list.append({"bool": {"should": f_list}}) + return {"bool": {"should": f_list}} + +def _create_and_append_bool_should_clauses(es_field_name, value_list, + filter_list, with_subitems=False, es_subitem_field_name=None): + + filter_clauses = _create_bool_should_clauses(es_field_name, value_list, + with_subitems, es_subitem_field_name) + + if filter_clauses: + filter_list.append(filter_clauses) # List of possible arguments: # - fmt: format to be returned: "json", "csv", "xls", or "xlsx" @@ -88,23 +209,6 @@ def search(**kwargs): "sort": "relevance_desc" } - OPTIONAL_FILTERS = ("product", "issue", "company", "state", "zip_code", "timely", - "company_response", "company_public_response", - "consumer_consent_provided", "submitted_via", "tag") - - OPTIONAL_FILTERS_PARAM_TO_ES_MAP = { - "product": "product.raw", - "sub_product": "sub_product.raw", - "issue": "issue.raw", - "sub_issue": "sub_issue.raw" - } - OPTIONAL_FILTERS_CHILD_MAP = { - "product": "sub_product", - "issue": "sub_issue" - } - - OPTIONAL_FILTERS_STRING_TO_BOOL = ("consumer_disputed", "has_narratives") - params.update(**kwargs) res = None @@ -150,6 +254,9 @@ def search(**kwargs): # post-filter body["post_filter"] = {"and": {"filters": []}} + ## Create base aggregation + body["aggs"] = _create_aggregation(**kwargs) + ## date if params.get("min_date") or params.get("max_date"): date_clause = {"range": {"date_received": {}}} @@ -160,18 +267,19 @@ def search(**kwargs): body["post_filter"]["and"]["filters"].append(date_clause) - ## Create bool should clauses for fields in OPTIONAL_FILTERS - for field in OPTIONAL_FILTERS: - if field in OPTIONAL_FILTERS_CHILD_MAP: - _create_and_append_bool_should_clauses(OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(field, field), + + ## Create bool should clauses for fields in _OPTIONAL_FILTERS + for field in _OPTIONAL_FILTERS: + if field in _OPTIONAL_FILTERS_CHILD_MAP: + _create_and_append_bool_should_clauses(_OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(field, field), params.get(field), body["post_filter"]["and"]["filters"], with_subitems=True, - es_subitem_field_name=OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(OPTIONAL_FILTERS_CHILD_MAP.get(field), - OPTIONAL_FILTERS_CHILD_MAP.get(field))) + es_subitem_field_name=_OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(_OPTIONAL_FILTERS_CHILD_MAP.get(field), + _OPTIONAL_FILTERS_CHILD_MAP.get(field))) else: - _create_and_append_bool_should_clauses(OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(field, field), + _create_and_append_bool_should_clauses(_OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(field, field), params.get(field), body["post_filter"]["and"]["filters"]) - for field in OPTIONAL_FILTERS_STRING_TO_BOOL: + for field in _OPTIONAL_FILTERS_STRING_TO_BOOL: if params.get(field): _create_and_append_bool_should_clauses(field, [ 0 if cd.lower() == "no" else 1 for cd in params.get(field) ], diff --git a/complaint_search/tests/expected_results/search_no_param__valid.json b/complaint_search/tests/expected_results/search_no_param__valid.json new file mode 100644 index 00000000..6203ca12 --- /dev/null +++ b/complaint_search/tests/expected_results/search_no_param__valid.json @@ -0,0 +1,314 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_company__valid.json b/complaint_search/tests/expected_results/search_with_company__valid.json new file mode 100644 index 00000000..dccf4413 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_company__valid.json @@ -0,0 +1,423 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [ + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company": ["Bank 1"]}}, + {"terms": {"company": ["Second Bank"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_company_public_response__valid.json b/complaint_search/tests/expected_results/search_with_company_public_response__valid.json new file mode 100644 index 00000000..a559e744 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_company_public_response__valid.json @@ -0,0 +1,421 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_public_response.raw": ["Response 1"]}}, + {"terms": {"company_public_response.raw": ["Response 2"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_company_response__valid.json b/complaint_search/tests/expected_results/search_with_company_response__valid.json new file mode 100644 index 00000000..f66ad3d8 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_company_response__valid.json @@ -0,0 +1,421 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"company_response": ["Closed"]}}, + {"terms": {"company_response": ["No response"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_consumer_consent_provided__valid.json b/complaint_search/tests/expected_results/search_with_consumer_consent_provided__valid.json new file mode 100644 index 00000000..74a30e22 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_consumer_consent_provided__valid.json @@ -0,0 +1,421 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_consent_provided.raw": ["yes"]}}, + {"terms": {"consumer_consent_provided.raw": ["no"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_consumer_disputed__valid.json b/complaint_search/tests/expected_results/search_with_consumer_disputed__valid.json new file mode 100644 index 00000000..5272cb4e --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_consumer_disputed__valid.json @@ -0,0 +1,421 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"consumer_disputed": [0]}}, + {"terms": {"consumer_disputed": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_field__valid.json b/complaint_search/tests/expected_results/search_with_field__valid.json new file mode 100644 index 00000000..45251787 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_field__valid.json @@ -0,0 +1,315 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "test_field" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "test_field": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } + +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_fmt_nonjson__valid.json b/complaint_search/tests/expected_results/search_with_fmt_nonjson__valid.json new file mode 100644 index 00000000..6203ca12 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_fmt_nonjson__valid.json @@ -0,0 +1,314 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_frm__valid.json b/complaint_search/tests/expected_results/search_with_frm__valid.json new file mode 100644 index 00000000..b3346d36 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_frm__valid.json @@ -0,0 +1,314 @@ +{ + "from": 20, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_has_narratives__valid.json b/complaint_search/tests/expected_results/search_with_has_narratives__valid.json new file mode 100644 index 00000000..c69a239f --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_has_narratives__valid.json @@ -0,0 +1,421 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"has_narratives": [0]}}, + {"terms": {"has_narratives": [1]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_issue__valid.json b/complaint_search/tests/expected_results/search_with_issue__valid.json new file mode 100644 index 00000000..2310ec58 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_issue__valid.json @@ -0,0 +1,512 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + { + "and": { + "filters": [ + {"terms": {"issue.raw": ["Communication tactics"]}}, + {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} + ] + } + }, + {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_max_date__valid.json b/complaint_search/tests/expected_results/search_with_max_date__valid.json new file mode 100644 index 00000000..ccda41ef --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_max_date__valid.json @@ -0,0 +1,352 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "to": "2017-04-14" + } + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_min_date__valid.json b/complaint_search/tests/expected_results/search_with_min_date__valid.json new file mode 100644 index 00000000..f837337a --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_min_date__valid.json @@ -0,0 +1,352 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": { + "from": "2014-04-14" + } + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_product__valid.json b/complaint_search/tests/expected_results/search_with_product__valid.json new file mode 100644 index 00000000..805f90e8 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_product__valid.json @@ -0,0 +1,527 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [ + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + + {"terms": {"product.raw": ["Payday Loan"]}}, + { + "and": { + "filters": [ + {"terms": {"product.raw": ["Mortgage"]}}, + {"terms": {"sub_product.raw": ["FHA Mortgage"]}} + ] + } + } + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_search_term__valid.json b/complaint_search/tests/expected_results/search_with_search_term__valid.json new file mode 100644 index 00000000..ccff66cd --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_search_term__valid.json @@ -0,0 +1,313 @@ +{ + "from": 0, + "size": 10, + "query": { + "match": { + "complaint_what_happened": { + "query": "test_term", + "operator": "and" + } + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_size__valid.json b/complaint_search/tests/expected_results/search_with_size__valid.json new file mode 100644 index 00000000..aa9b1f3b --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_size__valid.json @@ -0,0 +1,314 @@ +{ + "from": 0, + "size": 40, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_sort__valid.json b/complaint_search/tests/expected_results/search_with_sort__valid.json new file mode 100644 index 00000000..036fbe0e --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_sort__valid.json @@ -0,0 +1,313 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "post_filter": {"and": {"filters": []}}, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_state__valid.json b/complaint_search/tests/expected_results/search_with_state__valid.json new file mode 100644 index 00000000..80b75da6 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_state__valid.json @@ -0,0 +1,434 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"state": ["CA"]}}, + {"terms": {"state": ["VA"]}}, + {"terms": {"state": ["OR"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_submitted_via__valid.json b/complaint_search/tests/expected_results/search_with_submitted_via__valid.json new file mode 100644 index 00000000..0012d9f5 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_submitted_via__valid.json @@ -0,0 +1,421 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"submitted_via": ["mail"]}}, + {"terms": {"submitted_via": ["web"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_tag__valid.json b/complaint_search/tests/expected_results/search_with_tag__valid.json new file mode 100644 index 00000000..5749c1ab --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_tag__valid.json @@ -0,0 +1,421 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"tag": ["Older American"]}}, + {"terms": {"tag": ["Servicemember"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_timely__valid.json b/complaint_search/tests/expected_results/search_with_timely__valid.json new file mode 100644 index 00000000..e6253e7f --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_timely__valid.json @@ -0,0 +1,421 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"timely": ["Yes"]}}, + {"terms": {"timely": ["No"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/search_with_zip_code__valid.json b/complaint_search/tests/expected_results/search_with_zip_code__valid.json new file mode 100644 index 00000000..973cfde1 --- /dev/null +++ b/complaint_search/tests/expected_results/search_with_zip_code__valid.json @@ -0,0 +1,434 @@ +{ + "from": 0, + "size": 10, + "query": { + "query_string": { + "query": "*", + "fields": [ + "complaint_what_happened" + ], + "default_operator": "AND" + } + }, + "highlight": { + "fields": { + "complaint_what_happened": {} + }, + "number_of_fragments": 1, + "fragment_size": 500 + }, + "sort": [{"_score": {"order": "desc"}}], + "post_filter": { + "and": { + "filters": [{ + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + }] + } + }, + "aggs": { + "has_narratives": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "has_narratives": { + "terms": { + "field": "has_narratives", + "size": 10 + } + } + } + }, + "company": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "company": { + "terms": { + "field": "company", + "size": 10000 + } + } + } + }, + "product": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "product": { + "terms": { + "field": "product.raw", + "size": 10000 + }, + "aggs": { + "sub_product.raw": { + "terms": { + "field": "sub_product.raw", + "size": 10000 + } + } + } + } + } + }, + "issue": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "issue": { + "terms": { + "field": "issue.raw", + "size": 10000 + }, + "aggs": { + "sub_issue.raw": { + "terms": { + "field": "sub_issue.raw", + "size": 10000 + } + } + } + } + } + }, + "state": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "state": { + "terms": { + "field": "state", + "size": 50 + } + } + } + }, + "zip_code": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + } + ] + } + }, + "aggs": { + "zip_code": { + "terms": { + "field": "zip_code", + "size": 1000 + } + } + } + }, + "timely": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "timely": { + "terms": { + "field": "timely", + "size": 10 + } + } + } + }, + "company_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_response": { + "terms": { + "field": "company_response", + "size": 100 + } + } + } + }, + "company_public_response": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "company_public_response": { + "terms": { + "field": "company_public_response.raw", + "size": 100 + } + } + } + }, + "consumer_disputed": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_disputed": { + "terms": { + "field": "consumer_disputed", + "size": 100 + } + } + } + }, + "consumer_consent_provided": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "consumer_consent_provided": { + "terms": { + "field": "consumer_consent_provided.raw", + "size": 100 + } + } + } + }, + "tag": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "tag": { + "terms": { + "field": "tag", + "size": 100 + } + } + } + }, + "submitted_via": { + "filter": { + "and": { + "filters": [ + { + "range": { + "date_received": {} + } + }, + { + "bool": { + "should": [ + {"terms": {"zip_code": ["12345"]}}, + {"terms": {"zip_code": ["23435"]}}, + {"terms": {"zip_code": ["03433"]}} + ] + } + } + ] + } + }, + "aggs": { + "submitted_via": { + "terms": { + "field": "submitted_via", + "size": 100 + } + } + } + } + + } +} \ No newline at end of file diff --git a/complaint_search/tests/test_es_interface.py b/complaint_search/tests/test_es_interface.py index 3cf224e7..b3cc3296 100644 --- a/complaint_search/tests/test_es_interface.py +++ b/complaint_search/tests/test_es_interface.py @@ -5,9 +5,26 @@ import os import urllib import json +import deep import mock class EsInterfaceTest(TestCase): + # ------------------------------------------------------------------------- + # Helper Methods + # ------------------------------------------------------------------------- + + def to_absolute(self, fileName): + import os.path + # where is this module? + thisDir = os.path.dirname(__file__) + return os.path.join(thisDir, "expected_results", fileName) + + def load(self, shortName): + import json + fileName = self.to_absolute(shortName + '.json') + with open(fileName, 'r') as f: + return json.load(f) + def setUp(self): pass @@ -16,31 +33,14 @@ def setUp(self): @mock.patch('requests.get', ok=True, content="RGET_OK") def test_search_no_param__valid(self, mock_rget, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"and": {"filters": []}} - } + body = self.load("search_no_param__valid") res = search() - mock_search.assert_called_once_with(body=body, - index='INDEX') + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') mock_rget.assert_not_called() self.assertEqual('OK', res) @@ -51,36 +51,29 @@ def test_search_no_param__valid(self, mock_rget, mock_search): @mock.patch("complaint_search.es_interface._COMPLAINT_DOC_TYPE", "DOC_TYPE") @mock.patch.object(Elasticsearch, 'search') @mock.patch('requests.get', ok=True, content="RGET_OK") - def test_search_with_fmt_nonjson__valid(self, mock_rget, mock_search): + @mock.patch('json.dumps') + @mock.patch('urllib.urlencode') + def test_search_with_fmt_nonjson__valid(self, mock_urlencode, mock_jdump, mock_rget, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"and": {"filters": []}} - } + mock_jdump.return_value = 'JDUMPS_OK' + body = self.load("search_with_fmt_nonjson__valid") for fmt in ["csv", "xls", "xlsx"]: res = search(fmt=fmt) - param = urllib.urlencode({"format": fmt, "source": json.dumps(body)}) - other_args = {"auth": ("ES_USER", "ES_PASSWORD"), "verify": False, "timeout": 30} - url = "ES_URL/INDEX/DOC_TYPE/_data?{}".format(param) - mock_rget.assert_any_call(url, **other_args) + self.assertEqual(len(mock_jdump.call_args), 2) + self.assertEqual(1, len(mock_jdump.call_args[0])) + act_body = mock_jdump.call_args[0][0] + diff = deep.diff(body, act_body) + if diff: + print "fmt={}".format(fmt) + diff.print_full() + self.assertEqual(len(mock_urlencode.call_args), 2) + self.assertEqual(1, len(mock_urlencode.call_args[0])) + param = {"format": fmt, "source": "JDUMPS_OK"} + act_param = mock_urlencode.call_args[0][0] + self.assertEqual(param, act_param) + + self.assertEqual(mock_jdump.call_count, 3) + self.assertEqual(mock_urlencode.call_count, 3) mock_search.assert_not_called() self.assertEqual(3, mock_rget.call_count) @@ -89,31 +82,14 @@ def test_search_with_fmt_nonjson__valid(self, mock_rget, mock_search): @mock.patch.object(Elasticsearch, 'search') def test_search_with_field__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "test_field" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "test_field": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"and": {"filters": []}} - } + body = self.load("search_with_field__valid") res = search(field="test_field") - mock_search.assert_called_once_with(body=body, - index='INDEX') + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch.object(Elasticsearch, 'search') @@ -129,28 +105,7 @@ def test_search_with_fmt__invalid(self, mock_rget, mock_search): @mock.patch.object(Elasticsearch, 'search') def test_search_with_size__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 40, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"and": {"filters": []}} - } + body = self.load("search_with_size__valid") res = search(size=40) mock_search.assert_called_once_with(body=body, index="INDEX") @@ -160,31 +115,14 @@ def test_search_with_size__valid(self, mock_search): @mock.patch.object(Elasticsearch, 'search') def test_search_with_frm__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 20, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"and": {"filters": []}} - } + body = self.load("search_with_frm__valid") res = search(frm=20) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @@ -192,27 +130,7 @@ def test_search_with_frm__valid(self, mock_search): @mock.patch.object(Elasticsearch, 'search') def test_search_with_sort__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "post_filter": {"and": {"filters": []}} - } + body = self.load("search_with_sort__valid") sort_fields = [ ("relevance_desc", "_score", "desc"), @@ -232,134 +150,61 @@ def test_search_with_sort__valid(self, mock_search): @mock.patch.object(Elasticsearch, 'search') def test_search_with_search_term__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "match": { - "complaint_what_happened": { - "query": "test_term", - "operator": "and" - } - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"and": {"filters": []}} - } + body = self.load("search_with_search_term__valid") res = search(search_term="test_term") - mock_search.assert_called_once_with(body=body, - index='INDEX') + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_min_date__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"and": {"filters": [{"range": {"date_received": {"from": "2014-04-14"}}}]}} - } + body = self.load("search_with_min_date__valid") res = search(min_date="2014-04-14") - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_max_date__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"and": {"filters": [{"range": {"date_received": {"to": "2017-04-14"}}}]}} - } + body = self.load("search_with_max_date__valid") res = search(max_date="2017-04-14") - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_company__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"company": ["Bank 1"]}}, - {"terms": {"company": ["Second Bank"]}} - ] - } - }] - } - } - } + body = self.load("search_with_company__valid") res = search(company=["Bank 1", "Second Bank"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "company" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @@ -367,522 +212,233 @@ def test_search_with_company__valid(self, mock_search): @mock.patch.object(Elasticsearch, 'search') def test_search_with_product__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - - {"terms": {"product.raw": ["Payday Loan"]}}, - { - "and": { - "filters": [ - {"terms": {"product.raw": ["Mortgage"]}}, - {"terms": {"sub_product.raw": ["FHA Mortgage"]}} - ] - } - } - ] - } - }] - } - } - } + body = self.load("search_with_product__valid") res = search(product=["Payday Loan", u"Mortgage\u2022FHA Mortgage"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "product" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_issue__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - { - "and": { - "filters": [ - {"terms": {"issue.raw": ["Communication tactics"]}}, - {"terms": {"sub_issue.raw": ["Frequent or repeated calls"]}} - ] - } - }, - {"terms": {"issue.raw": ["Loan servicing, payments, escrow account"]}} - ] - } - }] - } - } - } + body = self.load("search_with_issue__valid") res = search(issue=[u"Communication tactics\u2022Frequent or repeated calls", "Loan servicing, payments, escrow account"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "issue" + diff.print_full() + print "body" + print body + print "act_body" + print act_body + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_state__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"state": ["CA"]}}, - {"terms": {"state": ["VA"]}}, - {"terms": {"state": ["OR"]}} - ] - } - }] - } - } - } + body = self.load("search_with_state__valid") res = search(state=["CA", "VA", "OR"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "state" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_zip_code__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"zip_code": ["12345"]}}, - {"terms": {"zip_code": ["23435"]}}, - {"terms": {"zip_code": ["03433"]}} - ] - } - }] - } - } - } + body = self.load("search_with_zip_code__valid") res = search(zip_code=["12345", "23435", "03433"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "zip_code" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_timely__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"timely": ["Yes"]}}, - {"terms": {"timely": ["No"]}} - ] - } - }] - } - } - } + body = self.load("search_with_timely__valid") res = search(timely=["Yes", "No"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "timely" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_company_response__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"company_response": ["Closed"]}}, - {"terms": {"company_response": ["No response"]}} - ] - } - }] - } - } - } + body = self.load("search_with_company_response__valid") res = search(company_response=["Closed", "No response"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "company_response" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_company_public_response__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"company_public_response": ["Response 1"]}}, - {"terms": {"company_public_response": ["Response 2"]}} - ] - } - }] - } - } - } + body = self.load("search_with_company_public_response__valid") res = search(company_public_response=["Response 1", "Response 2"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "company_public_response.raw" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_consumer_consent_provided__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"consumer_consent_provided": ["yes"]}}, - {"terms": {"consumer_consent_provided": ["no"]}} - ] - } - }] - } - } - } + body = self.load("search_with_consumer_consent_provided__valid") res = search(consumer_consent_provided=["yes", "no"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "consumer_consent_provided.raw" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_submitted_via__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"submitted_via": ["mail"]}}, - {"terms": {"submitted_via": ["web"]}} - ] - } - }] - } - } - } + body = self.load("search_with_submitted_via__valid") res = search(submitted_via=["mail", "web"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "submitted_via" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_tag__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"tag": ["Older American"]}}, - {"terms": {"tag": ["Servicemember"]}} - ] - } - }] - } - } - } + body = self.load("search_with_tag__valid") res = search(tag=["Older American", "Servicemember"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "tag" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_consumer_disputed__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"consumer_disputed": [0]}}, - {"terms": {"consumer_disputed": [1]}} - ] - } - }] - } - } - } + body = self.load("search_with_consumer_disputed__valid") res = search(consumer_disputed=["No", "Yes"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "consumer_disputed" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @mock.patch.object(Elasticsearch, 'search') def test_search_with_has_narratives__valid(self, mock_search): mock_search.return_value = 'OK' - body = { - "from": 0, - "size": 10, - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": { - "and": { - "filters": [{ - "bool": { - "should": [ - {"terms": {"has_narratives": [0]}}, - {"terms": {"has_narratives": [1]}} - ] - } - }] - } - } - } + body = self.load("search_with_has_narratives__valid") res = search(has_narratives=["No", "Yes"]) - mock_search.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(2, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + diff = deep.diff(body, act_body) + if diff: + print "has_narratives" + diff.print_full() + self.assertIsNone(deep.diff(body, act_body)) + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @@ -901,8 +457,12 @@ def test_suggest_with_text__valid(self, mock_suggest): mock_suggest.return_value = 'OK' body = {"sgg": {"text": "Mortgage", "completion": {"field": "suggest", "size": 6}}} res = suggest(text="Mortgage") - mock_suggest.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_suggest.call_args), 2) + self.assertEqual(0, len(mock_suggest.call_args[0])) + self.assertEqual(2, len(mock_suggest.call_args[1])) + act_body = mock_suggest.call_args[1]['body'] + self.assertDictEqual(mock_suggest.call_args[1]['body'], body) + self.assertEqual(mock_suggest.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @@ -911,8 +471,12 @@ def test_suggest_with_size__valid(self, mock_suggest): mock_suggest.return_value = 'OK' body = {"sgg": {"text": "Loan", "completion": {"field": "suggest", "size": 10}}} res = suggest(text="Loan", size=10) - mock_suggest.assert_called_once_with(body=body, - index="INDEX") + self.assertEqual(len(mock_suggest.call_args), 2) + self.assertEqual(0, len(mock_suggest.call_args[0])) + self.assertEqual(2, len(mock_suggest.call_args[1])) + act_body = mock_suggest.call_args[1]['body'] + self.assertDictEqual(mock_suggest.call_args[1]['body'], body) + self.assertEqual(mock_suggest.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") @@ -922,6 +486,11 @@ def test_document__valid(self, mock_search): mock_search.return_value = 'OK' body = {"query": {"term": {"_id": 123456}}} res = document(123456) - mock_search.assert_called_once_with(body=body, doc_type="DOC_TYPE", - index="INDEX") + self.assertEqual(len(mock_search.call_args), 2) + self.assertEqual(0, len(mock_search.call_args[0])) + self.assertEqual(3, len(mock_search.call_args[1])) + act_body = mock_search.call_args[1]['body'] + self.assertDictEqual(mock_search.call_args[1]['body'], body) + self.assertEqual(mock_search.call_args[1]['doc_type'], 'DOC_TYPE') + self.assertEqual(mock_search.call_args[1]['index'], 'INDEX') self.assertEqual('OK', res) \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt index 6eb6461c..421cfbeb 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1 +1,2 @@ -mock==2.0.0 \ No newline at end of file +mock==2.0.0 +deep==0.10 \ No newline at end of file