Merge 204aba7 into 599e632

arches/app/datatypes/base.py

@@ -1,6 +1,7 @@
 import json, urllib
 from django.urls import reverse
 from arches.app.models import models
+from arches.app.search.elasticsearch_dsl_builder import Bool, Terms, Exists, Nested
 from django.utils.translation import ugettext as _
 import logging
 
@@ -216,6 +217,22 @@ def append_search_filters(self, value, node, query, request):
         """
         pass
 
+    def append_null_search_filters(self, value, node, query, request):
+        """
+        Appends the search query dsl to search for fields that haven't been populated
+        """
+        base_query = Bool()
+        null_query = Bool()
+        data_exists_query = Exists(field="tiles.data.%s" % (str(node.pk)))
+        nested_query = Nested(path="tiles", query=data_exists_query)
+        null_query.must(nested_query)
+        base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
+        if value["op"] == "null":
+            base_query.must_not(null_query)
+        elif value["op"] == "not_null":
+            base_query.must(null_query)
+        query.must(base_query)
+
     def handle_request(self, current_tile, request, node):
         """
         Updates files

arches/app/datatypes/concept_types.py

@@ -84,6 +84,24 @@ def append_to_document(self, document, nodevalue, nodeid, tile, provisional=Fals
             )
             document["strings"].append({"string": value.value, "nodegroup_id": tile.nodegroup_id, "provisional": provisional})
 
+    def append_search_filters(self, value, node, query, request):
+        try:
+            if value["op"] == "null" or value["op"] == "not_null":
+                self.append_null_search_filters(value, node, query, request)
+            elif value["val"] != "":
+                base_query = Bool()
+                base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
+                match_query = Nested(path="tiles", query=Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"]))
+                if "!" in value["op"]:
+                    base_query.must_not(match_query)
+                    # base_query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
+                else:
+                    base_query.must(match_query)
+                query.must(base_query)
+
+        except KeyError as e:
+            pass
+
 
 class ConceptDataType(BaseConceptDataType):
     def validate(self, value, row_number=None, source="", node=None, nodeid=None):
@@ -131,19 +149,6 @@ def get_display_value(self, tile, node):
         else:
             return self.get_value(uuid.UUID(data[str(node.nodeid)])).value
 
-    def append_search_filters(self, value, node, query, request):
-        try:
-            if value["val"] != "":
-                match_query = Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"])
-                if "!" in value["op"]:
-                    query.must_not(match_query)
-                    query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
-                else:
-                    query.must(match_query)
-
-        except KeyError as e:
-            pass
-
     def get_rdf_uri(self, node, data, which="r"):
         if not data:
             return None
@@ -271,19 +276,6 @@ def get_display_value(self, tile, node):
                 new_values.append(new_val.value)
         return ",".join(new_values)
 
-    def append_search_filters(self, value, node, query, request):
-        try:
-            if value["val"] != "":
-                match_query = Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"])
-                if "!" in value["op"]:
-                    query.must_not(match_query)
-                    query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
-                else:
-                    query.must(match_query)
-
-        except KeyError as e:
-            pass
-
     def get_rdf_uri(self, node, data, which="r"):
         c = ConceptDataType()
         if not data:

arches/app/datatypes/datatypes.py

@@ -20,7 +20,7 @@
 from arches.app.utils.permission_backend import user_is_resource_reviewer
 from arches.app.utils.geo_utils import GeoUtils
 import arches.app.utils.task_management as task_management
-from arches.app.search.elasticsearch_dsl_builder import Bool, Match, Range, Term, Terms, Exists, RangeDSLException
+from arches.app.search.elasticsearch_dsl_builder import Bool, Match, Range, Term, Terms, Nested, Exists, RangeDSLException
 from arches.app.search.search_engine_factory import SearchEngineInstance as se
 from arches.app.search.mappings import RESOURCES_INDEX, RESOURCE_RELATIONS_INDEX
 from django.core.cache import cache
@@ -112,14 +112,19 @@ def get_search_terms(self, nodevalue, nodeid=None):
 
     def append_search_filters(self, value, node, query, request):
         try:
-            if value["val"] != "":
+            if value["op"] == "null" or value["op"] == "not_null":
+                self.append_null_search_filters(value, node, query, request)
+            elif value["val"] != "":
+                base_query = Bool()
+                base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
+                data_field = "tiles.data.%s" % (str(node.pk))
                 match_type = "phrase_prefix" if "~" in value["op"] else "phrase"
-                match_query = Match(field="tiles.data.%s" % (str(node.pk)), query=value["val"], type=match_type)
+                match_query = Nested(path="tiles", query=Match(field=data_field, query=value["val"], type=match_type))
                 if "!" in value["op"]:
-                    query.must_not(match_query)
-                    query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
+                    base_query.must_not(match_query)
                 else:
-                    query.must(match_query)
+                    base_query.must(match_query)
+                query.must(base_query)
         except KeyError as e:
             pass
 
@@ -189,14 +194,19 @@ def append_to_document(self, document, nodevalue, nodeid, tile, provisional=Fals
 
     def append_search_filters(self, value, node, query, request):
         try:
-            if value["val"] != "":
+            if value["op"] == "null" or value["op"] == "not_null":
+                self.append_null_search_filters(value, node, query, request)
+            elif value["val"] != "":
+                base_query = Bool()
+                base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
                 if value["op"] != "eq":
                     operators = {"gte": None, "lte": None, "lt": None, "gt": None}
                     operators[value["op"]] = value["val"]
                 else:
                     operators = {"gte": value["val"], "lte": value["val"]}
-                search_query = Range(field="tiles.data.%s" % (str(node.pk)), **operators)
-                query.must(search_query)
+                search_query = Nested(path="tiles", query=Range(field="tiles.data.%s" % (str(node.pk)), **operators))
+                base_query.must(search_query)
+                query.must(base_query)
         except KeyError:
             pass
 
@@ -248,12 +258,34 @@ def transform_value_for_tile(self, value, **kwargs):
 
     def append_search_filters(self, value, node, query, request):
         try:
-            if value["val"] != "":
+            if value["val"] == "null" or value["val"] == "not_null":
+                self.append_null_search_filters(value, node, query, request)
+            elif value["val"] != "" and value["val"] is not None:
+                base_query = Bool()
+                base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
                 term = True if value["val"] == "t" else False
-                query.must(Term(field="tiles.data.%s" % (str(node.pk)), term=term))
+                term_query = Nested(path="tiles", query=Term(field="tiles.data.%s" % (str(node.pk)), term=term))
+                base_query.must(term_query)
+                query.must(base_query)
         except KeyError as e:
             pass
 
+    def append_null_search_filters(self, value, node, query, request):
+        """
+        Appends the search query dsl to search for fields that haven't been populated
+        """
+        base_query = Bool()
+        null_query = Bool()
+        data_exists_query = Exists(field="tiles.data.%s" % (str(node.pk)))
+        nested_query = Nested(path="tiles", query=data_exists_query)
+        null_query.must(nested_query)
+        base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
+        if value["val"] == "null":
+            base_query.must_not(null_query)
+        elif value["val"] == "not_null":
+            base_query.must(null_query)
+        query.must(base_query)
+
     def to_rdf(self, edge_info, edge):
         # returns an in-memory graph object, containing the domain resource, its
         # type and the number as a numeric literal (as this is how it is in the JSON)
@@ -332,7 +364,11 @@ def append_to_document(self, document, nodevalue, nodeid, tile, provisional=Fals
 
     def append_search_filters(self, value, node, query, request):
         try:
-            if value["val"] != "" and value["val"] is not None:
+            if value["op"] == "null" or value["op"] == "not_null":
+                self.append_null_search_filters(value, node, query, request)
+            elif value["val"] != "" and value["val"] is not None:
+                base_query = Bool()
+                base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
                 try:
                     date_value = datetime.strptime(value["val"], "%Y-%m-%d %H:%M:%S%z").astimezone().isoformat()
                 except ValueError:
@@ -342,8 +378,9 @@ def append_search_filters(self, value, node, query, request):
                     operators[value["op"]] = date_value
                 else:
                     operators = {"gte": date_value, "lte": date_value}
-                search_query = Range(field="tiles.data.%s" % (str(node.pk)), **operators)
-                query.must(search_query)
+                search_query = Nested(path="tiles", query=Range(field="tiles.data.%s" % (str(node.pk)), **operators))
+                base_query.must(search_query)
+                query.must(base_query)
         except KeyError:
             pass
 
@@ -445,10 +482,15 @@ def add_date_to_doc(document, edtf):
 
     def append_search_filters(self, value, node, query, request):
         def add_date_to_doc(query, edtf):
+            base_query = Bool()
+            base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
             if value["op"] == "eq":
                 if edtf.lower != edtf.upper:
                     raise Exception(_('Only dates that specify an exact year, month, and day can be used with the "=" operator'))
-                query.should(Match(field="tiles.data.%s.dates.date" % (str(node.pk)), query=edtf.lower, type="phrase_prefix"))
+                match_query = Nested(
+                    path="tiles", query=Match(field="tiles.data.%s.dates.date" % (str(node.pk)), query=edtf.lower, type="phrase_prefix")
+                )
+                base_query.should(match_query)
             else:
                 if value["op"] == "overlaps":
                     operators = {"gte": edtf.lower, "lte": edtf.upper}
@@ -464,13 +506,21 @@ def add_date_to_doc(query, edtf):
                     operators = {value["op"]: edtf.lower or edtf.upper}
 
                 try:
-                    query.should(Range(field="tiles.data.%s.dates.date" % (str(node.pk)), **operators))
-                    query.should(Range(field="tiles.data.%s.date_ranges.date_range" % (str(node.pk)), relation="intersects", **operators))
+                    base_query.should(Nested(path="tiles", query=Range(field="tiles.data.%s.dates.date" % (str(node.pk)), **operators)))
+                    base_query.should(
+                        Nested(
+                            path="tiles",
+                            query=Range(field="tiles.data.%s.date_ranges.date_range" % (str(node.pk)), relation="intersects", **operators),
+                        )
+                    )
                 except RangeDSLException:
                     if edtf.lower is None and edtf.upper is None:
                         raise Exception(_("Invalid date specified."))
+            query.must(base_query)
 
-        if value["val"] != "" and value["val"] is not None:
+        if value["op"] == "null" or value["op"] == "not_null":
+            self.append_null_search_filters(value, node, query, request)
+        elif value["val"] != "" and value["val"] is not None:
             edtf = ExtendedDateFormat(value["val"])
             if edtf.result_set:
                 for result in edtf.result_set:
@@ -1518,14 +1568,17 @@ def transform_export_values(self, value, *args, **kwargs):
 
     def append_search_filters(self, value, node, query, request):
         try:
-            if value["val"] != "":
-                search_query = Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"])
-                # search_query = Term(field='tiles.data.%s' % (str(node.pk)), term=str(value['val']))
+            if value["op"] == "null" or value["op"] == "not_null":
+                self.append_null_search_filters(value, node, query, request)
+            elif value["val"] != "":
+                base_query = Bool()
+                base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
+                search_query = Nested(path="tiles", query=Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"]))
                 if "!" in value["op"]:
-                    query.must_not(search_query)
-                    query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
+                    base_query.must_not(search_query)
                 else:
-                    query.must(search_query)
+                    base_query.must(search_query)
+                query.must(base_query)
 
         except KeyError as e:
             pass
@@ -1623,14 +1676,17 @@ def transform_export_values(self, value, *args, **kwargs):
 
     def append_search_filters(self, value, node, query, request):
         try:
-            if value["val"] != "":
-                search_query = Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"])
-                # search_query = Term(field='tiles.data.%s' % (str(node.pk)), term=str(value['val']))
+            if value["op"] == "null" or value["op"] == "not_null":
+                self.append_null_search_filters(value, node, query, request)
+            elif value["val"] != "":
+                base_query = Bool()
+                base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
+                search_query = Nested(path="tiles", query=Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"]))
                 if "!" in value["op"]:
-                    query.must_not(search_query)
-                    query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
+                    base_query.must_not(search_query)
                 else:
-                    query.must(search_query)
+                    base_query.must(search_query)
+                query.must(base_query)
         except KeyError as e:
             pass
 
@@ -1832,14 +1888,19 @@ def transform_export_values(self, value, *args, **kwargs):
 
     def append_search_filters(self, value, node, query, request):
         try:
-            if value["val"] != "" and value["val"] != []:
-                # search_query = Match(field="tiles.data.%s.resourceId" % (str(node.pk)), type="phrase", query=value["val"])
-                search_query = Terms(field="tiles.data.%s.resourceId.keyword" % (str(node.pk)), terms=value["val"])
+            if value["op"] == "null" or value["op"] == "not_null":
+                self.append_null_search_filters(value, node, query, request)
+            elif value["val"] != "" and value["val"] != []:
+                base_query = Bool()
+                base_query.filter(Terms(field="graph_id", terms=[str(node.graph_id)]))
+                search_query = Nested(
+                    path="tiles", query=Terms(field="tiles.data.%s.resourceId.keyword" % (str(node.pk)), terms=value["val"])
+                )
                 if "!" in value["op"]:
-                    query.must_not(search_query)
-                    query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
+                    base_query.must_not(search_query)
                 else:
-                    query.must(search_query)
+                    base_query.must(search_query)
+                query.must(base_query)
         except KeyError as e:
             pass
 

arches/app/search/components/advanced_search.py

@@ -2,7 +2,7 @@
 from arches.app.models.system_settings import settings
 from arches.app.datatypes.datatypes import DataTypeFactory
 from arches.app.utils.betterJSONSerializer import JSONDeserializer
-from arches.app.search.elasticsearch_dsl_builder import Bool, Nested
+from arches.app.search.elasticsearch_dsl_builder import Bool, Nested, Terms
 from arches.app.search.components.base import BaseSearchFilter
 
 details = {
@@ -36,15 +36,15 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis
                     if self.request.user.has_perm("read_nodegroup", node.nodegroup):
                         datatype = datatype_factory.get_instance(node.datatype)
                         datatype.append_search_filters(val, node, tile_query, self.request)
-            nested_query = Nested(path="tiles", query=tile_query)
             if advanced_filter["op"] == "or" and index != 0:
                 grouped_query = Bool()
                 grouped_queries.append(grouped_query)
-            grouped_query.must(nested_query)
+            grouped_query.must(tile_query)
         for grouped_query in grouped_queries:
             advanced_query.should(grouped_query)
         search_query.must(advanced_query)
         search_results_object["query"].add_query(search_query)
+        print(search_results_object["query"])
 
     def view_data(self):
         ret = {}

arches/app/templates/views/components/datatypes/boolean.htm

@@ -8,7 +8,9 @@
     }, options: [
         {id: '', name: '{% trans "Select an Option" %}'},
         {id: 't', name: trueLabel},
-        {id: 'f', name: falseLabel}
+        {id: 'f', name: falseLabel},
+        {id: 'null', name: '{% trans "Has no value" %}'},
+        {id: 'not_null', name: '{% trans "Has any value" %}'}
     ], optionsText: 'name', optionsValue: 'id'"></select>
 </div>
 {% endblock search %}

arches/app/templates/views/components/datatypes/concept.htm

@@ -15,10 +15,12 @@
     <select class="resources" data-bind="value: op, chosen: {width: '100%', disable_search_threshold: 15}">
         <option value="">{% trans "Equals" %}</option>
         <option value="!">{% trans "Not" %}</option>
+        <option value="null">{% trans "Has no value" %}</option>
+        <option value="not_null">{% trans "Has any value" %}</option>
     </select>
 </div>
 
-<div class="col-md-8 col-lg-9">
+<div class="col-md-8 col-lg-9" data-bind="visible: op() !== 'null' && op() !== 'not_null'">
     <input style="display:inline-block;"
         data-bind="
             value: searchValue,

arches/app/templates/views/components/datatypes/date.htm

@@ -7,10 +7,12 @@
         <option value="lt"> < </option>
         <option value="gte"> >= </option>
         <option value="lte"> <= </option>
+        <option value="null">{% trans "Has no value" %}</option>
+        <option value="not_null">{% trans "Has any value" %}</option>
     </select>
 </div>
 
-<div class="col-md-8 col-lg-9">
+<div class="col-md-8 col-lg-9" data-bind="visible: op() !== 'null' && op() !== 'not_null'">
     <input type="" placeholder="{% trans "Date" %}" class="form-control input-md widget-input" data-bind="value: searchValue, datepicker: {format: dateFormat, viewMode: 'days', minDate: false, maxDate: false}">
 </div>
 <!-- /ko -->