Not indexed value support (MissingValue, EmptyValue)

src/Products/PluginIndexes/BooleanIndex/BooleanIndex.py

@@ -155,7 +155,7 @@ def _index_object(self, documentId, obj, threshold=None, attr=''):
         returnStatus = 0
 
         # First we need to see if there's anything interesting to look at
-        datum = self._get_object_datum(obj, attr)
+        datum = self.get_object_datum(obj, attr)
 
         # Make it boolean, int as an optimization
         if datum is not _marker:

src/Products/PluginIndexes/CompositeIndex/CompositeIndex.py

@@ -21,16 +21,31 @@
 from Acquisition import aq_parent
 from Acquisition import aq_inner
 from App.special_dtml import DTMLFile
-from BTrees.OOBTree import difference
 from BTrees.OOBTree import OOSet
 from Persistence import PersistentMapping
-from zope.interface import implementer
-
-from Products.PluginIndexes.interfaces import ITransposeQuery
+from zope.interface import implementer_only
+
+from Products.PluginIndexes.interfaces import (
+    ILimitedResultIndex,
+    IQueryIndex,
+    ISortIndex,
+    IUniqueValueIndex,
+    IRequestCacheIndex,
+    ITransposeQuery,
+    missing,
+    empty,
+)
+from Products.PluginIndexes.util import safe_callable
 from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
 from Products.PluginIndexes.unindex import _marker
 from Products.ZCatalog.query import IndexQuery
 
+try:
+    basestring
+except NameError:
+    # Python 3 compatibility
+    basestring = (bytes, str)
+
 LOG = logging.getLogger('CompositeIndex')
 
 QUERY_OPTIONS = {
@@ -172,7 +187,8 @@ def __repr__(self):
                 'attributes: {0.attributes}>').format(self)
 
 
-@implementer(ITransposeQuery)
+@implementer_only(ILimitedResultIndex, IQueryIndex, IUniqueValueIndex,
+                  ISortIndex, IRequestCacheIndex, ITransposeQuery)
 class CompositeIndex(KeywordIndex):
 
     """Index for composition of simple fields.
@@ -209,45 +225,8 @@ def __init__(self, id, ignore_ex=None, call_methods=None,
                                                    c_attributes)
         self.clear()
 
-    def _index_object(self, documentId, obj, threshold=None, attr=''):
-
-        # get permuted keywords
-        newKeywords = self._get_permuted_keywords(obj)
-
-        oldKeywords = self._unindex.get(documentId, None)
-
-        if oldKeywords is None:
-            # we've got a new document, let's not futz around.
-            try:
-                for kw in newKeywords:
-                    self.insertForwardIndexEntry(kw, documentId)
-                if newKeywords:
-                    self._unindex[documentId] = list(newKeywords)
-            except TypeError:
-                return 0
-        else:
-            # we have an existing entry for this document, and we need
-            # to figure out if any of the keywords have actually changed
-            if type(oldKeywords) is not OOSet:
-                oldKeywords = OOSet(oldKeywords)
-            newKeywords = OOSet(newKeywords)
-            fdiff = difference(oldKeywords, newKeywords)
-            rdiff = difference(newKeywords, oldKeywords)
-            if fdiff or rdiff:
-                # if we've got forward or reverse changes
-                if newKeywords:
-                    self._unindex[documentId] = list(newKeywords)
-                else:
-                    del self._unindex[documentId]
-                if fdiff:
-                    self.unindex_objectKeywords(documentId, fdiff)
-                if rdiff:
-                    for kw in rdiff:
-                        self.insertForwardIndexEntry(kw, documentId)
-        return 1
-
-    def _get_permuted_keywords(self, obj):
-        """ returns permutation tuple of object keywords """
+    def get_object_datum(self, obj, attr):
+        """ returns permutation of object keywords """
 
         components = self.getIndexComponents()
         kw_list = []
@@ -270,34 +249,48 @@ def _get_permuted_keywords(self, obj):
                 p = combinations(c, r)
                 pkl.extend(p)
 
-        return tuple(pkl)
+        return OOSet(pkl)
+
+    def _get_component_datum(self, obj, attr):
+        # self.id is the name of the index, which is also the name of the
+        # attribute we're interested in.  If the attribute is callable,
+        # we'll do so.
+        try:
+            datum = getattr(obj, attr)
+            if safe_callable(datum):
+                datum = datum()
+        except (AttributeError, TypeError):
+            datum = _marker
+        return datum
 
     def _get_component_keywords(self, obj, component):
 
         if component.meta_type == 'FieldIndex':
             # last attribute is the winner if value is not None
             for attr in component.attributes:
-                datum = self._get_object_datum(obj, attr)
+                datum = self._get_component_datum(obj, attr)
                 if datum is None:
                     continue
             if datum is None:
                 return ()
-            if isinstance(datum, list):
-                datum = tuple(datum)
+            if isinstance(datum, (list, OOSet)):
+                return tuple(datum)
             return (datum,)
 
         elif component.meta_type == 'KeywordIndex':
             # last attribute is the winner
             attr = component.attributes[-1]
-            datum = self._get_object_keywords(obj, attr)
+            datum = self._get_component_datum(obj, attr)
+            if isinstance(datum, basestring):
+                datum = (datum,)
             if isinstance(datum, list):
                 datum = tuple(datum)
             return datum
 
         elif component.meta_type == 'BooleanIndex':
             # last attribute is the winner
             attr = component.attributes[-1]
-            datum = self._get_object_datum(obj, attr)
+            datum = self._get_component_datum(obj, attr)
             if datum is not _marker:
                 datum = int(bool(datum))
             return (datum,)
@@ -380,6 +373,14 @@ def make_query(self, query):
             if c.meta_type == 'BooleanIndex':
                 rec.keys = [int(bool(v)) for v in rec.keys[:]]
 
+            # cannot currently support KeywordIndex's
+            # missing/empty feature
+            if c.meta_type == 'KeywordIndex':
+                if missing in rec.keys:
+                    continue
+                if empty in rec.keys:
+                    continue
+
             # rec with 'not' parameter
             not_parm = rec.get('not', None)
             if not_parm:

src/Products/PluginIndexes/CompositeIndex/tests/testCompositeIndex.py

@@ -21,32 +21,29 @@
 types = ['Document', 'News', 'File', 'Image']
 default_pages = [True, False, False, False, False, False]
 subjects = list(map(lambda x: 'subject_{0}'.format(x), range(6)))
-keywords = list(map(lambda x: 'keyword_{0}'.format(x), range(6)))
 
 
 class TestObject(object):
 
     def __init__(self, id, portal_type, review_state,
-                 is_default_page=False, subject=(), keyword=()):
+                 is_default_page=False, subject=()):
         self.id = id
         self.portal_type = portal_type
         self.review_state = review_state
         self.is_default_page = is_default_page
         self.subject = subject
-        self.keyword = keyword
 
     def getPhysicalPath(self):
         return ['', self.id, ]
 
     def __repr__(self):
         return ('< {id}, {portal_type}, {review_state},\
-        {is_default_page}, {subject} , {keyword}>'.format(
+        {is_default_page}, {subject}>'.format(
             id=self.id,
             portal_type=self.portal_type,
             review_state=self.review_state,
             is_default_page=self.is_default_page,
-            subject=self.subject,
-            keyword=self.keyword))
+            subject=self.subject))
 
 
 class RandomTestObject(TestObject):
@@ -63,11 +60,10 @@ def __init__(self, id):
         is_default_page = default_pages[i]
 
         subject = random.sample(subjects, random.randint(1, len(subjects)))
-        keyword = random.sample(keywords, random.randint(1, len(keywords)))
 
         super(RandomTestObject, self).__init__(id, portal_type,
                                                review_state, is_default_page,
-                                               subject, keyword)
+                                               subject)
 
 
 # Pseudo ContentLayer class to support quick
@@ -92,7 +88,7 @@ def setUp(self):
                          KeywordIndex('subject',
                                       extra={
                                           'indexed_attrs':
-                                          'keyword,subject'}
+                                          'subject'}
                                       ),
                          CompositeIndex('comp01',
                                         extra=[{'id': 'portal_type',
@@ -107,7 +103,7 @@ def setUp(self):
                                                {'id': 'subject',
                                                 'meta_type': 'KeywordIndex',
                                                 'attributes':
-                                                'keyword,subject'}
+                                                'subject'}
                                                ])
                          ]
 
@@ -206,9 +202,6 @@ def testPerformance(self):
                    ('query02_default_two_indexes',
                     {'portal_type': {'query': 'Document'},
                      'subject': {'query': 'subject_2'}}),
-                   ('query02_default_two_indexes_zero_hits',
-                    {'portal_type': {'query': 'Document'},
-                     'subject': {'query': ['keyword_1', 'keyword_2']}}),
                    ('query03_default_two_indexes',
                     {'portal_type': {'query': 'Document'},
                      'subject': {'query': ['subject_1', 'subject_3']}}),
@@ -340,8 +333,7 @@ def testSearch(self):
                          subject=('subject_1', 'subject_2'))
         self.populateIndexes(3, obj)
         obj = TestObject('obj_4', 'Event', 'private',
-                         subject=('subject_1', 'subject_2'),
-                         keyword=('keyword_1', ))
+                         subject=('subject_1', 'subject_2'))
         self.populateIndexes(4, obj)
 
         queries = [
@@ -379,12 +371,6 @@ def testSearch(self):
              'is_default_page': {'query': False},
              'subject': {'query': ('subject_1', 'subject_2'),
                          'operator': 'and'}},
-            # query on five attributes with
-            {'review_state': {'not': ('pending', 'visible')},
-             'portal_type': {'query': ('News', 'Document')},
-             'is_default_page': {'query': False},
-             'subject': {'query': ('subject_1', )},
-             'keyword': {'query': ('keyword_1',)}},
         ]
 
         for query in queries: