Consolidate special value handling step two

src/Products/PluginIndexes/BooleanIndex/BooleanIndex.py

@@ -155,7 +155,7 @@ def _index_object(self, documentId, obj, threshold=None, attr=''):
         returnStatus = 0
 
         # First we need to see if there's anything interesting to look at
-        datum = self._get_object_datum(obj, attr)
+        datum = self.get_object_datum(obj, attr)
 
         # Make it boolean, int as an optimization
         if datum is not _marker:

src/Products/PluginIndexes/CompositeIndex/CompositeIndex.py

@@ -36,10 +36,16 @@
     missing,
     empty,
 )
+from Products.PluginIndexes.util import safe_callable
 from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
 from Products.PluginIndexes.unindex import _marker
 from Products.ZCatalog.query import IndexQuery
 
+try:
+    basestring
+except NameError:
+    # Python 3 compatibility
+    basestring = (bytes, str)
 
 LOG = logging.getLogger('CompositeIndex')
 
@@ -281,34 +287,48 @@ def _get_permuted_keywords(self, obj):
                 p = combinations(c, r)
                 pkl.extend(p)
 
-        return tuple(pkl)
+        return OOSet(pkl)
+
+    def get_object_datum(self, obj, attr):
+        # self.id is the name of the index, which is also the name of the
+        # attribute we're interested in.  If the attribute is callable,
+        # we'll do so.
+        try:
+            datum = getattr(obj, attr)
+            if safe_callable(datum):
+                datum = datum()
+        except (AttributeError, TypeError):
+            datum = _marker
+        return datum
 
     def _get_component_keywords(self, obj, component):
 
         if component.meta_type == 'FieldIndex':
             # last attribute is the winner if value is not None
             for attr in component.attributes:
-                datum = self._get_object_datum(obj, attr)
+                datum = self.get_object_datum(obj, attr)
                 if datum is None:
                     continue
             if datum is None:
                 return ()
-            if isinstance(datum, list):
-                datum = tuple(datum)
+            if isinstance(datum, (list, OOSet)):
+                return tuple(datum)
             return (datum,)
 
         elif component.meta_type == 'KeywordIndex':
             # last attribute is the winner
             attr = component.attributes[-1]
-            datum = self._get_object_keywords(obj, attr)
+            datum = self.get_object_datum(obj, attr)
+            if isinstance(datum, basestring):
+                datum = (datum,)
             if isinstance(datum, list):
                 datum = tuple(datum)
             return datum
 
         elif component.meta_type == 'BooleanIndex':
             # last attribute is the winner
             attr = component.attributes[-1]
-            datum = self._get_object_datum(obj, attr)
+            datum = self.get_object_datum(obj, attr)
             if datum is not _marker:
                 datum = int(bool(datum))
             return (datum,)

src/Products/PluginIndexes/KeywordIndex/KeywordIndex.py

@@ -20,7 +20,6 @@
 from zope.interface import implementer
 
 from Products.PluginIndexes.unindex import UnIndex
-from Products.PluginIndexes.util import safe_callable
 from Products.PluginIndexes.interfaces import (
     IIndexingMissingValue,
     missing,
@@ -71,7 +70,7 @@ def _index_object(self, documentId, obj, threshold=None, attr=''):
         # attribute we're interested in.  If the attribute is callable,
         # we'll do so.
 
-        newKeywords = self._get_object_keywords(obj, attr)
+        newKeywords = self.get_object_datum(obj, attr)
         oldKeywords = self._unindex.get(documentId, _marker)
 
         if oldKeywords is _marker:
@@ -121,60 +120,22 @@ def _index_object(self, documentId, obj, threshold=None, attr=''):
 
         return 1
 
-    def _get_object_keywords(self, obj, attr):
-        newKeywords = getattr(obj, attr, None)
-
-        def _getSpecialValueFor(datum):
-            try:
-                special_value = self.special_values[datum]
-            except TypeError:
-                raise KeyError(datum)
-
-            if self.providesSpecialIndex(special_value):
-                return special_value
-            raise KeyError(datum)
-
-        if safe_callable(newKeywords):
-            try:
-                newKeywords = newKeywords()
-            except (AttributeError, TypeError):
-                LOG.debug('%(context)s: Cannot determine datum for attribute '
-                          '%(attr)s of object %(obj)r', dict(
-                              context=self.__class__.__name__,
-                              attr=attr,
-                              obj=obj),
-                          exc_info=True)
-
-                newKeywords = sys.exc_info()[0]
-                try:
-                    return _getSpecialValueFor(newKeywords)
-                except KeyError:
-                    return _marker
-
-        try:
-            return _getSpecialValueFor(newKeywords)
-        except KeyError:
-            pass
-
-        keywords = OOSet()
-        # normalize datum
-        if isinstance(newKeywords, basestring):
-            keywords.insert(newKeywords)
-        else:
-            try:
-                # unique
-                keywords.update(newKeywords)
-            except TypeError:
-                # Not a sequence
-                keywords.insert(newKeywords)
+    def map_value(self, value):
+        value = super(KeywordIndex, self).map_value(value)
+        if value is not missing:
+            # at this place, *value* is expected to be a sequence
+            if isinstance(value, basestring):
+                value = OOSet((value,))
+            if not value and self.providesSpecialIndex(empty):
+                value = empty
+            else:
+                value = OOSet(value)
 
-        try:
-            return _getSpecialValueFor(tuple(keywords))
-        except KeyError:
-            return keywords
+        return value
 
     def index_objectKeywords(self, documentId, keywords):
-        """ carefully index the object with integer id 'documentId'"""
+        """ carefully index keywords of object with integer id 'documentId'
+        """
 
         indexed_keys = OOSet()
         for kw in keywords:
@@ -195,7 +156,8 @@ def index_objectKeywords(self, documentId, keywords):
         return indexed_keys
 
     def unindex_objectKeywords(self, documentId, keywords):
-        """ carefully unindex the object with integer id 'documentId'"""
+        """ carefully unindex keywords of object with integer id 'documentId'
+        """
 
         if keywords is not None:
             for kw in keywords:

src/Products/PluginIndexes/tests/test_unindex.py

@@ -20,6 +20,7 @@
 from zope.interface import directlyProvides
 
 from Products.ZCatalog.query import IndexQuery
+
 from Products.PluginIndexes.interfaces import (
     missing,
     IIndexingMissingValue,
@@ -79,12 +80,12 @@ def test_get_object_datum(self):
         idx = self._makeOne('interesting')
 
         dummy = object()
-        self.assertEqual(idx._get_object_datum(dummy, 'interesting'), _marker)
+        self.assertEqual(idx.get_object_datum(dummy, 'interesting'), _marker)
 
         class DummyContent2(object):
             interesting = 'GOT IT'
         dummy = DummyContent2()
-        self.assertEqual(idx._get_object_datum(dummy, 'interesting'),
+        self.assertEqual(idx.get_object_datum(dummy, 'interesting'),
                          'GOT IT')
 
         class DummyContent3(object):
@@ -95,14 +96,14 @@ def interesting(self):
                     raise self.exc
                 return 'GOT IT'
         dummy = DummyContent3()
-        self.assertEqual(idx._get_object_datum(dummy, 'interesting'),
+        self.assertEqual(idx.get_object_datum(dummy, 'interesting'),
                          'GOT IT')
 
         dummy.exc = AttributeError
-        self.assertEqual(idx._get_object_datum(dummy, 'interesting'), _marker)
+        self.assertEqual(idx.get_object_datum(dummy, 'interesting'), _marker)
 
         dummy.exc = TypeError
-        self.assertEqual(idx._get_object_datum(dummy, 'interesting'), _marker)
+        self.assertEqual(idx.get_object_datum(dummy, 'interesting'), _marker)
 
     def test_cache(self):
         idx = self._makeOne(id='foo')
@@ -258,7 +259,8 @@ def __init__(self, value):
 
         values = ((0, Dummy('a')),
                   (1, Dummy('b')),
-                  (2, Dummy(None)))
+                  (2, Dummy(None)),
+                  (3, Dummy('')))
 
         for i, obj in values:
             index.index_object(i, obj)
@@ -270,14 +272,14 @@ def __init__(self, value):
         self.assertEqual(tuple(apply(req)[0]), (0, 2))
 
         req = {'foo': {'not': 'a'}}
-        self.assertEqual(tuple(apply(req)[0]), (1, 2,))
+        self.assertEqual(tuple(apply(req)[0]), (1, 2, 3,))
 
         req = {'foo': {'not': ['a', missing]}}
-        self.assertEqual(tuple(apply(req)[0]), (1,))
+        self.assertEqual(tuple(apply(req)[0]), (1, 3,))
 
         index.unindex_object(2)
         req = {'foo': {'not': 'a'}}
-        self.assertEqual(tuple(apply(req)[0]), (1,))
+        self.assertEqual(tuple(apply(req)[0]), (1, 3,))
 
     def test_emptyvalue(self):
         index = self._makeOne('foo')
@@ -287,8 +289,18 @@ def test_emptyvalue(self):
         index.clear()
         # activate `not`, `operator`
         index.query_options = 'not', 'operator'
-        # define empty string as empty value
-        index.special_values.update({'': empty})
+
+        # patch method `map_value` in order to map empty
+        # strings to special value `empty`
+        def map_value(self, value):
+            if value is None:
+                return value
+            elif not value:
+                return empty
+            return value
+
+        import types
+        index.map_value = types.MethodType(map_value, index)
 
         apply = index._apply_index
 
@@ -300,7 +312,8 @@ def __init__(self, value):
 
         values = ((0, Dummy('a')),
                   (1, Dummy('b')),
-                  (2, Dummy('')))
+                  (2, Dummy('')),
+                  (3, Dummy(None)))
 
         for i, obj in values:
             index.index_object(i, obj)

src/Products/PluginIndexes/unindex.py

@@ -31,6 +31,7 @@
 from BTrees.OOBTree import OOBTree
 from OFS.SimpleItem import SimpleItem
 from ZODB.POSException import ConflictError
+from zope.deprecation import deprecation
 from zope.interface import implementer
 
 from Products.PluginIndexes.cache import RequestCache
@@ -311,7 +312,7 @@ def _index_object(self, documentId, obj, threshold=None, attr=''):
         returnStatus = 0
 
         # First we need to see if there's anything interesting to look at
-        datum = self._get_object_datum(obj, attr)
+        datum = self.get_object_datum(obj, attr)
         if datum is None:
             # Prevent None from being indexed. None doesn't have a valid
             # ordering definition compared to any other object.
@@ -366,7 +367,7 @@ def map_value(self, value):
         else:
             return value
 
-    def _get_object_datum(self, obj, attr):
+    def get_object_datum(self, obj, attr):
         # self.id is the name of the index, which is also the name of the
         # attribute we're interested in.  If the attribute is callable,
         # we'll do so.
@@ -381,6 +382,12 @@ def _get_object_datum(self, obj, attr):
             else:
                 return _marker
 
+    _get_object_datum = get_object_datum
+    _get_object_datum = deprecation.deprecated(_get_object_datum,
+                                               '`_get_object_datum` is moved '
+                                               'to public method '
+                                               '`get_object_datum`')
+
     def _increment_counter(self):
         if self._counter is None:
             self._counter = Length()