Add new IQueryIndex interface for indices.

This introduces a new `query` method on each index with a simplified
contract compared to `_apply_index`. The responsibility for parsing and
skipping the query has moved into the catalog, and the return value no
longer has to be a tuple of (result, used_attributes), as the later
wasn't used by the catalog.

CHANGES.rst

@@ -4,6 +4,13 @@ Changelog
 4.0 (unreleased)
 ----------------
 
+- Add new IQueryIndex interface for indices. This introduces a new
+  `query` method on each index with a simplified contract compared to
+  `_apply_index`. The responsibility for parsing and skipping the query
+  has moved into the catalog, and the return value no longer has to be
+  a tuple of (result, used_attributes), as the later wasn't used by the
+  catalog.
+
 - Rename `parseIndexRequest` to `IndexQuery` and move it to `ZCatalog.query`.
 
 - Remove unused ZMI icons.

src/Products/PluginIndexes/BooleanIndex/BooleanIndex.py

@@ -194,26 +194,21 @@ def unindex_object(self, documentId):
             LOG.debug('Attempt to unindex nonexistent document'
                       ' with id %s' % documentId, exc_info=True)
 
-    def _apply_index(self, request, resultset=None):
-        record = IndexQuery(request, self.id, self.query_options)
-        if record.keys is None:
-            return None
-
+    def query(self, record, resultset=None):
         index = self._index
         indexed = self._index_value
 
         for key in record.keys:
             if bool(key) is bool(indexed):
                 # If we match the indexed value, check index
-                return (intersection(index, resultset), (self.id, ))
+                return intersection(index, resultset)
             else:
                 # Otherwise, remove from resultset or _unindex
                 if resultset is None:
-                    return (union(difference(self._unindex, index), IISet([])),
-                            (self.id, ))
+                    return union(difference(self._unindex, index), IISet([]))
                 else:
-                    return (difference(resultset, index), (self.id, ))
-        return (IISet(), (self.id, ))
+                    return difference(resultset, index)
+        return IISet()
 
     def indexSize(self):
         """Return distinct values, as an optimization we always claim 2."""

src/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py

@@ -245,32 +245,22 @@ def getRequestCacheKey(self, record, resultset=None):
         return (iid, rid)
 
     def _apply_index(self, request, resultset=None):
-        """Apply the index to query parameters given in 'request', which
-        should be a mapping object.
-
-        If the request does not contain the needed parameters, then
-        return None.
-
-        Otherwise return two objects.  The first object is a ResultSet
-        containing the record numbers of the matching records.  The
-        second object is a tuple containing the names of all data fields
-        used.
-        """
         record = IndexQuery(request, self.id, self.query_options)
         if record.keys is None:
             return None
+        return (self.query(record, resultset=resultset),
+                (self._since_field, self._until_field))
 
+    def query(self, record, resultset=None):
         cache = self.getRequestCache()
         if cache is not None:
             cachekey = self.getRequestCacheKey(record, resultset)
             cached = cache.get(cachekey, None)
             if cached is not None:
                 if resultset is None:
-                    return (cached,
-                            (self._since_field, self._until_field))
+                    return cached
                 else:
-                    return (difference(resultset, cached),
-                            (self._since_field, self._until_field))
+                    return difference(resultset, cached)
 
         term = self._convertDateTime(record.keys[0])
         if resultset is None:
@@ -288,7 +278,7 @@ def _apply_index(self, request, resultset=None):
             if cache is not None:
                 cache[cachekey] = result
 
-            return (result, (self._since_field, self._until_field))
+            return result
         else:
             # Compute the inverse and subtract from res
             until_only = multiunion(self._until_only.values(None, term - 1))
@@ -300,8 +290,7 @@ def _apply_index(self, request, resultset=None):
             if cache is not None:
                 cache[cachekey] = result
 
-            return (difference(resultset, result),
-                    (self._since_field, self._until_field))
+            return difference(resultset, result)
 
     def _insert_migrate(self, tree, key, value):
         treeset = tree.get(key, None)

src/Products/PluginIndexes/PathIndex/PathIndex.py

@@ -27,9 +27,12 @@
 from zope.interface import implements
 
 from Products.PluginIndexes.common import safe_callable
-from Products.PluginIndexes.interfaces import IPathIndex
-from Products.PluginIndexes.interfaces import ISortIndex
-from Products.PluginIndexes.interfaces import IUniqueValueIndex
+from Products.PluginIndexes.interfaces import (
+    IPathIndex,
+    IQueryIndex,
+    ISortIndex,
+    IUniqueValueIndex,
+)
 from Products.ZCatalog.query import IndexQuery
 
 LOG = getLogger('Zope.PathIndex')
@@ -50,7 +53,7 @@ class PathIndex(Persistent, SimpleItem):
     - the value is a mapping 'level of the path component' to
       'all docids with this path component on this level'
     """
-    implements(IPathIndex, IUniqueValueIndex, ISortIndex)
+    implements(IPathIndex, IQueryIndex, IUniqueValueIndex, ISortIndex)
 
     meta_type = "PathIndex"
     query_options = ('query', 'level', 'operator')
@@ -153,19 +156,21 @@ def unindex_object(self, docid):
         del self._unindex[docid]
 
     def _apply_index(self, request):
-        """ See IPluggableIndex.
-
-        o Unpacks args from catalog and mapps onto '_search'.
-        """
         record = IndexQuery(request, self.id, self.query_options)
         if record.keys is None:
             return None
+        return (self.query(record), (self.id, ))
 
-        level = record.get("level", 0)
+    def query(self, record, resultset=None):
+        """See IPluggableIndex.
+
+        o Unpacks args from catalog and mapps onto '_search'.
+        """
+        level = record.get('level', 0)
         operator = record.get('operator', self.useOperator).lower()
 
         # depending on the operator we use intersection of union
-        if operator == "or":
+        if operator == 'or':
             set_func = union
         else:
             set_func = intersection
@@ -176,9 +181,8 @@ def _apply_index(self, request):
             res = set_func(res, rows)
 
         if res:
-            return res, (self.id,)
-        else:
-            return IISet(), (self.id,)
+            return res
+        return IISet()
 
     def numObjects(self):
         """ See IPluggableIndex.

src/Products/PluginIndexes/TopicIndex/TopicIndex.py

@@ -22,8 +22,10 @@
 from Persistence import Persistent
 from zope.interface import implements
 
-from Products.PluginIndexes.interfaces import IPluggableIndex
-from Products.PluginIndexes.interfaces import ITopicIndex
+from Products.PluginIndexes.interfaces import (
+    IQueryIndex,
+    ITopicIndex,
+)
 from Products.PluginIndexes.TopicIndex.FilteredSet import factory
 from Products.ZCatalog.query import IndexQuery
 
@@ -37,7 +39,7 @@ class TopicIndex(Persistent, SimpleItem):
     Every FilteredSet object consists of an expression and and IISet with all
     Ids of indexed objects that eval with this expression to 1.
     """
-    implements(ITopicIndex, IPluggableIndex)
+    implements(ITopicIndex, IQueryIndex)
 
     meta_type = "TopicIndex"
     query_options = ('query', 'operator')
@@ -89,13 +91,15 @@ def search(self, filter_id):
             return f.getIds()
 
     def _apply_index(self, request):
-        """hook for (Z)Catalog
-        'request' --  mapping type (usually {"topic": "..." }
-        """
         record = IndexQuery(request, self.id, self.query_options)
         if record.keys is None:
             return None
+        return (self.query(record), (self.id, ))
 
+    def query(self, record, resultset=None):
+        """Hook for (Z)Catalog
+        'record' --  mapping type (usually {"topic": "..." }
+        """
         operator = record.get('operator', self.defaultOperator).lower()
         if operator == 'or':
             set_func = union
@@ -108,9 +112,8 @@ def _apply_index(self, request):
             res = set_func(res, rows)
 
         if res:
-            return res, (self.id,)
-        else:
-            return IITreeSet(), (self.id,)
+            return res
+        return IITreeSet()
 
     def uniqueValues(self, name=None, withLength=0):
         """ needed to be consistent with the interface """

src/Products/PluginIndexes/common/UnIndex.py

@@ -33,24 +33,27 @@
 
 from Products.PluginIndexes.common import safe_callable
 from Products.PluginIndexes.common.util import RequestCache
-from Products.PluginIndexes.interfaces import ILimitedResultIndex
-from Products.PluginIndexes.interfaces import ISortIndex
-from Products.PluginIndexes.interfaces import IUniqueValueIndex
-from Products.PluginIndexes.interfaces import IRequestCacheIndex
+from Products.PluginIndexes.interfaces import (
+    ILimitedResultIndex,
+    IQueryIndex,
+    ISortIndex,
+    IUniqueValueIndex,
+    IRequestCacheIndex,
+)
 from Products.ZCatalog.query import IndexQuery
 
 _marker = []
 LOG = getLogger('Zope.UnIndex')
 
 
 class UnIndex(SimpleItem):
-
     """Simple forward and reverse index.
     """
-    implements(ILimitedResultIndex, IUniqueValueIndex,
+    implements(ILimitedResultIndex, IQueryIndex, IUniqueValueIndex,
                ISortIndex, IRequestCacheIndex)
 
     _counter = None
+    query_options = ()
 
     def __init__(self, id, ignore_ex=None, call_methods=None,
                  extra=None, caller=None):
@@ -385,42 +388,31 @@ def getRequestCacheKey(self, record, resultset=None):
     def _apply_index(self, request, resultset=None):
         """Apply the index to query parameters given in the request arg.
 
-        The request argument should be a mapping object.
+        If the query does not match the index, return None, otherwise
+        return a tuple of (result, used_attributes), where used_attributes
+        is again a tuple with the names of all used data fields.
+        """
+        record = IndexQuery(request, self.id, self.query_options)
+        if record.keys is None:
+            return None
+        return (self.query(record, resultset=resultset), (self.id, ))
 
-        If the request does not have a key which matches the "id" of
-        the index instance, then None is returned.
+    def query(self, record, resultset=None):
+        """Search the index with the given IndexQuery object.
 
-        If the request *does* have a key which matches the "id" of
+        If the query has a key which matches the 'id' of
         the index instance, one of a few things can happen:
 
-          - if the value is a blank string, None is returned (in
-            order to support requests from web forms where
-            you can't tell a blank string from empty).
-
-          - if the value is a nonblank string, turn the value into
+          - if the value is a string, turn the value into
             a single-element sequence, and proceed.
 
           - if the value is a sequence, return a union search.
 
           - If the value is a dict and contains a key of the form
             '<index>_operator' this overrides the default method
-            ('or') to combine search results. Valid values are "or"
-            and "and".
-
-        If None is not returned as a result of the abovementioned
-        constraints, two objects are returned.  The first object is a
-        ResultSet containing the record numbers of the matching
-        records.  The second object is a tuple containing the names of
-        all data fields used.
-
-        FAQ answer:  to search a Field Index for documents that
-        have a blank string as their value, wrap the request value
-        up in a tuple ala: request = {'id':('',)}
+            ('or') to combine search results. Valid values are 'or'
+            and 'and'.
         """
-        record = IndexQuery(request, self.id, self.query_options)
-        if record.keys is None:
-            return None
-
         index = self._index
         r = None
         opr = None
@@ -431,7 +423,7 @@ def _apply_index(self, request, resultset=None):
         # experimental code for specifing the operator
         operator = record.get('operator', self.useOperator)
         if operator not in self.operators:
-            raise RuntimeError("operator not valid: %s" % escape(operator))
+            raise RuntimeError('operator not valid: %s' % escape(operator))
 
         cachekey = None
         cache = self.getRequestCache()
@@ -463,7 +455,7 @@ def _apply_index(self, request, resultset=None):
                         exclude = self._apply_not(not_parm, resultset)
                         cached = difference(cached, exclude)
 
-                    return cached, (self.id,)
+                    return cached
 
         if not record.keys and not_parm:
             # convert into indexed format
@@ -518,7 +510,7 @@ def _apply_index(self, request, resultset=None):
                 if not_parm:
                     exclude = self._apply_not(not_parm, resultset)
                     result = difference(result, exclude)
-                return result, (self.id,)
+                return result
 
             if operator == 'or':
                 tmp = []
@@ -576,7 +568,7 @@ def _apply_index(self, request, resultset=None):
                         # If operator is 'and', we have to cache a list of
                         # IISet objects
                         cache[cachekey] = [IISet()]
-                    return IISet(), (self.id,)
+                    return IISet()
                 elif isinstance(s, int):
                     s = IISet((s,))
                 setlist.append(s)
@@ -596,7 +588,7 @@ def _apply_index(self, request, resultset=None):
                 if not_parm:
                     exclude = self._apply_not(not_parm, resultset)
                     result = difference(result, exclude)
-                return result, (self.id,)
+                return result
 
             if operator == 'or':
                 # If we already get a small result set passed in, intersecting
@@ -639,11 +631,11 @@ def _apply_index(self, request, resultset=None):
         if isinstance(r, int):
             r = IISet((r, ))
         if r is None:
-            return IISet(), (self.id,)
+            return IISet()
         if not_parm:
             exclude = self._apply_not(not_parm, resultset)
             r = difference(r, exclude)
-        return r, (self.id,)
+        return r
 
     def hasUniqueValuesFor(self, name):
         """has unique values for column name"""

src/Products/PluginIndexes/interfaces.py

@@ -13,7 +13,7 @@
 """PluginIndexes interfaces.
 """
 
-from zope.interface import Interface
+from zope.interface import Attribute, Interface
 from zope.schema import Bool
 
 
@@ -96,6 +96,17 @@ def _apply_index(request, resultset=None):
         """
 
 
+class IQueryIndex(IPluggableIndex):
+
+    id = Attribute('Index id used to query the index.')
+    query_options = Attribute('Supported query options for the index.')
+
+    def query(record, resultset=None):
+        """Same as _apply_index, but the query is already a pre-parsed
+        IndexQuery object.
+        """
+
+
 class IUniqueValueIndex(IPluggableIndex):
     """An index which can return lists of unique values contained in it"""