Added support for not queries to FieldIndexes - note that these are…

…n't optimized in any way and could be slow.

CHANGES.txt

@@ -1,8 +1,14 @@
 Changelog
 =========
 
-2.13.23 (unreleased)
---------------------
+3.0 (unreleased)
+----------------
+
+- Added support for `not` queries to FieldIndexes. Both restrictions of normal
+  queries and range queries are supported, as well as purely exclusive
+  queries. For example: `{'foo': {'query': ['a', 'ab'], 'not': 'a'}}`,
+  `{'query': 'a', 'range': 'min', 'not': ['a', 'e', 'f']}}` and
+  `{'foo': {'not': ['a', 'b']}}`.
 
 - Updated deprecation warnings to point to Zope 4 instead of 2.14.
 

setup.py

@@ -15,7 +15,7 @@
 from setuptools import setup, find_packages
 
 setup(name='Products.ZCatalog',
-      version = '2.13.23dev',
+      version = '3.0dev',
       url='http://pypi.python.org/pypi/Products.ZCatalog',
       license='ZPL 2.1',
       description="Zope 2's indexing and search solution.",

src/Products/PluginIndexes/FieldIndex/FieldIndex.py

@@ -29,7 +29,7 @@ class FieldIndex(UnIndex):
         {'label': 'Browse', 'action': 'manage_browse'},
     )
 
-    query_options = ["query","range"]
+    query_options = ["query", "range", "not"]
 
     manage = manage_main = DTMLFile('dtml/manageFieldIndex', globals())
     manage_main._setName('manage_main')

src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py

@@ -37,8 +37,6 @@ class FieldIndexTests(unittest.TestCase):
     """
 
     def setUp( self ):
-        """
-        """
         self._index = FieldIndex( 'foo' )
         self._marker = []
         self._values = [ ( 0, Dummy( 'a' ) )
@@ -57,23 +55,30 @@ def setUp( self ):
             keys = self._forward.get( v, [] )
             self._forward[v] = keys
 
-        self._noop_req  = { 'bar': 123 }
-        self._request   = { 'foo': 'abce' }
-        self._min_req   = { 'foo': {'query': 'abc'
-                          , 'range': 'min'}
-                          }
-        self._max_req   = { 'foo': {'query': 'abc'
-                          , 'range': 'max' }
-                          }
-        self._range_req = { 'foo': {'query': ( 'abc', 'abcd' )
-                          , 'range': 'min:max' }
-                          }
-        self._zero_req  = { 'foo': 0 }
-        self._none_req  = { 'foo': None }
-
-    def tearDown( self ):
-        """
-        """
+        self._noop_req  = {'bar': 123}
+        self._request   = {'foo': 'abce'}
+        self._min_req   = {'foo':
+            {'query': 'abc', 'range': 'min'}}
+        self._min_req_n = {'foo':
+            {'query': 'abc', 'range': 'min', 'not': 'abca'}}
+        self._max_req   = {'foo':
+            {'query': 'abc', 'range': 'max'}}
+        self._max_req_n = {'foo':
+            {'query': 'abc', 'range': 'max', 'not': ['a', 'b', None, 0]}}
+        self._range_req = {'foo':
+            {'query': ( 'abc', 'abcd' ), 'range': 'min:max'}}
+        self._range_ren = {'foo':
+            {'query': ( 'abc', 'abcd' ), 'range': 'min:max', 'not': 'abcd'}}
+        self._range_non = {'foo':
+            {'query': ( 'a', 'aa' ), 'range': 'min:max', 'not': 'a'}}
+        self._zero_req  = {'foo': 0 }
+        self._none_req  = {'foo': None }
+        self._not_1     = {'foo': {'query': 'a', 'not': 'a'}}
+        self._not_2     = {'foo': {'query': ['a', 'ab'], 'not': 'a'}}
+        self._not_3     = {'foo': {'not': 'a'}}
+        self._not_4     = {'foo': {'not': [0, None]}}
+        self._not_5     = {'foo': {'not': ['a', 'b']}}
+        self._not_6     = {'foo': 'a', 'bar': {'query': 123, 'not': 1}}
 
     def _populateIndex( self ):
         for k, v in self._values:
@@ -115,11 +120,15 @@ def testEmpty( self ):
         assert not self._index.hasUniqueValuesFor( 'bar' )
         assert len( self._index.uniqueValues( 'foo' ) ) == 0
 
-        assert self._index._apply_index( self._noop_req ) is None
-        self._checkApply( self._request, [] )
-        self._checkApply( self._min_req, [] )
-        self._checkApply( self._max_req, [] )
-        self._checkApply( self._range_req, [] )
+        assert self._index._apply_index(self._noop_req) is None
+        self._checkApply(self._request, [])
+        self._checkApply(self._min_req, [])
+        self._checkApply(self._min_req_n, [])
+        self._checkApply(self._max_req, [])
+        self._checkApply(self._max_req_n, [])
+        self._checkApply(self._range_req, [])
+        self._checkApply(self._range_ren, [])
+        self._checkApply(self._range_non, [])
 
     def testPopulated( self ):
         """ Test a populated FieldIndex """
@@ -142,10 +151,21 @@ def testPopulated( self ):
 
         assert self._index._apply_index( self._noop_req ) is None
 
-        self._checkApply( self._request, values[ -4:-2 ] )
-        self._checkApply( self._min_req, values[ 2:-2 ] )
-        self._checkApply( self._max_req, values[ :3 ] + values[ -2: ] )
-        self._checkApply( self._range_req, values[ 2:5 ] )
+        self._checkApply(self._request, values[-4:-2])
+        self._checkApply(self._min_req, values[2:-2])
+        self._checkApply(self._min_req_n, values[2:3] + values[4:-2])
+        self._checkApply(self._max_req, values[:3] + values[-2:])
+        self._checkApply(self._max_req_n, values[1:3])
+        self._checkApply(self._range_req, values[2:5])
+        self._checkApply(self._range_ren, values[2:4])
+        self._checkApply(self._range_non, [])
+
+        self._checkApply(self._not_1, [])
+        self._checkApply(self._not_2, values[1:2])
+        self._checkApply(self._not_3, values[1:])
+        self._checkApply(self._not_4, values[:7])
+        self._checkApply(self._not_5, values[1:])
+        self._checkApply(self._not_6, values[0:1])
 
     def testZero( self ):
         """ Make sure 0 gets indexed """

src/Products/PluginIndexes/common/UnIndex.py

@@ -17,6 +17,7 @@
 from logging import getLogger
 import sys
 
+from BTrees.IIBTree import difference
 from BTrees.IIBTree import intersection
 from BTrees.IIBTree import IITreeSet
 from BTrees.IIBTree import IISet
@@ -293,6 +294,18 @@ def unindex_object(self, documentId):
             LOG.debug('Attempt to unindex nonexistent document'
                       ' with id %s' % documentId,exc_info=True)
 
+    def _apply_not(self, not_parm, resultset=None):
+        index = self._index
+        setlist = []
+        for k in not_parm:
+            s = index.get(k, None)
+            if s is None:
+                continue
+            elif isinstance(s, int):
+                s = IISet((s, ))
+            setlist.append(s)
+        return multiunion(setlist)
+
     def _apply_index(self, request, resultset=None):
         """Apply the index to query parameters given in the request arg.
 
@@ -336,6 +349,13 @@ def _apply_index(self, request, resultset=None):
         r     = None
         opr   = None
 
+        # not / exclude parameter
+        not_parm = record.get('not', None)
+        if not record.keys and not_parm:
+            # we have only a 'not' query
+            record.keys = [k for k in index.keys() if k not in not_parm]
+            not_parm = None
+
         # experimental code for specifing the operator
         operator = record.get('operator',self.useOperator)
         if not operator in self.operators :
@@ -371,6 +391,9 @@ def _apply_index(self, request, resultset=None):
                 result = setlist[0]
                 if isinstance(result, int):
                     result = IISet((result,))
+                if not_parm:
+                    exclude = self._apply_not(not_parm, resultset)
+                    result = difference(result, exclude)
                 return result, (self.id,)
 
             if operator == 'or':
@@ -417,6 +440,9 @@ def _apply_index(self, request, resultset=None):
                 result = setlist[0]
                 if isinstance(result, int):
                     result = IISet((result,))
+                if not_parm:
+                    exclude = self._apply_not(not_parm, resultset)
+                    result = difference(result, exclude)
                 return result, (self.id,)
 
             if operator == 'or':
@@ -442,8 +468,10 @@ def _apply_index(self, request, resultset=None):
             r = IISet((r, ))
         if r is None:
             return IISet(), (self.id,)
-        else:
-            return r, (self.id,)
+        if not_parm:
+            exclude = self._apply_not(not_parm, resultset)
+            r = difference(r, exclude)
+        return r, (self.id,)
 
     def hasUniqueValuesFor(self, name):
         """has unique values for column name"""

src/Products/PluginIndexes/common/tests/test_util.py

@@ -53,6 +53,30 @@ def test_get_string(self):
         self.assertEqual(parser.get('level'), 0)
         self.assertEqual(parser.get('operator'), 'and')
 
+    def test_get_not_dict(self):
+        request = {'path': {'query': 'foo', 'not': 'bar'}}
+        parser = self._makeOne(request, 'path', ('query', 'not'))
+        self.assertEqual(parser.get('keys'), ['foo'])
+        self.assertEqual(parser.get('not'), ['bar'])
+
+    def test_get_not_dict_list(self):
+        request = {'path': {'query': 'foo', 'not': ['bar', 'baz']}}
+        parser = self._makeOne(request, 'path', ('query', 'not'))
+        self.assertEqual(parser.get('keys'), ['foo'])
+        self.assertEqual(parser.get('not'), ['bar', 'baz'])
+
+    def test_get_not_string(self):
+        request = {'path': 'foo', 'path_not': 'bar'}
+        parser = self._makeOne(request, 'path', ('query', 'not'))
+        self.assertEqual(parser.get('keys'), ['foo'])
+        self.assertEqual(parser.get('not'), ['bar'])
+
+    def test_get_not_string_list(self):
+        request = {'path': 'foo', 'path_not': ['bar', 'baz']}
+        parser = self._makeOne(request, 'path', ('query', 'not'))
+        self.assertEqual(parser.get('keys'), ['foo'])
+        self.assertEqual(parser.get('not'), ['bar', 'baz'])
+
 
 def test_suite():
     suite = unittest.TestSuite()

src/Products/PluginIndexes/common/util.py

@@ -118,6 +118,11 @@ def __init__(self, request, iid, options=[]):
                     setattr(self, op, request[field])
 
         self.keys = keys
+        not_value = getattr(self, 'not', None)
+        if not_value is not None:
+            if isinstance(not_value, basestring):
+                not_value = [not_value]
+                setattr(self, 'not', not_value)
 
     def get(self, k, default_v=None):
         if hasattr(self, k):