From 824b26f36b76330a2c61bf454c7dc86d9ccc6c94 Mon Sep 17 00:00:00 2001 From: Hanno Schlichting Date: Sun, 25 Mar 2012 12:49:53 +0000 Subject: [PATCH] Added support for `not` queries to FieldIndexes - note that these aren't optimized in any way and could be slow. --- CHANGES.txt | 10 ++- setup.py | 2 +- .../PluginIndexes/FieldIndex/FieldIndex.py | 2 +- .../FieldIndex/tests/testFieldIndex.py | 76 ++++++++++++------- src/Products/PluginIndexes/common/UnIndex.py | 32 +++++++- .../PluginIndexes/common/tests/test_util.py | 24 ++++++ src/Products/PluginIndexes/common/util.py | 5 ++ 7 files changed, 117 insertions(+), 34 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7a66f503..01b43e90 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,8 +1,14 @@ Changelog ========= -2.13.23 (unreleased) --------------------- +3.0 (unreleased) +---------------- + +- Added support for `not` queries to FieldIndexes. Both restrictions of normal + queries and range queries are supported, as well as purely exclusive + queries. For example: `{'foo': {'query': ['a', 'ab'], 'not': 'a'}}`, + `{'query': 'a', 'range': 'min', 'not': ['a', 'e', 'f']}}` and + `{'foo': {'not': ['a', 'b']}}`. - Updated deprecation warnings to point to Zope 4 instead of 2.14. diff --git a/setup.py b/setup.py index dae778b2..b9885ba1 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ from setuptools import setup, find_packages setup(name='Products.ZCatalog', - version = '2.13.23dev', + version = '3.0dev', url='http://pypi.python.org/pypi/Products.ZCatalog', license='ZPL 2.1', description="Zope 2's indexing and search solution.", diff --git a/src/Products/PluginIndexes/FieldIndex/FieldIndex.py b/src/Products/PluginIndexes/FieldIndex/FieldIndex.py index 011fb9e0..8fb545ab 100644 --- a/src/Products/PluginIndexes/FieldIndex/FieldIndex.py +++ b/src/Products/PluginIndexes/FieldIndex/FieldIndex.py @@ -29,7 +29,7 @@ class FieldIndex(UnIndex): {'label': 'Browse', 'action': 'manage_browse'}, ) - query_options = ["query","range"] + query_options = ["query", "range", "not"] manage = manage_main = DTMLFile('dtml/manageFieldIndex', globals()) manage_main._setName('manage_main') diff --git a/src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py b/src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py index 21f6d822..5aa35ff8 100644 --- a/src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py +++ b/src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py @@ -37,8 +37,6 @@ class FieldIndexTests(unittest.TestCase): """ def setUp( self ): - """ - """ self._index = FieldIndex( 'foo' ) self._marker = [] self._values = [ ( 0, Dummy( 'a' ) ) @@ -57,23 +55,30 @@ def setUp( self ): keys = self._forward.get( v, [] ) self._forward[v] = keys - self._noop_req = { 'bar': 123 } - self._request = { 'foo': 'abce' } - self._min_req = { 'foo': {'query': 'abc' - , 'range': 'min'} - } - self._max_req = { 'foo': {'query': 'abc' - , 'range': 'max' } - } - self._range_req = { 'foo': {'query': ( 'abc', 'abcd' ) - , 'range': 'min:max' } - } - self._zero_req = { 'foo': 0 } - self._none_req = { 'foo': None } - - def tearDown( self ): - """ - """ + self._noop_req = {'bar': 123} + self._request = {'foo': 'abce'} + self._min_req = {'foo': + {'query': 'abc', 'range': 'min'}} + self._min_req_n = {'foo': + {'query': 'abc', 'range': 'min', 'not': 'abca'}} + self._max_req = {'foo': + {'query': 'abc', 'range': 'max'}} + self._max_req_n = {'foo': + {'query': 'abc', 'range': 'max', 'not': ['a', 'b', None, 0]}} + self._range_req = {'foo': + {'query': ( 'abc', 'abcd' ), 'range': 'min:max'}} + self._range_ren = {'foo': + {'query': ( 'abc', 'abcd' ), 'range': 'min:max', 'not': 'abcd'}} + self._range_non = {'foo': + {'query': ( 'a', 'aa' ), 'range': 'min:max', 'not': 'a'}} + self._zero_req = {'foo': 0 } + self._none_req = {'foo': None } + self._not_1 = {'foo': {'query': 'a', 'not': 'a'}} + self._not_2 = {'foo': {'query': ['a', 'ab'], 'not': 'a'}} + self._not_3 = {'foo': {'not': 'a'}} + self._not_4 = {'foo': {'not': [0, None]}} + self._not_5 = {'foo': {'not': ['a', 'b']}} + self._not_6 = {'foo': 'a', 'bar': {'query': 123, 'not': 1}} def _populateIndex( self ): for k, v in self._values: @@ -115,11 +120,15 @@ def testEmpty( self ): assert not self._index.hasUniqueValuesFor( 'bar' ) assert len( self._index.uniqueValues( 'foo' ) ) == 0 - assert self._index._apply_index( self._noop_req ) is None - self._checkApply( self._request, [] ) - self._checkApply( self._min_req, [] ) - self._checkApply( self._max_req, [] ) - self._checkApply( self._range_req, [] ) + assert self._index._apply_index(self._noop_req) is None + self._checkApply(self._request, []) + self._checkApply(self._min_req, []) + self._checkApply(self._min_req_n, []) + self._checkApply(self._max_req, []) + self._checkApply(self._max_req_n, []) + self._checkApply(self._range_req, []) + self._checkApply(self._range_ren, []) + self._checkApply(self._range_non, []) def testPopulated( self ): """ Test a populated FieldIndex """ @@ -142,10 +151,21 @@ def testPopulated( self ): assert self._index._apply_index( self._noop_req ) is None - self._checkApply( self._request, values[ -4:-2 ] ) - self._checkApply( self._min_req, values[ 2:-2 ] ) - self._checkApply( self._max_req, values[ :3 ] + values[ -2: ] ) - self._checkApply( self._range_req, values[ 2:5 ] ) + self._checkApply(self._request, values[-4:-2]) + self._checkApply(self._min_req, values[2:-2]) + self._checkApply(self._min_req_n, values[2:3] + values[4:-2]) + self._checkApply(self._max_req, values[:3] + values[-2:]) + self._checkApply(self._max_req_n, values[1:3]) + self._checkApply(self._range_req, values[2:5]) + self._checkApply(self._range_ren, values[2:4]) + self._checkApply(self._range_non, []) + + self._checkApply(self._not_1, []) + self._checkApply(self._not_2, values[1:2]) + self._checkApply(self._not_3, values[1:]) + self._checkApply(self._not_4, values[:7]) + self._checkApply(self._not_5, values[1:]) + self._checkApply(self._not_6, values[0:1]) def testZero( self ): """ Make sure 0 gets indexed """ diff --git a/src/Products/PluginIndexes/common/UnIndex.py b/src/Products/PluginIndexes/common/UnIndex.py index 720c766a..bc1fb041 100644 --- a/src/Products/PluginIndexes/common/UnIndex.py +++ b/src/Products/PluginIndexes/common/UnIndex.py @@ -17,6 +17,7 @@ from logging import getLogger import sys +from BTrees.IIBTree import difference from BTrees.IIBTree import intersection from BTrees.IIBTree import IITreeSet from BTrees.IIBTree import IISet @@ -293,6 +294,18 @@ def unindex_object(self, documentId): LOG.debug('Attempt to unindex nonexistent document' ' with id %s' % documentId,exc_info=True) + def _apply_not(self, not_parm, resultset=None): + index = self._index + setlist = [] + for k in not_parm: + s = index.get(k, None) + if s is None: + continue + elif isinstance(s, int): + s = IISet((s, )) + setlist.append(s) + return multiunion(setlist) + def _apply_index(self, request, resultset=None): """Apply the index to query parameters given in the request arg. @@ -336,6 +349,13 @@ def _apply_index(self, request, resultset=None): r = None opr = None + # not / exclude parameter + not_parm = record.get('not', None) + if not record.keys and not_parm: + # we have only a 'not' query + record.keys = [k for k in index.keys() if k not in not_parm] + not_parm = None + # experimental code for specifing the operator operator = record.get('operator',self.useOperator) if not operator in self.operators : @@ -371,6 +391,9 @@ def _apply_index(self, request, resultset=None): result = setlist[0] if isinstance(result, int): result = IISet((result,)) + if not_parm: + exclude = self._apply_not(not_parm, resultset) + result = difference(result, exclude) return result, (self.id,) if operator == 'or': @@ -417,6 +440,9 @@ def _apply_index(self, request, resultset=None): result = setlist[0] if isinstance(result, int): result = IISet((result,)) + if not_parm: + exclude = self._apply_not(not_parm, resultset) + result = difference(result, exclude) return result, (self.id,) if operator == 'or': @@ -442,8 +468,10 @@ def _apply_index(self, request, resultset=None): r = IISet((r, )) if r is None: return IISet(), (self.id,) - else: - return r, (self.id,) + if not_parm: + exclude = self._apply_not(not_parm, resultset) + r = difference(r, exclude) + return r, (self.id,) def hasUniqueValuesFor(self, name): """has unique values for column name""" diff --git a/src/Products/PluginIndexes/common/tests/test_util.py b/src/Products/PluginIndexes/common/tests/test_util.py index d9df2303..1709dcc2 100644 --- a/src/Products/PluginIndexes/common/tests/test_util.py +++ b/src/Products/PluginIndexes/common/tests/test_util.py @@ -53,6 +53,30 @@ def test_get_string(self): self.assertEqual(parser.get('level'), 0) self.assertEqual(parser.get('operator'), 'and') + def test_get_not_dict(self): + request = {'path': {'query': 'foo', 'not': 'bar'}} + parser = self._makeOne(request, 'path', ('query', 'not')) + self.assertEqual(parser.get('keys'), ['foo']) + self.assertEqual(parser.get('not'), ['bar']) + + def test_get_not_dict_list(self): + request = {'path': {'query': 'foo', 'not': ['bar', 'baz']}} + parser = self._makeOne(request, 'path', ('query', 'not')) + self.assertEqual(parser.get('keys'), ['foo']) + self.assertEqual(parser.get('not'), ['bar', 'baz']) + + def test_get_not_string(self): + request = {'path': 'foo', 'path_not': 'bar'} + parser = self._makeOne(request, 'path', ('query', 'not')) + self.assertEqual(parser.get('keys'), ['foo']) + self.assertEqual(parser.get('not'), ['bar']) + + def test_get_not_string_list(self): + request = {'path': 'foo', 'path_not': ['bar', 'baz']} + parser = self._makeOne(request, 'path', ('query', 'not')) + self.assertEqual(parser.get('keys'), ['foo']) + self.assertEqual(parser.get('not'), ['bar', 'baz']) + def test_suite(): suite = unittest.TestSuite() diff --git a/src/Products/PluginIndexes/common/util.py b/src/Products/PluginIndexes/common/util.py index 6b973c6a..58dd67d7 100644 --- a/src/Products/PluginIndexes/common/util.py +++ b/src/Products/PluginIndexes/common/util.py @@ -118,6 +118,11 @@ def __init__(self, request, iid, options=[]): setattr(self, op, request[field]) self.keys = keys + not_value = getattr(self, 'not', None) + if not_value is not None: + if isinstance(not_value, basestring): + not_value = [not_value] + setattr(self, 'not', not_value) def get(self, k, default_v=None): if hasattr(self, k):