Merge 9d8aea6 into 2a21568

setup.py

@@ -65,6 +65,7 @@
         'zope.interface',
         'zope.schema',
         'zope.testing',
+        'plone.memoize',
     ],
     include_package_data=True,
     zip_safe=False,

src/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py

@@ -248,8 +248,9 @@ def getRequestCacheKey(self, record, resultset=None):
         tid = str(term)
 
         # unique index identifier
-        iid = '_%s_%s_%s' % (self.__class__.__name__,
-                             self.id, self.getCounter())
+        iid = (self.__class__.__name__,
+               self.id, self.getCounterKey())
+
         # record identifier
         if resultset is None:
             rid = '_%s' % (tid, )

src/Products/PluginIndexes/PathIndex/PathIndex.py

@@ -24,21 +24,25 @@
 from BTrees.OOBTree import OOBTree
 from BTrees.Length import Length
 from Persistence import Persistent
+from ZODB.utils import newTid
 from zope.interface import implementer
 
+
 from Products.PluginIndexes.interfaces import (
     IPathIndex,
     IQueryIndex,
     ISortIndex,
     IUniqueValueIndex,
+    IIndexCounter,
 )
 from Products.PluginIndexes.util import safe_callable
 from Products.ZCatalog.query import IndexQuery
 
 LOG = getLogger('Zope.PathIndex')
 
 
-@implementer(IPathIndex, IQueryIndex, IUniqueValueIndex, ISortIndex)
+@implementer(IPathIndex, IQueryIndex, IUniqueValueIndex,
+             ISortIndex, IIndexCounter)
 class PathIndex(Persistent, SimpleItem):
 
     """Index for paths returned by getPhysicalPath.
@@ -60,6 +64,7 @@ class PathIndex(Persistent, SimpleItem):
     operators = ('or', 'and')
     useOperator = 'or'
     query_options = ('query', 'level', 'operator')
+    _counter = None
 
     manage_options = (
         {'label': 'Settings', 'action': 'manage_main'},
@@ -128,6 +133,7 @@ def index_object(self, docid, obj, threshold=100):
         for i in range(len(comps)):
             self.insertEntry(comps[i], docid, i)
         self._unindex[docid] = path
+        self._increment_counter()
         return 1
 
     def unindex_object(self, docid):
@@ -154,6 +160,7 @@ def unindex_object(self, docid):
                           % docid)
 
         self._length.change(-1)
+        self._increment_counter()
         del self._unindex[docid]
 
     def _apply_index(self, request):
@@ -186,6 +193,26 @@ def query_index(self, record, resultset=None):
             return res
         return IISet()
 
+    def _increment_counter(self):
+        if self._counter is None:
+            self._counter = Length()
+        self._counter.change(1)
+
+    def getCounter(self):
+        """Return a counter which is increased on index changes"""
+        return self._counter is not None and self._counter() or 0
+
+    def getCounterKey(self):
+        """Returns an unique key indicating an uniqe state of the index"""
+        if self._counter is not None:
+            key = (self.getCounter(), self._counter._p_serial)
+        else:
+            # generate new serial for backward compatibility
+            # if counter is not set
+            key = (self.getCounter(), newTid(None))
+
+        return key
+
     def numObjects(self):
         """ See IPluggableIndex.
         """
@@ -203,6 +230,10 @@ def clear(self):
         self._index = OOBTree()
         self._unindex = IOBTree()
         self._length = Length(0)
+        if self._counter is None:
+            self._counter = Length(0)
+        else:
+            self._increment_counter()
 
     # IUniqueValueIndex implementation
 

src/Products/PluginIndexes/PathIndex/tests.py

@@ -533,3 +533,23 @@ def test__search_w_level_0(self):
         self.assertEqual(list(index._search('bb', 1)), [1])
         self.assertEqual(list(index._search('aa/bb', 0)), [1])
         self.assertEqual(list(index._search('aa/bb', 1)), [])
+
+    def test_getCounter(self):
+        index = self._makeOne()
+
+        self.assertEqual(index.getCounter(), 0)
+
+        doc = Dummy('/aa/bb')
+        index.index_object(1, doc)
+        self.assertEqual(index.getCounter(), 1)
+
+        index.unindex_object(1)
+        self.assertEqual(index.getCounter(), 2)
+
+        # unknown id
+        index.unindex_object(1)
+        self.assertEqual(index.getCounter(), 2)
+
+        # clear changes the index
+        index.clear()
+        self.assertEqual(index.getCounter(), 3)

src/Products/PluginIndexes/TopicIndex/FilteredSet.py

@@ -41,10 +41,7 @@ def index_object(self, documentId, obj):
         raise RuntimeError('index_object not defined')
 
     def unindex_object(self, documentId):
-        try:
-            self.ids.remove(documentId)
-        except KeyError:
-            pass
+        self.ids.remove(documentId)
 
     def getId(self):
         return self.id
@@ -75,19 +72,23 @@ class PythonFilteredSet(FilteredSetBase):
     meta_type = 'PythonFilteredSet'
 
     def index_object(self, documentId, o):
+        res = 0
         try:
             if RestrictionCapableEval(self.expr).eval({'o': o}):
                 self.ids.insert(documentId)
+                res = 1
             else:
                 try:
                     self.ids.remove(documentId)
+                    res = 1
                 except KeyError:
                     pass
         except ConflictError:
             raise
         except Exception:
             LOG.warn('eval() failed Object: %s, expr: %s' %
                      (o.getId(), self.expr), exc_info=sys.exc_info())
+        return res
 
 
 def factory(f_id, f_type, expr):

src/Products/PluginIndexes/TopicIndex/TopicIndex.py

@@ -18,13 +18,16 @@
 from BTrees.IIBTree import intersection
 from BTrees.IIBTree import union
 from BTrees.OOBTree import OOBTree
+from BTrees.Length import Length
 from OFS.SimpleItem import SimpleItem
 from Persistence import Persistent
+from ZODB.utils import newTid
 from zope.interface import implementer
 
 from Products.PluginIndexes.interfaces import (
     IQueryIndex,
     ITopicIndex,
+    IIndexCounter,
 )
 from Products.PluginIndexes.TopicIndex.FilteredSet import factory
 from Products.ZCatalog.query import IndexQuery
@@ -33,7 +36,7 @@
 LOG = getLogger('Zope.TopicIndex')
 
 
-@implementer(ITopicIndex, IQueryIndex)
+@implementer(ITopicIndex, IQueryIndex, IIndexCounter)
 class TopicIndex(Persistent, SimpleItem):
     """A TopicIndex maintains a set of FilteredSet objects.
 
@@ -43,6 +46,7 @@ class TopicIndex(Persistent, SimpleItem):
 
     meta_type = "TopicIndex"
     query_options = ('query', 'operator')
+    _counter = None
 
     manage_options = (
         {'label': 'FilteredSets', 'action': 'manage_main'},
@@ -51,6 +55,7 @@ class TopicIndex(Persistent, SimpleItem):
     def __init__(self, id, caller=None):
         self.id = id
         self.filteredSets = OOBTree()
+        self._counter = Length()
         self.operators = ('or', 'and')
         self.defaultOperator = 'or'
 
@@ -60,22 +65,54 @@ def getId(self):
     def clear(self):
         for fs in self.filteredSets.values():
             fs.clear()
+        self._increment_counter()
 
     def index_object(self, docid, obj, threshold=100):
         """ hook for (Z)Catalog """
+        res = 0
         for fid, filteredSet in self.filteredSets.items():
-            filteredSet.index_object(docid, obj)
-        return 1
+            res += filteredSet.index_object(docid, obj)
+
+        if res > 0:
+            self._increment_counter()
+            return 1
+
+        return 0
 
     def unindex_object(self, docid):
         """ hook for (Z)Catalog """
+        res = 0
         for fs in self.filteredSets.values():
             try:
                 fs.unindex_object(docid)
+                res += 1
             except KeyError:
                 LOG.debug('Attempt to unindex document'
                           ' with id %s failed' % docid)
-        return 1
+        if res > 0:
+            self._increment_counter()
+            return 1
+
+        return 0
+
+    def _increment_counter(self):
+        if self._counter is None:
+            self._counter = Length()
+        self._counter.change(1)
+
+    def getCounter(self):
+        """Return a counter which is increased on index changes"""
+        return self._counter is not None and self._counter() or 0
+
+    def getCounterKey(self):
+        """Returns an unique key indicating an uniqe state of the index"""
+        if self._counter is not None:
+            key = (self.getCounter(), self._counter._p_serial)
+        else:
+            # counter is not set, generate new serial
+            key = (self.getCounter(), newTid(None))
+
+        return key
 
     def numObjects(self):
         """Return the number of indexed objects."""

src/Products/PluginIndexes/TopicIndex/tests.py

@@ -92,3 +92,23 @@ def testRemoval(self):
         self.TI.index_object(1, Obj('1', 'doc2'))
         self._searchOr('doc1', [2])
         self._searchOr('doc2', [1, 3, 4])
+
+    def test_getCounter(self):
+        index = self.TI
+        index._counter.set(0)
+
+        self.assertEqual(index.getCounter(), 0)
+
+        index.index_object(1, Obj(1, 'doc1'))
+        self.assertEqual(index.getCounter(), 1)
+
+        index.unindex_object(1)
+        self.assertEqual(index.getCounter(), 2)
+
+        # unknown id
+        index.unindex_object(1)
+        self.assertEqual(index.getCounter(), 2)
+
+        # clear changes the index
+        index.clear()
+        self.assertEqual(index.getCounter(), 3)

src/Products/PluginIndexes/interfaces.py

@@ -280,3 +280,13 @@ def make_query(query):
 
     def getIndexNames():
         """ returns index names that are optimized by index """
+
+
+class IIndexCounter(Interface):
+    """ Invalidation helper API for pluggable indexes"""
+
+    def getCounter():
+        """Return a counter which is increased on index changes"""
+
+    def getCounterKey():
+        """Returns an unique key indicating an uniqe state of the index"""

src/Products/PluginIndexes/tests/test_unindex.py

@@ -16,6 +16,9 @@
 from BTrees.IIBTree import difference
 from OFS.SimpleItem import SimpleItem
 from Testing.makerequest import makerequest
+from Acquisition import aq_base
+import ZODB
+import transaction
 
 
 class TestUnIndex(unittest.TestCase):
@@ -164,3 +167,51 @@ class Dummy(object):
         # clear changes the index
         index.clear()
         self.assertEqual(index.getCounter(), 3)
+
+    def test_getCounterKey(self):
+        index = self._makeOne('counter')
+
+        class Dummy(object):
+            def __init__(self, obj_id):
+                self.id = obj_id
+                self.counter = 'counter_{0}'.format(obj_id)
+
+        # check counter key of initialized empty index
+        # counter key is a tuple of the counts of index operations and
+        # transaction id (tid) of the counter variable
+
+        key0 = index.getCounterKey()
+        self.assertEqual(key0, (0, b'\x00\x00\x00\x00\x00\x00\x00\x00'))
+
+        connection = ZODB.connection(None)
+        connection.add(aq_base(index))
+
+        # first object to index
+        obj = Dummy(1)
+        index.index_object(obj.id, obj)
+
+        # indexing of object changes counter but not the tid
+        key1 = index.getCounterKey()
+        self.assertEqual(key1, (1, b'\x00\x00\x00\x00\x00\x00\x00\x00'))
+
+        transaction.commit()
+
+        # commit changes the tid but not the counter
+        key2 = index.getCounterKey()
+        self.assertEqual(key2[0], key1[0])
+        self.assertFalse(key2[1] == key1[1])
+
+        # second object to index
+        obj = Dummy(2)
+        index.index_object(obj.id, obj)
+
+        # indexing of object changes counter but not the tid
+        key3 = index.getCounterKey()
+        self.assertFalse(key3[0] == key2[0])
+        self.assertEqual(key3[1], key2[1])
+
+        transaction.abort()
+
+        # abort resets counter key to previos state
+        key4 = index.getCounterKey()
+        self.assertEqual(key4, key2)