Merge branch 'master' into andbag-compositeindex

Conflicts:
	src/Products/PluginIndexes/interfaces.py

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 sudo: false
 python:
-    - 2.6
     - 2.7
 install:
     - python bootstrap.py

CHANGES.txt → CHANGES.rst

@@ -1,11 +1,41 @@
 Changelog
 =========
 
-3.1.1 (unreleased)
+4.0 (unreleased)
+----------------
+
+
+4.0a1 (2016-07-22)
 ------------------
 
-- Update to ZODB 4.0 as direct dependency.
+- Moved `Products.ZCatalog.Lazy` module to `ZTUtils.Lazy`.
+
+- Add configure.zcml with deprecatedManageAddDelete directives.
 
+3.2 (2016-07-18)
+----------------
+
+- #12: Add request cache for index results to all UnIndex subclasses.
+
+- Add dependency on `five.globalrequest`.
+
+3.1.2 (2016-07-17)
+------------------
+
+- #6, #7, #11: Run ZODB cache garbage collection during queries.
+
+- #13: Deal with threshold value of None in add/delColum.
+
+3.1.1 (2016-07-17)
+------------------
+
+- Make index-listing compatible with Zope 4.
+
+- #5: Ignore None values in UnIndex instead of raising a TypeError.
+
+- Add a new getCounter method to indices.
+
+- Update to ZODB 4.0 as direct dependency.
 
 3.1 (2014-11-02)
 ----------------

buildout.cfg

@@ -2,12 +2,10 @@
 extends = https://raw.githubusercontent.com/zopefoundation/Zope/master/versions.cfg
 
 develop = .
-parts = interpreter test test-globalrequest
+parts = interpreter test
 
 [versions]
-five.globalrequest = 1.0
 Products.ZCatalog =
-zope.globalrequest = 1.1
 
 [interpreter]
 recipe = zc.recipe.egg
@@ -17,9 +15,3 @@ eggs = Products.ZCatalog
 [test]
 recipe = zc.recipe.testrunner
 eggs = Products.ZCatalog
-
-[test-globalrequest]
-recipe = zc.recipe.testrunner
-eggs =
-    five.globalrequest
-    Products.ZCatalog

setup.cfg

@@ -2,3 +2,7 @@
 ignore =
     *.cfg
     bootstrap.py
+
+[flake8]
+ignore = N801,N802,N803,N805,N806,N812,E301
+exclude = bootstrap.py

setup.py

@@ -16,14 +16,14 @@
 
 setup(
     name='Products.ZCatalog',
-    version='3.1.1',
+    version='4.0.dev0',
     url='https://pypi.python.org/pypi/Products.ZCatalog',
     license='ZPL 2.1',
     description="Zope 2's indexing and search solution.",
     author='Zope Foundation and Contributors',
     author_email='zope-dev@zope.org',
-    long_description=open('README.txt').read() + '\n' +
-                     open('CHANGES.txt').read(),
+    long_description=(open('README.rst').read() + '\n' +
+                      open('CHANGES.rst').read()),
     packages=find_packages('src'),
     namespace_packages=['Products'],
     package_dir={'': 'src'},
@@ -35,7 +35,6 @@
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 2 :: Only",
-        "Programming Language :: Python :: 2.6",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: Implementation :: CPython",
     ],
@@ -47,15 +46,17 @@
         'DateTime',
         'DocumentTemplate',
         'ExtensionClass',
+        'five.globalrequest',
         'Missing',
         'Persistence',
         'Products.ZCTextIndex',
         'Record',
         'RestrictedPython',
         'zExceptions',
         'ZODB',
-        'Zope2',
+        'Zope2 >= 4.0.dev0',
         'zope.dottedname',
+        'zope.globalrequest',
         'zope.interface',
         'zope.schema',
         'zope.testing',

src/Products/PluginIndexes/BooleanIndex/BooleanIndex.py

@@ -62,6 +62,7 @@ def clear(self):
         self._index_value = 1
         self._unindex = IIBTree()
         self._length = BTrees.Length.Length()
+        self._counter = BTrees.Length.Length()
 
     def histogram(self):
         """Return a mapping which provides a histogram of the number of
@@ -144,10 +145,12 @@ def removeForwardIndexEntry(self, entry, documentId, check=True):
             except ConflictError:
                 raise
             except Exception:
-                LOG.exception('%s: unindex_object could not remove '
-                              'documentId %s from index %s. This '
-                              'should not happen.' % (self.__class__.__name__,
-                              str(documentId), str(self.id)))
+                LOG.exception(
+                    '%s: unindex_object could not remove documentId %s '
+                    'from index %s. This should not happen.' % (
+                        self.__class__.__name__,
+                        str(documentId),
+                        str(self.id)))
         elif check:
             length = self._length.value
             index_length = self._index_length.value
@@ -200,6 +203,8 @@ def unindex_object(self, documentId):
         if unindexRecord is _marker:
             return None
 
+        self._increment_counter()
+
         self.removeForwardIndexEntry(unindexRecord, documentId)
 
         try:
@@ -251,7 +256,8 @@ def items(self):
 
 
 def manage_addBooleanIndex(self, id, extra=None,
-                REQUEST=None, RESPONSE=None, URL3=None):
+                           REQUEST=None, RESPONSE=None, URL3=None):
     """Add a boolean index"""
-    return self.manage_addIndex(id, 'BooleanIndex', extra=extra, \
-             REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
+    return self.manage_addIndex(
+        id, 'BooleanIndex', extra=extra,
+        REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)

src/Products/PluginIndexes/BooleanIndex/tests.py

@@ -131,9 +131,11 @@ def test_index_many_false(self):
 
     def test_index_many_change(self):
         index = self._makeOne()
+
         def add(i, value):
             obj = Dummy(i, value)
             index._index_object(obj.id, obj, attr='truth')
+
         # First lets index only True values
         for i in range(0, 4):
             add(i, True)
@@ -235,3 +237,22 @@ def test_reindexation_when_index_reversed(self):
         res = index._apply_index({'truth': True})[0]
         self.assertEqual(list(index._index), [2])
         self.assertEqual(list(res), [1, 3, 4])
+
+    def test_getCounter(self):
+        index = self._makeOne()
+
+        self.assertEqual(index.getCounter(), 0)
+
+        obj = Dummy(1, True)
+        index.index_object(obj.id, obj)
+        self.assertEqual(index.getCounter(), 1)
+
+        index.unindex_object(obj.id)
+        self.assertEqual(index.getCounter(), 2)
+
+        # unknown id
+        index.unindex_object(1234)
+        self.assertEqual(index.getCounter(), 2)
+
+        index.clear()
+        self.assertEqual(index.getCounter(), 0)

src/Products/PluginIndexes/DateIndex/DateIndex.py

@@ -18,10 +18,6 @@
 
 from App.special_dtml import DTMLFile
 from BTrees.IIBTree import IIBTree
-from BTrees.IIBTree import IISet
-from BTrees.IIBTree import union
-from BTrees.IIBTree import intersection
-from BTrees.IIBTree import multiunion
 from BTrees.IOBTree import IOBTree
 from BTrees.Length import Length
 from DateTime.DateTime import DateTime
@@ -31,7 +27,6 @@
 
 from Products.PluginIndexes.common import safe_callable
 from Products.PluginIndexes.common.UnIndex import UnIndex
-from Products.PluginIndexes.common.util import parseIndexRequest
 from Products.PluginIndexes.interfaces import IDateIndex
 
 LOG = getLogger('DateIndex')
@@ -100,13 +95,14 @@ class DateIndex(UnIndex, PropertyManager):
     manage_main._setName('manage_main')
     manage_options = ({'label': 'Settings', 'action': 'manage_main'},
                       {'label': 'Browse', 'action': 'manage_browse'},
-                     ) + PropertyManager.manage_options
+                      ) + PropertyManager.manage_options
 
     def clear(self):
         """ Complete reset """
         self._index = IOBTree()
         self._unindex = IIBTree()
         self._length = Length()
+        self._counter = Length()
 
     def index_object(self, documentId, obj, threshold=None):
         """index an object, normalizing the indexed value to an integer
@@ -148,6 +144,9 @@ def index_object(self, documentId, obj, threshold=None):
 
             returnStatus = 1
 
+        if returnStatus > 0:
+            self._increment_counter()
+
         return returnStatus
 
     def _convert(self, value, default=None):
@@ -189,5 +188,5 @@ def _convert(self, value, default=None):
 
 def manage_addDateIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None):
     """Add a Date index"""
-    return self.manage_addIndex(id, 'DateIndex', extra=None, \
-                    REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
+    return self.manage_addIndex(id, 'DateIndex', extra=None,
+                                REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)