From bdcd3f8255322d2c13776140f1514eaaa97d6b51 Mon Sep 17 00:00:00 2001 From: mamico Date: Thu, 21 Jul 2022 13:23:47 +0200 Subject: [PATCH 1/6] plan as daterangeindex --- .github/workflows/tests.yml | 60 +++++++ .gitignore | 10 +- buildout.cfg | 23 +++ buildout4.cfg | 7 + setup.py | 8 +- src/Products/DateRecurringIndex/index.py | 28 ++-- src/Products/DateRecurringIndex/testing.py | 3 +- src/Products/DateRecurringIndex/tests.py | 178 ++++++++++++--------- tox.ini | 88 ++++++++++ 9 files changed, 310 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 buildout.cfg create mode 100644 buildout4.cfg create mode 100644 tox.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..c700792 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,60 @@ +# Generated from: +# https://github.com/zopefoundation/meta/tree/master/config/zope-product +name: tests + +on: + push: + pull_request: + schedule: + - cron: '0 12 * * 0' # run once a week on Sunday + # Allow to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + build: + strategy: + # We want to see all failures: + fail-fast: false + matrix: + os: + - ubuntu + config: + # [Python version, tox env] + - ["3.9", "lint"] + - ["2.7", "py27"] + - ["3.6", "py36"] + - ["3.7", "py37"] + - ["3.8", "py38"] + - ["3.9", "py39"] + - ["3.10", "py310"] + - ["3.9", "coverage"] + + runs-on: ${{ matrix.os }}-latest + name: ${{ matrix.config[1] }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.config[0] }} + - name: Pip cache + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.config[0] }}-${{ hashFiles('setup.*', 'tox.ini') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.config[0] }}- + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox + - name: Test + run: tox -e ${{ matrix.config[1] }} + - name: Coverage + if: matrix.config[1] == 'coverage' + run: | + pip install coveralls coverage-python-version + coveralls --service=github + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index fe9b528..f4a44db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ *.egg-info *.mo *.py? -.* dist/ -!.gitattributes -!.gitignore +.coverage +.coverage.* +.eggs/ +.installed.cfg +.mr.developer.cfg +.tox/ +.vscode/ diff --git a/buildout.cfg b/buildout.cfg new file mode 100644 index 0000000..0179d68 --- /dev/null +++ b/buildout.cfg @@ -0,0 +1,23 @@ +[buildout] +extends = + https://zopefoundation.github.io/Zope/releases/master/versions.cfg +develop = . +parts = + interpreter + test + +[versions] +Products.DateRecurringIndex = +RestrictedPython = >= 5.1 + +[interpreter] +recipe = zc.recipe.egg +interpreter = py +eggs = + Products.DateRecurringIndex + tox + +[test] +recipe = zc.recipe.testrunner +eggs = + Products.DateRecurringIndex diff --git a/buildout4.cfg b/buildout4.cfg new file mode 100644 index 0000000..58835f2 --- /dev/null +++ b/buildout4.cfg @@ -0,0 +1,7 @@ +[buildout] +extends = + buildout.cfg + http://zopefoundation.github.io/Zope/releases/4.x/versions.cfg + +[versions] +Products.DateRecurringIndex = diff --git a/setup.py b/setup.py index 13a05be..16ac9a2 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- """Installer for the bda.aaf.site package.""" -from setuptools import find_packages -from setuptools import setup - +from setuptools import find_packages, setup version = '3.0.2.dev0' short_description = "Zope 2 date index with support for recurring events." @@ -31,6 +29,9 @@ 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', ], keywords='zope zope2 index catalog date recurring', @@ -45,6 +46,7 @@ zip_safe=False, install_requires=[ 'setuptools', + 'six', 'BTrees', 'plone.event', 'Products.ZCatalog', diff --git a/src/Products/DateRecurringIndex/index.py b/src/Products/DateRecurringIndex/index.py index 88cde78..d3afbec 100644 --- a/src/Products/DateRecurringIndex/index.py +++ b/src/Products/DateRecurringIndex/index.py @@ -1,26 +1,25 @@ +# -*- coding: utf-8 -*- +from logging import getLogger + from AccessControl.class_init import InitializeClass from App.special_dtml import DTMLFile -from BTrees.IIBTree import difference -from BTrees.IIBTree import IISet -from logging import getLogger +from BTrees.IIBTree import IISet, difference from OFS.PropertyManager import PropertyManager from plone.event.recurrence import recurrence_sequence_ical -from plone.event.utils import dt2int -from plone.event.utils import pydt +from plone.event.utils import dt2int, pydt from Products.PageTemplates.PageTemplateFile import PageTemplateFile +from Products.PluginIndexes.interfaces import IDateRangeIndex from Products.PluginIndexes.unindex import UnIndex from Products.PluginIndexes.util import safe_callable from ZODB.POSException import ConflictError from zope.interface import implementer -from zope.interface import Interface from zope.schema import Text - LOG = getLogger('Products.DateRecurringIndex') _marker = object() -class IDateRecurringIndex(Interface): +class IDateRecurringIndex(IDateRangeIndex): attr_recurdef = Text( title=u"Attribute- or fieldname of recurrence rule definition." u"RFC2445 compatible string or timedelta." @@ -48,7 +47,6 @@ class DateRecurringIndex(UnIndex, PropertyManager): {'label': 'Browse', 'action': 'manage_browse'}, ) + PropertyManager.manage_options - def __init__(self, id, ignore_ex=None, call_methods=None, extra=None, caller=None): """ Initialize the index @@ -87,7 +85,7 @@ def index_object(self, documentId, obj, threshold=None): if not recurdef: dates = [pydt(date_attr)] else: - until = getattr(obj, self.attr_until, None) + until = getattr(obj, self.getUntilField(), None) if safe_callable(until): until = until() @@ -156,6 +154,16 @@ def _convert(self, value, default=None): """ return dt2int(value) or default + def getSinceField(self): + """Get the name of the attribute indexed as start date. + """ + return None + + def getUntilField(self): + """Get the name of the attribute indexed as end date. + """ + return self.attr_until + manage_addDRIndexForm = DTMLFile('www/addDRIndex', globals()) diff --git a/src/Products/DateRecurringIndex/testing.py b/src/Products/DateRecurringIndex/testing.py index 4975d87..2c9773a 100644 --- a/src/Products/DateRecurringIndex/testing.py +++ b/src/Products/DateRecurringIndex/testing.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from plone.testing import z2 -from plone.testing import Layer +from plone.testing import Layer, z2 class DRILayer(Layer): diff --git a/src/Products/DateRecurringIndex/tests.py b/src/Products/DateRecurringIndex/tests.py index ce4af3d..7e33728 100644 --- a/src/Products/DateRecurringIndex/tests.py +++ b/src/Products/DateRecurringIndex/tests.py @@ -1,12 +1,22 @@ # -*- coding: utf-8 -*- -import unittest import doctest +import unittest +from datetime import datetime + +import pytz +from OFS.Folder import Folder +from Products.ZCatalog.Catalog import Catalog +from Products.ZCatalog.ZCatalog import ZCatalog +from zope.testing import cleanup + +from Products.DateRecurringIndex.index import DateRecurringIndex optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS class DummyEvent(object): """some dummy with a start, delta and until to index""" + def __init__(self, id=None, start=None, recurdef=None, until=None): self.id = id self.start = start @@ -15,161 +25,175 @@ def __init__(self, id=None, start=None, recurdef=None, until=None): class DummyExtras(object): - def __init__(self, recurrence_type=None, - recurdef=None, until=None): + def __init__(self, recurrence_type=None, recurdef=None, until=None): self.recurrence_type = recurrence_type self.recurdef = recurdef self.until = until -class TestIndex(unittest.TestCase): +class TestIndex(cleanup.CleanUp, unittest.TestCase): + def setUp(self): + cleanup.CleanUp.setUp(self) def test_index(self): - """Test the index in icalendar/rfc5545 recurrence mode. - """ - + """Test the index in icalendar/rfc5545 recurrence mode.""" # Initialize the catalog with DateRecurringIndex - from Products.DateRecurringIndex.index import DateRecurringIndex - dri = DateRecurringIndex( - 'start', + "start", extra=DummyExtras( - recurrence_type='ical', - recurdef='recurdef', - until='until') + recurrence_type="ical", recurdef="recurdef", until="until" + ), ) # Index must have be the same name as dri's id - from Products.ZCatalog.Catalog import Catalog - cat = Catalog() - cat.addIndex('start', dri) - cat.addColumn('id') + cat.addIndex("start", dri) + cat.addColumn("id") # catalog needs to be contained somewhere, otherwise # aquisition-wrapping of result brains doesn't work - from OFS.Folder import Folder - portal = Folder(id='portal') + portal = Folder(id="portal") cat.__parent__ = portal # Let's define some dummy events and catalog them. - from datetime import datetime - import pytz - cet = pytz.timezone('CET') + cet = pytz.timezone("CET") # Index the same event more than once and test if index size changes. test_event = DummyEvent( - id='test_event', + id="test_event", start=datetime(2001, 1, 1), - recurdef='RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5' - ) - self.assertEqual( - cat.catalogObject(test_event, 'test_event'), - 1 + recurdef="RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5", ) + self.assertEqual(cat.catalogObject(test_event, "test_event"), 1) self.assertEqual(dri.indexSize(), 5) test_event = DummyEvent( - id='test_event', + id="test_event", start=datetime(2001, 1, 1), - recurdef='RRULE:FREQ=DAILY;INTERVAL=1;COUNT=3' - ) - self.assertEqual( - cat.catalogObject(test_event, 'test_event'), - 1 + recurdef="RRULE:FREQ=DAILY;INTERVAL=1;COUNT=3", ) + self.assertEqual(cat.catalogObject(test_event, "test_event"), 1) self.assertEqual(dri.indexSize(), 3) test_event = DummyEvent( - id='test_event', + id="test_event", start=datetime(2001, 1, 1), - recurdef='RRULE:FREQ=DAILY;INTERVAL=1;COUNT=8' - ) - self.assertEqual( - cat.catalogObject(test_event, 'test_event'), - 1 + recurdef="RRULE:FREQ=DAILY;INTERVAL=1;COUNT=8", ) + self.assertEqual(cat.catalogObject(test_event, "test_event"), 1) self.assertEqual(dri.indexSize(), 8) - cat.uncatalogObject('test_event') + cat.uncatalogObject("test_event") self.assertEqual(dri.indexSize(), 0) # Index for querying later on... nonr = DummyEvent( - id='nonr', start=datetime(2010, 10, 10, 0, 0, tzinfo=cet) - ) + id="nonr", start=datetime(2010, 10, 10, 0, 0, tzinfo=cet)) days = DummyEvent( - id='days', start=datetime(2010, 10, 10, 0, 0, tzinfo=cet), - recurdef='RRULE:FREQ=DAILY;INTERVAL=10;COUNT=5' + id="days", + start=datetime(2010, 10, 10, 0, 0, tzinfo=cet), + recurdef="RRULE:FREQ=DAILY;INTERVAL=10;COUNT=5", ) mins = DummyEvent( - id='mins', start=datetime(2010, 10, 10, 0, 0, tzinfo=cet), - recurdef='RRULE:FREQ=MINUTELY;INTERVAL=10;COUNT=5' + id="mins", + start=datetime(2010, 10, 10, 0, 0, tzinfo=cet), + recurdef="RRULE:FREQ=MINUTELY;INTERVAL=10;COUNT=5", ) dstc = DummyEvent( - id='dstc', start=datetime(2010, 10, 20, 0, 0, tzinfo=cet), - recurdef='RRULE:FREQ=HOURLY;INTERVAL=1;COUNT=7' + id="dstc", + start=datetime(2010, 10, 20, 0, 0, tzinfo=cet), + recurdef="RRULE:FREQ=HOURLY;INTERVAL=1;COUNT=7", ) - cat.catalogObject(nonr, 'nonr') - cat.catalogObject(days, 'days') - cat.catalogObject(mins, 'mins') - cat.catalogObject(dstc, 'dstc') + cat.catalogObject(nonr, "nonr") + cat.catalogObject(days, "days") + cat.catalogObject(mins, "mins") + cat.catalogObject(dstc, "dstc") # Query min one specific date query = { - 'start': { - 'query': datetime(2010, 10, 10, 0, 0, tzinfo=cet), - 'range': 'min', + "start": { + "query": datetime(2010, 10, 10, 0, 0, tzinfo=cet), + "range": "min", }, } res = cat(**query) self.assertEqual( - sorted([it.id for it in res]), - ['days', 'dstc', 'mins', 'nonr'] + sorted([it.id for it in res]), ["days", "dstc", "mins", "nonr"] ) # Query max one specific date query = { - 'start': { - 'query': datetime(2010, 10, 10, 0, 0, tzinfo=cet), - 'range': 'max', + "start": { + "query": datetime(2010, 10, 10, 0, 0, tzinfo=cet), + "range": "max", }, } res = cat(**query) self.assertEqual( sorted([it.id for it in res]), - ['days', 'mins', 'nonr'] + ["days", "mins", "nonr"] ) # Query timerange over days and dstc set query = { - 'start': { - 'query': [ + "start": { + "query": [ datetime(2010, 10, 11, 0, 0, tzinfo=cet), - datetime(2010, 11, 20, 0, 0, tzinfo=cet) + datetime(2010, 11, 20, 0, 0, tzinfo=cet), ], - 'range': 'min:max', + "range": "min:max", }, } res = cat(**query) - self.assertEqual( - sorted([brain.id for brain in res]), - ['days', 'dstc'] - ) + self.assertEqual(sorted([brain.id for brain in res]), ["days", "dstc"]) # Query timerange over mins set query = { - 'start': { - 'query': [ + "start": { + "query": [ datetime(2010, 10, 10, 0, 10, tzinfo=cet), - datetime(2010, 10, 10, 0, 40, tzinfo=cet) + datetime(2010, 10, 10, 0, 40, tzinfo=cet), ], - 'range': 'min:max', + "range": "min:max", }, } res = cat(**query) - self.assertEqual( - sorted([brain.id for brain in res]), - ['mins'] + self.assertEqual(sorted([brain.id for brain in res]), ["mins"]) + + def test_plan(self): + zcat = ZCatalog("catalog") + # Initialize the catalog with DateRecurringIndex + dri = DateRecurringIndex( + "start", + extra=DummyExtras( + recurrence_type="ical", recurdef="recurdef", until="until" + ), + ) + # Index must have be the same name as dri's id + cat = zcat._catalog + cat.addIndex("start", dri) + cat.addColumn("id") + test_event = DummyEvent( + id="test_event", + start=datetime(2001, 1, 1), + recurdef="RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5", + ) + cat.catalogObject(test_event, "test_event") + query = { + "start": { + "query": datetime(2010, 10, 10, 0, 0), + "range": "min", + }, + } + zcat.search(query) + # Wrong benchmark key + self.assertNotIn( + "(('start', \"{'query': datetime.datetime(2010, 10, 10, 0, 0), 'range': 'min'}\"),): {", # NOQA + zcat.getCatalogPlan(), + ) + # Correct benchmark key + self.assertIn( + """('start',): {""", + zcat.getCatalogPlan(), ) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f206a25 --- /dev/null +++ b/tox.ini @@ -0,0 +1,88 @@ +# Generated from: +# https://github.com/zopefoundation/meta/tree/master/config/zope-product +[tox] +minversion = 3.18 +envlist = + lint + py27 + py36 + py37 + py38 + py39 + py310 + coverage + +[testenv] +skip_install = true +deps = + zc.buildout >= 3.0.0rc3 + wheel > 0.37 +commands_pre = + py27: {envbindir}/buildout -nc {toxinidir}/buildout4.cfg buildout:directory={envdir} buildout:develop={toxinidir} install test + !py27: {envbindir}/buildout -nc {toxinidir}/buildout.cfg buildout:directory={envdir} buildout:develop={toxinidir} install test +commands = + {envbindir}/test {posargs:-cv} + +[testenv:lint] +basepython = python3 +commands_pre = + mkdir -p {toxinidir}/parts/flake8 +allowlist_externals = + mkdir +commands = + isort --check-only --diff {toxinidir}/src {toxinidir}/setup.py + - flake8 {toxinidir}/src {toxinidir}/setup.py + flake8 {toxinidir}/src {toxinidir}/setup.py + check-manifest + check-python-versions +deps = + check-manifest + check-python-versions + flake8 + isort + # Useful flake8 plugins that are Python and Plone specific: + flake8-coding + flake8-debugger + mccabe + +[testenv:isort-apply] +basepython = python3 +commands_pre = +deps = + isort +commands = + isort {toxinidir}/src {toxinidir}/setup.py [] + +[testenv:coverage] +basepython = python3 +skip_install = true +allowlist_externals = + mkdir +deps = + {[testenv]deps} + coverage + coverage-python-version +commands = + mkdir -p {toxinidir}/parts/htmlcov + coverage run {envbindir}/test {posargs:-cv} + coverage html + coverage report -m --fail-under=85 + +[coverage:run] +branch = True +plugins = coverage_python_version +source = Products.ZCatalog + +[coverage:report] +precision = 2 +exclude_lines = + pragma: no cover + pragma: nocover + except ImportError: + raise NotImplementedError + if __name__ == '__main__': + self.fail + raise AssertionError + +[coverage:html] +directory = parts/htmlcov From 4fe759a4aa4dba08d36c4d76edf636b1dc73a738 Mon Sep 17 00:00:00 2001 From: mamico Date: Thu, 21 Jul 2022 13:27:20 +0200 Subject: [PATCH 2/6] lint manifest --- MANIFEST.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 2e0911a..f1b4d12 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,7 @@ include *.rst +include buildout.cfg +include tox.ini +include buildout4.cfg recursive-include docs * recursive-include src * global-include *.mo From e47d9cc129c868babb516abe4adeb3358ae8467f Mon Sep 17 00:00:00 2001 From: mamico Date: Thu, 21 Jul 2022 13:28:55 +0200 Subject: [PATCH 3/6] coverage --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f206a25..8521b68 100644 --- a/tox.ini +++ b/tox.ini @@ -71,7 +71,7 @@ commands = [coverage:run] branch = True plugins = coverage_python_version -source = Products.ZCatalog +source = Products.DateRecurringIndex [coverage:report] precision = 2 From 7fcb60ea4941762f3a227b89c8e8d6a7d96c24ca Mon Sep 17 00:00:00 2001 From: mamico Date: Thu, 21 Jul 2022 13:34:45 +0200 Subject: [PATCH 4/6] changes --- CHANGES.rst | 3 ++- tox.ini | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 34f6422..e54c829 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,8 @@ Changelog 3.0.2 (unreleased) ------------------ -- Nothing changed yet. +- Implements IDateRangeIndex so is excluded by index that could have value in the keys of the catalog plan + [mamico] 3.0.1 (2019-10-03) diff --git a/tox.ini b/tox.ini index 8521b68..13d44e4 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ commands = mkdir -p {toxinidir}/parts/htmlcov coverage run {envbindir}/test {posargs:-cv} coverage html - coverage report -m --fail-under=85 + coverage report -m --fail-under=80 [coverage:run] branch = True From e01c69fd986cf559fb2482ee2f1ef5c8406734fe Mon Sep 17 00:00:00 2001 From: mamico Date: Thu, 21 Jul 2022 22:13:04 +0200 Subject: [PATCH 5/6] gha --- .github/workflows/tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c700792..456ca24 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,12 +1,8 @@ -# Generated from: -# https://github.com/zopefoundation/meta/tree/master/config/zope-product name: tests on: push: pull_request: - schedule: - - cron: '0 12 * * 0' # run once a week on Sunday # Allow to run this workflow manually from the Actions tab workflow_dispatch: From e4b4b2c0f976ce736b4953411fb6fd58aa376784 Mon Sep 17 00:00:00 2001 From: Mauro Amico Date: Fri, 22 Jul 2022 10:05:26 +0200 Subject: [PATCH 6/6] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e54c829..6a54c87 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changelog 3.0.2 (unreleased) ------------------ -- Implements IDateRangeIndex so is excluded by index that could have value in the keys of the catalog plan +- Implements IDateRangeIndex to exclude DateRecurringIndex by indexes with value in the keys of the catalog plan [mamico]