/
DateRangeIndex.py
384 lines (312 loc) · 13.4 KB
/
DateRangeIndex.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import os
from AccessControl.class_init import InitializeClass
from AccessControl.Permissions import manage_zcatalog_indexes
from AccessControl.Permissions import view
from AccessControl.SecurityInfo import ClassSecurityInfo
from App.Common import package_home
from App.special_dtml import DTMLFile
from BTrees.IIBTree import IITreeSet
from BTrees.IIBTree import difference
from BTrees.IIBTree import intersection
from BTrees.IIBTree import multiunion
from BTrees.IOBTree import IOBTree
from BTrees.Length import Length
from zope.interface import implementer
from Products.PluginIndexes.interfaces import IDateRangeIndex
from Products.PluginIndexes.unindex import UnIndex
from Products.PluginIndexes.util import datetime_to_minutes
from Products.PluginIndexes.util import safe_callable
from Products.ZCatalog.query import IndexQuery
_dtmldir = os.path.join(package_home(globals()), 'dtml')
MAX32 = int(2 ** 31 - 1)
@implementer(IDateRangeIndex)
class DateRangeIndex(UnIndex):
"""Index for date ranges, such as the "effective-expiration" range in CMF.
Any object may return None for either the start or the end date: for the
start date, this should be the logical equivalent of "since the beginning
of time"; for the end date, "until the end of time".
Therefore, divide the space of indexed objects into four containers:
- Objects which always match (i.e., they returned None for both);
- Objects which match after a given time (i.e., they returned None for the
end date);
- Objects which match until a given time (i.e., they returned None for the
start date);
- Objects which match only during a specific interval.
"""
security = ClassSecurityInfo()
meta_type = 'DateRangeIndex'
query_options = ('query', )
manage_options = ({'label': 'Properties',
'action': 'manage_indexProperties'},
)
since_field = until_field = None
# int(DateTime('1000/1/1 0:00 GMT-12').millis() / 1000 / 60)
floor_value = -510162480
# int(DateTime('2499/12/31 0:00 GMT+12').millis() / 1000 / 60)
ceiling_value = 278751600
# precision of indexed time interval in minutes
precision_value = 1
def __init__(self, id, since_field=None, until_field=None,
caller=None, extra=None, floor_value=None,
ceiling_value=None, precision_value=None):
if extra:
since_field = extra.since_field
until_field = extra.until_field
floor_value = getattr(extra, 'floor_value', None)
ceiling_value = getattr(extra, 'ceiling_value', None)
precision_value = getattr(extra, 'precision_value', None)
self._setId(id)
self._edit(since_field, until_field, floor_value,
ceiling_value, precision_value)
self.clear()
@security.protected(view)
def getSinceField(self):
"""Get the name of the attribute indexed as start date.
"""
return self._since_field
@security.protected(view)
def getUntilField(self):
"""Get the name of the attribute indexed as end date.
"""
return self._until_field
@security.protected(view)
def getFloorValue(self):
""" """
return self.floor_value
@security.protected(view)
def getCeilingValue(self):
""" """
return self.ceiling_value
@security.protected(view)
def getPrecisionValue(self):
""" """
return self.precision_value
manage_indexProperties = DTMLFile('manageDateRangeIndex', _dtmldir)
@security.protected(manage_zcatalog_indexes)
def manage_edit(self, since_field, until_field, floor_value,
ceiling_value, precision_value, REQUEST):
""" """
self._edit(since_field, until_field, floor_value, ceiling_value,
precision_value)
REQUEST['RESPONSE'].redirect('{0}/manage_main'
'?manage_tabs_message=Updated'.format(
REQUEST.get('URL2')))
@security.private
def _edit(self, since_field, until_field, floor_value=None,
ceiling_value=None, precision_value=None):
"""Update the fields used to compute the range.
"""
self._since_field = since_field
self._until_field = until_field
if floor_value not in (None, ''):
self.floor_value = int(floor_value)
if ceiling_value not in (None, ''):
self.ceiling_value = int(ceiling_value)
if precision_value not in (None, ''):
self.precision_value = int(precision_value)
@security.protected(manage_zcatalog_indexes)
def clear(self):
"""Start over fresh."""
self._always = IITreeSet()
self._since_only = IOBTree()
self._until_only = IOBTree()
self._since = IOBTree()
self._until = IOBTree()
self._unindex = IOBTree() # 'datum' will be a tuple of date ints
self._length = Length()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def getEntryForObject(self, documentId, default=None):
"""Get all information contained for the specific object
identified by 'documentId'. Return 'default' if not found.
"""
return self._unindex.get(documentId, default)
def index_object(self, documentId, obj, threshold=None):
"""Index an object:
- 'documentId' is the integer ID of the document
- 'obj' is the object to be indexed
- ignore threshold
"""
if self._since_field is None:
return 0
since = getattr(obj, self._since_field, None)
if safe_callable(since):
since = since()
since = self._convertDateTime(since)
until = getattr(obj, self._until_field, None)
if safe_callable(until):
until = until()
until = self._convertDateTime(until)
datum = (since, until)
old_datum = self._unindex.get(documentId, None)
if datum == old_datum: # No change? bail out!
return 0
self._increment_counter()
if old_datum is not None:
old_since, old_until = old_datum
self._removeForwardIndexEntry(old_since, old_until, documentId)
self._insertForwardIndexEntry(since, until, documentId)
self._unindex[documentId] = datum
return 1
def unindex_object(self, documentId):
"""Remove the object corresponding to 'documentId' from the index.
"""
datum = self._unindex.get(documentId, None)
if datum is None:
return
self._increment_counter()
since, until = datum
self._removeForwardIndexEntry(since, until, documentId)
del self._unindex[documentId]
def uniqueValues(self, name=None, withLengths=0):
"""Return a sequence of unique values for 'name'.
If 'withLengths' is true, return a sequence of tuples, in
the form '(value, length)'.
"""
if name not in (self._since_field, self._until_field):
return
if name == self._since_field:
sets = (self._since, self._since_only)
else:
sets = (self._until, self._until_only)
if not withLengths:
for s in sets:
for key in s.keys():
yield key
else:
for s in sets:
for key, value in s.items():
if isinstance(value, int):
yield (key, 1)
else:
yield (key, len(value))
def getRequestCacheKey(self, record, resultset=None):
term = self._convertDateTime(record.keys[0])
tid = str(term)
# unique index identifier
iid = '_{0}_{1}_{2}'.format(self.__class__.__name__,
self.id, self.getCounter())
# record identifier
if resultset is None:
rid = '_{0}'.format(tid)
else:
rid = '_inverse_{0}'.format(tid)
return (iid, rid)
def _apply_index(self, request, resultset=None):
record = IndexQuery(request, self.id, self.query_options,
self.operators, self.useOperator)
if record.keys is None:
return None
return (self.query_index(record, resultset=resultset),
(self._since_field, self._until_field))
def query_index(self, record, resultset=None):
cache = self.getRequestCache()
if cache is not None:
cachekey = self.getRequestCacheKey(record, resultset)
cached = cache.get(cachekey, None)
if cached is not None:
if resultset is None:
return cached
else:
return difference(resultset, cached)
term = self._convertDateTime(record.keys[0])
if resultset is None:
# Aggregate sets for each bucket separately, to avoid
# large-small union penalties.
until_only = multiunion(self._until_only.values(term))
since_only = multiunion(self._since_only.values(None, term))
until = multiunion(self._until.values(term))
since = multiunion(self._since.values(None, term))
bounded = intersection(until, since)
# Merge from smallest to largest.
result = multiunion([bounded, until_only, since_only,
self._always])
if cache is not None:
cache[cachekey] = result
return result
else:
# Compute the inverse and subtract from res
until_only = multiunion(self._until_only.values(None, term - 1))
since_only = multiunion(self._since_only.values(term + 1))
until = multiunion(self._until.values(None, term - 1))
since = multiunion(self._since.values(term + 1))
result = multiunion([since, since_only, until_only, until])
if cache is not None:
cache[cachekey] = result
return difference(resultset, result)
def _insert_migrate(self, tree, key, value):
treeset = tree.get(key, None)
if treeset is None:
tree[key] = IITreeSet((value, ))
else:
if isinstance(treeset, IITreeSet):
treeset.insert(value)
elif isinstance(treeset, int):
tree[key] = IITreeSet((treeset, value))
else:
tree[key] = IITreeSet(treeset)
tree[key].insert(value)
def _insertForwardIndexEntry(self, since, until, documentId):
"""Insert 'documentId' into the appropriate set based on 'datum'.
"""
if since is None and until is None:
self._always.insert(documentId)
elif since is None:
self._insert_migrate(self._until_only, until, documentId)
elif until is None:
self._insert_migrate(self._since_only, since, documentId)
else:
self._insert_migrate(self._since, since, documentId)
self._insert_migrate(self._until, until, documentId)
def _remove_delete(self, tree, key, value):
treeset = tree.get(key, None)
if treeset is not None:
if isinstance(treeset, int):
del tree[key]
else:
treeset.remove(value)
if not treeset:
del tree[key]
def _removeForwardIndexEntry(self, since, until, documentId):
"""Remove 'documentId' from the appropriate set based on 'datum'.
"""
if since is None and until is None:
self._always.remove(documentId)
elif since is None:
self._remove_delete(self._until_only, until, documentId)
elif until is None:
self._remove_delete(self._since_only, since, documentId)
else:
self._remove_delete(self._since, since, documentId)
self._remove_delete(self._until, until, documentId)
def _convertDateTime(self, value):
value = datetime_to_minutes(value, self.precision_value)
if value is None:
return None
if (value > self.ceiling_value or value < self.floor_value):
# handle values outside our specified range
return None
return value
InitializeClass(DateRangeIndex)
manage_addDateRangeIndexForm = DTMLFile('addDateRangeIndex', _dtmldir)
def manage_addDateRangeIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""Add a date range index to the catalog, using the incredibly icky
double-indirection-which-hides-NOTHING.
"""
return self.manage_addIndex(id, 'DateRangeIndex', extra,
REQUEST, RESPONSE, URL3)