-
Notifications
You must be signed in to change notification settings - Fork 22
/
KeywordIndex.py
213 lines (174 loc) · 7.7 KB
/
KeywordIndex.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import sys
from logging import getLogger
from BTrees.OOBTree import difference
from BTrees.OOBTree import OOSet
from App.special_dtml import DTMLFile
from zope.interface import implementer
from Products.PluginIndexes.unindex import UnIndex
from Products.PluginIndexes.interfaces import (
IIndexingMissingValue,
missing,
IIndexingEmptyValue,
empty,
)
_marker = []
LOG = getLogger('Zope.KeywordIndex')
try:
basestring
except NameError:
# Python 3 compatibility
basestring = (bytes, str)
@implementer(IIndexingMissingValue, IIndexingEmptyValue)
class KeywordIndex(UnIndex):
"""Like an UnIndex only it indexes sequences of items.
Searches match any keyword.
This should have an _apply_index that returns a relevance score
"""
meta_type = 'KeywordIndex'
query_options = ('query', 'range', 'not', 'operator')
special_values = {TypeError: missing,
AttributeError: missing,
None: missing,
(): empty}
manage_options = (
{'label': 'Settings', 'action': 'manage_main'},
{'label': 'Browse', 'action': 'manage_browse'},
)
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index an object 'obj' with integer id 'i'
Ideally, we've been passed a sequence of some sort that we
can iterate over. If however, we haven't, we should do something
useful with the results. In the case of a string, this means
indexing the entire string as a keyword."""
# First we need to see if there's anything interesting to look at
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
newKeywords = self.get_object_datum(obj, attr)
oldKeywords = self._unindex.get(documentId, _marker)
if oldKeywords is _marker:
# we've got a new document, let's not futz around.
if newKeywords in (missing, empty):
self.insertSpecialIndexEntry(newKeywords, documentId)
else:
newKeywords = self.index_objectKeywords(documentId,
newKeywords)
# TODO: What do we do if none of the keywords
# is indexable?
if not newKeywords:
return 0
else:
# we have an existing entry for this document, and we need
# to figure out if any of the keywords have actually changed
if oldKeywords in (missing, empty):
self.removeSpecialIndexEntry(oldKeywords, documentId)
oldSet = OOSet()
else:
if not isinstance(oldKeywords, OOSet):
oldKeywords = OOSet(oldKeywords)
oldSet = oldKeywords
if newKeywords in (missing, empty):
self.insertSpecialIndexEntry(newKeywords, documentId)
newSet = OOSet()
else:
newSet = newKeywords = OOSet(newKeywords)
fdiff = difference(oldSet, newSet)
rdiff = difference(newSet, oldSet)
if fdiff or rdiff:
# if we've got forward or reverse changes
if fdiff:
self.unindex_objectKeywords(documentId, fdiff)
if rdiff:
newKeywords = self.index_objectKeywords(documentId,
rdiff)
# TODO: What do we do if none of the keywords
# is indexable?
if not newKeywords:
return 0
self._unindex[documentId] = newKeywords
return 1
def map_value(self, value):
value = super(KeywordIndex, self).map_value(value)
if value is not missing:
# at this place, *value* is expected to be a sequence
if isinstance(value, basestring):
value = OOSet((value,))
if not value and self.providesSpecialIndex(empty):
value = empty
else:
value = OOSet(value)
return value
def index_objectKeywords(self, documentId, keywords):
""" carefully index keywords of object with integer id 'documentId'
"""
indexed_keys = OOSet()
for kw in keywords:
try:
self.insertForwardIndexEntry(kw, documentId)
indexed_keys.insert(kw)
except TypeError:
# key is not valid for this Btree so we have to
# log and ignore keyword not indexable
LOG.error('%(context)s: Unable to insert forward '
'index entry for document with id '
'%(doc_id)s and keyword %(kw)r '
'for index %{index}r.', dict(
context=self.__class__.__name__,
kw=kw,
doc_id=documentId,
index=self.id))
return indexed_keys
def unindex_objectKeywords(self, documentId, keywords):
""" carefully unindex keywords of object with integer id 'documentId'
"""
if keywords is not None:
for kw in keywords:
self.removeForwardIndexEntry(kw, documentId)
def unindex_object(self, documentId):
""" carefully unindex the object with integer id 'documentId'"""
keywords = self._unindex.get(documentId, _marker)
if keywords is _marker:
return
self._increment_counter()
if keywords in (missing, empty):
try:
if not self.removeSpecialIndexEntry(keywords, documentId):
raise KeyError
del self._unindex[documentId]
except KeyError:
LOG.debug('%(context)s: Attempt to unindex nonexistent '
'document with id %(doc_id)s', dict(
context=self.__class__.__name__,
doc_id=documentId),
exc_info=True)
return None
self.unindex_objectKeywords(documentId, keywords)
try:
del self._unindex[documentId]
except KeyError:
LOG.debug('%(context)s: Attempt to unindex nonexistent '
'document with id %(doc_id)s', dict(
context=self.__class__.__name__,
doc_id=documentId),
exc_info=True)
manage = manage_main = DTMLFile('dtml/manageKeywordIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_addKeywordIndexForm = DTMLFile('dtml/addKeywordIndex', globals())
def manage_addKeywordIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""Add a keyword index"""
return self.manage_addIndex(id, 'KeywordIndex', extra=extra,
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)