Skip to content
This repository has been archived by the owner on May 13, 2020. It is now read-only.

Commit

Permalink
Comment the special status of wid 0.
Browse files Browse the repository at this point in the history
globToWordIds():  This was building a list of words and then throwing
it away without referencing it.  Deleted the code.
  • Loading branch information
Tim Peters committed May 17, 2002
1 parent 5ff679e commit 48deb36
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions Lexicon.py
Expand Up @@ -26,7 +26,11 @@ class Lexicon:
def __init__(self, *pipeline):
self._wids = OIBTree() # word -> wid
self._words = IOBTree() # wid -> word
# XXX we're reserving wid 0, but that might be yagni
# wid 0 is reserved for words that aren't in the lexicon (OOV -- out
# of vocabulary). This can happen, e.g., if a query contains a word
# we never saw before, and that isn't a known stopword (or otherwise
# filtered out). Returning a special wid value for OOV words is a
# way to let clients know when an OOV word appears.
self._nextwid = 1
self._pipeline = pipeline

Expand Down Expand Up @@ -78,12 +82,10 @@ def globToWordIds(self, pattern):
assert prefix and not prefix.endswith("*")
keys = self._wids.keys(prefix) # Keys starting at prefix
wids = []
words = []
for key in keys:
if not key.startswith(prefix):
break
wids.append(self._wids[key])
words.append(key)
return wids

def _getWordIdCreate(self, word):
Expand Down

0 comments on commit 48deb36

Please sign in to comment.