Merge pull request #344 from Fideln8/master

Update tfidf.py

dedupe/tfidf.py

@@ -3,6 +3,8 @@
 import logging
 from index import CanopyIndex
 import math
+import collections
+import itertools
 
 logger = logging.getLogger(__name__)
 
@@ -13,15 +15,13 @@ def __init__(self, field, stop_words=[]) :
         self._index = CanopyIndex(stop_words)
 
         self._i_to_id = {}
-        self._parseTerms = self._index.lexicon.parseTerms
-
-    def _hash32(self, x) :
-        i = hash(x)
-        return int(math.copysign(i % (2**31), i))
 
+        self._id_to_i = collections.defaultdict(itertools.count(-2**31).next)
+
+        self._parseTerms = self._index.lexicon.parseTerms
 
     def index(self, record_id, doc) :
-        i = self._hash32(record_id)
+        i = self._id_to_i[record_id]
         self._i_to_id[i] = record_id
 
         try :
@@ -31,7 +31,7 @@ def index(self, record_id, doc) :
             raise
 
     def unindex(self, record_id) :
-        i = self._hash32(record_id)
+        i = self._id_to_i.pop(record_id)
         del self._i_to_id[i]
         self._index.unindex_doc(i)