make sure that metaphoneToken returns unique bkv

dedupe/predicates.py

@@ -221,10 +221,10 @@ def doubleMetaphone(field) :
     return [metaphone for metaphone in doublemetaphone(field) if metaphone]
 
 def metaphoneToken(field) :
-    return [metaphone_token for metaphone_token 
-            in itertools.chain(*(doublemetaphone(token) 
-                                 for token in set(field.split())))
-            if metaphone_token]
+    return set([metaphone_token for metaphone_token 
+                in itertools.chain(*(doublemetaphone(token) 
+                                     for token in set(field.split())))
+                if metaphone_token])
 
 def existsPredicate(field) :
     try :

setup.py

@@ -34,7 +34,7 @@
 setup(
     name='dedupe',
     url='https://github.com/datamade/dedupe',
-    version='0.7.7.1.0',
+    version='0.7.7.1.1',
     description='A python library for accurate and scaleable data deduplication and entity-resolution',
     packages=['dedupe', 'dedupe.variables'],
     ext_modules=[Extension('dedupe.cpredicates', ['src/cpredicates.c'])],

tests/test_predicates.py

@@ -1,6 +1,11 @@
 import unittest
 from dedupe import predicates
 
+class TestMetaphone(unittest.TestCase):
+    def test_metaphone_token(self) :
+        block_val = predicates.metaphoneToken('9301 S. State St. ')
+        assert block_val == set([u'STT', u'SS'])
+
 class TestWholeSet(unittest.TestCase):
     def setUp(self):
         self.s1 = set(['red', 'blue', 'green'])