Merge pull request #837 from vintasoftware/fix-836-escape-colon

Closes #836: escape colon at CompoundPredicate
dedupeio · Jul 20, 2020 · 6c85d53 · 6c85d53
2 parents 5092f25 + 9862fef
commit 6c85d53
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 3 deletions.
diff --git a/dedupe/predicates.py b/dedupe/predicates.py
@@ -329,9 +329,14 @@ def __name__(self):
     def __call__(self, record, **kwargs):
         predicate_keys = [predicate(record, **kwargs)
                           for predicate in self]
-        return [u':'.join(block_key)
-                for block_key
-                in itertools.product(*predicate_keys)]
+        return [
+            u':'.join(
+                # must escape : to avoid confusion with : join separator
+                b.replace(u':', u'\\:') for b in block_key
+            )
+            for block_key
+            in itertools.product(*predicate_keys)
+        ]
 
 
 def wholeFieldPredicate(field: Any) -> Tuple[str]:

diff --git a/tests/test_predicates.py b/tests/test_predicates.py
@@ -80,5 +80,47 @@ def test_round_to_1(self):
         assert predicates.roundTo1(-22315) == (u'-20000',)
 
 
+class TestCompoundPredicate(unittest.TestCase):
+    def test_escapes_colon(self):
+        '''
+        Regression test for issue #836
+        '''
+        predicate_1 = predicates.SimplePredicate(
+            predicates.commonSetElementPredicate, 'col_1')
+        predicate_2 = predicates.SimplePredicate(
+            predicates.commonSetElementPredicate, 'col_2')
+        record = {
+            'col_1': ['foo:', 'foo'],
+            'col_2': [':bar', 'bar']
+        }
+
+        block_val = predicates.CompoundPredicate([
+            predicate_1,
+            predicate_2
+        ])(record)
+        assert len(set(block_val)) == 4
+        assert block_val == ['foo\\::\\:bar', 'foo\\::bar', 'foo:\\:bar', 'foo:bar']
+
+    def test_escapes_escaped_colon(self):
+        '''
+        Regression test for issue #836
+        '''
+        predicate_1 = predicates.SimplePredicate(
+            predicates.commonSetElementPredicate, 'col_1')
+        predicate_2 = predicates.SimplePredicate(
+            predicates.commonSetElementPredicate, 'col_2')
+        record = {
+            'col_1': ['foo\\:', 'foo'],
+            'col_2': ['\\:bar', 'bar']
+        }
+
+        block_val = predicates.CompoundPredicate([
+            predicate_1,
+            predicate_2
+        ])(record)
+        assert len(set(block_val)) == 4
+        assert block_val == ['foo\\\\::\\\\:bar', 'foo\\\\::bar', 'foo:\\\\:bar', 'foo:bar']
+
+
 if __name__ == '__main__':
     unittest.main()