when pickling variables and predicates remove unnecessary bits

this dramatically reduces the size of the settings file
dedupeio · Feb 4, 2022 · 0a2d785 · 0a2d785 · az0 · Feb 14, 2022
1 parent 9ba44fa
commit 0a2d785
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 0 deletions.
diff --git a/dedupe/predicates.py b/dedupe/predicates.py
@@ -126,6 +126,9 @@ def __init__(self, threshold, field):
     def __getstate__(self):
         odict = self.__dict__.copy()
         odict['index'] = None
+        odict['_cache'] = {}
+        if 'canopy' in odict:
+            odict['canopy'] = {}
         return odict
 
     def __setstate__(self, d):

diff --git a/dedupe/variables/base.py b/dedupe/variables/base.py
@@ -28,6 +28,12 @@ def __init__(self, definition):
         else:
             self.has_missing = False
 
+    def __getstate__(self):
+        odict = self.__dict__.copy()
+        odict['predicates'] = None
+
+        return odict
+
 
 class DerivedType(Variable):
     type = "Derived"