fix pickling of dynamic classes for python3 (2/2)

We need to have a __reduce__ function.

haystack/model.py

@@ -88,8 +88,9 @@ def __create_POPO_classes(self, targetmodule):
         # python 3 fix
         # create the package module hierachy in this model
         # use python.pyObj as module class object (lazyness)
+        # need module class
         _prev = None
-        for _hierarchy_module in targetmodule.__name__.split('.'):
+        for i, _hierarchy_module in enumerate(targetmodule.__name__.split('.')):
             # create intermediate module in haystack.model
             _new = python.pyObj()
             if _prev is not None:
@@ -127,9 +128,7 @@ def __create_POPO_classes(self, targetmodule):
                 # add the structure size to the class
                 setattr(kpy, '_len_', self._ctypes.sizeof(klass))
                 _created += 1
-        log.debug(
-            'created %d POPO types in %s' %
-            (_created, targetmodule.__name__))
+        log.debug('created %d POPO types in %s' % (_created, targetmodule.__name__))
         return _created
 
     def build_python_class_clones(self, targetmodule):

haystack/outputters/python.py

@@ -244,6 +244,30 @@ def __getstate__(self):
             d['_ctype_'] = d['_ctype_'].__class__.__name__
         return d
 
+    def __reduce__(self):
+        """Explains how to rebuild this class once pickled."""
+        state = self.__dict__.copy()
+        if '_ctype_' in state:
+            state['_ctype_'] = state['_ctype_'].__class__.__name__
+        name = self.__class__.__name__
+        modulename = self.__module__
+        return (_pyObjBuilder(), # __call__
+                (modulename, name), # arg for builder
+                state
+                )
+
+
+class _pyObjBuilder:
+    """Builder of pickled instance of pyObj into properly named POPO"""
+    def __call__(self, modulename, classname):
+        # make a simple object which has no complex __init__ (this one will do)
+        obj = pyObj()
+        # class = getattr(containing_class, class_name)
+        # kpy = type('%s.%s' % (modulename, classname), (pyObj,), {})
+        kpy = type(classname, (pyObj,), {})
+        obj.__class__ = kpy
+        return obj
+
 
 def findCtypesInPyObj(memory_handler, obj):
     """ check function to help in unpickling errors correction """

haystack/search/api.py

@@ -126,16 +126,16 @@ def output_to_json(memory_handler, results):
 
 def output_to_pickle(memory_handler, results):
     """
-    Transform ctypes results in a pickled format
+    Transform ctypes results in a pickled format.
+    To load the pickled objects, you need to have haystack in your path.
+    
     :param memory_handler: IMemoryHandler
     :param results: results from the search_record
     :return:
     """
     if not isinstance(results, list):
         raise TypeError('Feed me a list of results')
     ret = output_to_python(memory_handler, results)
-    #import code
-    #code.interact(local=locals())
     return pickle.dumps(ret)
 
 

test/haystack/search/test_api.py

@@ -166,14 +166,17 @@ def test_weird_py3_bug(self):
         retstr = api.output_to_string(self.memory_handler, [(results, validated)])
         self.assertTrue(isinstance(retstr, str))
         ret = api.output_to_python(self.memory_handler, [(results, validated)])
-        model = self.memory_handler.get_model()
         # check subclass in model module
-        # from haystack import model as mm
         x = pickle.dumps(ret)
         # Python 2
         # self.assertIn(b'test.src.ctypes6_gen32.struct_usual_py', x)
         # Python 3
         self.assertIn(b'struct_usual_py', x)
+        # TODO TEST really, you should be able to load the pickled code as long as haystack.outputters.python is there
+        # and still be able to load the object graph.
+        obj = pickle.loads(x)
+        self.assertEqual(obj[0][0].root.blink, obj[0][0].root.flink)
+        self.assertEqual(obj[0][0].root.blink.flink, obj[0][0].root.flink.flink)
         return
 
     def test_refresh(self):