Merge pull request #54 from georgianpartners/issue_53_concrete_serial…

…ization

Revise smart transformer serialization (#53)

foreshadow/optimizers/param_mapping.py

@@ -76,6 +76,7 @@ def _param_mapping(pipeline, X_df, y_df):
         # One parameter is from_json (the configuration dict we calculated)
         # Other parameters are pulled from config_dict using
         # extract_config_params()
+
         explicit_params = {
             "{}__from_json".format(k): [param[k].serialize()]
             for k, v in task.items()

foreshadow/preprocessor.py

@@ -6,7 +6,7 @@
 
 from foreshadow.intents import GenericIntent
 from foreshadow.intents.registry import registry_eval
-from foreshadow.transformers.base import ParallelProcessor
+from foreshadow.transformers.base import ParallelProcessor, SmartTransformer
 from foreshadow.utils import PipelineStep, check_df
 
 
@@ -397,7 +397,10 @@ def serialize(self):
 
         # Serialize intent multi processors
         json_intents = {
-            k: {l: serialize_pipeline(j) for l, j in v.items()}
+            k: {
+                l: serialize_pipeline(j, include_smart=True)
+                for l, j in v.items()
+            }
             for k, v in self._intent_pipelines.items()
         }
 
@@ -485,7 +488,7 @@ def inverse_transform(self, X):
         return self.pipeline.inverse_transform(X)
 
 
-def serialize_pipeline(pipeline):
+def serialize_pipeline(pipeline, include_smart=False):
     """Serializes sklearn Pipeline object into JSON object for reconstruction.
 
     Args:
@@ -495,14 +498,55 @@ def serialize_pipeline(pipeline):
     Returns:
         list: JSON serializable object of form ``[cls, name, {**params}]``
     """
-    return [
-        {
-            "transformer": type(step[PipelineStep["CLASS"]]).__name__,
-            "name": step[PipelineStep["NAME"]],
-            "parameters": step[PipelineStep["CLASS"]].get_params(deep=False),
+
+    def ser_params(trans):
+        from foreshadow.transformers.externals import no_serialize_params
+
+        bad_params = [
+            "name",
+            *no_serialize_params.get(type(trans).__name__, []),
+        ]
+        return {
+            k: v
+            for k, v in trans.get_params(deep=False).items()
+            if k not in bad_params
         }
-        for step in pipeline.steps
-        if pipeline.steps[0][PipelineStep["NAME"]] != "null"
+
+    return [
+        p
+        for s in [
+            (
+                [
+                    {
+                        "transformer": type(
+                            step[PipelineStep["CLASS"]].transformer
+                        ).__name__,
+                        "name": step[PipelineStep["NAME"]],
+                        "parameters": ser_params(
+                            step[PipelineStep["CLASS"]].transformer
+                        ),
+                    }
+                ]
+                if not isinstance(
+                    step[PipelineStep["CLASS"]].transformer, Pipeline
+                )
+                else serialize_pipeline(
+                    step[PipelineStep["CLASS"]].transformer
+                )
+            )
+            if isinstance(step[PipelineStep["CLASS"]], SmartTransformer)
+            and not include_smart
+            else [
+                {
+                    "transformer": type(step[PipelineStep["CLASS"]]).__name__,
+                    "name": step[PipelineStep["NAME"]],
+                    "parameters": ser_params(step[PipelineStep["CLASS"]]),
+                }
+            ]
+            for step in pipeline.steps
+            if pipeline.steps[0][PipelineStep["NAME"]] != "null"
+        ]
+        for p in s
     ]