Fixed serialization to always explicitly output single pipeline. Reso…

…lves issue #2
georgian-io-archive · Nov 30, 2018 · d69197a · d69197a
1 parent 5777789
commit d69197a
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 9 deletions.
diff --git a/foreshadow/preprocessor.py b/foreshadow/preprocessor.py
@@ -166,6 +166,11 @@ def _map_pipelines(self):
                     if len(v.multi_pipeline) > 0
                     else [("null", None)]
                 ),
+                "single": Pipeline(
+                    deepcopy(v.single_pipeline)
+                    if len(v.single_pipeline) > 0
+                    else [("null", None)]
+                ),
                 # Extract multi pipeline from JSON config (highest priority)
                 **{k: v for k, v in self._intent_pipelines.get(v.__name__, {}).items()},
             }
@@ -350,11 +355,7 @@ def serialize(self):
 
         # Serialize intent multi processors
         json_intents = {
-            k: {
-                l: serialize_pipeline(j)
-                for l, j in v.items()
-                if j.steps[0][PipelineStep["NAME"]] != "null"
-            }
+            k: {l: serialize_pipeline(j) for l, j in v.items()}
             for k, v in self._intent_pipelines.items()
         }
 

diff --git a/foreshadow/tests/test_configs/test_serialize.json b/foreshadow/tests/test_configs/test_serialize.json
@@ -1 +1 @@
-{"columns": {"crim": ["TestGenericIntent", [["StandardScaler", "Scaler", {"copy": true, "keep_columns": false, "name": "Scaler", "with_mean": false, "with_std": true}]], ["TestGenericIntent", "TestNumericIntent"]], "zn": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "indus": ["TestGenericIntent", [], ["TestGenericIntent", "TestNumericIntent"]], "chas": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "nox": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "rm": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "age": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "dis": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "rad": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "tax": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "ptratio": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "b": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "lstat": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "medv": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]]}, "postprocess": [["pca", ["age"], [["PCA", "PCA", {"copy": true, "iterated_power": "auto", "keep_columns": false, "n_components": 2, "name": "PCA", "random_state": null, "svd_solver": "auto", "tol": 0.0, "whiten": false}]]]], "intents": {"TestGenericIntent": {"multi": [["PCA", "pca", {"copy": true, "iterated_power": "auto", "keep_columns": false, "n_components": 2, "name": "pca", "random_state": null, "svd_solver": "auto", "tol": 0.0, "whiten": false}]]}, "TestNumericIntent": {"multi": [["PCA", "pca", {"copy": true, "iterated_power": "auto", "keep_columns": false, "n_components": 5, "name": "pca", "random_state": null, "svd_solver": "auto", "tol": 0.0, "whiten": false}]], "single": [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]]}}}
+{"columns": {"crim": ["TestGenericIntent", [["StandardScaler", "Scaler", {"copy": true, "keep_columns": false, "name": "Scaler", "with_mean": false, "with_std": true}]], ["TestGenericIntent", "TestNumericIntent"]], "zn": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "indus": ["TestGenericIntent", [], ["TestGenericIntent", "TestNumericIntent"]], "chas": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "nox": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "rm": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "age": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "dis": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "rad": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "tax": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "ptratio": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "b": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "lstat": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]], "medv": ["TestNumericIntent", [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]], ["TestGenericIntent", "TestNumericIntent"]]}, "postprocess": [["pca", ["age"], [["PCA", "PCA", {"copy": true, "iterated_power": "auto", "keep_columns": false, "n_components": 2, "name": "PCA", "random_state": null, "svd_solver": "auto", "tol": 0.0, "whiten": false}]]]], "intents": {"TestGenericIntent": {"multi": [["PCA", "pca", {"copy": true, "iterated_power": "auto", "keep_columns": false, "n_components": 2, "name": "pca", "random_state": null, "svd_solver": "auto", "tol": 0.0, "whiten": false}]], "single": []}, "TestNumericIntent": {"multi": [["PCA", "pca", {"copy": true, "iterated_power": "auto", "keep_columns": false, "n_components": 5, "name": "pca", "random_state": null, "svd_solver": "auto", "tol": 0.0, "whiten": false}]], "single": [["Imputer", "impute", {"axis": 0, "copy": true, "keep_columns": false, "missing_values": "NaN", "name": "impute", "strategy": "mean", "verbose": 0}]]}}}
diff --git a/foreshadow/tests/test_preprocessor.py b/foreshadow/tests/test_preprocessor.py
@@ -748,7 +748,4 @@ def test_preprocessor_serialize():
     proc.fit(df)
     out = proc.serialize()
 
-    print(json.dumps(truth))
-    print(json.dumps(out))
-
     assert json.loads(json.dumps(truth)) == json.loads(json.dumps(out))