disable multiprocessing for preprocessing elements in optimum_pipe

wwu-mmll · Nov 23, 2020 · 0a77ea8 · 0a77ea8
1 parent ded13c3
commit 0a77ea8
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 3 deletions.
diff --git a/photonai/base/hyperpipe.py b/photonai/base/hyperpipe.py
@@ -792,8 +792,8 @@ def _finalize_optimization(self):
 
             self.optimum_pipe.fit(self.data.X, self.data.y, **self.data.kwargs)
 
-            # Before saving the optimum pipe, add preprocessing
-            self.optimum_pipe._add_preprocessing(self.preprocessing)
+            # Before saving the optimum pipe, add preprocessing without multiprocessing
+            self.optimum_pipe._add_preprocessing(self.disable_multiprocessing_recursively(self.preprocessing))
 
             # Now truly set to no caching (including single_subject_caching)
             self.recursive_cache_folder_propagation(self.optimum_pipe, None, None)

diff --git a/test/base_tests/test_hyperpipe.py b/test/base_tests/test_hyperpipe.py
@@ -413,7 +413,7 @@ def setup_crazy_pipe(self):
         nmb_list = list()
         for i in range(5):
             nmb = ParallelBranch(name=str(i), nr_of_processes=i+3)
-            sp = PipelineElement('PCA', hyperparameters= {'n_components': IntegerRange(1, 50)})
+            sp = PipelineElement('PCA', hyperparameters={'n_components': IntegerRange(1, 50)})
             nmb += sp
             nmb_list.append(nmb)
 
@@ -504,6 +504,32 @@ def test_finalize_optimization(self):
         loaded_array = np.loadtxt(open(backmapped_feature_importances, 'rb'), delimiter=",")
         self.assertEqual(loaded_array.shape[0], self.__X.shape[1])
 
+    def test_finalize_optimization_preprocessing(self):
+        self.hyperpipe.elements = list()
+
+        pre_proc = Preprocessing()
+        pre_proc += PipelineElement('StandardScaler')
+        self.hyperpipe.add(pre_proc)
+        self.hyperpipe.add(PipelineElement('SVC'))
+        self.hyperpipe.fit(self.__X, self.__y)
+
+        self.assertTrue(os.path.isfile(os.path.join(self.hyperpipe.output_settings.results_folder,
+                                                    'photon_best_model.photon')))
+
+    def test_finalize_optimization_preprocessing_with_client(self):
+        self.hyperpipe.elements = list()
+
+        pb = ParallelBranch(name="ParallelBranch", nr_of_processes=2)
+        pb += PipelineElement('LabelEncoder')
+        pre_proc = Preprocessing()
+        pre_proc += pb
+        self.hyperpipe.add(pre_proc)
+        self.hyperpipe.add(PipelineElement('SVC'))
+        self.hyperpipe.fit(self.__X, self.__y)
+
+        self.assertTrue(os.path.isfile(os.path.join(self.hyperpipe.output_settings.results_folder,
+                                                    'photon_best_model.photon')))
+
     def test_optimum_pipe_predict_and_predict_proba_and_transform(self):
         # find best config and test against sklearn
         self.hyperpipe.elements[-1] = PipelineElement('RandomForestClassifier', {'n_estimators': IntegerRange(4, 20,