Adding a proper API for grid search CV, in the midst of change

Author: raghukiran1224

codeflare/pipelines/Datamodel.py

@@ -2,8 +2,10 @@
 from enum import Enum
 
 import sklearn.base as base
+from codeflare.pipelines.Datamodel import PipelineParam
 from sklearn.base import TransformerMixin
 from sklearn.base import BaseEstimator
+from sklearn.model_selection import ParameterGrid
 
 import ray
 import pickle5 as pickle
@@ -158,6 +160,8 @@ class Node(ABC):
     """
 
     def __init__(self, node_name, node_input_type: NodeInputType, node_firing_type: NodeFiringType, node_state_type: NodeStateType):
+        if '__' in node_name:
+            raise pe.PipelineException("Node name cannot have __, please rename")
         self.__node_name__ = node_name
         self.__node_input_type__ = node_input_type
         self.__node_firing_type__ = node_firing_type
@@ -510,6 +514,24 @@ def save(self, filehandle):
         saved_pipeline = _SavedPipeline(nodes, edges)
         pickle.dump(saved_pipeline, filehandle)
 
+    def set_param_grid(self, pipeline_param: PipelineParam):
+        result = Pipeline()
+        pipeline_params = pipeline_param.get_all_params()
+        parameterized_nodes = {}
+        for node_name, params in pipeline_params.items():
+            node_name_part, num = node_name.split('__', 1)
+            if node_name_part not in parameterized_nodes.keys():
+                parameterized_nodes[node_name_part] = []
+            node = self.__node_name_map__[node_name_part]
+            estimator = node.get_estimator()
+            cloned_estimator = estimator.clone()
+            cloned_estimator.set_params(**params)
+
+            parameterized_nodes[node_name_part].append()
+            result.add_node()
+
+        # construct nodes
+
     @staticmethod
     def load(filehandle):
         saved_pipeline = pickle.load(filehandle)
@@ -597,3 +619,46 @@ def add_xy_arg(self, node: Node, xy: Xy):
 
     def get_in_args(self):
         return self.__in_args__
+
+
+class PipelineParam:
+    def __init__(self):
+        self.__node_name_param_map__ = {}
+
+    @staticmethod
+    def from_param_grid(fit_params: dict):
+        pipeline_param = PipelineParam()
+        fit_params_nodes = {}
+        for pname, pval in fit_params.items():
+            if '__' not in pname:
+                raise ValueError(
+                    "Pipeline.fit does not accept the {} parameter. "
+                    "You can pass parameters to specific steps of your "
+                    "pipeline using the stepname__parameter format, e.g. "
+                    "`Pipeline.fit(X, y, logisticregression__sample_weight"
+                    "=sample_weight)`.".format(pname))
+            node_name, param = pname.split('__', 1)
+            if node_name not in fit_params_nodes.keys():
+                fit_params_nodes[node_name] = {}
+
+            fit_params_nodes[node_name][param] = pval
+
+        # we have the split based on convention, now to create paramter grid for each node
+        for node_name, param in fit_params_nodes.items():
+            pg = ParameterGrid(param)
+            pg_list = list(pg)
+            for i in range(len(pg_list)):
+                p = pg_list[i]
+                curr_node_name = node_name + '__' + str(i)
+                pipeline_param.add_param(curr_node_name, p)
+
+        return pipeline_param
+
+    def add_param(self, node_name: str, params: dict):
+        self.__node_name_param_map__[node_name] = params
+
+    def get_param(self, node_name: str):
+        return self.__node_name_param_map__[node_name]
+
+    def get_all_params(self):
+        return self.__node_name_param_map__

codeflare/pipelines/test_Datamodel.py renamed to codeflare/pipelines/tests/test_Datamodel.py

@@ -60,7 +60,8 @@ def test_grid_search():
     xy = dm.Xy(X_train, y_train)
     pipeline_input.add_xy_arg(input_node, xy)
 
-    kf = KFold(2)
+    k = 2
+    kf = KFold(k)
     result = rt.grid_search_cv(kf, pipeline, pipeline_input)
     node_rf = pipeline.get_node('random_forest')
     node_gb = pipeline.get_node('gradient_boost')
@@ -73,7 +74,7 @@ def test_grid_search():
             node_rf_pipeline = True
         elif out_node.get_node_name() == node_gb.get_node_name():
             node_gb_pipeline = True
-        if len(scores) != 2:
+        if len(scores) != k:
             assert False
     assert node_rf_pipeline
     assert node_gb_pipeline

codeflare/pipelines/tests/test_runtime.py

@@ -5,7 +5,6 @@ codeflare/pipelines/Datamodel.py
 codeflare/pipelines/Exceptions.py
 codeflare/pipelines/Runtime.py
 codeflare/pipelines/__init__.py
-codeflare/pipelines/test_Datamodel.py
 codeflare_pipelines.egg-info/PKG-INFO
 codeflare_pipelines.egg-info/SOURCES.txt
 codeflare_pipelines.egg-info/dependency_links.txt

codeflare_pipelines.egg-info/SOURCES.txt

@@ -12,7 +12,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "id": "da96167d",
    "metadata": {},
    "outputs": [],
@@ -22,20 +22,75 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 17,
    "id": "a559b7bb",
    "metadata": {},
    "outputs": [],
    "source": [
     "param_grid = {\n",
     "    'pca__n_components': [5, 15, 30, 45, 64],\n",
+    "    'pca__m_components': [6, 10],\n",
     "    'logistic__C': np.logspace(-4, 4, 4),\n",
     "}"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 18,
+   "id": "4e1ff38c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import codeflare.pipelines.Datamodel as dm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "4691ea01",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipeline_params = dm.PipelineParam.from_param_grid(param_grid)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "1da83403",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'pca__0': {'m_components': 6, 'n_components': 5},\n",
+       " 'pca__1': {'m_components': 6, 'n_components': 15},\n",
+       " 'pca__2': {'m_components': 6, 'n_components': 30},\n",
+       " 'pca__3': {'m_components': 6, 'n_components': 45},\n",
+       " 'pca__4': {'m_components': 6, 'n_components': 64},\n",
+       " 'pca__5': {'m_components': 10, 'n_components': 5},\n",
+       " 'pca__6': {'m_components': 10, 'n_components': 15},\n",
+       " 'pca__7': {'m_components': 10, 'n_components': 30},\n",
+       " 'pca__8': {'m_components': 10, 'n_components': 45},\n",
+       " 'pca__9': {'m_components': 10, 'n_components': 64},\n",
+       " 'logistic__0': {'C': 0.0001},\n",
+       " 'logistic__1': {'C': 0.046415888336127774},\n",
+       " 'logistic__2': {'C': 21.54434690031882},\n",
+       " 'logistic__3': {'C': 10000.0}}"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pipeline_params.__node_name_param_map__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
    "id": "ec28e356",
    "metadata": {},
    "outputs": [],
@@ -45,7 +100,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 9,
    "id": "fca4ae99",
    "metadata": {},
    "outputs": [],
@@ -55,34 +110,62 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
+   "id": "d24579ee",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
    "id": "acc69456",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'logistic__C': 0.0001, 'pca__n_components': 5}\n",
-      "{'logistic__C': 0.0001, 'pca__n_components': 15}\n",
-      "{'logistic__C': 0.0001, 'pca__n_components': 30}\n",
-      "{'logistic__C': 0.0001, 'pca__n_components': 45}\n",
-      "{'logistic__C': 0.0001, 'pca__n_components': 64}\n",
-      "{'logistic__C': 0.046415888336127774, 'pca__n_components': 5}\n",
-      "{'logistic__C': 0.046415888336127774, 'pca__n_components': 15}\n",
-      "{'logistic__C': 0.046415888336127774, 'pca__n_components': 30}\n",
-      "{'logistic__C': 0.046415888336127774, 'pca__n_components': 45}\n",
-      "{'logistic__C': 0.046415888336127774, 'pca__n_components': 64}\n",
-      "{'logistic__C': 21.54434690031882, 'pca__n_components': 5}\n",
-      "{'logistic__C': 21.54434690031882, 'pca__n_components': 15}\n",
-      "{'logistic__C': 21.54434690031882, 'pca__n_components': 30}\n",
-      "{'logistic__C': 21.54434690031882, 'pca__n_components': 45}\n",
-      "{'logistic__C': 21.54434690031882, 'pca__n_components': 64}\n",
-      "{'logistic__C': 10000.0, 'pca__n_components': 5}\n",
-      "{'logistic__C': 10000.0, 'pca__n_components': 15}\n",
-      "{'logistic__C': 10000.0, 'pca__n_components': 30}\n",
-      "{'logistic__C': 10000.0, 'pca__n_components': 45}\n",
-      "{'logistic__C': 10000.0, 'pca__n_components': 64}\n"
+      "{'logistic__C': 0.0001, 'pca__m_components': 6, 'pca__n__components': 5}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 6, 'pca__n__components': 15}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 6, 'pca__n__components': 30}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 6, 'pca__n__components': 45}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 6, 'pca__n__components': 64}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 10, 'pca__n__components': 5}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 10, 'pca__n__components': 15}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 10, 'pca__n__components': 30}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 10, 'pca__n__components': 45}\n",
+      "{'logistic__C': 0.0001, 'pca__m_components': 10, 'pca__n__components': 64}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 6, 'pca__n__components': 5}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 6, 'pca__n__components': 15}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 6, 'pca__n__components': 30}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 6, 'pca__n__components': 45}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 6, 'pca__n__components': 64}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 10, 'pca__n__components': 5}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 10, 'pca__n__components': 15}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 10, 'pca__n__components': 30}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 10, 'pca__n__components': 45}\n",
+      "{'logistic__C': 0.046415888336127774, 'pca__m_components': 10, 'pca__n__components': 64}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 6, 'pca__n__components': 5}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 6, 'pca__n__components': 15}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 6, 'pca__n__components': 30}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 6, 'pca__n__components': 45}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 6, 'pca__n__components': 64}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 10, 'pca__n__components': 5}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 10, 'pca__n__components': 15}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 10, 'pca__n__components': 30}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 10, 'pca__n__components': 45}\n",
+      "{'logistic__C': 21.54434690031882, 'pca__m_components': 10, 'pca__n__components': 64}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 6, 'pca__n__components': 5}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 6, 'pca__n__components': 15}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 6, 'pca__n__components': 30}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 6, 'pca__n__components': 45}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 6, 'pca__n__components': 64}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 10, 'pca__n__components': 5}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 10, 'pca__n__components': 15}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 10, 'pca__n__components': 30}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 10, 'pca__n__components': 45}\n",
+      "{'logistic__C': 10000.0, 'pca__m_components': 10, 'pca__n__components': 64}\n"
      ]
     }
    ],