Merge code from branch refactor/xg-boost-model to refactor/major_change_with_k1st

MIGRATION_NOTES.md

@@ -9,4 +9,3 @@ Due to changes to the package structure, please update your H1st code to:
 - The `Modeler` class is introduced with `build_model` method which is used to build the corresponding `Model` instance. The `Model` class' `load_data`, `explore`, `evaluate` now belongs to the `Modeler` class with `explore` being renamed to `explore_data` and `evaluate` being renamed to `evaluate_model`.
 - The `Model` class' `predict` method is renamed to `process`. Its `PredictiveModel` subclass which is then inherited by `RuleBasedModel`, `MLModel` still possess the `predict` method which basically calls the `process` one.
 - The `Model` class' `load` is renamed to `load_params`.
-

h1st/model/k1st/collaborator.py

@@ -0,0 +1,73 @@
+import logging
+
+import pandas as pd
+
+from h1st.model.wrapper.multi_model import MultiModel
+from h1st.model.ml_model import MLModel
+
+
+class kCollaboratorModel(MultiModel):
+
+    name: str = 'k-Collaborator'
+    data_key = 'X'
+    output_key = 'predictions'
+
+    def __init__(self):
+        '''
+        Model for running predictions with multiple models and combining the outputs.
+        Also handles persistance and loading of submodels
+
+        knowledge and ml models must take that same input and output a dict
+        with the "predictions" key and value as list of dict or dict (matching
+        the input format with key "X")
+        i.e. model.predict({"X": {'k1': 12, 'k2': 22}}) ->
+            {"predictions": {'output': 10}}
+        '''
+        super().__init__()
+        self.ensemble = None
+
+    def predict(self, input_data: dict) -> dict:
+        submodel_out = super().predict(input_data)[self.output_key]
+
+        # Inject original x value into input feature of Ensembler
+        if (
+            isinstance(self.ensemble, MLModel)
+            and (
+                isinstance(input_data["X"], pd.DataFrame)
+                or isinstance(input_data["X"], pd.Series)
+            )
+            and self.stats["inject_x_in_ensembler"]
+        ):
+            submodel_out = pd.concat([submodel_out, input_data['X']], axis=1)
+
+        if self.ensemble is None:
+            return {self.output_key: submodel_out}
+
+        ensemble_input = {'X': submodel_out}
+        ensemble_key = getattr(self.ensemble, 'output_key', 'predictions')
+        ensemble_pred = self.ensemble.predict(ensemble_input)[ensemble_key]
+        return {self.output_key: ensemble_pred}
+
+    def persist(self, version=None):
+        if self.ensemble is not None:
+            ensemble_version = self.ensemble.persist(version)
+            self.stats['ensemble'] = {
+                'version': ensemble_version,
+                'model_class': self.ensemble.__class__,
+            }
+        else:
+            self.stats['ensemble'] = None
+
+        version = super().persist(version)
+        return version
+
+    def load(self, version=None):
+        super().load(version)
+
+        if self.stats['ensemble'] is not None:
+            ensemble_version = self.stats['ensemble']['version']
+            ensemble_class = self.stats['ensemble']['model_class']
+            self.ensemble = ensemble_class().load(ensemble_version)
+        else:
+            self.ensemble = None
+        return self

h1st/model/k1st/collaborator_modeler.py

@@ -0,0 +1,101 @@
+import logging
+from typing import List
+
+import pandas as pd
+
+from h1st.model.wrapper.multi_modeler import MultiModeler
+from h1st.model.oracle.ensembler_models import MajorityVotingEnsembleModel
+from h1st.model.rule_based_modeler import RuleBasedModeler
+from h1st.model.k1st.collaborator import kCollaboratorModel
+from h1st.model.predictive_model import PredictiveModel
+from h1st.model.modeler import Modeler
+from h1st.model.model import Model
+from h1st.model.ml_modeler import MLModeler
+
+
+class kCollaboratorModeler(MultiModeler):
+
+    model_class = kCollaboratorModel
+
+    def __init__(self):
+        super().__init__()
+
+    def build_model(
+        self,
+        prepared_data: dict,
+        modelers: List[MLModeler] = [],
+        ensemble_modeler: Modeler = RuleBasedModeler(MajorityVotingEnsembleModel),
+        models: List[PredictiveModel] = None,
+        inject_x_in_ensembler: bool = False,
+        parallel: bool = False,
+    ) -> kCollaboratorModel:
+        '''
+        prepared_data must be in the format necessary for modelers
+        '''
+        self.stats['inject_x_in_ensembler'] = inject_x_in_ensembler
+        model = super().build_model(prepared_data, modelers, parallel)
+        for i, m in enumerate(models):
+            model.add_model(m, name=f'prebuilt-{model.__class__.__name__}-{i}')
+
+        if prepared_data is None:
+            model.stats['input_features'] = models[0].stats['input_features']
+            return model
+        else:
+            model.stats['input_features'] = list(prepared_data['X_train'].columns)
+
+        # train ensemble
+        raw_pred = model.predict({model.data_key: prepared_data['X_train']})[
+            model.output_key
+        ]
+
+        # If there is labeled_data and ensembler_modeler is MLModeler,
+        # then prepare the training data of ensembler.
+        labeled_data = prepared_data.get("labeled_data", prepared_data)
+        if isinstance(ensemble_modeler, MLModeler) and labeled_data is None:
+            raise ValueError("No data to train the machine-learning-based ensembler")
+
+        ensembler_data = {}
+        if labeled_data:
+            ensembler_train_input = model.predict(
+                {model.data_key: labeled_data["X_train"]}
+            )[model.output_key]
+
+            ensembler_test_input = model.predict(
+                {model.data_key: labeled_data["X_test"]}
+            )[model.output_key]
+
+            ensembler_data = {
+                'X_train': ensembler_train_input,
+                'y_train': labeled_data['y_train'],
+                'X_test': ensembler_test_input,
+                'y_test': labeled_data['y_test'],
+            }
+        else:
+            ensembler_data = None
+
+        ensemble = ensemble_modeler.build_model(ensembler_data)
+        model.ensemble = ensemble
+
+        # Generate metrics of all sub models (teacher, student, ensembler).
+        if labeled_data:
+            test_data = {"X": labeled_data["X_test"], "y": labeled_data["y_test"]}
+            try:
+                model.metrics = self.evaluate_model(test_data, model)
+            except Exception as e:
+                logging.error(
+                    (
+                        "Couldn't complete the submodel evaluation. "
+                        "Got the following error."
+                    )
+                )
+                logging.error(e)
+            else:
+                logging.info("Evaluated all sub models successfully.")
+
+        return model
+
+    def evaluate_model(self, test_data: dict, model: Model):
+        submodel_metrics = super().evaluate_model(test_data, model)
+        metrics = {'submodel_metrics': submodel_metrics}
+        # TODO: Compute overall model metrics
+        return metrics

h1st/model/k1st/oracle.py

@@ -0,0 +1,74 @@
+import logging
+
+import pandas as pd
+
+from h1st.model.wrapper.multi_model import MultiModel
+from h1st.model.ml_model import MLModel
+
+
+class kOracleModel(MultiModel):
+
+    name: str = 'k-Oracle'
+    data_key = 'X'
+    output_key = 'predictions'
+
+    def __init__(self):
+        '''
+        Model for running predictions with multiple models and combining the outputs.
+        Also handles persistance and loading of submodels
+
+        knowledge and ml models must take that same input and output a dict
+        with the "predictions" key and value as list of dict or dict (matching
+        the input format with key "X")
+        i.e. model.predict({"X": {'k1': 12, 'k2': 22}}) ->
+            {"predictions": {'output': 10}}
+        '''
+        super().__init__()
+        self.ensemble = None
+
+    def predict(self, input_data: dict) -> dict:
+        submodel_out = super().predict(input_data)[self.output_key]
+
+        # Inject original x value into input feature of Ensembler
+        if (
+            isinstance(self.ensemble, MLModel)
+            and (
+                isinstance(input_data["X"], pd.DataFrame)
+                or isinstance(input_data["X"], pd.Series)
+            )
+            and self.stats["inject_x_in_ensembler"]
+        ):
+            submodel_out = pd.concat([submodel_out, input_data['X']], axis=1)
+
+        if self.ensemble is None:
+            return {self.output_key: submodel_out}
+
+        ensemble_input = {'X': submodel_out}
+        ensemble_key = getattr(self.ensemble, 'output_key', 'predictions')
+        ensemble_pred = self.ensemble.predict(ensemble_input)[ensemble_key]
+        return {self.output_key: ensemble_pred}
+
+    def persist(self, version=None):
+        if self.ensemble:
+            ensemble_version = self.ensemble.persist()
+            self.stats['ensemble'] = {
+                'version': ensemble_version,
+                'model_class': self.ensemble.__class__,
+            }
+        else:
+            self.stats['ensemble'] = None
+
+        version = super().persist(version)
+        return version
+
+    def load(self, version=None):
+        super().load(version)
+
+        if self.stats['ensemble']:
+            ensemble_version = self.stats['ensemble']['version']
+            ensemble_class = self.stats['ensemble']['model_class']
+            self.ensemble = ensemble_class().load(ensemble_version)
+        else:
+            self.ensemble = None
+
+        return self

h1st/model/k1st/oracle_modeler.py

@@ -0,0 +1,115 @@
+import logging
+from typing import List
+
+from h1st.model.wrapper.multi_modeler import MultiModeler
+from h1st.model.oracle.ensembler_models import MajorityVotingEnsembleModel
+from h1st.model.rule_based_modeler import RuleBasedModeler
+from h1st.model.k1st.oracle import kOracleModel
+from h1st.model.predictive_model import PredictiveModel
+from h1st.model.modeler import Modeler
+from h1st.model.model import Model
+from h1st.model.ml_modeler import MLModeler
+
+
+class kOracleModeler(MultiModeler):
+
+    model_class = kOracleModel
+
+    def __init__(self):
+        super().__init__()
+
+    def build_model(
+        self,
+        prepared_data: dict,
+        modelers: List[MLModeler] = [],
+        ensemble_modeler: Modeler = RuleBasedModeler(MajorityVotingEnsembleModel),
+        teacher: PredictiveModel = None,
+        inject_x_in_ensembler: bool = False,
+        parallel: bool = False,
+    ) -> kOracleModel:
+        '''
+        prepared_data must be in the format necessary for modelers
+        '''
+        if prepared_data is None:
+            model = kOracleModel()
+            model.add_model(teacher, f'prebuilt-{teacher.__class__.__name__}')
+            model.stats['input_features'] = teacher.stats['input_features']
+            return model
+
+        teacher_data_key = getattr(teacher, 'data_key', 'X')
+        teacher_output_key = getattr(teacher, 'output_key', 'predictions')
+        teacher_pred = teacher.predict({teacher_data_key: prepared_data['X_teacher_train']})[
+            teacher_output_key
+        ]
+        student_training_data = prepared_data.copy()
+        student_training_data['y_train'] = teacher_pred
+        if 'X_test' in prepared_data.keys():
+            student_training_data['y_test'] = teacher.predict(
+                {teacher_data_key: prepared_data['X_test']}
+            )[teacher_output_key]
+
+        self.stats['inject_x_in_ensembler'] = inject_x_in_ensembler
+        model = super().build_model(student_training_data, modelers, parallel)
+
+        # Add teacher to MultiModel
+        model.add_model(teacher, f'prebuilt-{teacher.__class__.__name__}')
+
+        # train ensemble
+        raw_pred = model.predict({model.data_key: prepared_data['X_train']})[
+            model.output_key
+        ]
+
+        # If there is labeled_data and ensembler_modeler is MLModeler,
+        # then prepare the training data of ensembler.
+        labeled_data = prepared_data.get("labeled_data", None)
+        if isinstance(ensemble_modeler, MLModeler) and labeled_data is None:
+            raise ValueError("No data to train the machine-learning-based ensembler")
+
+        ensembler_data = {}
+        if labeled_data:
+            x_train_input = {"X": labeled_data["X_train"]}
+            x_test_input = {"X": labeled_data["X_test"]}
+
+            ensembler_train_input = model.predict({model.data_key: x_train_input})[
+                model.output_key
+            ]
+
+            ensembler_test_input = model.predict({model.data_key: x_test_input})
+
+            ensembler_data = {
+                'X_train': ensembler_train_input,
+                'y_train': labeled_data['y_train'],
+                'X_test': ensembler_test_input,
+                'y_test': labeled_data['y_test'],
+            }
+        else:
+            ensembler_data = None
+
+        ensemble = ensemble_modeler.build_model(ensembler_data)
+        model.ensemble = ensemble
+
+        # Generate metrics of all sub models (teacher, student, ensembler).
+        if labeled_data:
+            test_data = {"X": labeled_data["X_test"], "y": labeled_data["y_test"]}
+            try:
+                model.metrics = self.evaluate_model(test_data, model)
+            except Exception as e:
+                logging.error(
+                    (
+                        "Couldn't complete the submodel evaluation. "
+                        "Got the following error."
+                    )
+                )
+                logging.error(e)
+            else:
+                logging.info("Evaluated all sub models successfully.")
+
+        model.stats['input_features'] = list(prepared_data['X_train'].columns)
+
+        return model
+
+    def evaluate_model(self, test_data: dict, model: Model):
+        submodel_metrics = super().evaluate_model(test_data, model)
+        metrics = {'submodel_metrics': submodel_metrics}
+        # TODO: Compute overall model metrics
+        return metrics