Refactor experiment/result.py (#65)

* Refactor experiment result

* Update test of experiment

* Update given_data.py example

cornac/eval_methods/base_method.py

@@ -12,6 +12,7 @@
 from ..utils.common import validate_format
 from ..metrics.rating import RatingMetric
 from ..metrics.ranking import RankingMetric
+from ..experiment.result import Result
 from collections import OrderedDict
 import numpy as np
 import tqdm
@@ -324,7 +325,7 @@ def evaluate(self, model, metrics, user_based):
             user_results = list(metric_user_results[mt.name].values())
             metric_avg_results[mt.name] = np.mean(user_results)
 
-        return metric_avg_results, metric_user_results
+        return Result(model.name, metric_avg_results, metric_user_results)
 
     @classmethod
     def from_splits(cls, train_data, test_data, val_data=None, data_format='UIR',

cornac/eval_methods/cross_validation.py

@@ -7,7 +7,7 @@
 import numpy as np
 from .base_method import BaseMethod
 from ..utils.common import safe_indexing
-from ..experiment.result import CVSingleModelResult
+from ..experiment.result import CVResult
 
 
 class CrossValidation(BaseMethod):
@@ -98,12 +98,11 @@ def _next_fold(self):
             self.current_fold = 0        
 
     def evaluate(self, model, metrics, user_based):
-        result = CVSingleModelResult(model.name, metrics)
-
+        result = CVResult(model.name)
         for fold in range(self.n_folds):
             self._get_train_test()
-            avg_res, per_user_res = BaseMethod.evaluate(self, model, metrics, user_based)
-            result.add_fold_res(fold=fold, metric_avg_results=avg_res)
+            fold_result = BaseMethod.evaluate(self, model, metrics, user_based)
+            result.append(fold_result)
             self._next_fold()
-        result._compute_avg_res()
+        result.organize()
         return result

cornac/eval_methods/ratio_split.py

@@ -7,7 +7,7 @@
 from ..utils.common import safe_indexing
 from math import ceil
 from .base_method import BaseMethod
-from ..experiment.result import SingleModelResult
+from ..experiment.result import Result
 import numpy as np
 
 
@@ -124,7 +124,4 @@ def split(self):
 
     def evaluate(self, model, metrics, user_based):
         self.split()
-        metric_avg_results, per_user_results = BaseMethod.evaluate(self, model, metrics, user_based)
-        res = SingleModelResult(model.name, metrics, metric_avg_results, per_user_results)
-        res.organize_avg_res()
-        return res
+        return BaseMethod.evaluate(self, model, metrics, user_based)

cornac/experiment/experiment.py

@@ -5,8 +5,11 @@
          Quoc-Tuan Truong <tuantq.vnu@gmail.com>
 """
 
-from .result import Result
-from .result import CVResult
+from .result import ExperimentResult
+from .result import CVExperimentResult
+from ..metrics.rating import RatingMetric
+from ..metrics.ranking import RankingMetric
+from ..models.recommender import Recommender
 
 
 class Experiment:
@@ -42,50 +45,39 @@ def __init__(self, eval_method, models, metrics, user_based=True, verbose=False)
         self.metrics = self._validate_metrics(metrics)
         self.user_based = user_based
         self.verbose = verbose
-        from ..eval_methods.ratio_split import RatioSplit
-        from ..eval_methods.cross_validation import CrossValidation
-        if isinstance(self.eval_method, RatioSplit):
-            self.results = Result()
-        elif isinstance(eval_method, CrossValidation):
-            self.results = CVResult(eval_method.n_folds)
 
     @staticmethod
     def _validate_models(input_models):
         if not hasattr(input_models, "__len__"):
             raise ValueError('models have to be an array but {}'.format(type(input_models)))
 
-        from ..models.recommender import Recommender
-
         valid_models = []
         for model in input_models:
             if isinstance(model, Recommender):
                 valid_models.append(model)
-
         return valid_models
 
     @staticmethod
     def _validate_metrics(input_metrics):
         if not hasattr(input_metrics, "__len__"):
             raise ValueError('metrics have to be an array but {}'.format(type(input_metrics)))
 
-        from ..metrics.rating import RatingMetric
-        from ..metrics.ranking import RankingMetric
-
         valid_metrics = []
         for metric in input_metrics:
             if isinstance(metric, RatingMetric) or isinstance(metric, RankingMetric):
                 valid_metrics.append(metric)
-
         return valid_metrics
 
-    # Check depth of dictionary
-    def dict_depth(self, d):
-        if isinstance(d, dict):
-            return 1 + (max(map(self.dict_depth, d.values())) if d else 0)
-        return 0
+    def _create_result(self):
+        from ..eval_methods.cross_validation import CrossValidation
+        if isinstance(self.eval_method, CrossValidation):
+            return CVExperimentResult()
+        return ExperimentResult()
 
     # modify this function to accommodate several models
     def run(self):
+        result = self._create_result()
+
         metric_names = []
         organized_metrics = {'ranking': [], 'rating': []}
 
@@ -97,7 +89,8 @@ def run(self):
         for model in self.models:
             if self.verbose:
                 print(model.name)
-            model_res = self.eval_method.evaluate(model=model, metrics=self.metrics, user_based=self.user_based)
-            self.results._add_model_res(res=model_res, model_name=model.name)
+            model_result = self.eval_method.evaluate(model=model, metrics=self.metrics, user_based=self.user_based)
+            result.append(model_result)
 
-        self.results.show()
+        print('\n{}'.format(result))
+        return result

cornac/experiment/result.py

@@ -2,117 +2,87 @@
 
 """
 @author: Aghiles Salah <asalah@smu.edu.sg>
+         Quoc-Tuan Truong <tuantq.vnu@gmail.com>
 """
 
 import numpy as np
 import pandas as pd
 
+pd.set_option('precision', 4)
 
-class SingleModelResult:
+
+class Result:
     """ Result Class for a single model
 
     Parameters
     ----------
     """
 
-    def __init__(self, model_name, metrics, metric_avg_results, metric_user_results=None):
+    def __init__(self, model_name, metric_avg_results, metric_user_results):
         self.model_name = model_name
-        self.metric_names = self._get_metric_names(metrics)
-        self.avg = metric_avg_results
-        self.per_user = metric_user_results
+        self.metric_user_results = metric_user_results
+        self.result_df = self.to_df(metric_avg_results)
 
-    def organize_avg_res(self):
-        self.avg = self._get_data_frame(avg_res=self.avg)
+    def __str__(self):
+        self.result_df.index = [self.model_name]
+        return self.result_df.__str__()
 
-    def _get_data_frame(self, avg_res):
-        avg_res = [avg_res.get(mt_name, np.nan) for mt_name in self.metric_names]
-        avg_res = np.asarray(avg_res)
-        avg_res = avg_res.reshape(1, len(self.metric_names))
-        avg_res = pd.DataFrame(data=avg_res, index=np.asarray([self.model_name]), columns=np.asarray(self.metric_names))
-        return avg_res
-
-    def _get_metric_names(self, metrics):
+    @staticmethod
+    def to_df(metric_avg_results):
         metric_names = []
-        for mt in metrics:
-            metric_names.append(mt.name)
-        return metric_names
+        metric_scores = []
+        for name, score in metric_avg_results.items():
+            metric_names.append(name)
+            metric_scores.append(score)
+
+        return pd.DataFrame(data=np.asarray([metric_scores]),
+                            columns=np.asarray(metric_names))
 
 
-class CVSingleModelResult(SingleModelResult):
+class CVResult(list):
     """ Cross Validation Result Class for a single model
 
     Parameters
     ----------
     """
 
-    def __init__(self, model_name, metrics, metric_avg_results={}):
-        SingleModelResult.__init__(self, model_name, metrics, metric_avg_results)
-        #self.avg = metric_avg_results
-        self.per_fold_avg = {}
-        #self.avg = {}
+    def __init__(self, model_name):
+        super().__init__()
+        self.model_name = model_name
 
-    def add_fold_res(self, fold, metric_avg_results):
-        # think to organize the results first
-        self.per_fold_avg[fold] = metric_avg_results
+    def __str__(self):
+        return '[{}]\n{}'.format(self.model_name, self.result_df.__str__())
 
-    def _compute_avg_res(self):
-        for mt in self.per_fold_avg[0]:
-            self.avg[mt] = 0.0
-        for f in self.per_fold_avg:
-            for mt in self.per_fold_avg[f]:
-                self.avg[mt] += self.per_fold_avg[f][mt] / len(self.per_fold_avg)
-        self._organize_avg_res()
+    def organize(self):
+        self.result_df = pd.concat([r.result_df for r in self])
+        self.result_df.index = ['Fold {}'.format(i + 1) for i in range(self.__len__())]
 
-    def _organize_avg_res(self):
-        # global avg
-        self.avg = self._get_data_frame(avg_res=self.avg)
-        # per_fold avg
-        for f in self.per_fold_avg:
-            self.per_fold_avg[f] = self._get_data_frame(avg_res=self.per_fold_avg[f])
+        self.result_df = self.result_df.T
+        mean = self.result_df.mean(axis=1)
+        std = self.result_df.std(axis=1)
+        self.result_df['Mean'] = mean
+        self.result_df['Std'] = std
 
 
-class Result:
+class ExperimentResult(list):
     """ Result Class
 
     Parameters
     ----------
     """
 
-    def __init__(self, avg_results=None):
-        self.avg = avg_results
-        self.per_user = {}
-
-    def _add_model_res(self, res, model_name):
-        self.per_user[model_name] = res.per_user
-        if self.avg is None:
-            self.avg = res.avg
-        else:
-            self.avg = self.avg.append(res.avg)
-
-    def show(self):
-        print(self.avg)
+    def __str__(self):
+        df = pd.concat([r.result_df for r in self])
+        df.index = [r.model_name for r in self]
+        return df.__str__()
 
 
-class CVResult(Result):
+class CVExperimentResult(ExperimentResult):
     """ Cross Validation Result Class
 
     Parameters
     ----------
     """
 
-    def __init__(self, n_folds, avg_results=None):
-        self.avg = avg_results
-        self.per_fold_avg = {}
-        for f in range(n_folds):
-            self.per_fold_avg[f] = None
-
-    def _add_model_res(self, res, model_name):
-        if self.avg is None:
-            self.avg = res.avg
-        else:
-            self.avg = self.avg.append(res.avg)
-        for f in res.per_fold_avg:
-            if self.per_fold_avg[f] is None:
-                self.per_fold_avg[f] = res.per_fold_avg[f]
-            else:
-                self.per_fold_avg[f] = self.per_fold_avg[f].append(res.per_fold_avg[f])
+    def __str__(self):
+        return '\n\n'.join([r.__str__() for r in self])

examples/given_data.py

@@ -25,7 +25,5 @@
         use_bias=True, early_stop=True, verbose=True)
 
 # Evaluation
-avg_results, _ = eval_method.evaluate(model=mf,
-                                      metrics=[MAE(), RMSE()],
-                                      user_based=True)
-print(avg_results)
+result = eval_method.evaluate(model=mf, metrics=[MAE(), RMSE()], user_based=True)
+print(result)

tests/cornac/experiment/test_experiment.py

@@ -23,14 +23,6 @@ def test_with_ratio_split(self):
                          verbose=True)
         exp.run()
 
-        self.assertSequenceEqual(exp.results.avg.shape, (1, 4))
-        self.assertEqual(len(exp.results.per_user), 1)
-        self.assertEqual(len(exp.results.per_user['PMF']), 4)
-        self.assertEqual(len(exp.results.per_user['PMF']['MAE']), 2)
-        self.assertEqual(len(exp.results.per_user['PMF']['RMSE']), 2)
-        self.assertEqual(len(exp.results.per_user['PMF']['Recall@1']), 2)
-        self.assertEqual(len(exp.results.per_user['PMF']['F1@1']), 2)
-
         try:
             Experiment(None, None, None)
         except ValueError:
@@ -52,4 +44,4 @@ def test_with_cross_validation(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()