metric

autokeras/image_classifier.py

@@ -14,6 +14,7 @@
 from torch.utils.data import DataLoader
 
 from autokeras.constant import Constant
+from autokeras.metric import Accuracy
 from autokeras.preprocessor import OneHotEncoder, DataTransformer
 from autokeras.search import BayesianSearcher, train
 from autokeras.utils import ensure_dir, has_file, pickle_from_file, pickle_to_file
@@ -158,6 +159,7 @@ def __init__(self, verbose=False, path=Constant.DEFAULT_SAVE_PATH, resume=False,
             self.path = path
             self.searcher_args = searcher_args
             self.augment = augment
+            self.metric = Accuracy
             ensure_dir(path)
 
     def fit(self, x_train=None, y_train=None, time_limit=None):
@@ -201,6 +203,7 @@ def fit(self, x_train=None, y_train=None, time_limit=None):
             self.searcher_args['n_classes'] = n_classes
             self.searcher_args['input_shape'] = input_shape
             self.searcher_args['path'] = self.path
+            self.searcher_args['metric'] = self.metric
             self.searcher_args['verbose'] = self.verbose
             searcher = BayesianSearcher(**self.searcher_args)
             self.save_searcher(searcher)
@@ -285,7 +288,7 @@ def final_fit(self, x_train, y_train, x_test, y_test, trainer_args=None, retrain
 
         if retrain:
             graph.weighted = False
-        _, _1, graph = train((graph, train_data, test_data, trainer_args, None, self.verbose))
+        _, _1, graph = train((graph, train_data, test_data, trainer_args, None, self.metric, self.verbose))
 
     def get_best_model_id(self):
         """ Return an integer indicating the id of the best model."""

autokeras/metric.py

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+
+from sklearn.metrics import accuracy_score
+
+
+class Metric:
+
+    @classmethod
+    @abstractmethod
+    def higher_better(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def compute(cls, prediction, target):
+        pass
+
+
+class Accuracy(Metric):
+    @classmethod
+    def higher_better(cls):
+        return True
+
+    @classmethod
+    def compute(cls, prediction, target):
+        return accuracy_score(prediction, target)

autokeras/search.py

@@ -40,7 +40,7 @@ class BayesianSearcher:
             when using this layer as the first layer in a model.
         verbose: Verbosity mode.
         history: A list that stores the performance of model. Each element in it is a dictionary of 'model_id',
-            'loss', and 'accuracy'.
+            'loss', and 'metric_value'.
         path: A string. The path to the directory for saving the searcher.
         model_count: An integer. the total number of neural networks in the current searcher.
         descriptors: A dictionary of all the neural network architectures searched.
@@ -56,7 +56,7 @@ class BayesianSearcher:
         t_min: A float. The minimum temperature during simulated annealing.
     """
 
-    def __init__(self, n_classes, input_shape, path, verbose,
+    def __init__(self, n_classes, input_shape, path, metric, verbose,
                  trainer_args=None,
                  default_model_len=Constant.MODEL_LEN,
                  default_model_width=Constant.MODEL_WIDTH,
@@ -83,6 +83,7 @@ def __init__(self, n_classes, input_shape, path, verbose,
         self.input_shape = input_shape
         self.verbose = verbose
         self.history = []
+        self.metric = metric
         self.path = path
         self.model_count = 0
         self.descriptors = []
@@ -108,19 +109,19 @@ def load_model_by_id(self, model_id):
     def load_best_model(self):
         return self.load_model_by_id(self.get_best_model_id())
 
-    def get_accuracy_by_id(self, model_id):
+    def get_metric_value_by_id(self, model_id):
         for item in self.history:
             if item['model_id'] == model_id:
-                return item['accuracy']
+                return item['metric_value']
         return None
 
     def get_best_model_id(self):
-        return max(self.history, key=lambda x: x['accuracy'])['model_id']
+        return max(self.history, key=lambda x: x['metric_value'])['model_id']
 
     def replace_model(self, graph, model_id):
         pickle_to_file(graph, os.path.join(self.path, str(model_id) + '.h5'))
 
-    def add_model(self, accuracy, loss, graph, model_id):
+    def add_model(self, metric_value, loss, graph, model_id):
         if self.verbose:
             print('Saving model.')
 
@@ -131,9 +132,9 @@ def add_model(self, accuracy, loss, graph, model_id):
         if self.verbose:
             print('Model ID:', model_id)
             print('Loss:', loss)
-            print('Accuracy', accuracy)
+            print('Metric Value:', metric_value)
 
-        ret = {'model_id': model_id, 'loss': loss, 'accuracy': accuracy}
+        ret = {'model_id': model_id, 'loss': loss, 'metric_value': metric_value}
         self.history.append(ret)
         if model_id == self.get_best_model_id():
             file = open(os.path.join(self.path, 'best_model.txt'), 'w')
@@ -142,7 +143,7 @@ def add_model(self, accuracy, loss, graph, model_id):
 
         descriptor = graph.extract_descriptor()
         self.x_queue.append(descriptor)
-        self.y_queue.append(accuracy)
+        self.y_queue.append(metric_value)
 
         return ret
 
@@ -176,21 +177,21 @@ def search(self, train_data, test_data):
         multiprocessing.set_start_method('spawn', force=True)
         pool = multiprocessing.Pool(1)
         train_results = pool.map_async(train, [(graph, train_data, test_data, self.trainer_args,
-                                                os.path.join(self.path, str(model_id) + '.png'), self.verbose)])
+                                                os.path.join(self.path, str(model_id) + '.png'),
+                                                self.metric, self.verbose)])
 
         # Do the search in current thread.
         if not self.training_queue:
             new_graph, new_father_id = self.maximize_acq()
             new_model_id = self.model_count
             self.model_count += 1
             self.training_queue.append((new_graph, new_father_id, new_model_id))
-            descriptor = new_graph.extract_descriptor()
             self.descriptors.append(new_graph.extract_descriptor())
 
-        accuracy, loss, graph = train_results.get()[0]
+        metric_value, loss, graph = train_results.get()[0]
         pool.terminate()
         pool.join()
-        self.add_model(accuracy, loss, graph, model_id)
+        self.add_model(metric_value, loss, graph, model_id)
         self.search_tree.add_child(father_id, model_id)
         self.gpr.fit(self.x_queue, self.y_queue)
         self.x_queue = []
@@ -209,21 +210,21 @@ def maximize_acq(self):
         pq = PriorityQueue()
         temp_list = []
         for model_id in model_ids:
-            accuracy = self.get_accuracy_by_id(model_id)
-            temp_list.append((accuracy, model_id))
+            metric_value = self.get_metric_value_by_id(model_id)
+            temp_list.append((metric_value, model_id))
         temp_list = sorted(temp_list)
-        for accuracy, model_id in temp_list:
+        for metric_value, model_id in temp_list:
             graph = self.load_model_by_id(model_id)
             graph.clear_operation_history()
-            pq.put(Elem(accuracy, model_id, graph))
+            pq.put(Elem(metric_value, model_id, graph))
 
         t = 1.0
         t_min = self.t_min
         alpha = 0.9
         max_acq = -1
         while not pq.empty() and t > t_min:
             elem = pq.get()
-            temp_exp = min((elem.accuracy - max_acq) / t, 709.0)
+            temp_exp = min((elem.metric_value - max_acq) / t, 709.0)
             ap = math.exp(temp_exp)
             if ap > random.uniform(0, 1):
                 graphs = transform(elem.graph)
@@ -304,29 +305,30 @@ def get_dict(self, u=None):
 
 @total_ordering
 class Elem:
-    def __init__(self, accuracy, father_id, graph):
+    def __init__(self, metric_value, father_id, graph):
         self.father_id = father_id
         self.graph = graph
-        self.accuracy = accuracy
+        self.metric_value = metric_value
 
     def __eq__(self, other):
-        return self.accuracy == other.accuracy
+        return self.metric_value == other.metric_value
 
     def __lt__(self, other):
-        return self.accuracy < other.accuracy
+        return self.metric_value < other.metric_value
 
 
 def train(args):
-    graph, train_data, test_data, trainer_args, path, verbose = args
+    graph, train_data, test_data, trainer_args, path, metric, verbose = args
     model = graph.produce_model()
     # if path is not None:
     #     plot_model(model, to_file=path, show_shapes=True)
-    loss, accuracy = ModelTrainer(model,
-                                  train_data,
-                                  test_data,
-                                  verbose).train_model(**trainer_args)
+    loss, metric_value = ModelTrainer(model,
+                                      train_data,
+                                      test_data,
+                                      metric,
+                                      verbose).train_model(**trainer_args)
     model.set_weight_to_graph()
-    return accuracy, loss, model.graph
+    return metric_value, loss, model.graph
 
 
 def same_graph(des1, des2):

autokeras/utils.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import pickle
+import numpy as np
 from copy import deepcopy
 
 import torch
@@ -74,7 +75,7 @@ class ModelTrainer:
         verbose: Verbosity mode.
     """
 
-    def __init__(self, model, train_data, test_data, verbose):
+    def __init__(self, model, train_data, test_data, metric, verbose):
         """Init the ModelTrainer with `model`, `x_train`, `y_train`, `x_test`, `y_test`, `verbose`"""
         self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
         self.model = model
@@ -85,6 +86,7 @@ def __init__(self, model, train_data, test_data, verbose):
         self.criterion = torch.nn.NLLLoss()
         self.optimizer = None
         self.early_stop = None
+        self.metric = metric
 
     def train_model(self,
                     max_iter_num=None,
@@ -154,7 +156,8 @@ def _train(self, loader, epoch):
     def _test(self, test_loader):
         self.model.eval()
         test_loss = 0
-        correct = 0
+        all_targets = []
+        all_predicted = []
         with torch.no_grad():
             for batch_idx, (inputs, targets) in enumerate(deepcopy(test_loader)):
                 targets = targets.argmax(1)
@@ -163,8 +166,9 @@ def _test(self, test_loader):
                 test_loss += self.criterion(outputs, targets)
 
                 _, predicted = outputs.max(1)
-                correct += predicted.eq(targets).sum().item()
-        return test_loss, correct * 100.0 / len(test_loader.dataset)
+                all_predicted = np.concatenate((all_predicted, predicted.numpy()))
+                all_targets = np.concatenate((all_targets, targets.numpy()))
+        return test_loss, self.metric.compute(all_predicted, all_targets)
 
 
 def ensure_dir(directory):

tests/test_search.py

@@ -1,6 +1,7 @@
 from copy import deepcopy
 from unittest.mock import patch
 
+from autokeras.metric import Accuracy
 from autokeras.search import *
 
 from tests.common import clean_dir, MockProcess, get_processed_data, get_add_skip_model, get_concat_skip_model
@@ -23,7 +24,7 @@ def mock_train(**_):
 def test_bayesian_searcher(_, _1):
     train_data, test_data = get_processed_data()
     clean_dir(default_test_path)
-    generator = BayesianSearcher(3, (28, 28, 3), verbose=False, path=default_test_path)
+    generator = BayesianSearcher(3, (28, 28, 3), verbose=False, path=default_test_path, metric=Accuracy)
     Constant.N_NEIGHBOURS = 1
     Constant.T_MIN = 0.8
     for _ in range(2):
@@ -47,7 +48,7 @@ def test_export_json(_, _1):
     train_data, test_data = get_processed_data()
 
     clean_dir(default_test_path)
-    generator = BayesianSearcher(3, (28, 28, 3), verbose=False, path=default_test_path)
+    generator = BayesianSearcher(3, (28, 28, 3), verbose=False, path=default_test_path, metric=Accuracy)
     Constant.N_NEIGHBOURS = 1
     Constant.T_MIN = 0.8
     for _ in range(3):
@@ -82,7 +83,7 @@ def test_max_acq(_, _1):
     Constant.SEARCH_MAX_ITER = 0
     Constant.T_MIN = 0.8
     Constant.BETA = 1
-    generator = BayesianSearcher(3, (28, 28, 3), verbose=False, path=default_test_path)
+    generator = BayesianSearcher(3, (28, 28, 3), verbose=False, path=default_test_path, metric=Accuracy)
     for _ in range(3):
         generator.search(train_data, test_data)
     for index1, descriptor1 in enumerate(generator.descriptors):

tests/test_utils.py

@@ -1,4 +1,5 @@
 from autokeras.generator import DefaultClassifierGenerator
+from autokeras.metric import Accuracy
 from autokeras.utils import *
 
 from tests.common import get_processed_data
@@ -7,4 +8,4 @@
 def test_model_trainer():
     model = DefaultClassifierGenerator(3, (28, 28, 3)).generate().produce_model()
     train_data, test_data = get_processed_data()
-    ModelTrainer(model, train_data, test_data, False).train_model(max_iter_num=3)
+    ModelTrainer(model, train_data, test_data, Accuracy, False).train_model(max_iter_num=3)