Devel (#27)

* default

* update

* bn

* middle

* middle

* graph test passed

* pytorch

* req

* ci

* ci

* ci

.travis.yml

@@ -3,6 +3,8 @@ python:
   - "3.6"
 before_install:
   - sudo apt-get install graphviz
+install:
+  - pip install -r requirements.txt --quiet
 script:
   - pytest tests --cov=autokeras
 after_success:

autokeras/bayesian.py

@@ -21,7 +21,7 @@ def layers_distance(list_a, list_b):
     for i in range(len_a):
         for j in range(len_b):
             f[i][j] = min(f[i][j - 1] + 1, f[i - 1][j] + 1, f[i - 1][j - 1] + layer_distance(list_a[i], list_b[j]))
-    return f[len_a][len_b]
+    return f[len_a - 1][len_b - 1]
 
 
 def skip_connection_distance(a, b):

autokeras/classifier.py

@@ -2,17 +2,19 @@
 import pickle
 import csv
 import time
-import tensorflow as tf
+from functools import reduce
+
+import torch
 
 import scipy.ndimage as ndimage
 
 import numpy as np
-from keras import backend
 from sklearn.metrics import accuracy_score
 from sklearn.model_selection import train_test_split
+from torch.utils.data import DataLoader
 
 from autokeras import constant
-from autokeras.preprocessor import OneHotEncoder
+from autokeras.preprocessor import OneHotEncoder, DataTransformer
 from autokeras.search import BayesianSearcher, train
 from autokeras.utils import ensure_dir, has_file, pickle_from_file, pickle_to_file
 
@@ -31,16 +33,12 @@ def _validate(x_train, y_train):
         raise ValueError('x_train and y_train should have the same number of instances.')
 
 
-def run_searcher_once(x_train, y_train, x_test, y_test, path):
+def run_searcher_once(train_data, test_data, path):
     if constant.LIMIT_MEMORY:
-        config = tf.ConfigProto()
-        config.gpu_options.allow_growth = True
-        sess = tf.Session(config=config)
-        init = tf.global_variables_initializer()
-        sess.run(init)
-        backend.set_session(sess)
+        # TODO: limit pytorch memory.
+        pass
     searcher = pickle_from_file(os.path.join(path, 'searcher'))
-    searcher.search(x_train, y_train, x_test, y_test)
+    searcher.search(train_data, test_data)
 
 
 def read_csv_file(csv_file_path):
@@ -150,6 +148,7 @@ def __init__(self, verbose=False, path=constant.DEFAULT_SAVE_PATH, resume=False,
             self.path = path
         else:
             self.y_encoder = None
+            self.data_transformer = None
             self.verbose = verbose
             self.searcher = False
             self.path = path
@@ -186,6 +185,10 @@ def fit(self, x_train=None, y_train=None, time_limit=None):
 
         y_train = self.y_encoder.transform(y_train)
 
+        # Transform x_train
+        if self.data_transformer is None:
+            self.data_transformer = DataTransformer(x_train)
+
         # Create the searcher and save on disk
         if not self.searcher:
             input_shape = x_train.shape[1:]
@@ -199,8 +202,12 @@ def fit(self, x_train=None, y_train=None, time_limit=None):
             self.searcher = True
 
         # Divide training data into training and testing data.
-        x_train, x_test, y_train, y_test = train_test_split(x_train, y_train, test_size=0.25, random_state=42)
+        x_train, x_test, y_train, y_test = train_test_split(x_train, y_train,
+                                                            test_size=constant.VALIDATION_SET_RATIO,
+                                                            random_state=42)
 
+        train_data = self.data_transformer.transform_train(x_train, y_train)
+        test_data = self.data_transformer.transform_test(x_test, y_test)
         pickle.dump(self, open(os.path.join(self.path, 'classifier'), 'wb'))
         pickle_to_file(self, os.path.join(self.path, 'classifier'))
 
@@ -209,7 +216,7 @@ def fit(self, x_train=None, y_train=None, time_limit=None):
 
         start_time = time.time()
         while time.time() - start_time <= time_limit:
-            run_searcher_once(x_train, y_train, x_test, y_test, self.path)
+            run_searcher_once(train_data, test_data, self.path)
             if len(self.load_searcher().history) >= constant.MAX_MODEL_NUM:
                 break
 
@@ -223,20 +230,19 @@ def predict(self, x_test):
             An numpy.ndarray containing the results.
         """
         if constant.LIMIT_MEMORY:
-            config = tf.ConfigProto()
-            config.gpu_options.allow_growth = True
-            sess = tf.Session(config=config)
-            init = tf.global_variables_initializer()
-            sess.run(init)
-            backend.set_session(sess)
-        x_test = x_test.astype('float32') / 255
+            # TODO: limit pytorch memory.
+            pass
+        test_data = self.data_transformer.transform_test(x_test)
+        test_loader = DataLoader(test_data, batch_size=constant.MAX_BATCH_SIZE, shuffle=True)
         model = self.load_searcher().load_best_model().produce_model()
-        return self.y_encoder.inverse_transform(model.predict(x_test, ))
+        model.eval()
 
-    def summary(self):
-        """Print the summary of the best model."""
-        model = self.load_searcher().load_best_model()
-        model.summary()
+        outputs = []
+        with torch.no_grad():
+            for index, inputs in enumerate(test_loader):
+                outputs.append(model(inputs).numpy())
+        output = reduce(lambda x, y: np.concatenate((x, y)), outputs)
+        return self.y_encoder.inverse_transform(output)
 
     def evaluate(self, x_test, y_test):
         """Return the accuracy score between predict value and test_y."""
@@ -262,25 +268,19 @@ def final_fit(self, x_train, y_train, x_test, y_test, trainer_args=None, retrain
         """
         if trainer_args is None:
             trainer_args = {}
+
         y_train = self.y_encoder.transform(y_train)
         y_test = self.y_encoder.transform(y_test)
+
+        train_data = self.data_transformer.transform_train(x_train, y_train)
+        test_data = self.data_transformer.transform_test(x_test, y_test)
+
         searcher = self.load_searcher()
         graph = searcher.load_best_model()
+
         if retrain:
             graph.weighted = False
-        _, _1, graph = train((graph, x_train, y_train, x_test, y_test, trainer_args, None))
-
-    def export_keras_model(self, path, model_id=None):
-        """Export the searched model as a Keras saved model.
-
-        Args:
-            path: A string. The path to the file to save.
-            model_id: A integer. If not provided, the function will export the best model.
-        """
-        if model_id is None:
-            model_id = self.get_best_model_id()
-        graph = self.load_searcher().load_model_by_id(model_id)
-        graph.produce_model().save(path)
+        _, _1, graph = train((graph, train_data, test_data, trainer_args, None))
 
     def get_best_model_id(self):
         """

autokeras/constant.py

@@ -1,5 +1,8 @@
 DEFAULT_SAVE_PATH = '/tmp/autokeras/'
 
+# Data
+
+VALIDATION_SET_RATIO = 0.08333
 
 # Searcher
 

autokeras/generator.py

@@ -1,16 +1,7 @@
-from random import randint, random
-
-from keras import Input, Model
-from keras.layers import MaxPooling1D, MaxPooling2D, MaxPooling3D, Dropout, Flatten, Dense, BatchNormalization, \
-    Activation, GlobalAveragePooling2D
-from keras.losses import categorical_crossentropy
-from keras.optimizers import Adadelta, Adam
-
 from autokeras import constant
 from autokeras.graph import Graph
-from autokeras.layers import get_conv_layer_func, get_ave_layer_func, StubBatchNormalization, StubActivation, StubConv, \
-    StubDropout, StubPooling, StubGlobalPooling, StubDense, StubInput
-from autokeras.stub import StubModel
+from autokeras.layers import StubBatchNormalization, StubConv, StubDropout, StubPooling, StubDense, StubFlatten, \
+    StubReLU, StubSoftmax
 
 
 class ClassifierGenerator:
@@ -22,10 +13,6 @@ def __init__(self, n_classes, input_shape):
         if len(self.input_shape) < 2:
             raise ValueError('The input dimension is too low.')
 
-    def _get_pool_layer_func(self):
-        pool_funcs = [MaxPooling1D, MaxPooling2D, MaxPooling3D]
-        return pool_funcs[len(self.input_shape) - 2]
-
     def _get_shape(self, dim_size):
         temp_list = [(dim_size,), (dim_size, dim_size), (dim_size, dim_size, dim_size)]
         return temp_list[len(self.input_shape) - 2]
@@ -36,69 +23,21 @@ def __init__(self, n_classes, input_shape):
         super().__init__(n_classes, input_shape)
 
     def generate(self, model_len=constant.MODEL_LEN, model_width=constant.MODEL_WIDTH):
-        pool = self._get_pool_layer_func()
-        conv = get_conv_layer_func(len(self._get_shape(3)))
-        ave = get_ave_layer_func(len(self._get_shape(3)))
-
         pooling_len = int(model_len / 4)
-        model = StubModel()
-        model.input_shape = self.input_shape
-        model.inputs = [0]
-        model.layers.append(StubInput())
+        graph = Graph(self.input_shape, False)
+        temp_input_channel = self.input_shape[-1]
+        output_node_id = 0
         for i in range(model_len):
-            model.layers += [StubActivation('relu'),
-                             StubConv(model_width, kernel_size=3, func=conv),
-                             StubBatchNormalization(),
-                             StubDropout(constant.CONV_DROPOUT_RATE)]
+            output_node_id = graph.add_layer(StubReLU(), output_node_id)
+            output_node_id = graph.add_layer(StubConv(temp_input_channel, model_width, kernel_size=3), output_node_id)
+            output_node_id = graph.add_layer(StubBatchNormalization(model_width), output_node_id)
+            output_node_id = graph.add_layer(StubDropout(constant.CONV_DROPOUT_RATE), output_node_id)
+            temp_input_channel = model_width
             if pooling_len == 0 or ((i + 1) % pooling_len == 0 and i != model_len - 1):
-                model.layers.append(StubPooling(func=pool))
-
-        model.layers.append(StubGlobalPooling(ave))
-        model.layers.append(StubDense(self.n_classes, activation='softmax'))
-        model.outputs = [len(model.layers)]
-        for index, layer in enumerate(model.layers):
-            layer.input = index
-            layer.output = index + 1
-        return Graph(model, False)
-
-
-class RandomConvClassifierGenerator(ClassifierGenerator):
-    def __init__(self, n_classes, input_shape):
-        super().__init__(n_classes, input_shape)
-
-    def generate(self):
-        """Return the random generated CNN model."""
-        conv_num = randint(1, 10)
-        dense_num = randint(1, 10)
-        dropout_rate = random()
-        filter_size = randint(1, 2) * 2 + 1
-        pool_size = randint(2, 3)
-        filter_shape = self._get_shape(filter_size)
-        pool_shape = self._get_shape(pool_size)
-        pool = self._get_pool_layer_func()
-        conv = get_conv_layer_func(len(filter_shape))
+                output_node_id = graph.add_layer(StubPooling(), output_node_id)
 
-        input_tensor = Input(shape=self.input_shape)
-        output_tensor = input_tensor
-        for i in range(conv_num):
-            kernel_num = randint(10, 30)
-            output_tensor = conv(kernel_num, filter_shape,
-                                 padding='same')(output_tensor)
-            output_tensor = BatchNormalization()(output_tensor)
-            output_tensor = Activation('relu')(output_tensor)
-            if random() > 0.5:
-                output_tensor = pool(pool_size=pool_shape, padding='same')(output_tensor)
-            if random() > 0.5:
-                output_tensor = Dropout(dropout_rate)(output_tensor)
-        output_tensor = Flatten()(output_tensor)
-        for i in range(dense_num):
-            node_num = randint(128, 1024)
-            output_tensor = Dense(node_num, activation='relu')(output_tensor)
-            if random() > 0.5:
-                output_tensor = Dropout(dropout_rate)(output_tensor)
-        output_tensor = Dense(self.n_classes, activation='softmax')(output_tensor)
-        model = Model(input_tensor, output_tensor)
-        model.compile(loss='categorical_crossentropy',
-                      optimizer=Adam(),
-                      metrics=['accuracy'])
-        return model
+        output_node_id = graph.add_layer(StubFlatten(), output_node_id)
+        output_node_id = graph.add_layer(StubDense(graph.node_list[output_node_id].shape[0], self.n_classes),
+                                         output_node_id)
+        graph.add_layer(StubSoftmax(), output_node_id)
+        return graph