bn-relu-conv

autokeras/constant.py

@@ -15,8 +15,8 @@
 
 DENSE_DROPOUT_RATE = 0.5
 CONV_DROPOUT_RATE = 0.25
-CONV_BLOCK_SIZE = 4
-DENSE_BLOCK_SIZE = 2
+CONV_BLOCK_DISTANCE = 2
+DENSE_BLOCK_DISTANCE = 2
 
 # ModelTrainer
 

autokeras/generator.py

@@ -54,31 +54,31 @@ def generate(self):
         conv = get_conv_layer_func(len(self._get_shape(3)))
         ave = get_ave_layer_func(len(self._get_shape(3)))
 
-        input_tensor = Input(shape=self.input_shape)
-        output_tensor = conv(32, kernel_size=self._get_shape(3), padding='same')(input_tensor)
+        output_tensor = input_tensor = Input(shape=self.input_shape)
         output_tensor = BatchNormalization()(output_tensor)
         output_tensor = Activation('relu')(output_tensor)
+        output_tensor = conv(32, kernel_size=self._get_shape(3), padding='same')(output_tensor)
         output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
         output_tensor = pool(padding='same')(output_tensor)
 
-        output_tensor = conv(64, kernel_size=self._get_shape(3), padding='same', activation='linear')(output_tensor)
         output_tensor = BatchNormalization()(output_tensor)
         output_tensor = Activation('relu')(output_tensor)
+        output_tensor = conv(64, kernel_size=self._get_shape(3), padding='same', activation='linear')(output_tensor)
         output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
         output_tensor = pool(padding='same')(output_tensor)
 
-        output_tensor = conv(64, kernel_size=self._get_shape(3), padding='same', activation='linear')(output_tensor)
         output_tensor = BatchNormalization()(output_tensor)
         output_tensor = Activation('relu')(output_tensor)
+        output_tensor = conv(64, kernel_size=self._get_shape(3), padding='same', activation='linear')(output_tensor)
         output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
         output_tensor = pool(padding='same')(output_tensor)
 
-        output_tensor = conv(64, kernel_size=self._get_shape(3), padding='same', activation='linear')(output_tensor)
         output_tensor = BatchNormalization()(output_tensor)
         output_tensor = Activation('relu')(output_tensor)
+        output_tensor = conv(64, kernel_size=self._get_shape(3), padding='same', activation='linear')(output_tensor)
         output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
         output_tensor = ave()(output_tensor)

autokeras/graph.py

@@ -429,7 +429,7 @@ def _block_end_node(self, layer_id, block_size):
         return ret
 
     def _dense_block_end_node(self, layer_id):
-        return self._block_end_node(layer_id, constant.DENSE_BLOCK_SIZE)
+        return self._block_end_node(layer_id, constant.DENSE_BLOCK_DISTANCE)
 
     def _conv_block_end_node(self, layer_id):
         """
@@ -441,7 +441,7 @@ def _conv_block_end_node(self, layer_id):
             The input node ID of the last layer in the convolutional block.
 
         """
-        return self._block_end_node(layer_id, constant.CONV_BLOCK_SIZE)
+        return self._block_end_node(layer_id, constant.CONV_BLOCK_DISTANCE)
 
     def to_add_skip_model(self, start_id, end_id):
         """Add a weighted add skip connection from before start node to end node.
@@ -456,13 +456,15 @@ def to_add_skip_model(self, start_id, end_id):
         self.operation_history.append(('to_add_skip_model', start_id, end_id))
         conv_layer_ids = self._conv_layer_ids_in_order()
         start_id = conv_layer_ids[conv_layer_ids.index(start_id) + 1]
-        conv_input_id = self.layer_id_to_input_node_ids[start_id][0]
+        conv_block_input_id = self.layer_id_to_input_node_ids[start_id][0]
+        conv_block_input_id = self.reverse_adj_list[conv_block_input_id][0][0]
+        conv_block_input_id = self.reverse_adj_list[conv_block_input_id][0][0]
 
         dropout_input_id = self._conv_block_end_node(end_id)
 
         # Add the pooling layer chain.
-        pooling_layer_list = self._get_pooling_layers(conv_input_id, dropout_input_id)
-        skip_output_id = conv_input_id
+        pooling_layer_list = self._get_pooling_layers(conv_block_input_id, dropout_input_id)
+        skip_output_id = conv_block_input_id
         for index, layer_id in enumerate(pooling_layer_list):
             layer = self.layer_list[layer_id]
             new_node_id = self._add_new_node()
@@ -490,14 +492,16 @@ def to_concat_skip_model(self, start_id, end_id):
         # start = self.layer_list[start_id]
         conv_layer_ids = self._conv_layer_ids_in_order()
         start_id = conv_layer_ids[conv_layer_ids.index(start_id) + 1]
-        conv_input_id = self.layer_id_to_input_node_ids[start_id][0]
+        conv_block_input_id = self.layer_id_to_input_node_ids[start_id][0]
+        conv_block_input_id = self.reverse_adj_list[conv_block_input_id][0][0]
+        conv_block_input_id = self.reverse_adj_list[conv_block_input_id][0][0]
 
         end = self.layer_list[end_id]
         dropout_input_id = self._conv_block_end_node(end_id)
 
         # Add the pooling layer chain.
-        pooling_layer_list = self._get_pooling_layers(conv_input_id, dropout_input_id)
-        skip_output_id = conv_input_id
+        pooling_layer_list = self._get_pooling_layers(conv_block_input_id, dropout_input_id)
+        skip_output_id = conv_block_input_id
         for index, layer_id in enumerate(pooling_layer_list):
             layer = self.layer_list[layer_id]
             new_node_id = self._add_new_node()
@@ -520,7 +524,7 @@ def to_concat_skip_model(self, start_id, end_id):
 
         self.pre_vis[dropout_output_id] = True
         dim = layer_width(end)
-        n_add = self._upper_layer_width(conv_input_id)
+        n_add = self._upper_layer_width(conv_block_input_id)
         self._search_next(dropout_output_id, dim, dim, n_add)
 
     def extract_descriptor(self):

autokeras/layer_transformer.py

@@ -39,9 +39,9 @@ def deeper_conv_block(conv_layer, kernel_size, weighted=True):
                        np.ones(n_filters, dtype=np.float32)]
         bn.set_weights(new_weights)
 
-    return [new_conv_layer,
-            bn,
+    return [bn,
             StubActivation('relu'),
+            new_conv_layer,
             StubDropout(constant.CONV_DROPOUT_RATE)]
 
 

tests/common.py

@@ -10,34 +10,34 @@
 
 
 def get_concat_skip_model():
-    input_tensor = Input(shape=(5, 5, 3))
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(input_tensor)
+    output_tensor = input_tensor = Input(shape=(5, 5, 3))
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     add_input = output_tensor
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     output_tensor = Concatenate()([output_tensor, add_input])
     add_input = output_tensor
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     output_tensor = Concatenate()([output_tensor, add_input])
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     output_tensor = Flatten()(output_tensor)
@@ -48,28 +48,28 @@ def get_concat_skip_model():
 
 
 def get_add_skip_model():
-    input_tensor = Input(shape=(5, 5, 3))
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(input_tensor)
+    output_tensor = input_tensor = Input(shape=(5, 5, 3))
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     add_input = output_tensor
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     output_tensor = WeightedAdd()([output_tensor, add_input])
     add_input = output_tensor
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     output_tensor = WeightedAdd()([output_tensor, add_input])
@@ -81,15 +81,19 @@ def get_add_skip_model():
 
 
 def get_conv_model():
-    input_tensor = Input(shape=(5, 5, 3))
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(input_tensor)
+    output_tensor = input_tensor = Input(shape=(5, 5, 3))
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
-    output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
     output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
+    output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
+    output_tensor = GlobalAveragePooling2D()(output_tensor)
+    output_tensor = Dense(5, activation='relu')(output_tensor)
+    output_tensor = Dropout(constant.DENSE_DROPOUT_RATE)(output_tensor)
+    output_tensor = Dense(5, activation='softmax')(output_tensor)
     return Model(inputs=input_tensor, outputs=output_tensor)
 
 
@@ -98,10 +102,10 @@ def get_conv_data():
 
 
 def get_conv_dense_model():
-    input_tensor = Input(shape=(5, 5, 3))
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(input_tensor)
+    output_tensor = input_tensor = Input(shape=(5, 5, 3))
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
     output_tensor = GlobalAveragePooling2D()(output_tensor)
     output_tensor = Dense(5, activation='relu')(output_tensor)
@@ -111,27 +115,27 @@ def get_conv_dense_model():
 
 
 def get_pooling_model():
-    input_tensor = Input(shape=(5, 5, 3))
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(input_tensor)
+    output_tensor = input_tensor = Input(shape=(5, 5, 3))
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     output_tensor = MaxPooling2D(padding='same')(output_tensor)
 
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
-    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
+    output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
     output_tensor = Flatten()(output_tensor)

tests/test_graph.py

@@ -6,22 +6,22 @@
 
 def test_graph():
     graph = Graph(get_conv_model())
-    assert graph.n_nodes == 9
+    assert graph.n_nodes == 13
 
 
 def test_conv_deeper_stub():
     model = get_conv_model()
     graph = Graph(model, False)
     layer_num = graph.n_layers
-    graph.to_conv_deeper_model(4, 4)
+    graph.to_conv_deeper_model(6, 3)
 
     assert graph.n_layers == layer_num + 4
 
 
 def test_conv_deeper():
     model = get_conv_model()
     graph = Graph(model, True)
-    graph.to_conv_deeper_model(4, 3)
+    graph.to_conv_deeper_model(6, 3)
     new_model = graph.produce_model()
     input_data = get_conv_data()
 
@@ -57,15 +57,15 @@ def test_conv_wider_stub():
     model = get_add_skip_model()
     graph = Graph(model, False)
     layer_num = graph.n_layers
-    graph.to_wider_model(8, 3)
+    graph.to_wider_model(10, 3)
 
     assert graph.n_layers == layer_num
 
 
 def test_conv_wider():
     model = get_concat_skip_model()
     graph = Graph(model, True)
-    graph.to_wider_model(4, 3)
+    graph.to_wider_model(6, 3)
     new_model = graph.produce_model()
     input_data = get_conv_data()
 
@@ -101,15 +101,15 @@ def test_skip_add_over_pooling_stub():
     model = get_pooling_model()
     graph = Graph(model, False)
     layer_num = graph.n_layers
-    graph.to_add_skip_model(0, 10)
+    graph.to_add_skip_model(2, 10)
 
     assert graph.n_layers == layer_num + 2
 
 
 def test_skip_add_over_pooling():
     model = get_pooling_model()
     graph = Graph(model, True)
-    graph.to_add_skip_model(0, 10)
+    graph.to_add_skip_model(2, 10)
     new_model = graph.produce_model()
     input_data = get_conv_data()
 
@@ -123,16 +123,16 @@ def test_skip_concat_over_pooling_stub():
     model = get_pooling_model()
     graph = Graph(model, False)
     layer_num = graph.n_layers
-    graph.to_concat_skip_model(0, 13)
+    graph.to_concat_skip_model(2, 15)
 
     assert graph.n_layers == layer_num + 2
 
 
 def test_skip_concat_over_pooling():
     model = get_pooling_model()
     graph = Graph(model, True)
-    graph.to_concat_skip_model(4, 9)
-    graph.to_concat_skip_model(4, 9)
+    graph.to_concat_skip_model(6, 11)
+    graph.to_concat_skip_model(6, 11)
     graph = Graph(graph.produce_model(), True)
     new_model = graph.produce_model()
     input_data = get_conv_data()

tests/test_layer_transformer.py

@@ -5,8 +5,8 @@
 
 def test_deeper_conv_block():
     model = to_stub_model(get_conv_model(), True)
-    layers = deeper_conv_block(model.layers[1], 3)
-    assert len(layers) == constant.CONV_BLOCK_SIZE
+    layers = deeper_conv_block(model.layers[3], 3)
+    assert len(layers) == constant.CONV_BLOCK_DISTANCE + 2
 
 
 def test_dense_to_deeper_layer():
@@ -26,7 +26,10 @@ def test_dense_to_wider_layer():
 
 def test_wider_bn():
     bn_layer = StubBatchNormalization()
-    bn_layer.set_weights(get_conv_model().layers[2].get_weights())
+    bn_layer.set_weights([np.ones(3, dtype=np.float32),
+                          np.zeros(3, dtype=np.float32),
+                          np.zeros(3, dtype=np.float32),
+                          np.ones(3, dtype=np.float32)])
     new_bn_layer = wider_bn(bn_layer, 1, 3, 4)
     assert new_bn_layer.get_weights()[0].shape[0] == 7
 
@@ -49,6 +52,6 @@ def test_wider_next_dense():
 def test_wider_conv():
     model = to_stub_model(get_conv_model(), True)
 
-    assert isinstance(wider_pre_conv(model.layers[1], 3), StubConv)
-    assert isinstance(wider_bn(model.layers[2], 3, 3, 3), StubBatchNormalization)
-    assert isinstance(wider_next_conv(model.layers[5], 3, 3, 3), StubConv)
+    assert isinstance(wider_pre_conv(model.layers[3], 3), StubConv)
+    assert isinstance(wider_bn(model.layers[1], 3, 3, 3), StubBatchNormalization)
+    assert isinstance(wider_next_conv(model.layers[7], 3, 3, 3), StubConv)

tests/test_net_transformer.py

@@ -31,7 +31,7 @@ def test_transform():
 
 def test_legal_graph():
     graph = Graph(get_pooling_model(), False)
-    graph.to_add_skip_model(0, 4)
+    graph.to_add_skip_model(2, 6)
     assert legal_graph(graph)
-    graph.to_add_skip_model(0, 4)
+    graph.to_add_skip_model(2, 6)
     assert not legal_graph(graph)