delete weighted add

autokeras/graph.py

@@ -5,7 +5,7 @@
 
 from keras import Input
 from keras.engine import Model
-from keras.layers import Concatenate, Dropout, Activation
+from keras.layers import Concatenate, Dropout, Activation, Add
 
 from autokeras import constant
 from autokeras.layer_transformer import wider_bn, wider_next_conv, wider_next_dense, wider_pre_dense, wider_pre_conv, \
@@ -51,7 +51,7 @@ class Graph:
     Graph extracts the neural architecture graph from a Keras model. Each node in the graph
     is a intermediate tensor between layers. Each layer is an edge in the graph.
 
-    Notably, multiple edges may refer to the same layer. (e.g. WeightedAdd layer is adding
+    Notably, multiple edges may refer to the same layer. (e.g. Add layer is adding
     two tensor into one tensor. So it is related to two edges.)
 
     Attributes:
@@ -400,19 +400,32 @@ def to_add_skip_model(self, start_id, end_id):
         dropout_input_id = self._conv_block_end_node(end_id)
 
         # Add the pooling layer chain.
-        pooling_layer_list = self._get_pooling_layers(conv_block_input_id, dropout_input_id)
+        layer_list = self._get_pooling_layers(conv_block_input_id, dropout_input_id)
         skip_output_id = conv_block_input_id
-        for index, layer_id in enumerate(pooling_layer_list):
+        for index, layer_id in enumerate(layer_list):
             layer = self.layer_list[layer_id]
             new_node_id = self._add_new_node()
             self._add_edge(deepcopy(layer), skip_output_id, new_node_id)
             skip_output_id = new_node_id
 
-        # Add the weighted add layer.
+        # Add the conv layer
+        layer2 = StubConv(self.layer_list[end_id].filters, 1, self.layer_list[end_id].func)
         new_node_id = self._add_new_node()
-        layer = StubAdd()
+        self._add_edge(layer2, skip_output_id, new_node_id)
+        skip_output_id = new_node_id
+
+        # Set weights to the additional conv layer.
         if self.weighted:
-            layer.set_weights([np.float32(1.0)])
+            filters_end = self.layer_list[end_id].filters
+            filters_start = self.layer_list[start_id].filters
+            filter_shape = (1,) * (len(self.layer_list[end_id].get_weights()[0].shape) - 2)
+            weights = np.zeros(filter_shape + (filters_start, filters_end))
+            bias = np.zeros(filters_end)
+            layer2.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1]))))
+
+        # Add the add layer.
+        new_node_id = self._add_new_node()
+        layer = StubAdd()
 
         dropout_output_id = self.adj_list[dropout_input_id][0][0]
         self._redirect_edge(dropout_input_id, dropout_output_id, new_node_id)
@@ -446,19 +459,18 @@ def to_concat_skip_model(self, start_id, end_id):
         new_node_id2 = self._add_new_node()
         layer2 = StubConv(self.layer_list[end_id].filters, 1, self.layer_list[end_id].func)
         if self.weighted:
-            if self.weighted:
-                filters_end = self.layer_list[end_id].filters
-                filters_start = self.layer_list[start_id].filters
-                filter_shape = (1,) * (len(self.layer_list[end_id].get_weights()[0].shape) - 2)
-                weights = np.zeros(filter_shape + (filters_end, filters_end))
-                for i in range(filters_end):
-                    filter_weight = np.zeros(filter_shape + (filters_end,))
-                    filter_weight[(0, 0, i)] = 1
-                    weights[..., i] = filter_weight
-                weights = np.concatenate((weights,
-                                          np.zeros(filter_shape + (filters_start, filters_end))), axis=2)
-                bias = np.zeros(filters_end)
-                layer2.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1]))))
+            filters_end = self.layer_list[end_id].filters
+            filters_start = self.layer_list[start_id].filters
+            filter_shape = (1,) * (len(self.layer_list[end_id].get_weights()[0].shape) - 2)
+            weights = np.zeros(filter_shape + (filters_end, filters_end))
+            for i in range(filters_end):
+                filter_weight = np.zeros(filter_shape + (filters_end,))
+                filter_weight[(0, 0, i)] = 1
+                weights[..., i] = filter_weight
+            weights = np.concatenate((weights,
+                                      np.zeros(filter_shape + (filters_start, filters_end))), axis=2)
+            bias = np.zeros(filters_end)
+            layer2.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1]))))
 
         dropout_output_id = self.adj_list[dropout_input_id][0][0]
         self._redirect_edge(dropout_input_id, dropout_output_id, new_node_id)
@@ -502,7 +514,7 @@ def extract_descriptor(self):
                 layer = self.layer_list[layer_id]
                 if is_layer(layer, 'Concatenate'):
                     ret.add_skip_connection(pos[u], pos[v], NetworkDescriptor.CONCAT_CONNECT)
-                if is_layer(layer, 'WeightedAdd'):
+                if is_layer(layer, 'Add'):
                     ret.add_skip_connection(pos[u], pos[v], NetworkDescriptor.ADD_CONNECT)
 
         return ret
@@ -540,7 +552,7 @@ def produce_model(self):
                 node_to_id[temp_tensor] = v
         model = Model(input_tensor, node_list[output_id])
         for layer in model.layers[1:]:
-            if not isinstance(layer, (Activation, Dropout, Concatenate)):
+            if not isinstance(layer, (Activation, Dropout, Concatenate, Add)):
                 old_layer = new_to_old_layer[layer]
                 if self.weighted:
                     layer.set_weights(old_layer.get_weights())

autokeras/layers.py

@@ -183,7 +183,7 @@ def to_real_layer(layer):
         return BatchNormalization()
     if is_layer(layer, 'Concatenate'):
         return Concatenate()
-    if is_layer(layer, 'WeightedAdd'):
+    if is_layer(layer, 'Add'):
         return Add()
     if is_layer(layer, 'Dropout'):
         return Dropout(layer.rate)
@@ -200,7 +200,7 @@ def to_stub_layer(layer, input_id, output_id):
         temp_stub_layer = StubConv(layer.filters, layer.kernel_size, layer.__class__, input_id, output_id)
     elif is_layer(layer, 'Dense'):
         temp_stub_layer = StubDense(layer.units, layer.activation, input_id, output_id)
-    elif is_layer(layer, 'WeightedAdd'):
+    elif is_layer(layer, 'Add'):
         temp_stub_layer = StubAdd(input_id, output_id)
     elif is_layer(layer, 'Concatenate'):
         temp_stub_layer = StubConcatenate(input_id, output_id)

tests/common.py

@@ -3,10 +3,9 @@
 from keras import Input
 from keras.engine import Model
 from keras.layers import Conv2D, BatchNormalization, Activation, Flatten, Dense, MaxPooling2D, Concatenate, Dropout, \
-    GlobalAveragePooling2D
+    GlobalAveragePooling2D, Add
 
 from autokeras import constant
-from autokeras.layers import WeightedAdd
 
 
 def get_concat_skip_model():
@@ -67,14 +66,14 @@ def get_add_skip_model():
     output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
-    output_tensor = WeightedAdd()([output_tensor, add_input])
+    output_tensor = Add()([output_tensor, add_input])
     add_input = output_tensor
     output_tensor = BatchNormalization()(output_tensor)
     output_tensor = Activation('relu')(output_tensor)
     output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor)
     output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor)
 
-    output_tensor = WeightedAdd()([output_tensor, add_input])
+    output_tensor = Add()([output_tensor, add_input])
     output_tensor = Flatten()(output_tensor)
     output_tensor = Dense(5, activation='relu')(output_tensor)
     output_tensor = Dropout(constant.DENSE_DROPOUT_RATE)(output_tensor)

tests/test_graph.py

@@ -105,22 +105,22 @@ def test_skip_add_over_pooling_stub():
     model = get_pooling_model()
     graph = Graph(model, False)
     layer_num = graph.n_layers
-    graph.to_add_skip_model(2, 10)
+    graph.to_add_skip_model(2, 11)
 
-    assert graph.n_layers == layer_num + 2
+    assert graph.n_layers == layer_num + 3
 
 
 def test_skip_add_over_pooling():
     model = get_pooling_model()
     graph = Graph(model, True)
-    graph.to_add_skip_model(2, 10)
+    graph.to_add_skip_model(2, 11)
     new_model = graph.produce_model()
     input_data = get_conv_data()
 
     output1 = model.predict_on_batch(input_data).flatten()
     output2 = new_model.predict_on_batch(input_data).flatten()
 
-    assert np.array_equal(output1, output2)
+    assert np.sum(np.abs(output1 - output2)) < 1e-4
 
 
 def test_skip_concat_over_pooling_stub():
@@ -187,10 +187,7 @@ def test_long_transform():
     history = [('to_wider_model', 2, 256), ('to_conv_deeper_model', 2, 3),
                ('to_concat_skip_model', 23, 7)]
     for args in history:
-        print(args)
-        if args[1] == 23:
-            print('now')
         getattr(graph, args[0])(*list(args[1:]))
         graph.produce_model()
-    # assert not legal_graph(graph)
+    assert legal_graph(graph)
 

tests/test_layers.py

@@ -9,23 +9,9 @@
 from tests.common import get_add_skip_model, clean_dir
 
 
-def test_weighted_add():
-    a1 = Input(shape=(3, 3, 2))
-    a2 = Input(shape=(3, 3, 2))
-    layer = WeightedAdd()
-    b = layer([a1, a2])
-    model = Model(inputs=[a1, a2], outputs=b)
-    data = np.ones((1, 3, 3, 2))
-    model.compile(optimizer='Adam', loss=mean_squared_error)
-    model.fit([data, data * 2], data * 2, epochs=10, verbose=False)
-    results = model.predict_on_batch([data, data * 2])
-    assert not np.array_equal(results, data)
-    assert constant_value(layer.one) == 1
-
-
 def test_save_weighted_add():
     model = get_add_skip_model()
     path = 'tests/resources/temp/m.h5'
     model.save(path)
-    load_model(path, {'WeightedAdd': WeightedAdd})
+    load_model(path)
     os.remove(path)