From 58b863192a390baa79fa2c799435eefac91f6b8b Mon Sep 17 00:00:00 2001 From: Haifeng Jin Date: Wed, 27 Jun 2018 15:01:57 -0500 Subject: [PATCH] delete weighted add --- autokeras/graph.py | 56 +++++++++++++++++++++++++++----------------- autokeras/layers.py | 4 ++-- tests/common.py | 7 +++--- tests/test_graph.py | 13 ++++------ tests/test_layers.py | 16 +------------ 5 files changed, 45 insertions(+), 51 deletions(-) diff --git a/autokeras/graph.py b/autokeras/graph.py index 43c75291f..6d9107cca 100644 --- a/autokeras/graph.py +++ b/autokeras/graph.py @@ -5,7 +5,7 @@ from keras import Input from keras.engine import Model -from keras.layers import Concatenate, Dropout, Activation +from keras.layers import Concatenate, Dropout, Activation, Add from autokeras import constant from autokeras.layer_transformer import wider_bn, wider_next_conv, wider_next_dense, wider_pre_dense, wider_pre_conv, \ @@ -51,7 +51,7 @@ class Graph: Graph extracts the neural architecture graph from a Keras model. Each node in the graph is a intermediate tensor between layers. Each layer is an edge in the graph. - Notably, multiple edges may refer to the same layer. (e.g. WeightedAdd layer is adding + Notably, multiple edges may refer to the same layer. (e.g. Add layer is adding two tensor into one tensor. So it is related to two edges.) Attributes: @@ -400,19 +400,32 @@ def to_add_skip_model(self, start_id, end_id): dropout_input_id = self._conv_block_end_node(end_id) # Add the pooling layer chain. - pooling_layer_list = self._get_pooling_layers(conv_block_input_id, dropout_input_id) + layer_list = self._get_pooling_layers(conv_block_input_id, dropout_input_id) skip_output_id = conv_block_input_id - for index, layer_id in enumerate(pooling_layer_list): + for index, layer_id in enumerate(layer_list): layer = self.layer_list[layer_id] new_node_id = self._add_new_node() self._add_edge(deepcopy(layer), skip_output_id, new_node_id) skip_output_id = new_node_id - # Add the weighted add layer. + # Add the conv layer + layer2 = StubConv(self.layer_list[end_id].filters, 1, self.layer_list[end_id].func) new_node_id = self._add_new_node() - layer = StubAdd() + self._add_edge(layer2, skip_output_id, new_node_id) + skip_output_id = new_node_id + + # Set weights to the additional conv layer. if self.weighted: - layer.set_weights([np.float32(1.0)]) + filters_end = self.layer_list[end_id].filters + filters_start = self.layer_list[start_id].filters + filter_shape = (1,) * (len(self.layer_list[end_id].get_weights()[0].shape) - 2) + weights = np.zeros(filter_shape + (filters_start, filters_end)) + bias = np.zeros(filters_end) + layer2.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) + + # Add the add layer. + new_node_id = self._add_new_node() + layer = StubAdd() dropout_output_id = self.adj_list[dropout_input_id][0][0] self._redirect_edge(dropout_input_id, dropout_output_id, new_node_id) @@ -446,19 +459,18 @@ def to_concat_skip_model(self, start_id, end_id): new_node_id2 = self._add_new_node() layer2 = StubConv(self.layer_list[end_id].filters, 1, self.layer_list[end_id].func) if self.weighted: - if self.weighted: - filters_end = self.layer_list[end_id].filters - filters_start = self.layer_list[start_id].filters - filter_shape = (1,) * (len(self.layer_list[end_id].get_weights()[0].shape) - 2) - weights = np.zeros(filter_shape + (filters_end, filters_end)) - for i in range(filters_end): - filter_weight = np.zeros(filter_shape + (filters_end,)) - filter_weight[(0, 0, i)] = 1 - weights[..., i] = filter_weight - weights = np.concatenate((weights, - np.zeros(filter_shape + (filters_start, filters_end))), axis=2) - bias = np.zeros(filters_end) - layer2.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) + filters_end = self.layer_list[end_id].filters + filters_start = self.layer_list[start_id].filters + filter_shape = (1,) * (len(self.layer_list[end_id].get_weights()[0].shape) - 2) + weights = np.zeros(filter_shape + (filters_end, filters_end)) + for i in range(filters_end): + filter_weight = np.zeros(filter_shape + (filters_end,)) + filter_weight[(0, 0, i)] = 1 + weights[..., i] = filter_weight + weights = np.concatenate((weights, + np.zeros(filter_shape + (filters_start, filters_end))), axis=2) + bias = np.zeros(filters_end) + layer2.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) dropout_output_id = self.adj_list[dropout_input_id][0][0] self._redirect_edge(dropout_input_id, dropout_output_id, new_node_id) @@ -502,7 +514,7 @@ def extract_descriptor(self): layer = self.layer_list[layer_id] if is_layer(layer, 'Concatenate'): ret.add_skip_connection(pos[u], pos[v], NetworkDescriptor.CONCAT_CONNECT) - if is_layer(layer, 'WeightedAdd'): + if is_layer(layer, 'Add'): ret.add_skip_connection(pos[u], pos[v], NetworkDescriptor.ADD_CONNECT) return ret @@ -540,7 +552,7 @@ def produce_model(self): node_to_id[temp_tensor] = v model = Model(input_tensor, node_list[output_id]) for layer in model.layers[1:]: - if not isinstance(layer, (Activation, Dropout, Concatenate)): + if not isinstance(layer, (Activation, Dropout, Concatenate, Add)): old_layer = new_to_old_layer[layer] if self.weighted: layer.set_weights(old_layer.get_weights()) diff --git a/autokeras/layers.py b/autokeras/layers.py index 83e72445c..f15676857 100644 --- a/autokeras/layers.py +++ b/autokeras/layers.py @@ -183,7 +183,7 @@ def to_real_layer(layer): return BatchNormalization() if is_layer(layer, 'Concatenate'): return Concatenate() - if is_layer(layer, 'WeightedAdd'): + if is_layer(layer, 'Add'): return Add() if is_layer(layer, 'Dropout'): return Dropout(layer.rate) @@ -200,7 +200,7 @@ def to_stub_layer(layer, input_id, output_id): temp_stub_layer = StubConv(layer.filters, layer.kernel_size, layer.__class__, input_id, output_id) elif is_layer(layer, 'Dense'): temp_stub_layer = StubDense(layer.units, layer.activation, input_id, output_id) - elif is_layer(layer, 'WeightedAdd'): + elif is_layer(layer, 'Add'): temp_stub_layer = StubAdd(input_id, output_id) elif is_layer(layer, 'Concatenate'): temp_stub_layer = StubConcatenate(input_id, output_id) diff --git a/tests/common.py b/tests/common.py index 76b958bee..030400b68 100644 --- a/tests/common.py +++ b/tests/common.py @@ -3,10 +3,9 @@ from keras import Input from keras.engine import Model from keras.layers import Conv2D, BatchNormalization, Activation, Flatten, Dense, MaxPooling2D, Concatenate, Dropout, \ - GlobalAveragePooling2D + GlobalAveragePooling2D, Add from autokeras import constant -from autokeras.layers import WeightedAdd def get_concat_skip_model(): @@ -67,14 +66,14 @@ def get_add_skip_model(): output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor) output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor) - output_tensor = WeightedAdd()([output_tensor, add_input]) + output_tensor = Add()([output_tensor, add_input]) add_input = output_tensor output_tensor = BatchNormalization()(output_tensor) output_tensor = Activation('relu')(output_tensor) output_tensor = Conv2D(3, kernel_size=(3, 3), padding='same', activation='linear')(output_tensor) output_tensor = Dropout(constant.CONV_DROPOUT_RATE)(output_tensor) - output_tensor = WeightedAdd()([output_tensor, add_input]) + output_tensor = Add()([output_tensor, add_input]) output_tensor = Flatten()(output_tensor) output_tensor = Dense(5, activation='relu')(output_tensor) output_tensor = Dropout(constant.DENSE_DROPOUT_RATE)(output_tensor) diff --git a/tests/test_graph.py b/tests/test_graph.py index 15ac37d60..f66fe6dd9 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -105,22 +105,22 @@ def test_skip_add_over_pooling_stub(): model = get_pooling_model() graph = Graph(model, False) layer_num = graph.n_layers - graph.to_add_skip_model(2, 10) + graph.to_add_skip_model(2, 11) - assert graph.n_layers == layer_num + 2 + assert graph.n_layers == layer_num + 3 def test_skip_add_over_pooling(): model = get_pooling_model() graph = Graph(model, True) - graph.to_add_skip_model(2, 10) + graph.to_add_skip_model(2, 11) new_model = graph.produce_model() input_data = get_conv_data() output1 = model.predict_on_batch(input_data).flatten() output2 = new_model.predict_on_batch(input_data).flatten() - assert np.array_equal(output1, output2) + assert np.sum(np.abs(output1 - output2)) < 1e-4 def test_skip_concat_over_pooling_stub(): @@ -187,10 +187,7 @@ def test_long_transform(): history = [('to_wider_model', 2, 256), ('to_conv_deeper_model', 2, 3), ('to_concat_skip_model', 23, 7)] for args in history: - print(args) - if args[1] == 23: - print('now') getattr(graph, args[0])(*list(args[1:])) graph.produce_model() - # assert not legal_graph(graph) + assert legal_graph(graph) diff --git a/tests/test_layers.py b/tests/test_layers.py index 37127f5b0..6997de7ee 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -9,23 +9,9 @@ from tests.common import get_add_skip_model, clean_dir -def test_weighted_add(): - a1 = Input(shape=(3, 3, 2)) - a2 = Input(shape=(3, 3, 2)) - layer = WeightedAdd() - b = layer([a1, a2]) - model = Model(inputs=[a1, a2], outputs=b) - data = np.ones((1, 3, 3, 2)) - model.compile(optimizer='Adam', loss=mean_squared_error) - model.fit([data, data * 2], data * 2, epochs=10, verbose=False) - results = model.predict_on_batch([data, data * 2]) - assert not np.array_equal(results, data) - assert constant_value(layer.one) == 1 - - def test_save_weighted_add(): model = get_add_skip_model() path = 'tests/resources/temp/m.h5' model.save(path) - load_model(path, {'WeightedAdd': WeightedAdd}) + load_model(path) os.remove(path)