Skip to content

Commit

Permalink
Modified layer connections. Get rid of old modules.
Browse files Browse the repository at this point in the history
  • Loading branch information
Yurii Shevchuk committed Nov 27, 2016
1 parent 2c6f295 commit 9d2a34f
Show file tree
Hide file tree
Showing 90 changed files with 1,088 additions and 2,046 deletions.
89 changes: 43 additions & 46 deletions examples/autoencoder/stacked_conv_autoencoders.py
View file
Expand Up @@ -5,7 +5,7 @@
from sklearn import datasets, metrics
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
from neupy import algorithms, layers, environment, surgery
from neupy import algorithms, layers, environment


environment.reproducible()
Expand All @@ -32,85 +32,82 @@
x_labeled_4d = x_labeled.reshape((n_labeled, 1, 28, 28))
x_unlabeled_4d = x_unlabeled.reshape((n_unlabeled, 1, 28, 28))

conv_autoencoder = algorithms.Momentum(
[
layers.Input((1, 28, 28)),
encoder = layers.join(
layers.Input((1, 28, 28)),

layers.Convolution((16, 3, 3)) > layers.Relu(),
layers.Convolution((16, 3, 3)) > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Convolution((16, 3, 3)) > layers.Relu(),
layers.Convolution((16, 3, 3)) > layers.Relu(),
layers.MaxPooling((2, 2)),

layers.Convolution((32, 3, 3)) > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Convolution((32, 3, 3)) > layers.Relu(),
layers.MaxPooling((2, 2)),

layers.Reshape(),
layers.Reshape(),

layers.Relu(512),
layers.Relu(256),
surgery.CutLine(),
layers.Relu(256),
layers.Relu(128),
)

layers.Relu(512),
layers.Relu(800),
decoder = layers.join(
layers.Relu(256),
layers.Relu(32 * 5 * 5),

layers.Reshape((32, 5, 5)),
layers.Reshape((32, 5, 5)),

layers.Upscale((2, 2)),
layers.Convolution((16, 3, 3), padding='full') > layers.Relu(),
layers.Upscale((2, 2)),
layers.Convolution((16, 3, 3), padding='full') > layers.Relu(),

layers.Upscale((2, 2)),
layers.Convolution((16, 3, 3), padding='full') > layers.Relu(),
layers.Convolution((1, 3, 3), padding='full') > layers.Sigmoid(),
layers.Upscale((2, 2)),
layers.Convolution((16, 3, 3), padding='full') > layers.Relu(),
layers.Convolution((1, 3, 3), padding='full') > layers.Sigmoid(),

layers.Reshape(),
],
layers.Reshape(),
)

conv_autoencoder = algorithms.Momentum(
connection=encoder > decoder,
verbose=True,
step=0.1,
momentum=0.99,
shuffle_data=True,
batch_size=128,
batch_size=64,
error='binary_crossentropy',
)
conv_autoencoder.architecture()
conv_autoencoder.train(x_unlabeled_4d, x_unlabeled,
x_labeled_4d, x_labeled, epochs=10)

classifier_structure, _ = surgery.cut_along_lines(conv_autoencoder)
x_labeled_encoded = encoder.output(x_labeled_4d).eval()
x_unlabeled_encoded = encoder.output(x_unlabeled_4d).eval()

x_labeled_encoded = classifier_structure.output(x_labeled_4d).eval()
x_unlabeled_encoded = classifier_structure.output(x_unlabeled_4d).eval()
classifier_network = layers.join(
layers.PRelu(512),
layers.Dropout(0.25),
layers.Softmax(10),
)

linear_classifier = algorithms.Adadelta(
[
layers.Input(classifier_structure.output_shape),
layers.PRelu(512),
layers.Dropout(0.25),
layers.Softmax(10),
],
encoder_classifier = algorithms.Adadelta(
layers.Input(encoder.output_shape) > classifier_network,
verbose=True,
step=0.05,
shuffle_data=True,
batch_size=128,
batch_size=64,
error='categorical_crossentropy',
)
linear_classifier.architecture()
linear_classifier.train(x_labeled_encoded, y_labeled,
x_unlabeled_encoded, y_unlabeled, epochs=100)

classification_layer = surgery.cut(linear_classifier, start=1, end=4)
classifier_structure = surgery.sew_together([classifier_structure,
classification_layer])
encoder_classifier.architecture()
encoder_classifier.train(x_labeled_encoded, y_labeled,
x_unlabeled_encoded, y_unlabeled, epochs=100)

classifier = algorithms.MinibatchGradientDescent(
classifier_structure,
encoder > classifier_network,
verbose=True,
step=0.1,
step=0.01,
shuffle_data=True,
batch_size=128,
batch_size=64,
error='categorical_crossentropy',
)
classifier.architecture()
classifier.train(x_labeled_4d, y_labeled, epochs=1000)
classifier.train(x_labeled_4d, y_labeled, epochs=100)

unlabeled_predicted = classifier.predict(x_unlabeled_4d).argmax(axis=1)
y_unlabeled_classes = np.asarray(y_unlabeled).argmax(axis=1)
Expand Down
69 changes: 33 additions & 36 deletions examples/cnn/alexnet.py
View file
Expand Up @@ -72,47 +72,44 @@ def __repr__(self):
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.LocalResponseNorm(),

layers.Parallel(
[[
SliceChannels(0, 48),
layers.Convolution((128, 5, 5), padding=2, **conv_2_1),
layers.Relu(),
], [
SliceChannels(48, 96),
layers.Convolution((128, 5, 5), padding=2, **conv_2_2),
layers.Relu(),
]],
layers.Concatenate(),
),
[[
SliceChannels(0, 48),
layers.Convolution((128, 5, 5), padding=2, **conv_2_1),
layers.Relu(),
], [
SliceChannels(48, 96),
layers.Convolution((128, 5, 5), padding=2, **conv_2_2),
layers.Relu(),
]],
layers.Concatenate(),

layers.MaxPooling((3, 3), stride=(2, 2)),
layers.LocalResponseNorm(),

layers.Convolution((384, 3, 3), padding=1, **conv_3) > layers.Relu(),

layers.Parallel(
[[
SliceChannels(0, 192),
layers.Convolution((192, 3, 3), padding=1, **conv_4_1),
layers.Relu(),
], [
SliceChannels(192, 384),
layers.Convolution((192, 3, 3), padding=1, **conv_4_2),
layers.Relu(),
]],
layers.Concatenate(),
),
layers.Parallel(
[[
SliceChannels(0, 192),
layers.Convolution((128, 3, 3), padding=1, **conv_5_1),
layers.Relu(),
], [
SliceChannels(192, 384),
layers.Convolution((128, 3, 3), padding=1, **conv_5_2),
layers.Relu(),
]],
layers.Concatenate(),
),
[[
SliceChannels(0, 192),
layers.Convolution((192, 3, 3), padding=1, **conv_4_1),
layers.Relu(),
], [
SliceChannels(192, 384),
layers.Convolution((192, 3, 3), padding=1, **conv_4_2),
layers.Relu(),
]],
layers.Concatenate(),

[[
SliceChannels(0, 192),
layers.Convolution((128, 3, 3), padding=1, **conv_5_1),
layers.Relu(),
], [
SliceChannels(192, 384),
layers.Convolution((128, 3, 3), padding=1, **conv_5_2),
layers.Relu(),
]],
layers.Concatenate(),

layers.MaxPooling((3, 3), stride=(2, 2)),

layers.Reshape(),
Expand Down
4 changes: 1 addition & 3 deletions examples/cnn/googlenet.py
View file
Expand Up @@ -7,7 +7,7 @@


def Inception(nfilters):
return layers.Parallel(
return layers.join(
[[
layers.MaxPooling((3, 3), stride=1, padding=(1, 1)),
layers.Convolution((nfilters[0], 1, 1)),
Expand Down Expand Up @@ -60,6 +60,4 @@ def Inception(nfilters):

layers.Softmax(1000),
)
googlenet.initialize()

plots.layer_structure(googlenet)
90 changes: 39 additions & 51 deletions examples/cnn/inception_v3.py
View file
Expand Up @@ -12,7 +12,7 @@ def Conv_Relu_BatchNorm(*conv_args, **conv_kwargs):


def Inception_1(conv_filters):
return layers.Parallel(
return layers.join(
[[
Conv_Relu_BatchNorm((conv_filters[0][0], 1, 1)),
], [
Expand All @@ -32,7 +32,7 @@ def Inception_1(conv_filters):


def Inception_2(conv_filters):
return layers.Parallel(
return layers.join(
[[
Conv_Relu_BatchNorm((conv_filters[0][0], 1, 1)),
], [
Expand Down Expand Up @@ -64,34 +64,28 @@ def Inception_3(conv_filters, pooling):
elif pooling == 'average':
Pooling = partial(layers.AveragePooling, mode='exclude_padding')

return layers.Parallel(
return layers.join(
[[
Conv_Relu_BatchNorm((conv_filters[0][0], 1, 1)),
], [
Conv_Relu_BatchNorm((conv_filters[1][0], 1, 1)),
layers.Parallel(
[[
Conv_Relu_BatchNorm((conv_filters[1][1], 1, 3),
padding=(0, 1)),
], [
Conv_Relu_BatchNorm((conv_filters[1][2], 3, 1),
padding=(1, 0)),
]],
layers.Concatenate(),
)
[[
Conv_Relu_BatchNorm((conv_filters[1][1], 1, 3),
padding=(0, 1)),
], [
Conv_Relu_BatchNorm((conv_filters[1][2], 3, 1),
padding=(1, 0)),
]],
], [
Conv_Relu_BatchNorm((conv_filters[2][0], 1, 1)),
Conv_Relu_BatchNorm((conv_filters[2][1], 3, 3), padding=1),
layers.Parallel(
[[
Conv_Relu_BatchNorm((conv_filters[2][2], 1, 3),
padding=(0, 1)),
], [
Conv_Relu_BatchNorm((conv_filters[2][3], 3, 1),
padding=(1, 0)),
]],
layers.Concatenate(),
)
[[
Conv_Relu_BatchNorm((conv_filters[2][2], 1, 3),
padding=(0, 1)),
], [
Conv_Relu_BatchNorm((conv_filters[2][3], 3, 1),
padding=(1, 0)),
]],
], [
Pooling((3, 3), stride=(1, 1), padding=1),
Conv_Relu_BatchNorm((conv_filters[3][0], 1, 1)),
Expand All @@ -116,39 +110,35 @@ def Inception_3(conv_filters, pooling):
Inception_1([[64], [48, 64], [64, 96, 96], [64]]),
Inception_1([[64], [48, 64], [64, 96, 96], [64]]),

layers.Parallel(
[[
Conv_Relu_BatchNorm((384, 3, 3), stride=2),
], [
Conv_Relu_BatchNorm((64, 1, 1)),
Conv_Relu_BatchNorm((96, 3, 3), padding=1),
Conv_Relu_BatchNorm((96, 3, 3), stride=2),
[[
Conv_Relu_BatchNorm((384, 3, 3), stride=2),
], [
Conv_Relu_BatchNorm((64, 1, 1)),
Conv_Relu_BatchNorm((96, 3, 3), padding=1),
Conv_Relu_BatchNorm((96, 3, 3), stride=2),

], [
layers.MaxPooling((3, 3), stride=(2, 2))
]],
layers.Concatenate(),
),
], [
layers.MaxPooling((3, 3), stride=(2, 2))
]],
layers.Concatenate(),

Inception_2([[192], [128, 128, 192], [128, 128, 128, 128, 192], [192]]),
Inception_2([[192], [160, 160, 192], [160, 160, 160, 160, 192], [192]]),
Inception_2([[192], [160, 160, 192], [160, 160, 160, 160, 192], [192]]),
Inception_2([[192], [192, 192, 192], [192, 192, 192, 192, 192], [192]]),

layers.Parallel(
[[
Conv_Relu_BatchNorm((192, 1, 1)),
Conv_Relu_BatchNorm((320, 3, 3), stride=2),
], [
Conv_Relu_BatchNorm((192, 1, 1)),
Conv_Relu_BatchNorm((192, 1, 7), padding=(0, 3)),
Conv_Relu_BatchNorm((192, 7, 1), padding=(3, 0)),
Conv_Relu_BatchNorm((192, 3, 3), stride=2),
], [
layers.MaxPooling((3, 3), stride=(2, 2))
]],
layers.Concatenate(),
),
[[
Conv_Relu_BatchNorm((192, 1, 1)),
Conv_Relu_BatchNorm((320, 3, 3), stride=2),
], [
Conv_Relu_BatchNorm((192, 1, 1)),
Conv_Relu_BatchNorm((192, 1, 7), padding=(0, 3)),
Conv_Relu_BatchNorm((192, 7, 1), padding=(3, 0)),
Conv_Relu_BatchNorm((192, 3, 3), stride=2),
], [
layers.MaxPooling((3, 3), stride=(2, 2))
]],
layers.Concatenate(),

Inception_3([[320], [384, 384, 384], [448, 384, 384, 384], [192]],
pooling='average'),
Expand All @@ -158,6 +148,4 @@ def Inception_3(conv_filters, pooling):
layers.GlobalPooling(),
layers.Softmax(1000),
)
inception_v3.initialize()

plots.layer_structure(inception_v3)
4 changes: 1 addition & 3 deletions examples/cnn/resnet50.py
View file
Expand Up @@ -23,7 +23,7 @@ def ResidualUnit(n_in_filters, n_out_filters, stride, has_branch=False):
layers.BatchNorm(),
)

return layers.Parallel(
return layers.join(
[main_branch, residual_branch],
layers.Elementwise() > layers.Relu(),
)
Expand Down Expand Up @@ -62,6 +62,4 @@ def ResidualUnit(n_in_filters, n_out_filters, stride, has_branch=False):
layers.Reshape(),
layers.Softmax(1000),
)
resnet50.initialize()

plots.layer_structure(resnet50)

0 comments on commit 9d2a34f

Please sign in to comment.