Skip to content

Commit

Permalink
Pushing new optimizer
Browse files Browse the repository at this point in the history
  • Loading branch information
Ragav Venkatesan committed Mar 26, 2017
1 parent 7c86f50 commit ae74f7b
Show file tree
Hide file tree
Showing 7 changed files with 532 additions and 229 deletions.
13 changes: 6 additions & 7 deletions pantry/tutorials/autoencoder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
* Need a validation and testing thats better than just measuring rmse. Can't find something
great.
* Loss increases after 3 epochs.
"""
from yann.network import network
Expand Down Expand Up @@ -73,15 +72,15 @@ def autoencoder ( dataset= None, verbose = 1 ):
origin = "flatten",
id = "encoder",
num_neurons = 64,
activation = 'relu',
activation = 'tanh',
verbose = verbose
)

net.add_layer ( type = "dot_product",
origin = "encoder",
id = "decoder",
num_neurons = 784,
activation = 'relu',
activation = 'tanh',
input_params = [net.dropout_layers['encoder'].w.T, None],
# Use the same weights but transposed for decoder.
learnable = False,
Expand Down Expand Up @@ -307,7 +306,7 @@ def convolutional_autoencoder ( dataset= None, verbose = 1 ):
verbose = verbose
)

learning_rates = (0, 0.01, 0.001)
learning_rates = (0.04, 0.0001, 0.00001)
net.cook( objective_layers = ['obj'],
datastream = 'data',
learning_rates = learning_rates,
Expand All @@ -329,7 +328,7 @@ def convolutional_autoencoder ( dataset= None, verbose = 1 ):
dataset = None
if len(sys.argv) > 1:
if sys.argv[1] == 'create_dataset':
from yann.special.datasets import cook_mnist_normalized as cook_mnist
from yann.special.datasets import cook_mnist_normalized_zero_mean as cook_mnist
data = cook_mnist (verbose = 2)
dataset = data.dataset_location()
else:
Expand All @@ -339,9 +338,9 @@ def convolutional_autoencoder ( dataset= None, verbose = 1 ):

if dataset is None:
print " creating a new dataset to run through"
from yann.special.datasets import cook_mnist_normalized as cook_mnist
from yann.special.datasets import cook_mnist_normalized_zero_mean as cook_mnist
data = cook_mnist (verbose = 2)
dataset = data.dataset_location()

#autoencoder ( dataset, verbose = 2 )
# autoencoder ( dataset, verbose = 2 )
convolutional_autoencoder ( dataset , verbose = 2 )
22 changes: 11 additions & 11 deletions pantry/tutorials/gan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ def deep_gan_mnist (dataset, verbose = 1 ):
"momentum_type" : 'polyak',
"momentum_params" : (0.5, 0.7, 20),
"regularization" : (0.000, 0.000),
"optimizer_type" : 'rmsprop',
"optimizer_type" : 'adagrad',
"id" : "main"
}

Expand Down Expand Up @@ -270,7 +270,7 @@ def deep_gan_mnist (dataset, verbose = 1 ):
#z - latent space created by random layer
net.add_layer(type = 'random',
id = 'z',
num_neurons = (100,10),
num_neurons = (100,32),
distribution = 'normal',
mu = 0,
sigma = 1,
Expand Down Expand Up @@ -455,7 +455,7 @@ def deep_gan_mnist (dataset, verbose = 1 ):

net.train( epochs = (20),
k = 1,
pre_train_discriminator = 0,
pre_train_discriminator = 3,
validate_after_epochs = 1,
visualize_after_epochs = 1,
training_accuracy = True,
Expand Down Expand Up @@ -491,10 +491,10 @@ def deep_deconvolutional_gan(dataset,
print (".. Creating a GAN network")

optimizer_params = {
"momentum_type" : 'polyak',
"momentum_type" : 'false',
"momentum_params" : (0.51, 0.95, 40),
"regularization" : (0.00001, 0.00001),
"optimizer_type" : 'adagrad',
"optimizer_type" : 'adam',
"id" : "main"
}

Expand Down Expand Up @@ -789,7 +789,7 @@ def deep_deconvolutional_gan(dataset,
game_layers = ("D(x)", "D(G(z))"),
verbose = verbose )

learning_rates = (0.0004, 0.001 )
learning_rates = (0.0004, 0.0001 )

net.train( epochs = (20),
k = 1,
Expand Down Expand Up @@ -1131,10 +1131,10 @@ def deep_deconvolutional_lsgan(dataset,
game_layers = ("D(x)", "D(G(z))"),
verbose = verbose )

learning_rates = (0.04, 0.01 )
learning_rates = (0.04, 0.001 )

net.train( epochs = (20),
k = 2,
k = 1,
pre_train_discriminator = 0,
validate_after_epochs = 1,
visualize_after_epochs = 1,
Expand All @@ -1148,7 +1148,7 @@ def deep_deconvolutional_lsgan(dataset,

if __name__ == '__main__':

#from yann.special.datasets import cook_mnist_normalized_zero_mean as c
# from yann.special.datasets import cook_mnist_normalized_zero_mean as c
from yann.special.datasets import cook_cifar10_normalized_zero_mean as c
import sys

Expand All @@ -1169,11 +1169,11 @@ def deep_deconvolutional_lsgan(dataset,

# net = shallow_gan_mnist ( dataset, verbose = 2 )
# net = deep_gan_mnist ( dataset, verbose = 2 )
"""net = deep_deconvolutional_gan ( batch_norm = True,
net = deep_deconvolutional_gan ( batch_norm = True,
dropout_rate = 0.5,
regularize = True,
dataset = dataset,
verbose = 2 )"""
verbose = 2 )
net = deep_deconvolutional_lsgan ( batch_norm = True,
dropout_rate = 0.5,
regularize = True,
Expand Down
25 changes: 15 additions & 10 deletions pantry/tutorials/lenet.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ def lenet5 ( dataset= None, verbose = 1 ):
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type" : 'polyak',
"momentum_type" : 'nesterov',
"momentum_params" : (0.65, 0.97, 30),
"optimizer_type" : 'adagrad',
"optimizer_type" : 'rmsprop',
"id" : "main"
}

Expand All @@ -25,7 +25,7 @@ def lenet5 ( dataset= None, verbose = 1 ):
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
}

visualizer_params = {
"root" : 'lenet5',
Expand Down Expand Up @@ -122,24 +122,25 @@ def lenet5 ( dataset= None, verbose = 1 ):
verbose = verbose
)

learning_rates = (0.05, 0.01, 0.001)
learning_rates = (0.05, .0001, 0.001)
net.pretty_print()
draw_network(net.graph, filename = 'lenet.png')
# draw_network(net.graph, filename = 'lenet.png')

net.cook()
net.train( epochs = (4, 4),

net.train( epochs = (20, 20),
validate_after_epochs = 1,
training_accuracy = True,
learning_rates = learning_rates,
show_progress = True,
early_terminate = True,
patience = 2,
verbose = verbose)

net.test(verbose = verbose)

# Advaned versions of the CNN
def lenet_maxout_batchnorm_before_activation ( dataset= None, verbose = 1 ):
def lenet_maxout_batchnorm_before_activation ( dataset= None, verbose = 1 ):
"""
This is a version with nesterov momentum and rmsprop instead of the typical sgd.
This also has maxout activations for convolutional layers, dropouts on the last
Expand Down Expand Up @@ -278,7 +279,7 @@ def lenet_maxout_batchnorm_before_activation ( dataset= None, verbose = 1 ):

net.test(verbose = verbose)

def lenet_maxout_batchnorm_after_activation ( dataset= None, verbose = 1 ):
def lenet_maxout_batchnorm_after_activation ( dataset= None, verbose = 1 ):
"""
This is a version with nesterov momentum and rmsprop instead of the typical sgd.
This also has maxout activations for convolutional layers, dropouts on the last
Expand Down Expand Up @@ -444,7 +445,10 @@ def lenet_maxout_batchnorm_after_activation ( dataset= None, verbose = 1 ):
if len(sys.argv) > 1:
if sys.argv[1] == 'create_dataset':
from yann.special.datasets import cook_cifar10
from yann.special.datasets import cook_mnist

data = cook_cifar10 (verbose = 2)
# data = cook_mnist()
dataset = data.dataset_location()
else:
dataset = sys.argv[1]
Expand All @@ -454,7 +458,8 @@ def lenet_maxout_batchnorm_after_activation ( dataset= None, verbose = 1 ):
if dataset is None:
print " creating a new dataset to run through"
from yann.special.datasets import cook_cifar10
# from yann.special.datasets import cook_mnist
from yann.special.datasets import cook_mnist

data = cook_cifar10 (verbose = 2)
# data = cook_mnist()
dataset = data.dataset_location()
Expand Down

0 comments on commit ae74f7b

Please sign in to comment.