Compile

examples/mnist_examples.py

@@ -30,7 +30,7 @@
 @yadll.utils.timer(' Loading the data')
 def load_data(data_loader):
     print '... Loading the data'
-    return yadll.data.Data(data_loader)
+    return yadll.data.Data(data_loader) #, preprocessing='Standardize')
 
 
 def build_network(network_name='Logistic_regression', input_var=None):

examples/networks.py

@@ -6,6 +6,7 @@
 __all__ = ['logistic_regression',
            'mlp',
            'dropout',
+           'snn',
            'dropconnect',
            'convpool',
            'lenet5',
@@ -93,12 +94,12 @@ def dropout(input_var=None):
     # Input layer
     l_in = InputLayer(input_shape=(None, 28 * 28), input_var=input_var, name='Input')
     # Dropout Layer
-    l_dro1 = Dropout(incoming=l_in, corruption_level=0.4, name='Dropout 1')
+    l_dro1 = Dropout(incoming=l_in, corruption_level=0.2, name='Dropout 1')
     # Dense Layer
     l_hid1 = DenseLayer(incoming=l_dro1, n_units=500, W=glorot_uniform,
                         activation=relu, name='Hidden layer 1')
     # Dropout Layer
-    l_dro2 = Dropout(incoming=l_hid1, corruption_level=0.2, name='Dropout 2')
+    l_dro2 = Dropout(incoming=l_hid1, corruption_level=0.1, name='Dropout 2')
     # Dense Layer
     l_hid2 = DenseLayer(incoming=l_dro2, n_units=500, W=glorot_uniform,
                         activation=relu, name='Hidden layer 2')
@@ -131,12 +132,12 @@ def snn(input_var=None):
     # Input layer
     l_in = InputLayer(input_shape=(None, 28 * 28), input_var=input_var, name='Input')
     # Dropout Layer
-    l_dro1 = Dropout(incoming=l_in, corruption_level=0.4, name='Dropout 1')
+    l_dro1 = AlphaDropout(incoming=l_in, corruption_level=0.2, name='Dropout 1')
     # Dense Layer
     l_hid1 = DenseLayer(incoming=l_dro1, n_units=500, W=selu_normal,
                         activation=selu, name='Hidden layer 1')
     # Dropout Layer
-    l_dro2 = Dropout(incoming=l_hid1, corruption_level=0.2, name='Dropout 2')
+    l_dro2 = AlphaDropout(incoming=l_hid1, corruption_level=0.1, name='Dropout 2')
     # Dense Layer
     l_hid2 = DenseLayer(incoming=l_dro2, n_units=500, W=selu_normal,
                         activation=selu, name='Hidden layer 2')

examples/normalization_example.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: UTF-8 -*-
-import os
 import yadll
 import logging
 
@@ -16,34 +15,25 @@
 
 # Hyperparameters
 hp = yadll.hyperparameters.Hyperparameters()
-hp('batch_size', 500)
+hp('batch_size', 512)
 hp('n_epochs', 1000)
-hp('learning_rate', 0.01)
-hp('momentum', 0.5)
-hp('l1_reg', 0.00)
-hp('l2_reg', 0.0000)
 hp('patience', 10000)
 
 # add the hyperparameters to the model
 model.hp = hp
 
 # Create connected layers
 # Input layer
-l_in = yadll.layers.InputLayer(input_shape=(hp.batch_size, 28 * 28), name='Input')
+l_in = yadll.layers.InputLayer(input_shape=(hp.batch_size, 28 * 28))
 # Dense Layer 1
-l_hid1 = yadll.layers.DenseLayer(incoming=l_in, n_units=500, W=yadll.init.glorot_uniform,
-                                 l1=hp.l1_reg, l2=hp.l2_reg, activation=yadll.activations.relu,
-                                 name='Hidden layer 1')
+l_hid1 = yadll.layers.DenseLayer(incoming=l_in, n_units=500)
 # Dense Layer 2
-l_hid2 = yadll.layers.DenseLayer(incoming=l_hid1, n_units=500, W=yadll.init.glorot_uniform,
-                                 l1=hp.l1_reg, l2=hp.l2_reg, activation=yadll.activations.relu,
-                                 name='Hidden layer 2')
+l_hid2 = yadll.layers.DenseLayer(incoming=l_hid1, n_units=500)
 # Logistic regression Layer
-l_out = yadll.layers.LogisticRegression(incoming=l_hid2, n_class=10, l1=hp.l1_reg,
-                                        l2=hp.l2_reg, name='Logistic regression')
+l_out = yadll.layers.LogisticRegression(incoming=l_hid2, n_class=10)
 
 # Create network and add layers
-net = yadll.network.Network('2 layers mlp with dropout')
+net = yadll.network.Network('2 layers mlp')
 net.add(l_in)
 net.add(l_hid1)
 net.add(l_hid2)
@@ -53,7 +43,7 @@
 model.network = net
 
 # updates method
-model.updates = yadll.updates.adagrad
+model.updates = yadll.updates.rmsprop
 
 # train the model and save it to file at each best
 model_report = model.train()
@@ -62,57 +52,45 @@
 ######################################################################################
 # Basic mlp with batch normalization model
 # create the model
-model_BN = yadll.model.Model(name='mlp with batch normalization', data=data)
-
-# # Hyperparameters
-# hp = yadll.hyperparameters.Hyperparameters()
-# hp('batch_size', 500)
-# hp('n_epochs', 1000)
-# hp('learning_rate', 0.9)
-# hp('momentum', 0.5)
-# hp('l1_reg', 0.00)
-# hp('l2_reg', 0.0000)
-# hp('patience', 10000)
-
-# add the hyperparameters to the model
-model_BN.hp = hp
+model_bn = yadll.model.Model(name='mlp with batch normalization', data=data, hyperparameters=hp)
 
 # Create connected layers
 # Input layer
-l_in = yadll.layers.InputLayer(input_shape=(hp.batch_size, 28 * 28), name='Input')
+l_in = yadll.layers.InputLayer(input_shape=(hp.batch_size, 28 * 28))
 # Batch Normalization 1
-l_bn1 = yadll.layers.BatchNormalization(incoming=l_in, name='Batch Normalization 1')
+l_bn1 = yadll.layers.BatchNormalization(incoming=l_in)
 # Dense Layer 1
-l_hid1 = yadll.layers.DenseLayer(incoming=l_bn1, n_units=500, W=yadll.init.glorot_uniform,
-                                 l1=hp.l1_reg, l2=hp.l2_reg, activation=yadll.activations.relu,
-                                 name='Hidden layer 1')
+l_hid1 = yadll.layers.DenseLayer(incoming=l_bn1, n_units=500, activation=yadll.activations.linear)
 # Batch Normalization 2
-l_bn2 = yadll.layers.BatchNormalization(incoming=l_hid1, name='Batch Normalization 2')
+l_bn2 = yadll.layers.BatchNormalization(incoming=l_hid1)
+# Activation 1
+l_act1 = yadll.layers.Activation(incoming=l_bn2, activation=yadll.activations.relu)
 # Dense Layer 2
-l_hid2 = yadll.layers.DenseLayer(incoming=l_bn2, n_units=500, W=yadll.init.glorot_uniform,
-                                 l1=hp.l1_reg, l2=hp.l2_reg, activation=yadll.activations.relu,
-                                 name='Hidden layer 2')
+l_hid2 = yadll.layers.DenseLayer(incoming=l_act1, n_units=500, activation=yadll.activations.linear)
 # Batch Normalization 3
-l_bn3 = yadll.layers.BatchNormalization(incoming=l_hid2, name='Batch Normalization 3')
+l_bn3 = yadll.layers.BatchNormalization(incoming=l_hid2)
+# Activation 2
+l_act2 = yadll.layers.Activation(incoming=l_bn3, activation=yadll.activations.relu)
 # Logistic regression Layer
-l_out = yadll.layers.LogisticRegression(incoming=l_bn3, n_class=10, l1=hp.l1_reg,
-                                        l2=hp.l2_reg, name='Logistic regression')
+l_out = yadll.layers.LogisticRegression(incoming=l_act2, n_class=10)
 
 # Create network and add layers
 net = yadll.network.Network('mlp with batch normalization')
 net.add(l_in)
 net.add(l_bn1)
 net.add(l_hid1)
 net.add(l_bn2)
+net.add(l_act1)
 net.add(l_hid2)
 net.add(l_bn3)
+net.add(l_act2)
 net.add(l_out)
 
 # add the network to the model
-model_BN.network = net
+model_bn.network = net
 
 # updates method
-model_BN.updates = yadll.updates.adagrad
+model_bn.updates = yadll.updates.rmsprop
 
 # train the model and save it to file at each best
-model_BN_report = model_BN.train()
+model_bn_report = model_bn.train()

examples/test.py

@@ -10,10 +10,10 @@
 
 logging.basicConfig(level=logging.DEBUG, format='%(message)s')
 
-_networks = [# 'logistic_regression',
-            # 'mlp',
-            # 'dropout',
-            # 'dropconnect',
+_networks = [ 'logistic_regression',
+            'mlp',
+            'dropout',
+            'dropconnect',
             'convpool',
             'lenet5',
             'autoencoder',

yadll/activations.py

@@ -225,3 +225,26 @@ def elu(x, alpha=1):
         Exponential Linear Units (ELUs)" <http://arxiv.org/abs/1511.07289>`.
     """
     return T.nnet.elu(x, alpha)
+
+
+def selu(x):
+    r"""
+    Compute the element-wise scaled exponential linear activation function.
+
+    Parameters
+    ----------
+    x : symbolic tensor
+        Tensor to compute the activation function for.
+
+    Returns
+    -------
+    symbolic tensor
+        Element-wise scaled exponential linear activation function applied to `x`.
+
+    References
+    -----
+    .. [1] https://arxiv.org/pdf/1706.02515.pdf
+    """
+    alpha = 1.6732632423543772848170429916717
+    scale = 1.0507009873554804934193349852946
+    return scale * elu(x, alpha)

yadll/data.py

@@ -9,8 +9,8 @@
 
 
 def normalize(x):
-    x_min = x.min()
-    x_max = x.max()
+    x_min = x.min(axis=0)
+    x_max = x.max(axis=0)
     z = apply_normalize(x, x_min, x_max)
     return z, x_min, x_max
 
@@ -23,9 +23,9 @@ def revert_normalize(z, x_min, x_max):
     return z * (x_max - x_min) + x_min
 
 
-def standardize(x):
-    x_mean = x.mean()
-    x_std = x.std()
+def standardize(x, epsilon=1e-6):
+    x_mean = x.mean(axis=0)
+    x_std = x.std(axis=0) + epsilon
     z = apply_standardize(x, x_mean, x_std)
     return z, x_mean, x_std
 
@@ -184,13 +184,13 @@ def __init__(self, data, preprocessing=None,
         if preprocessing == 'Normalize':
             train_set_x, self.min, self.max = normalize(train_set_x)
             test_set_x = apply_normalize(test_set_x, self.min, self.max)
-            if valid_set_x:
+            if valid_set_x is not None:
                 valid_set_x = apply_normalize(valid_set_x, self.min, self.max)
 
         if preprocessing == 'Standardize':
             train_set_x, self.mean, self.std = standardize(train_set_x)
             test_set_x = apply_standardize(test_set_x, self.mean, self.std)
-            if valid_set_x:
+            if valid_set_x is not None:
                 valid_set_x = apply_standardize(valid_set_x, self.mean, self.std)
 
         if shared:
@@ -214,3 +214,6 @@ def dataset(self):
 
     def shape(self):
         return [(data[0].shape, data[1].shape) for data in self.data]
+
+from sklearn.preprocessing import StandardScaler
+

yadll/layers.py

@@ -390,7 +390,8 @@ def get_output(self, stochastic=True, **kwargs):
             lst[0] = T.shape(X)[0]
             self.input_shape = tuple(lst)
         if self.p != 1 and stochastic:
-            X = X * T_rng.binomial(self.input_shape, n=1, p=self.p, dtype=floatX)
+            X *= T_rng.binomial(self.input_shape, n=1, p=self.p, dtype=floatX)
+            X /= self.p
         return X
 
     def to_conf(self):
@@ -399,6 +400,43 @@ def to_conf(self):
         return conf
 
 
+class AlphaDropout(Layer):
+    """
+    Alpha Dropout layer for zero mean and unit variance
+
+    References
+    ----------
+    .. [1] https://arxiv.org/pdf/1706.02515.pdf
+    """
+    n_instances = 0
+
+    def __init__(self, incoming, corruption_level=0.5,
+                 alpha_prime=-1.7580993408473766, **kwargs):
+        super(AlphaDropout, self).__init__(incoming, **kwargs)
+        self.p = 1 - corruption_level
+        self.alpha_prime = alpha_prime  # selu params : - alpha * scale
+
+    def get_output(self, stochastic=True, **kwargs):
+        X = self.input_layer.get_output(stochastic=stochastic, **kwargs)
+        if self.input_shape[0] is None:
+            lst = list(self.input_shape)
+            lst[0] = T.shape(X)[0]
+            self.input_shape = tuple(lst)
+        if self.p != 1 and stochastic:
+            binary_mask = T_rng.binomial(self.input_shape, n=1, p=self.p, dtype=floatX)
+            X = X * binary_mask + self.alpha_prime * (1 - binary_mask)
+            a = T.sqrt(1 / (self.p * (1 + (1-self.p) * T.pow(self.alpha_prime, 2))))
+            b = - a * (1 - self.p) * self.alpha_prime
+            X = a * X + b
+        return X
+
+    def to_conf(self):
+        conf = super(AlphaDropout, self).to_conf()
+        conf['corruption_level'] = 1 - self.p
+        conf['alpha_prime'] = self.alpha_prime
+        return conf
+
+
 class Dropconnect(DenseLayer):
     """
     DropConnect layer

yadll/model.py

@@ -1,5 +1,6 @@
 # -*- coding: UTF-8 -*-
 import cPickle
+import sys
 import yadll
 from .layers import *
 from .exceptions import *
@@ -29,8 +30,14 @@ def save_model(model, file=None):
             d_file = model.file
     else:
         d_file = file
-    with open(d_file, 'wb') as f:
-        cPickle.dump(model, f, cPickle.HIGHEST_PROTOCOL)
+
+    try:
+        with open(d_file, 'wb') as f:
+            cPickle.dump(model, f, cPickle.HIGHEST_PROTOCOL)
+    except RuntimeError:
+        sys.setrecursionlimit(5000)
+        with open(d_file, 'wb') as f:
+            cPickle.dump(model, f, cPickle.HIGHEST_PROTOCOL)
 
 
 def load_model(file):