New activation functions added, unit tests updated (part 3)

Examples checked
Prepared for AE release
Prepared for license change

README.md

@@ -10,14 +10,17 @@ its extensive use of unittests assures a high level of reliability and correctne
 
 News
 
-- The documentation is updated to restructured text
-- Documentation hosted
-- Next the unit tests will be added
-- Upcoming: Tutorials will be added
+- New activation functions added
+- Unitstest update to date
+- Prepared for Autoencoder release
+- Prepared to change License
+
+- Upcoming: Tutorials will be updated
 - Upcoming: Auto encoders will be added
-- Upcoming: MDP integration will be added
-- Upcoming: Deep Boltzmann machines will be added
-- Upcoming: Feed Forward neural networks will be added
+
+- Future: Feed Forward neural networks will be added
+- Future: Deep Boltzmann machines will be added
+- Future: MDP integration will be added
 
 Features
 

docs/tutorials/RBM_MNIST_big.rst

@@ -2,7 +2,8 @@ Big binary RBM on MNIST
 ==========================================================
 
 Example for training a centered and normal binary restricted Boltzmann machine on the MNIST handwritten digit dataset.
-The model has 500 hidden units, is trained for 200 epochs, and the log-likelihood is evaluated using annealed importance sampling.
+The model has 500 hidden units, is trained for 200 epochs (That takes a while, reduce it if you like), and the
+log-likelihood is evaluated using annealed importance sampling.
 
 It allows to reproduce the results from the publication `How to Center Deep Boltzmann Machines. Melchior et al. JMLR 2016. <http://jmlr.org/papers/v17/14-237.html>`_. Running the code as it is for example reproduces a single trial of the plot in Figure 9. (PCD-1) for $dd^b_s$.
 

pydeep/misc/io.py

@@ -350,3 +350,36 @@ def load_natural_image_patches(path):
         raise Exception('-> File reading Error: ')
     data = numx.array(data, dtype=numx.double)
     return data
+
+def load_olivetti_faces(path, correct_orientation=True):
+    """ Loads the Olivetti face dataset 400 images, size 64x64
+    :param path: Path and name of the file to load.
+    :type path: string
+    :param correct_orientation: Corrects the orientation of the images.
+    :type correct_orientation: bool
+    :return: Olivetti face dataset
+    :rtype: numpy array
+    """
+    if not os.path.isfile(path):
+        print('-> File not existing: ' + path)
+        try:
+            download_file('http://www.cs.nyu.edu/~roweis/data/olivettifaces.mat', path, buffer_size=10 * 1024 ** 2)
+        except:
+            try:
+                download_file('https://github.com/probml/pmtk3/tree/master/bigData/facesOlivetti/facesOlivetti.mat',
+                              path, buffer_size=10 * 1024 ** 2)
+            except:
+                raise Exception('Download failed, make sure you have internet connection!')
+    print('-> loading data ... ')
+    try:
+        data = scipy.io.loadmat(path)['faces'].T
+        if correct_orientation:
+            import pydeep.base.numpyextension as npext
+            for i in range(data.shape[0]):
+                data[i] = npext.rotate(data[i].reshape(64,64),270).reshape(64*64)
+            print('-> orientation corrected!')
+        print('-> done!')
+    except:
+        raise Exception('-> File reading Error: ')
+    data = numx.array(data, dtype=numx.double)
+    return data

pydeep/rbm/dbn.py

@@ -47,7 +47,9 @@ def __init__(self, list_of_rbms):
         """
         super(DBN, self).__init__(list_of_layers=list_of_rbms)
 
-    def forward_propagate(self, input_data, sample=False):
+    def forward_propagate(self,
+                          input_data,
+                          sample=False):
         """ Propagates the data through the network.
 
         :param input_data: Input data
@@ -68,7 +70,9 @@ def forward_propagate(self, input_data, sample=False):
                 self.states[l + 1] = self._layers[l].sample_h(self.states[l + 1])
         return self.states[len(self._layers)]
 
-    def backward_propagate(self, output_data, sample=False):
+    def backward_propagate(self,
+                           output_data,
+                           sample=False):
         """ Propagates the output back through the input.
 
         :param output_data: Output data.
@@ -89,7 +93,9 @@ def backward_propagate(self, output_data, sample=False):
                 self.states[l - 1] = self._layers[l - 1].sample_v(self.states[l - 1])
         return self.states[0]
 
-    def reconstruct(self, input_data, sample=False):
+    def reconstruct(self,
+                    input_data,
+                    sample=False):
         """ Reconstructs the data by propagating the data to the output and back to the input.
 
         :param input_data: Input data.
@@ -103,7 +109,10 @@ def reconstruct(self, input_data, sample=False):
         """
         return self.backward_propagate(self.forward_propagate(input_data, sample), sample)
 
-    def sample_top_layer(self, sampling_steps=100, initial_state=None, sample=True):
+    def sample_top_layer(self,
+                         sampling_steps=100,
+                         initial_state=None,
+                         sample=True):
         """ Samples the top most layer, if initial_state is None the current state is used otherwise sampling is \
             started from the given initial state
 
@@ -134,7 +143,10 @@ def sample_top_layer(self, sampling_steps=100, initial_state=None, sample=True):
                     self.states[len(self._layers)])
         return self.states[len(self._layers)]
 
-    def reconstruct_sample_top_layer(self, input_data, sampling_steps=100, sample_forward_backward=False):
+    def reconstruct_sample_top_layer(self,
+                                     input_data,
+                                     sampling_steps=100,
+                                     sample_forward_backward=False):
         """ Reconstructs data by propagating the data forward, sampling the top most layer and propagating the result \
             backward.
 

pydeep/rbm/estimator.py

@@ -18,7 +18,7 @@
         - Reverse annealed importance sampling to approximated the partition function.
       
     :Info:
-        For the derivations .. seealso:: \
+        For the derivations .. seealso::
         https://www.ini.rub.de/PEOPLE/wiskott/Reprints/Melchior-2012-MasterThesis-RBMs.pdf
         
     :Version:

pydeep/rbm/sampler.py

@@ -10,8 +10,8 @@
         - Independent Parallel Tempering Sampling
 
     :Info:
-        For the derivations \
-        .. seealso:: https://www.ini.rub.de/PEOPLE/wiskott/Reprints/Melchior-2012-MasterThesis-RBMs.pdf
+        For the derivations .. seealso::
+        https://www.ini.rub.de/PEOPLE/wiskott/Reprints/Melchior-2012-MasterThesis-RBMs.pdf
 
     :Version:
         1.1.0

pydeep/rbm/trainer.py

@@ -11,8 +11,8 @@
         - GD   (Exact Gradient descent (only for small binary models))
 
     :Info:
-        For the derivations \
-        .. seealso:: http://www.ini.rub.de/data/documents/tns/masterthesis_janmelchior.pdf
+        For the derivations .. seealso::
+        http://www.ini.rub.de/data/documents/tns/masterthesis_janmelchior.pdf
 
     :Version:
         1.1.0
@@ -590,7 +590,8 @@ def __init__(self,
                  num_samples,
                  betas=3,
                  data=None):
-        """ The constructor initializes the IPT trainer with a given model and data.
+        """ The constructor initializes the IPT trainer with a given model and
+            data.
 
         :param model: The model to sample from.
         :type model: Valid model class.

setup.py

@@ -43,12 +43,13 @@ def read(fname):
 
 setup(
     name="PyDeep",
-    version="1.1.0",
+    version="1.2.0",
     author="Jan Melchior",
     author_email="JanMelchior@gmx.de",
     description=("Machine learning library with focus on Restricted Boltzmann machines."),
     license="GNU",
-    keywords="Machine learning, Deep Learning, Restricted Boltzmann machines, Independent Component Analysis, "
+    keywords="Machine learning, Deep Learning, Feed-Forward Neural Networks, Autoencoder, "
+             "Restricted Boltzmann machines, Deep Boltzmann machines, Independent Component Analysis, "
              "Principal Component Analysis, Zero-Phase Component Analysis",
     url="https://github.com/MelJan/PyDeep",
     packages=['pydeep', 'pydeep.base', 'pydeep.misc', 'pydeep.rbm'],