Tutorials added ICA 2D, PCA 2D

docs/tutorial.rst

@@ -4,3 +4,8 @@ Tutorials
    :maxdepth: 2
 
    Eigenfaces<tutorials/eigenfaces.rst>
+
+   PCA 2D<tutorials/PCA_2D_example.rst>
+
+   ICA 2D<tutorials/ICA_2D_example.rst>
+

docs/tutorials/ICA_2D_example.rst

@@ -0,0 +1,59 @@
+Independent Component Analysis on a 2D example.
+=======================================================
+
+Example for Independent Component Analysis (`ICA <https://en.wikipedia.org/wiki/Principal_component_analysis>`_) used
+for a blind source separation on a linear 2D mixture.
+
+If you are new on blind source separation, a good theoretical introduction is given in the following video.
+
+.. raw:: html
+
+    <div style="margin-top:10px;">
+      <iframe width="560" height="315" src="http://www.youtube.com/embed/3eWuUWODE4o" frameborder="0" allowfullscreen></iframe>
+    </div>
+
+and the follow up video introduces to ICA.
+
+.. raw:: html
+
+    <div style="margin-top:10px;">
+      <iframe width="560" height="315" src="http://www.youtube.com/embed/ugiMhRbFnTo" frameborder="0" allowfullscreen></iframe>
+    </div>
+
+The code_ given below produces the following output.
+
+Visualization of the data and true mixing matrix projected to the whitened space.
+
+.. figure:: images/ICA_2D_mixing_whitened.png
+   :scale: 75 %
+   :alt: Examples of Mixing matrix 2D in whitened space
+
+Visualization of the whitened data with the ICA projection matrix, that is the estimation of the whitened mxining matrix.
+Note that ICA is invariant to sign flips of the sources. Thus the columns of the estimated mixing matrix are most likely a permutation of the columns of the original mixing matrix's and can also be the 180 degrees rotated version (original vector multiplied by -1).
+
+.. image:: images/ICA_2D_estimate_whitened.png
+   :scale: 75 %
+   :alt: Examples of ICA 2D in whitened space
+
+We can also back-project the ICA projection matrix back to the original space and compare the results in the original space.
+
+.. image:: images/ICA_2D_mixing.png
+   :scale: 75 %
+   :alt: Examples of Mixing matrix 2D
+
+.. image:: images/ICA_2D_estimate.png
+   :scale: 75 %
+   :alt: Examples of ICA 2D
+
+For a real-world application see the eigenfaces example.
+
+.. _code:
+
+Source code
+***********
+
+.. figure:: images/download_icon.png
+   :scale: 20 %
+   :target: https://github.com/MelJan/PyDeep/blob/master/examples/PCA_2D.py
+
+.. literalinclude:: ../../examples/PCA_2D.py

docs/tutorials/PCA_2D_example.rst

@@ -0,0 +1,45 @@
+Principal Component Analysis 2D example.
+============================================
+
+Example for Principal Component Analysis (`PCA <https://en.wikipedia.org/wiki/Principal_component_analysis>`_) on a linear 2D mixture.
+
+If you are new on PCA, a good theoretical introduction is given in the following video.
+
+.. raw:: html
+
+    <div style="margin-top:10px;">
+      <iframe width="560" height="315" src="http://www.youtube.com/embed/9H-1FH1gn6w" frameborder="0" allowfullscreen></iframe>
+    </div>
+
+The code_ given below produces the following output.
+
+The data is plotted with the extracted principal components.
+
+.. figure:: images/PCA_2D.png
+   :scale: 75 %
+   :alt: Examples of PCA 2D
+
+Data and extracted principal components can also be plotted in the projected space.
+
+.. image:: images/PCA_2D_projected.png
+   :scale: 75 %
+   :alt: Examples of PCA 2D in projected space
+
+The PCA class can also perform whitening and data and extracted principal components are plotted in the whitened space.
+
+.. image:: images/PCA_2D_whitened.png
+   :scale: 50 %
+   :alt: Examples of PCA 2D in whitened space
+
+For a real-world application see the eigenfaces example.
+
+.. _code:
+
+Source code
+***********
+
+.. figure:: images/download_icon.png
+   :scale: 20 %
+   :target: https://github.com/MelJan/PyDeep/blob/master/examples/PCA_2D.py
+
+.. literalinclude:: ../../examples/PCA_2D.py

examples/ICA_2D.py

@@ -0,0 +1,102 @@
+""" Example for the Independent Component Analysis on a 2D example.
+
+    :Version:
+        1.1.0
+
+    :Date:
+        22.04.2017
+
+    :Author:
+        Jan Melchior
+
+    :Contact:
+        JanMelchior@gmx.de
+
+    :License:
+
+        Copyright (C) 2017 Jan Melchior
+
+        This file is part of the Python library PyDeep.
+
+        PyDeep is free software: you can redistribute it and/or modify
+        it under the terms of the GNU General Public License as published by
+        the Free Software Foundation, either version 3 of the License, or
+        (at your option) any later version.
+
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU General Public License for more details.
+
+        You should have received a copy of the GNU General Public License
+        along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+
+# Import ZCA, ICA, the toy-problem, numpy, numpy extensions, and visualization module
+import numpy as numx
+import pydeep.misc.visualization as vis
+from pydeep.preprocessing import ZCA, ICA
+from pydeep.misc.toyproblems import generate_2d_mixtures
+import pydeep.base.numpyextension as numxext
+
+# Set the random seed
+# (optional, if stochastic processes are involved we get the same results)
+numx.random.seed(42)
+
+# Create 2D linear mixture
+data, mixing_matrix = generate_2d_mixtures(50000, 0, 3.0)
+
+# ZCA
+zca = ZCA(data.shape[1])
+zca.train(data)
+data_zca = zca.project(data)
+
+# ICA
+ica = ICA(data_zca.shape[1])
+ica.train(data_zca, iterations=1000)
+data_ica = ica.project(data_zca)
+
+# For better visualization the principal components are rescaled
+scale_factor = 3
+
+# Display results, the matrices are normalized such that the
+# column norm equals the scale factor.
+vis.figure(0, figsize=[7, 7])
+vis.title("Data and mixing matrix")
+vis.plot_2d_data(data)
+vis.plot_2d_weights(numxext.resize_norms(mixing_matrix,
+                                         norm=scale_factor,
+                                         axis=0))
+vis.axis('equal')
+vis.axis([-4, 4, -4, 4])
+
+vis.figure(1, figsize=[7, 7])
+vis.title("Data and mixing matrix in whitened space")
+vis.plot_2d_data(data_zca)
+vis.plot_2d_weights(numxext.resize_norms(scale_factor * zca.project(mixing_matrix.T).T,
+                                         norm=scale_factor,
+                                         axis=0))
+vis.axis('equal')
+vis.axis([-4, 4, -4, 4])
+
+vis.figure(2, figsize=[7, 7])
+vis.title("Data and ica estimation of the mixing matrix in whitened space")
+vis.plot_2d_data(data_zca)
+vis.plot_2d_weights(numxext.resize_norms(scale_factor * ica.projection_matrix,
+                                         norm=scale_factor,
+                                         axis=0))
+vis.axis('equal')
+vis.axis([-4, 4, -4, 4])
+
+vis.figure(3, figsize=[7, 7])
+vis.title("Data and ica estimation of the mixing matrix")
+vis.plot_2d_data(data)
+vis.plot_2d_weights(
+    numxext.resize_norms(scale_factor * zca.unproject(ica.projection_matrix.T).T,
+                         norm=scale_factor,
+                         axis=0))
+vis.axis('equal')
+vis.axis([-4, 4, -4, 4])
+
+vis.show()

examples/PCA_2D.py

@@ -0,0 +1,81 @@
+""" Example for the Principal Component Analysis on a 2D example.
+
+    :Version:
+        1.1.0
+
+    :Date:
+        22.04.2017
+
+    :Author:
+        Jan Melchior
+
+    :Contact:
+        JanMelchior@gmx.de
+
+    :License:
+
+        Copyright (C) 2017 Jan Melchior
+
+        This file is part of the Python library PyDeep.
+
+        PyDeep is free software: you can redistribute it and/or modify
+        it under the terms of the GNU General Public License as published by
+        the Free Software Foundation, either version 3 of the License, or
+        (at your option) any later version.
+
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU General Public License for more details.
+
+        You should have received a copy of the GNU General Public License
+        along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+# Import PCA, the toyproblem, numpy, numpy extensions, and visualization module
+import numpy as numx
+import pydeep.misc.visualization as vis
+from pydeep.preprocessing import PCA
+from pydeep.misc.toyproblems import generate_2d_mixtures
+
+# Set the random seed
+# (optional, if stochastic processes are involved we get the same results)
+numx.random.seed(42)
+
+# Create 2D linear mixture
+data = generate_2d_mixtures(50000, 0, 3.0)[0]
+
+# PCA
+pca = PCA(data.shape[1])
+pca.train(data)
+data_pca = pca.project(data)
+
+# Display results
+vis.figure(0, figsize=[7, 7])
+vis.title("Data with estimated principal components")
+vis.plot_2d_data(data)
+vis.plot_2d_weights(pca.projection_matrix)
+vis.axis('equal')
+vis.axis([-4, 4, -4, 4])
+
+vis.figure(2, figsize=[7, 7])
+vis.title("Data with estimated principal components in projected space")
+vis.plot_2d_data(data_pca)
+vis.plot_2d_weights(pca.project(pca.projection_matrix.T))
+vis.axis('equal')
+vis.axis([-4, 4, -4, 4])
+
+# PCA with whitened ouput
+pca = PCA(data.shape[1],True)
+pca.train(data)
+data_pca = pca.project(data)
+
+vis.figure(3, figsize=[7, 7])
+vis.title("Data with estimated principal components in whitened space")
+vis.plot_2d_data(data_pca)
+vis.plot_2d_weights(pca.project(pca.projection_matrix.T).T)
+vis.axis('equal')
+vis.axis([-4, 4, -4, 4])
+
+# Show all windows.
+vis.show()

examples/eigenfaces.py

@@ -1,10 +1,10 @@
-""" Example for Eigenfaces (Principal component analysis on face images).
+""" Example for Principal component analysis on face images (Eigenfaces).
 
     :Version:
         1.1.0
 
     :Date:
-        08.04.2017
+        22.04.2017
 
     :Author:
         Jan Melchior
@@ -38,7 +38,8 @@
 import pydeep.misc.io as io
 import pydeep.misc.visualization as vis
 
-# Set the random seed (optional, if stochastic processes are involved we always get the same results)
+# Set the random seed
+# (optional, if stochastic processes are involved we get the same results)
 numx.random.seed(42)
 
 # Load the data
@@ -92,7 +93,8 @@
                               border_size=1,
                               normalized=True)
 vis.imshow_matrix(matrix=images,
-                  windowtitle='First 100 Face images reconstructed from 50 principal components')
+                  windowtitle='First 100 Face images reconstructed from 50 '
+                              'principal components')
 
 # Show the first 100 Face images reconstructed from 120 principal components
 recon = pca.unproject(pca.project(data[0:100], num_components=200)).T
@@ -104,7 +106,8 @@
                               border_size=1,
                               normalized=True)
 vis.imshow_matrix(matrix=images,
-                  windowtitle='First 100 Face images reconstructed from 200 principal components')
+                  windowtitle='First 100 Face images reconstructed from 200 '
+                              'principal components')
 
 # Show all windows.
 vis.show()