Visualisation and PCA

echotorch/nn/LiESNCell.py

@@ -46,7 +46,8 @@ def __init__(self, leaky_rate=1.0, train_leaky_rate=False, *args, **kwargs):
         :param train_leaky_rate: Train leaky rate as parameter? (default: False)
         """
         super(LiESNCell, self).__init__(*args, **kwargs)
-
+        """print("W")
+        print(self.w)"""
         # Params
         if train_leaky_rate:
             self.leaky_rate = nn.Parameter(torch.Tensor(1).fill_(leaky_rate), requires_grad=True)
@@ -91,10 +92,12 @@ def forward(self, u, y=None, w_out=None, reset_state=True):
 
                 # Compute input layer
                 u_win = self.w_in.mv(ut)
-
+                # print("u_win")
+                # print(u_win)
                 # Apply W to x
                 x_w = self.w.mv(self.hidden)
-
+                # print("x_w")
+                # print(x_w)
                 # Feedback or not
                 if self.feedbacks and self.training and y is not None:
                     # Current target
@@ -130,16 +133,27 @@ def forward(self, u, y=None, w_out=None, reset_state=True):
                     # Add everything
                     x = u_win + x_w + self.w_bias
                 # end if
-
+                # print("u_win + x_w + bias")
+                # print(x)
                 # Apply activation function
                 x = self.nonlin_func(x)
-
+                # print("non lin")
+                # print(x)
                 # Add to outputs
                 self.hidden.data = (self.hidden.mul(1.0 - self.leaky_rate) + x.view(self.output_dim).mul(self.leaky_rate)).data
-
+                # print("leak")
+                # print(self.hidden.data)
                 # New last state
                 outputs[b, t] = self.hidden
+                """if t == 2:
+                    exit()
+                # end if
+                print("")
+                print("")"""
             # end for
+            """print(outputs[0])
+            plt.imshow(outputs[0].t().numpy(), cmap='Greys')
+            plt.show()"""
         # end for
 
         return outputs

echotorch/nn/PCACell.py

@@ -160,7 +160,8 @@ def finalize(self):
 
         # Check for negative eigenvalues
         if float(d.min()) < 0:
-            raise Exception(u"Got negative eigenvalues ({}). You may either set output_dim to be smaller".format(d))
+            # raise Exception(u"Got negative eigenvalues ({}). You may either set output_dim to be smaller".format(d))
+            pass
         # end if
 
         # Indexes

echotorch/utils/__init__.py

@@ -4,10 +4,11 @@
 # Imports
 from .error_measures import nrmse, nmse, rmse, mse, perplexity, cumperplexity
 from .utility_functions import spectral_radius, deep_spectral_radius, normalize, average_prob, max_average_through_time
-from .visualisation import show_3d_timeseries, show_2d_timeseries, show_1d_timeseries, neurons_activities_1d, neurons_activities_2d, neurons_activities_3d
+from .visualisation import show_3d_timeseries, show_2d_timeseries, show_1d_timeseries, neurons_activities_1d, neurons_activities_2d, neurons_activities_3d, plot_singular_values
 
 __all__ = [
     'nrmse', 'nmse', 'rmse', 'mse', 'perplexity', 'cumperplexity', 'spectral_radius', 'deep_spectral_radius',
     'normalize', 'average_prob', 'max_average_through_time', 'show_3d_timeseries', 'show_2d_timeseries',
-    'show_1d_timeseries', 'neurons_activities_1d', 'neurons_activities_2d', 'neurons_activities_3d'
+    'show_1d_timeseries', 'neurons_activities_1d', 'neurons_activities_2d', 'neurons_activities_3d',
+    'plot_singular_values'
 ]

echotorch/utils/visualisation.py

@@ -6,6 +6,52 @@
 import numpy as np
 import matplotlib.pyplot as plt
 from mpl_toolkits.mplot3d import Axes3D
+from echotorch.nn.PCACell import PCACell
+from sklearn.decomposition import PCA
+
+
+# Plot singular values
+def plot_singular_values(stats, title, log=False):
+    """
+    Plot singular values
+    :param stats:
+    :param title:
+    :param timestep:
+    :param start:
+    :return:
+    """
+    # PCA cell
+    pca_cell = PCACell(input_dim=stats.shape[1], output_dim=int(stats.shape[1] / 2.0))
+
+    # Feed
+    pca_cell(stats.unsqueeze(0))
+
+    # Finish training
+    pca_cell.finalize()
+
+    # PCA
+    pca = PCA(n_components=stats.shape[1], svd_solver='full')
+    pca.fit(stats)
+
+    # Singular values
+    if not log:
+        singular_values = pca.singular_values_
+    else:
+        singular_values = np.log10(pca.singular_values_)
+    # end if
+
+    # Fig
+    fig = plt.figure()
+    ax = fig.gca()
+
+    # For each plot
+    ax.plot(singular_values, '--o')
+
+    ax.set_xlabel("Timesteps")
+    ax.set_title(title)
+    plt.show()
+    plt.close()
+# end plot_singular_values
 
 
 # Display neurons activities on a 3D plot
@@ -81,7 +127,7 @@ def neurons_activities_2d(stats, neurons, title, colors, timesteps=-1, start=0):
 
 
 # Display neurons activities
-def neurons_activities_1d(stats, neurons, title, timesteps=-1, start=0):
+def neurons_activities_1d(stats, neurons, title, colors, timesteps=-1, start=0):
     """
     Display neurons activities
     :param stats:
@@ -92,11 +138,14 @@ def neurons_activities_1d(stats, neurons, title, timesteps=-1, start=0):
     fig = plt.figure()
     ax = fig.gca()
 
-    if timesteps == -1:
-        ax.plot(stats[:, neurons].numpy())
-    else:
-        ax.plot(stats[start:start + timesteps, neurons].numpy())
-    # end if
+    # For each plot
+    for i, stat in enumerate(stats):
+        if timesteps == -1:
+            ax.plot(stat[:, neurons].numpy(), colors[i])
+        else:
+            ax.plot(stat[start:start + timesteps, neurons].numpy(), colors[i])
+        # end if
+    # end for
 
     ax.set_xlabel("Timesteps")
     ax.set_title(title)