Feature/plots (#125)

* Fix SNR plot for 1 layer architecture

* Show epoch number in infoplane movie

* Readjust frequency of saved epoch activations

* Test activation patterns with re-normalizing weights

* Set max_norm_weights as experiment parameter

* Change keras import

* Improve parameter handling

* Add plot for activations of all neurons over epochs

* Experiment with modifications to entropy estimation

* Fix parameter layer size

* Refactor plotting code

* Fix first layer neuron count

* Include review fixes

iclr_wrap_up/configs/basic.json

@@ -1,15 +1,15 @@
 {
-  "epochs": 150,
+  "epochs": 1500,
   "batch_size": 256,
   "architecture": [10, 7, 5, 4, 3],
   "optimizer": "adam",
   "learning_rate": 0.0004,
-  "calculate_mi_for": "test",
-  "activation_fn": "tanh",
+  "calculate_mi_for": "full_dataset",
+  "activation_fn": "relu",
   "model": "models.feedforward",
   "dataset": "datasets.harmonics",
-  "estimator": "mi_estimator.upper",
-  "discretization_range": 1e-3,
+  "estimator": "mi_estimator.binning",
+  "discretization_range": 0.07,
   "callbacks": [],
-  "n_runs": 1
+  "n_runs": 2
 }

iclr_wrap_up/experiment.py

@@ -29,7 +29,8 @@ def hyperparams():
     plotters = [('plotter.informationplane', []),
                 ('plotter.snr', []),
                 ('plotter.informationplane_movie', []),
-                ('plotter.activations', [])
+                ('plotter.activations', []),
+                ('plotter.activations_single_neuron', [])
                 ]
 
 

iclr_wrap_up/plotter/activations_single_neuron.py

@@ -0,0 +1,84 @@
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import numpy as np
+import pandas as pd
+
+from iclr_wrap_up.plotter.base import BasePlotter
+
+
+def load(run, dataset):
+    return SingleNeuronActivityPlotter(run, dataset)
+
+
+class SingleNeuronActivityPlotter(BasePlotter):
+    plotname = 'single_neuron_activations'
+
+    def __init__(self, run, dataset):
+        self.dataset = dataset
+        self.run = run
+
+    def _grab_activations(self, measures_summary):
+        activations_summary = measures_summary['activations_summary']
+        activations_df = pd.DataFrame(activations_summary).transpose()
+        all_activations = activations_df['activations']
+        return all_activations
+
+    def _get_number_of_layers(self, all_activations):
+        return len(all_activations[0])
+
+    def _get_number_of_neurons_in_layer(self, all_activations, layer):
+        return all_activations[0][layer].shape[1]
+
+    def _create_histogram(self, all_activations, layer_number):
+        neurons_in_layer = self._get_number_of_neurons_in_layer(all_activations, layer_number)
+        hist = [[] for x in range(neurons_in_layer)]
+        for epoch, activations in all_activations.items():
+            layer_activations = activations[layer_number].transpose()
+            for neuron_number in range(neurons_in_layer):
+                histogram_per_neuron, _ = np.histogram(layer_activations[neuron_number], bins=30)
+                hist[neuron_number].append(histogram_per_neuron)
+        return hist
+
+    def plot(self, measures_summary):
+
+        all_activations = self._grab_activations(measures_summary)
+        neurons_in_first_layer = self._get_number_of_neurons_in_layer(all_activations, 0)
+        num_layers = self._get_number_of_layers(all_activations)
+
+        fig = plt.figure()
+        gs = gridspec.GridSpec(neurons_in_first_layer * 2, num_layers)
+
+        for layer_number in range(num_layers):
+
+            neurons_in_layer = self._get_number_of_neurons_in_layer(all_activations, layer_number)
+            hist = self._create_histogram(all_activations, layer_number)
+
+            for neuron_number in range(neurons_in_layer):
+                hist_df = pd.DataFrame(hist[neuron_number])
+
+                # vertical offset for plotting optic
+                plotting_offset = (neurons_in_first_layer - neurons_in_layer) / 2
+                gs_y_index = int((neuron_number + plotting_offset) * 2)
+
+                ax = fig.add_subplot(gs[gs_y_index:gs_y_index + 2, layer_number])
+
+                ax.set_ylabel("bins")
+                yticks = np.arange(0, hist_df.shape[1], 5)
+                ax.set_yticks(yticks)
+                ax.set_yticklabels(yticks)
+
+                ax.set_xlabel("epoch")
+                xticks = np.arange(0, hist_df.shape[0], 5)
+                ax.set_xticks(xticks)
+                ax.set_xticklabels(all_activations.index[xticks], rotation=90)
+
+                activity_map = ax.imshow(hist_df.transpose(), cmap="viridis", interpolation='nearest')
+                counts_colorbar = fig.colorbar(activity_map)
+                counts_colorbar.set_label("Absolute frequency")
+                ax.set_title(f"Layer {layer_number}, Neuron {neuron_number}")
+
+        fig.set_figheight(24)
+        fig.set_figwidth(48)
+        fig.tight_layout()
+
+        return fig

iclr_wrap_up/plotter/base.py

@@ -8,7 +8,7 @@ class BasePlotter:
     def generate(self, measures_summary):
         fig = self.plot(measures_summary)
         filename = f'plots/{self.plotname}.png'
-        fig.savefig(filename, bbox_inches='tight', dpi=600)
+        fig.savefig(filename, bbox_inches='tight', dpi=300)
 
         self.run.add_artifact(filename, name=self.plotname)