added batchwise prediction in validation step

NNFlow/binary_mlp.py

@@ -274,13 +274,13 @@ def train(self, train_data, val_data, epochs=10, batch_size=128,
             init = tf.global_variables_initializer()
             saver = tf.train.Saver(weights + biases + [x_mean, x_std])
 
-        train_start = time.time()
+
 
         # dont allocate all available gpu memory, remove if you can dont share a
         # machine with others
         config = tf.ConfigProto()
         config.gpu_options.allow_growth = True
-
+        #config.gpu_options.per_process_gpu_memory_fraction = 0.1
         with tf.Session(config=config, graph=train_graph) as sess:
             self.model_loc = self.savedir + '/{}.ckpt'.format(self.name)
             sess.run(init)
@@ -289,7 +289,7 @@ def train(self, train_data, val_data, epochs=10, batch_size=128,
             val_auc = []
             train_loss = []
             early_stopping  = {'auc': 0.0, 'epoch': 0}
-             
+            epoch_durations = []
             print(90*'-')
             print('Train model: {}'.format(self.model_loc))
             print(90*'-')
@@ -298,7 +298,8 @@ def train(self, train_data, val_data, epochs=10, batch_size=128,
                 'AUC Validation Score'))
             print(90*'-')
 
-            for epoch in range(epochs):
+            for epoch in range(1, epochs+1):
+                epoch_start = time.time()
                 total_batches = int(train_data.n/batch_size)
                 epoch_loss = 0
                 for _ in range(total_batches):
@@ -308,15 +309,24 @@ def train(self, train_data, val_data, epochs=10, batch_size=128,
                                                y: train_y,
                                                w: train_w})
                     epoch_loss += batch_loss
+
                 # monitor training
+                train_data.shuffle()
                 train_loss.append(epoch_loss/total_batches)
-                train_pre = sess.run(yy_, {x : train_data.x})
-                train_auc.append(roc_auc_score(train_data.y, train_pre))
+                train_pre = []
+                for batch in range(0, total_batches):
+                    train_x, _, _ = train_data.next_batch(batch_size)
+                    pred = sess.run(yy_, {x: train_x})
+                    train_pre.append(pred)
+                train_pre = np.concatenate(train_pre, axis=0)
+                train_auc.append(
+                    roc_auc_score(train_data.y[:total_batches*batch_size],
+                                  train_pre))
                 val_pre = sess.run(yy_, {x : val_data.x})
                 val_auc.append(roc_auc_score(val_data.y, val_pre))
 
                 print('{:^20} | {:^20.4e} | {:^20.4f} | {:^25.4f}'
-                      .format(epoch+1, train_loss[-1],
+                      .format(epoch, train_loss[-1],
                               train_auc[-1], val_auc[-1]))
 
                 if early_stop:
@@ -325,9 +335,9 @@ def train(self, train_data, val_data, epochs=10, batch_size=128,
                     if val_auc[-1] > early_stopping['auc']:
                         save_path = saver.save(sess, self.model_loc)
                         early_stopping['auc'] = val_auc[-1]
-                        early_stopping['epoch'] = epoch+1
+                        early_stopping['epoch'] = epoch
                         early_stopping['val_pre'] = val_pre
-                    elif (epoch+1 - early_stopping['epoch']) > early_stop:
+                    elif (epoch - early_stopping['epoch']) >= early_stop:
                         print(125*'-')
                         print('Validation AUC has not increased for {} epochs. ' \
                               'Achieved best validation auc score of {:.4f} ' \
@@ -339,18 +349,29 @@ def train(self, train_data, val_data, epochs=10, batch_size=128,
                     save_path = saver.save(sess, self.model_loc)
 
                 # set internal dataframe index to 0
-                train_data.shuffle()
 
-            train_end = time.time()
-            train_pre = sess.run(yy_, {x: train_data.x})
+                epoch_end = time.time()
+                epoch_durations.append(epoch_end - epoch_start)
+
+            # get training prediction for validation, use batches to prevent
+            # allocating to much gpu memory
+            train_data.shuffle()
+            evts_per_batch = 100
+            total_batches = int(train_data.n/evts_per_batch)
+            train_pre = []
+            for batch in range(0, total_batches):
+                train_x, _, _ = train_data.next_batch(evts_per_batch)
+                pred = sess.run(yy_, {x: train_x})
+                train_pre.append(pred)
+            train_pre = np.concatenate(train_pre, axis=0)
             print(90*'-')
-            self._validation(train_pre, train_data.y,
+            self._validation(train_pre, train_data.y[:total_batches*evts_per_batch],
                              early_stopping['val_pre'], val_data.y)
-            self._plot_auc_dev(train_auc, val_auc, epochs)
+            self._plot_auc_dev(train_auc, val_auc, early_stopping['epoch'])
             self._plot_loss(train_loss)
             self.trained = True
             self._write_parameters(batch_size, keep_prob, beta,
-                                   (train_end - train_start), early_stopping)
+                                   np.mean(epoch_durations), early_stopping)
             print('Model saved in: {}'.format(save_path))
             print(90*'-')
 
@@ -457,7 +478,7 @@ def _write_parameters(self, batch_size, keep_prob, beta, time,
             f.write('Batch Size: {}\n'.format(batch_size))
             f.write('Dropout: {}\n'.format(keep_prob))
             f.write('L2 Regularization: {}\n'.format(beta))
-            f.write('Training Time: {} s\n'.format(time))
+            f.write('Mean Training Time per Epoch: {} s\n'.format(time))
 
         with open('{}/NN_Info.txt'.format(self.savedir), 'r') as f:
             for line in f:
@@ -506,11 +527,11 @@ def seperate_sig_bkg(pred, labels):
 
         plt.legend(loc='upper left')
         plt.xlabel('Netzwerk Ausgabe')
-        plt.ylabel('normierte Ereignisse')
+        plt.ylabel('Ereignisse (normiert)')
         if self.variables:
             plt.title(self.variables, loc='left')
         plt.title(self.name, loc='center')
-        plt.title('CMS Private Work', loc='right')
+        # plt.title('CMS Private Work', loc='right')
 
 
         plt_name = self.name + '_dist'
@@ -538,7 +559,7 @@ def seperate_sig_bkg(pred, labels):
             plt.title(self.variables, loc='left')
         plt.grid(True)
         plt.title(self.name, loc='center')
-        plt.title('CMS Private Work', loc='right')
+        # plt.title('CMS Private Work', loc='right')
         plt.legend(loc='best')
 
         plt.savefig(self.savedir + '/' + plt_name + '.pdf')
@@ -549,41 +570,41 @@ def seperate_sig_bkg(pred, labels):
     def _plot_loss(self, train_loss):
         """Plot loss of training and validation data.
         """
-        train_loss = np.array(train_loss)*10**6
 
         plt.plot(train_loss, label= 'Trainingsfehler', color='#1f77b4', ls='',
                  marker='^')
         plt.xlabel('Epoche')
-        plt.ylabel('Fehlerfunktion (mult. mit 10^6)')
+        plt.ylabel('Fehlerfunktion')
 
         if self.variables:
             plt.title(self.variables, loc='left')
         plt.title(self.name, loc='center')
-        plt.title('CMS Private Work', loc='right')
-        # plt.ticklabel_format(axis='y', style='sci', scilimits=(-2,2))
-        plt.legend(loc=0, frameon=False)
+        # plt.title('CMS Private Work', loc='right')
+        plt.ticklabel_format(axis='y', style='sci', scilimits=(-2,2))
+        plt.legend(loc=0)
 
         plt_name = self.name + '_loss'
         plt.savefig(self.savedir + '/' + plt_name + '.pdf')
         plt.savefig(self.savedir + '/' + plt_name + '.png')
         plt.savefig(self.savedir + '/' + plt_name + '.eps')
         plt.clf()
 
-    def _plot_auc_dev(self, train_auc, val_auc, nepochs):
+    def _plot_auc_dev(self, train_auc, val_auc, stop):
         """Plot ROC-AUC-Score development
         """
-        plt.plot(train_auc, color='#1f77b4', label='Training', ls='',
+        plt.plot(range(1, len(train_auc)+1), train_auc, color='#1f77b4', label='Training', ls='',
                  marker='^')
-        plt.plot(val_auc, color='#ff7f0e', label='Validierung', ls='',
+        plt.plot(range(1, len(val_auc)+1), val_auc, color='#ff7f0e', label='Validierung', ls='',
                  marker='^')
 
         # make plot nicer
         plt.xlabel('Epoche')
         plt.ylabel('ROC Integral')
+        plt.axvline(x=stop, color='r')
         if self.variables:
             plt.title(self.variables, loc='left')
         plt.title(self.name, loc='center')
-        plt.title('CMS Private Work', loc='right')
+        # plt.title('CMS Private Work', loc='right')
         plt.legend(loc='best', frameon=False)
 
         # save plot

NNFlow/preprocessing.py

@@ -211,7 +211,7 @@ def _check_category(self, N_LL, N_TL, N_J, N_BTM, name):
                     '44': (N_LL == 1 and N_TL == 1 and N_J == 4 and N_BTM == 4),
                     '53': (N_LL == 1 and N_TL == 1 and N_J == 5 and N_BTM == 3),
                     '54': (N_LL == 1 and N_TL == 1 and N_J == 5 and N_BTM >= 4),
-                    '62': (N_LL == 1 and N_TL == 1 and N_J >= 6 and N_BTM == 2),
+                    '62': (N_LL == 1 and N_TL == 1 and N_J >= 6 and N_BTM >= 2),
                     '63': (N_LL == 1 and N_TL == 1 and N_J >= 6 and N_BTM == 3),
                     '64': (N_LL == 1 and N_TL == 1 and N_J >= 6 and N_BTM >= 4),
                     'all': True}

README.md

@@ -15,7 +15,7 @@ name = 'array.npy'
 my_tree = 'tree'
 my_branches = ['branch_1', 'branch_2']
 
-conv_root_to_np(save_to, name, ntuples, tree=my_tree, branches=my_branches)
+conv_root_to_np(save_to, name, ntuples, treename=my_tree, branches=my_branches)
 ```
 If there is only one TTree in the nTuple you do not have to provide the name of the tree which should be converted.
 If you want to convert all branches of a tree, do not provide a branchlist.
@@ -61,9 +61,15 @@ import numpy as np
 from NNFlow.binary_mlp import BinaryMLP
 from NNFlow.data_frame import Dataframe
 
-train = 'my_variables/category/train.npy'
-val = 'my_variables/category/val.npy'
-test = 'my_variables/category/test.npy'
+# load numpy arrays
+train = np.load('my_variables/category/train.npy')
+val = np.load('my_variables/category/val.npy')
+test = np.load('my_variables/category/test.npy')
+
+# use DataFrame
+train = DataFrame(train)
+val = DataFrame(val)
+test = DataFrame(test)
 
 save_model_to = 'my_variables/category/models/2x100'
 hidden_layers = [100, 100]