More cleaning up

hf_example.py

@@ -1,13 +1,18 @@
-""" Sanity check dataset: RNN trained with hf optimizer
-PYTHONPATH=$HOME/python/theano-hf:$PYTHONPATH ipython -i hf_example.py
+"""
+This code uses the recurrent neural net implementation in rnn.py
+but trains it using Hessian-Free optimization.
+
+It requires the theano-hf package:
+https://github.com/boulanni/theano-hf
+
 """
 from rnn import MetaRNN
 from hf import SequenceDataset, hf_optimizer
 import numpy as np
 import matplotlib.pyplot as plt
+import logging
 
-
-def test_real():
+def test_real(n_updates=100):
     """ Test RNN with real-valued outputs. """
     n_hidden = 10
     n_in = 5
@@ -43,23 +48,25 @@ def test_real():
                        s=model.rnn.y_pred,
                        costs=[model.rnn.loss(model.y)], h=model.rnn.h)
 
-    opt.train(gradient_dataset, cg_dataset)
+    opt.train(gradient_dataset, cg_dataset, num_updates=n_updates)
 
     plt.close('all')
     fig = plt.figure()
     ax1 = plt.subplot(211)
     plt.plot(seq[0])
+    ax1.set_title('input')
     ax2 = plt.subplot(212)
     true_targets = plt.plot(targets[0])
 
     guess = model.predict(seq[0])
     guessed_targets = plt.plot(guess, linestyle='--')
     for i, x in enumerate(guessed_targets):
         x.set_color(true_targets[i].get_color())
+    ax2.set_title('solid: true output, dashed: model output')
 
 
 def test_binary(multiple_out=False, n_updates=250):
-    """ Test RNN with binary-valued outputs. """
+    """ Test RNN with binary outputs. """
     n_hidden = 10
     n_in = 5
     if multiple_out:
@@ -114,6 +121,7 @@ def test_binary(multiple_out=False, n_updates=250):
         fig = plt.figure()
         ax1 = plt.subplot(211)
         plt.plot(seq[seq_num])
+        ax1.set_title('input')
         ax2 = plt.subplot(212)
         true_targets = plt.step(xrange(n_steps), targets[seq_num], marker='o')
 
@@ -123,9 +131,11 @@ def test_binary(multiple_out=False, n_updates=250):
         for i, x in enumerate(guessed_targets):
             x.set_color(true_targets[i].get_color())
         ax2.set_ylim((-0.1, 1.1))
+        ax2.set_title('solid: true output, dashed: model output (prob)')
 
 
 def test_softmax(n_updates=250):
+    """ Test RNN with softmax outputs. """
     n_hidden = 10
     n_in = 5
     n_steps = 10
@@ -161,7 +171,7 @@ def test_softmax(n_updates=250):
 
     model = MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out,
                     activation='tanh', output_type='softmax',
-                    symbolic_softmax=True)
+                    use_symbolic_softmax=True)
 
     # optimizes negative log likelihood
     # but also reports zero-one error
@@ -181,17 +191,21 @@ def test_softmax(n_updates=250):
         fig = plt.figure()
         ax1 = plt.subplot(211)
         plt.plot(seq[seq_num])
-        ax2 = plt.subplot(212)
+        ax1.set_title('input')
 
+        ax2 = plt.subplot(212)
         # blue line will represent true classes
         true_targets = plt.step(xrange(n_steps), targets[seq_num], marker='o')
 
         # show probabilities (in b/w) output by model
         guess = model.predict_proba(seq[seq_num])
         guessed_probs = plt.imshow(guess.T, interpolation='nearest',
                                    cmap='gray')
+        ax2.set_title('blue: true class, grayscale: probs assigned by model')
+
 
 if __name__ == "__main__":
-    # test_real()
+    logging.basicConfig(level=logging.INFO)
+    #test_real(n_updates=20)
     #test_binary(multiple_out=True, n_updates=20)
     test_softmax(n_updates=20)

rnn.py

@@ -451,6 +451,7 @@ def fit(self, X_train, Y_train, X_test=None, Y_test=None,
 
 
 def test_real():
+    """ Test RNN with real-valued outputs. """
     n_hidden = 10
     n_in = 5
     n_out = 3
@@ -491,6 +492,7 @@ def test_real():
 
 
 def test_binary(multiple_out=False, n_epochs=250):
+    """ Test RNN with binary outputs. """
     n_hidden = 10
     n_in = 5
     if multiple_out:
@@ -541,6 +543,7 @@ def test_binary(multiple_out=False, n_epochs=250):
 
 
 def test_softmax(n_epochs=250):
+    """ Test RNN with softmax outputs. """
     n_hidden = 10
     n_in = 5
     n_steps = 10
@@ -592,7 +595,7 @@ def test_softmax(n_epochs=250):
 
 
 if __name__ == "__main__":
-    # logging.basicConfig(level=logging.INFO)
+    logging.basicConfig(level=logging.INFO)
     t0 = time.time()
     test_real()
     # problem takes more epochs to solve

rnn_minibatch.py

@@ -782,6 +782,7 @@ def callback(theta_value):
 
 
 def test_real(n_epochs=1000):
+    """ Test RNN with real-valued outputs. """
     n_hidden = 10
     n_in = 5
     n_out = 3
@@ -824,6 +825,7 @@ def test_real(n_epochs=1000):
 
 
 def test_binary(multiple_out=False, n_epochs=1000, optimizer='cg'):
+    """ Test RNN with binary outputs. """
     n_hidden = 10
     n_in = 5
     if multiple_out:
@@ -878,6 +880,7 @@ def test_binary(multiple_out=False, n_epochs=1000, optimizer='cg'):
 
 
 def test_softmax(n_epochs=250, optimizer='cg'):
+    """ Test RNN with softmax outputs. """
     n_hidden = 10
     n_in = 5
     n_steps = 10
@@ -930,6 +933,7 @@ def test_softmax(n_epochs=250, optimizer='cg'):
                                    cmap='gray')
         ax2.set_title('blue: true class, grayscale: probs assigned by model')
 
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     t0 = time.time()