Single output for multiple timestep input for RNN

src/mlpack/methods/ann/rnn_impl.hpp

@@ -201,15 +201,18 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
   ResetCells();
 
   double performance = 0;
+  size_t responseSeq = 0;
 
   for (size_t seqNum = 0; seqNum < rho; ++seqNum)
   {
     // Wrap a matrix around our data to avoid a copy.
     arma::mat stepData(predictors.slice(seqNum).colptr(begin),
         predictors.n_rows, batchSize, false, true);
     Forward(std::move(stepData));
-    arma::mat respData(responses.slice(seqNum).colptr(begin),
-        responses.n_rows, batchSize, false, true);
+    if (!single)
+    {
+      responseSeq = seqNum;
+    }
 
     if (!deterministic)
     {
@@ -222,7 +225,7 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
 
     performance += outputLayer.Forward(std::move(boost::apply_visitor(
         outputParameterVisitor, network.back())),
-        std::move(arma::mat(responses.slice(seqNum).colptr(begin),
+        std::move(arma::mat(responses.slice(responseSeq).colptr(begin),
             responses.n_rows, batchSize, false, true)));
   }
 
@@ -283,6 +286,14 @@ void RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Gradient(
     {
       error.zeros();
     }
+    else if (single && seqNum == 0)
+    {
+      outputLayer.Backward(std::move(boost::apply_visitor(
+          outputParameterVisitor, network.back())),
+              std::move(arma::mat(responses.slice(0).colptr(begin),
+                  responses.n_rows, batchSize, false, true)),
+          std::move(error));
+    }
     else
     {
       outputLayer.Backward(std::move(boost::apply_visitor(

src/mlpack/tests/recurrent_network_test.cpp

@@ -12,9 +12,11 @@
 #include <mlpack/core.hpp>
 
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/rnn.hpp>
 #include <mlpack/core/data/binarize.hpp>
+#include <mlpack/core/math/random.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -24,6 +26,7 @@
 using namespace mlpack;
 using namespace mlpack::ann;
 using namespace mlpack::optimization;
+using namespace mlpack::math;
 
 BOOST_AUTO_TEST_SUITE(RecurrentNetworkTest);
 
@@ -1085,4 +1088,137 @@ BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest)
   ReberGrammarTestCustomNetwork(16, true);
 }
 
+/**
+ * @brief Generates noisy sine wave and outputs the data and the labels that
+ *        can be used directly for training and testing with RNN.
+ *
+ * @param data The data points as output
+ * @param labels The expected values as output
+ * @param rho The size of the sequence of each data point
+ * @param outputSteps How many output steps to consider for every rho inputs
+ * @param dataPoints  The number of generated data points. The actual generated
+ *        data points may be more than this to adjust to the outputSteps. But at
+ *        the minimum these many data points will be generated.
+ * @param gain The gain on the amplitude
+ * @param freq The frquency of the sine wave
+ * @param phase The phase shift if any
+ * @param noisePercent The percent noise to induce
+ * @param numCycles How many full size wave cycles required. All the data
+ *        points will be fit into these cycles.
+ * @param normalize Whether to normalise the data. This may be required for some
+ *        layers like LSTM. Default is true.
+ */
+void GenerateNoisySinRNN(arma::cube& data,
+                         arma::cube& labels,
+                         size_t rho,
+                         size_t outputSteps = 1,
+                         const int dataPoints = 100,
+                         const double gain = 1.0,
+                         const int freq = 10,
+                         const double phase = 0,
+                         const int noisePercent = 20,
+                         const double numCycles = 6.0,
+                         const bool normalize = true)
+{
+  int points = dataPoints;
+  int r = dataPoints % rho;
+  if (r == 0)
+  {
+    points += outputSteps;
+  }
+  else
+  {
+    points += rho - r + outputSteps;
+  }
+  arma::colvec x(points);
+  int i = 0;
+  double interval = numCycles / freq / points;
+  x.for_each([&i, gain, freq, phase, noisePercent, interval]
+             (arma::colvec::elem_type& val) {
+    double t = interval * (i++);
+    val = gain * ::sin(2 * M_PI * freq * t + phase) +
+          (noisePercent * gain / 100 * Random(0.0, 0.1));
+  });
+
+  arma::colvec y = x;
+  if (normalize)
+    y = arma::normalise(x);
+
+  // Now break this into columns of rho size slices.
+  size_t numColumns = y.n_elem / rho;
+  data = arma::cube(1, numColumns, rho);
+  labels = arma::cube(outputSteps, numColumns, 1);
+  for (size_t i = 0; i < numColumns; ++i)
+  {
+    data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1);
+    labels.subcube(0, i, 0, outputSteps - 1, i, 0) =
+        y.rows(i * rho + rho, i * rho + rho + outputSteps - 1);
+  }
+}
+
+/**
+ * @brief RNNSineTest Test a simple RNN using noisy sine. Use single output
+ *        for multiple inputs.
+ * @param hiddenUnits No of units in the hiddenlayer.
+ * @param rho The input sequence length.
+ * @param numEpochs The number of epochs to run.
+ * @return The mean squared error of the prediction.
+ */
+double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
+{
+  RNN<MeanSquaredError<> > net(rho, true);
+  net.Add<LinearNoBias<> >(1, hiddenUnits);
+  net.Add<LSTM<> >(hiddenUnits, hiddenUnits);
+  net.Add<LinearNoBias<> >(hiddenUnits, 1);
+
+  RMSProp opt(0.005, 100, 0.9, 1e-08, 50000, 1e-5);
+
+  // Generate data
+  arma::cube data;
+  arma::cube labels;
+  GenerateNoisySinRNN(data, labels, rho, 1, 2000, 20.0, 200, 0.0, 45, 20);
+
+  // Break into training and test sets. Simply split along columns.
+  size_t trainCols = data.n_cols * 0.8; // Take 20% out for testing.
+  size_t testCols = data.n_cols - trainCols;
+  arma::cube testData =
+      data.subcube(0, data.n_cols - testCols, 0, data.n_rows - 1,
+                   data.n_cols - 1, data.n_slices - 1);
+  arma::cube testLabels =
+      labels.subcube(0, labels.n_cols - testCols, 0, labels.n_rows - 1,
+                     labels.n_cols - 1, labels.n_slices - 1);
+
+  for (size_t i = 0; i < numEpochs; ++i)
+  {
+    net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1,
+                           data.n_slices - 1),
+              labels.subcube(0, 0, 0, labels.n_rows - 1, trainCols - 1,
+                             labels.n_slices - 1),
+              opt);
+  }
+  // Well now it should be trained. Do the test here.
+  arma::cube prediction;
+  net.Predict(testData, prediction);
+
+  // The prediction must really follow the test data. So convert both the test
+  // data and the pediction to vectors and compare the two.
+  arma::colvec testVector = arma::vectorise(testData);
+  arma::colvec predVector = arma::vectorise(prediction);
+  // Adjust the vectors for comparison, as the prediction is one step ahead.
+  testVector = testVector.rows(1, testVector.n_rows - 1);
+  predVector = predVector.rows(0, predVector.n_rows - 2);
+  double error = std::sqrt(arma::sum(arma::square(testVector - predVector))) /
+                 testVector.n_rows;
+  return error;
+}
+
+/**
+ * Test RNN using multiple timestep input and single output.
+ */
+BOOST_AUTO_TEST_CASE(MultiTimestepTest)
+{
+  double err = RNNSineTest(4, 10, 20);
+  BOOST_REQUIRE_LE(err, 1e-02);
+}
+
 BOOST_AUTO_TEST_SUITE_END();