Cost functions now support Stan Math, Kept the previous classes for backward compatability.

src/shogun/neuralnets/StanNeuralInputLayer.cpp

@@ -0,0 +1,51 @@
+
+
+#include <shogun/neuralnets/StanNeuralInputLayer.h>
+
+using namespace shogun;
+
+StanNeuralInputLayer::StanNeuralInputLayer() : StanNeuralLayer()
+{
+    init();
+}
+
+StanNeuralInputLayer::StanNeuralInputLayer(int32_t num_neurons, int32_t start_index):
+        StanNeuralLayer(num_neurons)
+{
+    init();
+    m_start_index = start_index;
+}
+
+StanNeuralInputLayer::StanNeuralInputLayer(int32_t width, int32_t height,
+                                     int32_t num_channels, int32_t start_index): StanNeuralLayer(width*height*num_channels)
+{
+    init();
+    m_width = width;
+    m_height = height;
+    m_start_index = start_index;
+}
+
+void StanNeuralInputLayer::compute_activations(StanMatrix& inputs, StanVector& parameters)
+{
+    auto biases = parameters.block(0,0,m_num_neurons, 1);
+    StanMatrix& A = m_stan_activations;
+    A.resize(m_num_neurons, m_batch_size);
+    A.colwise() = biases;
+
+    int32_t weights_index_offset = m_num_neurons;
+
+    auto W = parameters.block(weights_index_offset, 0, m_num_neurons * inputs.rows() , 1);
+    W.resize(m_num_neurons, inputs.rows());
+
+    A += W*inputs;
+}
+
+void StanNeuralInputLayer::init()
+{
+    m_start_index = 0;
+    gaussian_noise = 0;
+    SG_ADD(&m_start_index, "start_index",
+           "Start Index", MS_NOT_AVAILABLE);
+    SG_ADD(&gaussian_noise, "gaussian_noise",
+           "Gaussian Noise Standard Deviation", MS_NOT_AVAILABLE);
+}

src/shogun/neuralnets/StanNeuralInputLayer.h

@@ -0,0 +1,93 @@
+
+
+#ifndef SHOGUN_STANNEURALINPUTLAYER_H
+#define SHOGUN_STANNEURALINPUTLAYER_H
+
+#include <shogun/neuralnets/StanNeuralLayer.h>
+#include <shogun/lib/common.h>
+
+namespace shogun
+{
+/** @brief Represents an input layer. The layer can be either
+ * connected to all the input features that a network receives (default) or
+ * connected to just a small part of those features
+ */
+    class StanNeuralInputLayer : public StanNeuralLayer
+    {
+    public:
+        /** default constructor */
+        StanNeuralInputLayer();
+
+        /** Constuctor
+         *
+         * @param num_neurons Number of neurons in this layer
+         *
+         * @param start_index Index of the first feature that the layer connects to,
+         * i.e the activations of the layer are copied from
+         * input_features[start_index:start_index+num_neurons]
+         */
+        StanNeuralInputLayer(int32_t num_neurons, int32_t start_index = 0);
+
+        /** Constructs an input layer that deals with images (for convolutional nets).
+         * Sets the number of neurons to width*height*num_channels
+         *
+         * @param width Width of the image
+         *
+         * @param height Width of the image
+         *
+         * @param num_channels Number of channels
+         *
+         * @param start_index Index of the first feature that the layer connects to,
+         * i.e the activations of the layer are copied from
+         * input_features[start_index:start_index+num_neurons]
+         */
+        StanNeuralInputLayer(int32_t width, int32_t height, int32_t num_channels,
+                          int32_t start_index = 0);
+
+        virtual ~StanNeuralInputLayer() {}
+
+        /** Returns true */
+        virtual bool is_input() { return true; }
+
+        /** Copies inputs[start_index:start_index+num_neurons, :] into the
+         * layer's activations
+         *
+         * @param inputs Input features matrix, size num_features*num_cases
+         * @param parameters are the parameters of the neural network
+         */
+        virtual void compute_activations(StanMatrix& inputs, StanVector& parameters);
+
+        /** Gets the index of the first feature that the layer connects to,
+         * i.e the activations of the layer are copied from
+         * input_features[start_index:start_index+num_neurons]
+         */
+        virtual int32_t get_start_index() { return m_start_index; }
+
+        /** Sets the index of the first feature that the layer connects to,
+         * i.e the activations of the layer are copied from
+         * input_features[start_index:start_index+num_neurons]
+         */
+        virtual void set_start_index(int32_t i) { m_start_index = i; }
+
+        virtual const char* get_name() const { return "StanNeuralInputLayer"; }
+
+    private:
+        void init();
+
+    public:
+        /** Standard deviation of the gaussian noise added to the activations of
+         * the layer. Useful for denoising autoencoders. Default value is 0.0.
+         */
+        float64_t gaussian_noise;
+
+    protected:
+        /** Index of the first feature that the layer connects to,
+         * i.e the activations of the layer are copied from
+         * input_features[start_index:start_index+num_neurons]
+         */
+        int32_t m_start_index;
+    };
+}
+
+
+#endif //SHOGUN_STANNEURALINPUTLAYER_H

src/shogun/neuralnets/StanNeuralLayer.cpp

@@ -0,0 +1,112 @@
+/*
+ * This software is distributed under BSD 3-clause license (see LICENSE file).
+ *
+ * Authors: Elfarouk, Khaled Nasr
+ */
+
+#include <shogun/neuralnets/StanNeuralLayer.h>
+#include <shogun/base/Parameter.h>
+#include <shogun/lib/SGVector.h>
+#include <shogun/mathematics/Math.h>
+
+using namespace shogun;
+
+StanNeuralLayer::StanNeuralLayer()
+: CSGObject()
+{
+	init();
+}
+
+
+StanNeuralLayer::StanNeuralLayer(int32_t num_neurons)
+: CSGObject()
+{
+	init();
+	m_num_neurons = num_neurons;
+	m_width = m_num_neurons;
+	m_height = 1;
+}
+
+StanNeuralLayer::~StanNeuralLayer()
+{
+}
+
+void StanNeuralLayer::initialize_neural_layer(CDynamicObjectArray* layers,
+		SGVector< int32_t > input_indices)
+{
+	m_input_indices = input_indices;
+	m_input_sizes = SGVector<int32_t>(input_indices.vlen);
+
+	for (int32_t i=0; i<m_input_sizes.vlen; i++)
+	{
+		StanNeuralLayer* layer = (StanNeuralLayer*)layers->element(m_input_indices[i]);
+		m_input_sizes[i] = layer->get_num_neurons();
+		SG_UNREF(layer);
+	}
+}
+
+void StanNeuralLayer::set_batch_size(int32_t batch_size)
+{
+	m_batch_size = batch_size;
+
+  m_stan_activations.resize(m_num_neurons, m_batch_size);
+	m_dropout_mask = SGMatrix<bool>(m_num_neurons, m_batch_size);
+}
+
+void StanNeuralLayer::dropout_activations()
+{
+	if (dropout_prop==0.0) return;
+
+	if (is_training)
+	{
+    for(int32_t i=0; i<m_num_neurons; ++i)
+    {
+      for(int32_t j = 0; j<m_batch_size; ++j)
+      {
+        m_dropout_mask(i,j) = CMath::random(0.0,1.0) >= dropout_prop;
+        m_stan_activations(i,j) *= m_dropout_mask(i,j);
+      }
+    }
+	}
+	else
+	{
+    for(int32_t i=0; i<m_num_neurons; ++i)
+    {
+      for(int32_t j = 0; j<m_batch_size; ++j)
+      {
+        m_stan_activations(i,j) *= (1.0 - dropout_prop);
+      }
+    }
+	}
+}
+
+void StanNeuralLayer::init()
+{
+	m_num_neurons = 0;
+	m_width = 0;
+	m_height = 0;
+	m_num_parameters = 0;
+	m_batch_size = 0;
+	dropout_prop = 0.0;
+	is_training = false;
+
+	SG_ADD(&m_num_neurons, "num_neurons",
+	       "Number of Neurons", MS_NOT_AVAILABLE);
+	SG_ADD(&m_width, "width",
+	       "Width", MS_NOT_AVAILABLE);
+	SG_ADD(&m_height, "height",
+	       "Height", MS_NOT_AVAILABLE);
+	SG_ADD(&m_input_indices, "input_indices",
+	       "Input Indices", MS_NOT_AVAILABLE);
+	SG_ADD(&m_input_sizes, "input_sizes",
+	       "Input Sizes", MS_NOT_AVAILABLE);
+	SG_ADD(&dropout_prop, "dropout_prop",
+	       "Dropout Probabilty", MS_NOT_AVAILABLE);
+	SG_ADD(&is_training, "is_training",
+	       "is_training", MS_NOT_AVAILABLE);
+	SG_ADD(&m_batch_size, "batch_size",
+	       "Batch Size", MS_NOT_AVAILABLE);
+	SG_ADD(&m_dropout_mask, "dropout_mask",
+	       "Dropout mask", MS_NOT_AVAILABLE);
+
+}