Suggested improvements + linalg usage

src/shogun/lib/SGMatrix.cpp

@@ -227,6 +227,20 @@ void SGMatrix<T>::zero()
 	set_const(static_cast<T>(0));
 }
 
+template <class T>
+void SGMatrix<T>::random(T min_value, T max_value)
+{
+	for (index_t idx=0; idx<num_rows*num_cols; ++idx) {
+			matrix[idx] = CMath::random(min_value, max_value);
+	}
+}
+
+template <>
+void SGMatrix<complex128_t>::random(complex128_t min_value, complex128_t max_value)
+{
+	SG_SNOTIMPLEMENTED
+}
+
 template <class T>
 bool SGMatrix<T>::is_symmetric() const
 {

src/shogun/lib/SGMatrix.h

@@ -309,6 +309,12 @@ template<class T> class SGMatrix : public SGReferencedData
 		/** fill matrix with zeros */
 		void zero();
 
+		/**
+		 * fill matrix with random numbers between
+		 * [min_range, max_range] closed interval
+		 */
+  		void random(T min_value, T max_value);
+
 		/**
 		 * Checks whether the matrix is symmetric or not. The equality check
 		 * is performed using '==' operators for discrete types (int, char,

tests/unit/neuralnets/NeuralLayerTestFixture.h

@@ -1,45 +1,19 @@
 /*
- * Copyright (c) 2014, Shogun Toolbox Foundation
- * All rights reserved.
+ * This software is distributed under BSD 3-clause license (see LICENSE file).
  *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
-
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * 3. Neither the name of the copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived from this
- * software without specific prior written permission.
-
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- * Written (W) 2018 Saatvik Shah
+ * Authors: Saatvik Shah
  */
 
-#pragma once
+#ifndef NEURAL_LAYER_TEST_FIXTURE_H
+#define NEURAL_LAYER_TEST_FIXTURE_H
 
-#include "shogun/lib/SGMatrix.h"
-#include "shogun/lib/SGVector.h"
-#include "shogun/mathematics/Math.h"
-#include "shogun/neuralnets/NeuralInputLayer.h"
-#include "shogun/neuralnets/NeuralLinearLayer.h"
+#include <shogun/lib/SGMatrix.h>
+#include <shogun/lib/SGVector.h>
+#include <shogun/mathematics/Math.h>
+#include <shogun/neuralnets/NeuralInputLayer.h>
+#include <shogun/neuralnets/NeuralLinearLayer.h>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include <memory>
 #include <tuple>
@@ -49,42 +23,62 @@ using namespace shogun;
 class NeuralLayerTestFixture : public ::testing::Test
 {
 public:
+    /**
+     * Generates a random dataset according to provided specifications.
+     * Then performs the boilerplate needed to setup the input layer
+     * for this dataset.
+     * @tparam T: Type of input data eg. float64_t
+     * @param num_features: For the randomized dataset
+     * @param num_samples: For the randomized dataset
+     * @param lower_bound/upper_bound: Dataset generated has values between: [lower_bound, upper_bound]
+     * @param add_to_layers: Should this layer be added to the dynamic layer list(`m_layers`)
+     * This is normally done when expecting to connect this layer ahead to some other layer
+     * @return: The randomized dataset and corresponding Neural input layer
+     */
 	template <typename T>
-	SGMatrix<T> create_rand_sgmat(
-	    int32_t num_rows, int32_t num_cols, T lower_bound, T upper_bound) const
-	{
-		auto data_v = SGVector<T>(num_rows * num_cols);
-		data_v.random(lower_bound, upper_bound);
-		return SGVector<T>::convert_to_matrix(data_v, num_rows, num_cols, true);
-	}
-
-	template <typename T>
-	auto create_rand_input_layer(
+	auto setup_input_layer(
 	    int32_t num_features, int32_t num_samples, T lower_bound,
-	    T upper_bound) const
-	{
-		auto data_batch = create_rand_sgmat(
-		    num_features, num_samples, lower_bound, upper_bound);
-		// Can't use unique_ptr here due to "unref_all"
+	    T upper_bound, bool add_to_layers = true) {
+		auto data_batch = SGMatrix<T>(num_features, num_samples);
+        data_batch.random(lower_bound, upper_bound);
 		auto input_layer = new CNeuralInputLayer(data_batch.num_rows);
 		input_layer->set_batch_size(data_batch.num_cols);
 		input_layer->compute_activations(data_batch);
+		if(add_to_layers) {
+		    m_layers->append_element(input_layer);
+		}
 		return std::make_tuple(data_batch, input_layer);
 	}
 
-	auto init_neural_linear_layer(
-	    CNeuralLinearLayer* layer, CDynamicObjectArray* layers,
-	    const SGVector<int32_t>& input_indices, int32_t batch_size,
-	    double sigma) const
-	{
-		layer->initialize_neural_layer(layers, input_indices);
+	/**
+	 * Initializes Linear layer metadata according to specifications provided
+	 * @param layer: The layer to initialize
+	 * @param input_indices: the indices of layers from `m_layers` that are inputs to this layer
+	 * @param batch_size: Batch size for this layer
+	 * @param sigma: The parameters are initialized as Normal(0 mean, sigma stdev)
+	 * @param add_to_layers: Whether this layer should be added to the dynamic layer list(`m_layers`)
+	 * This is normally done when expecting to connect this layer ahead to some other layer.
+	 * @return: The initialized parameters
+	 */
+	auto init_linear_layer(
+			CNeuralLinearLayer* layer,
+			const SGVector<int32_t>& input_indices,
+			int32_t batch_size,
+			double sigma,
+			bool add_to_layers) const {
+		if(add_to_layers) {
+			m_layers->append_element(layer);
+		}
+		layer->initialize_neural_layer(m_layers.get(), input_indices);
 		SGVector<float64_t> params(layer->get_num_parameters());
 		SGVector<bool> param_regularizable(layer->get_num_parameters());
 		layer->initialize_parameters(params, param_regularizable, sigma);
 		layer->set_batch_size(batch_size);
-		layer->compute_activations(params, layers);
+		layer->compute_activations(params, m_layers.get());
 		return params;
 	}
+
+	// dynamic list of layers
 	std::unique_ptr<CDynamicObjectArray> m_layers;
 
 protected:
@@ -97,3 +91,4 @@ class NeuralLayerTestFixture : public ::testing::Test
 private:
 	const int SEED = 100;
 };
+#endif // NEURAL_LAYER_TEST_FIXTURE_H

tests/unit/neuralnets/NeuralRectifiedLinearLayer_unittest.cc

@@ -31,14 +31,19 @@
  * Written (W) 2014 Khaled Nasr
  */
 
-#include "shogun/neuralnets/NeuralRectifiedLinearLayer.h"
+#include <shogun/neuralnets/NeuralRectifiedLinearLayer.h>
 #include "NeuralLayerTestFixture.h"
-#include "shogun/lib/SGMatrix.h"
-#include "shogun/lib/SGVector.h"
-#include "shogun/neuralnets/NeuralInputLayer.h"
+#include <shogun/lib/SGMatrix.h>
+#include <shogun/lib/SGVector.h>
+#include <shogun/neuralnets/NeuralInputLayer.h>
+#include <shogun/mathematics/linalg/LinalgNamespace.h>
+#include <shogun/mathematics/linalg/LinalgSpecialPurposes.h>
 
 #include <memory>
 #include <tuple>
+#include <iostream>
+#include <iomanip>
+
 
 using namespace shogun;
 
@@ -52,35 +57,21 @@ TEST_F(NeuralRectifiedLinearLayerTest, compute_activations)
 	// initialize some random inputs
 	SGMatrix<float64_t> x;
 	CNeuralInputLayer* input;
-	std::tie(x, input) = create_rand_input_layer<float64_t>(12, 3, -10.0, 10.0);
-	m_layers->append_element(input);
+	std::tie(x, input) = setup_input_layer<float64_t>(12, 3, -10.0, 10.0);
 
 	// initialize the rectified linear layer
 	CNeuralRectifiedLinearLayer layer(9);
 	SGVector<int32_t> input_indices(1);
 	input_indices[0] = 0;
-	auto params = init_neural_linear_layer(
-	    &layer, m_layers.get(), input_indices, x.num_cols, 1.0);
+	auto params = init_linear_layer(&layer, input_indices, x.num_cols, 1.0, false);
 	SGMatrix<float64_t> A = layer.get_activations();
 
-	// manually compute the layer's activations
+	// Manually compute Recitified linear activations
+	auto biases = SGVector<float64_t>(params.vector, layer.get_num_neurons(), 0);
+	auto weights = SGMatrix<float64_t>(params.vector, layer.get_num_neurons(), x.num_rows, layer.get_num_neurons());
 	SGMatrix<float64_t> A_ref(layer.get_num_neurons(), x.num_cols);
-
-	float64_t* biases = params.vector;
-	float64_t* weights = biases + layer.get_num_neurons();
-
-	for (int32_t i = 0; i < A_ref.num_rows; i++)
-	{
-		for (int32_t j = 0; j < A_ref.num_cols; j++)
-		{
-			A_ref(i, j) = biases[i];
-
-			for (int32_t k = 0; k < x.num_rows; k++)
-				A_ref(i, j) += weights[i + k * A_ref.num_rows] * x(k, j);
-
-			A_ref(i, j) = CMath::max<float64_t>(0, A_ref(i, j));
-		}
-	}
+    shogun::linalg::add_vector(shogun::linalg::matrix_prod(weights, x), biases, A_ref);
+    shogun::linalg::rectified_linear(A_ref, A_ref);
 
 	// compare
 	EXPECT_EQ(A_ref.num_rows, A.num_rows);
@@ -95,33 +86,28 @@ TEST_F(NeuralRectifiedLinearLayerTest, compute_activations)
  */
 TEST_F(NeuralRectifiedLinearLayerTest, compute_parameter_gradients_hidden)
 {
-
 	// initialize some random inputs
 	SGMatrix<float64_t> x1, x2;
 	CNeuralInputLayer *input1, *input2;
-	std::tie(x1, input1) =
-	    create_rand_input_layer<float64_t>(12, 3, -10.0, 10.0);
-	std::tie(x2, input2) =
-	    create_rand_input_layer<float64_t>(7, 3, -10.0, 10.0);
-	m_layers->append_element(input1);
-	m_layers->append_element(input2);
+	std::tie(x1, input1) = setup_input_layer<float64_t>(12, 3, -10.0, 10.0);
+	std::tie(x2, input2) = setup_input_layer<float64_t>(7, 3, -10.0, 10.0);
 
 	// initialize the hidden rectified linear layer
 	CNeuralLinearLayer* layer_hid = new CNeuralRectifiedLinearLayer(5);
-	m_layers->append_element(layer_hid);
 	SGVector<int32_t> input_indices_hid(2);
 	input_indices_hid[0] = 0;
 	input_indices_hid[1] = 1;
-	auto param_hid = init_neural_linear_layer(
-	    layer_hid, m_layers.get(), input_indices_hid, x1.num_cols, 0.01);
+	auto param_hid = init_linear_layer(
+	    layer_hid, input_indices_hid, x1.num_cols, 0.01, true);
 
 	// initialize the output layer
-	auto y = create_rand_sgmat<float64_t>(9, 3, 0.0, 1.0);
+    auto y = SGMatrix<float64_t >(9, 3);
+    y.random(0.0, 1.0);
 	CNeuralLinearLayer layer_out(y.num_rows);
 	SGVector<int32_t> input_indices_out(1);
 	input_indices_out[0] = 2;
-	auto param_out = init_neural_linear_layer(
-	    &layer_out, m_layers.get(), input_indices_out, x1.num_cols, 0.01);
+	auto param_out = init_linear_layer(
+	    &layer_out, input_indices_out, x1.num_cols, 0.01, false);
 
 	// compute gradients
 	layer_hid->get_activation_gradients().zero();