Merge pull request #639 from karandesai-96/shape-infer-in-activation-…

…layers

Provide automatic shape inference in activation layers.

test/test_average_pooling_layer.h

@@ -10,16 +10,18 @@
 #include "testhelper.h"
 #include "tiny_dnn/tiny_dnn.h"
 
+using namespace tiny_dnn::activation;
+
 namespace tiny_dnn {
 
 TEST(ave_pool, gradient_check) {  // sigmoid - cross-entropy
-  using loss_func        = cross_entropy;
-  using activation_layer = sigmoid_layer;
-  using network          = network<sequential>;
+  using loss_func  = cross_entropy;
+  using activation = sigmoid;
+  using network    = network<sequential>;
 
   network nn;
-  nn << fully_connected_layer(3, 8) << activation_layer(8)
-     << average_pooling_layer(4, 2, 1, 2) << activation_layer(2);  // 4x2 => 2x1
+  nn << fully_connected_layer(3, 8) << activation()
+     << average_pooling_layer(4, 2, 1, 2) << activation();  // 4x2 => 2x1
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -29,14 +31,14 @@ TEST(ave_pool, gradient_check) {  // sigmoid - cross-entropy
 }
 
 TEST(ave_pool, gradient_check2) {  // x-stride
-  using loss_func        = cross_entropy;
-  using activation_layer = sigmoid_layer;
-  using network          = network<sequential>;
+  using loss_func  = cross_entropy;
+  using activation = sigmoid;
+  using network    = network<sequential>;
 
   network nn;
-  nn << fully_connected_layer(3, 8) << activation_layer(8)
+  nn << fully_connected_layer(3, 8) << activation()
      << average_pooling_layer(4, 2, 1, 2, 1, 2, 1)  // 4x2 => 2x2
-     << activation_layer(2, 2, 1);
+     << activation();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -46,14 +48,14 @@ TEST(ave_pool, gradient_check2) {  // x-stride
 }
 
 TEST(ave_pool, gradient_check3) {  // y-stride
-  using loss_func        = cross_entropy;
-  using activation_layer = sigmoid_layer;
-  using network          = network<sequential>;
+  using loss_func  = cross_entropy;
+  using activation = sigmoid;
+  using network    = network<sequential>;
 
   network nn;
-  nn << fully_connected_layer(3, 8) << activation_layer(8)
+  nn << fully_connected_layer(3, 8) << activation()
      << average_pooling_layer(4, 2, 1, 1, 2, 1, 2)  // 4x2 => 4x1
-     << activation_layer(4);
+     << activation();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -63,13 +65,13 @@ TEST(ave_pool, gradient_check3) {  // y-stride
 }
 
 TEST(ave_pool, gradient_check4) {  // padding-same
-  using loss_func        = cross_entropy;
-  using activation_layer = sigmoid_layer;
-  using network          = network<sequential>;
+  using loss_func  = cross_entropy;
+  using activation = sigmoid;
+  using network    = network<sequential>;
 
   network nn;
   nn << average_pooling_layer(4, 2, 1, 2, 2, 1, 1, padding::same)
-     << activation_layer(4, 2, 1);
+     << activation();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();

test/test_average_unpooling_layer.h

@@ -10,18 +10,19 @@
 #include "testhelper.h"
 #include "tiny_dnn/tiny_dnn.h"
 
+using namespace tiny_dnn::activation;
+
 namespace tiny_dnn {
 
 TEST(ave_unpool, gradient_check) {  // sigmoid - cross-entropy
-  using loss_func        = cross_entropy;
-  using activation_layer = sigmoid_layer;
-  using network          = network<sequential>;
+  using loss_func  = cross_entropy;
+  using activation = sigmoid;
+  using network    = network<sequential>;
 
   network nn;
-  nn << fully_connected_layer(3, 4) << activation_layer(4)
+  nn << fully_connected_layer(3, 4) << activation()
      << average_unpooling_layer(2, 2, 1, 2)  // 2x2 => 4x4
-     << activation_layer(4, 4, 1) << average_pooling_layer(4, 4, 1, 2)
-     << activation_layer(2, 2, 1);
+     << activation() << average_pooling_layer(4, 4, 1, 2) << activation();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();

test/test_convolutional_layer.h

@@ -10,6 +10,8 @@
 #include "testhelper.h"
 #include "tiny_dnn/tiny_dnn.h"
 
+using namespace tiny_dnn::activation;
+
 namespace tiny_dnn {
 
 TEST(convolutional, setup_internal) {
@@ -555,7 +557,7 @@ TEST(convolutional, fprop_nnp) {
 
 TEST(convolutional, gradient_check) {  // tanh - mse
   network<sequential> nn;
-  nn << convolutional_layer(5, 5, 3, 1, 1) << tanh_layer(3, 3, 1);
+  nn << convolutional_layer(5, 5, 3, 1, 1) << tanh();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -565,7 +567,7 @@ TEST(convolutional, gradient_check) {  // tanh - mse
 
 TEST(convolutional, gradient_check2) {  // sigmoid - mse
   network<sequential> nn;
-  nn << convolutional_layer(5, 5, 3, 1, 1) << sigmoid_layer(3, 3, 1);
+  nn << convolutional_layer(5, 5, 3, 1, 1) << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -576,7 +578,7 @@ TEST(convolutional, gradient_check2) {  // sigmoid - mse
 TEST(convolutional, gradient_check3) {  // rectified - mse
   network<sequential> nn;
 
-  nn << convolutional_layer(5, 5, 3, 1, 1) << relu_layer(3, 3, 1);
+  nn << convolutional_layer(5, 5, 3, 1, 1) << relu();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -598,7 +600,7 @@ TEST(convolutional, gradient_check4) {  // identity - mse
 TEST(convolutional, gradient_check5) {  // sigmoid - cross-entropy
   network<sequential> nn;
 
-  nn << convolutional_layer(5, 5, 3, 1, 1) << sigmoid_layer(3, 3, 1);
+  nn << convolutional_layer(5, 5, 3, 1, 1) << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -609,7 +611,7 @@ TEST(convolutional, gradient_check5) {  // sigmoid - cross-entropy
 TEST(convolutional, gradient_check6) {  // sigmoid - absolute
   network<sequential> nn;
 
-  nn << convolutional_layer(5, 5, 3, 1, 1) << sigmoid_layer(3, 3, 1);
+  nn << convolutional_layer(5, 5, 3, 1, 1) << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -620,7 +622,7 @@ TEST(convolutional, gradient_check6) {  // sigmoid - absolute
 TEST(convolutional, gradient_check7) {  // sigmoid - absolute eps
   network<sequential> nn;
 
-  nn << convolutional_layer(5, 5, 3, 1, 1) << sigmoid_layer(3, 3, 1);
+  nn << convolutional_layer(5, 5, 3, 1, 1) << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -633,7 +635,7 @@ TEST(convolutional, gradient_check8_pad_same) {  // sigmoid - mse - padding same
 
   nn << convolutional_layer(5, 5, 3, 1, 1, padding::same, true, 1, 1,
                             core::backend_t::internal)
-     << sigmoid_layer(5, 5, 1);
+     << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -646,7 +648,7 @@ TEST(convolutional, gradient_check9_w_stride) {  // sigmoid - mse - w_stride > 1
 
   nn << convolutional_layer(3, 3, 1, 1, 1, padding::valid, true, 2, 1,
                             core::backend_t::internal)
-     << sigmoid_layer(2, 3, 1);
+     << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size(), 1);
   nn.init_weight();
@@ -660,7 +662,7 @@ TEST(convolutional,
 
   nn << convolutional_layer(3, 3, 1, 1, 1, padding::valid, true, 1, 2,
                             core::backend_t::internal)
-     << sigmoid_layer(3, 2, 1);
+     << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size(), 1);
   nn.init_weight();
@@ -677,7 +679,7 @@ TEST(convolutional,
 
   nn << convolutional_layer(7, 7, 3, 3, 1, connections, padding::valid, true, 1,
                             1, core::backend_t::internal)
-     << sigmoid_layer(5, 5, 1);
+     << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -692,7 +694,7 @@ TEST(convolutional,
   nn << fully_connected_layer(10, 5 * 5)
      << convolutional_layer(5, 5, 3, 1, 1, padding::same, true, 1, 1,
                             core::backend_t::internal)
-     << sigmoid_layer(5, 5, 1);
+     << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();

test/test_deconvolutional_layer.h

@@ -10,6 +10,8 @@
 #include "testhelper.h"
 #include "tiny_dnn/tiny_dnn.h"
 
+using namespace tiny_dnn::activation;
+
 namespace tiny_dnn {
 
 TEST(deconvolutional, setup_tiny) {
@@ -195,7 +197,7 @@ TEST(deconvolutional, fprop2) {
 /*
 TEST(deconvolutional, gradient_check) {  // tanh - mse
   network<sequential> nn;
-  nn << deconvolutional_layer(2, 2, 3, 1, 1) << tanh_layer(4, 4, 1);
+  nn << deconvolutional_layer(2, 2, 3, 1, 1) << tanh();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -205,7 +207,7 @@ TEST(deconvolutional, gradient_check) {  // tanh - mse
 
 TEST(deconvolutional, gradient_check2) {  // sigmoid - mse
   network<sequential> nn;
-  nn << deconvolutional_layer(2, 2, 3, 1, 1) << sigmoid_layer(4, 4, 1);
+  nn << deconvolutional_layer(2, 2, 3, 1, 1) << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -216,7 +218,7 @@ TEST(deconvolutional, gradient_check2) {  // sigmoid - mse
 TEST(deconvolutional, gradient_check3) {  // rectified - mse
   network<sequential> nn;
 
-  nn << deconvolutional_layer(2, 2, 3, 1, 1) << relu_layer(4, 4, 1);
+  nn << deconvolutional_layer(2, 2, 3, 1, 1) << relu();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();
@@ -238,7 +240,7 @@ TEST(deconvolutional, gradient_check4) {  // identity - mse
 TEST(deconvolutional, gradient_check5) {  // sigmoid - cross-entropy
   network<sequential> nn;
 
-  nn << deconvolutional_layer(2, 2, 3, 1, 1) << sigmoid_layer(4, 4, 1);
+  nn << deconvolutional_layer(2, 2, 3, 1, 1) << sigmoid();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();

test/test_dropout_layer.h

@@ -12,6 +12,8 @@
 #include "testhelper.h"
 #include "tiny_dnn/tiny_dnn.h"
 
+using namespace tiny_dnn::activation;
+
 namespace tiny_dnn {
 
 TEST(dropout, randomized) {
@@ -73,8 +75,8 @@ TEST(dropout, full_net) {
     train.push_back(t2);
   }
 
-  nn << fully_connected_layer(4, 10) << relu_layer(10) << dropout(10, 0.5)
-     << fully_connected_layer(10, 2) << sigmoid_layer(2);
+  nn << fully_connected_layer(4, 10) << relu() << dropout(10, 0.5)
+     << fully_connected_layer(10, 2) << sigmoid();
 
   nn.train<mse>(optimizer, data, train, 1, 10);
   // batch = 11,20,50
@@ -103,8 +105,8 @@ TEST(dropout, full_net_batch) {
     train.push_back(t2);
   }
 
-  nn << fully_connected_layer(4, 10) << relu_layer(10) << dropout(10, 0.5)
-     << fully_connected_layer(10, 2) << sigmoid_layer(2);
+  nn << fully_connected_layer(4, 10) << relu() << dropout(10, 0.5)
+     << fully_connected_layer(10, 2) << sigmoid();
 
   nn.train<mse>(optimizer, data, train, 20, 10);
 }

test/test_fully_connected_layer.h

@@ -10,13 +10,15 @@
 #include "testhelper.h"
 #include "tiny_dnn/tiny_dnn.h"
 
+using namespace tiny_dnn::activation;
+
 namespace tiny_dnn {
 
 TEST(fully_connected, train) {
   network<sequential> nn;
   adagrad optimizer;
 
-  nn << fully_connected_layer(3, 2) << sigmoid_layer(2);
+  nn << fully_connected_layer(3, 2) << sigmoid();
 
   vec_t a(3), t(2), a2(3), t2(2);
 
@@ -58,7 +60,7 @@ TEST(fully_connected, train_different_batches) {
     network<sequential> nn;
     adagrad optimizer;
 
-    nn << fully_connected_layer(3, 2) << sigmoid_layer(2);
+    nn << fully_connected_layer(3, 2) << sigmoid();
 
     vec_t a(3), t(2), a2(3), t2(2);
 
@@ -97,8 +99,8 @@ TEST(fully_connected, train2) {
   network<sequential> nn;
   gradient_descent optimizer;
 
-  nn << fully_connected_layer(4, 6) << tanh_layer(6)
-     << fully_connected_layer(6, 3) << tanh_layer(3);
+  nn << fully_connected_layer(4, 6) << tanh() << fully_connected_layer(6, 3)
+     << tanh();
 
   vec_t a(4, 0.0), t(3, 0.0), a2(4, 0.0), t2(3, 0.0);
 
@@ -134,7 +136,7 @@ TEST(fully_connected, train2) {
 
 TEST(fully_connected, gradient_check) {
   network<sequential> nn;
-  nn << fully_connected_layer(50, 10) << tanh_layer(10);
+  nn << fully_connected_layer(50, 10) << tanh();
 
   const auto test_data = generate_gradient_check_data(nn.in_data_size());
   nn.init_weight();

test/test_large_thread_count.h

@@ -3,11 +3,13 @@
 #include "testhelper.h"
 #include "tiny_dnn/tiny_dnn.h"
 
+using namespace tiny_dnn::activation;
+
 namespace tiny_dnn {
 
 TEST(test_large_thread_count, test_large_thread_count) {
   network<sequential> net;
-  net << fully_connected_layer(1, 2) << tanh_layer(2);
+  net << fully_connected_layer(1, 2) << tanh();
   adagrad optimizer;
 
   std::vector<vec_t> data;