fix cnn demo with correct relu and stable softmax

cfg/eteq.yml

@@ -1146,13 +1146,18 @@ api:
                       logs::fatalf("cannot perform softmax on dimensions beyond %d",
                           teq::rank_cap);
                   }
-                  auto exarg = exp(arg);
-                  teq::Shape shape = exarg->shape();
+                  teq::Shape shape = arg->shape();
+                  auto overflow_preventer = ::tenncor::extend(
+                      ::tenncor::reduce_max(arg), 0,
+                          std::vector<teq::DimT>(shape.begin(), shape.end()));
+                  auto exarg = ::tenncor::exp(::tenncor::sub(arg, overflow_preventer));
                   auto it = shape.begin() + offset;
                   std::vector<teq::DimT> xlist(it, it + ndims);
                   auto out = ::tenncor::div(exarg,
-                      ::tenncor::extend(::tenncor::reduce_sum(exarg, offset, ndims),
-                      offset, xlist));
+                      ::tenncor::extend(::tenncor::add(
+                          ::tenncor::reduce_sum(exarg, offset, ndims),
+                          std::numeric_limits<T>::epsilon()),
+                          offset, xlist));
                   tag::recursive_group_tag(out->get_tensor(), "softmax", {
                       arg->get_tensor().get()});
                   return out;
@@ -1165,10 +1170,23 @@ api:
           type: eteq::NodeptrT<T>
           val: |
               //
-                  auto out = ::tenncor::log(::tenncor::add((T) 1, ::tenncor::exp(arg)));
+                  auto out = ::tenncor::max(arg, (T) 0);
                   tag::recursive_group_tag(out->get_tensor(), "relu", {
                       arg->get_tensor().get()});
                   return out;
+      - template: typename T
+        name: softplus
+        args:
+        - dtype: eteq::NodeptrT<T>
+          name: arg
+        out:
+          type: eteq::NodeptrT<T>
+          val: |
+              //
+                  auto out = ::tenncor::log(::tenncor::add((T) 1, ::tenncor::exp(arg)));
+                  tag::recursive_group_tag(out->get_tensor(), "softplus", {
+                      arg->get_tensor().get()});
+                  return out;
       - template: typename T
         name: sign
         args:
@@ -1209,14 +1227,6 @@ api:
                       eteq::convert_to_node(eteq::make_variable_scalar<T>((T) 1, shape)));
                   return ::tenncor::lt(trial, arg);
     tenncor::nn:
-      - template: typename T
-        name: relu
-        args:
-        - dtype: eteq::NodeptrT<T>
-          name: x
-        out:
-          type: eteq::NodeptrT<T>
-          val: return ::tenncor::max(x,eteq::make_constant_scalar<T>(0,x->shape()));
       - template: typename T
         name: fully_connect
         args:

eteq/functor.hpp

@@ -86,7 +86,7 @@ struct Functor final : public teq::iOperableFunc
 		{
 			initialize();
 		}
-		return out_->assign();
+		out_->assign();
 	}
 
 	/// Implementation of iData

eteq/test/ptest.py

@@ -423,7 +423,7 @@ def test_relu(self):
         if 'elementary.shape' in _test_data:
             shapes += _test_data['elementary.shape']
         for shape in shapes:
-            self._common_unary_tf(shape, tc.nn.relu, tf.nn.relu)
+            self._common_unary_tf(shape, tc.relu, tf.nn.relu)
 
     def test_square(self):
         shapes = [[3, 4, 5]]

rocnnet/demo/cnn_demo.py

@@ -31,6 +31,7 @@
 momentum = 0.9
 weight_decay = 0.0001
 show_every_n = 5
+nepochs = 10
 
 ds = tfds.load('cifar10',
     split=tfds.Split.TRAIN,
@@ -59,7 +60,7 @@
 model.add(rcn.Dense(10, 320,
     weight_init=rcn.unif_xavier_init(),
     bias_init=rcn.zero_init(), label="fc")) # outputs [nbatch, 10]
-model.add(rcn.softmax(0))
+model.add(rcn.softmax(1))
 
 sess = eteq.Session()
 
@@ -74,10 +75,11 @@
 raw_inshape[0] = nbatch
 train_inshape = raw_inshape
 train_outshape = [nbatch, 10]
-train_input = eteq.Variable(train_inshape)
-train_output = eteq.Variable(train_outshape)
+train_input = eteq.Variable(train_inshape, label="trainin")
+train_output = eteq.Variable(train_outshape, label="trainout")
+normalized = train_input / 255 - 0.5
 train = rcn.sgd_train(model, sess,
-    train_input, train_output, rcn.get_sgd(0.9))
+    normalized, train_output, rcn.get_sgd(0.5))
 
 sess.optimize("cfg/optimizations.rules")
 
@@ -86,14 +88,19 @@
     labels = np.zeros((nbatch, 10))
     for j, label in enumerate(data['label']):
         labels[j][label] = 1
+    print('expected label:\n{}'.format(labels))
     train_input.assign(data['image'].astype(np.float))
     train_output.assign(labels.astype(np.float))
-    trained_err = train()
-    print('done episode {}'.format(i))
-    if i % show_every_n == show_every_n - 1:
-        err = trained_err.as_numpy()
-        print('training {}\ntraining error:\n{}'
-            .format(i + 1, err))
+    for j in range(nepochs):
+        trained_err = train()
+        print('done epoch {}'.format(j))
+        if j % show_every_n == show_every_n - 1:
+            guess_err = trained_err.as_numpy()
+            err = np.max(guess_err, axis=1)
+            print(('training {}th image, epoch {}\n'+
+                'training error:\n{}\n'+
+                'training error max:\n{}')
+                .format(i, j + 1, guess_err, err))
 
 # test
 tds = tfds.load('cifar10',

todo

@@ -14,7 +14,7 @@ P2 - implement attention mechanisms
    - separate supervised and unsupervised trainers (since they have different statefulness)
 
 === demos ===
-P2 - demo convolutional nn
+P2 - improve convolutional nn demo
    - switch rbm and dbn demos to use tensorflow datasets
 
 === feature ===
@@ -44,8 +44,8 @@ P4 - implement GPU settings
 P4 - improve session implementation
 P4 - investigate remote sessions for distributed processing
 
-graph-construction-time performance:
-P4 - define optimization conversion rules to increase data accuracy
+stability:
+P3 - define optimization conversion rules to increase data stability
 
 compile-time performance:
 P3 - convert as many shared_ptr to unique_ptr