Preliminary support for exponential linear units.

aigamedev · Nov 26, 2015 · b29272a · b29272a
1 parent e0c553c
commit b29272a
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 4 deletions.
diff --git a/sknn/backend/lasagne/mlp.py b/sknn/backend/lasagne/mlp.py
@@ -28,6 +28,10 @@
 from ...nn import Layer, Convolution, ansi
 
 
+def explin(x):
+    return x * (x>=0) + (x<0) * (T.exp(x) - 1)
+
+
 class MultiLayerPerceptronBackend(BaseBackend):
     """
     Abstract base class for wrapping the multi-layer perceptron functionality
@@ -98,7 +102,8 @@ def _get_activation(self, l):
                           'Sigmoid': nl.sigmoid,
                           'Tanh': nl.tanh,
                           'Softmax': nl.softmax,
-                          'Linear': nl.linear}
+                          'Linear': nl.linear,
+                          'ExpLin': explin}
 
         assert l.type in nonlinearities,\
             "Layer type `%s` is not supported for `%s`." % (l.type, l.name)

diff --git a/sknn/nn.py b/sknn/nn.py
@@ -89,7 +89,7 @@ def __init__(
         assert warning is None,\
             "Specify layer parameters as keyword arguments, not positional arguments."
 
-        if type not in ['Rectifier', 'Sigmoid', 'Tanh', 'Linear', 'Softmax', 'Gaussian']:
+        if type not in ['Rectifier', 'Sigmoid', 'Tanh', 'Linear', 'Softmax', 'Gaussian', 'ExpLin']:
             raise NotImplementedError("Layer type `%s` is not implemented." % type)
 
         self.name = name

diff --git a/sknn/tests/test_deep.py b/sknn/tests/test_deep.py
@@ -23,6 +23,7 @@ def setUp(self):
             layers=[
                 L("Rectifier", units=16),
                 L("Sigmoid", units=12),
+                L("ExpLin", units=8),
                 L("Tanh", units=4),
                 L("Linear")],
             n_iter=1)
@@ -45,7 +46,7 @@ def setUp(self):
     def run_EqualityTest(self, copier, asserter):
         # Only PyLearn2 supports Maxout.
         extra =  ["Maxout"] if sknn.backend.name == 'pylearn2' else []
-        for activation in ["Rectifier", "Sigmoid", "Tanh"] + extra:
+        for activation in ["Rectifier", "Sigmoid", "Tanh", "ExpLin"] + extra:
             nn1 = MLPR(layers=[L(activation, units=16, pieces=2), L("Linear", units=1)], random_state=1234)
             nn1._initialize(self.a_in, self.a_out)
 
@@ -105,4 +106,3 @@ def test_UnusedParameterWarning(self):
 
         assert_in('Parameter `pieces` is unused', self.buf.getvalue())
         self.buf = io.StringIO() # clear
-