Use tensor.nnet.relu for ReLU calculations. (#1070)

* Use tensor.nnet.relu for ReLU calculations.

We punt the decision of how to implement max(0, x) or max(alpha * x, x) most efficiently to Theano, via `tensor.nnet.relu`. This is what Lasagne and Keras do, which between them have > 10x as many GitHub stars, so I am willing to treat this solution as "battle-tested".

(Finally) closes #960.

N.B. This was done via a quick edit on the GitHub web interface so the branch is on `mila-udem`, I'll delete it when merged.

* Fix LeakyRectifier test sensitivity.

Tests were failing with this new implementation due to an unduly high sensitivity.

blocks/bricks/simple.py

@@ -281,7 +281,7 @@ def apply(self, input_):
 class Rectifier(Activation):
     @application(inputs=['input_'], outputs=['output'])
     def apply(self, input_):
-        return tensor.switch(input_ > 0, input_, 0)
+        return tensor.nnet.relu(input_)
 
 
 class LeakyRectifier(Activation):
@@ -300,14 +300,15 @@ class LeakyRectifier(Activation):
     .. Maas, Andrew L., Awni Y. Hannun, and Andrew Y. Ng. Rectifier
        nonlinearities improve neural network acoustic models. Proc.
        ICML. Vol. 30. 2013.
+
     """
     def __init__(self, leak=0.01, **kwargs):
         super(LeakyRectifier, self).__init__(**kwargs)
         self._leak = leak
 
     @application(inputs=['input_'], outputs=['output'])
     def apply(self, input_):
-        return tensor.switch(input_ > 0, input_, self._leak * input_)
+        return tensor.nnet.relu(input_, alpha=self._leak)
 
 
 class Softmax(Brick):

tests/bricks/test_bricks.py

@@ -343,11 +343,13 @@ def test_activations():
     leaky_out_1 = x_val - 0.5
     leaky_out_1[leaky_out_1 < 0] *= 0.01
     assert_allclose(leaky_out_1,
-                    LeakyRectifier().apply(x).eval({x: x_val - 0.5}))
+                    LeakyRectifier().apply(x).eval({x: x_val - 0.5}),
+                    rtol=1e-5)
     leaky_out_2 = x_val - 0.5
     leaky_out_2[leaky_out_2 < 0] *= 0.05
     assert_allclose(leaky_out_2,
-                    LeakyRectifier(leak=0.05).apply(x).eval({x: x_val - 0.5}))
+                    LeakyRectifier(leak=0.05).apply(x).eval({x: x_val - 0.5}),
+                    rtol=1e-5)
 
 
 def test_mlp():