Added ReLU and some cleanup.

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 *.suo
 ConsoleExamples/bin/
 ConsoleExamples/obj/

encog-core-cs/Engine/Network/Activation/ActivationReLU.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Encog.Engine.Network.Activation
+{
+    /// <summary>
+    /// ReLU activation function. This function has a high and low threshold. If
+    /// the high threshold is exceeded a fixed value is returned.Likewise, if the
+    /// low value is exceeded another fixed value is returned.
+    /// </summary>
+    [Serializable]
+    public class ActivationReLU: IActivationFunction
+    {
+
+        /// <summary>
+        /// The ramp low threshold parameter.
+        /// </summary>
+        public const int PARAM_RELU_LOW_THRESHOLD = 0;
+
+        /// <summary>
+        /// The ramp low parameter.
+        /// </summary>
+        public const int PARAM_RELU_LOW = 0;
+
+        /// <summary>
+        /// The parameters.
+        /// </summary>
+        private readonly double[] _params;
+
+
+
+
+        /// <summary>
+        /// Default constructor.
+        /// </summary>
+        public ActivationReLU():
+            this(0,0)
+        {
+        }
+
+        /// <summary>
+        /// Construct a ramp activation function. 
+        /// </summary>
+        /// <param name="thresholdLow">The low threshold value.</param>
+        /// <param name="low">The low value, replaced if the low threshold is exceeded.</param>
+        public ActivationReLU(double thresholdLow, double low)
+        {
+            _params = new double[2];
+            _params[ActivationReLU.PARAM_RELU_LOW_THRESHOLD] = thresholdLow;
+		    _params[ActivationReLU.PARAM_RELU_LOW] = low;
+	    }
+
+
+
+        /// <returns>Return true, Rectifier has a derivative.</returns>
+        public bool HasDerivative
+        {
+            get
+            {
+                return true;
+            }
+        }
+
+        /// <summary>
+        /// Clone the object.
+        /// </summary>
+        /// <returns>The cloned object.</returns>
+        public object Clone()
+        {
+            return new ActivationReLU();
+        }
+
+        /// <inheritdoc />
+        public void ActivationFunction(double[] x, int start, int size)
+        {
+            for (int i = start; i < start + size; i++)
+            {
+                if (x[i] <= _params[ActivationReLU.PARAM_RELU_LOW_THRESHOLD]) {
+				x[i] = _params[ActivationReLU.PARAM_RELU_LOW];
+			}
+}
+        }
+
+        /// <inheritdoc />
+        public double DerivativeFunction(double b, double a)
+        {
+            return 1 / (1 + Math.Pow(Math.E, -a));
+        }
+
+        /// <inheritdoc />
+        public virtual String[] ParamNames
+        {
+            get
+            {
+                String[] result = { };
+                return result;
+            }
+        }
+
+        /// <inheritdoc />
+        public virtual double[] Params
+        {
+            get { return _params; }
+        }
+
+    }
+}

encog-core-cs/encog-core-cs.csproj

@@ -232,6 +232,7 @@
     <Compile Include="Engine\Network\Activation\ActivationLinear.cs" />
     <Compile Include="Engine\Network\Activation\ActivationLOG.cs" />
     <Compile Include="Engine\Network\Activation\ActivationRamp.cs" />
+    <Compile Include="Engine\Network\Activation\ActivationReLU.cs" />
     <Compile Include="Engine\Network\Activation\ActivationSmoothReLU.cs" />
     <Compile Include="Engine\Network\Activation\ActivationSigmoid.cs" />
     <Compile Include="Engine\Network\Activation\ActivationSIN.cs" />

encog-core-test/Engine/Network/Activation/TestActivationReLU.cs

@@ -0,0 +1,32 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Encog.Engine.Network.Activation
+{
+    [TestClass]
+    public class TestActivatioReLU
+    {
+        [TestMethod]
+        public void TestRELU()
+        {
+            var activation = new ActivationReLU();
+            Assert.IsTrue(activation.HasDerivative);
+
+            var clone = (ActivationReLU)activation.Clone();
+            Assert.IsNotNull(clone);
+
+            double[] input = { 0.0 };
+
+            activation.ActivationFunction(input, 0, 1);
+
+            Assert.AreEqual(0.0, input[0], EncogFramework.DefaultDoubleEqual);
+
+            // test derivative, wiki says this is logistic function (test may be wrong - jeroldhaas)
+            input[0] = activation.DerivativeFunction(input[0], input[0]);
+            Assert.AreEqual(0.5, input[0], EncogFramework.DefaultDoubleEqual);
+        }
+    }
+}

encog-core-test/Engine/Network/Activation/TestActivationRectifier.cs → encog-core-test/Engine/Network/Activation/TestActivationSmoothReLU.cs

@@ -4,26 +4,26 @@
 namespace Encog.Engine.Network.Activation
 {
     [TestClass]
-    public class TestActivationRectifier
+    public class TestActivationSmoothReLU
     {
         [TestMethod]
         public void TestRectifier()
         {
-            var activation = new ActivationRectifier();
+            var activation = new ActivationSmoothReLU();
             Assert.IsTrue(activation.HasDerivative);
 
-            var clone = (ActivationRectifier)activation.Clone();
+            var clone = (ActivationSmoothReLU)activation.Clone();
             Assert.IsNotNull(clone);
 
             double[] input = { 0.0 };
 
             activation.ActivationFunction(input, 0, 1);
 
-            Assert.AreEqual(0.0, input[0], 0.5);
+            Assert.AreEqual(0.69314718055994529, input[0], EncogFramework.DefaultDoubleEqual);
 
             // test derivative, wiki says this is logistic function (test may be wrong - jeroldhaas)
             input[0] = activation.DerivativeFunction(input[0], input[0]);
-            Assert.AreEqual(1.0, input[0], 0.1);
+            Assert.AreEqual(0.66666666666666666, input[0], EncogFramework.DefaultDoubleEqual);
         }
     }
 }

encog-core-test/encog-core-test.csproj

@@ -67,7 +67,8 @@
     <Compile Include="Engine\Network\Activation\TestActivationGaussian.cs" />
     <Compile Include="Engine\Network\Activation\TestActivationLinear.cs" />
     <Compile Include="Engine\Network\Activation\TestActivationLOG.cs" />
-    <Compile Include="Engine\Network\Activation\TestActivationRectifier.cs" />
+    <Compile Include="Engine\Network\Activation\TestActivationReLU.cs" />
+    <Compile Include="Engine\Network\Activation\TestActivationSmoothReLU.cs" />
     <Compile Include="Engine\Network\Activation\TestActivationSigmoid.cs" />
     <Compile Include="Engine\Network\Activation\TestActivationSIN.cs" />
     <Compile Include="Engine\Network\Activation\TestActivationSoftMax.cs" />