NN: Added ReLu Activation, Softmax Loss and TopologyMaker

mltool.cabal

@@ -31,8 +31,11 @@ library
                      , MachineLearning.NeuralNetwork
                      , MachineLearning.NeuralNetwork.Layer
                      , MachineLearning.NeuralNetwork.Regularization
+                     , MachineLearning.NeuralNetwork.ReluActivation
+                     , MachineLearning.NeuralNetwork.SoftmaxLoss
                      , MachineLearning.NeuralNetwork.Sigmoid
                      , MachineLearning.NeuralNetwork.Topology
+                     , MachineLearning.NeuralNetwork.TopologyMaker
                      , MachineLearning.NeuralNetwork.WeightInitialization
                      , MachineLearning.PCA
                      , MachineLearning.Clustering

samples/neural_networks/Main.hs

@@ -4,7 +4,7 @@ import qualified Numeric.LinearAlgebra as LA
 import qualified MachineLearning as ML
 import qualified MachineLearning.Optimization as Opt
 import qualified MachineLearning.NeuralNetwork as NN
-import qualified MachineLearning.NeuralNetwork.Sigmoid as Sigmoid
+import qualified MachineLearning.NeuralNetwork.TopologyMaker as TM
 import qualified MachineLearning.TerminalProgress as TP
 
 
@@ -18,7 +18,7 @@ main = do
   (xTest, yTest) <- pure ML.splitToXY <*> LA.loadMatrix "samples/digits_classification/optdigits.tes"
 
   -- Step 2. Initialize Neural Network.
-  let nnt = Sigmoid.makeTopology (LA.cols x) 10 [100, 100]
+  let nnt = TM.makeTopology TM.ASigmoid TM.LSigmoid (LA.cols x) 10 [100, 100]
       model = NN.NeuralNetwork nnt
   -- Step 3. Initialize theta with randon values.
       initTheta = NN.initializeTheta 5191711 nnt

src/MachineLearning/NeuralNetwork/ReluActivation.hs

@@ -0,0 +1,30 @@
+{-|
+Module: MachineLearning.NeuralNetwork.ReluActivation
+Description: ReLu Activation
+Copyright: (c) Alexander Ignatyev, 2017
+License: BSD-3
+Stability: experimental
+Portability: POSIX
+
+ReLu Activation.
+-}
+
+module MachineLearning.NeuralNetwork.ReluActivation
+(
+  relu
+  , gradient
+)
+
+where
+
+
+import qualified Numeric.LinearAlgebra as LA
+import MachineLearning.Types (Matrix)
+
+
+relu :: Matrix -> Matrix
+relu x = x * (LA.step x)
+
+
+gradient :: Matrix -> Matrix -> Matrix
+gradient x dx = dx * (LA.step x)  -- == dx[x<0] = 0

src/MachineLearning/NeuralNetwork/Sigmoid.hs

@@ -11,46 +11,26 @@ Sigmoid
 
 module MachineLearning.NeuralNetwork.Sigmoid
 (
-    makeTopology
+    LM.sigmoid
+    , gradient
+    , loss
+    , outputGradient
 )
 
 where
 
 
-import qualified Data.Vector.Storable as V
 import qualified Numeric.LinearAlgebra as LA
-import MachineLearning.Types (R, Vector, Matrix)
+import MachineLearning.Types (R, Matrix)
 import qualified MachineLearning.LogisticModel as LM
-import qualified MachineLearning.NeuralNetwork.Topology as T
-import MachineLearning.NeuralNetwork.Layer (Layer(..), affineForward, affineBackward)
-import MachineLearning.NeuralNetwork.WeightInitialization (nguyen)
-
-
--- | Creates toplogy. Takes number of inputs, number of outputs and list of hidden layers.
-makeTopology :: Int -> Int -> [Int] -> T.Topology
-makeTopology nInputs nOutputs hlUnits = T.makeTopology nInputs hiddenLayers outputLayer loss
-  where hiddenLayers = map mkAffineSigmoidLayer hlUnits
-        outputLayer = mkSigmoidOutputLayer nOutputs
-
-
-mkAffineSigmoidLayer nUnits = Layer {
-  lUnits = nUnits
-  , lForward = affineForward
-  , lActivation = LM.sigmoid
-  , lBackward = affineBackward
-  , lActivationGradient = \z da -> da * LM.sigmoidGradient z
-  , lInitializeThetaM = nguyen
-  }
-
-
-mkSigmoidOutputLayer nUnits = Layer {
-  lUnits = nUnits
-  , lForward = affineForward
-  , lActivation = LM.sigmoid
-  , lBackward = affineBackward
-  , lActivationGradient = \scores y -> scores - y
-  , lInitializeThetaM = nguyen
-  }
+
+
+gradient :: Matrix -> Matrix -> Matrix
+gradient z da = da * LM.sigmoidGradient z
+
+
+outputGradient :: Matrix -> Matrix -> Matrix
+outputGradient scores y = scores - y
 
 
 -- Sigmoid Loss function

src/MachineLearning/NeuralNetwork/SoftmaxLoss.hs

@@ -0,0 +1,41 @@
+{-|
+Module: MachineLearning.NeuralNetwork.SoftmaxLoss
+Description: Softmax Loss.
+Copyright: (c) Alexander Ignatyev, 2017
+License: BSD-3
+Stability: experimental
+Portability: POSIX
+
+Softmax Loss.
+-}
+
+module MachineLearning.NeuralNetwork.SoftmaxLoss
+(
+  scores
+  , gradient
+  , loss
+)
+
+where
+
+
+import qualified Data.Vector.Storable as V
+import qualified Numeric.LinearAlgebra as LA
+import MachineLearning.Types (R, Matrix)
+import MachineLearning.Utils (sumByRows, reduceByRows)
+
+scores x = x - reduceByRows V.maximum x
+
+
+gradient scores y =
+  let sum_probs = sumByRows $ exp scores
+      probs = (exp scores) / sum_probs
+  in probs - y
+
+
+-- Softmax Loss function
+loss :: Matrix -> [(Matrix, Matrix)] -> Matrix -> R
+loss scores thetaList y = LA.sumElements $ (log sum_probs) - t
+  where m = fromIntegral $ LA.rows scores
+        sum_probs = sumByRows $ exp scores
+        t = sumByRows $ scores * y

src/MachineLearning/NeuralNetwork/TopologyMaker.hs

@@ -0,0 +1,79 @@
+{-|
+Module: MachineLearning.NeuralNetwork.TopologyMaker
+Description: Topology Maker
+Copyright: (c) Alexander Ignatyev, 2017
+License: BSD-3
+Stability: experimental
+Portability: POSIX
+
+Topology Maker.
+-}
+
+module MachineLearning.NeuralNetwork.TopologyMaker
+(
+  Activation(..)
+  , Loss(..)
+  , makeTopology
+)
+
+where
+
+
+import qualified MachineLearning.NeuralNetwork.Topology as T
+import MachineLearning.NeuralNetwork.Layer (Layer(..), affineForward, affineBackward)
+import MachineLearning.NeuralNetwork.WeightInitialization (nguyen)
+import qualified MachineLearning.NeuralNetwork.ReluActivation as Relu
+import qualified MachineLearning.NeuralNetwork.SoftmaxLoss as Softmax
+import qualified MachineLearning.NeuralNetwork.Sigmoid as Sigmoid
+
+
+data Activation = ASigmoid | ARelu
+
+data Loss = LSigmoid | LSoftmax
+
+
+-- | Creates toplogy. Takes number of inputs, number of outputs and list of hidden layers.
+makeTopology :: Activation -> Loss -> Int -> Int -> [Int] -> T.Topology
+makeTopology a l nInputs nOutputs hlUnits = T.makeTopology nInputs hiddenLayers outputLayer (loss l)
+  where hiddenLayers = map (mkAffineLayer a) hlUnits
+        outputLayer = mkOutputLayer l nOutputs
+
+
+mkAffineLayer a nUnits = Layer {
+  lUnits = nUnits
+  , lForward = affineForward
+  , lActivation = hiddenActivation a
+  , lBackward = affineBackward
+  , lActivationGradient = hiddenGradient a
+  , lInitializeThetaM = nguyen
+  }
+
+
+mkOutputLayer l nUnits = Layer {
+  lUnits = nUnits
+  , lForward = affineForward
+  , lActivation = outputActivation l
+  , lBackward = affineBackward
+  , lActivationGradient = outputGradient l
+  , lInitializeThetaM = nguyen
+  }
+
+
+hiddenActivation ASigmoid = Sigmoid.sigmoid
+hiddenActivation ARelu = Relu.relu
+
+
+hiddenGradient ASigmoid = Sigmoid.gradient
+hiddenGradient ARelu = Relu.gradient
+
+
+outputActivation LSigmoid = Sigmoid.sigmoid
+outputActivation LSoftmax = Softmax.scores
+
+
+outputGradient LSigmoid = Sigmoid.outputGradient
+outputGradient LSoftmax = Softmax.gradient
+
+
+loss LSigmoid = Sigmoid.loss
+loss LSoftmax = Softmax.loss

test/MachineLearning/NeuralNetwork/TopologyTest.hs

@@ -12,9 +12,9 @@ import Test.HUnit.Approx
 import Test.HUnit.Plus
 import qualified Numeric.LinearAlgebra as LA
 import MachineLearning.NeuralNetwork.Topology
-import qualified MachineLearning.NeuralNetwork.Sigmoid as Sigmoid
+import qualified MachineLearning.NeuralNetwork.TopologyMaker as TM
 
-nnt = Sigmoid.makeTopology 15 2 [10]
+nnt = TM.makeTopology TM.ASigmoid TM.LSigmoid 15 2 [10]
 
 flattenTest = do
   theta <- initializeThetaIO nnt

test/MachineLearning/NeuralNetworkTest.hs

@@ -19,20 +19,19 @@ import qualified MachineLearning as ML
 import qualified MachineLearning.Optimization as Opt
 import MachineLearning.Model
 import MachineLearning.NeuralNetwork
-import qualified MachineLearning.NeuralNetwork.Sigmoid as Sigmoid
+import qualified MachineLearning.NeuralNetwork.TopologyMaker as TM
 
 (x, y) = ML.splitToXY dataset2
 
-nnt = Sigmoid.makeTopology (LA.cols x) 2 [10]
-model = NeuralNetwork nnt
-
 gradientCheckingEps = 0.1
 
-checkGradientTest lambda = do
-  let thetas = initializeTheta 1511197 nnt
+checkGradientTest eps activation loss lambda = do
+  let nnt = TM.makeTopology activation loss (LA.cols x) 2 [10]
+      model = NeuralNetwork nnt
+      thetas = initializeTheta 1511197 nnt
       diffs = take 5 $ map (\e -> Opt.checkGradient model lambda x y thetas e) [0.005, 0.0051 ..]
       diff = minimum $ filter (not . isNaN) diffs
-  assertApproxEqual (show thetas) gradientCheckingEps 0 diff
+  assertApproxEqual (show thetas) eps 0 diff
 
 
 xPredict = LA.matrix 2 [ -0.5, 0.5
@@ -42,10 +41,10 @@ xPredict = LA.matrix 2 [ -0.5, 0.5
                        , 0, 0]
 yExpected = LA.vector [1, 1, 0, 0, 1]
 
-learnTest minMethod =
+learnTest activation loss minMethod =
   let lambda = 0.5 / (fromIntegral $ LA.rows x)
       x1 = ML.mapFeatures 2 x
-      nnt = Sigmoid.makeTopology (LA.cols x1) 2 [10]
+      nnt = TM.makeTopology activation loss (LA.cols x1) 2 [10]
       model = NeuralNetwork nnt
       xPredict1 = ML.mapFeatures 2 xPredict
       initTheta = initializeTheta 5191711 nnt
@@ -57,10 +56,13 @@ learnTest minMethod =
 
 
 tests = [ testGroup "gradient checking" [
-            testCase "non-zero lambda" $ checkGradientTest 0.01
-            , testCase "zero lambda" $ checkGradientTest 0
+            testCase "Sigmoid: non-zero lambda" $ checkGradientTest 0.1 TM.ASigmoid TM.LSigmoid 0.01
+            , testCase "Sigmoid: zero lambda" $ checkGradientTest 0.1 TM.ASigmoid TM.LSigmoid 0
+            , testCase "ReLU - Softmax: non-zero lambda" $ checkGradientTest 12 TM.ARelu TM.LSoftmax 0.01
+            , testCase "ReLU - Softmax: zero lambda" $ checkGradientTest 12 TM.ARelu TM.LSoftmax 0
               ]
         , testGroup "learn" [
-            testCase "BFGS" $ learnTest (Opt.BFGS2 0.01 0.7)
+            testCase "Sigmoid: BFGS" $ learnTest TM.ASigmoid TM.LSigmoid (Opt.BFGS2 0.01 0.7)
+            , testCase "ReLU - Softmax: BFGS" $ learnTest TM.ARelu TM.LSoftmax (Opt.BFGS2 0.1 0.2)
             ]
         ]