added batch norm

code/layers.py

@@ -0,0 +1,67 @@
+import math
+import numpy as np
+
+
+def BatchNorm():
+    # From https://wiseodd.github.io/techblog/2016/07/04/batchnorm/
+    # TODO: Add doctring for variable names. Add momentum to init.
+    def __init__(self):
+        pass
+
+    def forward(self, X, gamma, beta):
+        mu = np.mean(X, axis=0)
+        var = np.var(X, axis=0)
+
+        X_norm = (X - mu) / np.sqrt(var + 1e-8)
+        out = gamma * X_norm + beta
+
+        cache = (X, X_norm, mu, var, gamma, beta)
+
+        return out, cache, mu, var
+
+    def backward(self, dout, cache):
+        X, X_norm, mu, var, gamma, beta = cache
+
+        N, D = X.shape
+
+        X_mu = X - mu
+        std_inv = 1. / np.sqrt(var + 1e-8)
+
+        dX_norm = dout * gamma
+        dvar = np.sum(dX_norm * X_mu, axis=0) * -.5 * std_inv**3
+        dmu = np.sum(dX_norm * -std_inv, axis=0) + dvar * np.mean(-2. * X_mu, axis=0)
+
+        dX = (dX_norm * std_inv) + (dvar * 2 * X_mu / N) + (dmu / N)
+        dgamma = np.sum(dout * X_norm, axis=0)
+        dbeta = np.sum(dout, axis=0)
+
+        return dX, dgamma, dbeta
+
+
+def Adagrad(data):
+    pass
+
+
+def Adam(data):
+    pass
+
+
+def LBFGS(data):
+    pass
+
+
+def RMSProp(data):
+    pass
+
+
+def SGD(data, batch_size, lr):
+    N = len(data)
+    np.random.shuffle(data)
+    mini_batches = np.array([data[i:i+batch_size]
+     for i in range(0, N, batch_size)])
+    for X,y in mini_batches:
+        backprop(X, y, lr)
+
+
+def SGD_Momentum():
+    pass

docs/index.rst

@@ -19,7 +19,6 @@ Brief visual explanations of machine learning concepts with diagrams, code examp
     training
     glossary
 
-
 .. toctree::
     :caption: Math
     :maxdepth: 1
@@ -30,7 +29,6 @@ Brief visual explanations of machine learning concepts with diagrams, code examp
     Statistics (empty) <statistics>
     math_notation
 
-
 .. toctree::
     :maxdepth: 1
     :caption: Neural Networks
@@ -39,8 +37,9 @@ Brief visual explanations of machine learning concepts with diagrams, code examp
     forwardpropagation
     backpropagation
     activation_functions
+    layers
     loss_functions
-    optimization
+    optimizers
     regularization
 
 .. toctree::

docs/layers.rst

@@ -0,0 +1,75 @@
+.. _layers:
+
+======
+Layers
+======
+
+.. contents:: :local:
+
+
+BatchNorm
+---------
+
+Accelerates convergence by reducing internal covariate shift inside each batch.
+If the individual observations in the batch are widely different, the gradient
+updates will be choppy and take longer to converge.
+
+The batch norm layer normalizes the incoming activations and outputs a new batch
+where the new mean equals 0 and standard deviation equals 1. It subtracts the mean
+and divides by the standard deviation of the batch.
+
+.. rubric:: Code
+
+Code example from `Agustinus Kristiadi <https://wiseodd.github.io/techblog/2016/07/04/batchnorm/>`_
+
+.. literalinclude:: ../code/layers.py
+      :pyobject: BatchNorm
+
+.. rubric:: Further reading
+
+- `Original Paper <https://arxiv.org/abs/1502.03167>`_
+- `Implementing BatchNorm in Neural Net <https://wiseodd.github.io/techblog/2016/07/04/batchnorm/>`_
+- `Understanding the backward pass through Batch Norm <https://kratzert.github.io/2016/02/12/understanding-the-gradient-flow-through-the-batch-normalization-layer.html>`_
+
+
+Convolution
+-----------
+
+Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
+
+
+Dropout
+-------
+
+Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
+
+
+Linear
+------
+
+Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
+
+
+LSTM
+----
+
+Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
+
+
+Pooling
+-------
+
+Max and average pooling layers.
+
+Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
+
+
+RNN
+---
+
+Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
+
+
+.. rubric:: References
+
+.. [1] http://www.deeplearningbook.org/contents/convnets.html

docs/optimization.rst → docs/optimizers.rst

@@ -1,34 +1,10 @@
 .. _optimizers:
 
-============
-Optimization
-============
-
-.. contents:: :local:
-
-
-General
-========
-
-Batch Normalization
--------------------
-
-Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
-
-First-order Methods
--------------------
-
-Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
-
-Second-order Methods
---------------------
-
-Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
-
-
+==========
 Optimizers
 ==========
 
+.. contents:: :local:
 
 Adadelta
 --------
@@ -86,52 +62,16 @@ RMSProp
 Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
 
 
-Stochastic Gradient Descent
----------------------------
+SGD
+---
+
+Stochastic Gradient Descent.
 
 .. literalinclude:: ../code/optimizers.py
     :language: python
     :pyobject: SGD
 
 
-
-Weight Initialization
-=====================
-
-`Good reference <https://github.com/alykhantejani/nninit/blob/master/nninit.py>`_
-
-Kaiming (He)
-------------
-
-Normal and uniform varieties..
-
-Normal
-------
-
-Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
-
-Orthogonal
-----------
-
-Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
-
-Sparse
-------
-
-Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
-
-Uniform
--------
-
-Be the first to `contribute! <https://github.com/bfortuner/ml-cheatsheet>`__
-
-Xavier
-------
-
-Normal and uniform varieties..
-
-
-
 .. rubric:: References
 
 .. [1] http://sebastianruder.com/optimizing-gradient-descent/