added math notation page

code/loss_functions.py

@@ -0,0 +1,33 @@
+import math
+import numpy as np
+
+
+def CrossEntropy(yHat, y):
+    pass
+
+
+def Hinge(yHat, y):
+    pass
+
+
+def KLDivergence(yHat, y):
+    pass
+
+
+def L1(yHat, y):
+    pass
+
+
+def L2(yHat, y):
+    pass
+
+
+def MLE(yHat, y):
+    pass
+
+
+def MSE(yHat, y):
+    return np.sum((yHat - y)**2) / 2.0
+
+def MSE_prime(yHat, y):
+    return yHat - y

code/nn_matrix.py

@@ -0,0 +1,82 @@
+import math
+import numpy as np
+
+# Neural Network w Matrices
+
+'''
+X  - Input matrix from training set
+Zh - Hidden layer weighted input matrix
+Zo - Output layer weighted input matrix
+Bh - Hidden layer bias matrix
+Bo - Output layer bias matrix
+H  - Hidden layer activation matrix
+yHat  - Output layer predictions
+'''
+
+# Initialize Weights
+Wh = np.random.randn(inputLayerSize, hiddenLayerSize) * \
+            np.sqrt(2.0/inputLayerSize)
+Wo = np.random.randn(hiddenLayerSize, outputLayerSize) * \
+            np.sqrt(2.0/hiddenLayerSize)
+
+# Initialize Biases
+Bh = np.full((1, hiddenLayerSize), 0.1)
+Bo = np.full((1, outputLayerSize), 0.1)
+
+
+def relu(Z):
+    return np.maximum(0, Z)
+
+def feed_forward(X):
+
+    # Hidden layer
+    Zh = np.dot(X, Wh) + Bh
+    H = relu(Zh)
+
+    # Output layer
+    Zo = np.dot(H, Wo) + Bo
+    yHat = relu(Zo)
+    return yHat
+
+def relu_prime(Z):
+    '''
+    Z - weighted input matrix
+    Returns the gradient of the
+    Z matrix where all negative
+    values are switched to 0 and
+    all positive values switched to 1
+    '''
+    Z[Z < 0] = 0
+    Z[Z > 0] = 1
+    return Z
+
+def cost(yHat, y):
+    cost = np.sum((yHat - y)**2) / 2.0
+    return cost
+
+def cost_prime(yHat, y):
+    return yHat - y
+
+def backprop(X, y, lr):
+
+    yHat = feed_forward(X)
+
+    # Layer Error
+    Eo = (yHat - y) * relu_prime(Zo)
+    Eh = np.dot(Eo, Wo.T) * relu_prime(Zh)
+
+    # Cost derivative for weights
+    dWo = np.dot(H.T, Eo)
+    dWh = np.dot(X.T, Eh)
+
+    # Cost derivative for bias
+    dBo = np.sum(Eo, axis=0, keepdims=True)
+    dBh = np.sum(Eh, axis=0, keepdims=True)
+
+    # Update weights
+    Wo -= lr * dWo
+    Wh -= lr * dWh
+
+    # Update biases
+    Bo -= lr * dBo
+    Bh -= lr * dBh

code/optimizers.py

@@ -0,0 +1,36 @@
+import math
+import numpy as np
+
+
+def Adadelta(data):
+    pass
+
+
+def Adagrad(data):
+    pass
+
+
+def Adam(data):
+    pass
+
+
+def LBFGS(data):
+    pass
+
+
+def RMSProp(data):
+    pass
+
+
+def SGD(data, batch_size, lr):
+    N = len(data)
+    np.random.shuffle(data)
+    mini_batches = np.array([data[i:i+batch_size]
+     for i in range(0, N, batch_size)])
+    for X,y in mini_batches:
+        backprop(X, y, lr)
+
+
+def SGD_Momentum():
+    pass
+

docs/images/calculus_symbol_table.tgn

@@ -0,0 +1 @@
+{"rows_views":[[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"middle","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"middle","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}]],"model":{"rows":[[{"value":"Symbol","cspan":1,"rspan":1},{"value":"Name","cspan":1,"rspan":1},{"value":"Description","cspan":1,"rspan":1},{"value":"Example","cspan":1,"rspan":1}],[{"value":"x'","cspan":1,"rspan":1},{"value":"derivative","cspan":1,"rspan":1},{"value":"first derivative","cspan":1,"rspan":1},{"value":"(x^2)' = 2x","cspan":1,"rspan":1}],[{"value":"x''","cspan":1,"rspan":1},{"value":"second derivative","cspan":1,"rspan":1},{"value":"second derivative","cspan":1,"rspan":1},{"value":"(x^2)'' = 2","cspan":1,"rspan":1}],[{"value":"lim(x-->0)","cspan":1,"rspan":1},{"value":"limit","cspan":1,"rspan":1},{"value":"function value as x approaches 0","cspan":1,"rspan":1},{"value":"","cspan":1,"rspan":1}],[{"value":"∇","cspan":1,"rspan":1},{"value":"nabla","cspan":1,"rspan":1},{"value":"gradient","cspan":1,"rspan":1},{"value":"∇f(a,b,c)","cspan":1,"rspan":1}]]},"theme":null,"fixed_layout":false}

docs/images/linearalgebra.tgn

@@ -0,0 +1 @@
+{"rows_views":[[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}],[{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}},{"style":{"borders":"lrtb","font_style":{},"text_color":"","bg_color":"","halign":"left","valign":"top","padding":{"top":10,"right":5,"bottom":10,"left":5}}}]],"model":{"rows":[[{"value":"Symbol","cspan":1,"rspan":1},{"value":"Name","cspan":1,"rspan":1},{"value":"Description","cspan":1,"rspan":1},{"value":"Example","cspan":1,"rspan":1}],[{"value":"[ ] ","cspan":1,"rspan":1},{"value":"brackets","cspan":1,"rspan":1},{"value":"matrix or vector","cspan":1,"rspan":1},{"value":"v = [1 3 5]","cspan":1,"rspan":1}],[{"value":"\\cdot","cspan":1,"rspan":1},{"value":"dot","cspan":1,"rspan":1},{"value":"dot product ","cspan":1,"rspan":1},{"value":"(Z = X \\cdot W","cspan":1,"rspan":1}],[{"value":"\\odot","cspan":1,"rspan":1},{"value":"hadamard","cspan":1,"rspan":1},{"value":"hadamard product","cspan":1,"rspan":1},{"value":"A = B \\odot C","cspan":1,"rspan":1}],[{"value":"X^T","cspan":1,"rspan":1},{"value":"transpose","cspan":1,"rspan":1},{"value":"matrix transpose","cspan":1,"rspan":1},{"value":"W^T \\cdot X ","cspan":1,"rspan":1}],[{"value":"\\arrow x","cspan":1,"rspan":1},{"value":"vector","cspan":1,"rspan":1},{"value":"","cspan":1,"rspan":1},{"value":"s = \\frac{1}{1+e^{-z}}","cspan":1,"rspan":1}],[{"value":"X","cspan":1,"rspan":1},{"value":"matrix","cspan":1,"rspan":1},{"value":"capitalized variables are matrices","cspan":1,"rspan":1},{"value":"X, W, B","cspan":1,"rspan":1}],[{"value":"\\hat x","cspan":1,"rspan":1},{"value":"unit vector","cspan":1,"rspan":1},{"value":"vector of magnitude 1","cspan":1,"rspan":1},{"value":"\\hat x = [0.2 0.5 0.3]","cspan":1,"rspan":1}]]},"theme":null,"fixed_layout":false}