Task 8 | Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IgRyImM8Uz-m"
   },
   "source": [
    "# Neural Network \n",
    "## with MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KxankQHIUz-n"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import keras\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "TMELl2RcUz-t",
    "outputId": "360bf9d4-491b-48c2-aaee-03554a0942b0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 1s 0us/step\n"
     ]
    }
   ],
   "source": [
    "# Load Data\n",
    "(train_x_orig, train_y_orig), (test_x_orig, test_y_orig) = keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "id": "yDIRMxBSUz-x",
    "outputId": "6b56412d-688e-4946-c479-b0472ef40b20"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples: 60000\n",
      "Number of testing examples: 10000\n",
      "Each image is of size: (28, 28, 3)\n",
      "train_x_orig shape: (60000, 28, 28)\n",
      "train_y shape: (60000,)\n",
      "test_x_orig shape: (10000, 28, 28)\n",
      "test_y shape: (10000,)\n"
     ]
    }
   ],
   "source": [
    "# Data Values\n",
    "\n",
    "m_train = train_x_orig.shape[0]\n",
    "num_px = train_x_orig.shape[1]\n",
    "m_test = test_x_orig.shape[0]\n",
    "\n",
    "print (\"Number of training examples: \" + str(m_train))\n",
    "print (\"Number of testing examples: \" + str(m_test))\n",
    "print (\"Each image is of size: (\" + str(num_px) + \", \" + str(num_px) + \", 3)\")\n",
    "print (\"train_x_orig shape: \" + str(train_x_orig.shape))\n",
    "print (\"train_y shape: \" + str(train_y_orig.shape))\n",
    "print (\"test_x_orig shape: \" + str(test_x_orig.shape))\n",
    "print (\"test_y shape: \" + str(test_y_orig.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-yqi8fyUUz-1"
   },
   "outputs": [],
   "source": [
    "# Prepare Training Labels\n",
    "\n",
    "digits = 10\n",
    "examples = train_y_orig.shape[0]\n",
    "\n",
    "train_y_orig = train_y_orig.reshape(1, examples)\n",
    "\n",
    "train_y = np.eye(digits)[train_y_orig.astype('int32')]\n",
    "train_y = train_y.T.reshape(digits, examples)\n",
    "#print(train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-9uc_vkAUz-4"
   },
   "outputs": [],
   "source": [
    "# Prepare Test Labels\n",
    "\n",
    "m_test = test_y_orig.shape[0]\n",
    "test_y_orig = test_y_orig.reshape(1, m_test)\n",
    "\n",
    "test_y = np.eye(digits)[test_y_orig.astype('int32')]\n",
    "test_y = test_y.T.reshape(digits, m_test)\n",
    "#print(test_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GLMBDNOMUz-6"
   },
   "source": [
    "## NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "WsKifZTtUz-8",
    "outputId": "8e8b5df9-aefe-4195-abd9-b2493454f803"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x's shape: (784, 60000)\n",
      "test_x's shape: (784, 10000)\n",
      "train_y's shape: (10, 60000)\n",
      "test_y's shape: (10, 10000)\n"
     ]
    }
   ],
   "source": [
    "#Model data For Neural Network\n",
    "\n",
    "# Convert into shape of (784,60000)\n",
    "train_x_flatten = train_x_orig.reshape(train_x_orig.shape[0], -1).T  \n",
    "# Convert into shape of (784,10000)\n",
    "test_x_flatten = test_x_orig.reshape(test_x_orig.shape[0], -1).T\n",
    "\n",
    "train_x = train_x_flatten / 255.\n",
    "test_x = test_x_flatten / 255.\n",
    "\n",
    "print (\"train_x's shape: \" + str(train_x.shape))\n",
    "print (\"test_x's shape: \" + str(test_x.shape))\n",
    "\n",
    "train_y = train_y\n",
    "test_y = test_y\n",
    "print (\"train_y's shape: \" + str(train_y.shape))\n",
    "print (\"test_y's shape: \" + str(test_y.shape))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kbZNZHlFWRfZ"
   },
   "outputs": [],
   "source": [
    "# Initializing hyperparameters\n",
    "n_x = train_x.shape[0]\n",
    "n_h = 50\n",
    "layers = [n_x, n_h]\n",
    "learning_rate = 1\n",
    "output = 10\n",
    "m = train_x.shape[1]\n",
    "epochs = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yI9nIuZGU-QC"
   },
   "outputs": [],
   "source": [
    "class NeuralNet:\n",
    "    def __init__(self, epochs=1000, layers = [784, 50], output=10):\n",
    "        # Intialize weights and biasis\n",
    "        self.epochs = epochs\n",
    "        self.layers = layers\n",
    "        self.output = output\n",
    "        self.W1 = np.random.randn(layers[1], layers[0])\n",
    "        self.b1 = np.zeros((layers[1], 1))\n",
    "        self.W2 = np.random.randn(output, layers[1])\n",
    "        self.b2 = np.zeros((output, 1))\n",
    "        self.costs = []\n",
    "    \n",
    "    \n",
    "    # activation Functions\n",
    "    def sigmoid(self,value):\n",
    "        return 1.0 / ( 1.0+np.exp(-value))\n",
    "\n",
    "    def sigmoid_prime(self,value):\n",
    "        return self.sigmoid(value) * (1- self.sigmoid(value))\n",
    "\n",
    "    def softmax(self,value):\n",
    "        return np.exp(value) / (np.sum(np.exp(value), axis = 0))\n",
    "\n",
    "    def sofmax_prime(self,value):\n",
    "        return self.softmax(value) * (1- self.softmax(value))\n",
    "    \n",
    "    # Cost Function\n",
    "    def compute_cost(self,Y, Y_hat):\n",
    "        cost_sum = np.sum(np.multiply(Y, np.log(Y_hat)))\n",
    "        m = Y.shape[1]\n",
    "        cost = -(1/m) * cost_sum\n",
    "        return cost\n",
    "\n",
    "    # Running the Neural Network\n",
    "    def fit(self, train_x, train_y):\n",
    "        m = train_x.shape[1]\n",
    "        for i in range(self.epochs):\n",
    "            # Forward Propogation\n",
    "            Z1 = np.matmul(self.W1,train_x) + self.b1\n",
    "            A1 = self.sigmoid(Z1)     # Activation Sigmoid\n",
    "            Z2 = np.matmul(self.W2,A1) + self.b2\n",
    "            A2 = self.softmax(Z2)     # Activation Softmax\n",
    "            \n",
    "            cost = self.compute_cost(train_y, A2)\n",
    "            \n",
    "            # Back Propogation\n",
    "            dZ2 = A2 - train_y\n",
    "            dW2 = (1./m) * np.matmul(dZ2, A1.T)\n",
    "            db2 = (1./m) * np.sum(dZ2, axis=1, keepdims=True)\n",
    "\n",
    "            dA1 = np.matmul(self.W2.T,dZ2)\n",
    "            dZ1 = dA1 * self.sigmoid_prime(Z1)\n",
    "            dW1 = (1./m) * np.matmul(dZ1, train_x.T)\n",
    "            db1 = (1./m) * np.sum(dZ1, axis = 1, keepdims=True)\n",
    "\n",
    "            # Updating Weights and biasis\n",
    "            self.W2 = self.W2 - learning_rate * dW2\n",
    "            self.b2 = self.b2 - learning_rate * db2\n",
    "            self.W1 = self.W1 - learning_rate * dW1\n",
    "            self.b1 = self.b1 - learning_rate * db1\n",
    "\n",
    "            if (i % 100 == 0):\n",
    "                print(\"Epoch\", i, \"cost: \", cost)\n",
    "\n",
    "        print(\"Final cost:\", cost)\n",
    "    \n",
    "    \n",
    "    def predict(self, test_x, test_y):\n",
    "        Z1 = np.matmul(self.W1, test_x) + self.b1\n",
    "        A1 = self.sigmoid(Z1)\n",
    "        Z2 = np.matmul(self.W2, A1) + self.b2\n",
    "        A2 = self.softmax(Z2)\n",
    "        predictions = np.argmax(A2, axis=0)\n",
    "        labels = np.argmax(test_y, axis=0)\n",
    "\n",
    "        correct = 0 \n",
    "        for i in range(labels.size):\n",
    "            if predictions[i] == labels[i]:\n",
    "                correct +=1\n",
    "        print('Test Accuracy : {}'.format(100*correct/labels.size))\n",
    "        \n",
    "        return predictions\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "Cwbw54IJUz_C",
    "outputId": "3209745b-3c80-4c68-a02b-6ab954e423ce"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 cost:  6.89133706375447\n",
      "Epoch 100 cost:  0.756986723610496\n",
      "Epoch 200 cost:  0.5708993450546843\n",
      "Epoch 300 cost:  0.49369314952488197\n",
      "Epoch 400 cost:  0.4475272593978021\n",
      "Epoch 500 cost:  0.41523351480846243\n",
      "Epoch 600 cost:  0.3905839974564431\n",
      "Epoch 700 cost:  0.37074442312719047\n",
      "Epoch 800 cost:  0.35419709447735975\n",
      "Epoch 900 cost:  0.34003993350927725\n",
      "Final cost: 0.3278180022413853\n"
     ]
    }
   ],
   "source": [
    "nn = NeuralNet()\n",
    "nn.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "WAHxnAFgbpAl",
    "outputId": "82d38b1e-d518-4c7b-f7ad-9baac72bfef0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy : 90.24\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([7, 2, 1, ..., 4, 5, 6])"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.predict(test_x, test_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wppAUHtsUz_o"
   },
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qCv0XyFPUz_q"
   },
   "outputs": [],
   "source": [
    "def relu(x):\n",
    "    return np.maximum(0, x)\n",
    "\n",
    "\n",
    "def sigmoid(x):\n",
    "    sig = 1 / (1 + np.exp(-x))\n",
    "    return sig\n",
    "\n",
    "\n",
    "def dSigmoid(x):\n",
    "    dsig = sigmoid(x) * (1 - sigmoid(x))\n",
    "    return dsig\n",
    "\n",
    "\n",
    "def softmax(x):\n",
    "    maxs = np.amax(x)\n",
    "    expScores = np.exp(x - maxs)\n",
    "    out = expScores / np.sum(expScores, axis=0, keepdims=True)\n",
    "    return out\n",
    "\n",
    "\n",
    "def AutoDiff_CL(partialL_Z, prevOut, summedVals):\n",
    "    dB = partialL_Z\n",
    "\n",
    "    dW = np.dot(partialL_Z, prevOut.transpose())\n",
    "\n",
    "    return dB, dW, partialL_Z\n",
    "\n",
    "\n",
    "def AutoDiff_FC(partialL_Z, prevWeights, prevOut, summedValues):\n",
    "    deltaSig = dSigmoid(summedValues)\n",
    "\n",
    "    partialLZ = np.dot(prevWeights.transpose(), partialL_Z) * deltaSig\n",
    "\n",
    "    dB = partialLZ\n",
    "\n",
    "    d0, d1, d2 = prevOut.shape\n",
    "    prevOut = prevOut.reshape((1, d0 * d1 * d2))\n",
    "    dW = np.dot(partialLZ, prevOut)\n",
    "    dW = dW.reshape((partialLZ.shape[0], d0, d1, d2))\n",
    "\n",
    "    return dB, dW, partialLZ\n",
    "\n",
    "\n",
    "def AutoDiff_PL(partialL_Z, prevWeights, prevOut, maxIndices, poolSize, output):\n",
    "    x, y, z = output.shape\n",
    "\n",
    "    a, b, c, d = prevWeights.shape\n",
    "\n",
    "    prevWeights = prevWeights.reshape((a, b * c * d))\n",
    "\n",
    "    output = output.reshape((x * y * z, 1))\n",
    "\n",
    "    maxIndices = maxIndices.reshape((x, y * z, 2))\n",
    "\n",
    "    sp = 1\n",
    "\n",
    "    partialL_Z = np.dot(prevWeights.transpose(), partialL_Z) * sp\n",
    "\n",
    "    partialL_Z = partialL_Z.reshape((x, y * z))\n",
    "    output = output.reshape((x, y * z))\n",
    "\n",
    "    depth, height, width = prevOut.shape\n",
    "\n",
    "    partialNew = np.zeros((depth, height, width))\n",
    "\n",
    "    for d in range(depth):\n",
    "\n",
    "        row = 0\n",
    "        col = 0\n",
    "\n",
    "        for i in range(maxIndices.shape[1]):\n",
    "\n",
    "            section = prevOut[d][row:row + poolSize[0], col:col + poolSize[0]]\n",
    "\n",
    "            partialPool = getPartialSec(output[d][i], partialL_Z[d][i], section)\n",
    "\n",
    "            partialNew[d][row:row + poolSize[0], col:col + poolSize[0]] = partialPool\n",
    "\n",
    "            col += poolSize[1]\n",
    "\n",
    "            if col >= width:\n",
    "                col = 0\n",
    "                row += poolSize[1]\n",
    "\n",
    "    return partialNew\n",
    "\n",
    "\n",
    "def AutoDiff_ConvL(partialL_Z, prevWeights, stride, im, summedValues):\n",
    "    numFilters, depth, filterSize, filterSize = prevWeights.shape\n",
    "\n",
    "    deltaB = np.zeros((numFilters, 1))\n",
    "    deltaW = np.zeros((prevWeights.shape))\n",
    "\n",
    "    convOutNum = (partialL_Z.shape[1]) * (partialL_Z.shape[2])\n",
    "\n",
    "    partialL_Z = partialL_Z.reshape((partialL_Z.shape[0], partialL_Z.shape[1] * partialL_Z.shape[2]))\n",
    "\n",
    "    for i in range(numFilters):\n",
    "\n",
    "        row = 0\n",
    "        col = 0\n",
    "\n",
    "        for j in range(convOutNum):\n",
    "\n",
    "            sec = im[:, row:row + filterSize, col:col + filterSize]\n",
    "\n",
    "            deltaW[i] += sec * partialL_Z[i][j]\n",
    "\n",
    "            deltaB[i] += partialL_Z[i][j]\n",
    "\n",
    "            col += stride\n",
    "\n",
    "            if (col + filterSize) - stride >= im.shape[2]:\n",
    "                col = 0\n",
    "                row += stride\n",
    "\n",
    "    return deltaB, deltaW\n",
    "\n",
    "\n",
    "def getPartialSec(val, partialL_Z, section):\n",
    "    dim1, dim2 = section.shape\n",
    "\n",
    "    section = section.reshape((dim1 * dim2))\n",
    "\n",
    "    partialSection = np.zeros((section.shape))\n",
    "\n",
    "    for i in range(len(section)):\n",
    "\n",
    "        num = section[i]\n",
    "\n",
    "        if num < val:\n",
    "\n",
    "            partialSection[i] = 0\n",
    "        else:\n",
    "\n",
    "            partialSection[i] = partialL_Z\n",
    "\n",
    "    return partialSection.reshape((dim1, dim2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "c3pMbhLEent0"
   },
   "outputs": [],
   "source": [
    "class CNN(object):\n",
    "\n",
    "    def __init__(self, inputShape, layers):\n",
    "\n",
    "        self.inputShape = inputShape\n",
    "\n",
    "        layerClasses = {'Convolution': ConvolutionalLayer,\n",
    "                        'Pooling': PoolingLayer,\n",
    "                        'fullyConnected': FullyConnectedLayer,\n",
    "                        'outputLayer': ClassificationLayer}\n",
    "\n",
    "        CNNLayers = []\n",
    "        shape = inputShape\n",
    "        for i in range(len(layers)):\n",
    "            layerName = list(layers[i].keys())[0]\n",
    "\n",
    "            layerClass = layerClasses[layerName]\n",
    "\n",
    "            layerArguments = list(layers[i].values())[0]\n",
    "\n",
    "            currentLayer = layerClass(shape, **layerArguments)\n",
    "\n",
    "            shape = currentLayer.output.shape\n",
    "\n",
    "            CNNLayers.append(currentLayer)\n",
    "\n",
    "        self.layers = CNNLayers\n",
    "\n",
    "        self.weightShapes = [currentLayer.weights.shape for currentLayer in self.layers if\n",
    "                             type(currentLayer).__name__ != 'PoolingLayer']\n",
    "        self.biasShapes = [currentLayer.biases.shape for currentLayer in self.layers if\n",
    "                           type(currentLayer).__name__ != 'PoolingLayer']\n",
    "\n",
    "    def forwardPass(self, im):\n",
    "\n",
    "        previousOutput = im\n",
    "\n",
    "        for currentLayer in self.layers:\n",
    "\n",
    "            inputData = previousOutput\n",
    "\n",
    "            className = type(currentLayer).__name__\n",
    "\n",
    "            if className == 'ConvolutionalLayer':\n",
    "                currentLayer.convolution(inputData)\n",
    "\n",
    "            if className == 'PoolingLayer':\n",
    "                currentLayer.pool(inputData)\n",
    "\n",
    "            if className == 'FullyConnectedLayer':\n",
    "                currentLayer.forwardPass(inputData)\n",
    "\n",
    "            if className == 'ClassificationLayer':\n",
    "                currentLayer.classify(inputData)\n",
    "\n",
    "            previousOutput = currentLayer.output\n",
    "\n",
    "        finalOutput = previousOutput\n",
    "        return finalOutput\n",
    "\n",
    "    def train(self, trainingData, batchSize, learningRate, numEpochs, lamdaVal=None):\n",
    "\n",
    "        trainSize = len(trainingData)\n",
    "\n",
    "        meanError = []\n",
    "\n",
    "        epochNum = 1\n",
    "\n",
    "        numTrainingRuns = len(trainingData) / batchSize * numEpochs * 1.0\n",
    "        currentRun = 1\n",
    "\n",
    "        for currentEpoch in range(numEpochs):\n",
    "            if epochNum %100 == 0:\n",
    "                print('Starting Epoch ', epochNum, ' of ', numEpochs)\n",
    "\n",
    "            random.shuffle(trainingData)\n",
    "\n",
    "            batches = [trainingData[i:i + batchSize] for i in range(0, trainSize, batchSize)]\n",
    "\n",
    "            losses = 0\n",
    "\n",
    "            batchNum = 1\n",
    "\n",
    "            cn = 0\n",
    "            for currentBatch in batches:\n",
    "\n",
    "                batchNum += 1\n",
    "                currentRun += 1\n",
    "                cn += 1\n",
    "\n",
    "                batchLoss = self.updateLoss(currentBatch, learningRate)\n",
    "                losses = losses + batchLoss\n",
    "\n",
    "            meanError.append(round(losses / cn, 2))\n",
    "            epochNum += 1\n",
    "\n",
    "        print('Done Training')\n",
    "\n",
    "    def updateLoss(self, batch, LearningRate):\n",
    "\n",
    "        derivW = [np.zeros(shape) for shape in self.weightShapes]\n",
    "        derivB = [np.zeros(shape) for shape in self.biasShapes]\n",
    "\n",
    "        batchLength = len(batch)\n",
    "\n",
    "        for image, label in batch:\n",
    "            im = image.reshape((1, 28, 28))\n",
    "\n",
    "            flag = self.forwardPass(im)\n",
    "\n",
    "            finalO, partialB, partialW = self.backpropogate(im, label)\n",
    "\n",
    "            derivB = [nb + db for nb, db in zip(derivB, partialB)]\n",
    "            derivW = [nw + dw for nw, dw in zip(derivW, partialW)]\n",
    "\n",
    "        error = crossELoss(label, finalO)\n",
    "\n",
    "        ind = 0\n",
    "\n",
    "        wIndex = []\n",
    "\n",
    "        for layer in self.layers:\n",
    "\n",
    "            if type(layer).__name__ != 'PoolingLayer':\n",
    "                wIndex.append(ind)\n",
    "\n",
    "            ind += 1\n",
    "\n",
    "        for iterationNum, (lnw, lnb) in enumerate(zip(derivW, derivB)):\n",
    "            layer = self.layers[wIndex[iterationNum]]\n",
    "\n",
    "            layer.weights -= LearningRate * lnw / batchLength\n",
    "            layer.biases -= LearningRate * lnb / batchLength\n",
    "\n",
    "        return error\n",
    "\n",
    "    def backpropogate(self, im, label):\n",
    "\n",
    "        derivW = [np.zeros(shape) for shape in self.weightShapes]\n",
    "        derivB = [np.zeros(shape) for shape in self.biasShapes]\n",
    "\n",
    "        prediction = self.layers[len(self.layers) - 1].output\n",
    "\n",
    "        partialL_Z = (prediction - label)\n",
    "\n",
    "        for layerNum in range(len(self.layers) - 1, -1, -1):\n",
    "\n",
    "            layer2 = layerNum\n",
    "            layer1 = layerNum - 1\n",
    "\n",
    "            currentLayer = self.layers[layer2]\n",
    "\n",
    "            if layer1 > -1:\n",
    "                prevOut = self.layers[layer1].output\n",
    "\n",
    "\n",
    "            elif layer1 == -1:\n",
    "                prevOut = im\n",
    "\n",
    "            if layer1 > -1: layer1Name = type(self.layers[layer1]).__name__\n",
    "            if layer1 == -1: layer1Name = 'image'\n",
    "            layer2Name = type(self.layers[layer2]).__name__\n",
    "\n",
    "            if layer1Name == 'FullyConnectedLayer' and layer2Name == 'ClassificationLayer':\n",
    "                deltaB, deltaW, partialL_Z = AutoDiff_CL(partialL_Z, prevOut,\n",
    "                                                         currentLayer.summedValues)\n",
    "\n",
    "            if layer1Name == 'PoolingLayer' and layer2Name == 'FullyConnectedLayer':\n",
    "                deltaB, deltaW, partialL_Z = AutoDiff_FC(partialL_Z, prevWeights,\n",
    "                                                         prevOut, currentLayer.summedValues)\n",
    "\n",
    "            if layer1Name == 'ConvolutionalLayer' and layer2Name == 'PoolingLayer':\n",
    "                partialL_Z = AutoDiff_PL(partialL_Z, prevWeights, prevOut,\n",
    "                                         currentLayer.maxIndices, currentLayer.poolSize, currentLayer.output)\n",
    "\n",
    "            if layer1Name == 'image' and layer2Name == 'ConvolutionalLayer':\n",
    "                prevWeights = currentLayer.weights\n",
    "\n",
    "                deltaB, deltaW = AutoDiff_ConvL(partialL_Z, prevWeights, currentLayer.stride, im,\n",
    "                                                currentLayer.outputValues)\n",
    "\n",
    "            if not (layer1Name == 'ConvolutionalLayer' and layer2Name == 'PoolingLayer'):\n",
    "\n",
    "                if layer1 == -1:\n",
    "                    layer1 = 0\n",
    "\n",
    "                derivB[layer1], derivW[layer1] = deltaB, deltaW\n",
    "                prevWeights = currentLayer.weights\n",
    "\n",
    "        return self.layers[-1].output, derivB, derivW\n",
    "\n",
    "\n",
    "class ConvolutionalLayer(object):\n",
    "\n",
    "    def __init__(self, inputShape, filterSize, numFilters, stride):\n",
    "\n",
    "        self.depth = inputShape[0]\n",
    "        self.height = inputShape[1]\n",
    "        self.width = inputShape[2]\n",
    "\n",
    "        self.filterSize = filterSize\n",
    "        self.stride = stride\n",
    "        self.numFilters = numFilters\n",
    "        self.padding = 0\n",
    "\n",
    "        self.weights = np.random.randn(numFilters, self.depth, filterSize, filterSize)\n",
    "        self.biases = np.random.rand(self.numFilters, 1)\n",
    "\n",
    "        self.outputRows = int((self.height - self.filterSize + 2 * self.padding) / self.stride + 1)\n",
    "        self.outputCols = int((self.width - self.filterSize + 2 * self.padding) / self.stride + 1)\n",
    "\n",
    "        self.output = np.zeros((self.numFilters, self.outputRows, self.outputCols))\n",
    "        self.outputValues = np.zeros((self.numFilters, self.outputRows, self.outputCols))\n",
    "\n",
    "        print('Convolutional Layer Initialized')\n",
    "\n",
    "    def convolution(self, inputData):\n",
    "\n",
    "        self.outputValues = self.outputValues.reshape((self.numFilters, self.outputRows * self.outputCols))\n",
    "        self.output = self.output.reshape((self.numFilters, self.outputRows * self.outputCols))\n",
    "\n",
    "        outputLength = self.outputRows * self.outputCols\n",
    "\n",
    "        for i in range(self.numFilters):\n",
    "            col = 0\n",
    "            row = 0\n",
    "\n",
    "            for j in range(outputLength):\n",
    "\n",
    "                dotProduct = inputData[:, row:row + self.filterSize, col:col + self.filterSize] * self.weights[i]\n",
    "\n",
    "                sumValue = np.sum(dotProduct)\n",
    "\n",
    "                self.outputValues[i][j] = sumValue + self.biases[i]\n",
    "\n",
    "                self.output[i][j] = sigmoid(self.outputValues[i][j])\n",
    "\n",
    "                col += self.stride\n",
    "\n",
    "                if col + self.filterSize - self.stride >= self.width:\n",
    "                    col = 0\n",
    "                    row += self.stride\n",
    "\n",
    "        self.outputValues = self.outputValues.reshape((self.numFilters, self.outputRows, self.outputCols))\n",
    "        self.output = self.output.reshape((self.numFilters, self.outputRows, self.outputCols))\n",
    "\n",
    "\n",
    "class PoolingLayer(object):\n",
    "\n",
    "    def __init__(self, inputShape, poolSize):\n",
    "\n",
    "        self.depth = inputShape[0]\n",
    "        self.height = inputShape[1]\n",
    "        self.width = inputShape[2]\n",
    "\n",
    "        self.poolSize = poolSize\n",
    "        self.stride = 2\n",
    "\n",
    "        self.outputHeight = (self.height - self.poolSize[0]) / self.stride + 1\n",
    "        self.outputWidth = (self.width - self.poolSize[0]) / self.stride + 1\n",
    "\n",
    "        self.outputHeight = int(self.outputHeight)\n",
    "        self.outputWidth = int(self.outputWidth)\n",
    "        self.output = np.empty((self.depth, self.outputHeight, self.outputWidth))\n",
    "\n",
    "        self.maxIndices = np.empty((self.depth, self.outputHeight, self.outputWidth, 2))\n",
    "\n",
    "        print('Pooling Layer Initialized')\n",
    "\n",
    "    def pool(self, inputData):\n",
    "\n",
    "        self.Length = self.outputHeight * self.outputWidth\n",
    "\n",
    "        self.output = self.output.reshape((self.depth, self.Length))\n",
    "        self.maxIndices = self.maxIndices.reshape((self.depth, self.Length, 2))\n",
    "\n",
    "        for i in range(self.depth):\n",
    "            row = 0\n",
    "            col = 0\n",
    "\n",
    "            for j in range(self.Length - 1):\n",
    "\n",
    "                section = inputData[i][row:row + self.poolSize[0], col:col + self.poolSize[0]]\n",
    "\n",
    "                maxVal = np.amax(section)\n",
    "                self.output[i][j] = maxVal\n",
    "\n",
    "                maxIndex = np.where(section == np.max(section))\n",
    "                if len(maxIndex[0]) > 1:\n",
    "                    maxIndex = [maxIndex[0][0], maxIndex[1][0]]\n",
    "\n",
    "                maxIndex = int(maxIndex[0]) + row, int(maxIndex[1]) + col\n",
    "\n",
    "                self.maxIndices[i][j] = maxIndex\n",
    "\n",
    "                col += self.stride\n",
    "\n",
    "                if col >= self.width:\n",
    "                    col = 0\n",
    "                    row += self.stride\n",
    "\n",
    "        self.output = self.output.reshape((self.depth, self.outputHeight, self.outputWidth))\n",
    "        self.maxIndices = self.maxIndices.reshape((self.depth, self.outputHeight, self.outputWidth, 2))\n",
    "\n",
    "\n",
    "class SingleLayer(object):\n",
    "\n",
    "    def __init__(self, inputShape, outputNum):\n",
    "        self.output = np.ones((outputNum, 1))\n",
    "        self.summedValues = np.ones((outputNum, 1))\n",
    "\n",
    "\n",
    "class FullyConnectedLayer(SingleLayer):\n",
    "\n",
    "    def __init__(self, inputShape, numOutput):\n",
    "        super(SingleLayer, self).__init__()\n",
    "\n",
    "        self.output = np.ones((numOutput, 1))\n",
    "        self.summedValues = np.ones((numOutput, 1))\n",
    "\n",
    "        self.depth = inputShape[0]\n",
    "        self.width = inputShape[1]\n",
    "        self.height = inputShape[2]\n",
    "\n",
    "        self.numOutput = numOutput\n",
    "\n",
    "        self.weights = np.random.randn(self.numOutput, self.depth, self.height, self.width)\n",
    "        self.biases = np.random.rand(self.numOutput, 1)\n",
    "\n",
    "        print('Fully Connected Layer Initialized')\n",
    "\n",
    "    def forwardPass(self, inputData):\n",
    "        self.weights = self.weights.reshape((self.numOutput, self.depth * self.height * self.width))\n",
    "        inputData = inputData.reshape((self.depth * self.height * self.width, 1))\n",
    "\n",
    "        self.summedValues = np.dot(self.weights, inputData) + self.biases\n",
    "\n",
    "        self.output = sigmoid(self.summedValues)\n",
    "\n",
    "        self.weights = self.weights.reshape((self.numOutput, self.depth, self.height, self.width))\n",
    "\n",
    "\n",
    "class ClassificationLayer(SingleLayer):\n",
    "\n",
    "    def __init__(self, inputShape, numClasses):\n",
    "        super(SingleLayer, self).__init__()\n",
    "\n",
    "        self.output = np.ones((numClasses, 1))\n",
    "        self.summedValues = np.ones((numClasses, 1))\n",
    "\n",
    "        self.numClasses = numClasses\n",
    "\n",
    "        self.weights = np.random.randn(self.numClasses, inputShape[0])\n",
    "        self.biases = np.random.randn(self.numClasses, 1)\n",
    "\n",
    "        print('Classification Layer Initialized')\n",
    "\n",
    "    def classify(self, data):\n",
    "        self.summedValues = np.dot(self.weights, data) + self.biases\n",
    "\n",
    "        self.output = softmax(self.summedValues)\n",
    "\n",
    "\n",
    "def getAccuracy(net, testData):\n",
    "    print('Begin Testing')\n",
    "    numCorrect = 0\n",
    "    for i in range(len(testData)):\n",
    "        im = testData[i][0].reshape(1, 28, 28)\n",
    "        label = testData[i][1]\n",
    "        prediction = np.argmax(net.forwardPass(im))\n",
    "\n",
    "        if prediction == label: numCorrect += 1\n",
    "\n",
    "        if (i + 1) % int(0.1 * len(testData)) == 0:\n",
    "            print('{0}% Completed'.format(int(float(i + 1) / len(testData) * 100)))\n",
    "\n",
    "    print('Accuracy: ', numCorrect / len(testData) * 100)\n",
    "\n",
    "\n",
    "def loss(desired, final):\n",
    "    return .5 * np.sum(1.0 * desired - final) ** 2\n",
    "\n",
    "\n",
    "def crossELoss(desired, final):\n",
    "    return -np.sum(desired * np.log(final + .000000000000000000000000000001))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ltyls3jph2n-"
   },
   "outputs": [],
   "source": [
    "trainData = train_x_orig\n",
    "testData = test_x_orig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "40gM-Sqggnki"
   },
   "outputs": [],
   "source": [
    "def oneHot(i):\n",
    "    vec = np.zeros((10,1))\n",
    "    vec[i] = 1\n",
    "    return vec\n",
    "\n",
    "\n",
    "\n",
    "trainLabel = [oneHot(x) for x in train_y_orig]\n",
    "testLabel = test_y_orig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zw8MaoxohIms"
   },
   "outputs": [],
   "source": [
    "training = [trainData,trainLabel]\n",
    "testing = [testData,testLabel]\n",
    "training = list(zip(trainData,trainLabel))\n",
    "testing = list(zip(testData,testLabel))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "bYwMeKqPiF3U",
    "outputId": "6acb421b-fc32-4093-827c-bb03d93fdb4a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convolutional Layer Initialized\n",
      "Pooling Layer Initialized\n",
      "Fully Connected Layer Initialized\n",
      "Classification Layer Initialized\n",
      " \n",
      "Model Initialized\n",
      "Starting Epoch  100  of  200\n",
      "Starting Epoch  200  of  200\n",
      "Done Training\n"
     ]
    }
   ],
   "source": [
    "smax = 1\n",
    "x, y = training[0][0].shape\n",
    "inputShape = (1, x, y)\n",
    "\n",
    "layers = [\n",
    "    {'Convolution': {'filterSize': 5, 'stride': 1, 'numFilters': 20}},\n",
    "    {'Pooling': {'poolSize': (2, 2)}},\n",
    "    {'fullyConnected': {'numOutput': 50}},\n",
    "    {'outputLayer': {'numClasses': 10}}\n",
    "]\n",
    "\n",
    "test = CNN(inputShape, layers)\n",
    "print(' ')\n",
    "print('Model Initialized')\n",
    "\n",
    "batchSize = 10\n",
    "learningRate = .1\n",
    "numEpochs = 200\n",
    "test.train(training, batchSize, learningRate, numEpochs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "id": "yRoCNjeLkHD1",
    "outputId": "4df76186-b463-407b-b348-4fcc96de8910"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Begin Testing\n",
      "Accuracy: 79.84%\n"
     ]
    }
   ],
   "source": [
    "def getAccuracy(net, testData):\n",
    "    print('Begin Testing')\n",
    "    numCorrect = 0\n",
    "\n",
    "    for i in range(len(testData)):\n",
    "\n",
    "        im = testData[i][0].reshape(1, 28, 28)\n",
    "        label = testData[i][1]\n",
    "\n",
    "        prediction = np.argmax(net.forwardPass(im))\n",
    "#         print(prediction)\n",
    "#         print(label[i])\n",
    "\n",
    "        if prediction == label[i]: numCorrect += 1\n",
    "\n",
    "    print('Accuracy: {}%'.format(numCorrect / len(testData) * 100)) \n",
    "\n",
    "\n",
    "getAccuracy(test, testing)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5tRuEiUviZIm"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "Task 8 | Neural Networks.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}