Simple-Neural-Network/Simple Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Neural Network:\n",
    "In this notebook, a fully vectorized 2-layer neural network will be implemented from scratch.   \n",
    "This will 'learn' the XOR gate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMPORTS:\n",
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DATA: The XOR truth table.\n",
    "inputs = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "expected_output = np.array([[0],[1],[1],[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ARCHITECTURE:\n",
    "n0 = 2\n",
    "n1 = 2\n",
    "n2 = 1\n",
    "m = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# INPUTS & OUTPUTS:\n",
    "A0 = inputs.T # Each datapoint is a column.\n",
    "Y = expected_output.T # Converted to row vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WEIGHTS: \n",
    "W1 = np.random.uniform(size = (n1, n0))\n",
    "W2 = np.random.uniform(size = (n2, n1))\n",
    "\n",
    "# BIASES: \n",
    "# Gets added to each column, although mathematically, W*A and b aren't of the same order. Python broadcasting helps us here.\n",
    "b1 = np.random.uniform(size = (n1, 1))\n",
    "b2 = np.random.uniform(size = (n2, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SIGMOID ACTIVATION FUNCTION:\n",
    "def sigmoid (z):\n",
    "    return 1/(1 + np.exp(-z))\n",
    "\n",
    "def sigmoidDerivative(z):\n",
    "    return sigmoid(z)*(1 - sigmoid(z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(Y_hat, Y):\n",
    "    L = -(1./m) * ( np.sum( np.multiply(np.log(Y_hat),Y) ) + np.sum( np.multiply(np.log(1-Y_hat),(1-Y)) ) )\n",
    "    return L"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GRADIENT DESCENT PARAMETERS:\n",
    "alpha = 1 # Learning rate.\n",
    "N_ITERS = 1500 # Total iterations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "costList = []\n",
    "for _ in range(N_ITERS):\n",
    "    \n",
    "    # FORWARD-PROPOGATION:\n",
    "    Z1 = np.matmul(W1,A0) + b1\n",
    "    A1 = sigmoid(Z1)\n",
    "    \n",
    "    Z2 = np.matmul(W2,A1) + b2\n",
    "    A2 = sigmoid(Z2)\n",
    "    Y_hat = A2 # The prediction for all datapoints.\n",
    "    \n",
    "    # FINDING OUT THE COST:\n",
    "    costList.append(cost(Y_hat, Y))\n",
    "\n",
    "    # BACKWARD-PROPOGATION:\n",
    "    dZ2 = A2 - Y\n",
    "    dW2 = (1/m)*np.matmul(dZ2, A1.T)\n",
    "    db2 = (1/m)*np.sum(dZ2, axis=1, keepdims=True)\n",
    "\n",
    "    dZ1 = np.matmul(W2.T, dZ2)*(sigmoidDerivative(Z1))\n",
    "    dW1 = (1/m)*np.matmul(dZ1, A0.T)\n",
    "    db1 = (1/m)*np.sum(dZ1, axis=1, keepdims=True)\n",
    "    \n",
    "    # GRADIENT-DESCENT:\n",
    "    W1 -= alpha*dW1\n",
    "    W2 -= alpha*dW2\n",
    "    b1 -= alpha*db1\n",
    "    b2 -= alpha*db2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our predictions:  [0.01215874 0.99085295 0.99085743 0.00985459]\n"
     ]
    }
   ],
   "source": [
    "print(\"Our predictions: \", Y_hat[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfN0lEQVR4nO3de3hU9b3v8fd3ZnK/QchFIBEQIhItiGZT6q0oVtFdwb23tdCbPXXXvWs93Z721GrtY91131q77eVoWz1tT29aq7VaamnRqq29CDWoIBfBiFwCAoFwD7nMzO/8MSthEiYwwUnWrOTzep48M+tC8nHBfFz5zW/WMuccIiISfCG/A4iISGao0EVEhgkVuojIMKFCFxEZJlToIiLDRMSvH1xRUeEmTpzo148XEQmkFStW7HbOVaba5luhT5w4kcbGRr9+vIhIIJnZ5v62achFRGSYOGGhm9n3zWyXma3uZ7uZ2TfNrMnMVpnZOZmPKSIiJ5LOGfoPgHnH2X4FUOd93QB8++3HEhGRgTphoTvnngdaj7PLAuBHLmEZMMrMxmYqoIiIpCcTY+jjga1Jy83eumOY2Q1m1mhmjS0tLRn40SIi0i0ThW4p1qW84pdz7gHnXINzrqGyMuWsGxEROUmZKPRmoDZpuQbYnoHvKyIiA5CJQl8MfMSb7TIb2O+ceysD3zelFze18t9PraczGh+sHyEiEkjpTFv8KfACMNXMms3sejP7ZzP7Z2+XJcBGoAn4v8CNg5YWeGnzXv7Ps01E4yp0EZFkJ/ykqHNu0Qm2O+CTGUt0AuaN2Md1Xw4RkV4C90lR896D1Z2WRER6C16he2foqnMRkd4CWOjeGbqG0EVEegleoXuPTufoIiK9BK7QQ91DLupzEZFeAlfo3UMucTW6iEgvASz0xKPqXESktwAWus7QRURSCV6hdz9Rn4uI9BK4Qg91T1v0OYeISLYJXKEf/ei/Kl1EJFnwCt17VJ+LiPQWuELXkIuISGqBK/TuU/S4LrcoItJL4Ao91f3uREQkgIXeM+SiE3QRkV4CV+ia5SIiklrgCl1vioqIpBa4QtcZuohIaoEr9G7qcxGR3gJX6N1DLhp0ERHpLXCFfnTIxd8cIiLZJniFjqYtioikErhC77kFnYZcRER6CVyh9wy5xP3NISKSbQJX6N0f/tcZuohIb4Er9J4hF/W5iEgvgSt007VcRERSClyh601REZHUAlfomocuIpJa8Aq9Zx66Gl1EJFnwCr1nyEVERJKlVehmNs/M1ptZk5ndmmL7qWb2nJm9bGarzOzKzEft+VmAztBFRPo6YaGbWRi4D7gCqAcWmVl9n92+ADzinJsJLAS+lemgPXm8R/W5iEhv6ZyhzwKanHMbnXOdwMPAgj77OKDUe14GbM9cxN50gwsRkdTSKfTxwNak5WZvXbI7gQ+ZWTOwBPifqb6Rmd1gZo1m1tjS0nIScZM/+q9KFxFJlk6hW4p1fdt0EfAD51wNcCXwYzM75ns75x5wzjU45xoqKysHnhZNWxQR6U86hd4M1CYt13DskMr1wCMAzrkXgHygIhMB+4qEEpF1CzoRkd7SKfQXgTozm2RmuSTe9FzcZ58twFwAM5tGotBPbkzlBMJe4qhO0UVEejlhoTvnosBNwFJgHYnZLGvM7EtmNt/b7TPAx81sJfBT4KNukOYVhrvP0FXoIiK9RNLZyTm3hMSbncnr7kh6vhY4P7PRUot4F3PRGbqISG+B+6Ro97TFmO5wISLSS+AKPRLuLnSfg4iIZJnAFXr3GXpUZ+giIr0ErtC7x9A1bVFEpLfAFXq4+03RmApdRCRZYAs9plkuIiK9BK7Qu4dcYhpyERHpJXCFHtIZuohISoEr9IgKXUQkpcAVut4UFRFJLXCFnp8TBqC9K+ZzEhGR7BK4Qs8Jh8iNhDjcqUIXEUkWuEIHKMoN09YZ9TuGiEhWCWShF+ZGONShQhcRSZbW5XOzTXVpHmu2HWDl1n2EQ4ZZ4s3SkBnhkBEJGZFwiEjo6HLiMdSz3D39UURkuAhkoV9QV8k3n3mdBff9+aS/h1liCmQk5BV/+Gjx50XCFOSEKcgNU5h79HlBTmK5KC9CeVEuowtzKS/Opbwwl/KiXCpL8nretBURGWqBLPSb59ZxwZQKDnV0EYsn5qQ754g5RyzuiMa8x7gjFo97j+7oY8wRTV4fO7pfNOboiMZo64xxpCvGkc4Y+490caTz6LpDHdF+58FXleRRW15IzegCJo4pon5cKfVjS6kZXYCZfisQkcETyEIPhYxZk8p9+/nOOQ60R9l7uJPWtk72Hu5kz+FOduxvZ2trG1v3trFi814Wr9xO9xUKSvMjzJpUznmTKzhvyhimVpeo4EUkowJZ6H4zM8oKcigryGEiRf3u19YZZf2Og6x96wCvNu9n2cY9/G7dLgAmjinkqhnjWHD2eKZUFQ9VdBEZxmyQ7uV8Qg0NDa6xsdGXn+2n5r1t/PH13Ty5ajsvvLGHuIOLTq/kHy+YxIV1FTprF5HjMrMVzrmGlNtU6P7ZdbCdRxub+cFfNtFysINzJ4zmC387jZmnjvY7mohkKRV6luuIxnhsxTbueXoDuw918A/n1HDHVfWUFeT4HU1EsszxCj2QHywabvIiYT7wzlP5/WfncOOcyTzxyjYu/9rz/Llpt9/RRCRAVOhZpDgvwi3zzuDxG8+jOD/Ch7+3nPv/8AZ+/RYlIsGiQs9C02tGsfim87nirLH8529e49OPrKQrFvc7lohkOU1bzFKFuRHu/cBMpj5bwj1Pb+Bgexf3fuAcfRJVRPqlM/QsZmZ8am4dd119Fs+8touP/eBFXQdeRPqlQg+AD8+ewD3XzuCFjXv45IMvafhFRFJSoQfE382s4a4FiTP1zz66krjuqSoifWgMPUA+NHsC+490cffS9VSX5nPbldP8jiQiWUSFHjA3zpnMW/uPcP/zG5lSVcz7Gmr9jiQiWSKtIRczm2dm682sycxu7Wefa81srZmtMbOHMhtTupkZX7zqTM6fMobbH19N46ZWvyOJSJY4YaGbWRi4D7gCqAcWmVl9n33qgNuA851zZwI3D0JW8eSEQ3zrA+cyfnQB//TjFTTvbfM7kohkgXTO0GcBTc65jc65TuBhYEGffT4O3Oec2wvgnNuV2ZjSV1lhDt+9roHOaJxPPvgSHVFNZxQZ6dIp9PHA1qTlZm9dstOB083sz2a2zMzmZSqg9G9yZTF3v28GK5v3829PrvM7joj4LJ1CT3WB7r5z5iJAHTAHWAR818xGHfONzG4ws0Yza2xpaRloVklh3lmncMNFp/HjZZt54uVtfscRER+lU+jNQPJUihpge4p9fumc63LOvQmsJ1HwvTjnHnDONTjnGiorK082s/Rxy+VTmTWpnNt+8Srrdxz0O46I+CSdQn8RqDOzSWaWCywEFvfZ5wngYgAzqyAxBLMxk0Glf5FwiHsXzaQ4P8InfrKCg+1dfkcSER+csNCdc1HgJmApsA54xDm3xsy+ZGbzvd2WAnvMbC3wHPBZ59yewQotx6oqzefeRTPZ3NrG5x9frUvuioxAumPRMHPfc03cvXQ9//X372DhrFP9jiMiGaY7Fo0gn3j3ZC6sq+DOX63ReLrICKNCH2ZCIeOea8+mOC+Hmx56ibbOqN+RRGSIqNCHocqSPL6x8GyaWg5x5+I1fscRkSGiQh+mzp9SwU0XT+GRxmbNTxcZIVTow9i/zK1j1sRybn/8VTa2HPI7jogMMhX6MBYJh/jGorPJjYS46aGXdfs6kWFOhT7MjS0r4L+vncHatw7wH0t0vReR4UyFPgJcckY1H79wEj96YTO/efUtv+OIyCBRoY8Qn738DGbUjuLWX7zKjv3tfscRkUGgQh8hciMhvv7+s+mMxrnlsVW6NIDIMKRCH0EmVRTx+SvP4PkNLTy4fIvfcUQkw1ToI8yHZk/gwroK/v3X69i0+7DfcUQkg1ToI4yZ8ZVrppMTNj79yCvE4hp6ERkuVOgj0NiyAv51wZm8tGUfDy7f7HccEckQFfoIdfXZ47lgSgV3/3Y9uw5o1ovIcKBCH6HMjLuuPouOWJy7fq0PHIkMByr0EWxSRRGfnDOFX63czp+bdvsdR0TeJhX6CPdP7z6NmtEF/MeSdcT1BqlIoKnQR7j8nDCfvXwqa7Yf4JcrdZldkSBToQtXTR/HWeNL+erSDboio0iAqdCFUMj4/JXT2LbvCA/pE6QigaVCFwDOm1zBOyeV88DzG+mI6ixdJIhU6NLjpkumsONAO4+t0Fi6SBCp0KXHBVMqmFE7im//oYmuWNzvOCIyQCp06WFmfHLOZLa2HuGpNTv9jiMiA6RCl17mTqumtryAH/5lk99RRGSAVOjSSzhkfGT2RP66qZW12w/4HUdEBkCFLse4tqGWgpywztJFAkaFLscoK8zh6pnjeeKVbew/0uV3HBFJkwpdUlo0q5aOaJwnV233O4qIpEmFLim9Y3wZp1cX82hjs99RRCRNKnRJycx437m1vLJ1H027DvodR0TSkFahm9k8M1tvZk1mdutx9rvGzJyZNWQuovjl6pnjCYeMR1foLF0kCE5Y6GYWBu4DrgDqgUVmVp9ivxLgU8DyTIcUf1SW5HHx1EqeeHmbrpUuEgDpnKHPApqccxudc53Aw8CCFPvdBXwF0A0qh5GrZoxj54EOGjfv9TuKiJxAOoU+HtiatNzsrethZjOBWufck8f7RmZ2g5k1mlljS0vLgMPK0Js7rZq8SIhfa7aLSNZLp9Atxbqe37/NLAR8DfjMib6Rc+4B51yDc66hsrIy/ZTim+K8CJecUcWS1TuIadhFJKulU+jNQG3Scg2QfLpWApwF/N7MNgGzgcV6Y3T4eO/0cbQc7OCvb7b6HUVEjiOdQn8RqDOzSWaWCywEFndvdM7td85VOOcmOucmAsuA+c65xkFJLEPu4jMqKcgJ60NGIlnuhIXunIsCNwFLgXXAI865NWb2JTObP9gBxX+FuRHmTqvit6t3ENV10kWyViSdnZxzS4Alfdbd0c++c95+LMk2f/uOsTy56i0aN+9l9mlj/I4jIinok6KSlotOryQ3EuLptbrxhUi2UqFLWoryIpw/eQxPrd2Bc5rtIpKNVOiStsvOPIWtrUdYv1PXdhHJRip0SdvcaVWYofuNimQpFbqkraokn5m1o3hq7Q6/o4hICip0GZD31J/C6m0H2L7viN9RRKQPFboMyGVnVgNototIFlKhy4BMrizmtMoiFbpIFlKhy4BdVn8KyzbuYX+bbiAtkk1U6DJg76mvJhp3/H7DLr+jiEgSFboM2MzaUVQU52n6okiWUaHLgIVCxnvqq/n9+l20d8X8jiMiHhW6nJTLzqzmcGeMF97Y43cUEfGo0OWknDd5DEW5YZ7SbBeRrKFCl5OSFwkzZ2oVT6/dSVy3phPJCip0OWmXnVnN7kMdvLx1n99RRAQVurwNc6ZWEQmZru0ikiVU6HLSygpymH3aGH1qVCRLqNDlbbnszGo2thymadchv6OIjHgqdHlbLp2WuFiXhl1E/KdCl7dl3KgCpteUadhFJAuo0OVtu6y+mpe37GPngXa/o4iMaCp0edvmnTUWgF+vesvnJCIjmwpd3rYpVcXUjy1l8crtfkcRGdFU6JIRV80Yxytb97G1tc3vKCIjlgpdMuK90xPDLjpLF/GPCl0yora8kHNOHcWvVOgivlGhS8bMnzGO13YcZMPOg35HERmRVOiSMVdOH0s4ZDz2UrPfUURGJBW6ZExVST4XT63isRXb6IrF/Y4jMuKo0CWj3v83tew+1MFzr+kG0iJDTYUuGXXx1EqqSvL42Ytb/Y4iMuKkVehmNs/M1ptZk5ndmmL7p81srZmtMrNnzGxC5qNKEETCIa45t4bn1u9ix35dCkBkKJ2w0M0sDNwHXAHUA4vMrL7Pbi8DDc656cDPga9kOqgEx7UNtcQdPPziFr+jiIwo6ZyhzwKanHMbnXOdwMPAguQdnHPPOee6PyK4DKjJbEwJkokVRcyZWslPlm2hIxrzO47IiJFOoY8HkgdEm711/bke+E2qDWZ2g5k1mlljS0tL+iklcK6/YBK7D3Ww+BV90EhkqKRT6JZiXcrbvJvZh4AG4O5U251zDzjnGpxzDZWVlemnlMC5YEoFU6tL+N6f3sS5lP9cRCTD0in0ZqA2abkGOOa0y8wuBW4H5jvnOjITT4LKzPjYBRN5bcdB/vj6br/jiIwI6RT6i0CdmU0ys1xgIbA4eQczmwncT6LMNQFZALh65njGluXz9d9t0Fm6yBA4YaE756LATcBSYB3wiHNujZl9yczme7vdDRQDj5rZK2a2uJ9vJyNIXiTMjRdP4aUt+3SWLjIEzK8zp4aGBtfY2OjLz5ah0xGNcfHdv6e6LJ9ffOI8zFK9JSMi6TKzFc65hlTb9ElRGVR5kTA3XVLHy1v28dvVO/yOIzKsqdBl0F3bUMPU6hL+fck62rs0L11ksKjQZdBFwiHuuKqe5r1H+N6f3vQ7jsiwpUKXIXH+lAouP7Oae59tYvOew37HERmWVOgyZO6cfyaRkPG5x1YRj2sao0imqdBlyIwtK+AL753Gso2t/GT5Zr/jiAw7KnQZUtc21PLu0yv5t1+vY832/X7HERlWVOgypMyMe66dQXlhLjc++BIH2rv8jiQybKjQZciNKc7j3g/MZNveI3z6Z68Q03i6SEao0MUXDRPL+eJV9fxu3S6+uHi1rvUikgERvwPIyPXhd01k2752vvOHN6gqyedTc+v8jiQSaCp08dUtl09l18F27nl6A7G44+ZL63S9F5GTpEIXX4VCxt3XzCBsxjeeeZ32rhifm3cGoZBKXWSgVOjiu3DI+PI/TCcvJ8T9z2+kee8Rvvq+GRTkhv2OJhIoKnTJCqGQcdeCszi1vJD//M1rbNpzmG998BwmjCnyO5pIYGiWi2QNM+OGiybz/ev+hi2tbVz5jT/ySONWzYARSZMKXbLOxWdU8dubL+Ks8WXc8vNVfPxHjWxtbfM7lkjWU6FLVho/qoCHPj6bz195Bn9u2sN7vvYH7n32dV1PXeQ4VOiStcKhxBDM7z7zbi45o4qvPrWBd9/9HD96YRMdURW7SF+6p6gExrKNe7jnqQ38dVMr48ry+ej5E3l/w6mUFeb4HU1kyBzvnqIqdAkU5xx/atrNvc82sfzNVgpywvz9OeNZNOtUzhxXqg8lybB3vELXtEUJFDPjwrpKLqyrZM32/fzwL5t4dEUzDy7fQl1VMVfPHM/8GeOoLS/0O6rIkNMZugTevrZOlry6g8dfbubFTXsBOOOUEi6dVs3caVXMqBmlT57KsKEhFxkxtra28dvVO/jdup00bt5LLO4oL8rlnZPKeeekcmZPHsPpVSUqeAksFbqMSPvaOvnDhhae37CbZRv3sG3fEQBGF+Ywo3YU08eX8Y6aUUyvKaO6NN/ntCLp0Ri6jEijCnNZcPZ4Fpw9HkicvS9/s5W/vrmHVc37eX5DC9331qgqyWPa2FLqqoqpqy5mSlUJU6qKKSvQDBoJDhW6jBi15YXUlhdyzbk1ABzpjLH2rf2sat7Pq837eW3HQZZt3ENHNN7zZ6pL85hUUcSp5YXUji7k1DGF1Iwu5NTyQiqKczWrRrKKCl1GrILcMOdOKOfcCeU962JxR/PeNpp2HeL1XYfYsPMgm/e08dz6FloOdvT+8zlhxo3K55SyfKpL8qkqzeeU0jyqSxPPq0vzqCrJJzeiz+/J0FChiyQJh4wJY4qYMKaIudOqe21r74rRvLeNLa1tbG09wpbWNrbvO8LOA+0sf7OVXQfb6Yod+55USX6EMUW5jC7Kpbwwl/Kio1+ji3IZU5TLqMJcygpyKM2PUFqQQ14kpLN/GTAVukia8nPC3th6Scrt8bhjb1snOw90sPNAu/fVwd62TvYc7mTv4U7e2t/Omu0HaD3cSWcsnvL7AOSEjdL8HEoLcijJj1Caf/SxtCBCSX4OhblhivIiFOaGKcyNUJQbpqDPusLcsP7nMIKo0EUyJBQyxhTnMaY4j/pxpcfd1znH4c4Yew930nq4k9a2Tg4c6eJge5QD7d5j0vKBI13sPNDes62tM/1r2YQMinIjPWVfkJMo/rxIiLxIiPyccMrHvH7W93rMCZETDpEb9h4jIXLC1rNO00OHlgpdxAdmRnFehOK8yEl9qrUrFqetM8aRzhiHO6O0dcRo60wU/WHvsa0jSltXjLaOxLrEvon1HdE47V0xDrZHae+K9SwnP2ZCOGS9Cj4nHCIn0me5e3ukz7K3PRI2IiEjHEo8D4eMnD7Lie2Jx0g41GddKGlb0nKfP5vTz58Lh4ywGaFQ4r8nZMnrsut/WGkVupnNA74BhIHvOuf+q8/2POBHwLnAHuD9zrlNmY0qIt1ywiHKCkKDNq3SOUdHNJ746qfwO6Ix2rvidMXidEbjdMUcXTFvORanK9pnOWldz7L3ZzqjcQ53ROmKOe97Hd0nFndE445YzNEVTyyneq/CL8mlnyj8o+tC1vuxe/vNl57OVTPGZTzLCQvdzMLAfcB7gGbgRTNb7Jxbm7Tb9cBe59wUM1sIfBl4f8bTisiQMDPyc8Lk54QhS+fix7uLPu6IxuNEY72XY8nbY94+ScvJ+3XF+vtzieWYS/y8mEus737e/RjtXheHuLdPz/Ze+ya+z6hBukJoOmfos4Am59xGADN7GFgAJBf6AuBO7/nPgXvNzJzuHSYigyQUMnJ7hjx0Q3FI7wYX44GtScvN3rqU+zjnosB+YEzfb2RmN5hZo5k1trS0nFxiERFJKZ1CTzXq3/fMO519cM494JxrcM41VFZWppNPRETSlE6hNwO1Scs1wPb+9jGzCFAGtGYioIiIpCedQn8RqDOzSWaWCywEFvfZZzFwnff8GuBZjZ+LiAytE74p6pyLmtlNwFIS7zx83zm3xsy+BDQ65xYD3wN+bGZNJM7MFw5maBEROVZa89Cdc0uAJX3W3ZH0vB14X2ajiYjIQOgycCIiw4QKXURkmPDtFnRm1gJsPsk/XgHszmCcwZDtGbM9HyhjJmR7Psj+jNmWb4JzLuW8b98K/e0ws8b+7qmXLbI9Y7bnA2XMhGzPB9mfMdvzJdOQi4jIMKFCFxEZJoJa6A/4HSAN2Z4x2/OBMmZCtueD7M+Y7fl6BHIMXUREjhXUM3QREelDhS4iMkwErtDNbJ6ZrTezJjO71acMtWb2nJmtM7M1ZvYv3vpyM3vazF73Hkd7683MvullXmVm5wxRzrCZvWxmT3rLk8xsuZfvZ97F1jCzPG+5yds+cYjyjTKzn5vZa96xfFcWHsP/5f0drzazn5pZvt/H0cy+b2a7zGx10roBHzczu87b/3Uzuy7Vz8pgvru9v+dVZva4mY1K2nabl2+9mV2etH7QXuupMiZt+99m5syswlse8mN40pxzgfkicXGwN4DTgFxgJVDvQ46xwDne8xJgA1APfAW41Vt/K/Bl7/mVwG9IXDd+NrB8iHJ+GngIeNJbfgRY6D3/DvAJ7/mNwHe85wuBnw1Rvh8C/+g9zwVGZdMxJHHjljeBgqTj91G/jyNwEXAOsDpp3YCOG1AObPQeR3vPRw9ivsuAiPf8y0n56r3XcR4wyXt9hwf7tZ4qo7e+lsSFCDcDFX4dw5P+7/Lzh5/EX8K7gKVJy7cBt2VBrl+SuOfqemCst24ssN57fj+wKGn/nv0GMVMN8AxwCfCk949xd9KLqudYev+A3+U9j3j72SDnK/XK0vqsz6Zj2H0nrnLvuDwJXJ4NxxGY2KcwB3TcgEXA/Unre+2X6Xx9tv0d8KD3vNdruPsYDsVrPVVGErfQnAFs4mih+3IMT+YraEMu6dwOb0h5v1bPBJYD1c65twC8xypvNz9yfx24BYh7y2OAfS5xi8C+GdK6hWCGnQa0AP/PGxb6rpkVkUXH0Dm3DfgqsAV4i8RxWUF2HcduAz1ufr6WPkbijJfj5BjyfGY2H9jmnFvZZ1PWZDyRoBV6Wre6GypmVgw8BtzsnDtwvF1TrBu03Gb2XmCXc25Fmhn8OK4REr/yfts5NxM4TGKooD9DntEbh15AYihgHFAEXHGcHFn179PTXyZfsprZ7UAUeLB7VT85hvo1UwjcDtyRanM/WbLu7ztohZ7O7fCGhJnlkCjzB51zv/BW7zSzsd72scAub/1Q5z4fmG9mm4CHSQy7fB0YZYlbBPbN4MctBJuBZufccm/55yQKPluOIcClwJvOuRbnXBfwC+A8sus4dhvocRvy4+m9afhe4IPOG6PIonyTSfyPe6X3uqkBXjKzU7Io4wkFrdDTuR3eoDMzI3GXpnXOuXuSNiXfiu86EmPr3es/4r1bPhvY3/3r8WBwzt3mnKtxzk0kcYyedc59EHiOxC0CU+Ub0lsIOud2AFvNbKq3ai6wliw5hp4twGwzK/T+zrszZs1xTDLQ47YUuMzMRnu/iVzmrRsUZjYP+Bww3znX1if3Qm+G0CSgDvgrQ/xad8696pyrcs5N9F43zSQmPuwgS45hWvwcwD/JNzKuJDGr5A3gdp8yXEDiV6tVwCve15UkxkufAV73Hsu9/Q24z8v8KtAwhFnncHSWy2kkXixNwKNAnrc+31tu8rafNkTZzgYaveP4BImZAll1DIF/BV4DVgM/JjEbw9fjCPyUxJh+F4niuf5kjhuJsewm7+t/DHK+JhLjzd2vl+8k7X+7l289cEXS+kF7rafK2Gf7Jo6+KTrkx/Bkv/TRfxGRYSJoQy4iItIPFbqIyDChQhcRGSZU6CIiw4QKXURkmFChi4gMEyp0EZFh4v8DHztyX7g4QlkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.linspace(1, N_ITERS, N_ITERS), costList)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cost function decreases monotonically, and good accuracy is achieved."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}