08-Dog Breed_v1.3.2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIR = \"C:/Users/Omkar/ML and DS/Dog/dataset\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "CATEGORIES = [\"Golden1\", \"German1\"]\n",
    "img_size = 64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data=[]\n",
    "\n",
    "def create_training_data():\n",
    "    for category in CATEGORIES:\n",
    "        path = os.path.join(DATADIR, category)\n",
    "        class_num = CATEGORIES.index(category)\n",
    "        for img in os.listdir(path):\n",
    "            img_array = cv2.imread(os.path.join(path, img))\n",
    "            new_array = cv2.resize(img_array,(img_size,img_size))\n",
    "            training_data.append([new_array, class_num])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "604\n"
     ]
    }
   ],
   "source": [
    "create_training_data()\n",
    "print(len(training_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(training_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Seperation of X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []\n",
    "for features, label in training_data:\n",
    "    X.append(features)\n",
    "    y.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(X)\n",
    "y = np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X = (604, 64, 64, 3)\n",
      "Shape of y = (604,)\n"
     ]
    }
   ],
   "source": [
    "print(\"Shape of X = \"+ str(X.shape))\n",
    "print(\"Shape of y = \" + str(y.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test=train_test_split(X,y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Reshaping y**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = y_train.reshape((1, y_train.shape[0]))\n",
    "y_test = y_test.reshape((1, y_test.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example of picture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y= [0]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztvWmwHNd1JnhOZtZeb1/wHtaHHeC+gCQoaqFI06Iojmi3KEtyh027Nc2OaNkh98jdkqYjJronemLsH932TMSEZ9i2R2yNbG2WhxSthRRJiPsCEgBJEDvwiIfl4a31aq/Kyrzzowp5FuABjwJRoKbuF/Hi3ap78+bNm3krz7nnnO+gMQYsLCw6C86VHoCFhUX7YRe+hUUHwi58C4sOhF34FhYdCLvwLSw6EHbhW1h0IOzCt7DoQFzSwkfEexHxACIeRsRvfFCDsrCwuLzAX9WBBxFdADgIAPcAwAkAeB0AvmSMefeDG56FhcXlgHcJx94KAIeNMUcBABDxuwDwAAAsuvDjyaxJdfU3P4TyBwcdJnzoHyPExeuAV7E61LX0Be8OVUMDi/eP7EDDxu+oc4ViHKryAr+z8keYjxcXbafPzfsXpzqnHX3helLwCxpBVPaDAJYCR4+RnV1UmXNuzOJjFH3QGM95WV1wvsXJFz/3BaC7lKc+/7nPmW7W7gLdyXlTdWHI5wNVXQgAALXqAjTqlQudAgAubeGvAIAJ9vkEANx2oQNSXf1w+wNfAwAApxaKulgiFZWN8UVd6MXpg9+Iivrq/EYtKruerqUHx3VcKrvyoQ/FypEPiuvScWGdxhhn3wMAVOo0DieRkMNosJun+q/51CcCXbMbl/036vWonPTULQxoXvnDFurr9Olz30CXqJufW4jKpwu5Rcfrsh/rOMpxBEBjjCfYvDXkOMSPmKPqgO51gs1jo9EQ7QKfrtlx5FxByM4NbH4vtJoVXHV/OXy/ft5251xLQOd2z/kBovE3TIO1k6hV2PhdOd+1WhUAAPa+8u1Fx8pxKTr++WbunJ9URHwYEXci4s56pXgJp7OwsPigcClv/BMAsIp9XgkAp3QjY8wjAPAIAEDP0BqDXvOX29XSvEe/6DFXviV99nvisd+bSrUq2vFfXM+Tv5cOsl9+Q7+wgVI5Avbr66m3aY29aZJJGqNWF7oSyahs6lKy4W+kmi9/CN0Y9eMAidiB+omNpWI0RpS/3bUyvYH4lYWhFNm9OElYueKc7D9J/ZuFkH2v3hMuk758Wec5NHfmAiJqg6kSus5l97DK7nUsFhPt+HG6DxS6kLNou18V+hk5i0CpSI5L5wt9+UzEEyTdIXvP10sVeS523b6Ses6qWku9rEt5478OABsRcS0ixgHgiwDw+CX0Z2Fh0Sb8ym98Y0wDEf8IAH4OTXXkb40xez+wkVlYWFw2XIqoD8aYnwDATz6gsVhYWLQJl7Tw3y/QccGNZwEAwAWp59SYHuglk/I4Q7vkHrJd4LrSK2Ns517t6nMVN2Q73/F4XLRrVMtROVC72HxXO2Q78EbtJIfs1EFcmcqq7DhPXic/XcgsFGmUY/SrpN/5Rup6aWC73+w68RzLAE1IXNWl4mnqj7YMIBaT46gzs59xpU5rmBnNMbx/OadJtlt/jt6N1GcyxfZN1L4MMP35XEsfe17cxTVb3u4c/Zzt0OsdfjEU1oc2OSJ7RrykXHb8eazX6PnwXLmXwccRhHL9QLTXsDQl37rsWlh0IOzCt7DoQLRX1EeEWLJpRkonpBiTBO5oIUUt5GJ1wExISjTkIpp2KDBM/ubimlEiU1ecRE8t8sWZWGqYeOYEsg+PmQuTSuVwmJdcMivNlpkMidIjvSNReTYvzW0LRZK/y4EU9RNM1OMmTaO8QWo1Oi6dkb//ldJMVN66mUTseXZeAIBckean4SgTG9DnkA8R5VwJj7YLeCgKk6mSZh1msqvX5Rhdpibx/s4R2cPFzbjCY/MCTkyhcEZaXORuqOcqYM+3yxxzGupaYkyVPef5fp+u9/aNb2HRgbAL38KiA2EXvoVFB6KtOr7jxiDTswwAAJINqb9ArURFFcATZzqzz3REF2UwT4ztDcS0Ky4Lkujr6Y/KiZicgkyadNMaC7YBAKhV6HOK6YjprrRoF4Q82EaNg6l+jiP18zCgOTmZK9D3vpyrRIJ+r1MxGWCTy81SXSITlX3VR38vuezOqT2Erh7aX2jMUX9daWnO6+ujPipF6X7sJWhcR8bPRGWj3LHdBE2I3lOpO3Svu1mglqMsWQHStTkN5arNdO1Q7BOovRd2P31fusry/QBUujt3seW6OjrabEl7JbWafK74sEJmrka1D1ZjAWqOI5/bRkNNykVg3/gWFh0Iu/AtLDoQbRX1e9Nx+O2b1gAAQCF3WtQtGyTxElGKfC+8+EZUDuo05FhcR+BROQikaOuxeHFjyDvPUWYdw0xgMSVT9neRuBbUSewyQUm0SzL1AbVXHysHgRQHnTgdx01KmYQU5xPMszGWkN5/ifWro/LcHInwlYoUX+t1+rxmeEDUTc9QDL4Tp0mNefpaaA6yPZqNhET/a8a66WvlhXh0gtQAL9Et6mINaltjnoaxtLzmRpHu5zlB7EyO5h5/ofb+4x6h6plw2b2o+/q5Or9Xn1ZbCky817H6Iesjxup8X0XgsWep4fuq7v29w+0b38KiA2EXvoVFB6K9on5PHO7/1AoAAEAzKupctltfC+Xv0a63d0Vlj22IGuUxV2eUXbGYFOUaIe28czFa+1elmAea9oXiBBXVBO26x0O5U224iKaDdFhZi99xRu5Rr1HdzJwUL7u6slG5NCPVDOSkDkyEzJcWRLtymY7bd3C/qLv++tujsslTuwSjRwMAiDFPxnxZu9Mxzz0mOWc9Kb5u2kDPwXxZ7nabKvV5Ypq8CV23R7Rz43QupSVCw+f8hNyrUVNX0XzHlfWCU30551CdUZGL21r05qQxgRokHxfXQDh5TLOS7+rL/n0l+l8M9o1vYdGBsAvfwqIDYRe+hUUHoq06vgGAs5yMGMhT+0ztmQ2krldhBAr9SdKH/JrUxVxmsqsqc14fI7IMGMlFxZfnmmf6V0x59WGFmVOYN6GnvOdcRljxwmvPi7pXXn89KhcreVGXSDEPMbb70JeS5raH/vAPo/LeQ5Lt7MCb1D+P7kqnpXdhwiWTWKjMdP0sgrDA9gxSaanjl5kn46h0MoOQfa4maE7Lvra3kW462KP2Q7pJp810DUblhXl5z86wSEPPlc8E99DjerHm/OA6v/aB44QsMaXjO+w+hcyEFypzXpoRZfK8BQCS8NU4fE9CzofD+tDmvLPReUuN0bNvfAuLDoRd+BYWHYi2ivoLpQr85JWmaJqUVh3oH14WlR9/7EeizvPJdHaCeXAVF2RwyalTk1SenBR1v/2b90flGCM7+Itv/e+iXSZG4qw2kTRCEp17UmTK+ti2O0W762/cHpX3vvGyqOuOMfEVpFjKJe4k41tDI9UWJ6TxL8/2irqjPpmNuOpzw5abRLvNV22i/tPDos44pAYsW8O47pQnmcdEZw+lN92ZM+SRFzqLi/rcg66ugqKq7L3kMBUvu1Kea4SJ1Qvz8p7l89RngZGxGOWVyU2fDZULwXM4cYsUpuvsY4IRZfh1aaoFYSaWy84w3kTP46QfchycS9+g8vpsfV5qtgD7xrew6EDYhW9h0YGwC9/CogPRVh1/4r334Gv/6mEAAGiUpN76n/7XP4/KqKLdvv2t/zMqNwJGbqhMcUmmiyVVNtEMT7jLcs+tz2RFO25qCVT6aGR6rMPIGSeOHRLtrr/5lqi8dfM1om60jyLQRoZGRF0fi5LrYuPq6R8U7Y6dor2NrZtk/zdefUNUdnkGYkbKAQAwx4g+qiqDbV8XM9sxwpS+HrmfgIz8oYHSnrd89fKoXCnRuRLKlMUjCAHl/ez2aMwN5hvLiVMBAIIYRefFBqX7dF8f3fhGlZ6P03OSOKTEyF+Mq/pnw4qpPAwBczX3mWkylpGmzxojKtWBdNzMyPV61118eeroP3yf7/CLtkbEv0XEKUR8h33Xj4hPIeKh1v++93VWCwuLK4ql/Ex8CwDuVd99AwCeNsZsBICnW58tLCx+TXBRUd8Y8xwijqmvHwCAO1vlRwFgBwB8/WJ9OYEPmXzTzKYyLsH4bvJwG+kdEnW9zGbCMjNBLCbFKZ6OacPataKOm2tcZlr5/d/772UfqRQ7Rop8HhP9QyayVhShRpH5T93yybtFXYxdgObj85k9b5KJ0ZOnp0U7w8x0FWV6QkYGkWVebI4yDXX1UIQbVlX+ABbtFrDx1tV18ujIupHmq4EeUmkSLrUrTOdEu4CZCANlLqyG5NnoedyDUHohFqpMXTuHLITG3NXDVIKsEtlZROjsjFRDC3k6Tudh4K9Oh6k7pYpM4e5w86wyJTpsjDydlqPbsWg9HSUY4tm5u7wptJYZY04DALT+D1+kvYWFxYcIl31zDxEfBoCHAQA89xxeJAsLiyuAX3Xhn0HEUWPMaUQcBYCpxRoaYx4BgEcAAK677nrzxOM/BwCAf/j7H4p2JeYRVVWRBv/yj/8N6495cylRvM7EsFDtzHLOtsYs7ehW1c59hakVWhTnbNhccnY9FaHCPP6yKuNpwMbvqnRSQZ1Th1O7bpU9eHiQBKyKL8e/98SxqNyfJJG4V+0Qc1prvUsuIlhiLG2YonTmKZ18X1piqlUS/fuytDvvTsvd9N5uojp/4/Wdou7J538clf/4T/40Ktd9KUZ3e3Sf4ik536USjavOyFgSiv4a2OfUctlHo5/GX69JtWhiap7a+XSc5ysyD+a9qGJvIMa4Fp3G4qogZ+kwRu3qt+7Z5fbcexwAHmqVHwKAx37FfiwsLK4AlmLO+3sAeBkANiPiCUT8MgD8GQDcg4iHAOCe1mcLC4tfEyxlV/9Li1Tdvcj3FhYWH3K01XOv4vuwpxU1V1X7fJwT31Emk4r4SLqNJiPkkV6O0luR6bjeheQcpjuF56Qe5h5WrP+GNEO5TEeuKzOa4R5oWuBiH7n5MUxJHX+B5eEKlcknnaDrzHYTQUhVRRom2Bj7eqX3osPMhZNTtH2T7pKEIw6bq7gmf2R7JXlkOjkqkx0zEa5eOybqrj+xPir/3X/7v6Lyb332d0U7l6Wamp+V203ZDOnnvWwaa74iw2BzWleKssuIW5Nqb2DdOpqToEbztlCU+xCnZ2hvA0HOdz5PbRNp6iOuKEEaLA2co/I14FkzpmYYWQTWV9/CogNhF76FRQeiraJ+qVCEN59/EQAAXBWsEWOirVkic1hoNDvaBdoyNYB75BklzmtvvcXAOdoaStTn0Pzn/HMYqmy57Hc4YLmgpvPKg6tI3m+jg1L83sCCYxoB9Y8g1YVknOp6uqToubBAHPzZDEsbprgQuzN07h5PBsfUa4zYgnmjIcyKdpUqia+1QM59dpA49/uL70XlRFyOd+bEvqj8wyekmfgr/5YcSo8dPxWVlw8sE+1SzKSWVCQXZcbDWHfl/fSY6B9nWXszSdmui81VwpUcinsPHI3KPDCnXJHifIOZgjUpCkbP8dLWjn3jW1h0IOzCt7DoQNiFb2HRgWirju8YgHjLjBLoOqYHNpyl6d1aP5e86fIYHlV1wXbm/HsBuo4TIVwoRfGF+tCmF64Lz5XJxJPKyijEMCRde76qTEOM9ILrlV5MzniDkYqcnpMRc1VGkhJPk+7eqMmotRpLiuerd4iMTqPjlg1J/XaqQJGH8YxMk73lGiI0+cLvfDYqjx8+I9od2Ee5/9YMybx63/vWt6LysdN03L/9UxlMmq9RBF5c7TWkk2QSzCrCkfwCXafncYJU2UcXU9d9b17Ubd5E99BBmoOTpySZLDdBFkvSXGi85v1c6h6VfeNbWHQg7MK3sOhAtFXUBzDE06Z+crgErEV4WEQ6PqcZM+9pvgRuItR85RwOi0ALQ62QLDYORVDB+1PHuTzlkhLLqnUybRlDInYuJ1Ncc8tZLZRiepZ57iEz2WXVuSoVZlYEaTZKdRO3XrFIqgMaeTW5MvVfXJBqwECWxNcay38gDYIAxSKLnlNpz6tMbXmdLHYwrFhcPvfgF6Pyt//rfxZ1SeYpeNUgidH/+K3/KtodnySPv69+7d+LOsNSkdVAekByykaPpRsrlyQxSYzPv0qrHrJwdY+ZBFcsU+nXmTdkrijv2XSxOSdLlPTtG9/CohNhF76FRQei7dlyIzpiJYvXeVZTtavKiQs4jTC6i++Yn0NUwD3mLuTxx8T7C+2QXmj33+VMQ0oNCBnRQl1ZL4qMVKPms4AjRaLBU02lPenBtbyPds2rbA7qikKbz89cqSzq0hmWZZd5qlXLkmxjoU7nVvwUkE7QGNcy/sPcKbkjz/nmZhekSuOyS/MrdO5MWonABRr/F/7gj0RdfmYmKj/+4x9E5Zi6tcuHiefRVRTaQ4yMZM97R0Rdpptk/Qzz+NOZlpNx6qNQl/MN7Jngj62nePUaDqM6T0hRv7ev2XZnfGnvcvvGt7DoQNiFb2HRgbAL38KiA9F2c17gNM0hRpmGPKbHxl2pw0mdmRVVpFStRnplqFIMOYKGcHH9nOv/2iOP6/W8znFkHwHbJ3AU/aFhx+V9udcwmScTUIylvOqT0wGVIpmU0ilFOMrSj9UZwaibkWm45pi+npdOYJArE2lElnHse4oYosLMj7mS1P+7E6SPHvJZf+qaKyzKrFiVJsGQjT9k7Bija2U6sBoz2Zm6eqQTpLv/3r/4SlQ+9d5h0eyZX/yczqUiR09P7I3Kr//yZ6LuJNtfuOWWj0flTZuuEu3cLjIlxtTeTpal2yqUqT9HmTdDTs6i9sHOcoXYNNkWFhaLwi58C4sORNvNeWelqLgniSE8hwWiKCIEZN503MQRKFNZinGq1+rSRyxkIqUkw9Bmv/MH4ujjZMCO4tXjZkWlLtRZ3WxBenflWXBMjCUXcJUpzkWS/bl6AwBQLJNatGqYsvE2VPqrBAtmOX3wpKhLJ8lEZRivYdCQOkEP4+CrF2dE3cE3Xo/Ku94+EJU/ff9vi3ZOgrjuMwk5H4FHqkSxQnOjCUGcGBujup/GI+G3xDzfBkZlirV/9oUvR+Wa8gh99pkdUdlX93qY8SGOH9wTlV987inR7s5P3BWVt15zk6hr1OjZjLvMJJiUZsVqlc5dPUcNOFuyQToWFhaLwC58C4sOhF34FhYdiLbq+IgOxGNNU4wJpcshV6drSk+LM478kBEQeAlJisBNgp4iTKixxHchM0u5yi3SZ/pjTOfE47o7d+1V+fEajIQhrggZGuy4EKQO57DYNR4RVlXXmWRd+o4aI3MV5Wmyiyjn9OQ4kVfW5qQbbXpoHfXRTaazYq4g2g31konq1MH9om73Cy9T/8jGUVP7BGkyM+ZQpgP3fZoPTmqZjMlnx+Vz7Czujm2YOUxlFweX5RlMqj5+5w/+VVTe8/YuUffum3SdIdubiodyv2L/O69F5Vd2virqvvSFh6PyINsz8NX+DbJ72N0l56DaMpM6OifgIlhKCq1ViPgsIu5DxL2I+NXW9/2I+BQiHmr971vSGS0sLK44liLqNwDga8aYrQCwHQC+gohXAcA3AOBpY8xGAHi69dnCwuLXAEvJnXcaAE63ygVE3AcAKwDgAQC4s9XsUQDYAQBfP08XEYLQQKFlllGU8hA2SDTyEur3iEU2xT0uzqvopfriKYa8OIlyAeObNypnccDUCu1dWGNiaqlAnmoNJV3l8lS3bEAKQoLkoirFwZD3wzwKSzU5Wak0jaugzZZl6uTMAnnMnchJPvsf/fVfR+Xufmla/cjdRGwxuGEVtUtLtaIrSfdl3fqNou7AyOqofOsnPhqVfcW/zwlSuvslX978OEXrjYyQSqD57IRHpbmAOYvNr6ZJ5GI0KNNnhTXees2Nou6qq6+Jym+8/iIdU5aEHVyjxLRU8eIx9tnQ/Tx08F3Rbu2WrVG5WJEqkzaPXwzva3MPEccA4EYAeBUAlrV+FM7+OAwvfqSFhcWHCUte+IiYBYB/AIA/Mcbk38dxDyPiTkTcWVHx3BYWFlcGS1r4iBiD5qL/jjHmR62vzyDiaKt+FACmznesMeYRY8w2Y8y2VDpzviYWFhZtxkV1fGyGr/0NAOwzxvwXVvU4ADwEAH/W+v/YRc9mKIcdqhTXcUZUmPAUeSXTd0POCKNSP/NIO53iulYk85gTi7GynALu1qndeUPDxpwkfTShTGoDyf6o7CsmoFqR9HXXkefmbsaclBOU63CQoM9dWakXJ9n+xYFWSnIAgGRS6oCb1pAOfvi4NMXlpuk4v7EhKifiKpdbg8ab6ZPj+Ni990fltZtWRuWjx6XJbjZHUmBBpZaOsXx2fWx/ITwnKwPhQrkWLtROPDvnMDtRuaFy09WY2fj6bRSdd/vtnxDtHnvse1F54qR8R8aY+dBhZJ7v7HpZtHtt986ovGmzjP4bW900wfL9qwthKXb8OwDg9wDgbUTc3fruf4Tmgv8+In4ZAI4DwOeXdEYLC4srjqXs6r8Ai3v+3/3BDsfCwqIdaKvnXjzmwqrhpnmrWpFiuuG/LcqMVmXiS5V5tGkhLpEgU1GgyDy5da/KvPNCkKIRF7e1eMzNhak0mQf9kryWLFNb6kaKx3VmmvN8aYqLszlIMa/B+bIkZ8wj7a12r5Tpnl2WbmtZSLd3oSZNhzfdsj0qr9u6Xo7RIxNknkeOoZzTgN0L1CSPcTJbTp2hdtMzc6odjbGqPNV4j3GHeTyGWhWERXFOjoZFwFVDne/AMKIVFTgqPAq5WXe+Jp+J3/7iQ1HZV2Qkc3m6n6MsNbgbyj7icTrBwX1virp39jS9AXM5Nb+LwPrqW1h0IOzCt7DoQLQ3W67jQDbbFJFjKntrjRGz+2q33mOeVC4TuyqKzD2R5IQMiqiAqxJ8p1eJbl0pqiuXpLuCYbvrYYxEalftAvNd2rragC4zLvpYSnpw9TQY0QILRurukt5/XAFxlKoyOUdjHhogIo5qWYr6V28ai8phfKuo23OUgnZ4KqjuQZnpNsdE0aSnPCUZ9/34BPVXKMpAH48dNzktxdSxYZZFln1fd+VNqzPVUN0KiLEDG+yxchVfo8tUsjhqtz6Wmk0HwTBPT4+pCNpik2NelEY9my6zcE3Os2dOe58yK4q2iiVaXpTuBTI3c9g3voVFB8IufAuLDoRd+BYWHYj2km0aE5ELeCpKy3HirKw49znBJtOzHe3hx0L+UCnvHvPQ4yQaRpluHCQNuqc/K+oazAxTZtzrmsyDe5alU9JNeSZPul46rqY/zvLq+WTOmy3KGIeuQfIMzE1JL7CZ+dNReXCAOOUbDbkXcLpACu/0gtStSzW6zhVZGn+gIsL6WHSezkcYMuJTn/HeJzPdop1fpflYOSzr4h67TzxXofKGZM0gofRih5mCHaaru+qdZ5gZN0zI++6wKFB9nS5wMyPV6VTsXPduKB3fY0OpMRLN3/0Xfyz7YBGE3370b+UYW/stS32T2ze+hUUHwi58C4sORFtFfc91oLe7ac7zAymSLeTJO02nGJYpr5n5RPWPrC6mzDXcqyqRJFNcXQU1VFl66qAh1QDOrVepkJmrUZOpn5J9JB5Xq9LrLpki8TiTlFfQzT6XGiR+V+alqF/2ic9+bIX03BtdQcEyhTyRb8RiMpDoyBSJ2BVl6osz4hOuWnkxaX50mfrQUKJtfy957mUyzAtxRppIa8zemXLkvRgdItG/xuYxpUxl61atiMorRyUtRJ2RnRgmKnd1S7VigaXofufQCVHHU7hpa16MjaVhGK9joD0GL5C2jasjTFXRqc1Y9/D5B39f1Pnl5vMy8ejfwFJg3/gWFh0Iu/AtLDoQduFbWHQg2syrj5BqRa45vnJzZTp5qSb14m6mj5VYOua4MocJMgVlMqn63PxG51IpyISeXatLndNn0WgJ5G6iyoTETE9plgIZAKAc0rXNzRdFXV8P6cL9PdRng+n0AADVKil/TkLq3b09NAeHdhPBRtfIBtHOBHTuuuK6B8aDzyfI9+Rk5QOaD1fz+zN352vXjUXlHROHRLMKS2s9NyNNk5PH347KN27bFpWvVsSecUZeWa7J/RDBM8/caxdycq+B5z/cuk7um5QqVDc7K12OuWmO73MEypzMh3FOxCB7lgxT5OVTBRCy13RZMbw6ydaeCuqjzg/7xrew6EDYhW9h0YFou+detUVmUVNkBMksecw1fCliC9MIS5MVS0jvP54y2lUmwWyMpZZmZqi4MnNxQgbNx4fsdxKZGSqp0hn3sNRSC4pwxGEkEqh+d4tV+tzP+Pf7XNn/O4xv3lGEDyfO5KKyx+anvJAT7VIpUh8ma/OiLsHq0kwMrTSkCjZ5ilJj16tyHCMDdD9Htmym8yakKJpnpqxiQfa/dcOWqDw2uiYqO0qMNuxZQldFtLE5bjg0Rk3mwQPy6g3pGciV0q6sfOZKVXrmWPAcoPI+NVynVMwh3BsQOekHyDWCIvWbqAK/FdlpdLjpIrBvfAuLDoRd+BYWHYj2ivoAkbBSVN5iXT0k2iZTciccmPiWckmErNfrqhnbCVdBKXH3/LudeoeVH6cMAyJ7bpyVUWWi5VlOHUeKdckkHec25AkqTEybydM4BlIqPVKcROKgKnnq0knq4+ZrScR+5uXdot2KbvLwi3tSlUAWxOSwzLQz40dFuwO7XqdjfDkHu+dph37Nv/lmVF61apVod2bvgai8etVKUZdnXHRcHNZBXPyzTp2GTIb3GEEKqvvOefY0yQXfade07dx6xIPJKoo/0FuUrxaAe/Xx/pwLnEtDB6VdDPaNb2HRgbAL38KiA2EXvoVFB6LN5jyAWr2pp6QzkuygXGFecUnlCcf0I4eZRQpKj+K8+q7S6TWBwllozSvG9DRXEWX6ATfFsVTbDWnKyqTJHNZQKa65KSqbUlGIzIy5UKN2BeVZV+akEUrvC3yax3KezH5Xb14r2p2aJhNeV0Z6BjaYRx6PJnzmn34k25Up+m/dCqmfQ43083ff2ROV16xZJ5qVpim1dPfwoKjLz5EXXort+8Tick+CRxDG9f4QM80FrNydkPsmC2znsC78AAAgAElEQVSOdfSc+KxUae5FaVg6LUdFh4YNmlNtcjNw/mfzQjr9peKib3xETCLia4i4BxH3IuJ/bH2/FhFfRcRDiPg9RIxfrC8LC4sPB5Yi6tcA4C5jzPUAcAMA3IuI2wHgzwHgL4wxGwFgHgC+fPmGaWFh8UFiKbnzDACcjeiItf4MANwFAL/b+v5RAPgPAPBXF+orDEMoV5piqhOTHlB5xis3PNwr6oB7uzFPrzRLYwUgzXue4sHjnGc8y64bl557homDnqoDnkKLBcegCqeIe4xfXVocIcO8/DzN/c+6MYZEyKTieefqyGxZBvqMrCZxea5IJtPloytEu+OnKWttKq7mm5nESmVSCUxdqhwnJ4kv/9TkjKgzTBW6f4BUiYRyOdvITHiTFRk4k8kQoclqZgacOvGeaBdnqc4ClVmY35ks6y+uTKncZJzQ3PSMc7+uArc4wckc485PZ1VKeJZ7S/PxcU0iFJ6dlw9L2txDRLeVKXcKAJ4CgCMAkDNk4DwBACsWO97CwuLDhSUtfGNMYIy5AQBWAsCtALD1fM3OdywiPoyIOxFxZ1FlUbGwsLgyeF/mPGNMDgB2AMB2AOhFjHiHVwLAqUWOecQYs80Ysy2b7TpfEwsLizbjojo+Ig4BgG+MySFiCgB+A5obe88CwIMA8F0AeAgAHrtYXw4ipOJNfWkqL3XTDHNlrVakma6LkTzGPdLrazXZR4NdThioqDhmXgm5dUbphMKSo3SxBud2B+o/rhK21Zj+qB2FYy7tbVRVtFuG7SmEzHU4pkyTsYD087Qn89kVSjQn6JDuO37suBxjlfTpbFYST6wfJt7+Ektr/eCXvija/fAH/09UPqDceYGZHHNTbD/BlfsyHtORu0Np4q0b6uOl116OyresHxPtzhRofyjbL3XrIjO1GrYHVPSlgOqyzyVF5lGr0nFBXeXtYzkOBRdJqDZ3DI+6U6Y+Zs5D4Tqscj6wvYdARRAuNR34WSzFjj8KAI9i04HZAYDvG2OeQMR3AeC7iPifAGAXACyN3tPCwuKKYym7+m8BwI3n+f4oNPV9CwuLXzO01XMvMJSeqao565lk5ClTi5MksSabZHUq8o2LOwlFsFHnBBsimkuKXVy88tU4eOpjw8q+igQUrA4gRbKApZZOqTTZAVMtMoY80MJQqi1bNi1np5J91FgasTjLA7B8peSRzyTpOk8uSCKOBkvEvXrlaFQuzkjVpJdF+K1dLj0DhwdIBZlgabJzC1KM9pj5ylUpvxPMQ+/Ie8R17ysxupdxL65fLnn1earzckDtHEfe2yrzViz5UtUsVWjMM5OTom7teoqAPDFHJk0evQkAUGeEKVosF+nB2POn23FPvvB9RuNpWF99C4sOhF34FhYdiLaK+mBM5JnkKw8ol3k2NZQYlmdZWlcOMLIDKQFDLMYCOVB5xTHRvMS8BLtVKiXu+WXCxUUyQQOoyB/ybJc5oXgBnRjz4FKWh4B5BnqMZ6+mLA/9XZw/UF5nuUYiZp2Nf2peesVhwLwLlYg9eYZE8xRTFwo5tdtdo/fGXXfdL+rqTFx+6/DJqLy5u1+0c1kkVEoFZ1UrNP6FKQo4mh2QfaSYylerSl+RdatI9K/PkaoyflRaIQrsXmy86mZRNztHVomRlBThj02TFdtn5qI4Lr60tHrJvfUWE/sBAAzPxnuJATz2jW9h0YGwC9/CogNhF76FRQeivSm0wICHTX0yHZf6y9nvAQByKoVWvkhte5gXX0Ol0Eoy77y4MvX5zMTGSR10NBdwE4oKj6oxHZyb+lJqHJpgU9TFFm8XZ0SiDcNNjtIbLc68Bh1P7iEgMx/W2O2dLso59Zi3YapL7nOcPD4RlY8eIx22Nn9atFu7hdw7xk/Pirp0F415+aaro3JdmblihqfhUpF1Mfq8eTUReLx78Jhol9w6FpWLFblfgdNkYkul6dyjY2Oi3UsvvBKVX5zYIeoqZdrbuHWj9JT87ve+E5W/8NC/jMrVojQ5et7i79jFUr9pQtALITpuiYfYN76FRQfCLnwLiw5Eez33QgPFclMEEmmxACDHRCP0pAi8apCIImqMrKFYkyaNWoJEslhW8fax4BiPiZtFlfrJZaZEzdEO3OzFqho60IcFZLjKrNjH+PjyKkiHH8d5/JLKzFWvs/lRWoWw7jEzUdKT6kKVmdvmipJgI9NF4qyTojndOip59UZHSGUql6Spz0eqK/j0mC2ofAolRqyyWqkcnD9vepJMgvPTMhD0eBfdz0NpqfpsXTESlQ3jvTtwTHrgTbDHoFFSz1WDxrz/iDz3xg2kxjz52ONR+eprrxPthpezuVMBQpxYBZlZ23e0yY6lcAut556FhcX7hF34FhYdCLvwLSw6EO3l1Q9DqLTIIbUZKsbMS9PavZSbsxj3fKDMPyHr01U6LXdt5Tq5dpsNmdJcrcsorSBgZB6G+mgoRZu72xaUGS0WWzyqz7DU2x4j5aypSEY/IJ0/r3TmODN3osPcP5WLJyeJ9BSZJ38okkhzmlRc8Vyv95Q+muki02SsRuOvlKXZD5j76kxemsAG2V5Mbz+RiN540zbR7vCRI1E5Nz8n6saR5qc3RXtF87MLol0wTdcyX5T6/1aW5rsxL4+Lp1nOxy6a78PjJ0S713a9GZU/98BviboKz5PAvncX4dsHAAgWtxgvCfaNb2HRgbAL38KiA9Fezz0kkd5RJrs4I13w5qUIX2LsvIFP5aTi1a8yTrV5xemHLBW0xzzmsiraqlTjEXJqegyPrKM+AuVhJZQHJUYHLILLKBWBp1Iql0lFSHRJklLOy649Dz2WGQo5R1tMio2cH15LjTxCzGW18Zi8zv4uIuKYzeVEXYId57l07t5eec8KCyTmThekWtTF1AXuHblyzZhol04RV9/evTIdeH7FUFQeHCCxH1PyqsdW0ri6F6R3nsfITWJdUoVM99IcIFOz8gtyPtav2xiVf/rzJ0Tdpi1EWj22dn1ULqg5jTG3T33fz94znZ5rMdg3voVFB8IufAuLDkRbRf3QhFBvZSWNKQEzMCROLe+Tom3ARKhcjmVGjWmaYkLdV1x3Pnmn9WRJhEzEpVdchR0XKHpth4nt3Nuqrnb/ayxFlxfIPhrsc6UqPea6k8wTjqk3iUBlBWZDHuiW2WEbzHoRhmyuPHmdBa7SKHWE3xmX1VUqcke7HmPBJZ6kxi4xr0qu0tRUTJRreLCTFFO5RaTqUt1An0z5hWz3/6bb7hZ1lSrNXZWJ7C4oavY6XVtXRor6LlNLnZicxyQbss/Uxt6s9HLM58lqEPfkPTt66FBUfu0VohF/4HOfE+38Ko3/nEy673OX377xLSw6EHbhW1h0IOzCt7DoQLQ9Om+updf2KBWFpyb2WNpjAAATkC4cYzzyaZT6ls9IHYxKXhUyE1WlzNr50pyXZvsJdZSedXXGb49M58x4crwBizxMJiXvfbVBOmcyLad/ao70TJ95KMZSsl08SSYl15Hjr7H0TzVGylGt6X0C8shzQEeLUdlDds3SSRAqZXYtcgoEUUnITJhOTXkrcm9ARcDixEkXdpkHYb4gvS1jzFMyUFGfBWbW5Tr+6KDcR1q9hvYNyiXtsUl9zsxIMhJIsP0iFnkZU3kdDNtjyfZdL+qKRTLbLWMeijufe1m0+/hdd0XlfFXejLNrwdUpvhfBkt/4rVTZuxDxidbntYj4KiIeQsTvIWL8Yn1YWFh8OPB+RP2vAsA+9vnPAeAvjDEbAWAeAL78QQ7MwsLi8mFJoj4irgSAzwDA/wIA/wM2ScLuAoDfbTV5FAD+AwD81QU7MgawZc6aU55NPSnynAp8JWIzX7gKEweTyrMuYPJmuSZNZdyqVmdiWMWXgSFdrI+YMhc6CRJZebLShgq2cZk6UlXZtTymnuTKsjJkKkOtQFzuYVVly43TXGl6vwZTVRaKJObWlIgd87hHniItYe+DsEqqQ3FBBth0DVF6rUZdip4hIxXxGfGEziyMTAQuOVJMd30K1upK90VlnSm2xBIsVAJJCFJlt7CPedmB4kk0zHPUrav0WlVSwWJpqSIk2dxxEy+fewCALPMyrVTkXMW46sZyQ/THJTHJUz8jj7+B0SFRNzS0GgDOk85tESz1jf+XAPDvgEzlAwCQMyYyvp8AgBVL7MvCwuIK46ILHxHvB4ApY8wb/OvzND2vkzAiPoyIOxFxZ6VSPl8TCwuLNmMpov4dAPBZRLwPAJIA0A1NCaAXEb3WW38lAJw638HGmEcA4BEAgGXLRi+NKMzCwuIDwUUXvjHmmwDwTQAARLwTAP7UGPPPEfEHAPAgAHwXAB4CgMeW0BcErQg6zlUOIM0QsR6p23iMiGP6NPGkB77cJ+geJFOI56kU1Ez3CXkqbFcTWZLOn4zJ6Umwjw1GxBlzVUpkZsoq+9L0VGO6MKr8aknWTWaA5gBDqdPy9NEVpbt7TGfmpBygzJsBI/1A5c4bY9eWzJBe7Mu4Q/B5SmclPDoxZt9j7tiOygeXTJKZLginRR23dubm6b7HNIkLM/t5MXnffZYnERu071NakHPqV+g4E8h9nyQzp57KSak1O0xmwICRoqRS0r4Zsn0rV73++F5V3md9ZKUb9Mjy1VF5dlrOVaXUzIVQr6uEkovgUhx4vg7Njb7D0NT5/+YS+rKwsGgj3pcDjzFmBwDsaJWPAsCtH/yQLCwsLjfay7lnDPitaKlkUkYopZlYk/OlOFXMk3iYK3K3MqkumDmq6++WEVw+MwEhizgLVJRTg5NqGBX9x8Qow8R0oywoNRZ1F0Mp12VSJEZ3ZeQc+Ixnf26O+nBdKdoWmJoUXCAdMxf7qzUpAnKBu1GXoi3nc09myHzVMyANN1U2H6FK1x2wtORFfm413xXmUdg/KPvv7WLXxsyDDXUuw9SHpPKiXDVIZq8ai9DU2alOTU5FZSeuUo+z44oVbSam5yrNzHlltf3No+mMivqMscF0MX2ypszasTSlB+/rlerl27ube+81Nb7FYH31LSw6EHbhW1h0INoq6tfrPkycPgMAAAMD/bKSiXLck6kJEqcSWSYK1eXO7FzIdnDVTnuqm0RAgyQOZWHxXf2ikte6GPddTXlmcSSYmJvtkuI8OiT21pWXGScP4bv/WcUtGAbcK072Ec+QitNgrBehmo9SlcYfUx5zHtD5kGV5rZUlEce+3URr3TMgr3NojMR2RDq3k5T3zGHWHE0WYpgHZCLBdv+V51u1QPezjFLU7eum8yVZtly/ZlQ7UjWnZ2UgTraLBTSF8tw+8xCtszFW8lJd5R6nK5YNirpcjua1UaDxvvjsK6JdnAUj9fZKVTbmNj8j6rRv54d941tYdCDswrew6EDYhW9h0YFoq47veQg9/U095cy0TFOEuIrahVIf9Q3pLT4jdUgkpJcWN+ukuqX3H7ffOMxcValJHTmdJP02bEh9sc4+pjyWnkp5o4k0SKHsI85+apOKrGGBmQGzcapDxYlfqnFdW547xUyVCdZ/fk56OQYsPXVcmRw95oG2f//hqJyJq7TeNbovM+PyfjI+U5hjavHwyKhoh4zkwlcpugzTw/MFitTTft98DyFUnpjAvPxS3NsyJk1lmKYBn56VfezeS9HoW667StQ5FbaPwuY0o3j73RqN8ac/+JmoM8xs3MU8JYe7hkW7mQWagymVAmzlyia5p3dcPlOLwb7xLSw6EHbhW1h0INoq6sdjMVi7rCm+jA4sE3XTOeKRz89LkbJvaCQq+8Hiv1VdaTIDNlRwTKaLRH8e44GKo6xemo/KSUUk12BmxRoTOHuUWsEFxZriusskSJQrKt60EJjZixFl+CojLue6D+pSLK2GFMySTZCJ7diJcdFubprmuFtfJ2P3ODy+Pypv2SJNSJUSHbfxmvWirpQjk1h+gca/fFgSWfgFMp96A1K05YFVPLgnCHX6KJZSTN13Jn3DVJHGobkQ6wHVVfIy424qQfc3PynJSOanSPyulshEOrRMzmlpilTKFSzYBgDg6NGTUdlPMVUtLftwC6SuCVIRIIITXCK/vn3jW1h0IOzCt7DoQNiFb2HRgWirjh9zPRjpbror1j1pouLpk+fmpI4/lSO91WUEGyaUw+/iHOcqH1w9ZBFoDVLy03Gpc/aziLBeRQiSZzpi4FAfdRVF5TKzVIhSHy0yYshESp47z0xDhrlenjh6RLQb6qH9kZde2inqnJDmsVaia+bkGgAAScbHX8/IfQg+V0lmVoyl5JwWp0lvDTPSjLT/9b1RecXaLVQekO7Y//BPP6W6628QdWtWr2OfuGuvipo0NKdxRz4THtC1VJkCfCYniSxeefn5qDy+75Cou+raj0Tldw4eE3XX37g9Kh+dIbNfMi5dmLtW0nWfmiiIugx7znoGaR+FuykDAMQ8ej6mzkjz6aGT4wAAUFXuzIvBvvEtLDoQduFbWHQg2ptCyxjIt0w0Tlz+5vRniDc9m5WRe8tGSC04evx4VHY8OfypqRNReeWqtaLOL5FY2kBSJTKDkp88YF6DVUVqkE6TSFYqkufUYFqaVp556umovHyd9FRr1EktePl5mSKJk1709ZDId/zIe6Jdl0Pndsziv93JBF2L5nmr+jSnPa5KScVETJd7uKm0Z4Z5p/VkpCrBrJGwevWaqDw/Lc1hcaaSnTopVZqNY+TNWaox4hOQ6lPIUp3VKtJD0auT6rNr956oPLZ5jWg32k338IQrL/Tg+JtR+VMP/L6omz9OIne2i+5ZV1re91KJuGhvvnmDqHv22bei8tF9pC74yhQcsinu7pPP3OZrrwUAgMlXxmEpsG98C4sOhF34FhYdiLaK+g46kEo0dzfny1KMmZojkckvShF75VoSxz3GS1cqy93/LVdfQ/2dkGQKiCS+vfEm5QZJpeTu60AXiYb3fepuUbdvL4l8u/YQSYLxpRxtiqQSrF47IuompmhcyV5Jn+wwL7z5k2z8vrxNFRbY4qostXEmYntsJz+tdoirNdr9NYqUosoIPGZ88mR0uqVnnYc0V309Uj3L9FHbKiMVqSuvuxrSrjtXgwAATs/SM5FOD0RlDFWKtRr1UanK4JU33ngmKhfm6Bno7ZNkGF29JJqPrb9G1JXYHCTjUi1yYzThK1aStcWE0oUuYKrVW3tfEnUnjpHn3tbNt0TlWEZaQMaPH43K01PSKtGXHWidd2mpK+wb38KiA2EXvoVFB8IufAuLDkRbdfyhZcPw8Ff/CAAAXn9bmqgmJshMd3jnq6LujV2MdLBOv1XL166S7d6k407tl6YhSDCvOBbNVa9J80+9Siawv/u7/ybqfI+lQWIxeGFd/n72DJBZLhaXZq4KMwPmi/OiLhmy6DdGPloJpDdWxmHjUASVLiNkDBhxqE7bDC7VdWek7n5ylnTOZRtoj+KWj94l+2Chb5WS5OZfvpJMVgNDpP9PHj0j2g0sp3tYd+WezWAf7ZWsHCXyztrClGj3xE+fjcqFcl7U3XEned298MzrUfm1F+QzdvfnPh2V49Nyf2jzCoo8/P63vyXq/vW//lpU/vEPHo/Kn/zYR0W7nTupz3vuu1PUDQ1PROUnn3gtKl+1VXoybtpAnox+Ve6DTZ9pzqvvLy2F1pIWPiKOA0ABmnS3DWPMNkTsB4DvAcAYAIwDwO8YY+YX68PCwuLDg/cj6n/SGHODMWZb6/M3AOBpY8xGAHi69dnCwuLXAJci6j8AAHe2yo9CM6fe1y90QK0ewLGJplDQPSI9p6rHyeuu7kixsVQkEbC/l0wmKlYDjh4ifri+lBSBF3wSdT3GRad/+QKeAVZx+iXS1Gd5lpmlGtIkuO2Om6NyoSQ91TJpatudknz5WzZeG5UnTpKnVyk8LNrxYKSPfewOOf46mXNefZk8A+tKNAyYSXCuLING+sfIK6x3kEyO5arK/FslAc/NS3Uk3UOemCtafHAAAKcOSVE/kyX1JmakCWzmNKkcI32kLjz2w++Kdql+Mnvdvf3jou6Jn/48Kl+95eqo/NrUQdHuvQl6/rK9Mjhr/wGa/9tv/Yio++WLz0Xlz3yW1IXHfvBD0e7uez4flZ9/QfLlb7+DiDm2XEsBTQfeele027RlLCqvWyvXz7v7Wm11brBFsNQ3vgGAJxHxDUR8uPXdMmPM6ea5zGkAGF70aAsLiw8VlvrGv8MYcwoRhwHgKUTcf9EjWmj9UDwMADA6uvIirS0sLNqBJb3xjTGnWv+nAOAfoZke+wwijgIAtP5PLXLsI8aYbcaYbX39A+drYmFh0WZc9I2PiBkAcIwxhVb5NwHgfwaAxwHgIQD4s9b/xy7WV8MPYXaqlSZ7QLpW9veRvvsdRooAAHDVKJl8eteSTujUpY6cbpDuPhdKA0OcEUomWHnr1i2yHePLP3JYmgQbVdLrGwGVXUel2g7IHTmuXGULjMzj6quvF3WnpxjpYoX03X6mLwMA1BYo0nBBkZZ0Z+ja4ix68aobpGnonT3kfpxTUWC/8999LioXmbnz3bePi3ZX30QRkEcPvyPqME7zU2cuwR+78xOi3Y9/TEQcuZwkl8gvEOllNyOruPnjnxTtGnWag1defl3Ubbvxpqg8MEhuubt37RPt3np1V1T+3O8/KOq6GZnqUEq60Z5g+SHe2b87Kt+y7TbRLrdAZDJlRZ767DMvROXt2+6MynOn5d7LFCP2HByWUaWbrmo+SxMLB2ApWIqovwwA/rGVNMIDgL8zxvwMEV8HgO8j4pcB4DgAfP4CfVhYWHyIcNGFb4w5CgDXn+f7WQC4+9wjLCwsPuxoq+deIhWHsWuaYvuwDEyDV9/5ZVTesFaK3ytWUCRVwSfR86XnpVkEmBfbVRtlqqNEmkTufYzsYN9eaTJxGEfe9u23i7ply8mL7eB+Mv/MTcs0XDfeQSa2kwfHRd1ED22FDK+U/OoHxin6qruXxOiVK6SH4vg+isx6UalFvRma2BuZeP/mm2+Kdhu3bo3Kx09KT7U3d5HI2jtA6tSK5StEuzRL312tSxPsbTeS6aynn1SVl56VovjMFInAoxvldW7aQM/Be+Pk3XbggDTFpRgv4OCwjIYssSjQ6iSZSO+79zdEu5deeTsq12tSdVs3RvfpyB45jzsZmcqGG+j9uGL5OtHuFz/ZEZU/+hH5vnxrL/Em7jlI5cywVGVNlUyanjIFn332EZdmqLO++hYWHQi78C0sOhB24VtYdCDaquMbMNBoRXTlQqlHJaqkn6/fKPUjh7G0HHiOdL3uLrlR0DtCZpetSl90Y2TmMswUV1YsPuvXUySWzp33s5/+Iirnpskks/1muRcQMM/T+ZzMw7awQGbGoCHn4KpNpBeXZuk3OZ+fEe1OnCKzWtyVc3DttcQes2PHjqj86U9/WrQ7NE5uqLfddLOoW76VTKbPvUTXvHn9mGj35i7Sd6fnpfl0/gztZfT2U7Ti7Z/YLtodfov2WxYWJLPO089S1J3H8ikkFGvSli20F/Duu3LPZmyMxtybovv5/PM7RLsTZ8hUdujRb4m6j931saicjUndevU6cp09epj2HjJ9MjfkDTdfF5VfeeVFUbeW7Uclumm/yPiSrWjyEJn3CnnpmpvMNMdlPmCXXQsLi/8fwS58C4sORFtFfXQQEtmmSL/n4FFRlzYsNdaATMd8Zp7E5UqRxJ+RlbLd2AYyN03OSBPVe8fo87p1JM7fcI0kVswVSZx64WVJinj9VSSK76vT+Osqau3VV+m43IQkRSz71H9PRqbQmn2PxOMVI+Te/O4Rab66lnn8bd0kTZ/Tp+h8H2VkEPPzUuXIV2kcp+dkXaZMps/bb7sxKu98WY6je4REbgykiHmGkUFeu51UiVhcRuCtWkOmsk03bhR15RqpODxCMzRSRVrIUeSlNjlOHCfVcA/zsssOSHLQz32cSEb2vCFDUdauJ9Pn2jHJl1/I03UOj5Ap0fVkSrGBfhL9r90mvf88j1TUqVkyP3b3yuWZ6SOVNz8tn7npQjMK1Ffp3BaDfeNbWHQg7MK3sOhAtHdX3xjwWym0oCQDFQolEnMbDSmuFIsk/tSqtOtZ9yVRxu6dxJefSMlLc1nAypFjxPeXVjPw9gEKzLnrTulhhQ1SM7JdpDrcfa9s5ydI7N1XleKxlyXrxdCwpDBwQhKxiyxAZdM1m0S75x6jnev5CanSXHcD8bIXyiQCH1UZdz+6ndplMpJj3q8SWcbzL++Iym4ow6pHlpGYHgQy0Gf9JhLb3ztEalFXr9yRn58jcf7t12TglsmQOBtjonM8Lu97mu3Wp5OybvIUeViuZbv/x0+dEu1e+OWOqOxX5UPxC5YS7XMPfkrUPfhb/ywq//ynT0bl06flfRldT+duTEtLUsh48gIgVUhny928hdSTN+ZkRt9UK12ag1KVWgz2jW9h0YGwC9/CogNhF76FRQeirTp+uVyCXa83+cxfYXoTAEAP88y6jnk5AQDsfIV09+uu3RyVN10rCQcdl/Sbel3uIUzNkP5YrpBZJJmSOuEd24lMcW5KeszteZvSGRfmibyyobjME1nSY0dUtNiWIfIo7OuTJqX5adLJ+wZJ704qta1SIX3aeNKT7Je/pCjHez51T1Q+rvTiF1+kyMagIR+DVdfS+PsHaR8im5TzvXotRRD++Ml/EnWjy6ntR24hMozJWUnUVGWek0Eg30Mllgugp5tMt41QElSYBtsDUqQiDssP7jg0kTdfLyPNOelKYV7q4CfP0HOwMCcjMY+U6TlYKDGToyPNeTVmglzPnmEAgHd30nM1OrI8Kq8akvsh9QKbu4YcB/ituVOmzsVg3/gWFh0Iu/AtLDoQbRX1FxZy8E8/a6YZiqtU2OlB5qn2zm5Rd+rIeFQu9JI57Myc4sRjQS9crAMAOH6aTFSxOIlQo/3Si6pWJ5PdfUxUBgBYMXZfVN63h8bU19sj2mGczJF+SQavuP3kpXXisDTJ1AqkglSZB+GZWTBlMv4AAAw5SURBVMl1d/M15Am39aq1oo7ztz39NKlTw8p0ePPHKVimocyiq7bSnDz/8lNRecWIVE2eenpHVH7g0/eJugP76drWrSOzXyIj52rNZvKiPDYur/OjH78zKgcssEqniUKm4qWSUjyuVkjlc1x6do7sl955B4+RybE3LQOfhrvp8+gyadI8eZKewS3XUF6Eg0ekuXDqPfIgLPdLVaKnm5ntmMm4UFBmvypdC88NAQBQq52dA2vOs7CwWAR24VtYdCDswrew6EC0l2wzHof1q5o6UndVRnONrCczRqjqNmwgfSkzQHraxCkZ4VdibsCaYOOmmyjKbNUqMqkZI0kiX3yOzFw/feYXoi5gJAf1GdLF3lYRW2Ga9hr275Z572IpivSaOCbNUsk4maxOnSKd8MgBeZ2fupvcRA8fOSbqRpdRFNit22gv4MknnxTt9rxK5qCZeakzP7SZSDs2byHT01uv7RXtbriD3HLffUeSUG67hchJxjZQyuxyWe7tVGp0nzIZqVtzl9Uqy/2nXXZrTOev1uW1VGp0f1OMwGPVWkn2cic7d9yRy+LI/vGo/PZe6YJ9080UsZmO0Xv02EkZlXlqgu7n1YMysczACJluZ46Tq/bEe3J/KHDJJF1Vbu1+2NTtLRGHhYXForAL38KiA9FeIo7QgFtrishBTYoqJ09QFNXUqZyoOz5O4qyZIs+sdLf0WtvEuOJPnjwp6va+TSmeEglSF44dk2L0jTdT1Fo8IVNtv/g8mcpCIHEzcOU09vSRyW7ztdK8tPU64ldbpXKIFubp2kZGyfw28Z70dvvZz38SlYf7ZCqlm7dti8oLCyQa3nf//aJdOkm/+Tte2iXqaiGNI9tF5rdbbpOmw9ff2hGV77xTpqf2GTHHYXb/1q6SuQQ8h+Z4xQrp5cjNdly8r6qU3x6LvNSRnS4z4eXzxKsXi2miDDJV5nN5UdfbRyrYifFxUXf1NaTuBHU697IhGfHoOTTf61ZIPr5KgVQ+M0A5COZmpbrgJek6B4ZkWrVqrnlufr0XwpLe+IjYi4g/RMT9iLgPEW9HxH5EfAoRD7X+9128JwsLiw8Dlirq/28A8DNjzBZoptPaBwDfAICnjTEbAeDp1mcLC4tfAywlW243AHwcAP4AAMA0t8HriPgAANzZavYoAOwAgK9fqK9SsQiv/rKZ8unm6yXX3RDLZBrU5E77Jka3veYaEoERpLjGvbtSMbnzOzdJnntHD9FO+7Zbt4l20zO0k3pkt1QD1q4jQozZSbIgDIxKKu+FIgV1vLZTitF1hwJPJo/KXVus0+3wmdi4aZMk4sgmSPxOx6W689zzlFJr3QbyinPjcq5yBRbYAjKgyWHsJGGejuvulRyHv3nvvVE5k5WkEQcPjUflzepec/Cd9qFBqbbEE+SFFjI6dk17zq0tuq7CPPf486F3v3ldCDLQZWiExlUrSPHbYSqNl6FnbnhACsB9WZrTMyelx+ZWRqv+1gmyGqTTUmyfLdDzkjCSrzHmNedqiTwcS3rjrwOAaQD4vxFxFyL+dStd9jJjzGkAgNb/4Qt1YmFh8eHBUha+BwA3AcBfGWNuBIASvA+xHhEfRsSdiLizVq9d/AALC4vLjqUs/BMAcMIY82rr8w+h+UNwBhFHAQBa/6fOd7Ax5hFjzDZjzLZEPHG+JhYWFm3GRXV8Y8wkIk4g4mZjzAEAuBsA3m39PQQAf9b6/9jF+urp6ob7725GvA2vUUSTDulHL+94TdQVGCHGZIl0oHpFKjTczKP14luZmWt4GXlO7Xl7j2jHyRTuuOOjom7yJI2jr5/G29Mr9ezhPjJ7Ze+U0X9jm8iUU16tSEXnSCJ69WUiH9m3V0aSfeRWGtcvfiG9CzduJVLHw0dpj4LrsAAAGzfTGD/1gIxCTHaR3lor0buhAbKPLpYXYH5Kmk97s+QJ5zCu96Iv9xO4WY2b2wAAli8j3ZrfW26+AwBwmWJbKEhvSGR1MXZcoFK4cTNgLCH3Q7yQdO1EKOdg+gxx9Q+uomd6aEB6550+za9beRdW6JmrVWkPCNV8Lx+h/qtn5PjRa75UvSWa85Zqx/9jAPgOIsYB4CgA/CE0pYXvI+KXAeA4AHx+iX1ZWFhcYSxp4RtjdgPAtvNU3X2e7ywsLD7kaKvnXrVWg4MtbrNiIINofJ/x5eWkuDbIPOGGlpOZZOKM3FZYYAEgvtpPSDKChmeeIw+8DSoD7NVbiR/u1TdkCq3qHE2Xx8TBA/sPiHbTJykgY/z4e6LulsJtUblwRvKmzeeYCOjS+G/dLjPM1gMyd97zGZkFt9agORjbQGbQM2quBobJNLdilRRL/QaJm9UqnWuoRz4uZSZWl0ryWnp6qE8PSV3w4tLc1psl0+SZ6VlRVyjQONJZOq5Rk2JujfErusqeJUx4fOwqiMtlnnWIcusrnqZ7HbhSPXv1xZej8mcefIDOlZDjSKVJNazk5fN9coK8Vn3GGZhNS5P0AJv/vUekqbkw37xPvr+0DXTrq29h0YGwC9/CogNhF76FRQeirTp+MpWEzdc0I+iGR5ReyTjVdzNTFgDAsjUUxpYrEFHB8HJJgLF8PZmoMsq91GUumrez9NGTjPACAOAlFoF3822S33/vTtLF7vkM5VAbWC6vxYxRBNqaMRmNtnkruWfmTktd78wU6Z2zs1QX+NKFOZEgfdeLKdNWksyHZUZMMrp8uWhnXNJ9Z6bPiLruHupjdITmeDYn88Ft2krznV+Qrqz1Oo2Za7uOI981mQydK5yU+xA8D1ypQKY+BLl/wyPSdNQdN9M1mAlPt+MuvH5V6vGex0x9yvWZm+3iCdLJXU9eZw8zbx7PS/KUBpuhF9mewehy6cI8NED3empG7ocEtea4tJlyMdg3voVFB8IufAuLDgQulaPrAzkZ4jQAvAcAgwAwc5HmlxsfhjEA2HFo2HFIvN9xrDHGDF2sUVsXfnRSxJ3GmPM5BHXUGOw47Diu1DisqG9h0YGwC9/CogNxpRb+I1fovBwfhjEA2HFo2HFIXJZxXBEd38LC4srCivoWFh2Iti58RLwXEQ8g4mFEbBsrLyL+LSJOIeI77Lu204Mj4ipEfLZFUb4XEb96JcaCiElEfA0R97TG8R9b369FxFdb4/hei3/hsgMR3Raf4xNXahyIOI6IbyPibkTc2fruSjwjbaGyb9vCR0QXAP4PAPg0AFwFAF9CxKsufNQHhm8BwL3quytBD94AgK8ZY7YCwHYA+EprDto9lhoA3GWMuR4AbgCAexFxOwD8OQD8RWsc8wDw5cs8jrP4KjQp28/iSo3jk8aYG5j57Eo8I+2hsjfGtOUPAG4HgJ+zz98EgG+28fxjAPAO+3wAAEZb5VEAONCusbAxPAYA91zJsQBAGgDeBIDboOko4p3vfl3G869sPcx3AcAT0HTtvxLjGAeAQfVdW+8LAHQDwDFo7b1dznG0U9RfAQA8IuZE67srhStKD46IYwBwIwC8eiXG0hKvd0OTJPUpADgCADljzNmIlHbdn78EgH8HEJHZD1yhcRgAeBIR30DEh1vftfu+tI3Kvp0L/3xU/x1pUkDELAD8AwD8iTEmf7H2lwPGmMAYcwM037i3AsDW8zW7nGNAxPsBYMoYw8Mxr9Rzcocx5iZoqqJfQcSPX+yAy4BLorJ/P2jnwj8BADzlzEoAOLVI23ZgSfTgHzQQMQbNRf8dY8yPruRYAACMMTloZkHaDgC9iHg29rMd9+cOAPgsIo4DwHehKe7/5RUYBxhjTrX+TwHAP0Lzx7Dd9+WSqOzfD9q58F8HgI2tHds4AHwRAB5v4/k1HocmLTjAEunBLxXY5Hr+GwDYZ4z5L1dqLIg4hIi9rXIKAH4DmptIzwLAg+0ahzHmm8aYlcaYMWg+D88YY/55u8eBiBlE7DpbBoDfBIB3oM33xRgzCQATiLi59dVZKvsPfhyXe9NEbVLcBwAHoalP/vs2nvfvAeA0NAnNT0Bzl3gAmptKh1r/+9swjo9CU2x9CwB2t/7ua/dYAOA6ANjVGsc7APA/tb5fBwCvAcBhAPgBACTaeI/uBIAnrsQ4Wufb0/rbe/bZvELPyA0AsLN1b/5fAOi7HOOwnnsWFh0I67lnYdGBsAvfwqIDYRe+hUUHwi58C4sOhF34FhYdCLvwLSw6EHbhW1h0IOzCt7DoQPx/K2vCj8+kLXYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(x_train[15])\n",
    "print(\"y= \"+ str(y_train[:,15]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explore Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples: m_train = 483\n",
      "Number of testing examples: m_test = 121\n",
      "Height/Width of each image: num_px = 64\n",
      "Each image is of size: (64, 64, 3)\n",
      "train_set_x shape: (483, 64, 64, 3)\n",
      "train_set_y shape: (1, 483)\n",
      "test_set_x shape: (121, 64, 64, 3)\n",
      "test_set_y shape: (1, 121)\n"
     ]
    }
   ],
   "source": [
    "m_train = x_train.shape[0]\n",
    "m_test = x_test.shape[0]\n",
    "num_px = x_train.shape[1]\n",
    "\n",
    "print (\"Number of training examples: m_train = \" + str(m_train))\n",
    "print (\"Number of testing examples: m_test = \" + str(m_test))\n",
    "print (\"Height/Width of each image: num_px = \" + str(num_px))\n",
    "print (\"Each image is of size: (\" + str(num_px) + \", \" + str(num_px) + \", 3)\")\n",
    "print (\"train_set_x shape: \" + str(x_train.shape))\n",
    "print (\"train_set_y shape: \" + str(y_train.shape))\n",
    "print (\"test_set_x shape: \" + str(x_test.shape))\n",
    "print (\"test_set_y shape: \" + str(y_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reshaping Training and Test examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_set_x_flatten shape: (12288, 483)\n",
      "train_set_y shape: (1, 483)\n",
      "test_set_x_flatten shape: (12288, 121)\n",
      "test_set_y shape: (1, 121)\n",
      "sanity check after reshaping: [45 47 47 58 56]\n"
     ]
    }
   ],
   "source": [
    "train_set_x_flatten = x_train.reshape(x_train.shape[0], -1).T\n",
    "test_set_x_flatten = x_test.reshape(x_test.shape[0], -1).T\n",
    "\n",
    "print (\"train_set_x_flatten shape: \" + str(train_set_x_flatten.shape))\n",
    "print (\"train_set_y shape: \" + str(y_train.shape))\n",
    "print (\"test_set_x_flatten shape: \" + str(test_set_x_flatten.shape))\n",
    "print (\"test_set_y shape: \" + str(y_test.shape))\n",
    "print (\"sanity check after reshaping: \" + str(train_set_x_flatten[0:5,0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Standardizing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = train_set_x_flatten/255.\n",
    "test_x = test_set_x_flatten/255."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(Z):\n",
    "    A = 1/(1+np.exp(-Z))\n",
    "    cache = Z\n",
    "    \n",
    "    return A,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(Z):\n",
    "    A = np.maximum(0,Z)\n",
    "    \n",
    "    assert(A.shape == Z.shape)\n",
    "    cache = Z\n",
    "    return A,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid_backward(dA, cache):\n",
    "    Z = cache\n",
    "    s = 1/(1+np.exp(-Z))\n",
    "    dZ = dA * s * (1-s)\n",
    "    \n",
    "    assert(dZ.shape == Z.shape)\n",
    "    return dZ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu_backward(dA, cache):\n",
    "    Z = cache\n",
    "    dZ = np.array(dA, copy=True)\n",
    "    \n",
    "    dZ[Z <= 0] = 0\n",
    "    #dZ[Z[:,:4]<=0] = 0\n",
    "    assert(dZ.shape == Z.shape)\n",
    "    return dZ"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_parameters_deep(layer_dims):\n",
    "    np.random.seed(1)\n",
    "    parameters = {}\n",
    "    L = len(layer_dims)\n",
    "    \n",
    "    for l in range(1,L):\n",
    "        parameters[\"W\" + str(l)] = np.random.randn(layer_dims[l],\n",
    "                                                   layer_dims[l-1])*np.sqrt(2/layer_dims[l-1])\n",
    "        parameters[\"b\" + str(l)] = np.zeros((layer_dims[l],1))*0.01\n",
    "        \n",
    "        assert(parameters['W' + str(l)].shape == (layer_dims[l],\n",
    "                                                  layer_dims[l-1]))\n",
    "        assert(parameters['b' + str(l)].shape == (layer_dims[l], 1))\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Forward Prop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_forward(A, W, b):\n",
    "    Z = np.dot(W,A)+b\n",
    "    assert(Z.shape == (W.shape[0], A.shape[1]))\n",
    "    cache = (A, W, b)\n",
    "    \n",
    "    return Z, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_activation_forward(A_prev, W, b, activation,keep_prob):\n",
    "    if activation == \"sigmoid\":\n",
    "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\"\n",
    "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
    "        A, activation_cache = sigmoid(Z)\n",
    "        if keep_prob < 1:\n",
    "            D = np.random.rand(A.shape[0],A.shape[1])\n",
    "            D = (D < keep_prob).astype(int)\n",
    "            A = A * D\n",
    "            A = A / keep_prob\n",
    "            #cache = (D)\n",
    "    \n",
    "    elif activation == \"relu\":\n",
    "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\"\n",
    "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
    "        A, activation_cache = relu(Z)\n",
    "        if keep_prob < 1:\n",
    "            D = np.random.rand(A.shape[0],A.shape[1])\n",
    "            D = (D < keep_prob).astype(int)\n",
    "            A = A * D\n",
    "            A = A / keep_prob\n",
    "            #cache = (D)\n",
    "    \n",
    "    assert (A.shape == (W.shape[0], A_prev.shape[1]))\n",
    "    if keep_prob < 1:\n",
    "        cache = (linear_cache, activation_cache, D)\n",
    "    else:\n",
    "        cache = (linear_cache, activation_cache)\n",
    "\n",
    "    return A, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def L_model_forward(X, parameters, keep_prob):\n",
    "    caches = []\n",
    "    A = X\n",
    "    L = len(parameters) // 2    # number of layers in the neural network\n",
    "    \n",
    "    # Implement [LINEAR -> RELU]*(L-1). Add \"cache\" to the \"caches\" list.\n",
    "    for l in range(1, L):\n",
    "        A_prev = A \n",
    "        A, cache = linear_activation_forward(A_prev, parameters['W'+ str(l)],\n",
    "                                             parameters['b'+ str(l)],\n",
    "                                             activation = \"relu\",\n",
    "                                             keep_prob = 0.8)\n",
    "        caches.append(cache)\n",
    "    \n",
    "    # Implement LINEAR -> SIGMOID. Add \"cache\" to the \"caches\" list.\n",
    "    AL, cache = linear_activation_forward(A, parameters['W' + str(L)],\n",
    "                                          parameters['b' + str(L)],\n",
    "                                          activation= \"sigmoid\",keep_prob = 1)\n",
    "    caches.append(cache)\n",
    "    \n",
    "    assert(AL.shape == (1,X.shape[1]))\n",
    "            \n",
    "    return AL, caches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cost Function\n",
    "\n",
    "$$-\\frac{1}{m} \\sum\\limits_{i = 1}^{m} (y^{(i)}\\log\\left(a^{[L] (i)}\\right) + (1-y^{(i)})\\log\\left(1- a^{[L](i)}\\right)) \\tag{7}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cost(AL, Y):\n",
    "    m = Y.shape[1]\n",
    "\n",
    "    cost = -(1/m) * np.sum(np.multiply(np.log(AL),Y) + \n",
    "                           np.multiply(np.log(1-AL),1-Y))\n",
    "    \n",
    "    cost = np.squeeze(cost)\n",
    "    assert(cost.shape == ())\n",
    "    \n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Backward Prop\n",
    "\n",
    "#### Linear back prop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_backward(dZ, cache):\n",
    "    A_prev, W, b = cache\n",
    "    m = A_prev.shape[1]\n",
    "\n",
    "    dW = (1/m) * np.dot(dZ,A_prev.T)\n",
    "    db = (1/m) * np.sum(dZ, axis=1, keepdims=True)\n",
    "    dA_prev = np.dot(W.T,dZ)\n",
    "    \n",
    "    assert (dA_prev.shape == A_prev.shape)\n",
    "    assert (dW.shape == W.shape)\n",
    "    assert (db.shape == b.shape)\n",
    "    \n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_activation_backward(dA, cache, activation, keep_prob):\n",
    "    if keep_prob<1:\n",
    "        linear_cache, activation_cache, D = cache\n",
    "    elif keep_prob == 1:\n",
    "        linear_cache, activation_cache = cache\n",
    "    if activation == \"relu\":\n",
    "        dZ = relu_backward(dA,activation_cache)\n",
    "        dA_prev, dW, db = linear_backward(dZ,linear_cache)\n",
    "        if keep_prob < 1:\n",
    "            D = D.reshape((D.shape[1], D.shape[0])) #jugaad\n",
    "            #dA_prev = dA_prev * D\n",
    "            dA_prev = np.dot(dA_prev,D)\n",
    "            dA_prev = dA_prev / keep_prob\n",
    "        \n",
    "    elif activation == \"sigmoid\":\n",
    "        dZ = sigmoid_backward(dA,activation_cache)\n",
    "        dA_prev, dW, db = linear_backward(dZ,linear_cache)\n",
    "        if keep_prob < 1:\n",
    "            D = D.reshape((D.shape[1], D.shape[0]))\n",
    "            #dA_prev = dA_prev * D\n",
    "            dA_prev = np.dot(dA_prev,D)\n",
    "            dA_prev = dA_prev / keep_prob\n",
    "    \n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_parameters(parameters, grads, learning_rate):\n",
    "    L = len(parameters) // 2 # number of layers in the neural network\n",
    "\n",
    "    for l in range(L):\n",
    "        parameters[\"W\" + str(l+1)] = parameters[\"W\"+ str(l+1)] - learning_rate*grads[\"dW\" + str(l+1)]\n",
    "        parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate*grads[\"db\" + str(l+1)]\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(X, y, parameters):\n",
    "    m = X.shape[1]\n",
    "    n = len(parameters) // 2 # number of layers in the neural network\n",
    "    p = np.zeros((1,m))\n",
    "    \n",
    "    # Forward propagation\n",
    "    probas, caches = L_model_forward(X, parameters, keep_prob=1)\n",
    "    \n",
    "    # convert probas to 0/1 predictions\n",
    "    for i in range(0, probas.shape[1]):\n",
    "        if probas[0,i] > 0.5:\n",
    "            p[0,i] = 1\n",
    "        else:\n",
    "            p[0,i] = 0\n",
    "    \n",
    "    #print results\n",
    "    #print (\"predictions: \" + str(p))\n",
    "    #print (\"true labels: \" + str(y))\n",
    "    print(\"Accuracy: \"  + str(np.sum((p == y)/m)))\n",
    "        \n",
    "    return p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep NN\n",
    "\n",
    "### L-Model Backward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def L_model_backward(AL, Y, caches, keep_prob):\n",
    "    grads = {}\n",
    "    L = len(caches) # the number of layers\n",
    "    m = AL.shape[1]\n",
    "    Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL\n",
    "    \n",
    "    # Initializing the backpropagation\n",
    "    dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n",
    "    # Lth layer (SIGMOID -> LINEAR) gradients.\n",
    "    current_cache = caches[L-1]\n",
    "    grads[\"dA\" + str(L-1)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] = linear_activation_backward(dAL, current_cache, activation = \"sigmoid\",keep_prob=1)\n",
    "\n",
    "    \n",
    "    # Loop from l=L-2 to l=0\n",
    "    for l in reversed(range(L-1)):\n",
    "        # lth layer: (RELU -> LINEAR) gradients.\n",
    "        current_cache = caches[l]\n",
    "        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(grads[\"dA\" + str(l + 1)],\n",
    "                                   current_cache, activation = \"relu\",keep_prob=0.8)\n",
    "        grads[\"dA\" + str(l)] = dA_prev_temp\n",
    "        grads[\"dW\" + str(l + 1)] = dW_temp\n",
    "        grads[\"db\" + str(l + 1)] = db_temp\n",
    "\n",
    "    return grads"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L-Layer NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def L_layer_model(X, Y, layers_dims, learning_rate = 0.002, num_iterations = 3000, print_cost=False,keep_prob = 1):#lr was 0.009\n",
    "    np.random.seed(1)\n",
    "    costs = []                  # keep track of cost\n",
    "    \n",
    "    # Parameters initialization\n",
    "    parameters = initialize_parameters_deep(layers_dims)\n",
    "    \n",
    "    # Loop (gradient descent)\n",
    "    for i in range(0, num_iterations):\n",
    "\n",
    "        # Forward propagation: [LINEAR -> RELU]*(L-1) -> LINEAR -> SIGMOID\n",
    "        AL, caches = L_model_forward(X, parameters, keep_prob)\n",
    "        # Compute cost\n",
    "        cost = compute_cost(AL, Y)\n",
    "    \n",
    "        # Backward propagation\n",
    "        grads = L_model_backward(AL, Y, caches, keep_prob)\n",
    " \n",
    "        # Update parameters\n",
    "        parameters = update_parameters(parameters, grads, learning_rate)\n",
    "                \n",
    "        if print_cost and i % 1000 == 0:\n",
    "            print (\"Cost after iteration %i: %f\" %(i, cost))\n",
    "        if print_cost and i % 1000 == 0:\n",
    "            costs.append(cost)\n",
    "            \n",
    "    # plot the cost\n",
    "    plt.plot(np.squeeze(costs))\n",
    "    plt.ylabel('cost')\n",
    "    plt.xlabel('iterations (per hundreds)')\n",
    "    plt.title(\"Learning rate =\" + str(learning_rate))\n",
    "    plt.show()\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "boolean index did not match indexed array along dimension 1; dimension is 4 but corresponding boolean dimension is 483",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-81-4d223c7f1747>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mlayer_dims\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m12288\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;31m# 4-layer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mparameters\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mL_layer_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_x\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlayer_dims\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnum_iterations\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m15000\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mprint_cost\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mkeep_prob\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-53-b9a0e7207b5d>\u001b[0m in \u001b[0;36mL_layer_model\u001b[1;34m(X, Y, layers_dims, learning_rate, num_iterations, print_cost, keep_prob)\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[1;31m# Backward propagation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m         \u001b[0mgrads\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mL_model_backward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mAL\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcaches\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkeep_prob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m         \u001b[1;31m# Update parameters\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-52-2730b4cf36ad>\u001b[0m in \u001b[0;36mL_model_backward\u001b[1;34m(AL, Y, caches, keep_prob)\u001b[0m\n\u001b[0;32m     17\u001b[0m         \u001b[0mcurrent_cache\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcaches\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m         dA_prev_temp, dW_temp, db_temp = linear_activation_backward(grads[\"dA\" + str(l + 1)],\n\u001b[1;32m---> 19\u001b[1;33m                                    current_cache, activation = \"relu\",keep_prob=0.8)\n\u001b[0m\u001b[0;32m     20\u001b[0m         \u001b[0mgrads\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"dA\"\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdA_prev_temp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m         \u001b[0mgrads\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"dW\"\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ml\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdW_temp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-49-e9a6c7ab6b4e>\u001b[0m in \u001b[0;36mlinear_activation_backward\u001b[1;34m(dA, cache, activation, keep_prob)\u001b[0m\n\u001b[0;32m      5\u001b[0m         \u001b[0mlinear_cache\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mactivation_cache\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mactivation\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relu\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m         \u001b[0mdZ\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrelu_backward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdA\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mactivation_cache\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m         \u001b[0mdA_prev\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdW\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlinear_backward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdZ\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlinear_cache\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mkeep_prob\u001b[0m \u001b[1;33m<\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-64-b224dade6a83>\u001b[0m in \u001b[0;36mrelu_backward\u001b[1;34m(dA, cache)\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0mdZ\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdA\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0mdZ\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mZ\u001b[0m \u001b[1;33m<=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[1;31m#dZ[Z[:,:4]<=0] = 0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[1;32massert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdZ\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mZ\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: boolean index did not match indexed array along dimension 1; dimension is 4 but corresponding boolean dimension is 483"
     ]
    }
   ],
   "source": [
    "layer_dims = [12288,20,4,1] # 4-layer\n",
    "parameters = L_layer_model(train_x,y_train,layer_dims,num_iterations=15000,print_cost=True,keep_prob=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9875776397515525\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict(train_x, y_train, parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.768595041322314\n"
     ]
    }
   ],
   "source": [
    "pred_test = predict(test_x, y_test, parameters)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}