1_multilayer_ann_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Multilayer Neural Network to classify hand written digits in MNIST dataset\n",
    "### Training Data: 60,000 digits\n",
    "### Test Data: 10,000 digits\n",
    "### Each row: 784 pixels (28x28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import struct\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist(path, kind='train'):\n",
    "    \"\"\"Load MNIST data from path\"\"\"\n",
    "    labels_path=os.path.join(path, '%s-labels.idx1-ubyte' % kind)\n",
    "    images_path=os.path.join(path, '%s-images.idx3-ubyte' % kind)\n",
    "    \n",
    "    with open(labels_path, 'rb') as lbpath:\n",
    "        magic, n=struct.unpack('>II', lbpath.read(8))\n",
    "        labels=np.fromfile(lbpath, dtype=np.uint8)\n",
    "    \n",
    "    with open(images_path, 'rb') as imgpath:\n",
    "        magic, num, rows, cols=struct.unpack('>IIII', imgpath.read(16))\n",
    "        images=np.fromfile(imgpath, dtype=np.uint8).reshape(len(labels), 784)\n",
    "        images=((images/255.0)-0.5)*2 #normalized to -1 to 1 range\n",
    "    \n",
    "    return images, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows: 60000, columns: 784\n"
     ]
    }
   ],
   "source": [
    "X_train, y_train=load_mnist('data/',kind='train')\n",
    "print('Rows: %d, columns: %d' % (X_train.shape[0], X_train.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows: 10000, columns: 784\n"
     ]
    }
   ],
   "source": [
    "X_test, y_test=load_mnist('data/', kind='t10k')\n",
    "print('Rows: %d, columns: %d' % (X_test.shape[0], X_test.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAHiFJREFUeJzt3XmcndP9wPFPaol9D9WiscTWWqO28qMIIUrtUcReYt+ptrZYqwhiX0PUWpRWa19KEQa1i6X2fYt9CfP7w+v7POfO3MRMcufeM3c+73/y9DzP3DlOnzvf55znnO/p1draiiRJuflBoysgSVI1BihJUpYMUJKkLBmgJElZMkBJkrJkgJIkZckAJUnKkgFKkpQlA5QkKUuTd+bi2WabrbVv375dVJXuo6Wl5d3W1tY+k/o5tud3bM/aq0Wb2p4l79Ha6mh7dipA9e3blwcffHDia9UkevXq9VItPsf2/I7tWXu1aFPbs+Q9WlsdbU+H+CRJWTJASZKyZICSJGXJACVJypIBSpKUJQOUJClLBihJUpYMUJKkLBmgJElZMkBJkrJkgJIkZckAJUnKkgFKkpQlA5QkKUsGKElSlgxQkqQsGaAkSVkyQEmSsmSAkiRlyQAlScqSAUqSlCUDlCQpSwYoSVKWDFCSpCwZoCRJWTJATYT+/fv3b3QdmontKfUsHf3OG6AkSVkyQEmSsmSAkiRlyQAlScqSAUqSlCUDlCQpSwYoSVKWDFCSpCwZoCRJWZq80RWYFK+88goAJ598clF20kknAbD33nsDsOeeexbn5p577jrWTpI0KexBSZKy1O16UK+99lpxvNRSSwHw4YcfFmW9evUCYPjw4QCMHDmyOPfOO+/Uo4pN6ZxzzgFg5513Lsq+/fZbAJ555pmibMEFF6xvxTL25ZdfAvD1118XZXfffTdQ3sdbb711cW7yybvd17Em3n333eJ43LhxAIwePRqA9ddfvzj3gx907nl62223BeCss84qyiabbLKJrqfgqaeeAmCNNdYoyh555BEA+vTpU/PfZw9KkpSlbvPI9tJLLwGw6qqrFmUffPABUPaaAGaccUYAevfuDcDbb79dnHvhhRcA+MlPflKU+UQ1YbfeeisA++yzD1D9KTZt/54qevEnnHBCUXbbbbcBcP/994/359IRgUMOOaSLapeXN998E4CLLroIgLPPPrs4F73yl19+Gai83zp7n1144YUAzDzzzEXZkUceCZR/H3L17LPPAuXfuGWXXbaR1SnEvbz66qvX5ffZg5IkZckAJUnKUpZDfOlL5RjaGzhwIFBOLR+fJZdcEoCjjjoKgJVWWqk4169fP6BySGH77bevQY2b15gxYwD44osvGlyTfKSTbWKJQ/z7+eefF+daW1sBmHfeeYuyWWedFYCWlhag8gX+0KFDga552ZyTgw46CIBRo0bV5ffF0hMoJ/nMP//8dfndEyuG1p9++mmgsUN8cR9DOfQYfxe6mj0oSVKWsuxB7b///sXxiBEjOvWzd955JwCffvopABtssEFx7uqrrwbg4YcfntQqNrUnn3yyOD7ssMMqzi299NLF8U033QTAtNNOW5d6NUr0HuMF+xlnnFGcGzt27Hh/brHFFgPKexLKadRzzDEHAG+99Va7z2r2HtSvfvUroHoP6kc/+hEA++23H1BOmoDqE3T+/e9/A3DNNdfUvJ6NdMoppwCw5pprNrgm8MknnxTHxxxzDFCZAKEr71d7UJKkLBmgJElZymqILyZApF3/9AUdVA7ZbbTRRgBsueWWRVnk21tkkUUAOPDAA4tzV111VdXP1Heee+45ANZZZ52i7P3336+45thjjy2OY81Zs7vnnnuAyv/28Vl00UWL47vuuguAGWaYoSh77733aly77ie+w23vLSiH8aabbroOfdZOO+0ElN/3WD+V2m677YrjdA1kzr755ptGV6GQZo8J0d5dzR6UJClLWfSgYjX9hHLrbbHFFkCZEw7Kl/lp2eDBgwGYZpppgPKlK5RPZxdffHFRFlNezXQO5557LlB9Kv+GG24IwC9/+cu61ikHkZGgmsg9uNpqqwHl8gao7DmFWDbRk8X3sFr7dNZDDz0EVObza2ueeeYpjnPOd/j6668Xx2mGkUar1tMdMGBAXX63PShJUpYa9jiRPvEcd9xxQJl3KqbgQrnIMRYxTjnllMW5WJQb/3bUZ599Vhwff/zxQDmts6ep1hbpdN5YWDps2LD6Viwjp59+OgArrLACUC4ah/Je7ehU+zQ3pCZOZISHcoF0eh+3lS5byVks24AJ//fUSyzVeeyxx9qdi78LXc0elCQpSwYoSVKW6j7EFyvpY6U4lNPKY9ryjTfeWJxbYIEFgMr8fLX0v//9r0s+N3cxESXdEK6ayCSx8MILd3WVsjX99NMDsMsuu0zyZ8UWHOqYmKoPsO+++wLwxBNPFGVfffXVeH925ZVXBjq/0WGjPP744+3KOvv6opZ+//vfA5WTNxZffHGg8lVLV+oe/89JknqcuvegYiFdtTxc9913H1B92/Cpp566ayvWw0QOs//85z/tzm2yySbF8TbbbFOvKnVrsQj8o48+KspiQXi60V5kMQ+DBg0qjuebb76urGI2ovd+xRVXAHDDDTeM99rrr7++OJ7QhoUzzTQTUG6CCOVOBlNMMcXEV7bBlltuuS79/C+//BKovC9jt4fLL7+83fUxmWyqqabq0noFe1CSpCzVvQe16667ApXphiL1SbWeUy1FZuR0TLqnpT164IEHANh6663bnYss0+nC53o9KXUH8R40HZOPbdqrjQhUu99CLAy/4IILirLu8q5kYrzxxhvF8aqrrgrA888/X7PPj3s3TdPVDNKkBRMS92Tcc2kG/XjPHu/rTj311OJcpFRKl0lEBvX47qfv/+uV4ig07zdCktStGaAkSVmqyxBfukFgTBtNX3imL+W7UgyhpL97mWWWqcvvbqR0mGD55Zcf73Uxpb/ZNyDsiDSb9KuvvgqUQ1NprsLI+RhDdmuvvXZx7tJLLwUqN3wLsdziH//4R1H2m9/8BoDJJptskuufsxhW78jw+vdtWBhickS6kV4jp2hPjLiXoPwbtd566wGw0EILTfBn7733XqBs0zTnYGSGjwkX6RKfmIqftlV8/+OejowSUP/NNO1BSZKyVJceVGyZDeW0xjTLeDrVtlbiCbVajr2NN964OD744INr/rtzc8IJJxTHE3oKTffO6qmi5/TII48UZW2n+kZuPoDVV18dgPnnnx+Azz//vDj36KOPAnD//fe3+z1vvvkmANtuu21RFtPM09+Xc/btzphzzjmL45ioc+WVVwKV25p3ZAHoeeedVxwfeuihtapiwx1xxBHFcdxPd9xxR4d+tl+/fkDZC4/RECjzmXZUTPuPe7SRi/TtQUmSsmSAkiRlqWHjB+n6mo5u79wRMbR3xhlnAHDAAQcU5/r27QuUOaagfjmlGiE2PYssB9WkQ0z1fgGai3RCRGzfkN43IYZPhgwZUpTFfRzbI6y77rrFuciM0rt376IstjSJIcR0HdQqq6wCwKabblqUxTqrat+Rueaa63v+y/IUOTd32GGHifr5yMkHzTXEl4p1itXWK3a1v//97xX/e7vttqt7HYI9KElSlhrWg9pqq61q9lnp9six+WG8yE57CGmGhJ4gptBX2w57rbXWAmDEiBF1rVNOYgrz8OHDi7KYKBIZzKHc8j3aLO39xxbuO+64I1CZfXuxxRYD4LLLLivK4oVzTBbafffdi3Pnn38+ACNHjizKIl9dSPP1jRkz5vv+E5tSbPOu+thwww0b9rvtQUmSslSXHlS6IC+O46kU4I9//ONEfW4shEyfQmPb+D322AOAk046aaI+uxnE9uLVppZHT6GZ38F9nxhrT6fXx7ueNIt2//79AXjmmWcAOPPMM4tzkYMvppenPdJ4ZzXDDDO0+93xXir214GyJ7fRRhsVZW17/d3hfk7f6cV24T/96U+LsonNLn7zzTcD9VvYr8azByVJypIBSpKUpboM8aW57+I48ptBuYJ6++23BypfUMf2zmeddRZQbrQH8OKLLwLlqmuAwYMHA+UQX08UubbSPGZtpUNLPVW1LdxjmUK6FGHs2LFA9S25QyxriHsYJn77jMiP1vY4d88++ywAhx12WFEWm969//77RVlHhvhiyHT06NFFWXy3q+U2jDx2bg9TO/E6JiYCQf031bQHJUnKUsOmmacvUqMHFTm2ZpllluJcvGStJjJHDxw4sCjbbbfdalrP7iKdah8Lc+MJPl0oGgsbzVheLtyOnGNQ5o2855572l2/5ZZbAjBgwICiLO7B2HK8mTcd/D7bbLMNUD33YDq5o9qkkbZikkq68V61Ld9jCnQs3m1k3rhmE+09oZGYrtZzv02SpKwZoCRJWarLEF+6BmKNNdYA4JZbbml3XUycSIerwuyzzw7A0KFDi7KJXT/VjNIXx23bL4aywC01UrfeeitQbvYG5dBeuj3EZpttBpQv4Jt9Q8GuMGzYsEn+jNiiJ81Cc/jhhwPNsy1Jjm677bbiOLaXqRd7UJKkLNXlsSN9KRov8GOLZpjwlPAjjzwSKHOdzTrrrF1RRfVAMXkktnJve6zOiSnl6SahJ554Yqc+Y9FFFwXKvxnpZobxNyDt3arrpBmAGsUelCQpS3UfuI1cZ+kiyWoLJtU5P/7xj4vjQYMGAZX55KSuFvtTHX300UXZ//3f/wGVez9Fdv3YZ2i99dYrzkUPtpZ7xKlzIhdkmnOyUexBSZKyZICSJGXJuZlNIh0SufbaaxtYE/V06ZTvddddF6jM1qG8xVTyRmaQCPagJElZMkBJkrJkgJIkZckAJUnKkgFKkpQlA5QkKUsGqInQ0tLS0ug6NBPbU+pZOvqdN0BJkrJkgJIkZckAJUnKkgFKkpQlA5QkKUsGKElSlgxQkqQsGaAkSVkyQEmSsmSAkiRlyQAlScqSAUqSlCUDlCQpSwYoSVKWDFCSpCwZoCRJWTJASZKyZICSJGXJACVJypIBSpKUJQOUJClLBihJUpYMUJKkLBmgJElZMkBJkrJkgJIkZalXa2trxy/u1esd4KWuq0638ZPW1tY+k/ohtmfB9qy9SW5T27OC92htdag9OxWgJEmqF4f4JElZMkBJkrJkgJIkZckAJUnKkgFKkpQlA5QkKUsGKElSlgxQkqQsGaAkSVkyQEmSsmSAkiRlyQAlScqSAUqSlCUDlCQpSwYoSVKWDFCSpCxN3pmLZ5tttta+fft2UVW6j5aWlndrsbum7fkd27P2atGmtmfJe7S2OtqenQpQffv25cEHH5z4WjWJXr161WTLZtvzO7Zn7dWiTW3PkvdobXW0PR3ikyRlyQAlScqSAUqSlCUDlCQpSwYoSVKWDFCSpCwZoCRJWTJASZKyZICSJGXJACVJypIBSpKUJQOUJClLBihJUpYMUJKkLBmgJElZMkBJkrJkgJIkZckAJUnKkgFKkpQlA5QkKUsGKElSlgxQkqQsGaAkSVkyQEmSsmSAmgj9+/fv3+g6NBPbU+pZOvqdN0BJkrJkgJIkZckAJUnKkgFKkpSlyRtdAeXn3XffLY5/8YtfADBu3DgAnn/++YbUSVLPYw9KkpQle1AqHH744QCceeaZRdk777wDwJAhQxpSJ0k9lz0oSVKWDFCSpCw5xNdDffrppwBssskmRdmNN94IQK9evYqy5ZZbDoDTTjutjrWTJHtQkqRMZd+D+vbbbwH48ssvx3vNyJEji+PoGTz55JMADB8+vDh38MEHAzBixIiibOqppwbghBNOAGDo0KG1qHa2Ygr5fvvtB8BNN93U7poLLrigOP75z38OlO0k5eyrr74qjgcOHAhULo3473//C8BMM81U34ppotiDkiRlqWE9qLFjxxbH33zzDVA+3aRP9R9++CEAZ599dqc+v2/fvgDsu+++Rdl5550HwIwzzliUrbzyygCsttpqnfr87uqjjz4CYNSoUeO9JtoOYOGFF+7qKkkd8vHHH1f8m5p22mkBaGlpKcruuOMOAJZYYomizJGA7sUelCQpSwYoSVKW6j7E9+qrrwKw5JJLFmUffPBBzT7/Bz/4LubGcF7apd9+++0BmH322Yuy6aabDoA+ffrUrA65SXPrrb322gC0tra2u+7+++8HYJlllqlPxZrcX/7yFwC++OKLouyxxx4D4JRTTml3/VJLLQXAgw8+WIfa5eONN94ojqNdXnzxxXbXxfBdtXyQMckp2hfKe7xfv35FWUy66kmiLS+88EIA/vWvfxXnHnjggXbXX3LJJQDMPffcANx8883FuW222QaofA3QlexBSZKyVPce1KyzzgrAHHPMUZR1pge15pprtvusq6++uijr3bs3AKuuuuqkVLOpXHrppcVxPH1uueWWQOWU++mnn76+FWsCY8aMAcplDbHYGeDcc88FqvdW08XQ4dFHHwVg6aWXLsoeeuih2lU2U/fcc09x/Kc//Wm810011VQA7LnnnkVZfPfTyVAh2njXXXctynrKJIm0TTfddFMA3nrrLaDyftxwww0BeOWVV4qy+NsQ0usjN2e9Fu7bg5IkZckAJUnKUt2H+KKLHS/sAK666ioAVlhhBQA22mijdj+30korAfC3v/2tKJtyyikBePPNN4uyk08+ubYV7sZiQsRdd91VlC244IIAnHjiiYDDeuPzySefFMdbbbUVUK7TS8XwdKzNSYdDYpj5zjvv7NDvjBf46RrBZnb66acDcMABB7Q7t88++wCVrwJ22WUXAKaZZpqiLIb2IuNJDGMB/PCHPwTKTTebWdw7MSFi0KBBxbm4l3/9618DcOSRRxbnYgJJrEUF2G677QC47LLL2v2eFVdcsYa1/n72oCRJWWpYJol44gFYfPHFgbJHlD5RxUvTYcOGVVyTiiclgGOOOab2le1mYppyZORIX8jvsMMOAEwxxRT1r1g3EJMd4mkT4IUXXujwz6e9+VjCkPbG3nvvPQDWXXddoPp06uWXX77jFe7Gol0+++yzomyBBRYA4NBDDwXKNky9//77xXH0BqLdI6MEwBlnnAHA5JNnn3J0kt1+++0ArLXWWu3ObbbZZgCcf/75QDmRLHX33XcXx217TumU8g022GCS69oZ9qAkSVnK4tGibUSfeeaZ210TC/gidx5Un6rbU6WLQW+99dbxXjfbbLMBMMMMM3Toc6+88kqgei/iwAMP7EwVu4UjjjgCmHCvKaY7A1x00UUA9O/fH6i+4Dud2nzqqacC1XtO8X7wnHPO6WStu6eY/hz3GJTT6g855BAAjj322OJc7GgQ76cALr74YqBs9/Qd9Prrr98V1c5Guth77733Bsq/idF+UH5Pq/Wcwl577TXec5dffnlxnL7/qwd7UJKkLBmgJElZymKIr620uzl69GgArrnmGgCeeOKJ4tzPfvaz+lYsY+lwZ7RZTD2N/IRQOUTaVmScSD8rXlY/99xz7a4/6KCDgHILD+ie09Yff/zx4jjNU9bW/PPPD8ANN9zQrqyjXn755fGeGzJkCFD/YZRGmWuuuQBYffXVi7IY4osMEZtvvnlxbosttgCq5+KLKevVlqg0mzPPPBMoh/WgHL4bPHgwAL/73e+Kc20nRI0bN644jqUTzz77bFEWSyViCLGRuTntQUmSspRlDyqdSh4bFcaL//TFZ0wFThfixTTInjaBIqZHQ7mYOXpO6VN+28kRr732WnEcbZwuog7RM5pvvvmKsnj62mSTTYqyeKGabgqZu6OOOqo4TqeEh1j0GC/sO9priokr0aMFuO6666p+NjT/S/22Yvp3te3XIzdcOuU+nuzT73YsSRkwYECX1TMH6SSoWHKTtkP0nGIqeTUxPT+mnUM5PT210047AbDjjjtOQo1rwx6UJClLWfagUrPMMgtQZokeOHBgcW748OEV/0L5BBFj0dUW+jWTmHpbbVp07Oeyxx57FGWRAT72iDruuOOKcxdccAFQmV4mekf7778/ULmocpFFFgHg7bffnsT/isZK33m+/vrrQOV08ehRdvZeiv2gfvvb37Y7FwvVY++difn8ZhGLczsqzbYdqY46umyiu0pTEaXpnMJJJ50EwKeffgqU6eOgHNW49957gcp3xtELq7aYv1pShHqzByVJypIBSpKUpeyH+MKyyy4LVE4zj2mW6Ur0yMQbU1FjaAq65xTo7/P0008DlS8+Q0wD33nnnYuyGALYb7/9ABg1alRxLiY2pENSf/jDH4BySDD9PXH9euut166sO1luueWK445mHh+fdIPB3Xbbrd35mPIb/9/01GE9KJdBpFuKV9vcMURW+ZEjR3ZtxTI02WSTFceRezTN+xivQiY0OWyeeeYBKielxGSUdFg/3TCz0exBSZKy1G16UGHOOecsjuPlddpDWGONNYBy6vAzzzxTnEtzSjWLRx55ZLzn0nYJMekhMp2n7rvvPqDMCQfl5Iu0LEQbN2NOvomVZumv9jT717/+FYB11lmnbnXK1dChQwE499xzi7IJ9QB62tKRVJr/MTKPp1PwYyv2RRddFCh7m1AuAI9M7+m56EHF/xe5sQclScqSAUqSlKVuN8SXim5vbK0N5cvEyDd17bXXFudiuG+hhRaqUw27XmyAl75c3nbbbSuuSbNFxCSTuD7W6kA5jBcTIqDcNr7a9dUmZvRUsQ4lXvxDZQ7EkA4B9iQff/xxcRxD7bGtSDp0t8oqqwBlO/35z38uzsUatZ4uNhBMJ0l0ROTbS/8mxj268MIL16ZyNWYPSpKUpW7Xg0qfoiLjcayQhspMvVD5xFrtRX+zSJ9CJ/QyOZ6Y4prYHh7KDMiff/55URYZ4+O6CW161hPFCv9on7TXFG2cruqPDSN7mpaWluI4cr2FdIPGyFge3+m0B7XEEkt0ZRWbXuTzq3aPxkhJbuxBSZKylH0PKqZPnnbaaUCZLw7g1VdfHe/PxbuoGK+F5pymGhndI6szlG0UPaJ0cfPYsWMrfj7enUD5nildtHf88ccDzbnIeWJ9/fXXxXEsMq22hCEW6qb5I5vxHpyQeO9bbZ+m6FUttthiRVlkk991113bXd/ZvbdUKW3n7sIelCQpSwYoSVKWshrii+799ddfX5QdccQRAIwZM6ZDn7HaaqsB5eZy/fv3r2UVsxO53dKcbtGO/fr1Azo+rFQtF9+SSy5Zk3o2g9jaZJ999inKzjrrrIpr0qG+GNbqacN6qX/+858AfPDBB0VZbCq61FJLAZVbSdx2221AubleunwizSKjznvssccaXYVOswclScpSw3pQkVUbynxQsRHZww8/3KHPWHPNNQE4/PDDi7KYVt5TnlpjU8I77rijKIsceTENv5roBaQ9zHiibebp+JMiJpi07TVBmQNt4403rmudctd2WUN6HD2n0aNHF+ciV2RMx0/zPK6//vpdW9kmV21T09zZg5IkZckAJUnKUl2G+NLMBHvttRdQpoyHctO9CYntCQ455JCiLF7gx0SBniydzJBu4KhJF2vxTjzxxHbnFl98cQBuv/32utapu3jrrbfalc0+++xAORx63XXXtbsmJlfktHledxebvn5fvsic5F07SVKP1SU9qBdffBGAo48+GoBbbrmlOPfSSy99789PM800xfGwYcMA2GWXXQCYcsopa1VNqUPiHjz99NPbnTv00EOB7rnVfT1EDzMVk0xiCnmfPn2KczFC0h2zHuQupulHfk2Ap556Cqjs6c4777z1rdgE2IOSJGWpS3pQsa31eeedN95r0rHlzTff/LvKTP5dddKFoulWx1K9pHvttM1fePDBBxfHK664Yt3q1B3F1PA0h2bkKBwwYABQTi0HGDx4cB1r1zMNHz68OF5rrbWAylyeI0aMACpzcjaKPShJUpYMUJKkLHXJEN++++5b8a/U3YwaNao4vuSSS4Ayt+Huu+9enEtf8Ku9GKIfMmRIUZYeq/5WWmml4njTTTcF4IorrijKIovHySefDDR2Ypo9KElSlrLKZi7lYtCgQcXxQQcdBMDFF18M2GtS99a7d+/iOCavLLTQQkVZLKs47LDDgMZOlrAHJUnKkgFKkpQlh/ikKhZZZJHieNy4cQ2sidR1YrgvMqK0PW40e1CSpCwZoCZCS0tLS6Pr0ExsT6ln6eh33gAlScqSAUqSlCUDlCQpSwYoSVKWDFCSpCwZoCRJWTJASZKyZICSJGXJACVJypIBSpKUJQOUJClLBihJUpYMUJKkLBmgJElZMkBJkrJkgJIkZckAJUnKkgFKkpQlA5QkKUsGKElSlgxQkqQsGaAkSVkyQEmSsmSAkiRlyQAlScqSAUqSlKVera2tHb+4V693gJe6rjrdxk9aW1v7TOqH2J4F27P2JrlNbc8K3qO11aH27FSAkiSpXhzikyRlyQAlScqSAUqSlCUDlCQpSwYoSVKWDFCSpCwZoCRJWTJASZKyZICSJGXp/wExUf3fhO0TIwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15537729748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the individual digit examples\n",
    "\n",
    "fig, ax=plt.subplots(nrows=2, ncols=5, sharex=True, sharey=True)\n",
    "ax=ax.flatten()\n",
    "for i in range(10):\n",
    "    img=X_train[y_train==i][0].reshape(28, 28)\n",
    "    ax[i].imshow(img, cmap='Greys')\n",
    "ax[0].set_xticks([])\n",
    "ax[0].set_yticks([])\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcjfX///HHGU1j7DIYPsgSKiVZSn1iWrRIkiVC5CtpsSR9SFJRQlla+EhEIilFlg9ZspRKU0xDSQiRncIYs885vz/O73rPHDOYM87MdZ2Z5/1262auc11nvq/v+3PN9breu8vj8SAiIuI0IXYHICIikh0lKBERcSQlKBERcSQlKBERcSQlKBERcSQlKBERcSQlKBERcSQlKBERcSQlKBERcaRL/Lk4IiLCU7169TwKJXhs2rTpuMfjKX+xv0fl6aXyDLxAlKnKM4Pu0cDKaXn6laCqV6/Oxo0bcx9VAeFyufYG4veoPL1UnoEXiDJVeWbQPRpYOS1PNfGJiIgjKUGJiIgjKUGJiIgjKUGJiIgjKUGJiIgjKUGJiIgjKUGJiIgj+TUPyik+/vhjAJKSkvjll1945513zLnrr79e8wwkoNLS0khISDDHW7ZswePxALB169Zsv3PppZcC0KFDB0qUKEFIiN4Fc8Pj8RAfH8/y5csBmDVrFps3b+aXX34BoHTp0naG5yjWPblgwYJznnO5XBw+fJj+/fsD8Omnn+JyuWjRogXgvPLUX42IiDhSUNSgduzYAcBvv/3GihUreP/99wHftwLLli1baNiwIQAxMTH5HGlwc7vdnDhxwuez0qVLc8klQXGb5Jmnn36ad999N1ff7dWrFz169OC///0vAOHh4YEMrUCKi4tjzZo1AEyfPp2lS5f6nC9evDihoaF2hOZICQkJREdH8/TTTwPe5+TZzn5WWv926tQJl8tF1apVAejfvz9dunShYsWK+RH6BTnuyRMfH0+3bt0A2Lx5M4B5aJ4+fRqPx8Ott94KwNdff53l+263m1OnTuVPsDbatGkTAI0aNfL7u8nJyQD8/PPPTJo0iZSUFABSUlJYvHixz7Xvvfcejz322EVGG9ySkpKIioryK7mkpaUB8NVXXzFz5kzTpNKgQYM8iTHYHTx4EIBRo0Yxffp0c4/Wrl2b4cOHm/IcOXIknTp1olixYrbF6jR9+/Zl1qxZF/U7/vrrLwAGDRrElClTePTRRwEYMGCAaa62g5r4RETEkRxTg7KqpQ888AC7d+8+53WHDx+mRIkSgLe29ffff3PfffcB8OeffwLQtGnTvA3WAerXr5/ja8+cOQN4a11btmzhq6++AjC1paioKAA6d+7MmTNnWL16tfnugAEDCn0NasqUKYSEhFCkSJEcf8ftdgPQrl07Fi9ezMqVKwHVoDL7/fffAbj//vs5cOAAAImJiTz//PP06NED8C6uGhoaas6PHDmS66+/3pZ4nSq7QWHWwLHIyMgLfr9fv34cOXLEHP/xxx8MHToUgJYtW3LttdcGKFL/OSZBvfLKKwBZklPRokVN9bVRo0aUL5+xQnt4eDgTJ040iQmgTp06TJs2Le8Dtpk/bfDNmjUDIDY2Fo/HY9qfPR4PvXr14q233gK8bfs1atQwCcrj8fDMM88EOPLgk5v+DqvZ1HoJ6NSpU0BjKgispvtmzZqZl86HH36Yxo0b+/Qrn6148eL5El+wWLVqFWPHjjV/x4B5Zq5YseKCI/PatWvH33//DeCYvieLIxLUr7/+aoaRZlarVi2WLVtGrVq1zvndffv2+Rx3795d7dNnsYY4FytWjGbNmjF69GgAatSoQalSpczDYPbs2Tz55JPme1deeSVDhgzJ/4ClULjpppt8/j2f5557zvz80EMP5VlMwahixYq8/vrrpqbz7rvvmlrVvHnz6Nixo9/Dx62aV5kyZQIbrJ/UByUiIo7kiBrUa6+9Rnx8vDlu1aoVAGPGjMm29pSUlATAjz/+6DPqrFWrVrRp0yaPow0+M2bMAKBKlSpcdtllWc5bIwKfeOIJEhMTqV27NgAbNmwwTS/in++//97uEAqUzM34klWRIkV45JFHAG8//sMPPwx4m5g7dux4we9bU3csnTt3BjDDz+3iiAQ1YMAAM8y0fPnyzJw5E+CcD0drJYnevXsD0KRJEwDmzJmjB2o2zjegYvbs2fTp0wfwdlDXqFHDPFztrt4Hs2PHjpmfa9as6dO2Hxsba4ZRn+3GG2/M89iC2W233WbrsOdgULp0aZYsWZLj648cOWKeueBNcK+++moeROY/RySoG2+8Mds5TdmJiYmhb9++5jg0NNT0kyg5+ScmJoZevXqRmpoKePuc5s6dS7ly5WyOLPgcPXoU8A6OOH36NIMHDzbn/vzzTypVqmSO4+LizMTJUqVKmXl9Xbt2VYLKRlxcnKnl9+jRQ8tGBViLFi3MYgjgnQtVtGhRGyPKoP+lRUTEkRxRg/JHkyZNfIagzp8/n3vvvdfGiIJPbGwsALfffjupqanUrFkTgPXr16v2dB7W0PHffvuN7du3A7B06VJ2795tltWy+kfPZt2zNWvWpG3btjzwwAMAVKpUKdt+QcmwZs0a0yQ6cOBAm6MpWNauXcuuXbvM/dm4cWNHzSMNqgT15ptv4na7far4Vv+T5ExsbKwps/T0dK666iq+++47QH1O55OQkMAVV1wBeCeLn0+NGjXYs2ePOd6wYYPu04uwevVq8zdfoUIFm6MpGKzV+QcPHkxKSgrVqlUDvPOmnCQoElR6ejrgnTEdEhJisv3nn39ORESEnaEFlZiYGG6//XZTngDr1q1TYsqBIkWKmNrl4cOHGTduHOCdL9KmTRufl6aUlBQzMOWvv/7immuuyf+AC5CDBw9y8803A87bDiJYTZgwAfA+E1wuFwMGDACcV77qgxIREUdyfA0qNTWVVatWAd7NtQAziu+ee+4575Io4hUXFwdAnz59iIuLMyPKFixYoBpoDoWFhTF79mzAW57WG312W5GEh4eb+SPWKtEiTrFixQpefvllwLtmZJ06dRy7OoejE1RycjIDBw7kvffeM599+umntG/fHkDJKQcSEhLMAqV//vknNWvWZNmyZYB33ULJOS30mv+Sk5NZvnw5rVu3tjuUAiEhIYFx48aZZ2dISAgrVqxwbN+eoxPUqVOnfJLT1VdfTYcOHWyMKPjs37/fzMK/5JJLWLlypRm1J3nHqpla850kd6Kjo0lMTPSZVya5Fx0dzdq1a81xjx49+Ne//mVjROenPigREXEkR9agrGVirJEm1oiozJlfcqZEiRJme4JOnTqp9pRPrLXMzt6hWPzz4YcfAs7bBiJYWdvCW4YMGZKr7WTyiyMTlLUO1OTJkwFMh57ThkAGg8qVK7N//34AxyxfIuKPMmXKUKpUKbvDCGqPP/44AFu3bgUy1jHdvHkzcXFxNGrUyLbYzkdNfCIi4kiOq0EdPnyYU6dOmeOhQ4eaIb2SO6p55r/7778fgJ07dxIWFmZzNMErJiaG8uXLU7JkSbtDCWrWdhrW6D1r1/GpU6eya9cu2+K6EMclqI8++og5c+YAULt2bfr16+ezzbtIMAgPDwc4727Qcm7WljqxsbG88MILNkdTcA0ePNjRo/gcl6BatWplts+YPXu2kpNIIXTkyBHzc5cuXWyMpGCy+qRGjx5tcyTnpz4oERFxJMfVoK666irS0tLsDkNEbPTMM8/4/CsXJ/MC0cHE5c9Md5fLdQzYm3fhBI3LPR7PRbc9qjwNlWfgXXSZqjx96B4NrByVp18JSkREJL+oD0pERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBxJCUpERBzJrx11IyIiPNWrV8+jUILHpk2bjgdi8zKVp5fKM/ACUaYqzwy6RwMrp+XpV4KqXr06GzduzH1UBYTL5QrIjpgqTy+VZ+AFokxVnhl0jwZWTstTTXwiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJIfs2DkoLv5MmTzJkzB4D+/ftz//33M3fuXACKFi1qZ2j5KiEhgVtuuQWAzZs38/zzz+f4u6NGjcLlcgHg8Xjo1asXFSpUAKBTp05ce+21gQ9YJAfS0tLYuXMnixYtAuDLL7/km2++MeeHDh3Ka6+9Zld4WagGJSIijhRUNaj09HRSUlKYMmWK+ezQoUOMHTvWHDdo0MCcv/HGG/M9xmCSkJBAdHQ0Tz/9NOB924+Pj+evv/4CwOVysXjxYoYNGwbAuHHjbIs1vx04cICQEO/7m8vlYvTo0T61orN/9ng85tqGDRsSGxtrfteMGTPM+TFjxvDdd9/p3vTDjBkzzM//+c9/OHXqFFdffTUAzzzzDAD33nsvAJGRkfkfoIMdOnSIxMRE1q1bB8Dy5cuZP3++Oe/xeMx9Dt7700k1KMcnqNTUVCZOnAjAqlWrWL58uXk4WDIfb9682fwPoIeAr/T0dE6ePGkS0pEjR1i7dq05n/nBm1nXrl3zLUanqF27tlmS5ujRo+dcnqZcuXI5us/27dsHQI0aNfj7778DF2gBk5SUxPLly5k3bx7gfaCeOnXK5750uVxs27YNgN69ewNQt25dALZu3ZrPETvPZ599ZpL6Dz/8QFxcnM8LFED9+vXNvx999JH5bp8+ffI52vNTE5+IiDiSI2tQcXFxAOzcuZPhw4ezdOlSn/NFihQB4MorrwQgPj4egL17A7KeY4FjvT3Nnj2bXr16ZXmbupDLL788z2ILBhUqVDBNSLm1cOFCIOdlXpjExsayadMmAEaMGMHBgwf9/h2HDx8GYM+ePdSoUSOg8QWbxx57jNOnT/t81rZtWwDatWtH69atufTSSwH49ddffWpQL774Yv4FmgOOSVCpqakAvPPOO6ZP6ejRo1muCw8P57vvvgPg+uuvJzU11bSv3n333QBUqlQpHyJ2th07dgDwyy+/0L9/f8DbpHe2Hj16mD/o//u//6NJkyY+140YMYLLLrssHyIu2Kxm1czt/YVVTEwMGzZs4IMPPgBg+/btJCYm5vj7TZo04aeffvL5rEyZMgCFPjkBREdHExMTY47btWtHWFhYtteuWrUKj8fDzTffDECpUqXyJcacckSCSk1NNR3xmQc8gPeGq127NgB33XUXd999N/Xq1TPnt2zZYhITwP3330/fvn3zIWrn2rx5My1atADgxIkTPucaN25MnTp1ALj22msZMGAAoaGhAFSuXNnnpaBx48YMHDgwn6IuuI4ePeoz4OLWW2+1NyCbtWjRwrSSnEuDBg0Ab1/gmDFjfM7t27eP2267zeczp73526lu3bqmT+5crBrW3LlzcblctGzZEuCcicwuep0TERFHckQNCqBs2bKAd8TY448/DniHjEZERFCyZEkgo+/JsmPHDlq1amWO+/Xrx7Bhw0yNoLByuVzmjf3aa6/l7bffBrxlXK1aNUqXLu1zfUJCAuBtAnS5XOYt6vXXX6dYsWL5GHnBNGrUKNPv99Zbb6lMgYoVK5qRZM899xxffPEFAFFRUVx99dVUrlwZyL7J6eym6lq1atGhQ4c8jrhgeeONNwBvH1TFihV54oknbI4oe45IUKGhoQwZMiTH11v9VUOHDuXo0aP069cP8DYPWp1/hVn9+vX59ttvAUzz6LkkJCRwzz33mOPGjRvz+uuvA96HhVyco0ePMmPGDDM44oEHHrA5IvutXr2aiIgIqlataj7zp9lz+vTp5udixYoxevRoSpQoEcgQCzxrtRjw9uWXK1fOxmjOzREJyh8pKSlmqZidO3fSqFEjM4G0sNecMrtQYrL07t3bDDpp2rQpq1at0ht+AK1Zs4YzZ86YznunPgjy0/XXX5/r744cOdJnoumUKVNo165dIMIqNA4cOMA///wDeAftjBo1yuaIzk19UCIi4khBV4PavXs3O3fuBLx9Kp988olqTrm0ePFi/ve//5nmJ9WeAscaDTlw4EBcLhcff/wxgMo3F9LS0vj9998BGD9+PElJSabV5L777rMztKA0YcIEM4rv9ttvNyMmnSioEtSuXbuoX7++aW/euHGj5j3kgrVsT/fu3YmPjzfrl+nhGTiLFy8GvBNIIyMjueGGG2yOKHjNmjXLLGkE0LNnTwYMGGBjRMHr0KFDPmsb9uzZ08ZoLiwoEtT27dsBTN/Tr7/+CkC1atVsiymYjR49GsAkpx9//NHmiAqWhIQEM0rK5XKZUZTiv+3bt/Poo4+aUak33HADb731ls1RBa/ExMQLzkFzEvVBiYiIIzm+BpWammpmjaelpfH000+r5pRLSUlJdO3alQULFgBQp04d1q9fbzbTk8AYNWoUu3btArwroWSeqyc5k5ycDEDDhg0JCQnhpptuAmDlypVqis4Ft9sNwPDhw/F4PGZ+WefOne0M64IcnaD27t1L3759zUKQ3bp1480337Q5quC1detWFi1aZJpLMu/0KoGTed5T69at9UD1U2pqqpnbmJyczA033GD2KFJZ5o61x9ucOXNwuVz897//tTminHF0gho7dixLly41nfhO2kgrmDz33HMAZtXiHj16AKijOQ8sW7aMw4cPmwTlz1bx4k1IAwcO9OnIHzNmjCaNX6S5c+f6HLdu3dqmSPyjPigREXEkR9agoqOjAW91tHLlynz//fcAVKlSxc6wgtKOHTvM26i1srm1rJTmjwWe1YTSq1cvADWh5pDVjN++fXuio6PN+puxsbGFfj+yQOvevXvQLA3luASVmJhotssoWbIk69ev16CIXLAWJ73qqqvMZ/Xq1WPLli12hVSgpaSkAN5h0R6Ph2uuucbmiIJHUlKSab6Pjo6mWbNmTJ06FdBmmYHy5ZdfAt7nwksvvRQ0+5IFR5QiIlLoOKYGlZSUBMCHH35otn+eOHGi3qByadq0aYB3oqg1yCTzCsYSWNYWELGxsURGRtK9e3fAu+SRmvmyZw0lHzRoEB9++CEAbdq04YMPPnDczq7B7PTp06xfvx7wPg+OHTsWNCvwOCZBHT9+HICnnnqKESNGAJh9ocQ/p0+f9hmOv2TJEiBjJQ7JOx6Ph8OHD3PHHXcA3t1N9WKQVWpqqtmtedGiRbRv3x7AbAMvgWONKLXcf//9Zp5e8eLF7QgpxxyRoJKSksyGWdWqVTN9UJdc4ojwgk5KSopZULdt27Y+/VCSN6w3/sjISJ9h5lZNVjKkp6fTp08fPvvsMwC2bNniszeUBFaJEiXMMP09e/YwadIkwsPDbY4qZ9QHJSIijuSIKsqrr75KbGws4F1p29r+XXKnXLlypKWl2R1GoVK6dGnAuxmcnF+vXr3Yt28fBw8eBAiat/lgtnbtWrtDyBWXNRw5Rxe7XMeAvXkXTtC43OPxlL/YX6LyNFSegXfRZary9KF7NLByVJ5+JSgREZH8oj4oERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJCUoERFxJL+2fI+IiPBUr149j0IJHps2bToeiN01VZ5eKs/AC0SZqjwz6B4NrJyWp18Jqnr16mzcuDH3URUQLpcrIFs2qzy9VJ6BF4gyVXlm0D0aWDktTzXxiYiIIylBiYiIIylBiYiIIylBiYiII/k1SEJEJNBmzpzJo48+ao5fe+01n/M9e/akQoUK+R2WOIASlIjYKioqivHjx5vjxYsX880335jjYcOGsWTJEgBatmyZ7/GJfQpUE5/b7SYtLc3853a77Q7J8TweD8nJySQnJ/Ptt98yePBgXC4XLpeLAQMGsH79ertDlAKuRo0aDBgwwPy3cuVKdu7cyc6dO03Nqn379rRv394kKgmsH3/8kR9//JHmzZvjcrl49NFHfWq1dilQCUpERAqOoGzis2pGqampzJ07l+PHjwOwceNG5s2bZ64bO3Yszz77rC0xOtHOnTuZO3euz2cpKSmMGjXK5zOXywXAO++8w+rVq4mOjgagWLFi+RNoEDl+/DhfffWVz2cej4d+/foBcOLECZ9zbrebZs2amTK/5ZZb8ifQIHLJJZdQo0YNACZNmsR1111nyvOhhx5i/vz53HPPPXaGGPTi4+MBWLBgAe+//z7ff/894L0/XS4XCxYsAGD69Om2xQhBkqDS09MB2L9/Px999BE7duwAYPbs2T7XeTwe83AFWL9+faFNUNYNuH//fnOTzZo1i2PHjvlcl7nMQkNDqVOnDn/++ScAZ86cYevWrSQlJQFKULt37wbgyJEjrFy5EoApU6act0wz348AISEhfP/999x9990AfPfddzRo0CCvQw9aoaGhPPXUU6xatQrw9k8tWbJECcpPJ0+eBGDixImMHz+etLQ0ABISErK93il9fWriExERR3JcDcrtdpusfuLECd5//332798PwAcffODX7+revXvA4wsGixYtYsCAAQDs3XvuJa+GDRtGWFgYd9xxB+CtIV111VW0bdsWgGXLlnHPPfdQokSJvA/awbZt28azzz7Lzz//DMCxY8fweDxA1hoSeN8+s/scvGUKkJycDGBqp3J+o0ePBmDJkiWsWLGCxMREAMLDw+0My/HWr1/P2rVrefPNNwE4derUBb9Tvnx5Zs2aldeh5YhjEtTq1asB+Pzzz3nvvffOeV2xYsVo06YNkJGAfvjhBwBeffVV3G439957LwAPPPBAXobsWPv37/dJTDfddBMAERERNGzY0CSga665hpCQkCzftR6i4B0CfOmll+ZD1M518uRJ06RnsVakLlKkCCNGjDB9JgBNmzbN8jusRFS8eHEA6tevD8DVV1+dFyEXOJdffjngLfc///zT3N9XXnmlnWE50vHjx5k8eTIAI0eONM15mVWsWBGAVq1aMWPGDJ9z48aN45JLnJEabInC7XabyXj//PMPgHkAbNu2zefa8PBwWrduDcATTzxBZGRklpvSulnDwsJITEzkwQcfBLJ/uy0Mevfubfo4AKpWrQp4y+dCrNoqQGRkJL179w58gEGmdu3a1K5d29yHjRo1olOnTjn+flJSknlpslg13FKlSgUu0ALMqindddddTJ06lcWLFwNKUJlZg8X+/e9/s3PnTp9z1apVA7wv7TVr1qRjx44AZvCJ5aGHHjLnnEB9UCIi4ki21KA2bNhghjFu3rw5y/moqCjAWz0tW7bseZtBDh48yMKFCwFITEzk4Ycf5s477wQKbw0qNDSUK664Ilff/fDDD83PX3zxBWXLlg1UWEErIiIiS80+p06fPk3Lli3ZsGED4G09eOqppwpt/2huWVNLzpw5g8fjMf2mkuHw4cOAtxXk4YcfBqBEiRLUq1ePbt26Ad4ae0JCgpmOM3/+fAA6d+4MeIeV56SlJb/YkqA+/vhjk5gGDRpEhw4dfM7XrVsXyFnzR+ZlUTweD8OHD6dy5coBjrhwGD9+PHPmzDHll9skJxnWrVvHDz/8YF6WGjRokGWtObkwa9rEnDlzaNmypenDkwzXXHMNAHv27KFcuXKAt4/0bN988w09e/Y0x7Vr1zZTUYoWLZoPkeacLQlq0qRJPPbYY4C307NMmTK5+j2HDx9mwYIFZtTf119/bTpTJedSUlIAWLFiBSVKlDCT9qybXPxn1bgeeeQRAOrUqQN4E1bp0qVtiytYHTx40Px8+eWXExoaamM0zna+hXW/+eYbU1sC6Nq1K2PHjnVcYrKoD0pERBzJlhqUy+UKyOz5N998k++//55mzZoBcOONN2ZbpZVzS0lJMTWlM2fO8NJLL5kRP5I7cXFxZimjuLg4rr76atauXQug2lMuWVMlwPt3Lv6x5o0NGjSIU6dOmftwxIgRREZG2hnaeTljsLufpk2bBsDkyZNJSEhgzJgxAI4Zux9M0tPTTft+o0aNGD58uL0BFQBly5b1GaAzdepUNZdehB07dpilepo2bUq7du1sjii4JCYmmi6Vn376iRtuuIFx48YBULNmTTtDu6Cge6Lv2rWLiRMnAt43/vfff58mTZoA2XcIyrmlpKRw7733mnJ7++23bY4oeFmrTIwcORK3221aCF544YVsJ+5Kzg0bNszMg/rggw8K/com/lq5ciUff/wx4K3Bf/bZZ2ZupNOpD0pERBwp6GpQr732Gr/++qs5btasmZr2cumtt95i3bp1NGrUCPDOQBf/JSYm8uKLLwKwfPlyQkJCzAz9s6dQiH+WLFnC0qVLzbxJrRzhn+joaHr06GGO582bFzS1JwiyBDVw4EDmzZvHddddB3gnmalD33+HDh0C4JVXXiE8PDzLOnOSc9u2baNnz5789NNP5rM//vhD9+VFSE5ONhNLly5dStOmTbW9hp+sPdysJc+sieHBtv9YUCQoa6299957j8TERPMWVa1aNdWecsHqs0tISGDQoEFcdtllNkcUfOLi4gAYNWoUP/30k1ntZOrUqT4Lx4r/7rnnHjP5/r777vPZhFQubPv27WY1nfj4eHr06JFlQdhgoT4oERFxpKCoflhNUNZYfmsotGpP/vv888/NDrADBgzghRdesDmi4GStUWgNJ586dSqQ/VYbcmHW3/bLL7/MN998Y+bpvPfee45aGy4YjBo1ykwdqVSpkk8fVLBx/BN+3759DB061BxPnjxZa8Tl0q+//kq3bt1ITU0FoE2bNtruwQ/WJoNdunQxi5c2aNBAyxddpN27d/PZZ58BMGHCBCBjrqOTJ5E6jbW+6YIFC8zSRb/88ktQN+GriU9ERBzJ8TWodevW8ffff5vj8PBwTcjNpeHDh5OcnMycOXMAaN68uc0RBReraXnRokVmJ+J+/fqp9pRL1uTmqKgos+AzwI8//qjVyv2Unp5OixYtAO8E/DVr1gAEde0JgiBBne3mm2+2O4SgZe0XY+0MW1j3y8qN5ORkdu/ebY7Hjx8PZKxWLv67/vrrgYwRkZI7breb7t27m36ndevW+axdGMwcn6C6detG165dzbH15iqSnw4cOGA2d4OMLduzOaTSAAAVo0lEQVRF7PbRRx8xd+5czpw5A2CWhSoI9LQXERFHcnyCcrlcFClSxPynZqnc+/bbb3G73ZQoUUILbooUEN27d8ftdhMeHl6gak8ALo/Hk/OLXa5jwN68CydoXO7xeMpf7C9ReRoqz8C76DJVefrQPRpYOSpPvxKUiIhIfnF8E5+IiBROSlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJISlAiIuJIl/hzcUREhKd69ep5FErw2LRp0/FA7K6p8vRSeQZeIMpU5ZlB92hg5bQ8/UpQ1atXZ+PGjbmPqoBwuVwB2bJZ5eml8gy8QJSpyjOD7tHAyml5qolPREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcKagS1IEDB3jssccICQkhJCSETp06cfr0abvDEhGRPODXMHO7LFy4EIB27dpRvnx5HnzwQQBiY2OpWbMm27ZtAyAiIsK2GKXgi4uLA6Bs2bK43W4AQkJCeOWVV6hVq5a5zuPxcMsttwDw3XffAfDvf/8bgKpVq+ZnyCJBzfEJ6vfff6dz584AlC5dmq+//porr7wSgCNHjlC5cmV+/vlnAO68807b4nSSffv2UaNGDQDS09NtjqbgCA0NBeC6665j8+bNALhcLl5++WWf6zweD5dddhkAJ06cALxJDWDRokU0adLE/C4RJ0lOTubDDz80x0WKFOHRRx+1LZ6gauITEZHCw9E1KLfbzcKFC7nkEm+YmzZtombNmuZ8yZIlKVq0KK1atQLg5MmTFCtWzJZYncTlcuFyuQCIjo7mxhtvtDmigiE8PByAoUOH0qlTp/Nea9Wczj5u1qwZJ0+eVA3qAqy+5R9++AGAtLQ0AHr06JHt9U888YT5t1KlSnkfYJCLj48HYP78+QDMmjULgPXr15OammqeHwDr1q0DYPbs2fkbJA5PUNu2beOFF15gxowZAD7JCaBYsWL07NmTyZMnA96mFfGWg1UWTZs2pVGjRgAsW7aMChUq5Pj3REdHEx0dzZNPPgmgh+r/16FDB9O3BLB7926GDx9ujk+cOJElQcn5HTt2DPA2Kc2bN4833ngDgD179uTo+6+88goAU6dO5dChQ3kTZJBat24dR44cAWDs2LHs2LHD9KEmJCRkuT5zcgKYM2cOYE+CUhOfiIg4kqNrUFbGbtOmjc2RBJfMTXyNGjUiNjYWgCuuuILGjRubwSQdOnSgdu3aPt+Njo5m7ty5AMyYMYMzZ87Qtm1bQCPQMmvatKnPz126dDHHn3zyCQ8//HC232vbti1hYWF5Hp8T7du3D/AO3Jk4cSI7d+4057Zu3Qp4a+k7duzI9f+Nxx577OKCLADWrFnDmjVrAJg2bRr//PPPRQ2WmjBhQqBC85ujExTAgw8+SMmSJbM95/F4SEtLo3HjxoCaoCyZm/hKlSpl2u+XLVsGwMqVKwFv8oqPj6dXr14AvP/++7hcLjMCMDIyErfbTbly5fL7/4Wg1rlzZ0JCsm+cGDJkCJdeemk+R2S/H3/8kWbNmgGQkpKS4+9VrVqV6tWrM3bsWJ/Pv/76awCee+45n8/PfuEqDA4ePMjbb79tXiwPHjxomvAupGPHjjRp0oTHH38c8N6fVpcJwN13322a+O3g6ARVrVo1jh07Zh6wRYoU8TmfkJDA1KlT6dOnD0Ch/MPPTuYaVGb33nuvz78vvPACSUlJ5nzFihXp1KmT6etbt24dAwYMyIeIC44pU6YQEhKSpfwnTpwIYF6mCpvx48eTmpp6zvNFixYFICwsjIkTJ5ph+c2bN6dUqVI+1+7YsYOYmBhzHBISwqBBgwC44447Ah264zVu3JjDhw+f83y3bt3MVAhr+oOlePHihIaGmjl+H330kc/5Ll262FrjVx+UiIg4kqNrUE899dR5z1sjU7p27Zof4QSNqlWr0rx5c8A79N56cz27CbR8ed8NLV999dUsv6tKlSoauu+HU6dOZfmsaNGi1K9f34ZonOPTTz/lgQceADKGOGd22223Ad5+0gvZvn07n376qTkuX768aUWpXLlyIMINCi+99BJAltrTI488YmpMxYsXp2zZsmaqztmSkpJYtmwZQ4YMAbyrpURFRTFs2DAAbr755rwKP0ccnaAuxLqpK1asaHMkzmMl7d69e5sb2N9BDtbwUrmwxMREwDtA4mzXXXedWeqoMLNWhLkYf/75p+lLtbRs2bJQrphi9Q0dOnSIW265hbvuuguAChUqZOkOOZd//vnHDIIC71y/t99+2zEvVI5MUFYH/8mTJ30+T0hI4IMPPuCXX34BvKOCXC4XR48eBbyJyppMWdhZkxU9Hg9Tp04Fsq8hnY/mk+TMyZMnTSe+dW9mNn78+PwOqcCxOv379u3L0qVLzed16tRh/PjxWfpWCgPrb3zatGl+f9d6abVGSFutJAsXLnRMcgL1QYmIiEM5pgZljSbbs2cPb731FuAd9pyZx+PxGR0VEhJCmTJlzBvq+PHjqVKlSj5F7GzWcNvsRvNdiDW7fP/+/WYZKTm3f/75hzFjxphjt9tthpm3bNnSZ86U+O/gwYOmlcSa02fVHh588EGKFy9uW2zBKCkpyfRfWcvHrV27FnDeXEdHJKjPPvvMdOpt377dfB4REUHPnj19rp0/fz67du0CvJ2j27ZtM0NSJYOVoJYsWeL33JC///4bgF27dvm1NFJhFBcXxxNPPJHlxck6zs0LgviaPXu26cS3rFixAoBrr73WjpCC2v79+9mwYYM5jo+Pz3GfVX5zRIJq164d1apVA7wzyh966CHAO+os88gzt9vN3r17TYJas2aNktMFWHOe/GHNO4mMjOS6664LdEgFyqJFi8ys/eycvX6kZLDWKzx58qSZHG6x5j7u37/f56UVvPMjreeF+Ofo0aP07t3brNxRr149Ro0a5djRj+qDEhERR3JEDapIkSJmS4jzbQ0xefJkPv30U9q3bw9gNi6UwCpdujQAdevWZdy4cbmqhRUWmVcxz6xEiRIAZoUDycpqSs5u7pM14tRapdxqgpo9ezYdO3Z0bJOUUyUnJwMwc+ZM1q1bZ0bt9e7dm9atW9sZ2nk5IkFdiLUUf//+/XG5XIwYMQLgnOudieQ1a0HTc22rsWTJEgAN2jmPc03KPXToENOnT8/2XIMGDZSc/JScnGyaoYcMGUKxYsXMPMl+/frZGdoFOT5Bud1uWrZsaY5Hjx5N3bp1bYyo4LMW84yLi8uyDpp4bdq0Cch+5Qi3202DBg3yO6QC4eDBgzRu3NjMwatQoQKDBw+mb9++AIV2JfjcSklJ4ZlnnmHKlCkAtGrVitGjR3PNNdfYHFnOqAoiIiKO5Pga1M8//8zPP/8MeFeKePzxx9W0l8esNQ5jY2PNmn7i63zDyHV/5l6TJk18VjDp0qULzz77rI0RBafdu3cDMHjwYBYsWGD6kT/55JOgmjfm+AT1xx9/mJ+nTJliOvAl71j9Kh6Px2xuKL6svqWwsDDTAS25c+jQIbp3725+rlSpEosWLQI0zyk3EhMTzcTxBQsWcNlll/Hee+8BBFVyAjXxiYiIQzm+BhUZGcktt9wCaOv3/DJv3jzA23ylibrZs+7J+vXr89NPP/mcK126tFaQ8MOkSZP46quvzHGHDh1o0qSJjREFt+uuu86n5SkmJoZ//etfNkaUe46vQUVFRfH111+bLZ4l/3g8Hs2BuoDslpL64YcfKFGihJkLJedXsmRJKlasaP7r0aOH3SEFtcmTJxMWFkZYWBhRUVHn3W3X6RxfgxL7NGzY0O4QHC8iIoJt27bZHUZQGzJkSJa19iT3WrRoYfYnC3aOr0GJiEjhpBqUZDFy5Eiff0VE7OCydq/N0cUu1zFgb96FEzQu93g85S/2l6g8DZVn4F10mao8fegeDawcladfCUpERCS/qA9KREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcSQlKREQcya8t3yMiIjzVq1fPo1CCx6ZNm44HYndNlaeXyjPwAlGmKs8MukcDK6fl6VeCql69Ohs3bsx9VAWEy+UKyJbNKk8vlWfgBaJMVZ4ZdI8GVk7LU018IiLiSEpQIiLiSEpQIiLiSEpQIiLiSEpQIiLiSH6N4pPg5/F4ePfddwHo06cPAHPnzgXgoYcesi0uEZGzKUEVMu+//z79+vUDICTEW4Fes2YNoASVG3v37uW3337jxIkTAHTt2pXGjRtTuXJlAJ544gkALrnE+6d255132hOow+3ZsweAmjVrUqZMGVOeF7J7926+/fZbOnbsCEDRokXzLMZg5Ha7SU1NZd68eQD8/fffAKxcuRKA5cuXA/Doo48CUK9ePa666ipzn1rPCLuoiU9ERBzJthrUyZMnAVi9enWWc4cPHwagf//+uN3uLFnc7XYD3uweGRlJ06ZNAejcuTP33Xef3qLO4fjx40yYMCHL5xs2bAAgPj6eEiVK5HdYjjRp0iTatWsHQK9evUhLS8v2uh07dvDXX3+Z45CQEGJiYoiJiQHgf//7HwBhYWEAHDlyhJIlS+Zl6EEtJCSE06dPM2DAAADuu+8+qlWrRqVKlQDvPbp161ZSU1MB6NSpE2fOnGHQoEEAbNy4kapVq9oTvIMkJiYC8MYbb/DKK69kOe/xeABwuVwAzJgxw+f8zJkzAejWrVseRnlhtiWovXu9E4k7dep0zmtcLhchISGmEC1WwnK5XBw5coRFixYBsHDhQo4dO6YEdZbjx48D0Lx5c3bs2EG9evUA78330ksv8dtvvwFKUJZdu3YxZswYnn766YD9zuTkZABatmzJl19+qSR1Hh6Ph4kTJwIwceJEypQpQ9myZQFISkri0KFDWb5j3ePWy2thduDAAZo1awZ4n7O1atWiWLFi5vzo0aMJDQ0FMhLUzp07gYx+aevYbmriExERR7KtBhUeHg5AyZIlOX36tM85qzmkTp06pKenU6RIEZ/z6enpgPeNf9++ffkQbfA6duwYUVFRAGzfvp1KlSoxfPhwANq1a8fq1atZtWoVAAkJCXaF6Sg1atSgbt26Pm/qZcqUoUePHrn6fTNnzjRN2hs2bKB58+Z89dVXAJQrV+6i4y3oTp48acqvePHiNG/enG+++cbnmrZt2wIQGRmZ7/E5zfTp000LVZcuXZg2bdoFW5XKl89Yt7VMmTJmIJXdbEtQderUAWDevHm0bNky23MxMTFZmvcyO3XqFHfddRebNm3Ku0CDlJVsoqKi2L59O+Ad4bRmzRpTvmd78803TdNKYRYSEsLYsWPp1asXAB9//DHFihWjWrVqufp9d911F/fee6853rJli1kw9O677774gAuADz/80Oe4b9++gPcBm1np0qWpXLmyafKzPP/880DGy21h9txzz/HAAw8A3mfphZJTamoqr776qjl+6qmnfBKWnWwfZt6wYcMsn/3yyy8AfPHFF6ajOjtJSUnExcX5fLZp0yYN5SVjWHOLFi245pprAHjppZfOmZzEV8OGDc1Ah4tVs2bNgPyegmzz5s0+x9deey0AN954Y5ZrP//8c5/jSy+9lFq1auVdcEEmLCyM+vXr5+jatLQ0hg0bxoIFCwBvTcrqh3IC9UGJiIgj2V6DKlOmDNOnTzcTxQAzwulCG3tVrFiRxo0b+4w4iY2NVQ0K71slwDvvvGNzJHK2W2+9lRYtWtgdhqPcd999ACxevJiiRYty6623ZntdfHw848aN8/ls0KBBlClTJq9DLJBmzpzpU54ff/yxo/rxbE9QRYoUoXv37qapb+jQoSZBZdf8dyGZE52I3ZKSkhg5cqTPZ6GhoVkG/hR27du3B7xzIDt27MgVV1yR7XWPP/44P/30kzm+8847GTZsWL7EWJDs3r0bgCFDhgAZw8ut4elOYXuCAu9YfKvNecmSJRf1u5YsWcIjjzwSiLAKnZ49e9odQoEzf/58PvroI7vDcDyrBvTCCy9ke95qJVm4cCGQMRdy+PDhprVAcq5BgwYAnDlzhrp165oJ/FbftVOoD0pERBzJWenSTydPnuT33383y3a43W5at25tc1Qi3qY9yBgubalbty4ffPCBHSEFrfj4eP7zn/8A3nItWrSoWYHfWuZMcu6xxx4jPj4e8A7bX7p0qeNqThZnRpVDe/fu5eeffzZzpexeeTfYzZgxQ/OgAuzsaRCdO3c268pJzixatMisaQjewVH333+/jREFr88//5zp06ebl/pZs2ZRo0YNm6M6t6BOUCJSsA0cOJBp06aZ40svvdQsDCs5Z03cX7x4MS6XyyzG6/QWJ1U5RETEkVSDEuPs/hLJvbOHllvNzzmd4V/YWauSv/vuu6SkpJjPJ0yYwJNPPmlXWEEpOTnZjNrbtWsX5cuXz3bbHScK+gRltaWCltq/WKVLl7Y7hAIhPT09y5YQ1hpxbdq0sSOkoJKWlmZ2d7aS05VXXgmcf3seySo5OZk+ffqwa9cuwLuYbnR0tM1R5VxQJ6jnn3/eZzHZmTNn6iF7ESZNmpTlzV/8t23bNrPhm0UP1pwbM2YMX3zxhTmuV68e69atA+Cyyy6zKarg9Pvvv/uMGn3jjTeoUqWKjRH5R31QIiLiSEFZg1qxYgUA33//PZCxB4xWkLg4p06dsjuEAmHHjh0+x2XLlmXgwIE2RRNcTp06laV/ZMWKFao55dL06dOBjHVNrSWlgkVQJihrbx2reU8LogbGli1bzERIyb3Bgwf7HEdFRVGvXj2bogkOycnJgHfposwvSuPHj6dChQp2hRW0/vrrL8A7zwkwm5IG29+2mvhERMSRgrIGZY3W08oRgfXtt98yYsQIRo8ebXcoUsgcPHgQyNoa0q9fP638ngvWFhqnT5+mbt26VKxY0eaIcicoE5SVmM63HbzkzKxZsxg+fDjg3Y34xRdftDegAui1116zO4SgY225I/5LSEgw/fPg3eGhePHiNkaUe0GZoDILCwujXLlydocRtCpUqMDkyZPtDqNAeeaZZ+jfvz/gXXvvQhtviq+iRYsSExMDoNpTLvzzzz+m/B566CFHr7V3IWojExERRwrKGtRnn30GeLcn7tevH1FRUTZHJJKhT58+ZodSyRnrLT89Pd3mSIJflSpVCkw5ujIvFXTBi12uY8DevAsnaFzu8XjKX+wvUXkaKs/Au+gyVXn60D0aWDkqT78SlIiISH5RH5SIiDiSEpSIiDiSEpSIiDiSEpSIiDiSEpSIiDiSEpSIiDiSEpSIiDiSEpSIiDiSEpSIiDjS/wP9oERMAInE+QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15537fcb940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting multiple examples of the same digit.\n",
    "\n",
    "fig, ax=plt.subplots(nrows=5, ncols=5, sharex=True, sharey=True)\n",
    "ax=ax.flatten()\n",
    "for i in range(25):\n",
    "    img=X_train[y_train==7][i].reshape(28, 28)\n",
    "    ax[i].imshow(img, cmap='Greys')\n",
    "ax[0].set_xticks([])\n",
    "ax[0].set_yticks([])\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving scaled images in .npz files in .npy format\n",
    "np.savez_compressed('mnist_scaled.npz', X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['X_train', 'y_train', 'X_test', 'y_test']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Loading preprocessed MNIST image arrays\n",
    "mnist=np.load('mnist_scaled.npz')\n",
    "mnist.files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implementing a MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNetMLP(object):\n",
    "    \"\"\" Feedforward Neural Network - MLP Classifier \"\"\"\n",
    "    def __init__(self, n_hidden=30, l2=0, epochs=100, eta=0.001, shuffle=True, minibatch_size=1, seed=None):\n",
    "        self.random=np.random.RandomState(seed)\n",
    "        self.n_hidden=n_hidden\n",
    "        self.l2=l2\n",
    "        self.epochs=epochs\n",
    "        self.eta=eta\n",
    "        self.shuffle=shuffle\n",
    "        self.minibatch_size=minibatch_size\n",
    "        \n",
    "    def _onehot(self, y, n_classes):\n",
    "        \"\"\" Encode labels into one-hot representation \"\"\"\n",
    "        onehot=np.zeros((n_classes, y.shape[0]))\n",
    "        for idx, val in enumerate(y.astype(int)):\n",
    "            onehot[val, idx]=1.\n",
    "        return onehot.T\n",
    "    \n",
    "    def _sigmoid(self, z):\n",
    "        \"\"\" Compute logistic function (sigmoid) \"\"\"\n",
    "        return 1./(1+np.exp(-np.clip(z, -250, 250)))\n",
    "    \n",
    "    def _forward(self, X):\n",
    "        \"\"\" Compute forward propagation step \"\"\"\n",
    "        # step 1: net input of hidden layer\n",
    "        z_h=np.dot(X, self.w_h) + self.b_h\n",
    "        \n",
    "        # step 2: activation of hidden layer\n",
    "        a_h=self._sigmoid(z_h)\n",
    "        \n",
    "        # step 3: net input of output layer\n",
    "        z_out=np.dot(a_h, self.w_out) + self.b_out\n",
    "        \n",
    "        # step 4: activation output layer\n",
    "        a_out=self._sigmoid(z_out)\n",
    "        \n",
    "        return z_h, a_h, z_out, a_out\n",
    "    \n",
    "    def _compute_cost(self, y_enc, output):\n",
    "        \"\"\" Compute cost function \"\"\"\n",
    "        L2_term=(self.l2*(np.sum(self.w_h**2.0)+np.sum(self.w_out**2.0)))\n",
    "        term1=-y_enc * (np.log(output))\n",
    "        term2=(1.0-y_enc)*np.log(1.0-output)\n",
    "        cost=np.sum(term1-term2)+L2_term\n",
    "        return cost\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \"\"\" Predict class labels \"\"\"\n",
    "        z_h, a_h, z_out, a_out=self._forward(X)\n",
    "        y_pred=np.argmax(z_out, axis=1)\n",
    "        return y_pred\n",
    "    \n",
    "    def fit(self, X_train, y_train, X_valid, y_valid):\n",
    "        \"\"\" Learn weights from training data \"\"\"\n",
    "        n_output=np.unique(y_train).shape[0] # no. of class labels\n",
    "        n_features=X_train.shape[1]\n",
    "        \n",
    "        \"\"\" Weight Initialization \"\"\"\n",
    "        \n",
    "        # weights for input -> hidden\n",
    "        self.b_h=np.zeros(self.n_hidden)\n",
    "        self.w_h=self.random.normal(loc=0.0, scale=0.1, size=(n_features, self.n_hidden))\n",
    "        \n",
    "        # weights for hidden -> output\n",
    "        self.b_out=np.zeros(n_output)\n",
    "        self.w_out=self.random.normal(loc=0.0, scale=0.1, size=(self.n_hidden, n_output))\n",
    "        \n",
    "        epoch_strlen=len(str(self.epochs)) # for progr. format\n",
    "        self.eval_={'cost':[], 'train_acc':[], 'valid_acc':[]}\n",
    "        \n",
    "        y_train_enc=self._onehot(y_train, n_output)\n",
    "        \n",
    "        # iterate over training examples\n",
    "        for i in range(self.epochs):\n",
    "            # iterate over minibatches\n",
    "            indices=np.arange(X_train.shape[0])\n",
    "            if self.shuffle:\n",
    "                self.random.shuffle(indices)\n",
    "            for start_idx in range(0,indices.shape[0]-self.minibatch_size+1,self.minibatch_size):\n",
    "                batch_idx=indices[start_idx:start_idx+self.minibatch_size]\n",
    "                \n",
    "                # forward propagation\n",
    "                z_h, a_h, z_out, a_out=self._forward(X_train[batch_idx])\n",
    "                \n",
    "                \"\"\" Backpropagtion \"\"\"\n",
    "                sigma_out=a_out-y_train_enc[batch_idx]\n",
    "                sigmoid_derivative_h=a_h*(1.0-a_h)\n",
    "                sigma_h=(np.dot(sigma_out, self.w_out.T)*sigmoid_derivative_h)\n",
    "                grad_w_h=np.dot(X_train[batch_idx].T, sigma_h)\n",
    "                grad_b_h=np.sum(sigma_h, axis=0)\n",
    "                grad_w_out=np.dot(a_h.T, sigma_out)\n",
    "                grad_b_out=np.sum(sigma_out, axis=0)\n",
    "                \n",
    "                # Regularization and weight updates\n",
    "                delta_w_h=(grad_w_h+self.l2*self.w_h)\n",
    "                delta_b_h=grad_b_h  # bias is not regularized\n",
    "                self.w_h -= self.eta*delta_w_h\n",
    "                self.b_h -= self.eta*delta_b_h\n",
    "                \n",
    "                delta_w_out=(grad_w_out + self.l2*self.w_out)\n",
    "                delta_b_out=grad_b_out  # bias is not regularized\n",
    "                self.w_out -= self.eta*delta_w_out\n",
    "                self.b_out -= self.eta*delta_b_out\n",
    "                delta_b_out=grad_b_out  # bias is not regularized\n",
    "\n",
    "            \"\"\" Evaluation \"\"\"\n",
    "            \n",
    "            # Evaluation after each epoch during training\n",
    "            z_h, a_h, z_out, a_out=self._forward(X_train)\n",
    "            cost=self._compute_cost(y_enc=y_train_enc, output=a_out)\n",
    "            \n",
    "            y_train_pred=self.predict(X_train)\n",
    "            y_valid_pred=self.predict(X_valid)\n",
    "            \n",
    "            train_acc=((np.sum(y_train==y_train_pred)).astype(np.float)/X_train.shape[0])\n",
    "            valid_acc=((np.sum(y_valid==y_valid_pred)).astype(np.float)/X_valid.shape[0])\n",
    "            \n",
    "            sys.stderr.write('\\r%0*d/%d | Cost: %.2f | Train/Valid Acc.: %.2f%%/%.2f%% ' % (epoch_strlen, i+1, self.epochs, cost, train_acc*100, valid_acc*100))\n",
    "            sys.stderr.flush()\n",
    "            \n",
    "            self.eval_['cost'].append(cost)\n",
    "            self.eval_['train_acc'].append(train_acc)\n",
    "            self.eval_['valid_acc'].append(valid_acc)\n",
    "            \n",
    "        return self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initializing 784-100-10 MLP\n",
    "nn=NeuralNetMLP(n_hidden=100, l2=0.01, epochs=200, eta=0.0005, minibatch_size=100, shuffle=True, seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "200/200 | Cost: 5065.78 | Train/Valid Acc.: 99.28%/97.98%  "
     ]
    },
    {
     "data": {
      "text/plain": [
       "<__main__.NeuralNetMLP at 0x15538979940>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training the model on 55,000 samples and performing validation on 5,000 samples\n",
    "\n",
    "nn.fit(X_train=X_train[:55000], y_train=y_train[:55000], X_valid=X_train[55000:],y_valid=y_train[55000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x155386156a0>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcXXV9//HX524zk9mTTCaTTJIJJOyaAAEiWFtAIVIrVCtirQRL5WdFq9VfFR72V1q1v2rtrwKtGwoa3BAVJcpmRHAtS4IhLAlkSAhZJpkkk8y+3bmf3x/3O+ESZrkDd5lJ3s/H4z7m3O8959zPPbmZ93zP+Z5zzN0RERHJhUixCxARkSOHQkVERHJGoSIiIjmjUBERkZxRqIiISM4oVEREJGcUKiIikjMKFRERyRmFioiI5Eys2AUU2syZM72pqanYZYiITBnr1q3b5+512cx71IVKU1MTa9euLXYZIiJThplty3Ze7f4SEZGcUaiIiEjOKFRERCRnFCoiIpIzChUREckZhYqIiOSMQkVERHJGoZKlG+/fzK+e3VvsMkREJjWFSpa+8qvn+O1mhYqIyFgUKlmKRyMMJFPFLkNEZFJTqGQpEYswMOTFLkNEZFJTqGQpoZ6KiMi4FCpZSsQiDA4pVERExqJQyVI8auqpiIiMQ6GSpXhUPRURkfEoVLKUPlCvUBERGYtCJUsaUiwiMj6FSpZKdKBeRGRcCpUsxaPa/SUiMh6FSpYS0QiDSZ38KCIyFoVKluI6UC8iMi6FSpZ0Rr2IyPgUKllKxEw9FRGRcShUspTQyY8iIuNSqGRJ56mIiIxPoZIlXVBSRGR8eQ0VM6sxsx+a2SYz22hmrzOz6Wa2xsw2h5+1YV4zsxvNrNnMNpjZaRnrWRnm32xmKzPaTzezJ8IyN5qZ5euzpK/95aRSGlYsIjKafPdUbgDudfcTgCXARuAa4H53XwzcH54DvBlYHB5XAV8GMLPpwHXAWcCZwHXDQRTmeV/Gcivy9UESsfSmGkyptyIiMpq8hYqZVQNvAG4GcPcBdz8IXAysCrOtAi4J0xcDt3raQ0CNmTUAFwJr3L3N3Q8Aa4AV4bUqd3/I3R24NWNdOZeIhlDR3R9FREaVz57KQmAv8A0z+4OZfd3MyoF6d28J8+wG6sP0XGB7xvI7QttY7TtGaM+L4Z6KDtaLiIwun6ESA04DvuzupwLdvLirC4DQw8j7n/5mdpWZrTWztXv37n1F64gf6qkoVERERpPPUNkB7HD3h8PzH5IOmT1h1xXhZ2t4fScwL2P5xtA2VnvjCO0v4+43ufsyd19WV1f3ij5MPJoeA6CeiojI6PIWKu6+G9huZseHpvOBp4HVwPAIrpXAnWF6NXB5GAW2HGgPu8nuAy4ws9pwgP4C4L7wWoeZLQ+jvi7PWFfOHdr9pZ6KiMioYnle/4eA75hZAtgCvJd0kN1uZlcC24BLw7x3AxcBzUBPmBd3bzOzTwOPhvk+5e5tYfoDwDeBMuCe8MiL4QP16qmIiIwur6Hi7uuBZSO8dP4I8zpw9SjruQW4ZYT2tcApr7LMrBwaUqyeiojIqHRGfZbi6qmIiIxLoZIlHVMRERmfQiVL6qmIiIxPoZKlkpjOqBcRGY9CJUs6+VFEZHwKlSzpMi0iIuNTqGTp0Bn16qmIiIxKoZIl9VRERManUMlSQsdURETGpVDJknoqIiLjU6hkSaO/RETGp1DJUiyiS9+LiIxHoZIlMyMRizCgkx9FREalUJmARDSinoqIyBgUKhOQiEV0TEVEZAwKlQmIR02hIiIyBoXKBCRi2v0lIjIWhcoExKMRXaZFRGQMCpUJ0IF6EZGxKVQmQAfqRUTGplCZgIR2f4mIjEmhMgHxaITBpE5+FBEZjUJlAhKxCP3qqYiIjCqvoWJmz5vZE2a23szWhrbpZrbGzDaHn7Wh3czsRjNrNrMNZnZaxnpWhvk3m9nKjPbTw/qbw7KWz8+T7qkoVERERlOInsq57r7U3ZeF59cA97v7YuD+8BzgzcDi8LgK+DKkQwi4DjgLOBO4bjiIwjzvy1huRT4/SElMx1RERMZSjN1fFwOrwvQq4JKM9ls97SGgxswagAuBNe7e5u4HgDXAivBalbs/5O4O3JqxrrzQGfUiImPLd6g48HMzW2dmV4W2endvCdO7gfowPRfYnrHsjtA2VvuOEdrzJhHT7i8RkbHE8rz+17v7TjObBawxs02ZL7q7m1neh1OFQLsKYP78+a94PTqjXkRkbHntqbj7zvCzFfgx6WMie8KuK8LP1jD7TmBexuKNoW2s9sYR2keq4yZ3X+buy+rq6l7x54nrjHoRkTHlLVTMrNzMKoengQuAJ4HVwPAIrpXAnWF6NXB5GAW2HGgPu8nuAy4ws9pwgP4C4L7wWoeZLQ+jvi7PWFde6EC9iMjY8rn7qx74cRjlGwO+6+73mtmjwO1mdiWwDbg0zH83cBHQDPQA7wVw9zYz+zTwaJjvU+7eFqY/AHwTKAPuCY+8iUcjDOrOjyIio8pbqLj7FmDJCO37gfNHaHfg6lHWdQtwywjta4FTXnWxWUrEIgylnKGUE43k9ZQYEZEpSWfUT0A8mt5cGlYsIjIyhcoEJGLpzdWvg/UiIiNSqExAeSIKQHd/ssiViIhMTgqVCaguiwPQ0TdY5EpERCYnhcoEVA2HSq96KiIiI1GoTEBVaTpU2nvVUxERGYlCZQIO7f5SqIiIjEihMgFVZenTenRMRURkZAqVCajU7i8RkTEpVCYgGjEqS2I6UC8iMgqFygRVlcW1+0tEZBQKlQmqLI3pQL2IyCgUKhNUVRbXMRURkVEoVCaouixOR5+OqYiIjEShMkFVpXHt/hIRGYVCZYKqynRMRURkNAqVCaoui9PZn2QopTtAiogcTqEyQcPX/+rScRURkZdRqEzQ8JWKNQJMROTlFCoTVFWq63+JiIxGoTJBulKxiMjoFCoTVKW7P4qIjEqhMkE6piIiMrq8h4qZRc3sD2b2s/B8oZk9bGbNZvZ9M0uE9pLwvDm83pSxjmtD+zNmdmFG+4rQ1mxm1+T7s0Dm7i+N/hIROVwheiofBjZmPP8c8AV3XwQcAK4M7VcCB0L7F8J8mNlJwGXAycAK4EshqKLAF4E3AycB7wrz5lV5IkrEtPtLRGQkeQ0VM2sE/hT4enhuwHnAD8Msq4BLwvTF4Tnh9fPD/BcDt7l7v7tvBZqBM8Oj2d23uPsAcFuYN6/MTBeVFBEZRb57KtcDHwdS4fkM4KC7D+872gHMDdNzge0A4fX2MP+h9sOWGa0972ZWlNDa0V+ItxIRmVLyFipm9hag1d3X5es9JlDLVWa21szW7t2791Wvr6G6lJaOvhxUJiJyZMlnT+Uc4K1m9jzpXVPnATcANWYWC/M0AjvD9E5gHkB4vRrYn9l+2DKjtb+Mu9/k7svcfVldXd2r/mBzqstoOdj7qtcjInKkyVuouPu17t7o7k2kD7T/0t3fDTwA/EWYbSVwZ5heHZ4TXv+lu3tovyyMDlsILAYeAR4FFofRZInwHqvz9Xkyza4uZW9XPwPJ1Pgzi4gcRWLjz5JznwBuM7PPAH8Abg7tNwPfMrNmoI10SODuT5nZ7cDTQBK42t2HAMzsg8B9QBS4xd2fKsQHmFNTiju0dvbRWDutEG8pIjIlZBUqZvYtd3/PeG2jcfcHgQfD9BbSI7cOn6cPeMcoy/8r8K8jtN8N3J1NDbnUUF0GQEu7QkVEJFO2u79OznwSzhE5PfflTA0N1aUA7NJxFRGRlxgzVMKZ7J3Aa82sIzw6gVZePBZy1GmoSfdUdrdrBJiISKYxQ8Xd/83dK4HPu3tVeFS6+wx3v7ZANU46FSUxKktitChUREReItvdXz8zs3IAM/srM/tPM1uQx7omvYaaUu3+EhE5TLah8mWgx8yWkD5Dfhtwa96qmgIaqsvYrRMgRUReIttQSYZzRi4GbnD3G4DK/JU1+TVUl7LroEJFRCRTtqHSaWbXAu8B7jKzCBDPX1mTX0N1Gfu6+ulPDhW7FBGRSSPbUHkn0A/8tbvvJn1JlM/nraopoKEmPay4Rb0VEZFDsgqVECTfAarDhSL73P2oPqZyXH16798zezqLXImIyOSRVaiY2aWkr7f1DuBS4GEz+4uxlzqyHVdfgRlsbOkodikiIpNGttf++iRwhru3AphZHfALXrzZ1lFnWiLGwhnlbGpRT0VEZFi2x1Qiw4ES7J/AskesExoq2bhbPRURkWHZBsO9ZnafmV1hZlcAd1GECzlONifMrmLb/h66+5PjzywichQYc/eXmS0C6t39H8zsbcDrw0v/Q/rA/VHtxIYqADbt7uT0BbVFrkZEpPjG66lcD3QCuPsd7v5Rd/8o6V7K9fkubrI7YXZ6BNgm7QITEQHGD5Umd99weKO7rwWa8lLRFNJYW0ZlSUwjwEREgvFCpXSM18pyWchUZGacNKeKJ3YqVEREYPxQedTM3nd4o5n9DbAuPyVNLUvn1bBxV4cu1yIiwvjnqXwE+LGZvZsXQ2QZkAD+PJ+FTRVL59UwMJTi6V0dnDpfB+tF5Og2Zqi4+x7gbDM7FzglNN/l7r/Me2VTxNL5NQA8vv2gQkVEjnpZnVHv7g8AD+S5lilpdlUpsypLWL/9YLFLEREpuqP+rPhXy8xYOq9GoSIigkIlJ5bOr+H5/T0c6B4odikiIkWVt1Axs1Ize8TMHjezp8zsX0L7QjN72Myazez7ZpYI7SXheXN4vSljXdeG9mfM7MKM9hWhrdnMrsnXZxnPGU3TAfjdc/uKVYKIyKSQz55KP3Ceuy8BlgIrzGw58DngC+6+CDgAXBnmvxI4ENq/EObDzE4CLgNOBlYAXzKzqJlFgS8CbwZOAt4V5i240+bXMqM8wX1P7SnG24uITBp5CxVP6wpP4+HhwHm8eMn8VcAlYfri8Jzw+vlmZqH9Nnfvd/etQDNwZng0u/sWdx8AbgvzFlw0YrzppHoe2NSq81VE5KiW12MqoUexHmgF1gDPAQfdffiyvjuAuWF6LrAdILzeDszIbD9smdHai+LCU2bT1Z/k9837i1WCiEjR5TVU3H3I3ZeSvqf9mcAJ+Xy/0ZjZVWa21szW7t27Ny/vcfaxM6goiXHvk7vzsn4RkamgIKO/3P0g6fNcXgfUmNnw+TGNwM4wvROYBxBeryZ9M7BD7YctM1r7SO9/k7svc/dldXV1OflMhyuJRTnvhFms2biHoZTn5T1ERCa7fI7+qjOzmjBdBrwJ2Eg6XIbvb78SuDNMrw7PCa//0t09tF8WRoctBBYDjwCPAovDaLIE6YP5q/P1ebJx4cmzaese4NHn24pZhohI0WR7j/pXogFYFUZpRYDb3f1nZvY0cJuZfQb4A3BzmP9m4Ftm1gy0kQ4J3P0pM7sdeBpIAle7+xCAmX0QuA+IAre4+1N5/Dzj+pPj60jEItz31G6WHzOjmKWIiBSFpTsDR49ly5b52rVr87b+v1n1KBtbOvntJ84lPXhNRGRqM7N17r4sm3l1Rn2OXXDybHYe7OXxHe3FLkVEpOAUKjl24cmzKY1HuH3t9vFnFhE5wihUcqy6LM5Fr2lg9fpddPcnx19AROQIolDJg788cz5d/Unu2tBS7FJERApKoZIHpy+oZfGsCr7z8DaOtoEQInJ0U6jkgZmx8uwmHt/RziNbdc6KiBw9FCp58vbTGplenuBrv9lS7FJERApGoZInZYko71m+gF9sbOWZ3Z3FLkdEpCAUKnm08uwmqsviXHvHBl0PTESOCgqVPJpenuC6PzuJx144yKrfP1/sckRE8k6hkmd/fupc/vi4Or6w5lnaeweLXY6ISF4pVPLMzPj4iuPp7E+qtyIiRzyFSgGcPKeaN544i5t/u5XOPvVWROTIpVApkA+dt5j23kG+/putxS5FRCRvFCoFsmReDX/62ga++uvnaGnvLXY5IiJ5oVApoGtWnEDK4TN3bdTlW0TkiKRQKaB506fxwXMXcdeGFv7PnU+S0rkrInKEyefthGUEHzpvEd0DSb76qy1Ulcb5+IoTil2SiEjOKFQKzMy4ZsUJdPQO8qUHn+P0BbWcf2J9scsSEckJ7f4qAjPjuj87mZPnVPGxHzxOW/dAsUsSEckJhUqRlMajXP/OpXT1Jfn8fZuKXY6ISE4oVIpocX0l7z2nidse3c5DW/YXuxwRkVdNoVJkH37jccyfPo2VtzzCnet3FrscEZFXJW+hYmbzzOwBM3vazJ4ysw+H9ulmtsbMNoeftaHdzOxGM2s2sw1mdlrGulaG+Teb2cqM9tPN7ImwzI1mZvn6PPlSURLjjr89myXzavjI99ez9nndKVJEpq589lSSwMfc/SRgOXC1mZ0EXAPc7+6LgfvDc4A3A4vD4yrgy5AOIeA64CzgTOC64SAK87wvY7kVefw8eTOjooRbrjiDOdVl/MMPN9A7MFTskkREXpG8hYq7t7j7Y2G6E9gIzAUuBlaF2VYBl4Tpi4FbPe0hoMbMGoALgTXu3ubuB4A1wIrwWpW7P+Tp09NvzVjXlFNREuPz73gtW/d1875b17Kno6/YJYmITFhBjqmYWRNwKvAwUO/uLeGl3cDwSRpzge0Zi+0IbWO17xihfco6+9iZfPZtr2HttjZWXP9r1m07UOySREQmJO+hYmYVwI+Aj7h7R+ZroYeR92uVmNlVZrbWzNbu3bs332/3qlx25nzu+rs/oroszru//hAPPNNa7JJERLKW11AxszjpQPmOu98RmveEXVeEn8O/NXcC8zIWbwxtY7U3jtD+Mu5+k7svc/dldXV1r+5DFcCxdRX84P1nc2xdBe9btZY7Htsx/kIiIpNAPkd/GXAzsNHd/zPjpdXA8AiulcCdGe2Xh1Fgy4H2sJvsPuACM6sNB+gvAO4Lr3WY2fLwXpdnrGvKq6ss4barlnPmwul89PbHuffJ3cUuSURkXPnsqZwDvAc4z8zWh8dFwGeBN5nZZuCN4TnA3cAWoBn4GvABAHdvAz4NPBoenwpthHm+HpZ5Drgnj5+n4CpL49xyxRksmVfDR29fz9O7OsZfSESkiOxou6/HsmXLfO3atcUuY0L2dPTx1v/+LR29ST6x4nguf10TkciUOyVHRKYoM1vn7suymVdn1E8B9VWl/OTqczhz4XT++adPc9nXHmLb/u5ilyUi8jIKlSmiobqMb773DP79L17LxpYOVlz/G775u6260ZeITCoKlSnEzLh02Tx+/vdv4KxjXuy1PL9PvRYRmRwUKlNQQ3UZ37jiDD4/3Gu54dd843dbGVKvRUSKTKEyRZkZ71g2jzV//8e87pgZ/MtPn+ZtX/odT+5sL3ZpInIUU6hMcbOrS7nlijO4/p1L2Xmwj7d9+ffc80TL+AuKiOSBQuUIYGZccupc1vz9GzhlThUf+O5jXHvHBnYe7C12aSJylFGoHEFqyxN8933LWfm6Jn60bifn/ceDfPGBZvqTupS+iBSGTn48Qu082Munf/o09z61mznVpXzg3EW8Y1kjJbFosUsTkSlGJz8Kc2vK+Mp7TufbV57F7OpS/vEnT/Inn3+QH63bwdH2h4SIFI56KkcBd+e3zfv4j/ue4fEd7Zw8p4ol82p4+2mNnL6gdvwViMhRbSI9FYXKUSSVcr6/djs/WLudzXu66BpI8u6z5nPu8bNYfswMyktixS5RRCYhhcoYjuZQydTVn+Tf7t7I9x55gZRDfVUJ//SWk7noNbNJ30lARCRNoTIGhcpLdfcnWbftAJ+9ZxNPt3SwZF4Nf31OE2cfO5O6ypJilycik4BCZQwKlZElh1L8cN0Obrx/M7va+4gYXLJ0Lh88bxHH1FUUuzwRKSKFyhgUKmNLDqV4alcHP318F99+eBsDyRSXLJ3LO5bN48yF04nqPi4iRx2FyhgUKtnb29nPTb9+jm8/9AK9g0PMrChhxSn1nDKnmiXzajixoarYJYpIAShUxqBQmbiegSS/3NTKXRtaeOCZVvoGUwCcMreKd54xn7cumUN1WbzIVYpIvihUxqBQeXWGUs7OA7088Ewrtz26nY0tHcSjxpkLp3PeCfW88cRZLJhRXuwyRSSHFCpjUKjkjrvz5M4O7nqihV9u2sOze7oAOLaunGULprN0fg0rTp5NbXmiyJWKyKuhUBmDQiV/trf1cP/GPTzwzF6e2NlOW/cA8ahxXH0lx9VX8lfLF+gMfpEpSKEyBoVKYbg7G1s6uXP9Tp7Z08kfXjhIe+8gx9dX8qaT6jn/xFksaawhotFkIpOeQmUMCpXi6O5P8qPHdnDXhhYefb6NlENVaYzTF9SyrGk6py+oZUljDWUJXUVZZLKZFKFiZrcAbwFa3f2U0DYd+D7QBDwPXOruByx9XZAbgIuAHuAKd38sLLMS+Mew2s+4+6rQfjrwTaAMuBv4sGfxYRQqxXege4AHn23l4S1trN12gObW9LEYM6gsiXHSnCr+5vXH8LpjdT0ykclgsoTKG4Au4NaMUPl3oM3dP2tm1wC17v4JM7sI+BDpUDkLuMHdzwohtBZYBjiwDjg9BNEjwN8BD5MOlRvd/Z7x6lKoTD4Hugd47IUDh47D/OLpPexq7wOgsbaM4+srOWVuNUvn17CksYbpOvAvUlCTIlRCIU3AzzJC5RngT9y9xcwagAfd/Xgz+2qY/l7mfMMPd/9fof2rwIPh8YC7nxDa35U531gUKpPf4FCKXz+7l6d3dfBsaxebWjpo3tvF8Ff1pIYq/mjxTI6dVcHCmeUsnlVBzTQFjUi+TCRUCr1vod7dW8L0bqA+TM8FtmfMtyO0jdW+Y4R2OQLEoxHOP7Ge80+sP9TW1Z/kyZ3trNt2gAefaeXm324lmUqnTMRg+TEzOLGhiuqyONVlcZpmlnPKnCpqpyU0GECkgIq2w9rd3cwKMkrAzK4CrgKYP39+Id5ScqyiJMbyY2aw/JgZXH3uIpJDKXYe7GXLvm4e23aAe5/czfrtL9AzMPSyZV/bWM1fnjmf1zbWMLemjMrSmIJGJE8KHSp7zKwhY/dXa2jfCczLmK8xtO0kvQsss/3B0N44wvwjcvebgJsgvfvr1X0EmQxi0QgLZpSzYEY55x4/i49dcDwAA8kU7b2DPLunk40tHRzsGeS+p3ZzzR1PvLhsxFg0q4LXzK3mNY3VvGZuNdMSMbbs7WLBjHKOn12pC2eKvEKFDpXVwErgs+HnnRntHzSz20gfqG8PwXMf8H/NbPiMuQuAa929zcw6zGw56QP1lwP/VcgPIpNTIhahrrKEusoSzlk0E4CPXXAcm3Z30tzaRWtnP3s7+3m6pYP7N7Xyg3U7XraOmRUlXHbGPOLRCCl33rp0Dsfq8v8iWclbqJjZ90j3Mmaa2Q7gOtJhcruZXQlsAy4Ns99NeuRXM+khxe8FCOHxaeDRMN+n3L0tTH+AF4cU3xMeIi9jZpzYUPWyqyq7Oy3tfTyxs52+wSGaZpTz3N4ufrahhS8+2JxeFrjh/s0snFnOyXOqmF1VCkA0Ypx1zHRObKhiRnkJiVik0B9LZFLSyY8iI2jrHmBaIkpH7yCrH9/FI1vb2LS7k72d/ZhBcsgZGEpfrTliMH/6NBbXV7J4VgXH1Vcyq7KEaMSIRoxZlaXMm16m2zTLlDVphhRPRgoVyYW+wSHWbTvAC209tLT30dzayeY9XWzd131oVFqm4VsDlCeiLJ1fw2nza1k0q4JENMLCunJmV5UqdGTSmsxDikWOCKXxKOcsmsk5h7UPJFM8v7+btu4BUilnMOVsb+vhqV0dJKJGW88gj207wN1P7H7JcrXT0sOgm2aUM7MiQfdA+qZox8wspyQWYVZVCYvqKqmepvvWyOSmUBHJoUQswnH1lePO19rRx/YDvfQnh2hu7WJjSyfb9nfzyNY29nX1U1ES40DPAId3emZWlFAzLY4B08sTzKoqpaYszr6ufmZUJHj7aY3MKC8hEoHK0jiVJRo+LYWl3V8ik1Tf4BA7DvQykEyx62Avz+3torm1i56BIZKpFG3dA7R29nOwZ5AZFQl2Hew9dFfOYWZQOy3B/OnTWDBjGrMqS3CHaSUxasri1JbHqSlLUD0tTu20BHNryjToQF5Gu79EjgCl8SiLZqWHMp80p4o3Uj/m/O29gzywqZVkyhlKpejsS9LRl2RvZz8vtHWzbtsB9nX1EzGjd3CIkf6eLIlFOLGhikQsQlVpjLk1ZcypKWNubRn1YeTbYDJFymFmZYKGqjKqymI6HiSHKFREjhDVZXEuOTW7qxUNpZyO3kEO9g5ysGeAgz2DtHUP8NSuDjbt7mAo5ew40MvDW9vo7EuOua5piSizq0uJRYwX2npoqC7juPoKppcnqJmWYEZ5goUzy6mrLKGiJEZFaYyhlJMcchqqS4lF1TM6kihURI5C0YhRW54It3ouP9T+9tNfPm9H3yC7Dvayp6OfiEEsEiFisK9rgJb2Xlra+2hp72VwyDln0Ux2Huhly95u1m07SHvvAINDo+9ij0eNaYkYKXdqpyVorC1j0awKYpEI0Ui6t1YajxKLGCmHhupSZlaUkEylaKwto2lGuUJpklGoiMiYqkrjVM2Oc8LsiS/r7hzoGWTrvm4OdA/Q1Z+ksz9JLGJEDLbu66F3IImZ0dY9wLb93fzkDztxh2TK6UuOvJsuU3kiSkVpLPSC0oMTSuMRegeHmFFeQmNtGXs6+qmdFuc1jdX0DAwxlHJqpsU5Zma6R9WfHKKmLKFdeTmgUBGRvDEzppcnXvE9cNzTJ5kmhxwz2HWwl/1dA0Qjxrb9PbzQ1kNXf5KuvuShwOrqG6StO0VpPMK6fQf42YZd1FWWcKBnkIFkasz3S8QizCxPMOROxIyZFSXMqiyhLBGloy9Je88AA0PO7KoS5tSUMauylEQsQjyaPtHVHSpLY8ytLaOxZhqliQi9A0P0Dg5Rnkgfozp8NN5Qytm2v5s5NWWUxqf+nU8VKiIyaZkZJbEowzcAXTSrkkWz0tPLmqZntY5UyolEjL7BIZ7f301VaZxYxNjfPUBzaxedfUkSsQgHewbY29nPvq4BYhFjyJ19Xf3s7uijZ2CIqrI4NdMSxCJGS3sf67cf5EDP4IQ+T1kC3KE6AAAH7ElEQVQ8SmVpjNJ4lJJY+tpyrR39dPYnKYlFWDKvhrrKEqZPS1BdFicejRCPGYlohGmJGOUl0fTPRJRpoUeWHHL6kylS7sypKSMeNVo7+qmvKmVmRaLgPS+Fiogc0YZ7BqXxKCfMfvH6b7OqSl92PbiJGgy9qMFUiqHQmzrYkz4GteNgL/3JFNPiUcoSUQ72DPLc3i66+5P0DQ7Rn0wRMeN1x8Y5eU41z+zu5Imd7Wzc1cH+7gE6+gbH3fU3nkQ0QnlJlPKSGA3Vpfzg/We/uhVmQaEiIvIKxaMR4lEo48XdVjXTEjTNLB9jqewNpZzBoRT9yRS9A0N0DyTp6Q8/B5L0DqSIR41ELIKZseNAD0MpZ1ZlCS3tfeleVv8Q3f3Jgp1/pFAREZmk0hclTY+AG75+3GSnsXgiIpIzChUREckZhYqIiOSMQkVERHJGoSIiIjmjUBERkZxRqIiISM4oVEREJGeOujs/mtleYNsrXHwmsC+H5eSK6pq4yVqb6poY1TVxr6S2Be5el82MR12ovBpmtjbbW2oWkuqauMlam+qaGNU1cfmuTbu/REQkZxQqIiKSMwqVibmp2AWMQnVN3GStTXVNjOqauLzWpmMqIiKSM+qpiIhIzihUsmBmK8zsGTNrNrNriljHPDN7wMyeNrOnzOzDof2fzWynma0Pj4uKVN/zZvZEqGFtaJtuZmvMbHP4WVvgmo7P2C7rzazDzD5SjG1mZreYWauZPZnRNuL2sbQbw3dug5mdVoTaPm9mm8L7/9jMakJ7k5n1Zmy7rxS4rlH/7czs2rDNnjGzCwtc1/czanrezNaH9kJur9F+RxTue+bueozxAKLAc8AxQAJ4HDipSLU0AKeF6UrgWeAk4J+B/z0JttXzwMzD2v4duCZMXwN8rsj/lruBBcXYZsAbgNOAJ8fbPsBFwD2AAcuBh4tQ2wVALEx/LqO2psz5ilDXiP924f/C40AJsDD8v40Wqq7DXv9/wD8VYXuN9juiYN8z9VTGdybQ7O5b3H0AuA24uBiFuHuLuz8WpjuBjcDcYtQyARcDq8L0KuCSItZyPvCcu7/Sk19fFXf/NdB2WPNo2+di4FZPewioMbOGQtbm7j9392R4+hDQmK/3n0hdY7gYuM3d+919K9BM+v9vQesyMwMuBb6Xj/ceyxi/Iwr2PVOojG8usD3j+Q4mwS9yM2sCTgUeDk0fDN3XWwq9iymDAz83s3VmdlVoq3f3ljC9G6gvTmkAXMZL/6NPhm022vaZbN+7vyb9F+2whWb2BzP7lZn9URHqGenfbrJssz8C9rj75oy2gm+vw35HFOx7plCZgsysAvgR8BF37wC+DBwLLAVaSHe9i+H17n4a8GbgajN7Q+aLnu5vF2W4oZklgLcCPwhNk2WbHVLM7TMWM/skkAS+E5pagPnufirwUeC7ZlZVwJIm3b/dYd7FS/94Kfj2GuF3xCH5/p4pVMa3E5iX8bwxtBWFmcVJf1m+4+53ALj7HncfcvcU8DXy1OUfj7vvDD9bgR+HOvYMd6fDz9Zi1EY66B5z9z2hxkmxzRh9+0yK752ZXQG8BXh3+GVE2L20P0yvI33s4rhC1TTGv13Rt5mZxYC3Ad8fbiv09hrpdwQF/J4pVMb3KLDYzBaGv3YvA1YXo5Cwr/ZmYKO7/2dGe+Y+0D8Hnjx82QLUVm5mlcPTpA/yPkl6W60Ms60E7ix0bcFL/nqcDNssGG37rAYuD6NzlgPtGbsvCsLMVgAfB97q7j0Z7XVmFg3TxwCLgS0FrGu0f7vVwGVmVmJmC0NdjxSqruCNwCZ33zHcUMjtNdrvCAr5PSvEiISp/iA9QuJZ0n9hfLKIdbyedLd1A7A+PC4CvgU8EdpXAw1FqO0Y0iNvHgeeGt5OwAzgfmAz8AtgehFqKwf2A9UZbQXfZqRDrQUYJL3v+srRtg/p0ThfDN+5J4BlRaitmfT+9uHv2lfCvG8P/8brgceAPytwXaP+2wGfDNvsGeDNhawrtH8TeP9h8xZye432O6Jg3zOdUS8iIjmj3V8iIpIzChUREckZhYqIiOSMQkVERHJGoSIiIjmjUBHJATMbspdeDTlnV7MOV7kt1nk0IhMSK3YBIkeIXndfWuwiRIpNPRWRPAr31ficmT0SHotCe5OZ/TJcFPF+M5sf2ustfe+Sx8Pj7LCqqJl9Ldwj4+dmVhbm/7tw74wNZnZbkT6myCEKFZHcKDts99c7M17rcPczgf8Grg9t/wWscvfXkr5Q442h/UbgV+6+hPT9Op4K7YuBL7r7ycBB0mdpQ/reGKeG9bw/Xx9OJFs6o14kB8ysy90rRmh/HjjP3beEC/3tdvcZZraP9OVFBkN7i7vPNLO9QKO792esowlY4+6Lw/NPAHF3/4yZ3Qt0AT8BfuLuXXn+qCJjUk9FJP98lOmJ6M+YHuLF46F/SvraTacD68JVckWKRqEikn/vzPj5P2H696SveA3wbuA3Yfp+4G8BzCxqZtWjrdTMIsA8d3+A9NWEa4CX9ZZECkl/1YjkRpmZrc94fq+7Dw8rLjGzh0n/Efeu0PYh4Btm9g/AXuC9of3DwE1mdiXpHsnfkr4a7kiiwLdD8BjwBXc/mLNPJPIK6JiKSB6FYyrL3H1fsWsRKQTt/hIRkZxRT0VERHJGPRUREckZhYqIiOSMQkVERHJGoSIiIjmjUBERkZxRqIiISM78f8QbMPBvf+66AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15538979128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualizing the cost\n",
    "plt.plot(range(nn.epochs), nn.eval_['cost'])\n",
    "plt.ylabel('Cost')\n",
    "plt.xlabel('Epochs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x15537f58a20>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VOXZ+PHvncm+hyQgECBhk002I1IRUVGLWqVqVVyq+FapvrbWtrbVLmq11rav9Wdbl6qtWpdqrXXBFlRUrBsqoOyLhD2sCdn3ZOb+/fEcYAhZhsBkAtyf68rFzHPOmbnnJJx7nuU8j6gqxhhjTFuiIh2AMcaYrs+ShTHGmHZZsjDGGNMuSxbGGGPaZcnCGGNMuyxZGGOMaVfYkoWIPCEiO0VkWSvbRUT+KCIFIrJERMYGbbtaRNZ4P1eHK0ZjjDGhCWfN4ilgShvbzwYGeT8zgEcARKQbcAdwIjAOuENEMsIYpzHGmHaELVmo6vtASRu7TAWeVucTIF1EegJfBeaoaomqlgJzaDvpGGOMCbPoCL53b2Bz0PNCr6y18jZlZWVpbm7uoYzPGGOOeAsXLixW1ez29otksjhoIjID14RF3759WbBgQYQjMsaYw4uIbAxlv0iOhtoC9Al6nuOVtVa+H1V9TFXzVTU/O7vdxGiMMaaDIpksZgJXeaOixgPlqroNeBM4S0QyvI7ts7wyY4wxERK2ZigReR44FcgSkULcCKcYAFX9MzALOAcoAGqAa7xtJSJyNzDfe6m7VLWtjnJjjDFhFrZkoaqXtbNdgRtb2fYE8EQ44jLGGHPg7A5uY4wx7bJkYYwxpl2WLIwxxrTrsL7PwhhjjiSBgLJkSzl1jX58UcLmkhoyEmPp0y2RxZvLSEuI4eRBWajCos1lLNzoxv70zkjggjE5YY3NkoUxxoRIVSmtaSTGJyTE+Ij2ucaZ2gY/a4uqWFtURawvil7pCXy2voQYnzB1dG++2FzKwo2lbCurI6BKSU0jK7ZW4A8EiIv2ERcTRUKMj9KaBnZU1LcZQ5RAQPctG9s33ZKFMcYcqOr6Jqrqm0iJjyYxdu9lLhBQVmyroLy2kZT4aOas2MGu6gbG9s0gMymWgCo7Kur5eG0x28vrGNE7jaKqenZW1JGTkciizWWsL67e83rRUUJ8jI/qhiZUW4oE7nx9BQC+KOGY1HiifUJSbDSnD8kmPsZHXaOfusYAtY1+BnRP5oyh3emREk+DP0BORiI7K+soLKllZJ80dlTU8+m6XSTHRzMwO5mTBmYR64uiKRAI6/kEEG3tEx5m8vPz1ab7MObIs7aoinVF1XRLiqWu0c+u6gZKquoprmqgKaAM75XKlrJaVm2rIDY6iqVbKli5rWLP8VnJcTT6A4iAKpTXNu7ZFiWQFBtNZX3TPu+ZnRJHn4wElm+tICs5jl7p8WwqqaF/VjKnD+kOQG2jn7pGP7WNftITYhnYPZkB3ZNoaAqwcVcNY/tlsKuqntnLtjO2bwaTBmcTG931uolFZKGq5re3n9UsjDFh1+gPsLmkhp2V9aTGx9DoD7C1rJaV2ypo8Cs90+LpmRZP99R46hv9/Pm/a1m1vZKEGB/rgr7JB4sS92290e++8PZOT6DRHyA3M4kfnjmYbsmxlFQ1UFhaS1xMlBeHkt8vg57p8RRXNTA+rxtZyXGsK66msq4RESEzKZbe6QlERUmHP+/InPQ9Me1+fLizZGGMOSCBgPLOqp1E+4SspDjqmvy8tXw7y7dWkBjrIzE2Gn9AWV9cTVKcj/gYHws2lFLb6N/vtZpf8HdLT4zhtGO7U1nXxOUn9uX4fhmU1TaSGOOjW1IsGUmxZCTG4g8oX+6oJDsljh6p8R3+TAO7J3f42KOFJQtjjmKBgFLb6CegSnJcNCLu2/TmkhoWbS6jYGcVBUVVFFfWowqTh3Zn/oYS3l65c5/XifEJw3ulUV7bSLXXpJOblURNvZ/iqgYuyc9hZE46PVLjqaxrJMYXRY/UeAb1SCbWF8Wu6ga2lddSVFlPdYOfU4/NJjU+pt34fVHCiN5ph/7EmP1YsjDmCNDQFMAfUBJifXvKGv0B1hZVsa28Dp8ISwrLWLW9EhEhSqCsppGFG0up8i7uyXHR9EqPJ8YXxfKtrs0/SqBPt0R6pMZT09jEvbNX4YsSbv/aMEbmpFFa00iUwJi+GXRLiu1w/NkpcWSnxB3cSTBhZcnCmMNIIKAEVPFFCZtKali+tYLP1pfw2qItVNY1kZ+bQXJcDDsq6li9o5KGpn1HyfTtlogvSgioEh/tY+roXvTLTEQVtpXXsbWsloq6Rm49ewinDMqmf3YS8TF7E9CaHZX4ooT+2dZsc7SxZGFMhNU2+CnYWUWfbgkUltby3uqdzFq6nfomP/0yk5g8tDvJcdF8tr6E2cu2U1LdQGx01J5EEOMTzhzWg5yMROat3UVFbRPdkmKZflIuw3ulkpORSJM/wIDuyWQlH9y390E9Ug7FRzaHIUsWxnSS8tpGCnZW4YsS3lq+nQ27qhmYncyLCwrZXlG3z775/TLol5nIym0VvLvK9Q/Ex0RxxtAeDMhOpqahif7ZyQzvlcrgHin7fPs3JhwsWRhziFXVN1Fa3QDAm8u3s7WsjoTYKJ79ZNOeMf5RAj3TEpi1dDvH9U7jx1OOZVdVA91T4zgxL5Nj0tzIHlVlzc4q/AFlYPdkYnxdb5y+OTpYsjCmg+qb/BRV1rOrqoHVOypZvb2SpVvK+XxjKU1B8zHEx0RR1xhg4qAsvjm+H/6AMrZfBj1S492dxHHRrY7pFxEGW9OP6QIsWRjTil1V9Xumd1hXXM36omo27KomLSGG7qnxfLZ+F3WNezuQ46KjOPaYFK6d2J/+WUnUNfmZMDCL/llJVNY3tTgUNC2h/eGhxnQFlizMUW17ed2eqSBKqxvpn53E5tIafvHqMuZvKN2zX0ZiDHlZSXylfybF1Q1sL6/lkvw+DO+VSkaim+qhX2YSvlZqCKHcM2BMVxbWZCEiU4A/AD7gL6r6m2bb++GWT80GSoArVbXQ2/Y74FzcmhtzgO/pkTKRlYmIukY/Oyrq6JEaT1x0FI++v47fzF61zz6x0VGgkJoQzQ/PHMz4AZkMzE4m4yDuITDmSBC2ZCEiPuAh4EygEJgvIjNVdUXQbvcBT6vq30TkdOBe4JsichIwARjp7fchMAl4L1zxmiPLrqp6HvtgHR+uKWZAdjLfntSfG579nE0lNQCkxLnJ4849rieTjs0mEFBSE2JYsKGUBr+fm88YfNDDTM0RSBWqdoL6ITETor2/kaZ6WP8B9J8Evhi3nwTVMss2QeV26DMOqopg8fNQsg4GngFDzoW6cohNBl+zS3JdhXuP3e9TXgjPXwY5+ZDSC2pL4KSbILVn2D96OGsW44ACVV0HICIvAFOB4GQxDPiB93gu8Kr3WIF4IBYQIAbYEcZYzRGgYGcVry3aQmZSLI++v47iqnpG90ln1tJtzFy8lbSEGO6aOpzKuiZ2VNSRl5XE1V/J3adz+Zzjwv+f7ohVXwWvfBuO+wYMv+DQvGbhAujWH/wN8PkzMGwqZA/eu72hBl6+DmIS4ZQf7d0WCECUN3KstgxqdkHmgP1ff/UbsPVzyP8WpPTYW169C3augOTukJYDsUlQugH+dR0Ufub2iYqGY46Dy/8Jb/0clrwAvY+HfhNg86fwzVdgVwFUF8O/b3YJ40frIDYRPv4TNFTBwichLg3qy+HCv8DIi2HdezDrR+58Vm5175M5EC76i3u//qfCZ49DUy2ID9a+C9P/A0lZh+actyKcyaI3sDnoeSFwYrN9FgMX4pqqLgBSRCRTVeeJyFxgGy5ZPKiqK5u/gYjMAGYA9O3b99B/AtNl+QPKltJavthcSsHOKuqbAjw9b8OeDuc+3RJ49cYJDO+VxoINJfz5v+v4wZmDGdYrNbKBH2lUIeB334jfuBVW/RvWzIG0Pu7bb8APH/0BGmvdt+L6Cjj1NjhmhDt+xwrYsRzyJrpv5oEmOPZsSEh3F8FnLoCoGHfBbKqFBX+Fm74AxF1oN38Gq/4D8akw0fve+fK3YemLkJELTQ1QUQi9xsKMuW77qv+Av9HFuvSfrmzs1e7fL9+CDR/AgifcxRzg9F/AKbe4xFJeCGfcCfHpUL4ZNACJ3dyFesRFUPAObFsMAya7JPHPa1wNAoUz74KkTPeaNy9xn2vx81A43yXE3JPdtrgU6DEcouMhaxA0VMPOlZCU7bafdTec9lN3rrYugrd+5j5PmIVtPQsR+QYwRVWv9Z5/EzhRVb8TtE8v4EEgD3gfuAgYAWThEsil3q5zgB+r6getvZ+tZ3HkqqhrZHNJDdvL69hUUsO7q3by6boSGvz7TmUxaXA2v71oJI3+ANkpcZG9Ua2uAr58AwZ/FeI7MNFdXbn7ZpnW2z1XdRfh937tXu/8ByG9z/7HVe5wF5Xd36o/+L371h0V7S7CaUGrqa3/YO8FauXr0FgDo6a5JpWXZ0Df8S6OFTOhthQufBTyToENH8Fnj8KE77lv/stehsv/AR/e75LCl29CXRlc8Kh7z7d+AR//EZJ7uBpCfSVMfci916618Kex+36GqBh3MU3p6b6BV26DpjoYfiH4613TzYqZ8OI33f5n3Anjb4Ror1/p82egaJW7mPtiofsw6DUaBpzuPs9vvC+WUdFw8g/c54hNcs1Gj0xwyWvY+TDqcpdo+p4EPYa55qO4FIhpZXZbVXdhDzRCQoZ3Xv8N/7jCJchTbz2Qv4DQBdeiOiDU9SzCmSy+Atypql/1nt8GoKr3trJ/MrBKVXNE5EdAvKre7W27HahT1d+19n6WLI4cjf7AnhXM5q8vYdbSbfvct5CbmcjkoT0Y1D2ZEb3TGNozFVXds8TlIeVvdG3QoWhqgBWvwbZFsOjvrj05vS9c/5G7yKx/H9bNhVWz3AUztSec9jMYOBlqSmDTJ+6bZOU2eOFK92172nMu8fzzavdNO72fa1JJOQZu/AyifFC6EZa/DEtfgh3LXFv2tW+7fX6X5y6Q4Jo7hp0P5/wf+OLg98e6i39dORTMgb5fgWtmuwvtMxe6JhCAPuMhox9M+olryil42317b6h236z7n+qSxe7VhXathXfvglN/Ct2H7HuOqothzh2uOWX89e5Ct/lT2PiRS1xR0bB6tjsv7V0AN86DopVw/DX79g+0+ftscs1L6ofsIRCTsO/2knUu2cYdwntbSta7Wk6oMXayrpAsooEvgcnAFmA+cLmqLg/aJwsoUdWAiNwD+FX1dhG5FLgOmIJrhnoDeEBVX2/t/SxZHP52J4nfv7WatUVuwZuU+GguPr4P+bkZ3gI5CfRIjdszlfZB2fCR+6Y64HTXdPDOXXDCdXDsFHcR+/fNsOJVuPgpt0+wqp2unbpqh2tymHCTSwIvXOYuxHmnwMhLoWyDa0sH+N0A9w29/yRI7Q0bP4aTvw9jv+kuwM9e5L24QPaxcO79kDsBti2Bv1/qmkLGXgUVW6BiK/Q7yTWNPO9VwHNOcDWZ0o1w/p/cxSkQcN/yq3bC6ze5z3nVTNfR+sF98O6vXHPHGb+EE67d28Hqb3JNRlHRromnuaqd8Pw0F8e3P4Dk7IP/fZiIiHiy8II4B3gAN3T2CVW9R0TuAhao6kyvqepeXIf2+8CNqlrvjaR6GDjF2/aGqv6g5XdxLFkcXlSVT9eXMHvpNmoa/BRV1bNsSznFVQ3kZSVx29lDGN03nbSEGOKiQ2xOqilxnZCqkDVwb/NPbalr2ih4210YB57hLorPXOB9W+8LFdtc88GM9+CYUfDKDNeenZQN1UUw7e9u1ErpBnj//9xFur7SfXPuMQK+/rB7r02fuk7O5qNaALZ+4WoGid3cc3+Tu6BH+dxr7VgOWz53yeCUW/Y2ZTTVg0S1XMOp3AHLX3FJolte2+cnEICGyn2bxdZ/AKm9Wu78bU8g4PoRYpMO/FjTZXSJZNGZLFl0fQU7qygsrWFTSQ3PfbKJ1TsqSYz1kZ4QQ3piLIN7JPO1kb04bUj3Vm9uo67CtUMHtxsXvAOv3eiab3aLT3PfoHuNhiemwKZ5+77Opc+6GsGCJ9xomLhUmHz73gv574e4msHEH7oaR58T3UiV2bfCkn+4hHDW3dB96KE9ScZ0MksWJuLWFVXxxaYyFm0u45N1u1izs2rPtmE9U5l+Ui7njeq1z4I9raqrcKNqPv6jGwXSbQAM/zqc/nPYuQo+esB1ZGYO8DqD34Rzfu86PRf/A3atcU1D5YVuBMqU37gRJ62pr4K4FtZs2P3/pYu2PxtzoCxZmE4XCCjVDU3ERfu47eWl/OvzQgCSYn2M7pvOV4cfw/BeaaQlxDAgO2n/foeKrW70TqH3eww0wfHTYdx18M7dro198Nmuc3TnCtf8c9ptnfshjTnChJosbG4oc9DqGv289kUhn819jX+V9ictIZbx9R8xs08BiefcQ17fPvh8PncH64e/geMuBo7f/4Wen+aNuT/F9TGAOwbcGPbex7uhmPat3phOZzUL02GbS2p49tON/GP+Zr5a/xa/jXmc14bcR2VZEVdu/63bKaWnG9lz6XNQtBqenOLG2mfkuhulGqrcaKPYJNfZnJHXfketMeaQsZqFOXgBv7tLtrjAjRTKHAhjvsnOhmgeeW8tn38yl29F/YcPYleQElOG5p3C1Eu+Ba9eD8f+BPImwYtXuSkJfDGQczzcssbdObvsX7B9iRudVLxm701TxpguyWoWxnX6xibtHaq5exI0VfhdfwJNddQGYkjyu5u7pjX8nM90GDO7P8rQ+qX4jp0CPUe5O3IT0vd97cY6NwmaNR0Z0yVZzcK0rKoINn8CQ77mLuBVO+GPY9xQ09N+5r7tr30X/vcTdtYKfxv0DI8vrECBq3pv5Yyq/3DJcTn8auIkBsaNdhOttXWHc2tTIxhjDiuWLI4mO1fBcxdD+SaYcLObUye5O5z3B5j3kLtjOSaRqtwz+eUz7/NygR+Ab4zN4buTB5KTkQhcG/SCLQwtNcYckSxZHC3qK+Gpc93dwsMvcPclpPdxUzyMvpzF6ZN5753ZlKQO4aWlZfii4NqJeVx2Ql9ys+wOXWOOdpYsjkQl69xCKmWbYN6Dbg2A4Re4GkSP4W7Kie7DITWHbeW1vPLFFh6Ys4aE2F6oljOqTzr3XTyKXukJ7b+XMeaoYMniSBIIwD+uhNX/2VsWl+qmrQAY+rU9xbVf+QG/nrWSZ558F4DTjs3m/ktG2/KhxpgWWbI4nNSUuFlD8ybtO31zxTY33XVUlJvu4vSfu5oFAmOu3G/aiiWFZdz8j0WsK6pm+km5XDm+HwO7W/+DMaZ1liwOB031bpnFz5929yVc966bb3/md/Yu//g/b0KfE9zkdi2oaWjivdVFzFq6jTeWbScrOY7nrj2RCQPDuxSjMebIYMmiKwv4YfUs+OQRtzjMCdfCyGnujuinvuaGueadAgNOdYvTtKCirpE/vL2G5z7dSF1jgMykWK44sS/fP3Mw6YnW5GSMCY0li65MouDVG90CPbsXcwe34trmT+Drf4ZRl7Z6+PriaqY9No+dlfVcNDaHi8bmMC6vW+vTfxtjTCssWXRFxWvcgjzRcXDtHLducvACMxVb3brBIy/Z79C6Rj9vLt9ObYOfP71bQKNfefV/JzCqT/p++xpjTKgsWXQFRavdAvRDz4PtS2HuPW5Rncued01OzY2a1uLLvLNyBz97ZRnbK+oAtyTp89eNZ0TvtBb3N8aYUFmyiLRAAF7/Huxc6RauL5jjys/61QG9zAdrirj+2YUM7J7C/108krysJNISYkiJb2MqDmOMCVFU+7t0nIhMEZHVIlIgIre2sL2fiLwjIktE5D0RyQna1ldE3hKRlSKyQkRywxlrxHz+N7fk51fvcT8IZA+BY88J6XBV5Z8LNjPj6YUMyE7mhRnjmTgom5yMREsUxphDJmw1CxHxAQ8BZwKFwHwRmamqK4J2uw94WlX/JiKnA/cC3/S2PQ3co6pzRCQZCIQr1k5XVQTJ2bDwKTcfU+5EGH2Fm9hv6oMuWUS1nceXbSnnphe+oLS6gdKaRk7M68afLh9DWoIlCGPMoRfOZqhxQIGqrgMQkReAqUBwshgG/MB7PBd41dt3GBCtqnMAVLWKI0FdBXx4P8z/K/zPG66fIibRTcOxewrvMVe2+zJFlfXMeHoBflXOOa4nQ3qmcvm4vjbKyRgTNuFMFr2BzUHPC4ETm+2zGLgQ+ANwAZAiIpnAYKBMRF4G8oC3gVtV1R98sIjMAGYA9O3bNxyf4dCpq4CHx0PFFnevRNaxcMU/obYMkjJDfpnP1pdw67+WUFLTwEvXn2Sd18aYThHWPosQ3AJMEpEvgEnAFsCPS2ITve0nAP2B6c0PVtXHVDVfVfOzs7M7LegOWT3LJYrLXoALHwVftJsBNsREUVHXyE9fWcolj86jwR/gieknWKIwxnSacNYstgB9gp7neGV7qOpWXM0Cr1/iIlUtE5FCYFFQE9arwHjgr2GMN7yWvwJpfWDwlAM+tKahiYsfmceanZVce3IePzhrMImxNpDNGNN5wnnFmQ8MEpE8XJKYBlwevIOIZAElqhoAbgOeCDo2XUSyVbUIOB04vNdMHTUNGmsPaHlRf0CpqG3k7v+s4MudlTw5/QROPbZ7GIM0xpiWhS1ZqGqTiHwHeBPwAU+o6nIRuQtYoKozgVOBe0VEgfeBG71j/SJyC/COiAiwEHg8XLGGxar/wOIX3JQdvce6O64PQEVdI5f8eR6rtlcC8P0zBluiMMZEjKhqpGM4JPLz83XBgi5S+VCFPx0PNcWQlA1Zg93d2CEKBJQZzyzkvdU7+f6Zg+mXmcjZI3raaCdjzCEnIgtVNb+9/azhOxy2L4WStW5I7PHTXfI4AA/NLeDtlTu487xhTJ+QF54YjTHmAFiyCIeKrW4iwCHnuech9lOUVjfwYUEx97/9JReM6c3VJ+WGL0ZjjDkAlizC4dgpMPirB9SZ/fS8Ddz+2nIAhvZM5dcXHIccwPHGGBNOliwOtYJ3oO9XIDYx5EO2ltXym9mrGN+/G1ec2I9TBmeTEOsLY5DGGHNgIn1T3pFl5evw3Dfgv78J+RBV5a7XVxBQ5f++MYrzRvWy+Z2MMV2OJYtDpbYUZn4Xeo2BST8J+bAXF2zmjeXbuWnyIPp0C702YowxncmaoQ6V//4O6srhvD/uu6pdK1SV974s4vbXlnPywCy+fcqATgjSGGM6xpJFR9WVwxfPulliS9bBZ4/B2KvhmBHtHlrf5Oebf/2Mz9aX0KdbAg9MG233UBhjujRLFh318YPw/u+gbBOc/AMYeCac/vOQDn32k018tr6En587lCvH9yM+xjqzjTFdmyWLjvA3wudPQ1waTL7DjXy6/IWQDq2oa+TBd9cwcVAW107sH+ZAjTHm0LBk0RGrZ0PVdjfd+AEMkV1SWMav/r2S0ppGfjJlSBgDNMaYQ8uSRUcsfBJSc2DQWSEfsnp7JRc+/DGpCTH85sLjbC0KY8xhxZJFR5x5txsqGxV6X8O9s1eSGOtjzvdPITM5LozBGWPMoWfJoiNCGPEU7KOCYt5bXcRPzxliicIYc1iym/IO1KK/w5dvhrx7dX0TP3tlKTkZCVz1ldzwxWWMMWFkyeJAqMI7d8Oi50I+5O5/r2BjSQ2/v3iUDZE1xhy2LFkciB3LoXKru6ciBC8u2MwL8zdz/aQBnNg/M8zBGWNM+FiyCJW/Ed64FXxxIY2C+mTdLn768lImDsriB2cO7oQAjTEmfMKaLERkioisFpECEbm1he39ROQdEVkiIu+JSE6z7akiUigiD4YzzpDMuR02fOBWv0vp0eauu2eS7Z2RwIOXjyXGZznZGHN4C9tVTER8wEPA2cAw4DIRGdZst/uAp1V1JHAXcG+z7XcD74crxgNyzEj46q9h9GXt7rq4sJwV2yq4bmJ/m27cGHNECOfQ2XFAgaquAxCRF4CpwIqgfYYBP/AezwVe3b1BRI4HegBvAO0uJh52ISSJ3f7+6UYSY31MHd0rjAEZY0znCWf7SG9gc9DzQq8s2GLgQu/xBUCKiGSKSBTwe+CWtt5ARGaIyAIRWVBUVHSIwm7B2rlQXhjSrqu2V/D64m1MHd2LlHirVRhjjgyRbky/BZgkIl8Ak4AtgB/4X2CWqrZ5hVbVx1Q1X1Xzs7OzwxOhvwleusYNmW3Ho/9dyzl/+IC4mCi+dXJeeOIxxpgICGcz1BagT9DzHK9sD1XdilezEJFk4CJVLRORrwATReR/gWQgVkSqVHW/TvKwK/zMTe1x7JQ2d9tSVsvv3/qS047tzn0XjyIjKbaTAjTGmPALZ7KYDwwSkTxckpgGXB68g4hkASWqGgBuA54AUNUrgvaZDuRHJFEArPw3+GJhwOQ2d3vw3QIA7vr6CEsUxpgjTtiaoVS1CfgO8CawEnhRVZeLyF0icr6326nAahH5EteZfU+44umQQABWvAoDz4D41FZ321JWyz8XbGbauD70Tk/oxACNMaZzhHUiQVWdBcxqVnZ70OOXgJfaeY2ngKfCEF77ilZCxVY44842d3tz2XaaAmr9FMaYI5bNOtuWHsPhh6sgrvVaBcAHa4rIy0qiX2ZSJwVmjDGdK9Kjobq+lGPaXA2vvsnPJ+tKmDgoqxODMsaYztVushCR74pIRmcE06WsfRee/nq791cs3FBKbaOfUwaFaeiuMcZ0AaHULHoA80XkRW+uJwl3UF3C58/AtkWQ1HYS+O+aIqKjhPEDbFZZY8yRq91koao/BwYBfwWmA2tE5NciMiDMsUVObSms+g8cdwlEt76y3Y6KOl7+fAsn5HYjOc66f4wxR66Q+ixUVYHt3k8TkAG8JCK/C2NskbPsZfDXw5grW92lrtHPjKcXUF3fxC++1nx+RGOMObK0+3VYRL4HXAUUA38BfqSqjd78TWuAH4c3xAhY8xZ06w89R7a6y0NzC1hcWM7jV+UzrFfbo6WMMeZwF0rbSTfgQlXdGFyoqgER+Vp4woqwvEnQ/7RWN28uqeHR99cxdXQvzhzW9toWxhhzJAglWcwGSnY/EZFUYKiqfqqqK8MWWSR95X80Ya1CAAAdbElEQVTb3Py7N1cTJfCTKUM6KSBjjImsUPosHgGqgp5XeWVHpqIvoa6i1c1lNQ3MXrqNK0/sRy+b2sMYc5QIJVmI18ENuOYnjuQ7v1+9Hv5+Saub56zYQVNAOW+ULWxkjDl6hJIs1onITSIS4/18D1gX7sAioqEGtn4BuSe3usvsZdvpnZ7AyJy0TgzMGGMiK5RkcT1wEm6a8ULgRGBGOIOKmIotoAHIHNTy5rpGPlhTxDnHHcPRcm+iMcZACM1JqroTtxbFka9sk/s3LafFzXNX7aTRr0wZ0bMTgzLGmMgL5T6LeOBbwHAgfne5qv5PGOOKjN3zQKX3aXHzB2uKSU+MYUyf9E4MyhhjIi+UZqhngGOArwL/xS2PWhnOoCKm/yS44FFI2b/zWlX5uKCYkwZkEhVlTVDGmKNLKMlioKr+AqhW1b8B5wLHhTesCMnIhVHTwLd/hWt9cTVby+s4aYBNRW6MOfqEkiwavX/LRGQEkAbkhvLi3iy1q0WkQET2W0NbRPqJyDsiskRE3hORHK98tIjME5Hl3rZLQ/w8B2fdf2HHihY3fbR2FwATBlqyMMYcfUJJFo9561n8HJgJrAB+295BIuIDHgLOBoYBl4lI8xn37gOeVtWRwF3AvV55DXCVqg4HpgAPiEj4Owpevwk++H2Lmz4uKKZXWjy5ma0vhGSMMUeqNpOFN1lghaqWqur7qtpfVbur6qMhvPY4oEBV16lqA/ACMLXZPsOAd73Hc3dvV9UvVXWN93grsBMI7+pCgQCUb2mxc7vJH+DjtbuYMDDLhswaY45KbSYL727t73TwtXsDm4OeF3plwRYDF3qPLwBSRGSfVYREZBwQC6ztYByhqdoBgcYWh80u2FhKeW0jpw/pHtYQjDGmqwqlGWqOiNwiIn1EpNvun0P0/rcAk0TkC2AS7sY//+6NItITNxrrGi9x7UNEZojIAhFZUFRUdHCRlHt5La3vfpveWbmDWF8UEwfb0qnGmKNTKHM87b6f4sagMgX6t3PcFiC4TSfHK9v7Iq6J6UIAEUkGLlLVMu95KvAf4Geq+klLb6CqjwGPAeTn52tL+4RsT7LYv2bxzsqdnNjfVsMzxhy9QrmDO6+Drz0fGCQiebgkMQ24PHgHEckCSrxaw23AE155LPAKrvP7pQ6+/4Hpfxpc/W+36FGQtUVVrCuu5uqTcjslDGOM6YpCuYP7qpbKVfXpto5T1SYR+Q7wJuADnlDV5SJyF7BAVWcCpwL3iogC77O39nIJcAqQKSLTvbLpqrqo/Y/UQYndIG/ifsXvf+mat6y/whhzNAulXeWEoMfxwGTgc6DNZAGgqrOAWc3Kbg96/BKwX81BVZ8Fng0htkNnw0dQUwzD9h2wtWxLBdkpcfTpZkNmjTFHr1Caob4b/Ny73+FvYYsoUhY+BYXz90sWK7ZVMLSnrbFtjDm6hTIaqrlqYPChDiTiGmsgZt/aQ0NTgIKdlQyzZGGMOcqF0mfxOm70E7jkMgx4MZxBRURjLcTsu0xqwc4qGv3KsF6WLIwxR7dQ+izuC3rcBGxU1cIwxRM5LSSLFdvcWtxWszDGHO1CSRabgG2qWgcgIgkikquqG8IaWWdrrIHkHvsUrdhaQXxMFHlZSREKyhhjuoZQksU/ccuq7ub3yk5oeffD1CX799mv2FbOkGNS8dn6FcaYo1woHdzR3kSAAHiPY8MXUoRk5Lofj6qycluljYQyxhhCSxZFInL+7iciMhUoDl9IEfLZ47Dx4z1PN5fUUl7byHG90yIYlDHGdA2hJIvrgZ+KyCYR2QT8BPh2eMOKgDm3w+q99w8u2VIGwMgcSxbGGBPKTXlrgfHeRH+oalXYo+psqvvdZ7GksJxYXxSDe6REMDBjjOka2q1ZiMivRSRdVatUtUpEMkTkV50RXKdpqnP/Bg2dXVJYxtBeqcRGd+S+RWOMObKEciU8e/e04QCqWgqcE76QIqChxv3r1SwCAWXZlgpGWn+FMcYAoSULn4jE7X4iIglAXBv7H34adycLV7NYv6uaqvomjrP+CmOMAUK7z+I54B0ReRIQYDpH2kSCKT3h+ysg3g2TXVpYDljntjHG7BZKB/dvRWQxcAZujqg3gX7hDqxT+aIhbe/y4Ku2VxLjEwZmJ0cwKGOM6TpC7b3dgUsUFwOnAyvDFlEklG6E934LZZsAKCytoXd6AtE+69w2xhhoo2YhIoOBy7yfYuAfgKjqaZ0UW+fZVQDv/Rr6nwrpfSksrSUnwxY7MsaY3dr66rwKV4v4mqqerKp/ws0LdeRprHX/eh3cLlkktHGAMcYcXdpKFhcC24C5IvK4iEzGdXCHTESmiMhqESkQkVtb2N5PRN4RkSUi8p6I5ARtu1pE1ng/Vx/I+x6wxr1DZ+sa/RRX1VuyMMaYIK0mC1V9VVWnAUOAucDNQHcReUREzmrvhUXEBzwEnI1bMOkyERnWbLf7gKdVdSRwF3Cvd2w34A7gRGAccIeIZBzohwtZ0NDZwlJXy7BmKGOM2avdHlxVrVbVv6vqeUAO8AVufqj2jAMKVHWdN1PtC8DUZvsMA971Hs8N2v5VYI6qlng3Ac4BpoTwnh2zuxkqNpHCUpc4rGZhjDF7HdBwH1UtVdXHVHVyCLv3BjYHPS/0yoItxjV3AVwApIhIZojHHjonXAs/2QhxaVazMMaYFkR6bOgtwCQR+QKYBGzhADrRRWSGiCwQkQVFRUUdj8IXAwnpEBVFYWktMT6he8qRdZO6McYcjHAmiy1An6DnOV7ZHqq6VVUvVNUxwM+8srJQjvX2fUxV81U1Pzs7u+ORrnwd3r0H2HuPRZStjmeMMXuEM1nMBwaJSJ6IxALTgJnBO4hIlojsjuE24Anv8ZvAWd4MtxnAWV5ZeKx9FxY+CWD3WBhjTAvClixUtQn4Du4ivxJ4UVWXi8hdQSvvnQqsFpEvgR7APd6xJcDduIQzH7jLKwuPxlq7x8IYY9oQykSCHaaqs4BZzcpuD3r8EvBSK8c+wd6aRng1VO9zj0XvdEsWxhgTLNId3F1DYy3EJLKrugGAbOvcNsaYfViyALdSXkwiu6rqAchKtmRhjDHBwtoMddiY/m8I+Cn+shiAzOTYCAdkjDFdi9UsdovyUVzlmqGsZmGMMfuymgXAW7+A7sMorjoBsJqFMcY0ZzULgMXPw+ZP2VXVQGKsj8RYy6HGGBPMkgXsHQ1VVW9NUMYY0wJLFqpuivKYBIqrGqwJyhhjWmDJwt8AGoDYRIqtZmGMMS2yZNFYC3FpEJtCcVUDWVazMMaY/VhPbkI63LaJQEApeXUWmUlWszDGmOasZuEpq20koFjNwhhjWmDJwlPsTfWRaX0WxhizH0sWnmKbF8oYY1plycKzd6oPa4YyxpjmLFl4dlkzlDHGtMqShWdXVQO+KCE9ISbSoRhjTJdjycJTVd9EUqyPqCiJdCjGGNPlhDVZiMgUEVktIgUicmsL2/uKyFwR+UJElojIOV55jIj8TUSWishKEbktnHEC1Df5iY/xhfttjDHmsBS2ZCEiPuAh4GxgGHCZiAxrttvPgRdVdQwwDXjYK78YiFPV44DjgW+LSG64YgWobwwQG20VLWOMaUk4r47jgAJVXaeqDcALwNRm+yiQ6j1OA7YGlSeJSDSQADQAFWGMlXp/gDhLFsYY06JwXh17A5uDnhd6ZcHuBK4UkUJgFvBdr/wloBrYBmwC7lPVkuZvICIzRGSBiCwoKio6qGDrGwPERVszlDHGtCTSX6UvA55S1RzgHOAZEYnC1Ur8QC8gD/ihiPRvfrCqPqaq+aqan52dfVCB1Df5iYuJ9OkwxpiuKZxXxy1An6DnOV5ZsG8BLwKo6jwgHsgCLgfeUNVGVd0JfATkhzFWGpoCxPosWRhjTEvCeXWcDwwSkTwRicV1YM9sts8mYDKAiAzFJYsir/x0rzwJGA+sCmOs1DcFiLPRUMYY06KwJQtVbQK+A7wJrMSNelouIneJyPnebj8ErhORxcDzwHRVVdwoqmQRWY5LOk+q6pJwxQpesrAObmOMaVFY17NQ1Vm4juvgstuDHq8AJrRwXBVu+GynaWjy29BZY4xphV0dPVazMMaY1tnV0eOShfVZGGNMSyxZeBqsZmGMMa2yq6OnvslvycIYY1phV0dAVa3Pwhhj2mBXR6DRr6hio6GMMaYVdnUEGvwBAOvgNsaYVliyAOob/QA2N5QxxrTCro64YbOAzQ1ljDGtsKsjbtgsWM3CGGNaY1dH9tYsrM/CGGNaZskCd48FWDOUMca0xq6OWDOUMca0J6yzzh4urBnKmK6nsbGRwsJC6urqIh3KESE+Pp6cnBxiYmI6dLwlC4KaoeymPGO6jMLCQlJSUsjNzUVEIh3OYU1V2bVrF4WFheTl5XXoNezqSFAzlCULY7qMuro6MjMzLVEcAiJCZmbmQdXS7OpIcDOUnQ5juhJLFIfOwZ5LuzoC9Y27O7itz8IY45SVlfHwww8f8HHnnHMOZWVlbe5z++238/bbb3c0tIgIa7IQkSkislpECkTk1ha29xWRuSLyhYgsEZFzgraNFJF5IrJcRJaKSHy44rShs8aY5lpLFk1NTW0eN2vWLNLT09vc56677uKMM844qPg6W9iujiLiAx4CzgaGAZeJyLBmu/0ceFFVxwDTgIe9Y6OBZ4HrVXU4cCrQGK5Y623orDGmmVtvvZW1a9cyevRoTjjhBE477TQuv/xyRo4cCcDXv/51jj/+eIYPH85jjz2257jc3FyKi4vZsGEDQ4cO5brrrmP48OGcddZZ1NbWAjB9+nReeumlPfvfcccdjB07luOOO45Vq1YBUFRUxJlnnsnYsWP59re/Tb9+/SguLu7ks7BXOEdDjQMKVHUdgIi8AEwFVgTto0Cq9zgN2Oo9PgtYoqqLAVR1VxjjtD4LY7q4X76+nBVbKw7paw7rlcod5w1vdftvfvMbli1bxqJFi3jvvfc499xzWbZs2Z7RRE888QTdunWjtraWE044gYsuuojMzMx9XmPNmjU8//zzPP7441xyySX861//4sorr9zvvbKysvj88895+OGHue+++/jLX/7CL3/5S04//XRuu+023njjjX0SUiSE8+rYG9gc9LzQKwt2J3CliBQCs4DveuWDARWRN0XkcxH5cUtvICIzRGSBiCwoKirqcKA2kaAxpj3jxo3bZ9jpH//4R0aNGsX48ePZvHkza9as2e+YvLw8Ro8eDcDxxx/Phg0bWnztCy+8cL99PvzwQ6ZNmwbAlClTyMjIOISf5sBF+j6Ly4CnVPX3IvIV4BkRGeHFdTJwAlADvCMiC1X1neCDVfUx4DGA/Px87WgQDU0BYqOjbOSFMV1UWzWAzpKUlLTn8Xvvvcfbb7/NvHnzSExM5NRTT21xWGpcXNyexz6fb08zVGv7+Xy+dvtEIiWcX6W3AH2Cnud4ZcG+BbwIoKrzgHggC1cLeV9Vi1W1BlfrGBuuQG39bWNMcykpKVRWVra4rby8nIyMDBITE1m1ahWffPLJIX//CRMm8OKLLwLw1ltvUVpaesjf40CE8wo5HxgkInkiEovrwJ7ZbJ9NwGQAERmKSxZFwJvAcSKS6HV2T2Lfvo5DytbfNsY0l5mZyYQJExgxYgQ/+tGP9tk2ZcoUmpqaGDlyJL/4xS8YP378IX//O+64g7feeouxY8cye/ZsevbsSUpKyiF/n1CJaodbb9p/cTcU9gHABzyhqveIyF3AAlWd6Y2OehxIxnV2/1hV3/KOvRK4zSufpaot9lvslp+frwsWLOhQnLf8czHz1u7io1tP79DxxphDb+XKlQwdOjTSYURMfX09Pp+P6Oho5s2bxw033MCiRYsO6jVbOqdeE39+e8eGtc9CVWfhmpCCy24PerwCmNDKsc/ihs+GndUsjDFdzaZNm7jkkksIBALExsby+OOPRzSeSHdwdwn1jX6bRNAY06UMGjSIL774ItJh7GFXSKxmYYwx7bErJG7orK1lYYwxrbNkgRs6a81QxhjTOrtCYs1QxhjTHrtC4jVD2SSCxpiDkJycDMDWrVv5xje+0eI+p556Ku0N8X/ggQeoqanZ8zyUKc87g10hcTULmxfKGHMo9OrVa8+Msh3RPFmEMuV5Z7ArJLun+7AObmPMXrfeeisPPfTQnud33nknv/rVr5g8efKe6cRfe+21/Y7bsGEDI0aMAKC2tpZp06YxcuRILr300n3mhrrhhhvIz89n+PDh3HHHHYCbnHDr1q2cdtppnHbaacDeKc8B7r//fkaMGMGIESN44IEH9rxfa1OhH0p2nwXWDGXMYeHJc/cvG/51GHcdNNTAcxfvv3305TDmCqjeBS9ete+2a/7T5ttdeuml3Hzzzdx4440AvPjii7z55pvcdNNNpKamUlxczPjx4zn//PNbnYT0kUceITExkSVLlrBkyRLGjt07xd0999xDt27d8Pv9TJ48mSVLlnDTTTdx//33M3fuXLKysvZ5rYULF/Lkk0/y6aefoqqceOKJTJo0iYyMjJCnQj8YdoXEmqGMMfsbM2YMO3fuZOvWrSxevJiMjAyOOeYYfvrTnzJy5EjOOOMMtmzZwo4dO1p9jffff3/PRXvkyJF7Fk4Cl3zGjh3LmDFjWL58OStWtD393YcffsgFF1xAUlISycnJXHjhhXzwwQdA6FOhHwyrWeCNhrKahTFdW1s1gdjEtrcnZbZbk2jJxRdfzEsvvcT27du59NJLee655ygqKmLhwoXExMSQm5vb4tTk7Vm/fj333Xcf8+fPJyMjg+nTp3fodXYLdSr0g3HUXyGb/AH8AbU+C2PMfi699FJeeOEFXnrpJS6++GLKy8vp3r07MTExzJ07l40bN7Z5/CmnnMLf//53AJYtW8aSJUsAqKioICkpibS0NHbs2MHs2bP3HNPa1OgTJ07k1Vdfpaamhurqal555RUmTpx4CD9t2476mkWD35ZUNca0bPjw4VRWVtK7d2969uzJFVdcwXnnnUd+fj6jR49myJAhbR5/ww03cM011zBy5EhGjx7NuHHjABg1ahRjxoxh+PDh9O/fnwkT9s6nOmPGDKZMmUKvXr2YO3funvKxY8cyffr0Pa9x7bXXMmbMmLA0ObUkrFOUd6aOTlFeWt3AmLvncMd5w7hmQl77BxhjOsXRPkV5OBzMFOVH/dfpKBHOHdmT/tnJkQ7FGGO6rKO+GSotMYaHLg/biq3GGHNEOOprFsYYY9pnycIY02UdKX2qXcHBnsuwJgsRmSIiq0WkQERubWF7XxGZKyJfiMgSb83u5turROSWcMZpjOl64uPj2bVrlyWMQ0BV2bVrF/Hx8R1+jbD1WYiID3gIOBMoBOaLyExv3e3dfg68qKqPiMgw3HrduUHb7wdmY4w56uTk5FBYWEhRUVGkQzkixMfHk5OT0+Hjw9nBPQ4oUNV1ACLyAjAVCE4WCqR6j9OArbs3iMjXgfVAdRhjNMZ0UTExMeTl2XD2riKczVC9gc1Bzwu9smB3AleKSCGuVvFdABFJBn4C/LKtNxCRGSKyQEQW2LcPY4wJn0h3cF8GPKWqOcA5wDMiEoVLIv9PVavaOlhVH1PVfFXNz87ODn+0xhhzlApnM9QWoE/Q8xyvLNi3gCkAqjpPROKBLOBE4Bsi8jsgHQiISJ2qPhjGeI0xxrQibNN9iEg08CUwGZck5gOXq+ryoH1mA/9Q1adEZCjwDtBbg4ISkTuBKlW9r533KwLantWrbVlA8UEcHy4W14HpqnFB143N4jowXTUu6Fhs/VS13aaZsNUsVLVJRL4DvAn4gCdUdbmI3AUsUNWZwA+Bx0Xk+7jO7unawewVyodti4gsCGV+lM5mcR2YrhoXdN3YLK4D01XjgvDGFtbpPlR1Fq7jOrjs9qDHK4AJzY9rtv+dYQnOGGNMyCLdwW2MMeYwYMlir8ciHUArLK4D01Xjgq4bm8V1YLpqXBDG2I6Y9SyMMcaEj9UsjDHGtOuoTxbtTXbYiXH08SZVXCEiy0Xke175nSKyRUQWeT/ntPdaYYpvg4gs9WJY4JV1E5E5IrLG+zejk2M6Nui8LBKRChG5ORLnTESeEJGdIrIsqKzF8yPOH72/uSUiErYFVVqJ6/9EZJX33q+ISLpXnisitUHn7c/hiquN2Fr93YnIbd45Wy0iX+3kuP4RFNMGEVnklXfaOWvjGtE5f2eqetT+4Ib0rgX6A7HAYmBYhGLpCYz1Hqfg7lEZhrub/ZYucK42AFnNyn4H3Oo9vhX4bYR/l9uBfpE4Z8ApwFhgWXvnBzdbwWxAgPHAp50c11lAtPf4t0Fx5QbvF6Fz1uLvzvu/sBiIA/K8/7e+zoqr2fbfA7d39jlr4xrRKX9nR3vNYs9kh6raAOye7LDTqeo2Vf3ce1wJrGT/ubS6mqnA37zHfwO+HsFYJgNrVfVgbszsMFV9HyhpVtza+ZkKPK3OJ0C6iPTsrLhU9S1VbfKefoKbXaHTtXLOWjMVeEFV61V1PVCA+//bqXGJiACXAM+H473b0sY1olP+zo72ZBHKZIedTkRygTHAp17Rd7xq5BOd3dQTRIG3RGShiMzwynqo6jbv8XagR2RCA2Aa+/4H7grnrLXz05X+7v6HfZcByBO3vsx/RWRihGJq6XfXVc7ZRGCHqq4JKuv0c9bsGtEpf2dHe7LocsTNuPsv4GZVrQAeAQYAo4FtuCpwJJysqmOBs4EbReSU4I3q6r0RGVonIrHA+cA/vaKucs72iOT5aY2I/AxoAp7zirYBfVV1DPAD4O8iktra8WHS5X53zVzGvl9KOv2ctXCN2COcf2dHe7IIZbLDTiMiMbg/gudU9WUAVd2hqn5VDQCPE6aqd3tUdYv3707gFS+OHburtd6/OyMRGy6Bfa6qO7wYu8Q5o/XzE/G/OxGZDnwNuMK7wOA18ezyHi/E9QsM7sy42vjddYVzFg1cCPxjd1lnn7OWrhF00t/Z0Z4s5gODRCTP+3Y6DZgZiUC8ttC/AitV9f6g8uA2xguAZc2P7YTYkkQkZfdjXAfpMty5utrb7Wrgtc6OzbPPt72ucM48rZ2fmcBV3miV8UB5UDNC2InIFODHwPmqWhNUni1uhUtEpD8wCFjXWXF579va724mME1E4kQkz4vts86MDTgDWKWqhbsLOvOctXaNoLP+zjqjF78r/+BGDHyJ+0bwswjGcTKu+rgEWOT9nAM8Ayz1ymcCPSMQW3/cSJTFwPLd5wnIxM0UvAZ4G+gWgdiSgF1AWlBZp58zXLLaBjTi2oa/1dr5wY1Oecj7m1sK5HdyXAW4tuzdf2d/9va9yPv9LgI+B86LwDlr9XcH/Mw7Z6uBszszLq/8KeD6Zvt22jlr4xrRKX9ndge3McaYdh3tzVDGGGNCYMnCGGNMuyxZGGOMaZclC2OMMe2yZGGMMaZdliyMaYeI+GXf2W0P2ezE3qylkboPxJiQhXUNbmOOELWqOjrSQRgTSVazMKaDvHUNfisin3k/A73yXBF515sM7x0R6euV9xC3fsRi7+ck76V8IvK4t0bBWyKS4O1/k7d2wRIReSFCH9MYwJKFMaFIaNYMdWnQtgpVHQc8CDzglf0J+JuqjsRN0vdHr/yPwH9VdRRuvYTlXvkg4CFVHQ6U4e4KBrc2wRjvda4P14czJhR2B7cx7RCRKlVNbqF8A3C6qq7zJnjbrqqZIlKMm6ai0SvfpqpZIlIE5KhqfdBr5AJzVHWQ9/wnQIyq/kpE3gCqgFeBV1W1Kswf1ZhWWc3CmIOjrTw+EPVBj/3s7Us8Fze3z/HAQm/WU2MiwpKFMQfn0qB/53mPP8bNYAxwBfCB9/gd4AYAEfGJSFprLyoiUUAfVZ2LmyE2HdivdmNMZ7FvKsa0L0FEFgU9f0NVdw+fjRORT3FfvC7zyr4LPCkiPwKKgGu88u8Bj4nIt3A1iBtws5u2xAc86yUUAf6fqpYdsk9kzAGyPgtjOsjrs8hX1eJIx2JMuFkzlDHGmHZZzcIYY0y7rGZhjDGmXZYsjDHGtMuShTHGmHZZsjDGGNMuSxbGGGPaZcnCGGNMu/4/zv0wdJ7WQGoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15537dc7ba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualizing the training and validation accuracy\n",
    "plt.plot(range(nn.epochs), nn.eval_['train_acc'], label='training')\n",
    "plt.plot(range(nn.epochs), nn.eval_['valid_acc'], label='validation', linestyle='--')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 97.54%\n"
     ]
    }
   ],
   "source": [
    "# Calculating prediction accuracy on test set\n",
    "y_test_pred=nn.predict(X_test)\n",
    "acc = (np.sum(y_test==y_test_pred).astype(np.float)/X_test.shape[0])\n",
    "print('Test accuracy: %.2f%%'%(acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaYAAAEYCAYAAAAXsVIGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXd4VFX6xz/vTAglgVBClRIRkGJBQYSfAiIg6qIo6CKiYBcLiO4q6woKyrr2rrgKKmBHpVhXFwQsIEVpgiIoJfQAgUAS0t7fH/dOnAyZycxkbmYmnM/z3CeZe9p7v/fc8542d0RVMRgMBoMhVnBF2wCDwWAwGLwxjslgMBgMMYVxTAaDwWCIKYxjMhgMBkNMYRyTwWAwGGIK45gMBoPBEFNE1DGJyCki8n0k8zyWMXpGHqNpZBGRi0TkvWjbUVkwelqE7JhE5HYRWSYiR0TkDe8wVV0FZIrIRQHSzxeRG0IsU0XksIgcso/JododQlk1ROQlEckQkQMistDBsqqKyBQR2SwiWSKyQkQu8IQ7padX2mG2tmGlD7KMG0Rkg33fvhCRJk6V5VXmmyKyQ0QOish67+uLd01FpKuIfCUi+0Rkj4jMEJHGTpRVStmtRSRXRN70nFPVj4EOInJKgHSbRKRPiGVV5HPYW0R+EZFsEflaRFo4VZZd3nxbR0979qsnzAk9K7LOiMhQr+s6ZGuqItIplHzCGTFtByYCr/kJfwu4OYx8y+JUVU22D8caUuAVoC7Qzv57p4NlJQBbgZ5ACjAWeF9E0rziOKKniNQB/gn8HOm8vco4B3gYGICl5R/AO06V58W/gTRVrQVcDEz0eTDiVlOgDlYdTQNaAFnA6w6W582LwNJSzr8D3BThsirkORSRVOAjYJxdzjKgIkYst3u1Zyf6hEVazwqrM6r6ltd1JQO3Ar8DP4aaUVgHlnN6o5TzxwE5QNVSwv4FFAK5wCHghSDLUqBVkHHnYzVMS4CDwGygbpBp29ppaoWrS3kPYBUwyEk97bQv25VmPnCDQ3o+Abzo9bmJfS9PqEA9TwR2AH+tDJqWktfpQFYF6HgF8D4wHnjTJ+ws4A8/6aYDRbbeh4B7gigrpOewnHX0JuB7r89Jtq1tHdSyrPoRUT1DrTMRrp9fAw+Emi7imx9UdRuQj9Ug+IbdB3zDn72F2wFE5BMR+UcZWS8UkZ0i8pHPiKI0hgHXAY2BAuA5T4CIrBKRK/2k6wJsBibYUwirRWRQGWVFDBFpCLTBq8fthJ4i0gXojNWQBkO4egJIKf+fFGS5YWNPA2UDv2A5ps88YZVAU2964OwIDRGpBTwI3OUnyjogzY5XAlW9GtgCXGTr+ZidZ6Sfw3D17ACs9LL3MLDRPu8k/7av7Tt7ZsGbSOvpSzB1ptz1054S7QFMC9KuYhJCTRAkWUDtYCOrav8yovQEFgM1sEZqn4hIR1Ut8BN/uqquARCRccAKERmuqoWq6nfuFmiK1Wh+iNW77wZ8KiJrVXVdsNcTDiJSBWuKaaqq/uITHDE9RcQNvITV8BaJiL+o3oSr5xfAuyLyMvAbcD/WiKlGkJcSNqp6q4iMxLqH5wBHfKLEq6be5Z6CpemAYOKXg4eAKaqa7ufasuy/tbF62GXiwHMYrp7JwB6fcweAmsFcR5iMAdYCeVgj0Y/t9myjHR5pPYsJoc6Uu35iObdvVPWPIOMX49R28ZpAZqQyU9WFqpqnqpnAHcDxWHPP/tjq9f9moAqQGkRROVg96Yl2eQuwhqLnhWd5cIiIC2uIngfcXkqUSOp5K7BKVReHkCYsPVX1f8ADWA3MJvvIAtJDKDts7IfoW6yG7haf4LjU1IOItAI+B+5Q1W9CKDckRKQj0Ad4OkA0TyMeKT3DeQ7D1fMQ4DsyqcWfziHiqOoPqpqlqkdUdSrwHXChV5RI6wmEXGfKVT9thgFTQ0wDODBiEpHjgETgVz9RIvE6c6XkFJEvzbz+b45VyTOCyHeVn7IcQ6wu6BSgIXChqub7hEdaz95ATxHxPAh1gdPsHltpThHC1xNVfRFr0RwRaYO1wWNNiDaXlwTgBM+HeNfUniL5H/CQqk4P0dZQOQdr0XyLPVpKBtwi0l5VT7fjtAM2qaq/3n2oeobzHIar58/AcM8HEUnCqiuOTo/64NueRVrPcOpM2PXTLu8srNHuB6HY6SGc7eIJIlINcGNV0Goi4u3gegLzVNV36sTDLqBlCOV1EJGOIuIWkWTgSWAb1jysP64SkfYiUgNrbvwDVS0MoriFWPO399rXeRbQC/hvsPaGwSSsiniRquaUEh5RPYFr7PI62scyYAJwX4A0Yelp142TxKI51s6gZ1V1fwj2hoSINBCRK0Qk2a4z/YAhwFyvaPGs6XHAPKxNGcGuZ5WHV7Aaas+1vQx8CvTzitMTqyfuj1D1DOc5DPeZnwmcJCKD7HbtfqzRr+90ekQQkdoi0s/TborIUKx1mC+8okVUzzDrTLh6ehgOfKiq4Y08w9hlMR7LY3sf473CPwUuDpC+G7Ae2A88Z5/7HPinn/jnYvVsDwO7gVlA6wD5z6fkjpKPgVSv8J+BoQHSdwAW2eWtBS4NVaMQtGxh6+fZAeY5hnrFiaiefvQKZQdZ0HpizZGvsrXcaefjdkpPu8z6wAKsaZCDwGrgRp848azpA3ad8a4vh5zU1Kf88Ry9K2811tc5/KUZgOVoMoG/l3WNdnjQz2EEnvk+WJtkcuy80hyun0uxpgozsdbO+zqpZ6h1JgJ6VrNt6x2uTmJnFBHshbX/qGq3iGUaug3zsR4cx76EW1EYPSOP0TSyiPVF5atV9a9RtGE+Rs9I2jCfKOsZ0TUmtb5VH7UHvrJh9Iw8RtPIotabCj6Oth2VBaOnhXmJq8FgMBhiiohO5RkMBoPBUF7MiMlgMBgMMUVYa0ypqamalpYWYVNil+XLl2eoan2n8jd6RhajZ2Q51vQEo2mkCVXPsBxTWloay5YtCydpXCIim53M3+gZWYyekeVY0xOMppEmVD3NVJ7BEGHy8/O55pprmDhxYrRNMRjiEuOYDIYIc+DAAaZNm8YjjzxCbm5utM2JO3bs2EGjRo0QEbZs2RJtcwxRIG4dU25uLtu3b2f79u3k5uby4osvsmjRIrZv3x5t0+KKjz/+GBHhhx9+iLYplY4mTZrgcsXtIxYVxowZQ4sWLdizZw8nn3wytWsH/QJ4QyXCqZ+9cIwVK1YwY8YMPv74Y37+2XrP4imnnML69euLe6eFhaG80unYZfbs2QwcOBARITk5OdrmVDquuOIKEhMTo21GXPHee+9RWFjIySefzPz586lV66ifJDIcA8SFY1q/fj3t2vn/lYuuXbvStWvXCrQovtm1axfdunVj69atvP/++wwaVGG/hVjpOXDgAG3btqVevXo8+OCD0TYnrmjUqBF79uwhPT2dxo0bR9ucuGfhwoXcddddrFixovicqnLWWWeRk5PDTz/9BMCGDRuItR2CMemYsrOzmTx5Mueccw6nnHIKCQkJ1KlTh5o1a5KVlcXgwYPp2LEj//d//8fxxx9PjRqO/+5cpSEzM5OuXbuydetW3n33XeOUIsySJUvYt28fr7/+erRNiSu++uor9u7dyy233ELDhg2Lz2dlZfHjjz8CcPrpp1OzppO/31d5WLhwIf379yc7O5vU1FSuv/56unTpwplnnkmdOnVQVerXr09OTg7Tp09n3Lhx0Ta5BDHnmPLy8jj//PP57rvvWLJkCQAtW7Zkw4YN1K5dmwMHDlCrVi2C/JVQgxdFRUX8/e9/Z8uWLTz88MNcdtllJcJExOhaDg4fPszdd99NkyZNSmhrKJt9+/ZRVFRE9+7dcblcrFy5kvHjx7Ny5Uo2b7Z2Grdo0YKUlBSaNm3K9OnTzfqTH/Lz83n++efp3bs3//rXv2jdujVVqlQ5Ko7nWb/yymB/kb0CCeeV5J06dVKnuOaaa9Tlcumzzz6reXl5jpUTCsAydeg1+Oqwnt68+eabCuhtt91W4nxRUZHedNNN+vnnn1eIHZVFT1/mzp2rIqIjRoxQVdW8vDzNzc11vNzKoGf79u3V5XJpVlaW/vjjj1q9enV1uVx+j86dO+v+/fsds6cyaBqIV155Rd1ut5566qmanZ3teHmh6hlToh45ckRdLpc2btxYc3JyHCkjHCpDJc3MzNTjjz9ezzzzTN29e/dRYYB27txZ8/PzHbelMujpy5EjR/Tyyy9XEdHNmzdrYWGhXnPNNdq3b1/HH/x413Pu3LmakJCgLpdLH3nkEW3btq26XC69+OKLddmyZbply5YSx2OPPVYc1yniXdNAbNq0SWvWrKlut1vff//9Cikzrh3T/PnztXXr1nrgwAFH8g+XylBJx44dq8BRFbGgoEDvuOMOBfTyyy933A7VyqGnL6+88oqKiF522WWqqrp69WoVERUR3bZtm6Nlx7ueU6dOPWpEdNxxx+m+ffv8pnG5XJqYmKjff/+9IzbFu6b+KCoq0ldffVXdbrempqY6Xjc9hKpnTH3JYu7cufTo0cNsEY0w+/bt44knnuDqq68+au3j4MGDPPvsswAMHjw4GubFPTt37uTf//43ACNHjiQ7O7t43r5Zs2akpKRE07y4o1GjRsybN486der4jdO4cWMKCgpYuHBhBVoW/3zxxReMGDECgKlTp9KkSZMoW1Q6MeWYXn31Vd555x0mTZpEenp6tM2pNHz55Zfk5uZaQ2Qf5syZU/x/v379KtKsuKewsJAff/yRrl27smnTJgC6devGmjVrWLNmDQBut5uioqIoWhlfNGzYkPvuu482bdoEjDdv3jwAHn300Yowq9Lw/vvvA3DCCSdw3nnnRdka/8TUrrydO3ficrm4/fbbGTVqFGPHjqVXr15s2LCBdu3a0bJlSwB+//13OnToYEZWQdKrVy8SEhJ48803qVq1Kvfffz/Nmzdn7969jBw5ErC+cZ+UlBRlS+OH3NxcrrrqKj766KMS56tWrVri86ZNm0hJSWH16tV06NChIk2MS+bNm0fbtm3LjGe+5xQ66enpTJs2DRHhscceIyEhppr/EsTUiMm791NUVMSDDz5Ir169uPHGGzn77LNp0qQJTZo04eyzz6Zdu3bcdtttUbQ2fmjYsCGTJk1CRJgyZQqnnXYaQ4YM4fjjjycrKwu3283YsWPNVvEgyc3NZfz48cVOKTk5mRdffJG3336bXr16lZrmWHqTdHlo2rRpUPGysrIctqRykZeXx7hx4ygqKmLQoEEMGDAg2iYFJpQFKXV44a6wsFBPOukkbdOmTfEunUCH2+3WV155xRFbvKGSLIR+8803evzxxytQ4hg4cGCFlO8h3vUcM2ZM8cYGEdG3335bVVXz8/M1NTVVRURdLpeeddZZ+sgjj+gvv/ziqD3xrqf35oesrKyg0txyyy3qcrn0nnvuccSmeNfUl5kzZ6rb7dakpCRdsGBBhZatGrqeMSvq2rVrdeXKldq3b9+Azum6665z3JbKVEkLCgo0Ly9PN2/erIAmJydrZmZmhZWvGv96ejul0aNHa2Fhoapa254BFRG96qqrHLXBm3jXc/v27VqjRg11uVz65ptvBvVVEZfLpbVr19adO3c6YlO8a+pNRkaG1q5dW91ut77wwgsVVq43oeoZU1N53rRr145TTjmleHdTQkIC9957L5s2beL222+PsnXxi9vtpkqVKuTk5ADQvHlzs2ssTE488UTGjx9f/AbxHTt2ICKMHj2aadOmRdm6+KFx48bccMMNAAwbNoyuXbsya9asUjeNFBUVsWfPHgBuuummEq8vMhyNqvLII48UT332798/yhYFSSheTKPg7fPy8vS22247aqRUpUoVve666/TIkSOO20Al6j2pqqanp2uVKlUU0AceeKBCy1aNbz2zsrJURHTt2rWlhm3dutWxsv0Rz3r6sn37dr3jjjs0MTHR7yxJ8+bNHbejsmg6efJkdbvd6na7j3pTxrffflshNqhWohGThypVqvDoo48ydOjQ4p04bdq04ZlnnmHKlCnmZwXCIDc3l/z8fGrWrMmwYcOibU5ckZycTFFRUalvu09OTg568d5QOo0bN+aZZ55hy5YtxTtGPbRo0YL//Oc/LF26NErWxR+LFi0C4LnnnqNWrVoUFBTwzTff0LlzZ+rWrRtl6/wTu/sFvUhKSmLatGnMnz+fr7/+mnvuucdsbS4HKSkpJCUl0bt37+It+AZDLNGwYUOeeeYZnnnmmWibEtd88cUXAGRkZLBr1y4uvfRSfv31V5599llOPPHEKFvnn5gfMXlzzjnnMGHCBOOUyklqaiqHDh1i5syZ0TbFYDA4yPXXXw/Agw8+SLNmzVi6dCm33XYbw4YNi+lfV45dywwGg8FQLsaMGcMZZ5wBQPfu3Vm0aBEPPPBAlK0qm7iYyjMYDAZD6NSoUaN4nSmeMCMmg8FgMMQUYu3kCzGRyB5gc+TNiVlaqGp9pzI3ekYWo2dkOQb1BKNppAlJz7Ack8FgMBgMTmGm8gwGg8EQUxjHZDAYDIaYwjgmg8FgMMQUxjEZDAaDIaYwjslgMBgMMYVxTAaDwWCIKYxjMhgMBkNMYRyTwWAwGGIK45gMBoPBEFMYx2QwGAyGmMI4JoPBYDDEFMYxGQwGgyGmMI7JYDAYDDFFRB2TiJwiIt9HMs9jGaNn5BGRhiKyTkSqRtuWyoCpo84iIh+KyAXRtqOiCcsxicgV9sN9WEQ2ikh3AFVdBWSKyEUB0s4XkRtCLE/tsg7Zx+Rw7A6inPYiskxE9tvH/0SkvRNleZV5yOcoFJHnwRk9RSRVRL4Tkb0ikikii0TkrAhcSmlldS/l+lREBjlRnle5aSLymX0Pd4rICyKSAKCqu4CvgZsCpH9DRCaGUe4dIvKHXVfXiUib8K8iYDlpIvK1iGSLyC8i0seJcrzKayci80TkgIhsEJFLPWHxXkft8hxtX0TkdrtdOSIib5QS3tu+j9n2fW3hFfwo4Lcuisg5IpIeoj1VReRlEdklIvtE5GMROS6UPEIs7077OTwoIq8F1SlU1ZAOoC/WD1x1xXJsxwHHeYUPBT4JkH4+cEOIZSrQKlRbw7i22kAaIIAbGAWscrpcr/KTgUNAD6f0BKoBJ9r3ToBLgH1AQgVc3zlAFpDkcDmfAW/Y19oIWA2M8go/C1gTIP0bwMQQy7wBWAW0t3U9Aajr0PUtAp4CqgODgEygvkNlJQDrgbvsZ+Jc4DDQprLUUafbF2CgfQ2TgDd8wlKBA8Dl9nU/Diz2ifMb0NlP3ucA6SHacw+wEmholzkN+Miha+8H7AI6AHXsuvBImenCKOh74PoA4ccBOUDVUsL+BRQCuXYD/EKkK4594f8GlgAHgdnhNBD2A3kbkO1UhS2lzOHA79g/4OiUnl7pXcBFtr4N/MR5A3gZ+ArLqSzA+jXKcK7vdeD1CtBxHXCh1+fHgf/43Nvs0q4DaySVD+TZmn4cpI5bgd5B2jce+AB4z9b0R+DUINO2AY4ANb3OfQOMcEjLk2wdvOvkl8BDlaWOhtK+lFPLiRztmG4Cvvf6nGRr2dbr3KvAA6Xk54lbZGt7CGgShB2TgMe8Pv8F+LUMfUbZbVOG/Ty5grzmt4GHvT73BnaWmS5EYd32A/sPYAOQDrwAVPeJdxA4xU8e8/HpPQGfAP8oQ5jtwE7gIyAtQNz5wDb7gUoCPgTe9ApfBVxZxnVmAgX2DR/rdIX1KnceML6U8xHV00uHPFvbVwPEe8N+2HsAVYFngW9DKcuOl2Tnc04F6HgzVi+wBlajuQa4tJTrvzjANU/0OfcS8JKf+M1tHe/AclB/ABP8PbxYjikfuAyoAvzdTlMliLIuBdb5nHsBeN4hLUtzTF8BMytLHSWE9qWcWpbmmJ4FJvmcWwMM8vp8F35GNJQyYgLOBjID2NEZ+A5oYj8jbwPPlKHP10Bdu66v99xP+3Mm0NxP2pXAYK/PqXZ+9QJplUBoNMR6kC4DumM9XLOBscB9XvGysKbFgkJV+5cRpSewGEvEicAnItJRVQv8xJ+uqmsARGQcsEJEhqtqoaqeEoQ9tUUkCWsEsznY6ygP9rxyT+D6UoIjrSeqeoqIVMNq6BLLiP6pqi607bwPOCAizVR1azBl2QzE6m0tCDJ+eViI1RM9iNWZmgrM8okTqqa3Bghuav89DzjZzvdLrI7bq37SLFfVDwBE5Cngb1jT49+UUVYy1tSPNwewHLAT/ArsBu4WkaeBXlj19GufePFcR0NtXyJJMrDH59wBoKbX51C1/baM+L9hdaC2YY1mVwO3l5Hto6q6D9gnIs8AQ4DJqrqljLJ866vn/5rAXn+JQt38kGP/fV5Vd6hqBtZc94U+8WpiedGIoKoLVTVPVTOxeqXHA+0CJNnq9f9mLGeaGmKZh7GmB6aJSIMQTQ6Hq7F6eX+UEhZRPT2oaq6qvgP8Q0RODRB1q1eaQ1jz/U1CLG44ME3tbpNTiIgL+AKr55uEdd/rYC0iexNJTT3PxWOqmqmqm4D/cPRz4Y23pkVYTiwYTQ8BtXzO1cJqvCKOquZjrY/8BWtE8TfgfSx7vYnbOhpG+xJJgrmfkdb2RayRZT2sZ+Qj4PMy0vi2qcE+/77X5/k/YH0NyTGp6n6sCunduJRoaOzdHYlYPa1SswmlzAB5SIDwZl7/N8ca2WWEUY6LP6eDnGYYVs++BBWkZxWgZYDwYj1FJBlrSL892MxFpBnWlMO0MO0LBc90wwuqekRV92KtbRU7CXuHXiusaYbSCFXTX/lzyinYPLw1dWGNuoLR9GegpYh496hPtc87gqquUtWeqlpPVfth1ZUlnvDKUEd9KKt9iSQ/Y90/AOyZmhMoeT/bEbm6CtARa0pxn6oeAZ4HuohIoM67b5sarLYlrs/+f5f9XPolnO3irwMjRaSBiNQB7sSaw/XQE5hnX3Bp7CJwBSuBiHQQkY4i4rYr3JNYQ9B1AZJdZW/9rgE8CHygqoVBlNVXRE6zy6qFNRrcX0ZZ5UZE/g/L+c0oJTjSenYVkbNFJFFEqovIGKwp2h8CJLvQkwZ4CGvX0NYA8X25GmuBd2MIacLCHsX/AdwiIgkiUhtrtLbKK1oXYJOq+pumDUlTVc3G2shwj4jUFJGmWFOJnwRI1klEBtpOcjTWhobFQZS1HlgBPCAi1eyt26dgraU6gljfVaomIjVE5O9AY6x1HQ9xW0fDbF9Cwq6H1bCmld22lp5llJnASSIyyI5zP9ZO4F+8suiJ/xHNLqCeiKSEYNJSYJiIpIhIFeBWYLv97PjjbhGpY3cy78Cq78EwDbjebo9rYy37vFFmqjAW8KpgLc5mYg3tnwOqeYV/ip9FZTu8G9bi2X7gOfvc58A//cQ/F6sndhhrrnsW0DpA/vMpuSvvYyDVK/xnYKiftJcDv2ANP/fY11Lqgm4kD6xpn+l+wiKtZ0+s3lcW1nTHAry2p5cS/w3+3PF0CGv95nivcL9lecX5hQA7OR3Qs6NdD/ZjjZTfBxp6hb+I1/bxUtK3xmr8M4FZ9rmXgZcDpKkFvGvruhWrgRE/ccdTclfeT8DpXuFllZVmX1+O/Wz0cVjPx20tD9n3u5VPeNzWUUJsX8LUbzzWyMb7GO8V3sd+RnLs+5rmFXYG8GMZ+b+GtV6TiTXF1h04FCB+PeAt+3ozgW+BLgHiK3/uytuL5bzddlhzW/NSNz/Yce7CcqAHsQY2R+3e9D3EThgRROQUrG253SKWaeg2zMfahefIl3ArkhjR8w2sXT9jo2VDJLHXCxcAp6lqbpRsGI/VuF8VjfIjiamjziIiHwJTVPWzKNqgWM56Q0WVGequvICo9S3wqFXQyobRM/Ko6m4qbmG70mPqqLOoqqNvSYlVzEtcDQaDwRBTRHQqz2AwGAyG8mJGTAaDwWCIKcJaY0pNTdW0tLQImxK7LF++PENV6zuVv9Ezshg9I8uxpicYTSNNqHqG5ZjS0tJYtmxZOEnjEhFx9LVERs/IYvSMLMeanmA0jTSh6hl3U3kZGRkMGjSIzz8v6w0aBoPBYPBl69atpKamcuKJJ5KREc4LcZwnrhxTRkYGPXv2ZPbs2dxxxx3RNsdgMBjihvz8fEaPHs1pp51GZmYmGzdu5KKL/P6+Y1SJG8c0Y8YMzjrrLH791XodV4cOHaJskcFg8cEHH0TbhLhkypQpvPbaa9E245hgx44d9OvXjxdeeIH9+/cXn7/wwkDvGY4eceGYZs2axZAhQ9iwwfri8QUXXMDMmTOjbFXsk52dTdeuXbn11lvZuHEjGzZsICvLkZdQH5Ps3LkTt9tNly5dom1KXHLjjTdy44034na7/R4ul+uoc+eff360TY8rHnzwQZo1a8bChQuPCpswYQIJCQl8/bXvr5hEl4i++cEJsrKyeOKJJ1BVioqKcLlcPPfcc9E2K2ZZsWIFN954I7/99hv5+fnUqVOHJUuWMHnyZIqKiqhatSpVqlTh+uuv59prrwWgVatWVKtWLcqWxx+jRo2iS5cuJCaW9VNBhtKYPXs2c+bMAWD37t188kmgd94awmHr1q0899xzeH9f1dOOevPBBx/Qq1evijbPLzHvmPbs2cPixYsREVwuF8OHD6d58+bRNitmuffee1m+fDlnnHEGLVu2ZOzYseTn51OrVi0KCgpYvHgxH3/8MVu2bKF79+4cOHCABg0a8Pjjj3P++edTv75jO2QrFT///DP5+fksWrSo1PB9+/Zx+PBhmjVrxi+//MI333wDQKdOnTj99NMr0tSY5aKLLipe41i7dm2pjqlVq1bFep188sk0aNCAiy++uELtjGeeffZZMjMzEbF+xaN79+7MmTOHjRs38sgjjxRPQ0+fPp1LL72UPn36RNPcYmLeMdWoUYMGDRqwe/duAL744gv27NlDo0aNomxZbPLGG29w5ZVX0rFjR5588smjwtu0acOwYcMAyM3NJT09nXfffZfRo0fTsmVL3nrrLdq0aVPRZscdn3zyCW63u8S5rKwsLrnkEgD27t1LdnY2bdq0IT09ndWrVwPQsGFD2rVrx9y5cyvc5lhmx44dJT673W6eeeYZrrjiCurWrRslq+Kfp59+utgp9e3blxkzZpBMSImlAAAgAElEQVScnEzHjh15+OGHix1TcnIyqakh/Zaqo8T8GlOjRo2Kp5zAGvJPmjQpihbFNg0bNuTTTz/lkUceKTNutWrVOP744xkxYgSXXHIJy5cvZ8KECeTk5JSZ9lgmKyuLH3/8kVdffZUtW7Zw+PBhAG6++Wb27t3Lnj17WL16NRs3buTCCy/k+uuvL067a9cumjZt6i/rY5IjR44wduyfLwavX78+S5Ys4dZbbzVOKYKMHj2ahIQEjhw5wsGDB/nPf/5THDZw4EA6duwYRet8COf3RTp16qQVyYEDB9TtdqvL5VK3261paWm6bdu2CisfWKYO/t5NRevp4aefftIWLVpocnKyioiKiKampuratWsdLTfe9bzsssvU5XLpX//6Vz355JN18+bNqqp644036vbt23Xbtm26Zs0aXbNmjebn5+vevXu1WrVq6nK5dOjQoXr48OGI2hPveq5YsaJYH5fLpTVr1tQ777xT77zzTv3yyy+1qKjI0fJLI9419eBpM91ut9atW1d79eqlAwcOLD7nORYuXOioHaHqGdOievPvf/9bgeLK27Jlyworu7JUUl8+++yzYofUtWtXbdCggc6ePdvxcuNdz5SUFD3nnHN0y5YtOnLkSN23b1+ZaUaPHq0ul0vvueeeiNsT73qqqr7yyivFz7b3ISJ633336fjx43Xv3r2am5urhYWFjttTGTT1bTM9R2nnnKbSOqacnBxdsGCB1qxZU91utyYmJuqECRM0PT3d8bIrQyUtjYKCAr3jjju0evXqumLFCs3KyqqQcuNdT5fLpe+8807IaYxj8s/+/fv19ddf1y5duhzlmHwb0ZEjRzpuT7xrOnbsWK1Tp06JEZPnKO2c01Rax+Thuuuu07p16xYL2rx5c8fLjPdKGojs7Gy9//779YQTTtDrr79ejxw54niZ8aznpEmTdMCAASH12l988UV9+OGHNT8/XwsKCiJuUzzr6Y+lS5fqU089pZdccknxqN77uOWWW/Stt95yrPx41nT79u3aqFGjEk6obt26+umnn+rTTz9dqmNyol56U+kdk6rqxx9/XML7O008V9JgKCws1BUrVmhSUpJedtlljpcXz3qKiA4cODDo+E899ZT26tVLN27c6JhN8axnWRQUFGhubq4+++yzevXVV5cYOSUlJemHH37oSLnxrOm4ceNKtI/9+/fXdevWFYdPmDBBExMTSzimyZMnO2aPauh6xvyuvNJo3759tE2oVLhcLk499VQ2btzImjVryM7OjrZJMYuqsm/fPg4dOhQw3pYtWxgzZgxjxoxh8+bNtGzZsoIsrFy43W6qVq3KqFGjmDJlCpmZmfz+++8A5OTkcPnll0fZwthj+vTpJT7fc889tG3btvjz/ffff9TO0J9++qlCbAuWmHRM69evZ/369aWGff7551xwwQXFnrWoqKiCrauc5OXlsXbtWnbv3m3eQxgAEWHhwoXceOONpW6r3759O88//zytW7cmIyODSZMm8eWXX0bB0spHlSpVqFmzJvn5+dE2JabZsmVLcfvYo0cPzjjjDMDalv/uu+8iImzatKnECKVv375RtrokMfcF26ysLLp27crFF1/MzTffXHz+vffeY86cOWzbto3CwkJEpPhbzIbwycrKYs6cOTz88MOsW7cOgBEjRkTZqtilXbt2/PLLL7z//vuoKi+99BJ33303a9asAaw34D/00ENs3ryZlJQUqlevHmWLY58vvviCJ598ko4dO/L444+XGsfziq2nn36aDz/8sPi8mT05Gs8XasF6Rdljjz3G1KlTKSwsJD09HZfLVSIOEHtvGQ9l3s9zODk/evDgQa1bt26pC3QtW7bUa6+9tsR8aUVAHM83+7JgwQK95557NCUlpXghuXPnzjpmzBidPHlyxL9jUxrxrmdhYaHu27dPO3furA0bNiw+Zs2a5Wi5/oh3PatXr17qVnF/W5uTkpK0X79++tNPPzlmUzxr2rFjx4A78Fq2bKmtW7fW1q1b65QpUxyzw5tQ9Yy5EVPNmjVZtGhR8Zzn+PHjueiii+jUqRODBw+OsnXxz5lnnsmCBQvo27cv7du3Z+DAgTRq1IiGDRtG27S4weVyUadOHZYuXRptUyoF/fr144cffmDXrl1+4yQlJZGamsrYsWM5/fTTY+stBTHG6NGjS7xtxMNdd90VN+1ozDkmsN7n5nlfWzyIGE9UrVqVcePGRdsMg6GYmTNncvDgQS666KLidwoOHz6cnj17Fsc58cQTadeuXbRMjCuGDx/O8OHDo21GuYhJx2QwGI4tatWqxYIFC6JthiFGiMldeQaDwWA4djGOyWAwGAwxhXFMBoPBYIgpxNrJF2IikT3A5sibE7O0UFXHftrV6BlZjJ6R5RjUE4ymkSYkPcNyTAaDwWAwOIWZyjMYDAZDTGEck8FgMBhiCuOYDAaDwRBTGMdkMBgMhpjCOCaDwWAwxBTGMRkMBoMhpjCOyWAwGAwxhXFMBoPBYIgpjGMyGAwGQ0xhHJPBYDAYYgrjmAwGg8EQUxjHZDAYDIaYwjgmg8FgMMQUjjomEXlSRG5xsoxjCaNn5DGaRhYR+VBELoi2HZWFY1ZPVQ14ALcDy4AjwBs+YYnAB8AmQIFzfMIbA1uBRD95p9npEsqywyvNy8Ahr+MIkBVs+lAO4CTgv0CGJVVE8gykZ1fgK2AfsAeYATR2WM8rgF+BA8BuYCpQyyE9Hbl3ZWja3g7bbx//A9o7rGnE602AsgSYCGyz7+F8oINTevrEu9/Wpo/XuS7A8gBpzgHSQ7TnbmANkAX8AdztsKanAwvtOroLuMPB+umpX97PxbjKoidQ1X7ud9nt2sfAcWWlC2bEtN2u+K/5Cf8WuArY6RugqjuAX4CLgygnKFR1hKomew7gHawG3AnygfeB6yOYZyA96wCvYFXWFlgV53VPoBN6At8BZ6lqCtASSLDtizgO3rtAmm4HLgPqAqnAHOBdL5uc0NSJeuOPy4HrgO5Y17gImF7OPMt65hGRE+yyd3ifV9UlQC0R6VxOG0oUBwzDej7OB24XkSsimP+fBYmkAl8A/wHqAa2AL8uZbZl6ArW9no2HPCfjXU/gDqAbcArQBKtz+HyZqULwfBMJ3HtKx2fEZJ+/D3jdT5otlOwtdAvRGydhNd49A8RRYBTwO1YP9nHAFWI5rYhwz7csPe04p+MzonBYz2RgGvCZk3oGe+8irSmW070NyK4ITYOtN+XRFBgDvO/1uQOQ67SeWI33hVizJX18wl4FHvBzz3OAIi89m4Rh13PA837C0mw9b8JyCDuAv4eQ98PA9EjVybL0JIgReZzrOQl4zOvzX4Bfy0pXEZsf1gGn+gnrYf/19BYWiUhzEckUkeZB5D0Ia8prYRnxLgU6YzX0A7B6mIRYVjToAfzscy7ieorI2SJyAMtRDAKeKcOuSOgZ7L2LCCKSCeRi9dYe9gl2so4GS7iavgucICJtRKQKMBzLaTiGiFwOHFHVz/xEKVVPVT0MXABs1z9HB9vt+pcZZNmCNTr0fS586QW0Bs4DxohIHzt9WWV1BfaJyPcisltEPq6g9mGziKSLyOv2qM2beNZzCnCWiDQRkRrAUODzsuyqCMeUBdQONrKqblHV2qq6JYjow4FparviADyqqvvsPJ8BhoRRVoUiIqdgzeHf7RMUcT1V9Vu1pvKaYvXWN5WRbST0DPbeRQRVrQ2kYM33/+QT7GQdDZZwNd2BNZ3+K1bv+XLgzgjaVQIRqYnl2O8IEC1UPb+1708wjMdqt14vI94EVT2sqqvtuB49yyqrKVbdvANojrUG806QtoVDBnAG1tR9J6Am8JZPnHjW8zesNdxtwEGgHfBgWUZVhGOqCQTlvUPB7sWcgzX1VBZbvf7fjDXXGbOISCusXsUdqvqNT7AjegKo6jas3va7ZUQtl54h3ruIYfcwXwamiUgDryDHNA2BcDW9H6thawZUAyYA8+zeqROMx5rq2hQgjlPP/O1YayN/UdUjZUQPV88cYKaqLlXVXCw9/09EUkI2OAhU9ZCqLlPVAlXdhdVxOs/uAHiIZz1fxNoAUQ9r6vEjYmTE1A5Y6SesPL3lq4HvVPX3IOI28/q/OdZcaUwiIi2wdo49pKqlLWI7paeHBOCEMuKUV89Q7l2kcQE1gOO8zjmtaTCEq2lH4D1VTbcbtzewFrXbR9g+D72BUSKyU0R2Ytn9voiM8YoTcT1F5DrgH0BvVU0PIkm4eq6ipI0Vdf99y/Num+NZz45Y62r7bOf3PNCllOnKEpTpmEQkQUSqAW7ALSLVRCTBK7yqHQ6QaIeLVxY98e8h92At3LUsy45SGAa8EWTcu0Wkjog0wxqivxdMIrGohrUtHvvaqoZhq3eefvUUkeOAecALqvqynywiqqeIDPXModtO8V/A3DKShaWnF6HcuzIpQ9O+InKaiLhFpBbwFNbOoHVeWURa03DqTbiaLgUuF5GGIuISkauBKsCGYO0txf5Az3xvrO3wHe1jO3AzVs/YQyA9dwH1QhmBiMhQrOnDviF0ZsaJSA0R6QBcS/B6vg5cKiId7TW7ccC3qnogWHt9KaN+nikiJ9r3rh7WRoT5PuXFs55LgWEikmLreSvWmlhGwFRB7KoYj+WVvY/xXuGbSglPs8MaY+3WK/U7InacB7Ee/kyshcfmWLtLmgdI0w04DNQMwn7vHU97gScBtx0WsCz+3JHifWwqq8xw9QQe4OjvNBzyShtxPbEcUbqtZzrWdvV6TugZ6r2LkKaXY20HP2Tr8ilwisOahlRvyllHq2E5hR1Yc/g/Auc7pWcpcTdR8ntMZwA/lpH/a/Z1ZmJNCXX3ruelxP8Dawu+93Pxchnae3aR7QTu8QoPWJYd5xasNZH9WN+7aeZg/RxiX99h+x5OAxpVFj2xpvDewvqOZCbWemiXsjQTO7EjiMiTwEZVfcmxQsq2QYHWqhp2DzJWMHpGHqNpZBGRD4Ep6n/HntPlp2E1vFVUtSAaNkSSY1VPRx1TLFCZHvpYwOgZeYymkaOyOaZoEy09zUtcDQaDwRBTVPoRk8FgMBjiCzNiMhgMBkNMkVB2lKNJTU3VtLS0CJsSuyxfvjxDVes7lb/RM7IYPSPLsaYnGE0jTah6huWY0tLSWLZsWThJ4xIR2exk/kbPyGL0jCzHmp5gNI00oepppvIMBkNMcvfddyMiXH99Rfx6iCGWMI7JYAiDBQsWkJCQQEJCAgsXVsgL0o85XC4XLpeLki+SMRwLxKRj+u9//4uIkJSUxJgxY/j992i8Uq3ykJeXR1paGv379ycvLy/a5lQKevfuXdxw9unTh08++YTffvst2mYZDAE5cOBAcd0VER577LFom1QqMemY+vXrh6py8OBBzjzzTP75z3/idrsZOnSoaVjDIDExkXnz5vHZZ59Rt25dTjzxRNxuN263G5fLxXHHHUejRo2Kz40fP57CwsJomx3TzJs3r8TnSy65hA4dOvDFF47+FJLBEBLZ2dnMmjWLunXr4na7qVu3LvPnz0dEYno0Gtbmh4rC7XYzcOBALrjgAvLz83nnnXe4+eab6dGjR9mJDcWoKosWLQIgPz8fgAkTJgBw9dVXk5qaSlFREfv27QOgZcuWdOzYkUsuuSQ6BscBaWlpFBUVFX/2/N+/f3/mzZtn6qghahw5coScnBw+//xzXnjhBRYvXhxtk0Imph2Th+rVq/Phhx/icrm4++67+eGHH6JtUlwxYcIEHnroIQBmzpzJhRdeWGq8mjX//AmYHTt2VIht8UpKSgqXXHIJc+bMKT7nclkTEH369DEje0PUuPnmm5k+/ehfzBk2bBjVqlXjlVdeiYJVoRGTU3n+aNy4MWvWrGHu3Ll069aNa665JtomxQX9+/fnyiuvZO3atX6dkoeNGzfSvn17BgwYUEHWxScpKSlMmjSJiy++ONqmVEoOHDjA5s2O7tiutJTmlPr168ekSZPips2MK8fUvn17cnNzGTx4MEuWLGHDBvPOy2Do3Lkz06dPp23btgHjffPNN3Tq1IlbbrmFJk1i+kd+Y4L69evTrl27aJtRKVm8eDEzZszg8ccf5+abb462OXHHqFGjiv8fPnw448aNo1q1akyePDmKVgVPXEzleZg7dy4iwv79+wE4++yzo2xR5WHhwoVceOGFJCYmMmLEiGibEzc89NBDfPfdd8yfP7/E+dNPP52vv/6alBRHfpH7mGHYsGGkpgb8sVODD2lpafzxxx8cOGD91mBiYiKJiYkAvP3229E0LWjiasTkS506daJtQtyTl5fHvffey4UXXki1atX44YcfitdKDMFx1VVXFW8d9xyrVq3iqaeeirZpccn333/PBRdcwH//+1/jlMLgq6++4tFHHyU5OZnk5ORip/S///2vePNTrBOTI6aMjAy2bt1Kw4YNmTVrFp9/fvSvCt9///2MHj06CtZVHn755RfOP/98tm7dSv/+/Xn11Vdp0KBBtM2KO6677jpuuummaJtRaRg4cGDMbmOOB1q2bFnq+V69elGlSpXir4I0bdqUv/3tbxVpWtDEpGNavnw5559//lGV0/MTHdOnT2fo0KHRMK1SkJeXx8yZMxkzZgybN2/mqaee4tZbb6Vq1arRNq1SMWXKFPr3788ZZ5wRbVMMxzg7duzggQceKLFb1PNdplgkJh1T27ZteeKJJxAR1q1bR7t27cjJyWHcuHEkJyfTp0+faJsYt6xevZpLLrmETZs2AdCiRQuysrJYvHgxPXv2jK5xcYz3d5o8n3fu3MnevXujZFF8cvDgQQoK/vyh1KKiohIajhgxgl69ejFx4kTq1q1Ly5YtWbt2LY0bNyYnJ4fXXnuNjh07RsP0mObAgQNMmTKFgQMH8tFHH0XbnDKJScfUokWLo4aYL730EgDJyck0bNgwGmbFNdu3b6dfv378+uuvFBYW4na7ad++PZs3b2bChAkkJiayceNGsxsvDDzrcr69T5fLxVdffUWPHj2oUaNGlKyLL0aPHl28uemrr75i//79XHHFFcXhjRo1ok2bNlx77bX07duXc889l9dee43zzjuP/fv3s3TpUuOYSiEhIYGJEyfSsWPHYsf07LPPRtkq/8SkY/KloKCAGTNmADBy5MgoWxOfbN68mT179vCPf/yDs846i/bt29OsWTNWrlxJt27dOHLkSNwsjMYa//znP/2GPf/889x5553GMQXBypUr+fbbb4s/P/HEEwD06NGD++67D4DU1NSjHM91110HWGsmJ598cgVZG1+0atWKzMxM+vfvX3zuxBNPjKJFgYkLx7R///7iNzibdZDw6NatGzt37ixxLjMzkwEDBnDkyBH69+9vRqJh8u6779K4ceNomxH3rFy5ko0bNxZ/TklJ4bvvvqNp06Yl3kpiCJ2MjIzikShYzvz4448v/pyZmUnt2rWjYVqpxObKVwD+7//+L9omxBW7d+9m1qxZbNq0iTfeeINZs2bx0EMP0apVK+rVq8egQYPIzs5m9uzZVKtWLdrmxiX169fnjz/+KDVswIAB1KpVq4Itij9cLhfXXnstf/vb38jNzaWwsJB9+/bRrl0745QiwK+//sqUKVOKP7/22mvUqFGDevXqcdppp/Gvf/2LI0eORNHCksTFiMkbz5fGDMHxwQcfFE9/qmrxTke3282UKVMYMmSIGYVGgHr16jFy5Eief/75Euevv/5645iCYM6cOcybN48JEyZQpUqVaJtT6ahXr16p5/v27cuQIUM477zzYqodiIsRU7169XjggQcAuOyyy+jVq5d5SWaQ+JtzHzBgANdcc01MVcZ4pkaNGvTu3TvaZsQt/fv356mnnqJ69erRNqVSUqtWLa666iquu+46Dh48WHy+U6dODBgw4KhdpdEmLhyTy+Uq3qWXnZ1N165dcbvdUbYqPjjzzDP54YcfGDVqFKNGjeLUU09l1KhRpb7o0VA+LrzwQvLy8sjLy6OgoIC8vDzOP//8aJtlMNCkSROmTp1KjRo1qF69Ovfeey9169ZlyJAhACQlJUXZwpLEzVReUlJSzHn1eCAxMZHOnTvTuXPnaJtiMBiijGeL+MSJE5k4cWKUrfFPXIyYDAaDwXDsYByTwWAwGGIK8bx/LqREInuAY+lXvFqoan2nMjd6RhajZ2Q5BvUEo2mkCUnPsByTwWAwGAxOYabyDAaDwRBTGMdkMBgMhpjCOCaDwWAwxBTGMRkMBoMhpjCOyWAwGAwxhXFMBoPBYIgpjGMyGAwGQ0xhHJPBYDAYYgrjmAwGg8EQUxjHZDAYDIaYwjgmg8FgMMQUxjEZDAaDIaYwjslgMBgMMYVxTAaDwWCIKRx1TCLypIjc4mQZxxJGz8gjIiNF5NFo21FZMHU0sojIhyJyQbTtqHBUNeAB3A4sA44Ab5QSXgN4CcgADgALvcIaA1uBRD95pwEKJJRlh1eal4FDXscRICvY9KEcwBXAr/Z17QamArXKmadfPYGhPteWbevTyUE9BZgIbLOvcz7QwQk9fcp9zba1VQTyKquO/hVYB2QBa4FLvMKqAelAgwD5h2QncA5Q5HMvhzukY8TvXxB63gBssK/rC6CJV1jE66hP+rnlSR9E/rXt53y3fYwvZ35VgSlYPwqYBawALvCJ0xv4xX7ev8b6UT1PWBdgeRl1LT2a11hGWZ/7PAd5wOqy0gUzYtpuV/zX/IS/AtQF2tl/7/QEqOoOW/CLgygnKFR1hKomew7gHWBGpPL34TvgLFVNAVoCCVhalAe/eqrqWz7XdivwO/CjHR5xPYHLgeuA7lj3bxEwPYL5H4WInA2cEMEs/WoqIscBbwJ3AbWAu4G3RaQBgKrmYj08wyJoD8B273upqlMjnL8HJ+5fID3PAR4GBtjl/YH1DAKO1VFP2UOBKpHO14ensTrbaVhO4WoRubYc+SVgOeqeQAowFnhfRNIARCQV+AgYh6XnMuA9T2JVXQLUEpHO5bDBl0hfo19U9QKfNu17gmmvQ/B8Ezm6h98WOEiAUQRwH/C6n7AtWL0fjzftFqI3TsLqhfQMEEeBUVgNfAbwOOAKw/MnA9OAzyLUkzhKz1LifA084KSewBjgfa/PHYDcAPE3AfdijTz2A68D1UK47gTgJ+AUIjRiKqOOngns9jm3x1sbrJHq137yXGjbedjWdHAQdpxDCL3Y8tTRUO9fBPR8AnjR63MT2/4TnKqjdroUYD3QlQAjJv4ckd2E5WB3AH8P4ZozgDO8Pv8T+CZSddTOcxUwyP7/JuB7r7AkIAdo63XuVd92wCeu9+i8SRDlB32N5dWzlLwKgbSy4pZ3jakL1hB1gohkiMhqERnkE2cdcKqf9D3sv7XV8qiLRKS5iGSKSPMgyh+E1cgsLCPepUBn4HSsnt51AMGUJSJni8gBLAc4CHgmCLvKjYi0wNJnmk9QpPV8FzhBRNqISBVgONb0TCCGAv2wRj1tsHqBHrsz7RGRP+7Emu5dVUYZkWIZsE5ELhYRt4hcgjVF5V2+X01V1aPpqbam70FQ19lARHaJyB8i8rSIJJVhZ7h1NJz7V16klP9P8jrnxDP/MDAJ2Bmkjb2A1sB5wBgR6QPFz3NmGWl9r+8kfxFDRUQaYj0zP9unOgArPeGqehjYaJ/3UKqedtwLKDk63+7QNZZHTw/DsBzgprIiltcxNcW6oANYPafbgaki0s4rThbWnGZQqOoWVa2tqluCiD4cmKa2Ow7Ao6q6z87zGWBIsGWp6rdqTeU1xerJbgrqQsqP5yb+4XM+0nruAL7FWkvLwZoautNPXA8vqOpWVd0H/AtbT7u82qr6bWmJRKQZcDNwf7D2lxdVLcRy7m9jOaS3gZvth9pDFlaPPJR8/V4n1lRWR6z1lnOBTsBTZWQZbh0N5/6Vhy+Av4rIKSJSHeteKtbUkIeI1lF7Guss4PkQ7JygqodVdTXWqN6j57eqGsi2L4B/iEhNEWmF1UGoESB+0Ngdh7eAqar6i306Gav99OYAUNPrc6h6OnGN4erpzTDgjWAiltcx5QD5wERVzVPVBVjTT+d5xakJBOtRg8buXZ3D0SOK0tjq9f9mLCcaEqq6DeuGvhtq2jAZhrVA6Uuk9bwfOANohrURYAIwT0QCVdRw9XwGeFBVfR9Ex7B7do9h1ZVErLn+ySLS0StaTY5uHMJGVXeq6lpVLbI7FvdgjbYDEa6m4dy/sFHV/wEPAB9iddI2YTWc6V7RIlZHRcSFtbnqDlUtCCFpuHqOwmrXfgNmY62fpQdMEQT2dUzHWvy/3SvoENbapze1sDT1EOlnPpxrLFcbas8uNAI+CCZ+eR1TadMxvqOXdngNVcuIGwpXA9+p6u9BxG3m9X9zrLnScEggsov2pSIiZ2Hd+NJuYqT17Ai8p6rpqlqgqm8AdYD2AdKEq2dv4HER2SkinimZRSJyZahGh0BHrKnDZbajWAr8APTxihNI00iglP2shatpOPevXKjqi6raWlUbYjmoBGCNV5RI1tFaWFOc79l1Zql9Pl1EugdIF5ae9qh1qKo2UtUOWPdtSYg2l0BEBGtnXkOstaV8r+Cf8Zqms6d8T+DPqT6I8DMf5jWWtw0dDnykqoeCNbKsBasErJ7Yv7E8fjXshUesHTIbsHaUJGANt7MouXD3JfBXP3nXwFoMaxPGQtqvwHVBxFOsLaZ1bHF/AW4KsoyhQHP7/xbAAlvc8ix8+tXTK84rWFOUpaWPqJ5Yvd9vsR4aF5bDP4y1BlBa/E3Aaqypzbp22oeDLKsBVq/JcyjWYnZ1pzTFGiFlAB3tz6cBe4HzfPS+J0D+O73jB2FPL7u+iF3nvsbPZoAI1NGQ7l8E9KyGNX0vWA3UfN/7H8k6apfjXWfOsPU6jlK2pPPnYv1bdlkdsLZEB3X/sJxCPcCNtX6TQfm3378MLAaSSwmrjzVaH2Rr+yiw2CfOeqCLn7zbYo1+UkKwJ/2iPUwAACAASURBVOhrLK+edh7V7Ws8N+g0QWQ63jbM+xjvFd4Ba4vqYaydWpd6hTXGGiKW+p0GO86DWBsYMrEaqeZYw9vmAdJ0s8urGYT93jue9gJPAm47LGBZWOsn6XZZ6VgNWL1yVtKy9Kxma9G7lLQR19Mu70WstYqDWFvTzw+Q/yb+3JWXiTXdWMMr/BDQPUgtlMh8j6ksTW/H6kBl2fXgbz7Xnw40DJD/CFufTOwGN9B1Ym1N34b1vZStwHOB6mo562hI96+8emKtdayyn4mdWM7L7WQd9UmbRmi78nbi1enA2lZ/KED+f7XTZWN956hfObVsYduTS8nv8wz1itMHqzOSg+Xo07zCzgB+LKOM1+x6k4k10xKxayyvnnacIVjTfxKsbmIndAQReRLYqKovOVZI2TYo0FpVN0TLhkgRI3puAm5Qa60h7hGRkUAzVb0nijaYOhq58tOwvltVRUNbk4pJRORDYIqqfhal8tOIgp6OOqZYoDI99LFAZXNMsYCpo5GjsjmmaBMtPc1LXA0Gg8EQU1T6EZPBYDAY4gszYjIYDAZDTJEQTqLU1FRNS0uLsCmxy/LlyzNUtb5T+Rs9I4vRM7Ica3qC0TTShKpnWI4pLS2NZcuWhZM0LhGRzU7mb/SMLEbPyHKs6QlG00gTqp5mKs9gMBgMMUVYIyaD4Vjmt99+Y/DgwaxcuZKLLrqIk06yXsy8Zs0a5syZg4igqsyfP58ePXqUkZvBYPAlJhxTYWEhOTk5pKenM2fOHGbMmMHBgwe54YYbALjllluoWrUqVao4/RthlY933nmH999/n9mzZyMi9O7dm2effZZ27dqVndhQKr/99hsrV65ERPj444/55JNPrG+riyAi3HDDDZx00kl069Yt2qYaDPFJOK/Z6NSpk0aKxx57TN1utzZu3FivvPJKHTJkiP7lL39Rt9tdfLhcLnW73Xruuedqbm5uxMoOFmCZluO1JGUdkdTTm2+//bZYw3Xr1unhw4d18eLFmpSUpG63W9evX+9IuWURr3p646mTixcvdryssqgMenqTm5urS5Ys0f3796vL5VIR0cTERN25c2eF2VCZNP300081NTVVzzzzTH3rrbd09+7dxWEFBQW6bds2x20IVc+orzHdcsstbN26lbVr1/LWW2/x9ttvM3PmTJYsWcJf/vKXEnEXLFjA0qVL/eRk8OXpp58G4O9//ztt27alRo0anHnmmcydO5fk5GS6du3Krl27omxlfHLRRRehqtx7773RNqVSsXPnTjp27EiPHj3Izs6mX79+ABQUFPDbb79F2br4YtWqVfzvf/9j3rx5jBw5knnz5nHllVdSv/6fm+Nmz55Ns2bNAuQSJULxYlrB3j4vL0+HDRtW3Dt1u92anJysK1asqJDyPRCnvadBgwap2+0utae5dOlSdblcOnjwYM3Ly3OkfH/Eq57erFq1St1ut9asWbNEDzQaVAY9VVVzcnK0a9euKiLavXt3VVWdOnWqioi6XC696aabKsQO1fjX9Pvvv9dhw4YFnGHKycnRE044QV0ul6O2qMbhiCkQVapUYerUqVx55ZXFBmdnZ3P66aeTl5cXbfNiHo9mn3zyyVFhnTt35pprrmHGjBn8+OOPUbAuvjn55JPp0aMHhw4dYufOYH/t2+CPjIwMhg8fzg8//ECXLl2YO3cu8OfIVFXZunVrGbkYPKxfv56uXbtStWpVv3FatWpFQUEBlt+ILWLaMXmYNGlS8cKy5xg6dGi0zYp5+vTpg4hw8803M2jQIGbMmFGiEZ00aRJnn312bA7l44BTTz0VEeH++yvsl+IrJZs2bWLIkCF88MEHiAhXX301LpfVNNWpU6f4mf/+++/JyMiIsrWxz/79+7nrrrto2rRpwHgul4uMjAwGDx5cQZaFQCjDK62gYagv+fn5Onjw4BIbImrXrl1hi6HE8bDedwNJSkqK3nnnnbp7927duHGjVq9eXadOnao5OTl66NCh4iMnJ8cxm+JZT2927dpVPM3ku5Fk5cqVet999xWHb9myxTE74l3PESNGqMvlKj5ERC+++GL98MMPNScnp1hDl8ula9ascdQWD/Gs6ZAhQ/Tyyy/XwsJCv3F++uknTUxM1CuvvLJCNpSFqmdcjJgSEhI4//zzS5xzuVwkJiZGyaL44e677yY5Obn486FDh3juuedo3LgxrVu3Ji8vj5deeokOHTqQkpJCrVq1SElJ4eqrr46i1fFBgwYNinvzv/32G7t372by5Mmce+65nHbaaTzyyCPF4V27/n97Zx4dRZXv8c/trHRCEghL2Bk0shMgBB0YdtQBQRGCorIZfA6b4FMwjoOCgMM8lTkYH4sRUEEGFAhn5Cj4ZHcDAVFImLAEE0CWsAUIkEjIfX9Up+2ETro76U5Vhfs5p0+6qm7V/fY3t+t31+r7yM7O1luyIdm+fbv9fYsWLXj++eepWbMmQ4cOpUWLFjoqMy9z5syxtzpLcunSJTp27EhERAQLFy4ss7tPL0wRmABGjBjBP/7xD/t2Tk6Ounm6wZw5czh27BivvfYaw4YNc5pmz549ZGX59AksVZaiMZABAwZQr149nn32WbZv346UkoSEBL777jtOnTrFmTNniImJ4fLly3pLNhy7du1i8+bNZGZmkpaWxj//+U8++OADMjMzycvLA37v2VG45sEHH6ROnTpOj2VnZ9O+fXuklKxevZqwsLBKVucepghMubm5bN26lYyMDHsBLSws5Ny5cxQUqN8Cc0XNmjWZPn06H3/8MQUFBWzYsIEGDRpQWFhYrPkcGRnJ2rVrycjIYPXq1XrLNgWzZs0qNvYZFRXFK6+8QlZWFvPnz+fee++lTp06jBkzhuzsbKZOnaq3ZMNRvXp1evbsedtYZ6NGjcjMzASw+6twTdE4XVFQL+LmzZucO3eOq1ev8uCDD3LffffppNA1hnjygzMuXbpEamoq69at45133rEXyqK/FouFvXv3Mnz4cFatWqWnVNNx//33k5SUxODBg4t92UNCQhg0aJCOysxHWlqaPbDPmzeP5557zmm65ORkFi9ezJIlS5g3bx5Wq7WSlZqT4OBg4uLi7qgHnlaUkSNHArBu3TqaNWvG0aNHyc7O5sCBA/Tr1w+AGTNmGHooxJAtpvz8fIYMGUKvXr1ISkoqM+2aNWtuqxko3Cc4ONg+BnX8+HE1w8xDUlNT7S2lohtCaRTV+rdt21Y54qoIXbt2tb//5ZdfdFRiHkaOHMn999/PV199xeXLl4mPj2fp0qXExsZy+fJlQ7eWwKCBKS8vj7p169q3e/ToweTJk5k8eTKRkZHF0jZq1Eg18StAv3792LdvH1FRUQDMnz9fZ0XmYuDAgTRr1oyZM2cSHh5earojR47YW1b9+/evRIXmp1evXnbvtmzZorcc01CrVi2mTZvG+PHjadSoEfn5+Tz00EOmKH+G7MoLDw9n+fLlLF68GNAW2hY1O3v16lWsu6lLly6GnFVidFq1agVozf1hw4bx8MMPk5ycTE5Ojs7KzEF2djbDhg1jy5Yt/PTTTy675mbMmIEQgo4dO1aSwqpDp06dsFqt3LhxQz0YtwJkZmZy+PBhVqxYobcUlxgyMIE2Rdzf/3Z5AwYMIC4ujh9++AGADRs2VLa0KkF0dDT79+8nJibGmAvsDE5SUhI7duygd+/eTJkyxb4/OjqaI0eO2P8CrFixgk8++YQxY8aoFmk5iIiIICQkhBs3bvDxxx8zdOhQvSWZjhMnTtChQwcaNGhgisqRYQNTWUycONHen9+hQwed1ZiXli1bEh8fz5o1a/SWYjrq1KmDlNpvLhVNDy/6HSZnf5OTkxkzZozesk1JtWrVCAgIQErJ+vXr9ZZjSo4cOUJeXh5vvvmm3lLcwpSB6dFHH6VTp07s3buX7du3c+PGDapVq6a3LNNhsVhITk4mJiaGV199FUDdPN1k3LhxCCHsi2ZTU1MBaNOmDampqZw4cYLY2Fj7ehLla8WwWCxqLLkCLFu2jLi4OB577DG9pbiFKQOT1Wpl0aJF/Otf/+L69ev4+fnpLcm0hIWFkZiYiL+/P5GRkSQkJOgtyRQEBASUOjVc4X2eeeYZXn/9db1lmJIzZ86wbNkyIiIiOHTokCl+JNSUgQm0LjzVjecdLBaLWvipMDSJiYk0adLENF1RRiI9PR2Al19+2RRBCUwcmBQKxZ1DUFAQo0aNYtSoUXpLMR09e/aksLBQbxkeYch1TAqFQqG4cxHleTCiEOIccCc99bOJlLK262TlQ/npXZSf3uUO9BOUp97GIz/LFZgUCoVCofAVqitPoVAoFIZCBSaFQqFQGAoVmBQKhUJhKFRgUigUCoWhUIFJoVAoFIZCBSaFQqFQGAoVmBQKhUJhKFRgUigUCoWhUIFJoVAoFIZCBSaFQqFQGAoVmBQKhUJhKFRgUigUCoWhUIFJoVAoFIbCp4FJCDFXCDHOl3ncaShPvYvy07sIIdYKIfrpraOqcMf6KaUs9QUEAUvQfjfkKvAT0M/heCCwBsgEJNCzxPn1gBNAYCnXb2o7z78sHSXOGQXsBa4AJ4E3PTnfkxewCMh1eOUDVyt4TVee3gd8BVwEzgGrgXo+9nQYcAi4DGQDHwFhPvJ0NHCrhK89K3A9V362AvYAl2yvTUArH/spgNnArzZPtwGtfeSnAP4GHLd9J1ZV5H/nys8SaV+zedPXYV9nYG8Z1+8JnPRQ0wzgZoky08wXfjrkWdP2/fumgtdxVT6LypfjZ3vVx372Arbaymamj32cCqTaPvsvwFR3znPVYvK3fWl7AOHANOBTIURThzTfAMOBMyVPllKeBtKBh13k4wlW4HmgFnAv0AeY4sXr25FSjpVShha9gJVogaIiuPK0BpCMVmCboP1DP3DQ5AtPvwW6SinDgWY2jbO9eP2SfO/oq5RyWwWu5crPU0A82o2mFvAZ2s0b8JmfQ4EEoJst3++B5V68viMjgRFAV6A+UA14twLXc+c7jxDiLrTPedpxv5TyByBMCNGpAhqc8UmJMnPMy9cvyf8A//HCddzyE4hw+Gyzinb6yM9rwFK0oOFrBFoZrQH8GZgohBjm8qxyRMD9wBAn+0/ipOaLVpv7oJRrHad4beGP5dDzArC+jOMSmAQcA84DbwGWcuQTghYkevigVuHUU9uxjpRopfnSUyAUWAZ84QtP0VpMFaqFVqCM+gMTgOu+9BNIBD512G4N5JWRPhP4K3AQrVX3ARDs5mddg0MtFOgC5AFWX/oJbAT627T3LXHsfWC6k+uEADeAQgc/67uR/wzgYze19rTdi16xlc1M4CkPP28XtMrE074oq45+4kaL3Nt+OpzfFxctJgd9z6JV8k4DUyrw2ZOAd12l82iMSQhRF7gHSPPgtP8AMaUc6277W1Rb+F4I0VgIkSOEaOzm9bu7oedRoBPaTf4RtNosHuY1BK1pv8NNXW7hhqfOPp/XPRVC/EkIcRkt+A4B5rmQXhFPOwghzgshDgshXhVC+LvIy21K81MIkYN2w34X+HuJ07zt5yrgLiHEPUKIALTu540upD8FPAjcZdM/zVG7EOJPZZwrSrwPAqJd5OcWzvwUQgwF8qWUX5RymlM/pZTXgH7AKfl76+CUrezluJAyUAhxUQiR5saYYBRa67gBmvfJQojmNu1PCiH2l3aiEMIP+F9gItoN2auU8X3PEkKcFEJ8IISoVeKYL/z0lF5oZeoBIFEI0df2edzOSwgh0HoRXMcPDyJdAFr//HulHC+txXQ/cMxFNC7XGBHazfAkUKuMNBL4s8P2eGBzOfLaDMwob02hnJ62Qxtr6laJnjZAq6He4wtP0boK/4A28aYtWivhr5XkZ4hN60O+9BNt7PUd23kFaH3rfygjfSYw1mG7P5DhZl7PAIdtOsPRuiol5eh9cMdPoDpwBGjqoL1ki+m/gC2lXLMnno+JtELrpvRDa82cBp4o4/oFQIjDvk9xGLdxkdd/Awtt70fjxRZTKX6GolXw/IG6aC3gL33pp8O5nrSYWjjsexNYUo78Xgd+BoJcpXWrxSSEsKD1kf+GVpPwhOqAt6M3QohBwBy0gcTzLpKfcHifhVbIPcmrMVoBWObJeS6uWaanQoi7gQ3AZCnl1yUO+8RTACnlr2i1+1UukpbLUynlMSnlL1LKQinlAWAm2hhQhXCnjEqthrkIWCaEqONwyNt+vgbEAY2AYLQv5BYhhLWMc8pbRpeijX1uQ6uJbrXtP+mB3tsow88ZwHIpZWYZp3vVTynlQSnlKSnlLSnld2hBv6wyc8n2vy7CLT+FEPXRuqj/ViHBzq/t1E8pZa6Uco+UskBKedZ27AEhRHWH0332ffeAit5DJ6KNNT0kpcx3ld5lYLI1v5agRfMhUsqbnggCWqJFSWeUq6kshPgzWr/rQNvNzRWNHN43Rusr9YQRwLfSSwOurjwVQjRBq1nNklI6GzT3uqcl8EfrUiqLinpahKR4V5THeFhGLWgTaBo47PO2n+3RButP2m44H6IN/rYq45xy+WkL8NOllE2llA3RgtOvtle5cOFnH2CSEOKMEOKMTfenQohEhzS+Lp+uykwNIUSIw7a7fnZGm6V50PbZ3gE62z6rX3nFelg+i/xxvDf72k93KPf3XQiRALwM9JFSuldhcqP5tQjYCYSWcjwIrVZ4Eq3/MRgQDsf/D3islHOtaFOHS+02cnJOb+AC0N3N9BKtG66Gzdx04FkPm6CHgARPm67l8RTthplBGQOMPvD0KaCx7X0TYDuQ4gtP0frE69ret0CbSjrdh37eD3RA6wYKQxt8PYXD5AIf+DkdbbZqXbQbzAi0mVARpaTPBA4ADdFm8X0D/N3NvGqiVSIEWuBL9bR8e+hnJNoYTtHrBNrsvFCHNIeBzqVcuwXagH24B3oesZU1gRY8fgVGlZK2J1pX3ttoXardbN63cCOfoBKfbTKwC4jyoZ/3As1t5SQS+ATYWiKNt/20oN2n+6G1foJxvVxihe270BptSckDbub1FNqM7ZYeeebiok1sovIoPs/+KYc0mbY0jq+mtmP10AKW0w9tSzMTbVJBDtoansa2PBqXkn6rreA56tlQxvUlv88guwDMBfxsx8rMy5bmj7aCXb0ihdNdT9FuarLEsVyH833h6Ru2a16z/U0GIn3hKdoN46wtr2M2rQE+9HMoWuDMtXnyOdDOx34GA/PRxkKuAD/iMCbnJH0mv8/Ky0FbR2Z1OJ5LiXFGh2P3oFWcrqPdZF7wZfksRbvjOqY44EcXeSy1lZsctC6hbo5l3En6lbb0ubb/5aQy0va0/T//hjYr7zgwwuH4U0Cam16MpuLrmFyVzyfQxiCv2crLMhwCoY/87Mnt9+xtpaRtSvFZeWeAlxyOu8rrF25fg7bIlW/CdrJPEELMRRvEXeCzTFxrkEC0lPKoXhq8ifLUuxjEz0zgGSnlJr00eAshxFq0gfHSZuz5Ov+eaFPLG+qRv7cxgJ9N0YJLgJSyoLLy9do0XWdIKV/05fXvRJSn3kX56V2klEP01lCVuFP9VA9xVSgUCoWh8GlXnkKhUCgUnqJaTAqFQqEwFOUaY6pVq5Zs2rSpl6UYl717956XUtb21fWVn95F+eld7jQ/QXnqbTz1s1yBqWnTpuzZs6c8p5oSIUSWL6+v/PQuyk/vcqf5CVXL0yFDtPkTa9eurZT8nOGpn6orT6FQKKowKSkppKSk2AOUGVCBSaFQKO4AUlJS9JbgNiow3aFkZGSQkZHBwoULCQkJYfTo0WzatImcHL2fFWk+8vPzWbp0KUuXLmXIkCFYrVbmzZvHjRs39JZmCq5du8YjjzzC2rVrSU9P11uOwgiU5zEbsbGx0tccPnxYvvnmmzIwMFAGBgbKgIAAaXvigBRCSCGEDA8PlzNnzpR+fn5y+PDhPtMC7JEVeCyJq5ev/XzppZekxWKR0dHR0mKxSIvFIoUQ9veDBg2y75s7d65PtUhpfj+LuHbtmoyPj5d+fn72l8ViKbYdEhIiR40a5VMdZvczKyvL/p0ODQ2VqampPs3PHczuqSPz58+3P3pILzz106dPfvCE48eP8/3337Nq1Sp+/vlnfv31V27dukVhYSHjxo2jfv36NG7cmBYtWhAT8/tvZgUEBDB9+nRWrFjB8uW++vVq8/LGG2/w1ltvIYQgIyODtm3bsnjxYho1aoQQgvfff5/du3cD0KVLFwYMGKCzYnNw7do1EhISWLduXZnp8vLyOHv2bCWpMi9RUVGcOXOGkJAQ/Pxuf5B3fn4+p0+fJiwsjIsXL3L+vPZLNxEREbRo0aKy5ZqKPn362N8fOnSI5s2b66jGPQwRmMaNG8fy5cu5fv26fd/EiROpUaMGY8eOpU6dOk4LK8DNm9oT5Pv27VspWs1Ggwa//7rDww8/zLJly6heXfuplwULFvDuu+9y7tw52rVrx5dffklISEhpl1I4sHXr1mKznAICAliyZAldu3ZlzZo1JCYmlnG2oohbt25Ro0YN4uLiWL9+PdnZ2XTr1u2273tBgfaYtuDgYCwWC9OmTcPf358ePXroIdtUNG/enMGDB5OSksLmzZtVYHKXgwcP2oPShAkTmDZtGrVr18ZicT0Elp6eTkBAAOvXr/e1TFMyevRonn76aQBiYmKYPXs26enpfPbZZwgh6NixI6+99hrjx4/XWam5SEpKKrYdExPDk08+CcBdd7n6KStFETdv3qRGjRoUFhYCULduXXsls3379ty8eZO0tDQmTZpEVFQUgYGBhIaGqgqUh/Tp04eUlBQmTJhgiu+6IQLTl19+SV5eHgBhYWFuBSSAAwcO0K1bN3r06EFQUJAvJZqaw4cP061bN2bNmmXf9/bbbxMfH0+9evUIDAzUUZ356dChAxs3brRvv/fee8WOR0VFVbYk0xAcHMz+/ftp06YNoPV8FHXJSynJz8/Hz8+PgIAAPWVWKRYsWGD44GSIWXnBwcFEREQQERHhdlD67bffeP3117ly5Qpz5szxsUJzEx0dTe/evYvtu3jxIjk5OSooeYG2bdsSEREBwI8//sj27duLHVfdemXz2Wef2d/v2LGD5ORkJk6cyIQJE7BarcTExJCSkmKvvCoqxubNm/WW4BJDBCZPyc/P5/nnnyclJYWgoCBiY2P1lmR4PvroI3bs2MFf/vIXGjZsyJw5c+jUqRO9evVi586d9rE6hWtOnTpVbFpzkyZN7O937tzJb7/9Zt9u37499evXr1R9ZuOZZ56h6PE8x48fZ+zYsSxYsIBFixbRuHFj2rZtS3x8PG3atOHoUdP/BJjumGGxrekC04kTJ3jhhRdYtGgRwcHBJCQk6C3JFAQEBNC1a1cWLFjAwYMH2blzJ+Hh4ezYsYOuXbsyadIk9u3bp7dMU3Dy5ElOnTpl387JyaGgoIB58+bx3HPPFUsbHR1NWFhYZUs0FbVq1WLEiBFYrVZq1qwJaF36L730EpmZmUydOhWAY8eO0bt3b9Vy8gKGX2zrydxyqcMcfEfy8vJkv3797Gse5s+fXyn5UoXWNJQkLS1NtmrVyr5GbPHixT7Psyr42bdvX/s6MMcXUGw7MzPT51qqgp9SSnnmzBl54cIFmZWVJbOzs4sdK1rHKISQjz/+uM+1VBVPHUlPT5fp6en2NU2DBw+utLw99dM0LabTp0/Tr18/Nm7ciNVqZeXKlYwcOVJvWaanVatW7N69mxUrVhAZGcm4cePQypGiLIQQTl8Wi6XYtsJ96tatS82aNWncuDG1a5f+IOqidXcKz2jevDnNmzdn/vz5gNZqWrBggc6qnGOKwJSfn88bb7zBtm3bAK175PHHHyc0NBSA3NxcHdWZH6vVyhNPPMGuXbu4deuW6ipxg9mzZ+st4Y5h37593Lp1y75t+G4og+M4I2/ChAkcOnRIRzXOMXxgunDhAi+++KI9sj/wwAOsXLkSgCtXrnDs2DF27dqlp0RDUlhYWGwQ3h2aNm1K69at+fzzz32kqurQvn17Dhw4QEJCAgkJCbRs2dJpurlz51aysqrH2bNni7XiHRfiK8rm0KFDTlv2jhjxyRmGWMdUGllZWcTGxnLx4kWsVitJSUkMHjyY8+fPM27cOL755htGjRrFlClT9JZqOLZt20ZiYqJH3R5+fn74+/urh4+6QWBgIC1btiQ5ORmA7Oxsp7Pvzp07V9nSqhxr1qyxv7darfj7G/q2ZSjMMDXcGYb8D+fm5tK5c2f7lNwffviBH3/8kVmzZjF27Fj740uKZvAobqd37958++23VKtWjdq1a7Ny5Uo6depU6kLkgoIChg8fzs8//0x8fHwlqzU/4eHhTvevXr2a6dOnG7JWagYSExNZunSpffv8+fMEBwfrqMhcjB8/nj59+jgNUBMmTNBBkXsYLjBlZGQQGxvLlStX7Pvi4uIQQhAREUFSUhJjx47VUaF5CAwM5MCBA/Tv35/u3bvTsGFD/v3vf9O+fXt7mtzcXHbv3s2MGTP4+uuvEUJQrVo1HVUrFBrHjx+3t0hr167N4sWL1YLwclA06aEkRn76g+EC09GjR4sFJYDQ0FBefvllJk6cWGrNVOGcu+++m9TUVNLT0xk0aBCdO3e2P3fsp59+4osvvuDy5cuA9nTxhQsX6qzYvAQFBd02cUQtsC0/M2bMsJfN7t27M3DgQJ0VKSoLwwWmtm3bEhkZyYULF7BYLMTGxrJp0yb7E7EVnhMYGEi7du1IS0vj9OnTAMycOdM+iWTSpEk8+eSTtGrVSj0cs5wEBQWxb9++YpMg4uLieOutt9QC23Jw9epVcnNzGTx4MMOGDaN///56S1JUIoYLTPXr12fRokVcvHiR1q1b06VLF70lVRmqVatGs2bNAPjwww/58MMP9RVUxbjnnnuKTWtWlJ/q1avz6aef6i1DoROGC0yA4Z/jpFAoFArfYfh1TAqFQqG4sxDlefyM/0ZlggAAAHxJREFUEOIckOV9OYaliZSy9GekVBDlp3dRfnqXO9BPUJ56G4/8LFdgUigUCoXCV6iuPIVCoVAYChWYFAqFQmEoVGBSKBQKhaFQgUmhUCgUhkIFJoVCoVAYChWYFAqFQmEoVGBSKBQKhaFQgUmhUCgUhkIFJoVCoVAYiv8Hvv2h2sDcF+8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15537f65c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Some images that our MLP struggles with\n",
    "miscl_img=X_test[y_test!=y_test_pred][:25]\n",
    "correct_lab=y_test[y_test!=y_test_pred][:25]\n",
    "miscl_lab=y_test_pred[y_test!=y_test_pred][:25]\n",
    "\n",
    "fig, ax=plt.subplots(nrows=5, ncols=5, sharex=True, sharey=True)\n",
    "ax=ax.flatten()\n",
    "for i in range(25):\n",
    "    img=miscl_img[i].reshape(28, 28)\n",
    "    ax[i].imshow(img, cmap='Greys', interpolation='nearest')\n",
    "    ax[i].set_title('%d) t: %d p: %d' % (i+1, correct_lab[i], miscl_lab[i]))\n",
    "ax[0].set_xticks([])\n",
    "ax[0].set_yticks([])\n",
    "plt.tight_layout()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}