neural_net_optimizers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 2: Adding optimizers to neural net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: (60000, 784)\n",
      "Testing set size: (10000, 784)\n"
     ]
    }
   ],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "labels={\n",
    "    0:'T-shirt/top',\n",
    "    1:'Trouser',\n",
    "    2:'Pullover',\n",
    "    3:'Dress',\n",
    "    4:'Coat',\n",
    "    5:'Sandal',\n",
    "    6:'Shirt',\n",
    "    7:'Sneaker',\n",
    "    8:'Bag',\n",
    "    9:'Ankle boot',\n",
    "}\n",
    "def load_mnist(path, kind='train'):\n",
    "    import os\n",
    "    import struct\n",
    "    import gzip\n",
    "    import numpy as np\n",
    "\n",
    "    \"\"\"Load MNIST data from `path`\"\"\"\n",
    "    labels_path = os.path.join(path,\n",
    "                               '%s-labels-idx1-ubyte.gz'\n",
    "                               % kind)\n",
    "    images_path = os.path.join(path,\n",
    "                               '%s-images-idx3-ubyte.gz'\n",
    "                               % kind)\n",
    "\n",
    "    with gzip.open(labels_path, 'rb') as lbpath:\n",
    "        struct.unpack('>II', lbpath.read(8))\n",
    "        labels = np.frombuffer(lbpath.read(), dtype=np.uint8)\n",
    "\n",
    "    with gzip.open(images_path, 'rb') as imgpath:\n",
    "        struct.unpack(\">IIII\", imgpath.read(16))\n",
    "        images = np.frombuffer(imgpath.read(), dtype=np.uint8).reshape(len(labels), 784)\n",
    "\n",
    "    return images, labels\n",
    "\n",
    "X_train, y_train = load_mnist('data/fashion', kind='train')\n",
    "X_test, y_test = load_mnist('data/fashion', kind='t10k')\n",
    "y_train,y_test = y_train.astype(np.int64),y_test.astype(np.int64)\n",
    "print('Training set size: {}'.format(X_train.shape))\n",
    "print('Testing set size: {}'.format(X_test.shape))\n",
    "\n",
    "from matplotlib import pyplot as plt,cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72.94035223214286 90.02118235130519\n",
      "(60000, 784)\n",
      "-1.74808013869143e-17\n"
     ]
    }
   ],
   "source": [
    "# Normalize dataset (preprocessing for Neural Net)\n",
    "\n",
    "n_mean = np.mean(X_train)\n",
    "n_std = np.std(X_train)\n",
    "print(n_mean,n_std)\n",
    "\n",
    "X_train = np.array((X_train - n_mean)/n_std)\n",
    "X_test = np.array((X_test - n_mean)/n_std)\n",
    "\n",
    "print(X_train.shape)\n",
    "\n",
    "print(np.mean(X_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model.metrics import accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model.activation_classes import ReLU,Sigmoid,LeakyReLU\n",
    "from model.neural_network import CustomNeuralNetwork"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model.optimizers import GradientDescent,Momentum,RMSProp,Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimizer 1: gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[784, 100, 10]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "act_obj = ReLU()\n",
    "layers = [X_train.shape[1],100,10]\n",
    "layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 2.9318330049378236, Val loss:2.8177904179101847\n",
      "Epoch 2. Training loss: 2.6887637758816396, Val loss:2.8940367419215995\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xd8VFX6x/HPQygRQpPQQQEr6QlDRFGpywKuqFioumBhQSyIoogFAXURERDFgiI/FVZWRRSxYFkWrMAQQugLCmogIEUQBIWE8/vjTOIkpEzIJHfK83698jLJ3Mw8F/CZk3PP/R4xxqCUUip8VHK6AKWUUhVLG79SSoUZbfxKKRVmtPErpVSY0cavlFJhRhu/UkqFGW38SikVZrTxK6VUmNHGr5RSYaay0wUUJjo62rRo0cLpMpRSKmisWrVqrzGmvi/HBmTjb9GiBW632+kylFIqaIjID74eq1M9SikVZrTxK6VUmNHGr5RSYSYg5/iVUhXr+PHjZGZm8vvvvztdiipBZGQkzZo1o0qVKqf8HNr4lVJkZmZSs2ZNWrRogYg4XY4qgjGGffv2kZmZScuWLU/5eXSqRynF77//Tr169bTpBzgRoV69emX+zUwbv1IKQJt+kPDH31NoNf7Ro+GOO2DvXqcrUUqpgBU6jX//fnj+eXjmGbj6ajhyxOmKlFI+2LdvH0lJSSQlJdGoUSOaNm2a9/WxY8d8eo7BgwezefPmYo+ZMWMGc+fO9UfJXHzxxaSnp/vluZwQOhd3Tz8dVq+G/v1h2TJITISXXoKOHZ2uTClVjHr16uU10UceeYSoqCjuueeefMcYYzDGUKlS4WPV2bNnl/g6w4cPL3uxISJ0RvwArVrBt9/Cf/4DxkCnTjB0KBw86HRlSqlS2rp1K3FxcQwdOpSUlBSysrIYMmQILpeL2NhYxo8fn3ds7gg8OzubOnXqMHr0aBITE7nwwgv5+eefAXjwwQeZNm1a3vGjR48mNTWV8847j6+//hqA3377jauvvprExET69euHy+UqcWQ/Z84c4uPjiYuLY8yYMQBkZ2dz/fXX531/+vTpAEydOpWYmBgSExMZOHCg3//MfBU6I35vnTpBRgY8/DBMnQqLFsELL8Df/uZ0ZUoFvhEjwJdpjOPHYdcuaNQISlpTnpQEnqZbGhs2bGD27Nm88MILAEycOJHTTz+d7OxsOnXqxDXXXENMTEy+nzl48CAdOnRg4sSJjBw5kldeeYXRo0ef9NzGGFasWMHChQsZP348H3/8Mc888wyNGjVi/vz5rFmzhpSUlGLry8zM5MEHH8TtdlO7dm26du3KokWLqF+/Pnv37mXt2rUAHDhwAIBJkybxww8/ULVq1bzvOSG0RvzeqleHyZPhm2+gbl24/HI7DbRnj9OVKRUadu2C77+3/y0nZ511Fm3bts37+o033iAlJYWUlBQ2btzIhg0bTvqZ0047jR49egDQpk0btm/fXuhz9+7d+6RjvvzyS/r27QtAYmIisbGxxda3fPlyOnfuTHR0NFWqVKF///4sW7aMs88+m82bN3PnnXeyePFiateuDUBsbCwDBw5k7ty5ZboBq6xCc8TvLTUVVq2Cf/4THnsMPv0Upk+Hvn1Bl68pdTJfR+Z798Ls2TB4MERHl0spNWrUyPt8y5YtPP3006xYsYI6deowcODAQtezV61aNe/ziIgIsrOzC33uatWqnXSMMaZU9RV1fL169cjIyOCjjz5i+vTpzJ8/n5kzZ7J48WKWLl3Ke++9x6OPPsq6deuIiIgo1Wv6Q+iO+L1VrQpjx0Jamr0O0L8/9OoFmZlOV6ZU8IqOhlGjyq3pF/Trr79Ss2ZNatWqRVZWFosXL/b7a1x88cW8+eabAKxdu7bQ3yi8tWvXjiVLlrBv3z6ys7OZN28eHTp0YM+ePRhjuPbaaxk3bhxpaWnk5OSQmZlJ586defLJJ9mzZw9HHFp9GPojfm9xcfD113bE/8ADEBsLTz4JN98MRawWUEoFhpSUFGJiYoiLi6NVq1a0b9/e769x++23c8MNN5CQkEBKSgpxcXF50zSFadasGePHj6djx44YY7j88su57LLLSEtL46abbsIYg4jwxBNPkJ2dTf/+/Tl06BAnTpzgvvvuo2bNmn4/B19IaX+1qQgul8uU+0Ys330Ht9wCS5bYJZ8vvQRnn12+r6lUgNq4cSOtW7d2ugzHZWdnk52dTWRkJFu2bKFbt25s2bKFypUDa4xc2N+XiKwyxrh8+fnwHeaedRZ8/rlt+GlpEB9vLwYXMR+olAp9hw8fpn379iQmJnL11Vfz4osvBlzT94fQO6PSELHTPD16wLBhdr7yzTdh1iz7RqCUCit16tRh1apVTpdR7sJ3xO+taVN47z2YNw+2b4eUFHsx+I8/nK5MKaX8Tht/LhHo0wc2bLD/HT8e2rSB5cudrkwppfxKG39B0dEwZ4692/fgQbjwQhg5En77zenKlFLKL7TxF+Wyy2D9epv1M3WqnfP//HOnq1JKqTLTxl+cWrXguefgv/+FiAjo2tUuAXUwY0OpUNSxY8eTbsiaNm0at956a7E/FxUVBcDOnTu55pprinzukpaHT5s2Ld/NVD179vRLls4jjzzC5MmTy/w8/qaN3xcdOtjQt3vvhVdegZgYezFYKeUX/fr1Y968efm+N2/ePPr16+fTzzdp0oS33377lF+/YOP/8MMPqVOnzik/X6DTxu+r006DJ56wF3ujo+HKK23ejyfyVSl16q655hoWLVrEH56VdNu3b2fnzp1cfPHFHD58mC5dupCSkkJ8fDzvFTLo2r59O3FxcQAcPXqUvn37kpCQQJ8+fTh69GjeccOGDcuLdR47diwA06dPZ+fOnXTq1IlOnToB0KJFC/Z6dvKbMmUKcXFxxMXF5cU6b9++ndatW3PLLbcQGxtLt27d8r1OYdLT02nXrh0JCQlcddVV/PLLL3mvHxMTQ0JCQl5A3NKlS/M2o0lOTubQoUOn/GdbmPBex38qXC5wu2HSJJgwwYa+Pf00DBigoW8qJIz4eATpu0qOZT6ec5xdh3fRKKoRVSKKT5pMapTEtO5Fh7/Vq1eP1NRUPv74Y6644grmzZtHnz59EBEiIyNZsGABtWrVYu/evbRr145evXoVuffs888/T/Xq1cnIyCAjIyNftPJjjz3G6aefTk5ODl26dCEjI4M77riDKVOmsGTJEqIL5A6tWrWK2bNns3z5cowxXHDBBXTo0IG6deuyZcsW3njjDV566SWuu+465s+fX2zG/g033MAzzzxDhw4dePjhhxk3bhzTpk1j4sSJbNu2jWrVquVNL02ePJkZM2bQvn17Dh8+TGRkZLF/vqWlI/5TUbUqPPig3fHr3HPh+utt1v9PPzldmVIVZtfhXXx/4Ht2HfZPLLP3dI/3NI8xhjFjxpCQkEDXrl3ZsWMHu3fvLvJ5li1blteAExISSEhIyHvszTffJCUlheTkZNavX19iCNuXX37JVVddRY0aNYiKiqJ379588cUXALRs2ZKkpCSg+PhnsHsEHDhwgA4dOgDw97//nWXLluXVOGDAAObMmZN3l3D79u0ZOXIk06dP58CBA36/e1hH/GUREwNffgnPPgtjxtivJ02Cf/xDQ99U0CpuZO5t75G9zF49m8HJg4muXvaEziuvvJKRI0eSlpbG0aNH80bqc+fOZc+ePaxatYoqVarQokWLQuOYvRX228C2bduYPHkyK1eupG7dugwaNKjE5ykuyyw31hlstHNJUz1F+eCDD1i2bBkLFy5kwoQJrF+/ntGjR3PZZZfx4Ycf0q5dOz777DPOP//8U3r+wmh3KquICLjzTli3Dtq1g1tvtaFv//uf05UpVa6iq0czqv0ovzR9sCt0OnbsyI033pjvou7Bgwdp0KABVapUYcmSJfzwww/FPs+ll16at6n6unXryMjIAGysc40aNahduza7d+/mo48+yvuZmjVrFjqPfumll/Luu+9y5MgRfvvtNxYsWMAll1xS6nOrXbs2devWzftt4fXXX6dDhw6cOHGCn376iU6dOjFp0iQOHDjA4cOH+e6774iPj+e+++7D5XKxadOmUr9mcUps/CLSXESWiMhGEVkvIncWckxdEVkgIhkiskJE4rwe6y4im0Vkq4icvP9ZqGjZEj75xOb8ZGTYzd4nTdLQN6VKoV+/fqxZsybvIifAgAEDcLvduFwu5s6dW+LId9iwYRw+fJiEhAQmTZpEamoqYHfUSk5OJjY2lhtvvDFfrPOQIUPo0aNH3sXdXCkpKQwaNIjU1FQuuOACbr75ZpKTk0/p3F599VVGjRpFQkIC6enpPPzww+Tk5DBw4EDi4+NJTk7mrrvuok6dOkybNo24uDgSExPz7SjmLyXGMotIY6CxMSZNRGoCq4ArjTEbvI55EjhsjBknIucDM4wxXUQkAvgf8BcgE1gJ9PP+2cJUSCxzedq5E4YPh3fftbk/r7xi3wiUClAayxxcyj2W2RiTZYxJ83x+CNgINC1wWAzwueeYTUALEWkIpAJbjTHfG2OOAfOAK3wpLKg1aQLvvANvvWV3+XK54KGHNPRNKRUQSjXHLyItgGSgYHLZGqC355hU4EygGfYNwnupSyYnv2mEJhG45hob+ta/Pzz6KCQn2x3AlFLKQT43fhGJAuYDI4wxvxZ4eCJQV0TSgduB1UA2UNhC20LnlkRkiIi4RcS9Z88eX8sKfPXqwauvwkcf2aC3iy+2F4MPH3a6MqXyCcTd+NTJ/PH35FPjF5Eq2KY/1xjzTiGF/GqMGWyMSQJuAOoD27Aj/OZehzYDdhb2GsaYmcYYlzHGVb9+/VKeRhDo3t2u/Ln1Vrvnb3y8vflLqQAQGRnJvn37tPkHOGMM+/btK/MNXSWu4xe7IHYWsNEYM6WIY+oARzzz+DcDy4wxv4rISuAcEWkJ7AD6Av3LVHEwq1nTrvnv08fu/NWtGwweDE89BXXrOl2dCmPNmjUjMzOTkPptO0RFRkbSrFmzMj2HLzdwtQeuB9Z6pnIAxgBnABhjXgBaA6+JSA6wAbjJ81i2iNwGLAYigFeMMevLVHEouOQSWLPGbvYyaZKdBnruObjqKqcrU2GqSpUqtGzZ0ukyVAUpcTmnE4J+OWdppKXBTTdBerq9GPzMM9CokdNVKaUqwKE/DrF612pW7ljJVz99xaa9m1g2eNkp3RRXmuWcGtngtJQUWLECJk+GcePsZi9Tp8INN2jom1Ih5MjxI6TvSse90533sWnvJoxnvUudyDoc+P0Ar6x+hXvb31uuteiIP5Bs2mRH/19/DX/9K7z4Ipx5ptNVKaVK6Y/sP8jYnZHX4FfuXMmGPRvIMTkANIpqRNsmbWnbpC2uJi7aNGlDJalUpuyj0oz4tfEHmhMn7Hz/aE+6xcSJdiWQhr4pFZCO5xxn/Z71tsHvWIk7y83a3Ws5fuI4YDONcht87keTmk38Xoc2/lDwww825XPxYmjf3mYAnXee01UpFdZyTuSwce/GfNM16bvS+SPH3pVfJ7KObe6N/2zyZ9Q+o8i9A/xJ5/hDwZln2tU+r70Gd91ls37GjoV77oEqxW96oZQquxPmBFv2bfmzyWe5SctK48hxu0VjVNUo2jRuw22pt+Fq4qJtk7a0qtuqQpp8WemIPxjs2gW33w5vvw1JSTb07RQTApVSJzPGsO3Atnwj+VVZq/j1DxtScFrl00hunJxvyubceudSSQJnClZH/KGmUSMb+PbOO3a+v21bGDXK/gbg5y3ZlAp1xhgyf83Mu+ia2+h/+d3ugVs1oipJjZIYGD8wr8m3rt+aypVCp13qiD/Y/PIL3H03zJ5tt32cNcvm/yilCrXr8C570dUzXePe6ebn334GoHKlysQ3iM934TWuQRxVI6o6XHXp6Yg/lNWta6d6+vaFIUPsXcDDh8M//2kjIZQKY3uP7M03XePe6WbHoR0AVJJKxNSPoec5PXE1dtG2aVsSGiYQWTn8fmvWxh+sunWzoW8PPGDv9n3/fZg5067/VyoMHPj9AKt2rsobya/csZIfDv65LeN59c6jY4uOefPySY2SqFG1hoMVBw6d6gkFX39tb/zatMne8Tt1Kpx+utNVKeU33tEGudM1W/dvzXv8rLpn5ZuuSWmcQq1qtRysuOLpVE+4uegiWL3abvbyxBPw8ccwYwZcfbXGPqigU1K0wRm1z8DVxMWNSTfm3fV6+mk60CkNHfGHmvR0O/pPS7NpnzNmQOPGTlelVKEKRhu4s9ys/3n9SdEGuevk2zRpQ4MaDRyuOjDpiD+cJSXB8uU243/sWIiJgSlTYNAgHf0rR3lHG+R+ZOzOOCna4IrzrijXaAOlI/7Q9r//2Q1fvvgCuna1F381c11VgJwTOWzauynfWvlAiTYIVTriV9a558J//2tTPu+9F+Li7LLP4cMhIsLp6lSIOGFOsHX/1nxr5YuLNnA1cXFW3bO0yTtIR/zh4scfYehQm/9z4YX2xq/WrZ2uSgUZX6MNckfybZu2Dbhog1ClI351sjPOgA8+gLlz4c477bWAhx6C++7T0DdVKO9oA++7Xvcf3Q/YaIPEhokhHW0QqnTEH45+/tmGvr35JiQk2DuB27RxuirlsF2Hd+XbOCRUow1ClY74VfEaNIB//xv69bOhb6mpNu75kUfgtNOcrk5VgNJEG7iauEhslBiW0QahSht/OLvySujY0SZ9TpoECxbAyy/DpZc6XZnyo4LRBu6dbrYf2J73eG60Qe5aeY02CH061aOszz+HW26Bbdtg2DC75WOt8LrlPRTkRht4j+S37N+S93iruq3yZconN0qmdmRtBytW/qJTPar0unSBtWvtBd9p02DRInjhBejZ0+nKVBGOHD/Cml1r8mXKFxZtMDhpsEYbqHx0xK9O9u23NvZhwwYYONCGvkVHO11VWCtNtIGriYs2jdvQMKqhw1WriqQjflU27drZrJ/HH7cfixfb6OfrrtPYhwrgS7SBq4mLXuf2om3TthptoEpNR/yqeBkZdvTvdkOvXvD889BEm4y/eEcb5C6j9I42qF2tdt5FV402UMXREb/yn4QE+OYbO+//0EM29G3yZPtmoM2nVHKjDdw73Xm58quzVvPb8d8AjTZQFUdH/Mp3W7fa0LelS6FzZ3jpJWjVyumqAlJpow1cTVycF32eRhuoU6YjflU+zj4b/vMfu9b/nnts6Ntjj8Edd4R16Juv0QYD4gfkTdtotIFyko741anJzLShbx98YO/8nTXLvhGEAe9og9yP3b/tBjTaQDlHR/yq/DVrZjd4nzfPjvhTUuzG7/ffD1VDp8ntPbI3767X3PXyBaMNepzTI2/KJqFhAqdV0dgLFdh0xK/Kbs8eGDEC/vUvO+p/5RVo29bpqkrNl2gD75F8UqMkoqpGOVewUl5KM+IvsfGLSHPgNaARcAKYaYx5usAxtYE5wBnY3yImG2Nmex7LAdZ6Dv3RGNOrpKK08Qep99+3cQ9ZWXDXXTB+PFSv7nRVhfIl2sB7GaVGG6hA5+/G3xhobIxJE5GawCrgSmPMBq9jxgC1jTH3iUh9YDPQyBhzTEQOG2NKNSzSxh/EDh60Gf8vvghnnWUvBHfs6GhJudEGuSP5lTtW5os2aF6rub0RyjNdo9EGKhj5dY7fGJMFZHk+PyQiG4GmwAbvw4CaYhccRwH7gezSFq5CQO3aNuOnTx8b+tapEwwZYtM/a5f/iPmP7D9Y+/PafGvlC4s26BvXV6MNVNgq1cVdEWkBJAPLCzz0LLAQ2AnUBPoYY054HosUETf2jWCiMebdshSsgkSnTvau37FjYcqUP0PfLr/cby9RUrRBvdPq0bZpW3qd2ytvG0CNNlCqFBd3RSQKWAo8Zox5p8Bj1wDtgZHAWcCnQKIx5lcRaWKM2SkirYD/AF2MMd8V8vxDgCEAZ5xxRpsffvihDKelAsqKFfZO33Xr7OYvTz8N9euX6ikKRhu4s9yk70rn9+zfgT+jDbzn5TXaQIUTv87xe56wCrAIWGyMmVLI4x9gR/NfeL7+DzDaGLOiwHH/Bywyxrxd3OvpHH8IOnbMZvw/+qjN+Z8+3b4JFNKYvaMNcj/SstJOijbwXmGj0QYq3Pn74q4ArwL7jTEjijjmeWC3MeYREWkIpAGJQA5wxBjzh4hEA98AV3hfGC6MNv4Qtn69Hf0vXw5/+xvmuefYHpWdb518SdEG59Y7l4hK4XunsFKF8Xfjvxj4ArskM3fefgx26SbGmBdEpAnwf0BjQLCj/zkichHwoufnKgHTjDGzSipKG39oMsaw49AOVv60HPfCF3Bv/g/uRifY77nfKTfawHskH1M/RqMNlPKBv1f1fIlt5sUdsxPoVsj3vwbifSlEhZ6Sog3iUs6n94ZfaPtZFq7GbYib/BpVz4txuGqlQp8OpZRfeEcb5K6V9442aB3duvBoA2Nszs/dd0NSG5gwwd4FXFn/aSpVXjSyQZXagd8PkJaVlm9evszRBjt2wK23wsKF4HLZN4OEhPI9EaVCiIa0Kb85fOxwXpMvKtogtWkqt7puxdXERUrjlFOLNmjaFN59F956C267Ddq0gTFj7Ee1an48I6WUjvhVnqPHj5K+Kz1fSNnGPRvzRRt4r5VPaZxCver1/F/Ivn12umfOHLvj16xZdh9gpVSRdMSvSuQdbZA7ZVNYtEGf2D4VH21Qrx68/rpd5/+Pf8BFF9k3ggkToEaNiqlBqRCmjT8MHM85zoY9G/LNyRcXbeBq4qJJzSbO3xDVs6dd9z96NEydaqeCXnoJunRxti6lgpxO9YSY0kQb5H6cWftM55t8SZYts/v9btlibwCbPBnq1HG6KqUCht8jGyqaNn7f+BJtkNI4BVdjG1DmauKiVd1Wwbuh99GjMG6cbfoNGsDzz8MVVzhdlVIBQRt/CDLGsP3A9nxz8t7RBpGVI0lulJwXUBbS0QarVtlR/5o1cN11NvenoUYrq/CmF3eDXG60gXemvHunm/1H9wN/RhsMiB8QntEGbdrAypU243/8ePjsM5v4OWBAoaFvSqn8dMQfAHYf3p130bVgtEGERBDfMD7vjte2TdsS1yCOqhGhs6F5mWzcaEf/33wDPXrYzP8zznC6KqUqnI74A1jBaAP3TjeZv2YCf0YbdD+7e96UTV60gSpc69bwxRcwYwbcfz/ExsITT8DQoVApSK9lKFXOdMRfjryjDXLn5cscbaCKtm2bXff/6adwySV2v99zz3W6KqUqhI74HXD42GFWZ63ON2VTLtEGqmgtW8LixfDqq3DXXTbrZ9w4GwCnoW9K5dER/ykoTbRB7l2v5RJtoIqWlQXDh8OCBZCSAq+8AomJTlelVLnREb8fFYw2cO90s+7ndXnRBg1rNKRtU4eiDVTRGjeGd96B+fPtG4DLBffdBw8+CJGRTlenlKN0xO/FO9ogdzSfsTuDYznHgD+jDby3AQyIaANVvP37YeRIOwV0/vk29O2ii5yuSim/0hu4fJBzIofN+zbnWysfEtEGqmiLF8OQIfDTTzb6+fHHIUovpqvQoI2/gBPmBN/t/y7fhdeiog1y18oHdbSBKtqhQzbjf8YMu95/5kzodtKuoUoFnbBu/AWjDdxZblbtXMXBPw4Cf0Yb5GbKh3S0gSral1/a0LfNm2HQIJgyBerWdboqpU5ZWF7cPZZzjO5zurM8czlHso8Af0Yb9I/vH57RBqpoF18M6ek28mHSJPj4Y/tbQO/eTlemVLkLmQ5YNaIqWYezOJJ9hN7n92bMJWOIaxBHtcq6bZ8qQmSknee/9lq48Ua4+mr78eyz0KiR09UpVW5CahL7i8FfMKnrJF68/EXaNGmjTV/5JjkZVqywbwKLFtntHl99FQJwGlQpfwipxh9dPZpR7UcRXT3a6VJUsKlSxWb9pKfbxj9oEHTvDtu3O12ZUn4XUo1fqTI7/3y729ezz8LXX0NcHDzzDJw44XRlSvmNNn6lCqpUyd7tu26dvQh8xx1w6aWwaZPTlSnlF9r4lSrKmWfCRx/Z+f4NG2zWz+OPw/HjTlemVJlo41eqOCJwww12w5deveCBByA1FdLSnK5MqVOmjV8pXzRsCG+9ZUPfdu2yzf/+++0G8EoFGW38SpVG79522ufvf4eJEyEpyd4FrFQQ0cavVGnVrWsTPj/9FI4ds7t93XabzQFSKgho41fqVHXtCmvXwp13wnPP2aWfH3/sdFVKlajExi8izUVkiYhsFJH1InJnIcfUFpH3RWSN55jBXo/9XUS2eD7+7u8TUMpRUVEwbRp89RXUqAE9ethpoH37nK5MqSL5MuLPBu42xrQG2gHDRSSmwDHDgQ3GmESgI/CUiFQVkdOBscAFQCowVkQ0AlGFngsvhNWr7Q5f//qXvfv3rbc09kEFpBIbvzEmyxiT5vn8ELARaFrwMKCm2F1KooD92DeMvwKfGmP2G2N+AT4FuvuxfqUCR7VqMGECuN3QvDlcd529GJyV5XRlSuVTqjl+EWkBJAPLCzz0LNAa2AmsBe40xpzAvkH85HVcJie/aeQ+9xARcYuIe8+ePaUpS6nAkpgI3377Z9xz69Z2s3cd/asA4XPjF5EoYD4wwhjza4GH/wqkA02AJOBZEakFFLZPYaH/+o0xM40xLmOMq379+r6WpVRgqlwZRo2CNWvsG8FNN9mdvrZtc7oypXxr/CJSBdv05xpj3inkkMHAO8baCmwDzseO8Jt7HdcM+1uBUuHh3HNhyRJ4/nlYvtyu/Hn6acjJcboyFcZ8WdUjwCxgozFmShGH/Qh08RzfEDgP+B5YDHQTkbqei7rdPN9TKnxUqgRDh8L69dChA4wYYdf+b9jgdGUqTPky4m8PXA90FpF0z0dPERkqIkM9x0wALhKRtcDnwH3GmL3GmP2ex1Z6PsZ7vqdU+GneHD74AObMgf/9z24A8+ij9iYwpSpQyG22rlRQ+Plne+PXvHmQkGDvBHb5tE+2UoUqzWbreueuUk5o0ADeeAPeew/27oULLoB779XQN1UhtPEr5aRevezc/003wZNP2tH/0qVOV6VCnDZ+pZxWpw7MnAmff263eOzYEYYNg18LrppWyj+08SsVKDp3hoyz9MwaAAAPWElEQVQMGDnSvhHExsKHHzpdlQpB2viVCiQ1asBTT9mN3mvVgssug4ED7XUApfxEG79SgeiCC+z2jmPHwptv2tiHefM09kH5hTZ+pQJVtWrwyCOwahW0bAn9+sGVV8KOHU5XpoKcNn6lAl18PHzzDUyebHf9iomBl17S0b86Zdr4lQoGERFw99324m9KCgwZAl26wHffOV2ZCkLa+JUKJmefbZd9vviinQKKj4cpUzT0TZWKNn6lgk2lSnbEv369HfXffTdcdBGsW+d0ZSpIaONXKlg1awYLF9roh++/t1NA48Zp6JsqkTZ+pYKZCPTtCxs3wrXX2lVAbdrAihVOV6YCmDZ+pUJBdDTMnQvvvw+//GI3f7/nHjhyxOnKVADSxq9UKPnb3+zc/y232DuA4+PtDmBKedHGr1SoqV0bXnjBNnwRmwH0j3/AwYNOV6YChDZ+pUJVx4523f+oUfDyy/bGr/ffd7oqFQC08SsVyqpXh0mT7Ebv9erZ/P9+/WDPHqcrUw7Sxq9UOHC5wO2G8eNh/nwb+vavf2nsQ5jSxq9UuKhaFR56CFavtncADxgAl18OP/3kdGWqgmnjVyrcxMbCV1/B1Kn2AnBsrI2AOHHC6cpUBdHGr1Q4ioiAESNg7VpITYWhQ+3qny1bnK5MVQBt/EqFs1atbNTzyy9Derrd7P3JJyE72+nKVDnSxq9UuBOBm26CDRvgr3+Fe++1d/5mZDhdmSon2viVUlaTJrBggd3q8ccfbebPww/DH384XZnyM238Sqk/idiwtw0b7Hr/CRNs6ue33zpdmfIjbfxKqZPVqwevvQYffgiHDtm8/7vugt9+c7oy5Qfa+JVSRevRw27wMmwYTJtmQ98++8zpqlQZaeNXShWvVi2YMQOWLYPKleEvf7EXgw8ccLoydYq08SulfHPJJbBmDYweDa++akPf3n3X6arUKdDGr5Ty3WmnwT//aUPfGjSAq66C666D3budrkyVQomNX0Sai8gSEdkoIutF5M5CjhklIumej3UikiMip3se2y4iaz2PucvjJJRSFaxNG1i5Eh57DN57z47+X39dQ9+ChC8j/mzgbmNMa6AdMFxEYrwPMMY8aYxJMsYkAfcDS40x+70O6eR53OW3ypVSzqpSBcaMsXf8nnce3HAD9Oxp7wFQAa3Exm+MyTLGpHk+PwRsBJoW8yP9gDf8U55SKuC1bg1ffAHTp9v/xsbai8Ea+hawSjXHLyItgGRgeRGPVwe6A/O9vm2AT0RklYgMObUylVIBLSICbr/dLv288EK47Tbo0AE2b3a6MlUInxu/iERhG/oIY8yvRRx2OfBVgWme9saYFKAHdpro0iKef4iIuEXEvUd3B1IqOLVoAYsXw+zZ9k0gMREmTtTQtwDjU+MXkSrYpj/XGPNOMYf2pcA0jzFmp+e/PwMLgNTCftAYM9MY4zLGuOrXr+9LWUqpQCQCgwbBxo1w2WVw//1wwQX2WoAKCL6s6hFgFrDRGDOlmONqAx2A97y+V0NEauZ+DnQD1pW1aKVUEGjUyG7z+PbbsGOH3f7xgQfg99+drizs+TLibw9cD3T2WrLZU0SGishQr+OuAj4xxniHeTQEvhSRNcAK4ANjzMd+q14pFfiuvtqGvl1/PTz+OCQn2x3AlGPEBOC6W5fLZdxuXfKvVMj55BMYMsQu+bztNvtGEBXldFUhQURW+bpkXu/cVUpVnG7d7EXf226DZ5+FuDj7ZqAqlDZ+pVTFior6c81/ZKTd9WvwYNi/v+SfVX6hjV8p5Yz27e1KnzFjbNxDTIy9GKzKnTZ+pZRzIiNt3o/bbbd+vOYa+7Frl9OVhTRt/Eop5yUl2cTPiRNh0SI7+v+//9PQt3KijV8pFRiqVIH77rOZ/7Gxdt7/r3+F7dudrizkaONXSgWW886DpUtt0Ns339iVP888o6FvfqSNXykVeCpVgltvtUs/L7kE7rjD/nfjRqcrCwna+JVSgevMM+HDD+G112DTJnst4PHH4fhxpysLatr4lVKBTcTGPWzYAFdeafN+UlMhLc3pyoKWNn6lVHBo2BD+/W9YsMAu90xNtcmfR486XVnQ0cavlAouV15pR/+DBtnln0lJ9i5g5TNt/Eqp4FO3Lrz8Mnz6KRw7BpdeCsOHw6FDTlcWFLTxK6WCV9euduXPiBHw/PN2/f9HHzldVcDTxq+UCm41asDUqTbjPyoKevaEG26AffucrixgaeNXSoWGCy+E1avhoYfgjTds7MNbb2nsQyG08SulQke1ajB+PKxaBc2bw3XXQe/esHOn05UFFG38SqnQk5AA334LkybBxx/b0f+sWTr699DGr5QKTZUrw6hRkJEBiYlw883wl7/A9987XZnjtPErpULbOefAkiV21c+KFRAfD9OmQU6O05U5Rhu/Uir0VaoEQ4fC+vXQsSPcdRdcfLG9ESwMaeNXSoWP5s3tRi9z58KWLZCcDBMm2JvAwog2fqVUeBGB/v1txHPv3vDww+BywcqVTldWYbTxK6XCU/36dr3/e+/Zm73atYN774UjR5yurNxp41dKhbdevexc/003wZNP2hVAS5c6XVW50savlFK1a8PMmfD553aLx44dYdgw+PVXpysrF9r4lVIqV+fOsHYt3H23fSOIjYUPPnC6Kr/Txq+UUt6qV4fJk+1G73XqwN/+BgMGwJ49TlfmN9r4lVKqMKmpNvPnkUds2FtMDMybFxKxD9r4lVKqKFWrwtixdn/fVq2gXz+44grYscPpyspEG79SSpUkLg6+/hqeego++8yO/l96KWhH/9r4lVLKFxERMHKkvfjbpg0MGQJdusB33zldWamV2PhFpLmILBGRjSKyXkTuLOSYUSKS7vlYJyI5InK657HuIrJZRLaKyOjyOAmllKowZ51ll33OnGmvAcTH298Egij0zZcRfzZwtzGmNdAOGC4iMd4HGGOeNMYkGWOSgPuBpcaY/SISAcwAegAxQL+CP6uUUkFHBG65xd741bUr3HOP3QFs3TqnK/NJiY3fGJNljEnzfH4I2Ag0LeZH+gFveD5PBbYaY743xhwD5gFXlK1kpZQKEE2b2siHefNg+3ZISbGrgAI89K1Uc/wi0gJIBpYX8Xh1oDsw3/OtpsBPXodkUvybhlJKBRcR6NPHjv6vuw7GjbNvACtWOF1ZkXxu/CIShW3oI4wxRd3HfDnwlTFmf+6PFXJMoZfBRWSIiLhFxL0nhG6UUEqFiehomDPHxj4fPGinfu6+OyBD33xq/CJSBdv05xpj3inm0L78Oc0DdoTf3OvrZkChux4bY2YaY1zGGFf9+vV9KUsppQLPZZfZDV+GDIEpU+zF3yVLnK4qH19W9QgwC9hojJlSzHG1gQ7Ae17fXgmcIyItRaQq9o1hYdlKVkqpAFerlt3q8b//tbt/de5s3wgOHHC6MsC3EX974Hqgs9eSzZ4iMlREhnoddxXwiTHmt9xvGGOygduAxdiLwm8aY9b7sX6llApcHTrYzd7vvRdmzbKhbwudH/uKCcA7z1wul3G73U6XoZRS/uN2w4032hvA+vSB6dOhQQO/Pb2IrDLGuHw5Vu/cVUqpiuBy2eY/YQIsWGBjH+bOdST2QRu/UkpVlKpV4cEHYfVqOOccGDgQLr8cfvqp5J/1I238SilV0WJi4MsvYdo0u+InNhZeeMHu/lUBtPErpZQTIiLgzjttzMMFF9itHs8+G374odxfWhu/Uko5qWVL+OQTuPZa2LYN3nyz3F+ycrm/glJKqeKJwHPPQdu2MHhwub+cNn6llAoE0dEwalSFvJRO9SilVJjRxq+UUmFGG79SSoUZbfxKKRVmtPErpVSY0cavlFJhRhu/UkqFmYCMZRaRPcCp3rccDez1YznBQM859IXb+YKec2mdaYzxafvCgGz8ZSEibl8zqUOFnnPoC7fzBT3n8qRTPUopFWa08SulVJgJxcY/0+kCHKDnHPrC7XxBz7nchNwcv1JKqeKF4ohfKaVUMYK28YtIdxHZLCJbRWR0IY9XE5F/ex5fLiItKr5K//HhfEeKyAYRyRCRz0XkTCfq9KeSztnruGtExIhI0K8A8eWcReQ6z9/1ehH5V0XX6G8+/Ns+Q0SWiMhqz7/vnk7U6S8i8oqI/Cwi64p4XERkuufPI0NEUvxehDEm6D6ACOA7oBVQFVgDxBQ45lbgBc/nfYF/O113OZ9vJ6C65/NhwXy+vp6z57iawDLgW8DldN0V8Pd8DrAaqOv5uoHTdVfAOc8Ehnk+jwG2O113Gc/5UiAFWFfE4z2BjwAB2gHL/V1DsI74U4GtxpjvjTHHgHnAFQWOuQJ41fP520AXEZEKrNGfSjxfY8wSY8wRz5ffAs0quEZ/8+XvGGACMAn4vSKLKye+nPMtwAxjzC8AxpifK7hGf/PlnA1Qy/N5bWBnBdbnd8aYZcD+Yg65AnjNWN8CdUSksT9rCNbG3xT4yevrTM/3Cj3GGJMNHATqVUh1/ufL+Xq7CTtiCGYlnrOIJAPNjTGLKrKwcuTL3/O5wLki8pWIfCsi3SusuvLhyzk/AgwUkUzgQ+D2iinNMaX9/73UgnXrxcJG7gWXJ/lyTLDw+VxEZCDgAjqUa0Xlr9hzFpFKwFRgUEUVVAF8+XuujJ3u6Yj9re4LEYkzxhwo59rKiy/n3A/4P2PMUyJyIfC655xPlH95jij33hWsI/5MoLnX1804+de/vGNEpDL2V8Tifr0KZL6cLyLSFXgA6GWM+aOCaisvJZ1zTSAO+K+IbMfOhS4M8gu8vv67fs8Yc9wYsw3YjH0jCFa+nPNNwJsAxphvgEhspk2o8un/97II1sa/EjhHRFqKSFXsxduFBY5ZCPzd8/k1wH+M58pJECrxfD3THi9im36wz/tCCedsjDlojIk2xrQwxrTAXtfoZYxxO1OuX/jy7/pd7IV8RCQaO/XzfYVW6V++nPOPQBcAEWmNbfx7KrTKirUQuMGzuqcdcNAYk+XPFwjKqR5jTLaI3AYsxq4KeMUYs15ExgNuY8xCYBb2V8Kt2JF+X+cqLhsfz/dJIAp4y3MN+0djTC/Hii4jH885pPh4zouBbiKyAcgBRhlj9jlXddn4eM53Ay+JyF3YKY9BQTyIQ0TewE7VRXuuW4wFqgAYY17AXsfoCWwFjgCD/V5DEP/5KaWUOgXBOtWjlFLqFGnjV0qpMKONXymlwow2fqWUCjPa+JVSKsxo41dKqTCjjV8ppcKMNn6llAoz/w829mRGv396JwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn = CustomNeuralNetwork(layers,act_obj)\n",
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.01,epochs=2,bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 1.767314073644877, Val loss:1.9621977696039812\n",
      "Epoch 2. Training loss: 1.713154670679888, Val loss:1.4291168740368305\n",
      "Epoch 3. Training loss: 1.620883903906978, Val loss:1.9126062679417812\n",
      "Epoch 4. Training loss: 1.610880603372522, Val loss:1.4527812798213797\n",
      "Epoch 5. Training loss: 1.55565546925381, Val loss:1.8240936093288085\n",
      "Epoch 6. Training loss: 1.4971411720699546, Val loss:1.4341424274760477\n",
      "Epoch 7. Training loss: 1.5026679973446964, Val loss:1.761068691280788\n",
      "Epoch 8. Training loss: 1.5002650159665911, Val loss:1.5787729785306737\n",
      "Epoch 9. Training loss: 1.4218177918516408, Val loss:1.8311250109777368\n",
      "Epoch 10. Training loss: 1.3775552256132313, Val loss:1.2085991712272899\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzs3XlYVVX3wPHvZlAUcQRzDixzDJXQNEe4TTbbqG/2K7Os3mbL6i0rHCozLRtssNKsTBvMskktxdnEedacFUdQMUVU4K7fHweQmQucey/i+jzPfYB99jlnHcF1991nn72NiKCUUur84ePtAJRSSnmWJn6llDrPaOJXSqnzjCZ+pZQ6z2jiV0qp84wmfqWUOs9o4ldKqfOMJn6llDrPFJn4jTEBxpg4Y8xqY8x6Y8zgfOpUNMZ8a4zZaoxZYowJzbbtfxnlm40x19gbvlJKqeLyc6HOaSBaRE4YY/yBBcaYP0Tk72x1+gFHReRiY0wv4E3gLmNMC6AX0BKoB/xljLlERNILO2FwcLCEhoaW5HqUUuq8tHz58kQRCXGlbpGJX6w5HU5k/Oif8co9z8PNQEzG9z8AHxhjTEb5ZBE5DewwxmwF2gOLCztnaGgoy5YtcyV+pZRSgDFml6t1XerjN8b4GmNWAYeAP0VkSa4q9YE9ACKSBhwDamUvzxCfUZbfOfobY5YZY5YlJCS4Gr9SSqlicinxi0i6iLQBGgDtjTGtclUx+e1WSHl+5xgrIpEiEhkS4tKnFaWUUiVQrFE9IpIEzAGuzbUpHmgIYIzxA6oBR7KXZ2gA7CthrEoppWxQZB+/MSYESBWRJGNMJeBKrJu32U0D7sXqu78dmC0iYoyZBnxjjHkb6+ZuEyDOzgtQSpVeamoq8fHxnDp1ytuhqCIEBATQoEED/P39S3wMV0b11AUmGGN8sT4hfCcivxpjhgDLRGQa8DnwVcbN2yNYI3kQkfXGmO+ADUAa8GhRI3qUUp4XHx9PUFAQoaGhWOMyVFkkIhw+fJj4+HjCwsJKfBxXRvWsAdrmU/5Ktu9PAXcUsP9rwGsljlAp5XanTp3SpH8OMMZQq1YtSjsARp/cVUoBaNI/R9jxeyo/iV8E/vc/ePppSEz0djRKKVVmlZ/Ef/QofPwxjB4Nr2nPklLnisOHD9OmTRvatGlDnTp1qF+/ftbPZ86ccekYffv2ZfPmzYXWGTNmDBMnTrQjZDp37syqVatsOZY3uHJz99xQsyb8+SdceSVMmAB9+0J4uLejUkoVoVatWllJNCYmhipVqvDss8/mqCMiiAg+Pvm3VcePH1/keR599NHSB1tOlJ8WP0BkJCxfDpUrg8MB69Z5OyKlVAlt3bqVVq1a8fDDDxMREcH+/fvp378/kZGRtGzZkiFDhmTVzWyBp6WlUb16dV544QVat25Nx44dOXToEACDBg1i9OjRWfVfeOEF2rdvT9OmTVm0aBEAycnJ3HbbbbRu3ZrevXsTGRlZZMv+66+/5tJLL6VVq1a8+OKLAKSlpXHPPfdklb/33nsAvPPOO7Ro0YLWrVvTp08f2//NXFV+WvyZLroI5syBbt0gOhpiY6FlS9tP4xQnCckJXFDlAtuPrZRXPfUUuNKNkZoKBw5AnTpQ1JjyNm2sbthi2rBhA+PHj+fjjz8GYPjw4dSsWZO0tDSioqK4/fbbadGiRY59jh07Rrdu3Rg+fDgDBgxg3LhxvPDCC3mOLSLExcUxbdo0hgwZwvTp03n//fepU6cOU6ZMYfXq1URERBQaX3x8PIMGDWLZsmVUq1aNK6+8kl9//ZWQkBASExNZu3YtAElJSQCMGDGCXbt2UaFChawybyhfLf5MF19sJX8/Pyv5b9hgy2HPpJ9h5raZPPLrI9QbVY86o+oQuyPWlmMrdc45cAC2b7e+uslFF11Eu3btsn6eNGkSERERREREsHHjRjbk83+7UqVK9OjRA4DLLruMnTt35nvsW2+9NU+dBQsW0KtXLwBat25NyyIajUuWLCE6Oprg4GD8/f35z3/+w7x587j44ovZvHkzTz75JDNmzKBatWoAtGzZkj59+jBx4sRSPYBVWuWvxZ+pSROrtR8Vdbbl37x5sQ9z4swJpm+dztRNU/ntn984dvoYgf6BhFYP5WDyQUYsHEFUWJQbLkApL3G1ZZ6YCOPHW/fTgoPdEkpgYGDW91u2bOHdd98lLi6O6tWr06dPn3yfNK5QoULW976+vqSlpeV77IoVK+apY01G7LqC6teqVYs1a9bwxx9/8N577zFlyhTGjh3LjBkzmDt3Lj///DPDhg1j3bp1+Pr6FuucdiifLf5MTZvC7NlgjPUGsGmTS7slnkxk/Mrx3DTpJkLeCuGO7+9gxtYZ3Nr8Vqb1mkbCwATm3DeHOlXqcDLtpJsvQqkyKjgYBg50W9LP7d9//yUoKIiqVauyf/9+ZsyYYfs5OnfuzHfffQfA2rVr8/1EkV2HDh2IjY3l8OHDpKWlMXnyZLp160ZCQgIiwh133MHgwYNZsWIF6enpxMfHEx0dzVtvvUVCQgInT3onf5TfFn+mZs2s5B8VZb3mzLHeEHLZfWw3P236iambpjJv1zyc4qRRtUY8dNlD9GzWk06NOuHnc/afq5J/JXq17MXHyz/mVNopAvwCPHhRSp1/IiIiaNGiBa1ataJx48Z06tTJ9nM8/vjj/N///R/h4eFERETQqlWrrG6a/DRo0IAhQ4bQvXt3RIQbb7yR66+/nhUrVtCvXz9EBGMMb775JmlpafznP//h+PHjOJ1Onn/+eYKCgmy/BleY4n608YTIyEixfSGWDRusxO/rC3PmIE2asCFhA1M3TeWnTT+xfP9yAFqGtKRns570bN6TtnXaFvqU3C+bf+GmyTcRe28s3UO72xuvUh60ceNGmpegK7S8SUtLIy0tjYCAALZs2cLVV1/Nli1b8PMrW23k/H5fxpjlIhLpyv5l62rcqUULnLP+Iq53V6Y+fxlTuwSz5fhOADo06MCbV75Jz2Y9aVKricuH7HphV3yMD7N3zNbEr1Q5cOLECRwOB2lpaYgIn3zySZlL+nYof1eUS2p6KnN2zmHqpqn8vPln9t2ehF86RG1M5emeMdzc5UHqBdUr0bGrBVSjXb12zNoxiyFRQ4reQSlVplWvXp3ly5d7Owy3K5eJP/lMMjO2zWDqpqn8+s+vJJ1KorJ/Za69+Fp6NuvJ9alh1LjmZpj+GczpA6XoZosOi+atRW9x/PRxgip6p79OKaWKo9wk/tNpp/l0+ad8tPwjth3Zxun009SsVJNbmt1Cz2Y9uarxVVTyr3R2h1mzrKd7M2/4Nm5covNGh0XzxoI3WLB7AT2a9LDnYpRSyo3KTeL39fHlub+eIyUthU4NOzE0aihdLuySYyRODq1bw19/5Uz+JVjY4IqGV1DBtwKzd8zWxK+UOieUm8Tv5+PHwvsX8uf2P7m/7f0EV3ZhbHGbNnmTf2hosc5b2b8yVzS8glk7ZpUobqWU8rRy9QBX27ptea7Tc64l/ayd2lrJ/9gxK/nv2lXs80aHRrPqwCoOnzxc7H2VUtC9e/c8D2SNHj2a//73v4XuV6VKFQD27dvH7bffXuCxixoePnr06BwPU1133XW2zKUTExPDyJEjS30cu5WrxF9iERFW8k9KspL/7t3F2j06LBpBmLtrrpsCVKp86927N5MnT85RNnnyZHr37u3S/vXq1eOHH34o8flzJ/7ff/+d6tWrl/h4ZV2Rid8Y09AYE2uM2WiMWW+MeTKfOgONMasyXuuMMenGmJoZ23YaY9ZmbLP5qSwbXXYZzJwJR45YyX/PHpd3bV+/PYH+gczeMduNASpVft1+++38+uuvnD59GoCdO3eyb98+OnfunDW2PiIigksvvZSff/45z/47d+6kVatWAKSkpNCrVy/Cw8O56667SElJyar3yCOPZE3r/OqrrwLw3nvvsW/fPqKiooiKsubdCg0NJTFjJb+3336bVq1a0apVq6xpnXfu3Enz5s158MEHadmyJVdffXWO8+Rn1apVdOjQgfDwcHr27MnRo0ezzt+iRQvCw8OzJoibO3du1mI0bdu25fjx4yX+t82PK338acAzIrLCGBMELDfG/CkiWZNYiMhbwFsAxpgbgadF5Ei2Y0SJSNlfD7FdOyv5X3XV2T7/Bg2K3M3f15+uF3bVfn5VLjw1/SlWHSh6WubU9FQOnDhAnSp18PctfKbJNnXaMPragid/q1WrFu3bt2f69OncfPPNTJ48mbvuugtjDAEBAUydOpWqVauSmJhIhw4duOmmmwp8qv6jjz6icuXKrFmzhjVr1uSYWvm1116jZs2apKen43A4WLNmDU888QRvv/02sbGxBOead2j58uWMHz+eJUuWICJcfvnldOvWjRo1arBlyxYmTZrEp59+yp133smUKVMKnWP///7v/3j//ffp1q0br7zyCoMHD2b06NEMHz6cHTt2ULFixazupZEjRzJmzBg6derEiRMnCAiwd0qYIlv8IrJfRFZkfH8c2AjUL2SX3sAke8LzgvbtreR/6JCV/PfudWm36LBoNiVuYt/xfW4OUKmy4cCJA2xP2s6BE/ZMy5y9uyd7N4+I8OKLLxIeHs6VV17J3r17OXjwYIHHmTdvXlYCDg8PJzzbSnzfffcdERERtG3blvXr1xc5CduCBQvo2bMngYGBVKlShVtvvZX58+cDEBYWRps2bYDCp38Ga42ApKQkunXrBsC9997LvHnzsmK8++67+frrr7OeEu7UqRMDBgzgvffeIykpyfanh4t1NGNMKNAWWFLA9srAtcBj2YoFmGmMEeATERlbokg96fLLYcYMuOaasy3/eoU/3RsdFg1A7I5Y7g6/2wNBKuUehbXMs8ucxbZv277FG1BRgFtuuYUBAwawYsUKUlJSslrqEydOJCEhgeXLl+Pv709oaGi+0zFnl9+ngR07djBy5EiWLl1KjRo1uO+++4o8TmFzmWVO6wzW1M5FdfUU5LfffmPevHlMmzaNoUOHsn79el544QWuv/56fv/9dzp06MBff/1Fs2bNSnT8/Lh8c9cYUwWYAjwlIv8WUO1GYGGubp5OIhIB9AAeNcZ0LeD4/Y0xy4wxyxISElwNy306doTp02H/fiv5799faPU2ddpQI6CGdveo80Zw5WAGdhpoS9IHa4RO9+7duf/++3Pc1D127Bi1a9fG39+f2NhYdhUx8q5r165Zi6qvW7eONWvWANa0zoGBgVSrVo2DBw/yxx9/ZO0TFBSUbz96165d+emnnzh58iTJyclMnTqVLl26FPvaqlWrRo0aNbI+LXz11Vd069YNp9PJnj17iIqKYsSIESQlJXHixAm2bdvGpZdeyvPPP09kZCSbXJxS3lUutfiNMf5YSX+iiPxYSNVe5OrmEZF9GV8PGWOmAu2Bebl3zPgkMBas2Tldit7drrjCSv7ZW/516uRb1cf4EBUWxawds7KmYlVKFU/v3r259dZbc4zwufvuu7nxxhuJjIykTZs2RbZ8H3nkEfr27Ut4eDht2rShffv2gLWiVtu2bWnZsmWeaZ379+9Pjx49qFu3LrGxZ1fVi4iI4L777ss6xgMPPEDbtm0L7dYpyIQJE3j44Yc5efIkjRs3Zvz48aSnp9OnTx+OHTuGiPD0009TvXp1Xn75ZWJjY/H19aVFixZZK4rZpchpmY2VwSYAR0TkqULqVQN2AA1FJDmjLBDwEZHjGd//CQwRkemFndMt0zKXxvz50KMHNGxoJf8L8l9nd0zcGB774zG2PbGNxjVKNgWEUt6g0zKfW0o7LbMrXT2dgHuA6GxDNq8zxjxsjHk4W72ewMzMpJ/hAmCBMWY1EAf8VlTSL5O6dIHff7fG90dFQQE3ljL7+XVYp1KqLCuyq0dEFgBF9luIyBfAF7nKtgOtSxhb2dK1q5X8r7vu7Bq+tWvnqNIsuBl1q9Rl1o5ZPBDxgJcCVUqpwumTu8XRrRv8+ivs2GEl/1w3oY0xRIdFM3vH7GIv2qyUt+nf7LnBjt+TJv7iioqykv+2bRAeDrlu8kSHRXMo+RAbEgofH6xUWRIQEMDhw4c1+ZdxIsLhw4dL/UBXuZmd06Oio+HOO+HLL2HQIPj667ObsvXzt6zd0lsRKlUsDRo0ID4+njIxlFoVKiAggAYuzChQGE38JfXmm/Dtt9bi7dmEVg+lcY3GzNoxi8cvf9xLwSlVPP7+/oSVYD0KdW7Srp6SqlMHrr4aFi7Msyk6NJo5O+eQ7kz3QmBKKVU4Tfyl4XBYff25niSMDovm2OljrDyw0kuBKaVUwTTxl4bDYX2dlXOahsx+/lnbdfoGpVTZo4m/NFq2tJ7izZX4L6hyAS1DWjJ7pz7IpZQqezTxl4Yx1gif2bMh1zC46LBo5u+az5n0M14KTiml8qeJv7QcDjhwAHLN6x0dFk1KWgpL4vOdwVoppbxGE39pFdDP3z20Oz7GR6dpVkqVOZr4Sys0FBo3zpP4qwdUJ6JuhE7YppQqczTx28HhsKZrTkvLURwdGs3f8X+TfCY5//2UUsoLNPHbweGAf/+F5ctzFEeHRZPqTGXB7gVeCkwppfLSxG+HaGvcfu7uns6NOuPv46/dPUqpMkUTvx1CQqyZOnMl/sAKgXRo0EHH8yulyhRN/HZxOKx5e1JSchRHh0WzYv8KjqYc9VJgSimVkyZ+uzgccPo0LFqUozg6LBqnOJm7a66XAlNKqZw08dula1fw88vT3dOhQQcq+VXSfn6lVJmhid8uQUHQvn2exF/BtwJdLuyiiV8pVWYUmfiNMQ2NMbHGmI3GmPXGmCfzqdPdGHPMGLMq4/VKtm3XGmM2G2O2GmNesPsCyhSHA5Ytg6SkHMXRodGsT1jPwRMHvRSYUkqd5UqLPw14RkSaAx2AR40xLfKpN19E2mS8hgAYY3yBMUAPoAXQu4B9yweHA5xOmJuzP9/R2JrWQVv9SqmyoMjELyL7RWRFxvfHgY1AfReP3x7YKiLbReQMMBm4uaTBlnkdOkClSnm6e9rWaUu1itU08SulyoRi9fEbY0KBtkB+U052NMasNsb8YYzJXGW8PrAnW514XH/TOPdUrAhduuRJ/L4+vnQP7a7j+ZVSZYLLid8YUwWYAjwlIv/m2rwCuFBEWgPvAz9l7pbPoSSfMowx/Y0xy4wxyxISElwNq+xxOKwpmvfvz1EcHRbN9qPb2Zm00ztxKaVUBpcSvzHGHyvpTxSRH3NvF5F/ReRExve/A/7GmGCsFn7DbFUbAPvyO4eIjBWRSBGJDAkJKeZllCGZ0zTPztm6d4RpP79SqmxwZVSPAT4HNorI2wXUqZNRD2NM+4zjHgaWAk2MMWHGmApAL2CaXcGXSW3aQM2aebp7WoS0oHZgbU38Simv83OhTifgHmCtMWZVRtmLQCMAEfkYuB14xBiTBqQAvUREgDRjzGPADMAXGCci622+hrLF1xeioqzEL2ItzwgYY4gOi2b2jtmICBnvk0op5XFFJn4RWUD+ffXZ63wAfFDAtt+B30sU3bnK4YApU2DbNrj44qzi6NBoJq+bzKbETTQPae7FAJVS5zN9ctcdCliOUcfzK6XKAk387tCkCTRokCfxh1UP48JqF+qwTqWUV2nidwdjrFb/7NnWk7xZxVY/f+yOWJziLOQASinlPpr43cXhgMOHYc2anMVhDo6eOsqqA6sK2FEppdxLE7+7FNDPHxUWBWg/v1LKezTxu0u9etCsWZ7EXy+oHs2Cm2niV0p5jSZ+d3I4YN48OHMmR3F0aDTzds3jTPqZAnZUSin30cTvTg4HJCdDXFzO4sYOklOTWbp3qZcCU0qdzzTxu1P37uDjk6e7p9uF3TAY7e5RSnmFJn53qlEDIiLyJP5alWvRpk4bHc+vlPIKTfzu5nDA339bXT7ZRIdFs2jPIk6mnvRSYEqp85UmfndzOCA1FebPz1kc5uBM+hkW7VnkpcCUUucrTfzu1qkTVKiQp7unc6PO+Pn4aT+/UsrjNPG7W+XKcMUVeRJ/UMUg2tdvr4lfKeVxmvg9weGAVausKRyyiQ6NZum+pRw7dcxLgSmlzkea+D3B4bAWZYmNzVnc2IFTnMzbNc9LgSmlzkea+D2hXTsICsrT3dOhQQcC/AK0u0cp5VGa+D3Bzw+6dcuT+AP8AujUsBOzdswqYEellLKfJn5PcThgyxbYsydncZiDtYfWcij5kJcCU0qdbzTxe0oB0zRHh0UDMGfnHA8HpJQ6XxWZ+I0xDY0xscaYjcaY9caYJ/Opc7cxZk3Ga5ExpnW2bTuNMWuNMauMMcvsvoBzRqtWULt2nsR/Wb3LCKoQpP38SimP8XOhThrwjIisMMYEAcuNMX+KyIZsdXYA3UTkqDGmBzAWuDzb9igRSbQv7HOQMRAdbSV+EetnwM/Hj26h3bSfXynlMUW2+EVkv4isyPj+OLARqJ+rziIROZrx499AA7sDLRccDti/HzZtylkc5mDrka3sPrbbS4Eppc4nxerjN8aEAm2BJYVU6wf8ke1nAWYaY5YbY/oXcuz+xphlxphlCQkJxQnr3JHZz//XXzmKM/v5Y3fE5t5DKaVs53LiN8ZUAaYAT4nIvwXUicJK/M9nK+4kIhFAD+BRY0zX/PYVkbEiEikikSEhIS5fwDklLMx65ernb1W7FcGVg3WaZqWUR7iU+I0x/lhJf6KI/FhAnXDgM+BmEcmam0BE9mV8PQRMBdqXNuhzmsMBc+ZAWlpWkY/xISo0ilnbZyEi3otNKXVecGVUjwE+BzaKyNsF1GkE/AjcIyL/ZCsPzLghjDEmELgaWGdH4OcshwOOHYMVK3IWhznYe3wvW45s8VJgSqnzhSujejoB9wBrjTGrMspeBBoBiMjHwCtALeBD632CNBGJBC4ApmaU+QHfiMh0W6/gXBNt9eczaxa0P/vhJ7Off/aO2VxS6xJvRKaUOk+Ysti1EBkZKcuWleMh/+Hh1pj+bDd5RYRGoxvRoUEHvr/jey8Gp5Q6Fxljlmc0uIukT+56g8MBCxfCqVNZRcYYHGEOYnfE4hSnF4NTSpV3mvi9weGwkv6inMsuRodFczjlMGsPrvVSYEqp84Emfm/o2hV8ffMM64wKjQLQ6RuUUm6lid8bqla1buzmSvwNqzWkSc0mOn2DUsqtNPF7i8MBS5daQzuzF4c5mLtrLqnpqV4KTClV3mni9xaHA5xOmDs3R3F0WDQnzpxg+f7lXgpMKVXeaeL3lo4doVKlPN093UO7AzBru3b3KKXcQxO/t1SsCJ0750n8IYEhhF8QrvP2KKXcRhO/NzkcsH49HDiQszjMwcLdCzmVdqqAHZVSquQ08XtT5jTNs3O27qPDojmdfprFexZ7ISilVHmnid+b2raF6tXzdPd0vbArvsZXh3UqpdxCE783+fpCVNTZ5RgzVK1Ylch6kfogl1LKLTTxe5vDAbt2wfbtOYvDHMTtjeP46eNeCkwpVV5p4ve2zH7+XN090WHRpEs683fP90JQSqnyTBO/tzVtCvXq5Un8VzS8goq+FXU8v1LKdpr4vc0Yq9U/e7b1JG+GSv6VuKLhFTqeXyllO038ZYHDAYmJsDbndMzRYdGsOrCKwycPF7CjUkoVnyb+sqCQfn6A2J2xno5IKVWOaeIvCxo0gEsuyZP429VrR5UKVXRYp1LKVkUmfmNMQ2NMrDFmozFmvTHmyXzqGGPMe8aYrcaYNcaYiGzb7jXGbMl43Wv3BZQbDgfMmwepZ6dj9vf1p+uFXTXxK6Vs5UqLPw14RkSaAx2AR40xLXLV6QE0yXj1Bz4CMMbUBF4FLgfaA68aY2rYFHv54nDAiRMQF5ejODo0ms2HN7P3371eCkwpVd4UmfhFZL+IrMj4/jiwEaifq9rNwJdi+RuoboypC1wD/CkiR0TkKPAncK2tV1BeREVZI3wK6OfXVr9Syi7F6uM3xoQCbYEluTbVB/Zk+zk+o6ygcpVbzZrW3D25En/rOq2pWammDutUStnG5cRvjKkCTAGeEpF/c2/OZxcppDy/4/c3xiwzxixLSEhwNazyxeGAxYshOTmryMf4EBUaxewdsxHJ959OKaWKxaXEb4zxx0r6E0Xkx3yqxAMNs/3cANhXSHkeIjJWRCJFJDIkJMSVsMofh8O6ubtgQY7i6LBodh/bzfaj2wvYUSmlXOfKqB4DfA5sFJG3C6g2Dfi/jNE9HYBjIrIfmAFcbYypkXFT9+qMMpWfzp3B37/Afn6dplkpZQc/F+p0Au4B1hpjVmWUvQg0AhCRj4HfgeuArcBJoG/GtiPGmKHA0oz9hojIEfvCL2cCA621eHMl/qa1mnJB4AV8uPRDbm1+K8GVg70UoFKqPCgy8YvIAvLvq89eR4BHC9g2DhhXoujORw4HxMTAkSPWDV/AGEP9qvVZsX8Fnyz7hJe6vuTdGJVS5zR9crescTisRVlic07TMDRqKAC+Pr7eiEopVY5o4i9r2reHKlXydPdc1+Q6OjboyNjlY0l3pnspOKVUeaCJv6zx94euXfMkfoBnOj7DjqQdTN001QuBKaXKC038ZZHDAf/8A/HxOYpvaXYLjWs0ZuSikTqmXylVYpr4y6ICpmn29fFlQIcBLNm7hEV7FnkhMPfYdmQbV391NTuTdno7FKXOC5r4y6JLL4Xg4Hy7e+5rcx81K9Vk5OKRXgjMPfpN68ef2/+k/y/9vR2KUucFTfxlkY8PREdbiT9Xl05ghUD+G/lfft70M/8c/sdLAdrnUPIh4vbGYTAs2rOIxJOJ3g5JqXJPE39Z5XDAvn2weXOeTY+1fwx/X3/eWfyOFwKz16hFozidfpopd04hJS2F4QuGezskpco9TfxlVQH9/AAXVLmAe8Lv4YvVX5CQfO5OaJeQnMAHSz+gd6ve9Gzek3vC7+GDuA+I/ze+6J2VUiWmib+satwYLrww38QPMKDjAE6lneKjZR95ODD7jFw0klNppxjUdRAAMd1jcIqTYfOGeTkypco3TfxllTFWqz82FtLzPrDVIqQF1ze5ng/iPiAlNcULAZZOZmu/V6teNAtuBkBo9VAeuuwhPl/5OVuPbPVyhEqVX5r4yzKHA5KSYOXKfDc/e8V7ODXZAAAgAElEQVSzJJxM4Ks1X3k4sNIbtXgUKakpvNz15RzlL3V9CX8ff16d86qXIlOq/NPEX5ZFW9MxF9Td0+3CblxW9zJGLR6FU5weDKx0EpIT+CDuA3pf2jurtZ+pTpU6PHn5k0xaO4k1B9d4KUKlyjdN/GVZnTrQsmWBid8YwzMdn+Gfw//w6z+/eji4khu1eBQnU08yqMugfLc/1+k5qlasysuxL+e7XSlPEBF2H9vt7TDcQhN/WedwWCtynT6d7+bbW9xOo2qNGLno3HigK/FkIh/EWX37zUOa51unRqUaPNfpOaZtnsbiPYs9HKH9ft70My/PflmfUTiHiAiP/v4oF46+kO/Xf+/tcGynib+sczggJcVaizcf/r7+PHX5U8zfPZ+4vXEeDq74Ri2yWvu5+/Zze+LyJ6gdWJsXZ794Ts9LtHjPYm759haGzR/G5ys+93Y4ykUxc2KyRswNmTfEy9HYTxN/Wdetm/UkbwHdPQAPRDxAtYrVGLV4lAcDK77Ek4m8H/d+oa39TFUqVOGlLi8xZ+ecc3bJydNpp+k3rR+B/oEA59R9mPPZiIUjGDJvCL1b9aZLoy5sTNjIwRMHvR2WrTTxl3XVqkG7doUm/qCKQTx02UP8sOEHdhzd4cHgisfV1n6mhy57iEbVGvHirHOz1T9s3jA2Jm7k+zu+5+qLruaNBW+w99+93g5LFWJM3Bie/+t5erXqxVc9v+KTGz4hXdLPyZFzhdHEfy5wOCAuDv79t8AqT1z+BD7Gh9F/j/ZgYK5LPJnIB0s/4K5WdxXZ2s9U0a8iMd1iWLpvKT9t+snNEdpr9YHVDF84nHvC76FHkx6MuW4MZ9LP8PSMp70dmirA+JXjeeyPx7ip6U18ecuX+Pr40jykOR0bdOTzlZ+fk42PgmjiPxc4HNZDXPPmFVilftX69G7Vm89Xfs6RlLK3nv3bi98m+Uyyy639TPe0voemtZoyKHbQObPyWJozjX7T+lGzUk3eucaaT+nimhfzUpeX+H7D9/yx5Q8vR6hy+3bdtzzwywNc1fgqvr39W/x9/bO29Wvbj02Jm1gcf+4PNMhUZOI3xowzxhwyxqwrYPtAY8yqjNc6Y0y6MaZmxradxpi1GduW2R38eeOKKyAgoNDuHrBW6EpOTeaTZZ94KDDXHD55mPfj3ufOlnfSIqRFsfb18/FjaNRQNiRs4Ju137gpQnu9s/gdlu9fzgc9PqBW5VpZ5c91eo6mtZry6O+PcjL1pBcjVNn9svkX+kztQ6eGnfip108E+AXk2H5nyzsJ9A8sXzfnRaTQF9AViADWuVD3RmB2tp93AsFF7Zf7ddlll4nKpUsXkYAAkdtvFxkzRmTlSpHU1DzVrvryKqk7sq6cSj3lhSDz9+JfL4qJMbLu4LoS7Z/uTJeITyIkbHSYnE47bXN09tqcuFkChgXILZNvEafTmWf77O2zhRjkxb9e9EJ09juVekoOnTjk7TBKbObWmVJhaAVpN7adHDt1rMB6/X7uJ4GvBcq/p/71YHTFAywTF3NskS1+EZkHuNp30BuYVOx3H1W0Sy+FU6dgxgx49FFo2xaqV7e6gQYNgt9/hyNHePaKZ9l/Yj+T1pWNX8Phk4d5L+497mx5Jy1rtyzRMXyMD69Fv8aOpB18tuIzmyO0j1OcPPjLg1T0rciY68ZgjMlTJyosinvC7+GtRW+xIWGDF6K0T7oznSu/upJ6b9fjvb/f83Y4xTZ/13xunnwzzYKbMb3PdKpWrFpg3X5t+5Gcmsy367/1YIRu5Mq7AxBKES1+oDLWG0TNbGU7gBXAcqC/q+9G2uLPR0KCyIgRIocOiezYITJxoshjj4lERIj4+opYS7aIs1lTufTFGtJyWD1xrl0rkp7u1bBL29rP5HQ6pcu4LlJnZB1JPpNsU3T2+mjpR0IM8vmKzwutd/DEQakxvIZ0Hd81308F54rBcwYLMWS97vvpvjL7u8ktLj5Ogl4PkqbvN5WDJw4WWd/pdErzD5pLh886eCC6kqEYLX47E/9dwC+5yuplfK0NrAa6FrJ/f2AZsKxRo0bu/Rcqb06cEImNFXn9dZEbbpAJnaoIMcgfFyNSrZrINdeIDB4sMnOmyLGCP87aLTE5Uaq8XkXu/P5OW443f9d8IQYZPn+4Lcez0+6k3RL0epA4JjhcSuZjl40VYpDxK8e7Pzg3mLNjjvgM9pHbv7tdhs8fLs/OeFZMjJGWY1rKhkMbvB1eoVYfWC01hteQsNFhEn8s3uX9Ri4cKcQg6w+td2N0JeetxD8V+E8h22OAZ105n7b4S+d06imp92ZtcQxvIfLQQyKXXipijPXrNsb6+aGHRL74QuSff0Tc1Op8adZLYmKMrD241rZjXjfxOqkxvIYcTTlq2zFLy+l0So+ve0jl1yrL9iPbXdon3ZkuV3x+hdR6s5YkJie6OUJ7JSQnSL1R9aTJe01y9HnP3DpTQkaESOBrgfL16q+9GGHBNiZslNpv1Zb6o+q7/LvKdPDEQfEb4icDpg9wU3Sl4/HED1TL6OYJzFYWCARl+34RcK0r59PEX3rD5w8XYpCV+1daBUlJVot/8GDrE0C1alndQxIcLHLjjdYnhjlzrE8QpZSYnChBrwfZ1trPtGLfCiEGGTRrkK3HLY2vV38txCCjF48u1n5rDqwR38G+cv9P97spMvs5nU65fuL1UmFohbN/W9nEH4uXLuO6CDFI/2n95eSZk16IMn/bj2yX+qPqS+23asumhE0lOsZt394mwSOCy+QgA1sTP9bN2v1AKhAP9AMeBh7OVuc+YHKu/RpndO+sBtYDL7kalCb+0juaclSqvF5F+vzYJ/8K6eki69aJfPqpSN++Is2anX0j8PUVad1a5KqrRFasKNH5X5r1khCDra39THd+f6cEvhYoB44fsP3YxXXwxEGp9WYt6fBZB0lLTyv2/gNnDhRikHk757khOvu9vehtIQZ5f8n7BdZJTU+VF/58QYhBWn/UWv5J/MeDEeZvz7E9EjY6TGoMryGrD6wu8XF+/+d3IQb5fv33NkZnD9tb/J5+aeK3x1N/PCV+Q/xkd9Ju13Y4fFjkt99EBg0SCQs7+0Zw5ZXWzeSTrrXeDp88LEGvB8kd391RiugLtjlxs/gO9pUn/3jSLccvjru+v0sqDK1Q4n7fE6dPSKN3GknLMS3LZCsyu6V7l4r/EH/pObmnS/cxfvvnN6n5Zk0Jej1Ivlv3nQcizN+B4wek6ftNJej1IImLjyvVsdLS06TB2w3k2q+vtSk6+2jiVyIisuPoDvEd7CvPzni2+DsnJIj8738izz9/9k2gWjWRRx4RWbq00PsCg2YNcltrP1O/n/tJhaEVZFfSLredoyg/bfxJiEGGzBlSquNM2zRNiEHemP+GTZHZLyklSRq/21gavdNIjpw84vJ+u5J2SYfPOggxyGO/Pebx50sOnzws4R+FS6VhlWz7VDVo1iAxMcb1BpWHaOJXWXr90EuCXg+SpJSkkh8kPV1k9myRPn2sh8jAukH8zjvW8NJs3N3az7QraZdUGFpB+v3cz63nKcjRlKNSb1Q9Cf8o3JaW+i2Tb5FKwyoV+4ajJzidTrnr+7vEd7CvLNy9sNj7n047LQOmDxBikMixkR67xmOnjkm7se2kwtAKMnPrTNuOu+3INlve8O2miV9lWbp3qRCDjFw40p4DJiWJfPyxSPv21p+Pv7/IbbdZXUSpqfLy7JeFGGTNgTX2nK8QT/3xlPgM9inxjbrSeODnB8RnsI8s3bvUluPtTtotga8FynUTrytzY/szh56W9hPJ1I1Tpdob1aTaG9Vk6sapNkWXvxOnT0iXcV3Eb4ifTNs0zfbjR0+IltDRoZLu9O5zMtlp4lc5dBvfTRq83UDOpJ2x98Br14oMGCASEiICcjisjlR9taLc/rln+j8Pnjgoga8F2j5yqCh/bftLiEEGzhxo63FHLRolxCA/rP/B1uOWxtqDayVgWIBc9eVVtiS57Ue2S+TYSCEGGTB9gP1/k2JNI3HVl1eJz2Afmbx2su3HFxH5Zs03Qgzy57Y/3XL8ktDEr3L4ZfMvQgwycc1E95zg9GmRH3+Ul/s3sVr7tRHp3Flk3DiR48fdc84MmfcTVuwr2eij4jpx+oQ0frexXPzexbYPVUxNT5XWH7WWeqPqlYk5YZLPJEuLMS3kgrcusHUE1anUU/L4748LMUiHzzrYep/mTNoZuWnSTW5/OC4lNUVqDK8hvX7o5bZzFJcmfpVDujNdmn3QTNp+3NZt3QhHTh6Rqm9UldsmXCcyfLjIJZdYf16BgSL33y+yYIFbHhQ7mnJUagyvIT2+7mH7sfOT2Vc9Z8cctxz/7z1/i4kx8tQfT7nl+MXR7+d+YmKM21q13637ToJeD5Kab9aUXzf/WurjpaWnyV3f3yXEIGPixtgQYeEe++0xqTi0ohw+edjt53KFJn6Vx6fLPxVikFnbZ7nl+K/MfkWI4ewYaadTZOFCkX79RKpUsf7ULrnEelPYt8/Wc7+54E2PjIX/e8/f4jPYRx7+5WG3nufhXx4Wn8E+snzfcreepzCZXRnunkX0n8R/pM3HbYQY5Pk/n5fU9Lwzzroi3ZkufX/qK8QgIxaMsDnK/K3cv1KIQd77+z2PnK8omvhVHimpKVL7rdpuaRlntfa/vS3/CsePi4wfb00tnfmA2A03iPz4o9VNVErJZ5Klzsg60nlcZ7d9ojmVekpajGkhDd5uUOj0vXY4mnJUar9VW9qNbVeih8JKa8vhLRL0epB0+rxTiRNxcaSkpshDvzwkxCBdxnUp1vw5Itaoo8d+e0yIQV6Z/YqbosxfxCcREv5ReJm4Ia+JX+VryJwhbhlfn6e1X5jNm63nA+rWtf78QkJEnn5aZPlykQMHrDeJEswoOiZujDUx3ZY/SnAFRXs19lUhBlu6JFwxcc1EIQb5YMkHHjlfplOpp+SyTy6TGsNrePwZiYlrJkrga4ESMiJEZmyd4dI+TqdTnv/zeSEGeWbGMx5PwB/GfSjEYNvortIoTuI3Vv2yJTIyUpYt0wW77JZ4MpFG7zSiV6tejLt5nC3HTDqVROjoUByNHUy5c4rrO6alwcyZMG4c/Pyz9XN2AQEQGGi9KlfO/2u2789UqkCztNFU9w1kWdgb+FQJyrde1ldfX5dDXXtwLRFjI7ir5V18fevXrl9jKYgIV399NXF749j06CbqBtX1yHmfnv40o5eM5udeP3NT05s8cs7sNiVu4vbvbmdDwgYGdR3Eq91exden4N/V0LlDeWXOKzwS+UiBayC4U9KpJOqOqst9re/joxs+8ui5czPGLBeRSJcqu/oO4cmXtvjd57+//lcqDK0g+/61p589syW8av+qkh/klVes1v8tt1hrDsTEiAwcKPLf/4rcd5/IHXeIXH+9SPfuIu3aibRsKRIaKlK7tnXzOGPm0S/DrXnhv2uBZE03UdArMNCarO7NN0XmzxdJSck3tNT0VIkcGykhI0IkITmh5NdYAv8k/iMVh1b02MiRzCeIn/j9CY+cryDJZ5Kz+uujvoiS/cf351svc/jr/039P6+Op+/zYx+p+kZVr69FgHb1qIJsObxFTIyx5abd0ZSjUu2NanLrt7eW7kCZi8wklDCxOp0iJ09K2sED0nL0JdJ0VJikLlksMneuyO+/i3z/vciECSIffigycqQ1+VzmrKSZbwQVKoh07CjyzDPWvYcD1vDFzDnY3TUevCgxsTFCDC53fZTU7qTdUvPNmhLxSUSZWbZz/MrxUmlYJbngrQtk9vbZObZ9vPRjIQa547s7PHIfojCxO2KFGOTLVV96NQ5N/KpQt357q9QYXkOOny7dGHtbWvs2m7pxatGrYGV/ozl0SOSnn0See06kUyeRihWz3gy2tGkklV7xlZveCBfnmjVeWc0sJTVFmrzXRC569yK3TXGcmp4qncd1liqvVykTM2lmt/bgWmn2QTPxGewjQ+cOlXRnuny56ksxMUaun3h9mZjYzul0ykXvXiRdx3f1ahya+FWhFu1eVOphaJmt/Z6Te9oYWek5nU5pN7adNHy7YclarqdOiSxaJOkj3pTuzwRL1f8ZiQ/K+FRQrZrItdeKDB0qMmuWLesWuCLzSeGXZ7/sluNnPgRXVhdPOX76uNw95W4hBmn7cVsxMUY6j+ssKan5d895w+vzXhdi8OobZ3ESf5GLravyp2PDjnRs0JF3/n6HNGda0Tvk492/3+XY6WO80u0Vm6MrHWMMrzteZ8+/e/hk+SfFP0DFitCxI59FV2dOUCKjbvuE+iu2wIQJcNddsGcPvPyytch9tWoQGQlPPAHffmttcwNHYwd3X3o3wxcMZ1PiJluPPWv7LF6b/xp92/Tl7vC7bT22XapUqMJXPb9i7A1jWXNwDYJwzUXXEOAX4O3Qstzb5l58jA/jVtozaMLtXH2H8ORLW/zuN2XDFOtGaAnmSS+rrf1MTqdTor6IkpARISXqztpzbI9UfaOqRH0Rlf/wwCNHrHsHL70kEhUlUrlyVveQNGwo0quXyHvvWUNUU+3pfz5w/IBUH15dun/R3bYhiwdPHJQ6I+tI8w+ay4nTnvn0UlrrD66X1+e/7vEb7a644ZsbpO7Iul6754B29aiipKWnyUXvXiTtP21f7EQyeM7gnMs6lkGL9ywWYpBhc4cVaz+n0yk3fHODVBpWSbYe3uraTmfOWGsUvPuuyJ13itSvLzlGD3XubC1m8+OPIsklH/mReUPTjpuI6c50uearayRgWIBHZlI9H2TeX3LHbKCu0MSvXJL50NP8XfNd3udoylGpPry63DL5FjdGZo+bJt0k1d6oVqy5VDKnKhi1aFTpTr5rl8g334g89phIvXpn3wj8/a3RQ88/b01lneT6OgnpznTp8FkHCRkRUur5YTKnufh46celOo4660zaGbngrQvkpkk3eeX8mviVS5LPJEutN2vJzZNudnmfzNa+p2bDLI01B9aIiTHywp8vuFT/0IlDEjwiWNp/2t7eqRISEkSGDBGZNEnkhResxO/nZ/338/ERiYgQeeop6xNBEUNaV+1fJb6DfeXBaQ+WOJzFexaL3xA/ueO7O8rEVAPlycCZA8V3sK9tz8kUhyZ+5bKXZ78sJsa4tJhJUkqSVB9evVhvFN5295S7pdKwSi79R/zPlP+I/xB/ty4ZmSU52RoZ9Oqr1n2CzJXNQKRFC5GHH7Y+McTnnbfmmRnPCDGUaDWsIyePyIXvXChho8NKtyqbytemhE1CDDJ8/nCPn9vWxA+MAw4B6wrY3h04BqzKeL2Sbdu1wGZgK/CCq0Fp4vecA8cPSMWhFeWhXx4qsm7mXD/nQms/05bDW8RviJ88+tujhdbLXLMgJjbGQ5HlcuqUNZvpG2+I9OghEhR09o3gootE+va1Jrrbtk2On/pXGr7dUFp92KpYC5k4nU659dtbxW+InyyJX+K+aznPdR7XWS55/xKPf5qyO/F3BSKKSPy/5lPuC2wDGgMVgNVAC1eC0sTvWQ/8/IAEDAuQQycOFVjnXGztZ3rol4fEf4h/gWu9JqUkSf1R9aXVh63KxANBImKNBlq2TOTtt62pLGrVOvtGUL++/PRgF2sK4h+fcXmdg8x7Om8tfMvNwZ/fxq8c75FpwnMrTuJ3aZI2Y0xoRnJvlc+27sCzInJDrvKOQIyIXJPx8/8yho++UdT5dJI2z9qYsJEWH7YgplsMr3Z/Nd86mZNhrei/grZ123o4wtLZ++9eLn7/Yu5seScTbpmQZ/vDvz7Mpys+ZXG/xbSv394LEbrA6YSNG2HePOs1dy43d9vPX41hw6QaXNg2Crp2tV7h4WcnoXM6ITWVVfHL6DDJQXS9Tvzq+Byf1DQ4cybnKzU1b1lR248dg7Vr4dJLz05+5+Njfc3+fUFfi7vtxAlYvx6efhqCg737OylA8plk6o6qS8/mPfP9e3MX2ydpA0IpvMV/GKtF/wfQMqP8duCzbPXuAT4o5Bz9gWXAskaNGrnxfVHl54ZvbpDgEcH5TguQlJIkNYbX8NpoBTs8O+NZMTFG1h1cl6M8c56VZ2Y846XISsjplJ2r5krlwRXkxoENxBkWevYTgZ+fSKVKWTeQj1dALnkMqTcAOVSZs/VK+6pY8ewUFxUrWs8zBARYI5d8fe07T36vhg2tCfa2ujjk1sMenPagVBpWyaP3UfBwi78q4BSRE8aY64B3RaSJMeYO4BoReSCj3j1AexF5vKjzaYvf8+bunEv3Cd35+PqPeSjyoRzbhs0bxsuxL7O8/3Ii6kZ4KcLSSTyZSON3G3PVRVdlTR99MvUk4R+FA7DmkTVU9q/szRBLZOSikQz8cyBT75rKLYGXwfPPw6RJVsu/c2eoUIH7fKbxlXMls4Ieo3vlFlChQv4vf3/Xt/n6gjGQmAjjx0Pfvvm3wEUgPd16OZ3F+5pf2cGD8P77sG8frF5tnaNNG7jtNuvVvLlnfwEFiNsbx+WfXZ7v/yd38WiLP5+6O4FgoCMwI1v5/4D/uXIM7eP3PKfTKZFjI6XJe01yTHFbHlr7mTJnuoyLjxMR61MAMeSZ+fFccibtjFz64aXS4O0G1lPKuWY6/XLVl0IM8mrsq94N1B127BAZNUrkiivOfhJo0ULk5ZdFVq1yyxrPrnI6ndLqw1bSbmw7j50Tu4dzFpb4gTqQ9cmhPbAbMIAfsB0I4+zN3ZaunE8Tv3dMWjtJiEF+2vhTVtnQuUOFGGTZ3mVejMwe/576V4JHBMtVX14lcfFx4jPYR/pP6+/tsEpt4e6FQgwyYPqAHOWbEjZJ4GuB0m18N68s4ehR8fEi779vrdng42Oltosvth6Ui4vzypvAO4vfcX1lOhvYmviBScB+IBWIB/oBDwMPZ2x/DFifkdj/Bq7Itu91wD9Yo3tecjUoTfzekZqeKo3eaSRdxnUREZFjp45JjeE15MZvbvRyZPbJXLyj7si6Un9U/XIzlr3/tP7iO9g3axqNlNQUaf1RawkeEVzsNWzPeQcPinzyicjVV599UK5RI2uJzwULPDa9dkJygvgP8ffYwja2t/g9/dLE7z2ZrZS/9/wtw+YOKzet/Uwnz5yUOm/VKdPTEJfE4ZOHJWREiFz+6eWSlp4mj/72qBCD/PbPb94OzbsOHxb54guRG26wFtsBa73n//7XeoDOpkn0CnLHd3dIzTdremRxm+Ikfp2WWeXQr20/qlWsxuC5gxm1eBQ3XnIjl9W7zNth2aaSfyV6Nu8JwL7j+7wcjX1qVqrJqKtHsWTvEnpP6c2YpWN4puMzXNfkOm+H5l01a8K998Ivv0BCAnzzDXTsaN2Qdjigbl148EGYPt0anmqzByIe4EjKEX7a9JPtxy4NXWxd5fHCXy/w5sI3AVj24LJylfjBGuEzfuV4+rbtS3DlsjkWvCREhC7ju7Bwz0LaXNCGJQ8uoYJvBW+HVTYlJ1vJfsoU603hxAmoXh1uuskaHXT11RBQ+vn+neIk7N0wmtZqysx7ZtoQeMGKM6pHW/wqj8fbP46f8aNZcDMurH6ht8OxXXDlYAZ2Gliukj5Yi9B0atQJgGsuvkaTfmECA60E/8031ieBadPg5pvPfg0JgVtvhT59YOVKa8xQCfgYH/q26ctf2/9iV9Iumy+i5DTxqzzqV63PI+0eYVPiJsavHO/tcFQxDLxiICOuHMGzVzzr7VDOHQEBcOON8MUX1nMC06dD794wcyZMnAgREVaXUbdu1mprn38Oy5ZBSopLh+/bpi8A41eVnf9L2tWj8lVeu0OUctmBAzBsGISGwrZt1gNja9ZY3URgTSPRtCm0bp3zVbeu9XBbNld/dTWbEjex48kd+Pr4uiXc4nT1aOJXSilXOZ2wfbv1JpD9tStbN05wcM43gvBwvmU9vX7uw/S7p3PNxde4JTRN/Eop5UlJSdangexvBuvWwalTAJyu6Eu9AeA4U5/v6jx+9k2hdm3bQihO4vez7axKKXW+ql797OyomdLSYMsWWL2aiqtXc8+RSXxYexeJrw4k+GRGnTp18nYVtWiRp6vIbtriV0opD1h7cC3hH4fzTuehPEXHnJ8ONmywpr2uVMnqNgoJKfbxdTinUkqVMZdecCnt6rXjs82TkehoGDAAJkyAVaus5wieftoaKfTFF26PRRO/Ukp5SL+2/VifsJ64vXE5N1SoAC++CCNGWFNcu5kmfqWU8pDel/amsn9lPl/5ed6NwcEwcKBHVhbTxK+UUh5StWJV7mhxB5PXTSb5TLLX4tDEr5RSHtSvbT+OnznO9xu+91oMmviVUsqDOjfqzCW1LuGzFZ95LQZN/Eop5UHGGO5vcz8L9yxkU+Imr8SgiV8ppTzs3jb34mt8GbdynFfOr4lfKaU8rE6VOtxwyQ1MWD2B1PRUj59fE79SSnlBv7b9OJR8iN+2/Obxc2viV0opL+jRpAd1q9T1yk3eIhO/MWacMeaQMWZdAdvvNsasyXgtMsa0zrZtpzFmrTFmlTFGJ99RSqkMfj5+3Nv6Xv7Y+gd7/93r0XO70uL/Ari2kO07gG4iEg4MBcbm2h4lIm1cnTxIKaXOF/e3vR+nOJmweoJHz1tk4heRecCRQrYvEpGjGT/+DTSwKTallCrXmtRqQrcLuzFu5Tic4vTYee3u4+8H/JHtZwFmGmOWG2P6F7ajMaa/MWaZMWZZQkKCzWEppVTZ1K9tP7Yd3ca8XfM8dk7bEr8xJgor8T+frbiTiEQAPYBHjTFd890ZEJGxIhIpIpEhJZiLWimlzkW3tbiNqhWrevQmry2J3xgTDnwG3CwihzPLRWRfxtdDwFSgvR3nU0qp8qKyf2X+0+o/TNk4haRTSR45Z6kTvzGmEfAjcI+I/JOtPNAYE5T5PXA1kO/IIKWUOp/1i+jHqbRTfLP2G4+cz5XhnJOAxUBTY0y8MaafMeZhY8zDGVVeAWoBH+YatkDWCqoAAAQqSURBVHkBsMAYsxqIA34TkeluuAallDqnXVb3MlqGtOT1+a+TeDLR7ecrcrF1EeldxPYHgAfyKd8OtM67h1JKqeyMMbQIacH3G77n0+Wf8r8u/3Pr+YpM/Eoppdzvw+s/pF29dvRt6/6lFzXxK6VUGRBcOZiBnQZ65Fw6V49SSp1nNPErpdR5RhO/UkqdZzTxK6XUeUYTv1JKnWc08Sul1HlGE79SSp1njIh4O4Y8jDEJwK4S7h4MuP+ZZ+/Qazt3lefr02srGy4UEZemNi6Tib80jDHLyutqX3pt567yfH16bece7epRSqnzjCZ+pZQ6z5THxJ97sffyRK/t3FWer0+v7RxT7vr4lVJKFa48tviVUkoVotwkfmPMtcaYzcaYrcaYF7wdj52MMQ2NMbHGmI3GmPXGmCe9HZPdjDG+xpiVxphfvR2LnYwx1Y0xPxhjNmX8/jp6OyY7GWOezvibXGeMmWSMCfB2TCVljBlnjDlkjFmXraymMeZPY8yWjK81vBmjXcpF4jfG+AJjgB5AC6C3MaaFd6OyVRrwjIg0BzoAj5az6wN4Etjo7SDc4F1guog0w1qRrtxcozGmPvAEECkirQBfoJd3oyqVL4Brc5W9AMwSkSbArIyfz3nlIvED7YGtIrJdRM4Ak4GbvRyTbURkv4isyPj+OFbyqO/dqOxjjGkAXA985u1Y7GSMqQp0BT4HEJEzIpLk3ahs5wdUMsb4AZWBfV6Op8REZB5wJFfxzcCEjO8nALd4NCg3KS+Jvz6wJ9vP8ZSjxJidMSYUaAss8W4kthoNPAc4vR2IzRoDCcD4jG6sz4wxgd4Oyi4ishcYCewG9gPHRGSmd6Oy3QUish+sBhhQ28vx2KK8JH6TT1m5G65kjKkCTAGeEpF/vR2PHYwxNwCHRGS5t2NxAz8gAvhIRNoCyZSTrgKAjP7um4EwoB4QaIzp492olCvKS+KPBxpm+7kB5/BHzvwYY/yxkv5EEfnR2/HYqBNwkzFmJ1YXXbQx5mvvhmSbeCBeRDI/nf2A9UZQXlwJ7BCRBBFJBX4ErvByTHY7aIypC5Dx9ZCX47FFeUn8S4EmxpgwY0wFrBtM07wck22MMQarn3ijiLzt7XjsJCL/E5EGIhKK9XubLSLlotUoIgeAPcaYphlFDmCDF0Oy226ggzGmcsbfqINydPM6wzTg3ozv7wV+9mIstvHzdgB2EJE0Y8xjwAyskQXjRGS9l8OyUyfgHmCtMWZVRtmLIvK7F2NSrnkcmJjRINkO9PVyPLYRkSXGmB+AFVgjz1ZyDj/paoyZBHQHgo0x8f/fvh2cAAjFQBTcFGkFFuLVWqzyH2zAgyC4MxXk9AKBJDmSnEmumdlzL7rtuwnf43MXoMxfTj0APCT8AGWEH6CM8AOUEX6AMsIPUEb4AcoIP0CZBf7W8xnwTJFjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.005,epochs=10,bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.002,epochs=40,bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 0.5836355078959558, Val loss:0.6150993410053911\n",
      "Epoch 2. Training loss: 0.5812619501018039, Val loss:0.6128409193151243\n",
      "Epoch 3. Training loss: 0.5789459293074573, Val loss:0.6106360768362856\n",
      "Epoch 4. Training loss: 0.5766850365091964, Val loss:0.6084904852793694\n",
      "Epoch 5. Training loss: 0.5744789296527971, Val loss:0.6063891978772881\n",
      "Epoch 6. Training loss: 0.5723373977192706, Val loss:0.6043485713369722\n",
      "Epoch 7. Training loss: 0.570255835669074, Val loss:0.6023475258266295\n",
      "Epoch 8. Training loss: 0.5682251860925956, Val loss:0.6004011843934712\n",
      "Epoch 9. Training loss: 0.5662459307847894, Val loss:0.5985146170509894\n",
      "Epoch 10. Training loss: 0.5643163289219851, Val loss:0.596682528276356\n",
      "Epoch 11. Training loss: 0.5624337348766189, Val loss:0.5949019343289694\n",
      "Epoch 12. Training loss: 0.5606009715041292, Val loss:0.5931705748089766\n",
      "Epoch 13. Training loss: 0.5588193118400651, Val loss:0.5914807632937185\n",
      "Epoch 14. Training loss: 0.5570907821927147, Val loss:0.5898443516087132\n",
      "Epoch 15. Training loss: 0.55540275294309, Val loss:0.5882449735384141\n",
      "Epoch 16. Training loss: 0.5537532419278757, Val loss:0.5866871591726496\n",
      "Epoch 17. Training loss: 0.5521374230369535, Val loss:0.5851619118837453\n",
      "Epoch 18. Training loss: 0.5505607162654491, Val loss:0.5836765068882681\n",
      "Epoch 19. Training loss: 0.5490220659178442, Val loss:0.582226620988888\n",
      "Epoch 20. Training loss: 0.5475161577492196, Val loss:0.580818635715793\n",
      "Epoch 21. Training loss: 0.5460421019715915, Val loss:0.5794464490374523\n",
      "Epoch 22. Training loss: 0.5446008248646742, Val loss:0.5781088281979572\n",
      "Epoch 23. Training loss: 0.5431946978984096, Val loss:0.5768081313286882\n",
      "Epoch 24. Training loss: 0.5418244724575391, Val loss:0.5755384413487639\n",
      "Epoch 25. Training loss: 0.540483439587076, Val loss:0.5743017946161555\n",
      "Epoch 26. Training loss: 0.5391687755442198, Val loss:0.573094126422322\n",
      "Epoch 27. Training loss: 0.5378802679166989, Val loss:0.5719119734560865\n",
      "Epoch 28. Training loss: 0.5366200750589931, Val loss:0.5707624623848314\n",
      "Epoch 29. Training loss: 0.5353876559777971, Val loss:0.5696344402557547\n",
      "Epoch 30. Training loss: 0.534175857916342, Val loss:0.568528834118634\n",
      "Epoch 31. Training loss: 0.5329828949469511, Val loss:0.5674478948197902\n",
      "Epoch 32. Training loss: 0.531812284993516, Val loss:0.5663860104455425\n",
      "Epoch 33. Training loss: 0.5306620934348415, Val loss:0.5653420245255694\n",
      "Epoch 34. Training loss: 0.5295331883936738, Val loss:0.5643198304595861\n",
      "Epoch 35. Training loss: 0.528425289160275, Val loss:0.5633211633010891\n",
      "Epoch 36. Training loss: 0.5273354915292968, Val loss:0.5623404266647857\n",
      "Epoch 37. Training loss: 0.5262633649440254, Val loss:0.5613869262956941\n",
      "Epoch 38. Training loss: 0.5252085134376023, Val loss:0.5604488077056702\n",
      "Epoch 39. Training loss: 0.5241684828525558, Val loss:0.5595246113388693\n",
      "Epoch 40. Training loss: 0.5231451682749585, Val loss:0.5586163705936582\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl8VPW9//HXdzKTmck62SCQkI2dhJDECCgoi9TdqEgr1AWjLT9rbWvby6319talva3btYq1WlxG3MAVSSlqa8sVba0WEJBFFlkkrNn3bZLv748zGSYQkgECkzN+no9HHpk5c+bM92TgPd/5nO/5HqW1RgghRGixBLsBQggh+p6EuxBChCAJdyGECEES7kIIEYIk3IUQIgRJuAshRAiScBdCiBAk4S6EECFIwl0IIUKQNVgvnJiYqDMyMoL18kIIYUpr1qwp11on9bZer+GulHIAqwC7d/03tNZ3H7WOHXgBOAuoAK7VWu/uabsZGRmsXr26t5cXQgjhRym1J5D1AinLtADTtdbjgDzgYqXUxKPWuQWo0loPA34HPHAijRVCCNG3eg13baj33rV5f46ebexKYJH39hvABUop1WetFEIIcUICOqCqlApTSq0DDgN/1Vp/ctQqKcBeAK21B6gBEvqyoUIIIQIX0AFVrXU7kKeUcgFLlVI5WuuNfqt010s/Zi5hpdQ8YB5AWlraSTRXCHGy2traKC0tpbm5OdhNEQFwOBykpqZis9lO6vknNFpGa12tlPo/4GLAP9xLgSFAqVLKCsQCld08fyGwEKCwsFAmkhfiDCotLSU6OpqMjAykatq/aa2pqKigtLSUzMzMk9pGr2UZpVSSt8eOUsoJzAC+OGq1EmCu9/Ys4O9argIiRL/S3NxMQkKCBLsJKKVISEg4pW9ZgfTcBwGLlFJhGB8Gr2mtlyul7gNWa61LgGeBF5VSOzB67LNPukVCiNNGgt08TvW96jXctdYbgPxulv/S73Yz8M1TakmgPvwQ7r4bFi6EYcPOyEsKIYTZmG/6gWefhZUrYcGCYLdECHECKioqyMvLIy8vj+TkZFJSUnz3W1tbA9pGcXExW7du7XGdJ554gpdffrkvmszkyZNZt25dn2zrTAva9AMnrbgYFi2CceOC3RIhxAlISEjwBeU999xDVFQU//Ef/9FlHa01Wmsslu77nW63u9fX+f73v3/qjQ0B5uu55+UZvysrWbV7FT9//+eUN5YHt01CiJO2Y8cOcnJyuPXWWykoKODAgQPMmzePwsJCsrOzue+++3zrdvakPR4PLpeLO++8k3HjxnHOOedw+PBhAH7xi1/w6KOP+ta/8847GT9+PCNHjuSf//wnAA0NDVxzzTWMGzeOOXPmUFhY2GsP/aWXXmLs2LHk5ORw1113AeDxeLjhhht8yxd4Kwq/+93vGDNmDOPGjeP666/v879ZIMzXc4+Nhbg4Wnbt4NJX7qWhrYF4ZzzzJ80PdsuEMI877oBAyg1tbXDwICQnQ2/jrfPywBuqJ2rz5s243W6eeuopAO6//37i4+PxeDxMmzaNWbNmMWbMmC7PqampYcqUKdx///385Cc/4bnnnuPOO+88Zttaaz799FNKSkq47777ePfdd3n88cdJTk7mzTffZP369RQUFPTYvtLSUn7xi1+wevVqYmNjmTFjBsuXLycpKYny8nI+//xzAKqrqwF48MEH2bNnD+Hh4b5lZ5r5eu4AmZksrf2EhrYGhscPpzi/ONgtEiI0HTwIO3cav0+joUOHcvbZZ/vuL168mIKCAgoKCtiyZQubN28+5jlOp5NLLrkEgLPOOovdu3d3u+2ZM2ces85HH33E7NnGoL5x48aRnZ3dY/s++eQTpk+fTmJiIjabjW9/+9usWrWKYcOGsXXrVn70ox/x3nvvERsbC0B2djbXX389L7/88kmfhHSqzNdzB8jI4I8x7wIwKHoQiRGJQW6QECYTaA+7vBzcbuNYV+Lp+38WGRnpu719+3Yee+wxPv30U1wuF9dff323473Dw8N9t8PCwvB4PN1u2263H7POiZ6Gc7z1ExIS2LBhA++88w4LFizgzTffZOHChbz33nt88MEHLFu2jF//+tds3LiRsLCwE3rNU2XKnvvWrBj+b2AjAM0eOZVaiNMmMRHmzz+twX602tpaoqOjiYmJ4cCBA7z33nt9/hqTJ0/mtddeA+Dzzz/v9puBv4kTJ7Jy5UoqKirweDwsWbKEKVOmUFZWhtaab37zm9x7772sXbuW9vZ2SktLmT59Og899BBlZWU0Njb2+T70xpQ994WJe7A2wNmDC6lvawp2c4QQfaigoIAxY8aQk5NDVlYWkyZN6vPX+MEPfsCNN95Ibm4uBQUF5OTk+Eoq3UlNTeW+++5j6tSpaK254ooruOyyy1i7di233HILWmuUUjzwwAN4PB6+/e1vU1dXR0dHBz/72c+Ijo7u833ojQrWLAGFhYX6ZC7W0expJuX+AUzfUIdtxoWsbtnFth9sOw0tFCK0bNmyhdGjRwe7Gf2Cx+PB4/HgcDjYvn07F154Idu3b8dq7V/93e7eM6XUGq11YW/P7V97EoBF6xZR2V7HtRthxbmtNGnpuQshTkx9fT0XXHABHo8HrTV//OMf+12wnyrT7c2B+gMA7IwHR30zzXapuQshTozL5WLNmjXBbsZpZboDqrePv50HZzzIzbvjcNY20iQ1dyGEOIbpwj0xIpH5k+aTOGgojqp6GS0jhBDdMF24+2Rk4KyspV234+nofnyrEEJ8XZk63B3lxmm9UpoRQoiuzB3uzUaPXUozQvR/U6dOPeaEpEcffZTbbrutx+dFRUUBsH//fmbNmnXcbfc2tPrRRx/tcjLRpZde2ifzvtxzzz08/PDDp7ydvmbqcHe2GTebPNJzF6K/mzNnDkuWLOmybMmSJcyZMyeg5w8ePJg33njjpF//6HBfsWIFLpfrpLfX35k33DMzcXhL7dJzF6L/mzVrFsuXL6elpQWA3bt3s3//fiZPnuwbd15QUMDYsWNZtmzZMc/fvXs3OTk5ADQ1NTF79mxyc3O59tpraWo60sH73ve+55su+O677wZgwYIF7N+/n2nTpjFt2jQAMjIyKC83pgt/5JFHyMnJIScnxzdd8O7duxk9ejTf/e53yc7O5sILL+zyOt1Zt24dEydOJDc3l6uvvpqqqirf648ZM4bc3FzfhGUffPCB72Il+fn51NXVnfTftjumG+fuk56O0xvuUnMX4sTc8e4drDvY+5S/be1tHKw/SHJUMrawnmc3zEvO49GLjz8hWUJCAuPHj+fdd9/lyiuvZMmSJVx77bUopXA4HCxdupSYmBjKy8uZOHEiRUVFx72O6JNPPklERAQbNmxgw4YNXabs/Z//+R/i4+Npb2/nggsuYMOGDfzwhz/kkUceYeXKlSQeNU/OmjVrcLvdfPLJJ2itmTBhAlOmTCEuLo7t27ezePFinn76ab71rW/x5ptv9jg/+4033sjjjz/OlClT+OUvf8m9997Lo48+yv3338+uXbuw2+2+UtDDDz/ME088waRJk6ivr8fhcPT49z1R5u25R0bicBrzNTSXn97pSIX4ujpYf5Cd1Ts5WN83/8f8SzP+JRmtNXfddRe5ubnMmDGDffv2cejQoeNuZ9WqVb6Qzc3NJTc31/fYa6+9RkFBAfn5+WzatKnXScE++ugjrr76aiIjI4mKimLmzJl8+OGHAGRmZpLnvUBQT9MKgzG/fHV1NVOmTAFg7ty5rFq1ytfG6667jpdeesl3JuykSZP4yU9+woIFC6iuru7zM2TN23MHnMroSTQvXwo5FwW5NUKYR089bH/ljeW4P3NTnF/cJ1NrX3XVVfzkJz9h7dq1NDU1+XrcL7/8MmVlZaxZswabzUZGRka30/z6665Xv2vXLh5++GH+/e9/ExcXx0033dTrdnqaX6tzumAwpgzurSxzPH/+859ZtWoVJSUl/OpXv2LTpk3ceeedXHbZZaxYsYKJEyfy/vvvM2rUqJPafnfM23MHHHFJADR9Y1qQWyJEaPKdNNhH10yIiopi6tSp3HzzzV0OpNbU1DBgwABsNhsrV65kz549PW7n/PPP910Ee+PGjWzYsAEwpguOjIwkNjaWQ4cO8c477/ieEx0d3W1d+/zzz+ftt9+msbGRhoYGli5dynnnnXfC+xYbG0tcXJyv1//iiy8yZcoUOjo62Lt3L9OmTePBBx+kurqa+vp6vvzyS8aOHcvPfvYzCgsL+eKLL074NXti6p67w2EMkWqOtPeyphCiv5gzZw4zZ87sMnLmuuuu44orrqCwsJC8vLxee7Df+973KC4uJjc3l7y8PMaPHw8YV1XKz88nOzv7mOmC582bxyWXXMKgQYNYuXKlb3lBQQE33XSTbxvf+c53yM/P77EEczyLFi3i1ltvpbGxkaysLNxuN+3t7Vx//fXU1NSgtebHP/4xLpeL//7v/2blypWEhYUxZswY31Wl+orppvz198XM8xk97kNemfkKc8YGNpxKiK8rmfLXfE5lyl9zl2VsTkCGQgohxNFMHe5Ou3HdRTmJSQghujJ1uDvCjXCXnrsQgQlWGVacuFN9r8wd7p0HVCXcheiVw+GgoqJCAt4EtNZUVFSc0olNph4tE+6MRGk5Q1WIQKSmplJaWkpZWVmwmyIC4HA4SE1NPennmzrclcOJwwPNUnMXolc2m43MzMxgN0OcIaYuy+B04myDppaGYLdECCH6FXOHu8Nh9Nwl3IUQootew10pNUQptVIptUUptUkp9aNu1pmqlKpRSq3z/vzy9DT3KE4nTgl3IYQ4RiA1dw/wU631WqVUNLBGKfVXrfXRU619qLW+vO+b2ANvz72ptbH3dYUQ4muk15671vqA1nqt93YdsAVIOd0NC4jTe0C1TcJdCCH8nVDNXSmVAeQDn3Tz8DlKqfVKqXeUUtl90LbeORzGAVUJdyGE6CLgoZBKqSjgTeAOrXXtUQ+vBdK11vVKqUuBt4Hh3WxjHjAPIC0t7aQb7ePtuTe0yUlMQgjhL6Ceu1LKhhHsL2ut3zr6ca11rda63nt7BWBTSh0zAbTWeqHWulBrXZiUlHSKTcfouXugSc5QFUKILgIZLaOAZ4EtWutHjrNOsnc9lFLjvdut6MuGdquz5t4u4S6EEP4CKctMAm4APldKdV5R9y4gDUBr/RQwC/ieUsoDNAGz9ZmYwMJbc29ubz3tLyWEEGbSa7hrrT8Cur8E+ZF1fg/8vq8aFTBvz72po+WMv7QQQvRnoXGGaof03IUQwp+5w917hmqTlnAXQgh/5g53b8+9lXY6dEewWyOEEP2G6cPd2WbclAt2CCHEEeYOd6sVR4exCxLuQghxhLnDHXBYbICEuxBC+DN9uDtVOCCX2hNCCH+mD3dHmBHu0nMXQogjTB/uTosdgCa5jqoQQviYPtwdYQ5Aeu5CCOHP9OHutEq4CyHE0Uwf7g6bEe5yQFUIIY4IgXCPAKTnLoQQ/kwf7k6bE5ADqkII4c/04e6wRwLScxdCCH+mD3dnuBHuUnMXQogjTB/u0nMXQohjmT/cHVGAhLsQQvgzfbhbnBGEe6CprTHYTRFCiH7D9OHuu9Req4S7EEJ0Mn+4d15qr7k+2C0RQoh+w/zh7uu5NwS7JUII0W+YP9ydTpxt0NQi4S6EEJ3MH+6dPXcZ5y6EED7mD3enU8JdCCGOYv5wdziMA6oS7kII4WP+cO/suctJTEII4WP+cHc4jAOq7RLuQgjRyfzh3tlzb28JdkuEEKLfMH+4d9bcJdyFEMLH/OHe2XPvaA12S4QQot8wf7h3jnPXbcFuiRBC9Bu9hrtSaohSaqVSaotSapNS6kfdrKOUUguUUjuUUhuUUgWnp7nd6DxDlTa01mfsZYUQoj8LpOfuAX6qtR4NTAS+r5Qac9Q6lwDDvT/zgCf7tJU98fbcNZq2Dum9CyEEBBDuWusDWuu13tt1wBYg5ajVrgRe0IZ/AS6l1KA+b213rFacHcZuyIlMQghhOKGau1IqA8gHPjnqoRRgr9/9Uo79ADhtHMoGyIlMQgjRKeBwV0pFAW8Cd2ita49+uJunHFMAV0rNU0qtVkqtLisrO7GW9sBhkXAXQgh/AYW7UsqGEewva63f6maVUmCI3/1UYP/RK2mtF2qtC7XWhUlJSSfT3m45LXYAmjxSlhFCCAhstIwCngW2aK0fOc5qJcCN3lEzE4EarfWBPmxnj1rDrQDsrzvm80QIIb6WAum5TwJuAKYrpdZ5fy5VSt2qlLrVu84KYCewA3gauO30NLd7/xhkjJJ5Y/MbZ/JlhRCi37L2toLW+iO6r6n7r6OB7/dVo07UrLIkns6sZFLapGA1QQgh+hXzn6EKpOtY44acwySEEECIhHusNRKA2pYjg3g2Ht7IsAXD2Fq+NVjNEkKIoAmNcLdFA1DTUuNb9tA/H+LLqi95+J8PB6tZQggRNCER7g57BNYOqGk+Eu5nJZ8FwITUCcFqlhBCBE1IhLtyRhDbaunSc2/X7cbvjvZgNUsIIYImJMIdh4PYlq419+rmagCqmquC1SohhAia0Ah3p5OY5q41985wr2yqDFarhBAiaEIj3B0OYps6utTcq1u8Pfemk+u5n+zzhBCiPwiNcHc6iW2GWr9w7wxn/7JMi6eFu96/i998+BvKG8uPu7ldVbtIeiiJ2/58W4/rCSFEf9XrGaqm4K2513hLMdB9zf2DPR/w23/8FgCbxcb8SfO73dzGwxtp1+08ufpJMl2Zx11PCCH6q5Dpuce0QE13B1T9yiuHGw4DcG32tRTnFx93c3tq9gBw1cirelxPCCH6q9AId4fDKMu01vmuo9pdz72swZhDfnzKeBIjEo+7uT3VRrhPTJ3Y43pCCNFfhUa4O53Ethhj2xvbGoEjoe7fcy9rNMK9tzp6Z8+9rrXudLRWCCFOu9AI97Y2YlqMmzUtNXg6PNS31mO1WKlpqfGdyNTZc/cPd601L214iQc+esC3vDPc61vrz+BOCCFE3wmNA6o7dxLrvcJeTXMN9jDjykxpsWnsrNpJTUsN8c74bnvun+77lBuW3gCARVmYP2m+ryxT1yI9dyGEOYVGz/3224kdWwhA7Ufv++rtma5M4EhpprtwL60tBWBG1gyK84tp9jRzqOEQIGUZIYR5hUa4JyYS+6sHAah54hGqGozw9oW7t/7eXVnmYP1BAFwOF4kRiXxV85XvMSnLCCHMKjTCHYiJMka11BzYTfUv/gOArLgs4EjPvXMoZGcPHo6E+6bDm4AjI2XCw8Kl5y6EMK2QCfdYh3E1ptpBcVSv/giAzLgjPffW9lZqWmqwWqxUNlX6DrJ2hvv2yu20trf6DqaOThwtPXchhGmFTrjbjXCvmXYu1Q5jmX/NvbMUMyx+GB26w1eXP9hghLunw8O2im3sqd5DmApjZOJIOaAqhDCtkAn3qPAoAGpyRx4Jd2/PvbKp0ldvH5M0BjhSdz9Yf5BBUYMAozSzu2Y3KTEpuOwuKcsIIUwrZMI9zBJGdHg0NbZ2qkakEdYBSRFJ2MPsVDVX+ersoxNHA13DfUrGFCzKwqayTeyp3kN6bDrR9mgpywghTCtkwh2MunttSy3Vo9JxNYPasoU4ZxxVTVW+nrt/uHfoDg7VHyIjNoNh8cOMcK/ZQ7ornejwaBrbGuVKTkIIUwqtcLfHUtNSQ3VqIq5m4PXXiXPEde25Jx0J96qmKto62kiOSiY7KZv1B9ezr3YfGbEZvjKP9N6FEGYUUuEeY4+hprmGatWCyxplhLvTG+4NZViUhREJIwAj3DtHynSG+5dVX9Ku242euz0akHAXQphTaEw/4BXriKWisQJrmxVX7EDYtIm4TcnsHxjB4dhMEpwJRIVH4bQ6jwl3jfZtJz023dfTl4OqQggzCq1wt8eys2onYSqMFIwhM3FfHWZTewdlWw+RNC4VgKTIJMoay7qEe7wz3reddFc6zR5jshrpuQshzCikwr2zLGNRFlyjL4AH5xKfuZWq9Ysoq20gab8xtj0xIvGYnrvD6iBMhdGu20mLTWN/3X5AJg8TQphTSNXcY+3e0TLN1bhik2H+fOISU6mxejgUG8aAg3XQ2tol3B1WBzH2GOxWO5lxmUTZoqhvrfcdUJWyjBDCjEIr3B2xNHmaaPI04XK4AIhzxAGwKw6SyhrhySePhHvDQZKjklFKAZAcmUx9Wz3uz9xEh8sBVSGEeYVUWaZzCgKAOGdcl98e3U5Schbccw+Jd4ykPNyouSdHJfues3T2UtyfuSnOL6bFY1z9Q8oyQggzCqmee4w9xnf76J47QNLl34LqahL//gk1rbXsrdnbJdwTIxKZP2k+iRGJMhRSCGFqvYa7Uuo5pdRhpdTG4zw+VSlVo5Ra5/35Zd83MzCdM0OCX7g7/cJ9RD7MnElii7Hb2yu3kxyZTHcibZGA1NyFEOYUSM/9eeDiXtb5UGud5/2579SbdXL8yzLd9twjkuD++0lsMO536I4uPXd/YZYwImwRUpYRQphSr+GutV4FVJ6Btpwy/557Z6h36blHJsHw4SROvdS3LNnmgrVrYds2qKwEfeRkpuhwmTxMCGFOfVVzP0cptV4p9Y5SKruPtnnCequ5D4gcAEDSd3/kW5Z8yx1w1lkwciQkJMCoUVBuzBgZbY+WsowQwpT6ItzXAula63HA48Dbx1tRKTVPKbVaKbW6rKzseKudtO7KMk6bE3uYHYUiwZkAQGJWjm+95JwJcMMNxp2UFKMHf//9gDFHvPTchRBmdMpDIbXWtX63Vyil/qCUStRal3ez7kJgIUBhYaE++vFT1dlzDw8Lx2F1+JbHOeNoa28jzBIG4At5gOQHn4SYITB2LEyebPw0NgJGWUZ67kIIMzrlnrtSKll5zwJSSo33brPiVLd7MuxWO/YwO3GOON+JSWCUZpIik3z3bWE2Xy9/YNpoSEyE+fPhnHPgmmvg9dehpYWo8Cg5oCqEMKVAhkIuBj4GRiqlSpVStyilblVK3epdZRawUSm1HlgAzNZa93mvPFCxjlhfSaZTQkQCng6P7+pLYIxpdzlcXXr4AHznO0bNvaRErsYkhDCtQEbLzNFaD9Ja27TWqVrrZ7XWT2mtn/I+/nutdbbWepzWeqLW+p+nv9nHF2s/NtwnD5nMjsoduD9zd1nPZrF1CXwAZsyA9HR45hkpywghTCukzlAFYyx7Q1tDl9D+6bk/5cEZD1KcX+xbNiByAGWNZV0CHwCLBW6+Gf7yF6L+vZ665lqEEMJsQi7cp2VOY+PhjV1C239agU4vznzxmMD3uflmsFqJ/sdq6tsaCGKVSQghTkpITRwGcMfEO4i1x3Yf2n46A79bqanw618T9ac70Wga2xqJDI88Da0VQojTI+R67t310k/Kf/4n0UNHAVD/xYY+aJkQQpw5IRfufUYpoouNAUF1xd/2nbUqhBBmIOHeg6iBaQDU7d8NbnfPKwshRD8i4d4D35zuzjAo7rmGL4QQ/YmEew86L7VXZ22HSDmgKoQwDwn3HnReJLs+HGM6YCGEMAkJ9x50lmXqwoGKoEyXI4QQJ0XCvQedPfc6O9JzF0KYioR7Dzpr7vXScxdCmIyEew9sYTbsYXajLCM9dyGEiUi49yIqPMooy0jPXQhhIhLuvYi2R1PvtEjPXQhhKhLuvYgKj6IuKlx67kIIU5Fw70V0eDT1Tqv03IUQpiLh3otoezR1TiU9dyGEqUi49yLcEs5OZyvl9YeD3RQhhAiYhHsvqpqrKLO18NyAfcFuihBCBEzCvRcXDr0QgJmfNYNcbk8IYRIS7r3ITsoGoM7igYaGILdGCCECI+Hei3RXOgB7XMiIGSGEaUi49yI91hvusciIGSGEaUi49yIxIpEIi0N67kIIU5Fw74VSivTIwex2IT13IYRpSLgHIN2VbpRlpOcuhDAJCfcApCcMNcoy0nMXQpiEhHsAMhKGUhEB9RUHgt0UIYQIiIR7AHwjZur2BrklQggRGAn3APjGujdJz10IYQ4S7gHIcGUAsKdNau5CCHPoNdyVUs8ppQ4rpTYe53GllFqglNqhlNqglCro+2YGV3JUMuEdFnZTHeymCCFEQALpuT8PXNzD45cAw70/84AnT71Z/YtFWRjSEcUem8wtI4Qwh17DXWu9CuhpgPeVwAva8C/ApZQa1FcN7C/SLXHscbbIzJBCCFPoi5p7CuA/jKTUuyykZNgHGicy1dYGuylCCNGrvgh31c2ybru3Sql5SqnVSqnVZWVlffDSZ056xGAOREPz4f3BbooQQvSqL8K9FBjidz8V6DYBtdYLtdaFWuvCpKSkPnjpM6dzOOTe/VuC3BIhhOhdX4R7CXCjd9TMRKBGax1yA8LTE4YC8NtNT1HeWB7k1gghRM8CGQq5GPgYGKmUKlVK3aKUulUpdat3lRXATmAH8DRw22lrbRCNHT4Zexu4y/7KjYuuorW9NdhNEkKI41I6SKM/CgsL9erVq4Py2idr7XXTucG1ks0DILHNxq/Ov5vvzriTMEtYsJsmhPiaUEqt0VoX9raenKF6Agoee41NsT/n5u3RlNva+N7Hv2DUfUl8695sSndtCHbzhBDCR8L9RCQmwm9+wwO//TcP1E3k2a2jqK+r4nU2M+K5cUz/r1Q+/nxFsFsphBBSljlVZX9dxn8vmcf/RRxma6KxLK9jICk6kruvfYqzs78R3AYKIUJKoGUZCfc+Ur5lDQt+fwO2mjr+OKCUfbHG8gGt4czMuIQrp/4/JqefR1R4VHAbKoQwNQn3ICr/19/5X/c8Pmsv5b0hLYS1Q3sYqA7IcQwhbkA6cwuKuWjoRaTEhNzJvEKI00jCvR8o37sV99O3MTv6XO77/HGeGVpDXCNURRxZJ6bDRkHK2dickczNm8vU9KkMjh6MUt2d+CuE+LqTcO9nyvduxf38HRR9aeWtL5dT2Oji9yOqKRkFka3QEH5k3UhbJOMGjsNqsTI7ZzaT0iYxKnEU4WHhx38BIcTXgoR7f1VeDm43FBdTvmcL7ud+SNGGFl5XW8g/AE+fBctGQ0xbGLW2dt/TLMpCVlwWUbYoLht+GRNSJ5AzIId0VzoWJYNSQ5a6AAATqElEQVSehPi6kHA3k87Av/BCyh+6F/eupRR95eTt9CYmfwVPTFAsztEkeRyUWZu7PNUZ5iQhMoHJaZMZlTCKr2q+4vbxtzMueZyEvhAhSMLdrDqDvqgIXnkFhg2jfNGTuJs+pmgrlIyEazbD/34jgj9kN5LeEcMeSy3R1kjqPEcuJmKz2BiTNIYRCSOoa6ljds5sxqeMZ2j8UCnvCGFiEu6hpDPwL74YFi6E5GTKly3GbdvkC/zidVA5Mp1HxzawNy2W5dYvGR6dweHWKmpaanybsmAhMy4Tu9XOxJSJ1LbUMjdvLhNTJ5IYkRjEnRRCBELCPdR1Bv7kyfDMM5CUBCUlsGUL5RHgzjMCnyFDeGpsC18NTeRp52amJ57NYdXExrKNKBTab+p9p9XJiIQRWC1WLhp6EfmD8hmRMILh8cNx2pzB21chhI+E+9dRZ+Cfc47xOzkZ/vQn+PzzroEfFYn77HAui5/AC1FfMm7ClbyiPmf5rveId8ZT2XTsVRVdDhdjB4wlKy6LisYK5oydQ+HgQjJdmdjCbGd+X4X4mpJwF4bOwL/gAnjuOcjMhDfegH/9C6xW8HiM1SLAPdFOkWU0JQOr+WbedTwav43H9r7OiPgRbKvcxpCYIZQ3ltPkafJtXqEYEjMEp83J+WnnkxWfxZeVX3Jr4a3kJefJjJlC9DEJd3F8nYF/xRXw4oswdqzxe8UKiInxXSe2s7dfdMhFyRgLxQMuQmdl8UTFO0y4+Ds8X72S1za/TnJkMgcbDmKz2GjraPO9TJgKIzMuk6y4LJramrgg8wL21e3jtrNvI3dgrozmEeIkSLiLE+M/SmfZMqOn/8QTxrLUVCgtBbsdWlqOPCUxAvfZNooiCyiJ3s9N597Goex0njj8Z0obDrB8+3Kyk7IpbyznUMOhLi9nD7PjcrjITsrG0+GhaGQR+YPyGRo3lNSYVOnxC3EcEu7i1PkHfkkJ3HQTHDoECxbA0KHGh8DHHx8T+ihF+dBBuHPaKB50KWRl8dz+PzP+8ltZ3PAxIxJGUrK1hFVfrSLSFklDW0OXl7UqK3HOOPIH5ZMZm8mhhkPMyZnDWYPPIt2VjtViPbN/ByH6EQl3cfr5h//rrxsHcp95BpYsMQ7mHjzYpa4PQHg4jBpF+YhU3JHbKBpzNcvKP+KiWT/nD4eWs3DtQrJcWeys3klyVDIVjRXHlHqGxAwhPCycwsGF1LbUcm3OteQn55MVlyWjekTIk3AXwXF0b3/uXNi/3yjx7NsH77wDI0fC4cNQVdX1qYNdRo0/soCSqFKKx/8/9LhxLKh+j9KWMp5f/zzTM6dT3ljOhkMbsCorHu3pso2kiCSiwqOYkDqB9Jh09tTs4cZxN5I/KJ+BkQNlQjZhehLuov/xm1cHgKefhnHj4LXXjFE8K1bAp58eW+bBG/z5UBwzBdLScNd9yBXfuJ0l9m2cM3w6i9YvYvHGxaTGpFJaW0qMPYbaltou23CEORgSO4QwSxgFyQXUtNRwzehryB2YS4Yrg3hnvIS/6Pck3IX5+Pf633jDOEHr2WeNkTydB3X9RvP4xMYaZZ70SoqSzqNEb6F4xnyaRw/nsb2vs6/+AIs3Lua8tPOobKpkU9mmbnv9EdYIou3RjB04ltToVA41HOKqUVcZE7TFppMclSwHekXQSbiL0HB0mae4GBob4bHHjDLPq6/CxIlQUQHbtx/7fIuF8mGDcY9qpjjhAhgyBHfl3yj6xg941baV80ZdyPPrFvHChhfIiM1gd81uBkcPpqqpqst4fjDq/TH2GIbGDaVdt3Ne2nmkxKSwpWwLN4y7gbEDxpIYkSi9f3FaSbiL0Hd0mcfthksvNXr6Z59tTLz21ltHDu46ndDUNbCJjKR8eAruzGqK4s6hxLKN4km3w5gxPFX9PvvbKnly9ZNcPepqqpuqWblnJXGOOKqaq7AoCx26o8vmbBYbg6IGYbFYGJ0wmvq2emZkzSA5Kpl1B9dRnFdMzoAcOfArTpqEuxDd9fpra+Hxx+HAAaPXP2kSVFbCli1gsUBH17AuT4zAfVYYxWFnQUoK7o41FJ31bUriDjN3/DwOR8DCz57mUP0hXtv8GlPSp1DTUsO6g+uIsEbQ6Gnstmkx9hgGRg6kQ3f4Dvbuqd7D1IypbKvYxtxxcxmVNIo4R5x8ExBdSLgL0ZPuev1XXGEM45w8GRYtgpdegvR02LPH6P1XVR1zoBerFaKjKR+VhjuljOLBl8KgQbgPrKBoxu0sC9/JzILrOdB4iBfWv0BZYxnLti7j3NRzqW2tZePhjcQ54qhtqaVdtx/TzDAVRmJEIlaLldFJo0mKSOJA3QHGp4znq5qvmDl6JsPih5EUmURSRBJ2q/0M/PFEMEm4C3Equuv1Azz5pDG086mn4KqroKYGVq6EuDgj/Lvp/WO1wqBBRv1/7FDcsbsoHv5NGDAA9443KL7ybvSoUfxh8yIKBxdSsrWEgkEFLN+2nOXbl/tGAKVEp1DXWnfMKCB/9jA7yVHJJDgTaPQ0MiphFBVNFZyXdh4JEQmsO7iOa0ZfQ1ZcFvHOeBIiEnBYHafxDyn6moS7EKdLd71+/3H9hw8bwzwPHTJKP1OnGh8Cn30GERHGAeHuhIfD4MHGB0RuLuXJMbhbP6Fo2OWUHP6Q4kvvgmHDeK50OVMzp/HG5jc4P+N8Xtv0Gi9ueNE3wVtech4NrQ1sr9yOw+qg2dPc/et5WS1W4h3GMNCsuCxi7bGUN5WTnZTNgboDTMuchsvu4tP9nzJrzCwyXBm4HC5cDhdOq1PKRmeYhLsQwXa8D4Fly2DWLKPuv2gR5OXB8uXGCV6dB38TEozjA21tx25XKeObQng4DB9ufAiEb6IoeQolDWspnnArZGbirvk/isZ+k5L9f2d2zhyqmqtwf+amsqmSFza8wMzRM2lsbeTdL9/1fTvIcmXR2NbIwYaD3R4wPppFWYgOjybMEkZKdArR9miqm6sZFjeM8sZyxqeMJzo8mo1lGzk39Vw2lm3kqpFXkRqbSnR4NDH2GKLt0UTaIuVDIkAS7kKYyfHKQE89BePHw9tvQ2Gh8diyZTBkCOzdC2lpUF9vHBQ+HovFuJhLQgK0tBjHB6K/pDj1Moh1GccHxt9IiX0XxTk3QHw87p1vUTTqSt764i2uHnk11S3VvPz5y1Q1VvHyxpeZOWomTZ4m3tnxDmmxaXxV8xWjE0fT5Glid/VunFYnTZ6mYy4I05NIWyQWZSHeEY9He0iLTSMyPJLDDYcZFjeMQw2HKBxcSIQ1go1lG5mWMY0IWwQfl37MFSOuYHD0YCLDI4mwRRBp8/4Ojwy5uYgk3IUIRcf7EPBeYJ1XXzVm9HzlFWP+/lGj4IsvjKGhDQ2weTNERhq3exMTYxwvGDLEd0yhPDsTd9jnFGdeDdHRuPe8TdGEuZTYdlKcfR3ExODevZSiMTMp2VrCTXk30djWyLOfPcukIZNYsX0F0zOns/SLpSxav4jspGw2lW1iRuYMWtpb+PCrDxkUNYgD9QfIcmXR0t7Cvrp9OMIcNLc3B/Rt4mhWZfUNPY21x9LW0cbg6ME4rU4qmipIi02jvLGcnAE5OMIcbK/cTvaAbHZV7WJy2mQcVgef7vuUc1LPYf2h9Vwx4goGRg3EaXPisDpwWo3fnT/hYeGn9VuIhLsQX2c9fQh0LrvxRuNYwDPPGD3/Z5+Fa681gn/5cuNbwVdfGXMBNTYa3xRstu5LRUez242y0YABEB1tfLvIyjJep6CA8pgw3FUrKRoyg5Kyj4xS0uDBuCv/TtHQS41lhd8BhwP3uucpGllEydYSivOLafG08MzaZ7hq1FU0tjWyZOMSalpqWLR+ETfk3kBreyuvbnqV0Ymj2VK+hWkZ02htb+Ufe//B4OjB7K/bz4j4EbR1tLGrehdRtijq2+qJDo+mrb2N5vaej1EEIkyFEWGLwGF10NbRhsvuorm9mUFRg7CH2RkcM5g/Xv7Hk7pusYS7ECJwPR0k7u6Ywbe+ZXwwvPCCMUrI7YbZs42TxJYtM+YK2rULcnOhuRm2bTvyjaGbuYOOy2IxPiRiY0FrSEkxtlNZaXwriYqCXbsozx2Gu+ljYxSS04l7x+sUjb6Kkp3vUDxjvjE8dd9y40Oi9G8UF34XlML9mbvLBweA+zM3lw2/jKVfLGXWmFk0e5p55fNXOHfIuby/831mZM3gT9v+xLOfPUvewDzWHVrH1aOupq29jeXblzMsbhg7qnYwacgk2jra+HTfp75vIyMTRtLW0cbOqp08OONB5k+af8JvVZ+Gu1LqYuAxIAx4Rmt9/1GP3wQ8BOzzLvq91vqZnrYp4S5EiDiRD4bOZR0dsHAhnH++sezCC435hDonk1u/Hi65xPhgWLnyyNxCI0cay/bsgfh440MikBLT0ZQyRi4pBS6X8W1k8GDjLObKSuNbS3m5cZUyp9P4cMrNhR07YOpUysM9uLe9StHIK40PkKl3wMCBuA+8Q1HWxZQc+MD4sHA4cH/+YrcfIMX5xcHtuSulwoBtwDeAUuDfwByt9Wa/dW4CCrXWtwfaQAl3IUQXgZSS/Jf53778cmOqiZkzjbB/5RWYMAH+9jfjA2TFCli8GMaMMY47TJ8Ora3w0UdGqO/fb3xwtLYa3zg6v2VERhrB39p68vullPEB0flB4nDA6NFGuxODG+7nAPdorS/y3v85gNb6t37r3ISEuxCivzrRD46jlz33nDFvUecHSFOTcTbzOecYHyDTpsGf/2yUqXJzYcMGuOwy44PhL385UqY66yzj4jXr18ODD8L8IJZllFKzgIu11t/x3r8BmOAf5N5w/y1QhtHL/7HWem9P25VwF0KElJP55hHknvs3gYuOCvfxWusf+K2TANRrrVuUUrcC39JaT+9mW/OAeQBpaWln7dmz50T2SQghvvYCDXdLANsqBYb43U8F9vuvoLWu0Fp3Hv5+Gjiruw1prRdqrQu11oVJSUkBvLQQQoiTEUi4/xsYrpTKVEqFA7OBEv8VlFKD/O4WAVv6rolCCCFOVK/n5WqtPUqp24H3MIZCPqe13qSUug9YrbUuAX6olCoCPEAlcNNpbLMQQoheyElMQghhIn1ZcxdCCGEyEu5CCBGCJNyFECIEBa3mrpQqA052oHsiUN6HzQmmUNkX2Y/+J1T2Rfajq3Stda9jyYMW7qdCKbU6kAMKZhAq+yL70f+Eyr7IfpwcKcsIIUQIknAXQogQZNZwXxjsBvShUNkX2Y/+J1T2RfbjJJiy5i6EEKJnZu25CyGE6IHpwl0pdbFSaqtSaodS6s5gtydQSqkhSqmVSqktSqlNSqkfeZfHK6X+qpTa7v0dF+y2BkIpFaaU+kwptdx7P1Mp9Yl3P171TjLX7ymlXEqpN5RSX3jfm3PM+J4opX7s/Xe1USm1WCnlMMt7opR6Til1WCm10W9Zt++BMizw/v/foJQqCF7LuzrOfjzk/be1QSm1VCnl8nvs59792KqUuqiv22OqcPde8u8J4BJgDDBHKTUmuK0KmAf4qdZ6NDAR+L637XcCf9NaDwf+5r1vBj+i6+yfDwC/8+5HFXBLUFp14h4D3tVajwLGYeyTqd4TpVQK8EOMq6HlYEzwNxvzvCfPAxcftex478ElwHDvzzzgyTPUxkA8z7H78VcgR2udi3Eho58DeP/vzwayvc/5gzff+oypwh0YD+zQWu/UWrcCS4Arg9ymgGitD2it13pv12GESApG+xd5V1sEXBWcFgZOKZUKXAY8472vgOnAG95VzLIfMcD5wLMAWutWrXU1JnxPMGZ4dSqlrEAEcACTvCda61UYs8n6O957cCXwgjb8C3AdNeV40HS3H1rrv2itPd67/8K4HgYY+7FEa92itd4F7MDItz5jtnBPAfwv31fqXWYqSqkMIB/4BBiotT4AxgcAMCB4LQvYo8B/Ah3e+wlAtd8/YrO8L1kYl4Z0e0tMzyilIjHZe6K13gc8DHyFEeo1wBrM+Z50Ot57YOYMuBl4x3v7tO+H2cJddbPMVMN9lFJRwJvAHVrr2mC350QppS4HDmut1/gv7mZVM7wvVqAAeFJrnQ800M9LMN3x1qOvBDKBwUAkRvniaGZ4T3pjyn9rSqn/wijNvty5qJvV+nQ/zBbuvV7yrz9TStkwgv1lrfVb3sWHOr9Wen8fDlb7AjQJKFJK7cYoi03H6Mm7vCUBMM/7UgqUaq0/8d5/AyPszfaezAB2aa3LtNZtwFvAuZjzPel0vPfAdBmglJoLXA5cp4+MPT/t+2G2cO/1kn/9lbcu/SywRWv9iN9DJcBc7+25wLIz3bYTobX+udY6VWudgfH3/7vW+jpgJTDLu1q/3w8ArfVBYK9SaqR30QXAZkz2nmCUYyYqpSK8/84698N074mf470HJcCN3lEzE4GazvJNf6SUuhj4GVCktW70e6gEmK2UsiulMjEOEH/apy+utTbVD3ApxlHnL4H/CnZ7TqDdkzG+dm0A1nl/LsWoV/8N2O79HR/stp7APk0FlntvZ3n/ce4AXgfswW5fgPuQB6z2vi9vA3FmfE+Ae4EvgI3Ai4DdLO8JsBjjWEEbRo/2luO9BxjljCe8//8/xxghFPR96GE/dmDU1jv/zz/lt/5/efdjK3BJX7dHzlAVQogQZLayjBBCiABIuAshRAiScBdCiBAk4S6EECFIwl0IIUKQhLsQQoQgCXchhAhBEu5CCBGC/j+U9tSNKvVq8QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.002,epochs=40,bs=512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normal Gradient Descent takes about 150 epochs (with manually tuned learning rate) to reach 80% accuracy on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8032"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accuracy on test set\n",
    "y_pred=nn.predict(X_test)\n",
    "accuracy(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimizer 2: Momentum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "act_obj = ReLU()\n",
    "layers = [X_train.shape[1],100,10]\n",
    "nn = CustomNeuralNetwork(layers,act_obj,opt=Momentum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.01,epochs=2,bs=512,beta1=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.01,epochs=20,bs=512,beta1=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.01,epochs=20,bs=512,beta1=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 0.4166741977541224, Val loss:0.45614356990020194\n",
      "Epoch 2. Training loss: 0.41565879820264495, Val loss:0.45553364681048425\n",
      "Epoch 3. Training loss: 0.41442260638028205, Val loss:0.4548421798664089\n",
      "Epoch 4. Training loss: 0.413212564313963, Val loss:0.4541376228101735\n",
      "Epoch 5. Training loss: 0.41203106429872943, Val loss:0.4534430697979531\n",
      "Epoch 6. Training loss: 0.4108719196903877, Val loss:0.45273883548476623\n",
      "Epoch 7. Training loss: 0.40973100081266095, Val loss:0.4520396109676074\n",
      "Epoch 8. Training loss: 0.4086102245053826, Val loss:0.45133821819854364\n",
      "Epoch 9. Training loss: 0.4075029789707413, Val loss:0.45064323623899283\n",
      "Epoch 10. Training loss: 0.40640367322253296, Val loss:0.4499640633935516\n",
      "Epoch 11. Training loss: 0.40532180953274344, Val loss:0.44931186488334396\n",
      "Epoch 12. Training loss: 0.404267287781448, Val loss:0.44868748555237226\n",
      "Epoch 13. Training loss: 0.40323629930404326, Val loss:0.4480778447635916\n",
      "Epoch 14. Training loss: 0.40222708500424403, Val loss:0.4474871313582692\n",
      "Epoch 15. Training loss: 0.40124064394652503, Val loss:0.44692525280341006\n",
      "Epoch 16. Training loss: 0.40027008665527286, Val loss:0.4463573543338698\n",
      "Epoch 17. Training loss: 0.3993170603246272, Val loss:0.4458260903242437\n",
      "Epoch 18. Training loss: 0.39838542749139777, Val loss:0.4452933246683409\n",
      "Epoch 19. Training loss: 0.39746943877362806, Val loss:0.44479593417077523\n",
      "Epoch 20. Training loss: 0.39657370304619144, Val loss:0.44429879773604636\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XlcVnX6//HXxc2+iQKKigq5CyISmo6aW1MuqS2aS7ZYjdX0bZmafjlNe818m2rKbMzGUsrG0a+lqZnpTGXZpoX7giamJbmBigvKct98fn+cW0JFQLjhyM31fDx4cM7hc59zHU+9z7k/ZxNjDEoppbyLj90FKKWU8jwNd6WU8kIa7kop5YU03JVSygtpuCullBfScFdKKS+k4a6UUl5Iw10ppbyQhrtSSnkhX7sWHBUVZeLi4uxavFJK1Ulr1qzJMcZEV9TOtnCPi4sjPT3drsUrpVSdJCI/VaaddssopZQX0nBXSikvVGG4i0gLEVkhIhkiskVE7i+jjYjIFBHJFJGNIpJSM+UqpZSqjMr0uTuBh4wxa0UkDFgjIv81xmwt1WYw0Nb9cxkwzf1bKXWRKCoqIisri/z8fLtLUZUQGBhIbGwsfn5+Vfp8heFujNkH7HMPHxeRDKA5UDrcRwCzjPVw+FUiEiEiTd2fVUpdBLKysggLCyMuLg4RsbscVQ5jDIcOHSIrK4v4+PgqzeOC+txFJA7oCqw+60/NgT2lxrPc05RSF4n8/HwiIyM12OsAESEyMrJa37IqHe4iEgrMBx4wxhw7+89lfOScVzyJyEQRSReR9Ozs7AurVClVbRrsdUd1t1Wlwl1E/LCCfbYxZkEZTbKAFqXGY4G9Zzcyxkw3xqQaY1Kjoyu8Br9sK1fCwIHwww9V+7xSStUDlblaRoAZQIYx5uXzNFsM3Oy+aqYHcLTG+tvfegs++wymTauR2SulasahQ4dITk4mOTmZmJgYmjdvXjJeWFhYqXlMmDCB7du3l9tm6tSpzJ492xMl07t3b9avX++RedW2ylwt0wu4CdgkIqfX8lGgJYAx5g1gKTAEyAROAhM8X6rb9dfDu+9Cnz41tgillOdFRkaWBOVTTz1FaGgof/zjH89oY4zBGIOPT9nHnWlpaRUu55577ql+sV6gwiN3Y8xXxhgxxiQZY5LdP0uNMW+4gx1juccY09oY09kYU3PPFWjd2vrtdNbYIpRStSczM5PExETuuusuUlJS2LdvHxMnTiQ1NZWEhASeeeaZkranj6SdTicRERFMmjSJLl260LNnTw4ePAjAY489xuTJk0vaT5o0ie7du9O+fXu++eYbAPLy8rj++uvp0qULY8eOJTU1tcIj9H/961907tyZxMREHn30UQCcTic33XRTyfQpU6YA8Morr9CpUye6dOnC+PHjPf5vVhm2PVumyiIjrd+HDtlbh1J12QMPQGW6G4qKYP9+iImBiq63Tk4Gd6heqK1bt5KWlsYbb7wBwPPPP0+jRo1wOp3079+fkSNH0qlTpzM+c/ToUfr27cvzzz/Pgw8+yMyZM5k0adI58zbG8N1337F48WKeeeYZli1bxmuvvUZMTAzz589nw4YNpKSUf99lVlYWjz32GOnp6TRo0IArrriCJUuWEB0dTU5ODps2bQIgNzcXgBdeeIGffvoJf3//kmm1re49fkDDXanas38//Pij9bsGtW7dmm7dupWMz5kzh5SUFFJSUsjIyGDr1q3nfCYoKIjBgwcDcOmll7J79+4y533ddded0+arr75izJgxAHTp0oWEhIRy61u9ejUDBgwgKioKPz8/xo0bx8qVK2nTpg3bt2/n/vvvZ/ny5TRo0ACAhIQExo8fz+zZs6t8E1J11b0jd39/CA3VcFeqOip7hJ2TA2lpMGECREXVWDkhISElwzt27ODVV1/lu+++IyIigvHjx5d5vbe/v3/JsMPhwHmertqAgIBz2lj3W1be+dpHRkayceNGPv74Y6ZMmcL8+fOZPn06y5cv54svvmDRokU899xzbN68GYfDcUHLrK66d+QO1tG7hrtSNS8qCh5+uEaD/WzHjh0jLCyM8PBw9u3bx/Llyz2+jN69ezNv3jwANm3aVOY3g9J69OjBihUrOHToEE6nk7lz59K3b1+ys7MxxjBq1Ciefvpp1q5di8vlIisriwEDBvDiiy+SnZ3NyZMnPb4OFal7R+5g/Yem4a6UV0pJSaFTp04kJiZyySWX0KtXL48v49577+Xmm28mKSmJlJQUEhMTS7pUyhIbG8szzzxDv379MMYwbNgwhg4dytq1a7n99tsxxiAi/O1vf8PpdDJu3DiOHz9OcXExjzzyCGFhYR5fh4rIhX498ZTU1FRT5Zd1XHUVHD0Kq1Z5tiilvFhGRgYdO3a0u4yLgtPpxOl0EhgYyI4dO7jyyivZsWMHvr4X1/FuWdtMRNYYY1Ir+uzFtSaVFRlpneRRSqkqOHHiBAMHDsTpdGKM4Z///OdFF+zVVTfXJjLSOtGjlFJVEBERwZo1a+wuo0bV3ROqubl6I5NSSp1H3Q13gCNH7K1DKaUuUnU73PWKGaWUKpOGu1JKeSENd6VUrejXr985NyRNnjyZ3//+9+V+LjQ0FIC9e/cycuTI8867okurJ0+efMbNREOGDPHIc1+eeuopXnrppWrPx9M03JVStWLs2LHMnTv3jGlz585l7Nixlfp8s2bNeP/996u8/LPDfenSpURERFR5fhc7DXelVK0YOXIkS5YsoaCgAIDdu3ezd+9eevfuXXLdeUpKCp07d2bRokXnfH737t0kJiYCcOrUKcaMGUNSUhKjR4/m1KlTJe3uvvvukscFP/nkkwBMmTKFvXv30r9/f/r37w9AXFwcOe5Lql9++WUSExNJTEwseVzw7t276dixI7/73e9ISEjgyiuvPGM5ZVm/fj09evQgKSmJa6+9liPuiz6mTJlCp06dSEpKKnlg2RdffFHyspKuXbty/PjxKv/blqVuXuceFga+vhruSlXRA8seYP3+ih/5W+QqYv+J/cSExuDnKP/phskxyUwedP4HkkVGRtK9e3eWLVvGiBEjmDt3LqNHj0ZECAwM5IMPPiA8PJycnBx69OjB8OHDz/se0WnTphEcHMzGjRvZuHHjGY/s/ctf/kKjRo1wuVwMHDiQjRs3ct999/Hyyy+zYsUKos56Ts6aNWtIS0tj9erVGGO47LLL6Nu3Lw0bNmTHjh3MmTOHN998kxtuuIH58+eX+3z2m2++mddee42+ffvyxBNP8PTTTzN58mSef/55du3aRUBAQElX0EsvvcTUqVPp1asXJ06cIDAwsNx/3wtVmdfszRSRgyKy+Tx/byAiH4rIBhHZIiI19xamXxeqDw9TqhbsP7GfH3N/ZP8Jzzzyt3TXTOkuGWMMjz76KElJSVxxxRX88ssvHDhw4LzzWblyZUnIJiUlkZSUVPK3efPmkZKSQteuXdmyZUuFDwX76quvuPbaawkJCSE0NJTrrruOL7/8EoD4+HiSk5OB8h8rDNbz5XNzc+nbty8At9xyCytXriyp8cYbb+Rf//pXyZ2wvXr14sEHH2TKlCnk5uZ6/A7ZysztbeAfwKzz/P0eYKsxZpiIRAPbRWS2MaZyL0WsKg13paqsvCPs0nJO5pC2Lo0JXScQFVz9J0Nec801PPjgg6xdu5ZTp06VHHHPnj2b7Oxs1qxZg5+fH3FxcWU+5re0so7qd+3axUsvvcT3339Pw4YNufXWWyucT3nP1zr9uGCwHhlcUbfM+Xz00UesXLmSxYsX8+yzz7JlyxYmTZrE0KFDWbp0KT169OCTTz6hQ4cOVZp/WSrzmr2VwOHymgBh7hdph7rb1vytoxruStW4qOAoHu71sEeCHawrX/r168dtt912xonUo0eP0rhxY/z8/FixYgU//fRTufO5/PLLS16CvXnzZjZu3AhYjwsOCQmhQYMGHDhwgI8//rjkM2FhYWX2a19++eUsXLiQkydPkpeXxwcffECfKryjuUGDBjRs2LDkqP/dd9+lb9++FBcXs2fPHvr3788LL7xAbm4uJ06cYOfOnXTu3JlHHnmE1NRUtm3bdsHLLI8nvgf8A1gM7AXCgNHGmGIPzLd8kZGQmVnji1FKedbYsWO57rrrzrhy5sYbb2TYsGGkpqaSnJxc4RHs3XffzYQJE0hKSiI5OZnu3bsD1luVunbtSkJCwjmPC544cSKDBw+madOmrFixomR6SkoKt956a8k87rjjDrp27VpuF8z5vPPOO9x1112cPHmSSy65hLS0NFwuF+PHj+fo0aMYY/jDH/5AREQEjz/+OCtWrMDhcNCpU6eSt0p5SqUe+SsiccASY0xiGX8bCfQCHgRaA/8FuhhjjpXRdiIwEaBly5aXVrR3Ltcdd8DSpbB3b9XnoVQ9oo/8rXuq88hfT1wKOQFYYCyZwC6gzN2uMWa6MSbVGJMaHR1dvaWefmGHTc+jV0qpi5knwv1nYCCAiDQB2gM1/7D1yEgoLIS8vBpflFJK1TUV9rmLyBygHxAlIlnAk4AfgDHmDeBZ4G0R2QQI8IgxpuYftl76Rib37clKqfKdfh2cuvhV9y15FYa7Mabce4ONMXuBK6tVxQX4fNfnPPn5k8wIGUsbsMK9VavaWrxSdVZgYCCHDh0iMjJSA/4iZ4zh0KFD1bqxqc7dofr2hrdZ+fNKXo2N4jXQyyGVqqTY2FiysrLIzs62uxRVCYGBgcTGxlb583Uu3H+X8jve2fAOiY07Aws03JWqJD8/P+Lj4+0uQ9WSOvfgsJSm1h1tBxzuO8U03JVS6hx1LtyD/IKIDY9lZ777+nYNd6WUOkedC3eANo3akJn7IzRooOGulFJlqJPh3rphazIPZ+rzZZRS6jzqZLi3adSGg3kHOd44QsNdKaXKUGfDHWBns0DIqfn7pZRSqq6pk+HeumFrAHZGO/TIXSmlylA3w72RFe6ZDVwa7kopVYY6Ge7hAeE0DmlMZnA+HDsGRUV2l6SUUheVOhnuYHXN7PQ7YY0cLu9FUUopVf/U2XBv06gNmcXuZ2Ts2GFvMUopdZGp0+Ge5TxCvi/w1lt2l6OUUheVOhvurRu2xgjsigBatrS7HKWUuqjU2XA/fa17ZptGsGePzdUopdTFpcJwF5GZInJQRDaX06afiKwXkS0i8oVnSyxbyY1MbSMhI6M2FqmUUnVGZY7c3wYGne+PIhIBvA4MN8YkAKM8U1r5GgU1okFAAzKbBsC2bfqibKWUKqXCcDfGrATKu9ZwHLDAGPOzu/1BD9VWLhGhTaM27GzggqNHYf/+2lisUkrVCZ7oc28HNBSRz0VkjYjc7IF5VkqbRm3IdBy1RrZtq63FKqXURc8T4e4LXAoMBa4CHheRdmU1FJGJIpIuIumeeI9j64at2V1wEKcP2u+ulFKleCLcs4Blxpg8Y0wOsBLoUlZDY8x0Y0yqMSY1Ojq62gtuEtIEp3GyLi5Ij9yVUqoUT4T7IqCPiPiKSDBwGVArh9HbD20H4I1+IXrkrpRSpVTmUsg5wLdAexHJEpHbReQuEbkLwBiTASwDNgLfAW8ZY8572aQn3X/Z/QC0CmmuR+5KKVWKb0UNjDFjK9HmReBFj1R0AdpFtSM6OJos8YOsLDh+HMLCarsMpZS66NTZO1RP6xDVgYygPGtk+3Z7i1FKqYtEnQ/3jlEd2eZyX+OuXTNKKQV4Qbh3iOpATsERckJ99KSqUkq5eUW4A2zroidVlVLqtDof7h2jOwKwrb0+QEwppU6r8+HeskFLAn0DyWjqB5mZ+j5VpZTCC8LdR3xoH9mebcF5VrCvWWN3SUopZbs6H+5gdc1sy8+yRqZNs7cYpZS6CHhFuHeI7MAux3FO+QLNm9tdjlJK2c4rwr1jdEcMhh0JMbB7t93lKKWU7bwi3E9fDpmRGANbt9pcjVJK2c8rwr1to7YIwraW7kf/ulx2l6SUUrbyinAP8gsivmE82xq6oKAAfvzR7pKUUspWXhHu4H6AmOOINaJdM0qpes57wj2yA9tP7qFYgC1b7C5HKaVs5TXhHhseS74rn3VJTfTIXSlV71XmTUwzReSgiJT7diUR6SYiLhEZ6bnyKm/nkZ0AvNbLV4/clVL1XmWO3N8GBpXXQEQcwN+A5R6oqUoe/s3DADQJb6pXzCil6r0Kw90YsxI4XEGze4H5wEFPFFUVrSJa0T6yPT80dEF+PuzaZVcpSillu2r3uYtIc+Ba4I3ql1M9yTHJrMf9Vibtd1dK1WOeOKE6GXjEGFNhP4iITBSRdBFJz87O9sCiz5Qck8zuU/vIDUT73ZVS9Zonwj0VmCsiu4GRwOsick1ZDY0x040xqcaY1OjoaA8s+kxdY7oCsCExWo/clVL1mm91Z2CMiT89LCJvA0uMMQurO9+qSI5JBmBdQiP6btJwV0rVXxWGu4jMAfoBUSKSBTwJ+AEYY2zvZy+tSWgTYkJjWN/cAfMyoLgYfLzmUn6llKq0CsPdGDO2sjMzxtxarWo8oGtMV9af3ACnTllvZerWze6SlFKq1nndYW1yTDJbXPspcABTpthdjlJK2cIrw90pxWyNcUBYmN3lKKWULbwy3AHW/yYeNmywuRqllLKH14V7m0ZtCPELYX2bMFi7FgoL7S5JKaVqndeFu4/40CWmC+sa5luPIdi40e6SlFKq1nlduAMkN0lmfZH72e6rVtldjlJK1TqvDPfWjVpzvOgEaxKjNNyVUvWSV4Z71rEsAF4aEKDhrpSql7wy3Cf1nkSwXzBERsLOnZCTY3dJSilVq7wy3BuHNGZI2yF867cfA7B6td0lKaVUrfLKcAcYEDeAPQUH2Rnlo10zSql6x3vDPX4AACt6NddwV0rVO14b7u0i29E0tCmfdQiA776znhCplFL1hNeGu4gwIH4AK4IPYI4dg6+/trskpZSqNV4b7gD94/pzwBwnIxp48UW7y1FKqVrj1eF+ut/9s86hkJdnczVKKVV7Kgx3EZkpIgdFZPN5/n6jiGx0/3wjIl08X2bVxDeMp1WDVnzWowl88431Ag+llKoHKnPk/jYwqJy/7wL6GmOSgGeB6R6oy2MGxA/g86ADFBfkw5df2l2OUkrVigrD3RizEjhczt+/McYccY+uAmI9VJtHdGvWjSOuE6xo6wvLltldjlJK1QpP97nfDnzs4XlWy/4T+wH43yFhsHy5zdUopVTt8Fi4i0h/rHB/pJw2E0UkXUTSs7OzPbXoct172b3ER8TzS0M/2LoV9uypleUqpZSdPBLuIpIEvAWMMMYcOl87Y8x0Y0yqMSY1OjraE4uuUFRwFPd2v5dt5iCZjdCjd6VUvVDtcBeRlsAC4CZjzA/VL8nzru14LQALu4drv7tSql6ozKWQc4BvgfYikiUit4vIXSJyl7vJE0Ak8LqIrBeR9Bqst0riIuJIjknmg66B8Mkn4HTaXZJSStUo34oaGGPGVvD3O4A7PFZRDbm2w7U8tf8p9rsMMR9/DMOG2V2SUkrVGK++Q7W0aztci8GwuD36KAKllNerN+Ge2DiRS8Lj+CA1BHbtAmPsLkkppWpMvQl3EeGqdoP5T9OT/HgiC9ats7skpZSqMfUm3MEK+GIMf74CmD/f7nKUUqrG1Ktwf7Lvk4QHhJPbtBEsWGB3OUopVWPqVbg3DmnMhOQJrIg8xrFd2yAjw+6SlFKqRtSrcAcY1WkUBTj5sB169K6U8lr1Ltx7tuhJ87DmvNe7oYa7Uspr1btw9xEfRnYaybImxzi2ZS2sWWN3SUop5XH1LtzhdNeMy+qaefppu8tRSimPq5fh3rNFT5qHNOW9lAA4dN6HWCqlVJ1VL8PdR3wY2n4YS+IK2ZXxjQa8Usrr1MtwB/D18cUlhsf6A0uW2F2OUkp5VL0N9yf6PkGwXzD5oYGwaJHd5SillEfV23BvEtqEGxJu4LM4Q9F/l8GpU3aXpJRSHlNvwx1geLvh5PoU8HXUKeslHkop5SUq8yammSJyUEQ2n+fvIiJTRCRTRDaKSIrny6wZv239W/wd/izu7A8LF9pdjlJKeUxljtzfBgaV8/fBQFv3z0RgWvXLqh2h/qEMjB/I4s7+mA8Xg8tld0lKKeURFYa7MWYlcLicJiOAWcayCogQkaaeKrCmDWs3jJ3+J9hmcuDeeyEnx+6SlFKq2jzR594c2FNqPMs9rU4Y1t56l+riDgLTpkFams0VKaVU9Xki3KWMaWW+w05EJopIuoikZ2dne2DR1RcbHktK0xQ+TA2D6GiYMMHukpRSqto8Ee5ZQItS47HA3rIaGmOmG2NSjTGp0dHRHli0ZwxrN4yvI47xZEI2Oft/tLscpZSqNk+E+2LgZvdVMz2Ao8aYfR6Yb60Z0X4EAM/0g7Qlz9pai1JKeYJvRQ1EZA7QD4gSkSzgScAPwBjzBrAUGAJkAieBOtevkRyTTMvwlgRl7WfCpsMwye6KlFKqeioMd2PM2Ar+boB7PFaRDUSE0YmjeeXo33F8swqOHIGGDe0uSymlqqxe36Fa2qhOo3BKMYvbFMPHH9tdjlJKVYuGu1tqs1RaNmjJ+1394cMP7S5HKaWqRcPdTUQY2XEk/2nl5OinS6GoyO6SlFKqyjTcSxnZaSSFUsySmGNwzz16t6pSqs7ScC/lstjLaB7ajPcTfeDNN/VuVaVUnaXhXoqP+HB9p5EsbWt4rr+DnGuutLskpZSqEg33s4xKGEWhj+Hxvi7SPnnR7nKUUqpKNNzP0qtFLzpGdaRpng+3/ufieP6NUkpdKA33s4gID/V8iH0hxWzZ9ImeVFVK1Uka7mUY13kcUf4NmdytGObNs7scpZS6YBruZQjyC+LO7nezuD38OP8tu8tRSqkLpuF+Hr/vfg8O8eE1v3XwyCPaPaOUqlM03M+jWVgzbmg9nDdT4NlVL5Az8x92l6SUUpWm4V6OPwz4M3kB8MQASIs7Ync5SilVaRru5Uhtlsqo+KtxuKDPCn1Dk1Kq7tBwr8Br171FqPjzRN4SzI4ddpejlFKVUqlwF5FBIrJdRDJF5Jz3FIlISxFZISLrRGSjiAzxfKn2aBLahOf6PMl/W8P7r9xhdzlKKVUpFYa7iDiAqcBgoBMwVkQ6ndXsMWCeMaYrMAZ43dOF2unuAY/Q1dWY+4JX8tyfepOzZ7vdJSmlVLkqc+TeHcg0xvxojCkE5gIjzmpjgHD3cANgr+dKtJ/Dx8Hro95mfyg8Hvg1abMetLskpZQqV2XCvTmwp9R4lntaaU8B490v0F4K3FvWjERkooiki0h6dnbdem5Lj86DuT+oHwDRJsjeYpRSqgKVCXcpY5o5a3ws8LYxJhYYArwrIufM2xgz3RiTaoxJjY6OvvBqbfbCQ8tIPRHOQyfm88vW1XaXo5RS51WZcM8CWpQaj+XcbpfbgXkAxphvgUAgyhMFXkz8fQOYffMi8n3hljeHUFzssrskpZQqU2XC/XugrYjEi4g/1gnTxWe1+RkYCCAiHbHCvW71u1RSu879eLXhOD6NOMzVDzfXk6tKqYtSheFujHEC/wMsBzKwrorZIiLPiMhwd7OHgN+JyAZgDnCrMebsrhuvcfsDs+hy2J+Pww/wxBuj7S5HKaXOIXZlcGpqqklPT7dl2Z6wa/Vyui0YhC8ONj2yi+hGLSr+kFJKVZOIrDHGpFbUTu9QraL4y67ik25TyPV3cdPfe1Nsiu0uSSmlSmi4V0PyyHt5tXAAy/1/ZsikFtr/rpS6aGi4V9PEpz8iOceX5cF7+fP0MXaXo5RSgIZ7tUlgIB9c/z7RebDAtYl9R362uySllNJw94S4fiNYkfgiJ8XFmFd64Sx22l2SUqqe03D3kISb/8gbJ/uz0pHFgD/Hav+7UspWGu4edNNfP+I3+/34MvAA90y/xu5ylFL1mIa7JwUF8f6Ni2l9RPhAtrFq5xd2V6SUqqc03D2sae9BrO4/m9hjMOLtQTz26Z/JOZljd1lKqXpGw70GRI4Yy+LwOzlKPn/56q+8tuJ5u0tSStUzGu41JPHJqbz3bSy+LljwzQxy83PtLkkpVY9ouNcUh4NhM75iyX8i2e7I5bdv9OK5lc9pF41SqlZouNekVq24auFm/u/zKNYc2crjKx5n8md/tbsqpVQ9oOFe02JiuHba58xZ5EuAE976/p+s2bvG7qqUUl5Ow702JCQw+uXlpM8KJPBEPr1n9OKG924gO88r32eilLoIaLjXlgEDSPzoe1YvaESzI0W8t/U9+s7ozf4T++2uTCnlhSoV7iIySES2i0imiEw6T5sbRGSriGwRkX97tkwvkZhIk2Vf8e2cEK7eBjsP7aDT1E6MmjeKAycO2F2dUsqLVBjuIuIApgKDgU7AWBHpdFabtsCfgF7GmATggRqo1Tu0b0/jz7/nwzVtWf+6Ifx4Ie9nvE+X15NYnrnc7uqUUl6iMkfu3YFMY8yPxphCYC4w4qw2vwOmGmOOABhjDnq2TC/Tvj1kZNDxrsf4/pU8xm+AwON5DJo9iLjJcczeNFvf7KSUqpbKhHtzYE+p8Sz3tNLaAe1E5GsRWSUig8qakYhMFJF0EUnPzq7nJxMdDnj2WaL/vZB3v4xi+//mMeJoM346+hPjF4wn4fUERs4bye7c3XZXqpSqgyoT7lLGtLPfqu0LtAX6AWOBt0Qk4pwPGTPdGJNqjEmNjo6+0Fq904gRsGcPAY8+zlvT9/G//4XXf2hL7skjzM+YT4d/dOC2hbdx/8f369U1SqlK861EmyygRanxWGBvGW1WGWOKgF0ish0r7L/3SJXeLjAQnnmGqKuvZtKdd8K/1zPykxCeuiaW7O4JzNo4C5dxMWfzHG7vejsu4+L/9fp/RAVH2V25UuoiJcacfRB+VgMRX+AHYCDwC1ZgjzPGbCnVZhAw1hhzi4hEAeuAZGPMofPNNzU11aSnp3tgFbzQhg0wbhxs3Qq+vmTeOpw/NdnE3tgIvjlg7S+bhDThpqSbKHQV8ufL/0zjkMY2F62Uqg0issYYk1phu4rC3T2zIcBkwAHMNMb8RUSeAdKNMYtFRIC/A4MAF/AXY8zc8uap4V6BnBx4/nnYvx/mzgWXC1q3ZuMTd/KEYyXZ+Yf5NutbDIYw/zCGtRtXIKZVAAAScklEQVSGiPDE5U/QLqqd3dUrpWqIR8O9Jmi4X4CtW+GPf4TNm2HPHggKgjFj2HbjVTx15AMKnAUs27mMfGc+AF2adCEiMII7L72Toe2GEh4QbvMKKKU8RcPdG7lccOedMGMG+PqC0wmNGsH48ewfczXPHVlEqF8o87bOY1fuLgAEoVlYMwa3GUz35t3ZnbubB3o8QHSIntBWqi7ScPdWOTmQlgbXXw+TJsF774EIGAMNG8KoUeRcP5h/+m+iU5NEpn4/lU93fYqvjy/OYicA4QHh9GnZh2JTzB1d72DAJQOICDzn4ial1EVIw70+OB30w4fDY4/B++9b18+7XBAQAL/9LTkDepB25DPG3/Yqm137mPr9VHx9fFmxewWHTx0umVWjoEb0btGbpCZJ7Duxjz/0+AMJjRNsXDmlVFk03Oub00E/ahQ8+STMmgXh4XDsmPX36Gi4+mqrK+fRR8lpGcXU76aSEJ3AjHUzWLZzGY2CGp0R+FHBUUQERDAwfiAdozuSeTiTh3o+RFzDOHvWUSml4V6vnQ76W2+1Tsa+9BLk58MXX0BRkdWmfXurG+eWW8jp1520Q58woett5J7K5e/f/p3Y8FgWbl9I+t70M7p0AJqFNiPEP4R+rfrRJrINmYcz+X2335PUJAkf0QeNKlWTNNzVufbuhb/+1Qr1efPghx9+/VtYGPTpA6mpcPAgPPIIOY1DSVuXxs1dbmZ37m6mpU+jeVhzlmYuZf3+9fj5+FFUXFQyiwBHAA0CG9A1pivxEfHsO7GP0Qmj6dq0K60atCLIL8iGlVbKu2i4q/Ll5FhX3Vx2Gbz6KixcaF15c/jXbhnat4fQUOvkbd++0KULhISQczKHtHVp3Jp8K/tO7GPqd1NpF9mOxdsXs/LnlTQJacKhU4fOONoHiA6OpkV4C5zGSZ+WfYgNj2VbzjZuSrqJxMaJRIdE65G/UhXQcFeVd7obZ8IEa/jFFyEyEj78ELZtO7Nt69bQubPVzXPTTXD55dC8OYiUhP6ErhMwxjBl9RR6tujJuxveZe6WuXRv1p3Dpw6TeSQThzhwGdcZs/YRH5qGNqVpaFPyXfn0aN6DJiFN+OnoT9x32X2kNkvFul9OqfpLw11VX04OzJwJgwfD3/8O77wDCQnwyy+Qm/tru8BASEqCDh2s6TffDD16QLNm54Q+QNq6NG5JvoXsvGymr5nOpc0uZUHGAhZtX0RKTApHC46y88hOAh2B5LvySxYT5h9GqH8oLRu0JNQ/lOy8bBIaJxDqF8rOIzvpGduTEP8Q1u1fx5WtryTEL4SVP61kZKeRxIbHEuIfQohfCMF+wfg7/HVHoeokDXflWaWP7gGmTYOuXa2unYULIT7eelTCqVO/fiYgADp2hLZtrat2Ro2y+vTbtIGQkDNnX8YOYELXCeQV5vHKqldo1aAV87bOY1XWKlo3bE2Bq4CsY1mEB4RT4CygwFVwQavjIz4E+wUT4hdCUXERMaExBPsGk5ufS+tGrQn2CybrWBZJjZMI8gti+6Ht/Cb2NwT5BbFm3xquuOQKgv2C+WL3F1zT4RqahTUjyC+IIN8ggv2CS4YdPo5q/bMrdTYNd1U7Soe+MfDaa1bov/MOLFpkBfnBg79eknlakybWDsHphCuugLg42LQJ7r4bOnWybsw6e1Hn2QEAzFw3k3Gdx3Gi4ATvbHyHoW2GkleUx/yM+Vze6nKW7ljKnM1zGN5uOIWuQpbtXEbvFr0pdBXy3d7v6BTdiSJXETsO76BZWDMKnYXknMohyDeIQlfhOV1IleUrvjh8HIQFhBHoCKTAVUCT0CYEOgI5kn+E+IbxBDoC+eX4LyREJxDgCGDH4R10a96NAEcAGw9spE+rPgQ4Alj9y2qubH0lAY4APt/9OcPbDycmNIZA30ACfAMI9A20hh3WsK+Pr3478UIa7speZx/pv/EG9OplHenPnm0dwefkwO7d5342MNC6eicxEVq0sE7y3nGHdVI3omp30pa3Y6jM8Iy1MxidOJpTRaeYtXEWw9oO45TzFPMz5tM/rj8f7fiIdza8w6hOoyhyFbFw+0KuvORKilxFrPhpBZc1v4yi4iLW7ltLx6iOFBUXkXk4k+ZhzSkqLuJg3kHCA8IpchVxynmqjDWomkBHIP6+/hQXF9MgsAEBjgDyivKICY3B3+HP4VOHaRXRCj8fP/Yd30e7qHb4+/jzY+6PJDVOwt/hT0ZOBt2aWTubdQfW0adlH/x8/FiVtYorLrkCf4c/n+/+nKFthxIdEo2/w/+MnwDfAPx8/M6Y5ufwO+Pk+dH8o+w5toftOdv5z87/MLz9cBqHNCbAN6BkZ1V6B+bv8K+3J9813NXF6ezQT0uDMWPg55+t4G/XDpYuhS+/tE7UHjkCJ0/++vmQEGt6ZCQcPWqdAwgLs3YSvXpZw2vXwtChVrvwcGtaWJg1HBxc5reCaq9WNXceZw/fmnwrha5CZqybwciOIylwFfDvTf9maNuhFBYX8kHGBwyIH1CyUxmdMBpnsZP5GfMZ2nYozmIny3cu5/KWl+MyLr7e8zWXNr0UZ7GTDQc20CGqA06Xk8wjmdYVTMVO9p3YR6OgRjiLnRwrOEaAIwBnsbPK31oq4oMPAb4BAFXaofmKLwG+ARgMDQKsHdcp5ymahDbB38efI/lHiIuIw9/hz97je+kQ1YEARwA7j+ykS5MuBPgGsOXgFnrE9sDf4c+afWvo26ovAY4Avs76mqtaX4W/jz+f7vqUER1G0Dik8Tk7rjN2WD5+tfJtScNd1V1n7wBee80K8bQ0WLbMOqI/eRJ+/NG6Zj8//8y+/vKIWDsIHx9o3NgaPnHi1/MAv/wCKSnWJaBbt0L//tb0b7+1HvPQvLk1Hhr66++goBrZYVSGp3cq5+vyuinpJgpcBby9/m3r20lxEXM2zWFYu2EUFheyIGMBV7W+ig9/+JA3177JLV1uwVnsZPam2YzqNApXsYsF2xYwtO1QXMUulu1cxoD4AVza9FJWZa3iy5+/ZFziOEYnjubjHR8zMH4gSzOXkrY+jbGJY3EWO3lv63tc3fZqnMVOlu1cRv9W/SkqLuKrPV+R2iwVp8vJ+gPrz9hxxYbH4nQ52Z+3n4jACJwuJyeKTuDr44ur2IU556Vy1ecQBwGOAPwcfjiLnYT5h+Hv8Oek8ySRQZH4O/y5pOElvDX8rSq9cEfDXXmfso76Tw/PnAljx1p9+7NmwZVXWjdqTZ9ufTMoKIAPPoCBA63hr76yrvApKIDt26FpU2v48GHw8/v1Tt7KOv2NIDLSGs7Lg1atrOEDB6wdUlAQ7NwJ3btb09evt+oJDrbqGT7cusIoJMSaFhxsfSY42HpmUB1woTubqOCoMz5TOuxqY8c1Y+0MxiSOodBVyKwNs7iu43UUuAp4b8t7DGk7hAJXAYu3L7Z2NjuW8vaGtxmXOA5nsZN5W+dxTYdrcLqcLNmxhKtaX4Wz2Mmnuz6lT8s+uIpdfJP1Dd2adcNZ7GTd/nUkNk7EVewiIyeDF654gYd7PXzB/8Ya7kqVtzOoaNjlsnYM115rBfWcOdaJ34ULre6jG26AwkJr/IorrB3Dl19aJ5MLC2HLFivci4qsO4PDw63p+fnn1lkZfn5WwDdoYAV+fr61QwoKgkOHrG8egYHW8/67dLGGt22Dnj2t4bVrrR1JYKBV5/DhEBNjjQcF/fr79LBvZd7AWb/UxM7G9iN392v0XsV6E9Nbxpjnz9NuJPAe0M0YU25ya7irOqk6O4zTw+PHW91Ip5/oeeqU9S1j4EDrxrG0tDN3Hr/9rbWT+Pxz66i/qAjWrbPOTxQVwa5d1tVHhYXWOYrAQGu6qxp95T4+vwa902l9IwkMhOPHITbWusz14EHrMteAAPjpJ+vmtoAAa6dy2WXW8Lp11onwwED4+msYNMjqDgsIsKad3pEcO2bdI5GVZX2LGTbM+hYTGnrmeRM/v6qvk5fwWLiLiAPrHaq/xXoR9vdY70vdela7MOAjwB/4Hw13parAEzuP08MzZsDo0dZR/qxZv+5I5s+3vm18+KHVnTVmjBXg778PQ4ZYw//5j/WsoaIiWLUKkpOt4S1b4JJLrDY//2w9bdTp/HWn4nRaPzXFz88KfLB2aKGh1on1tm2t4T174NJLreEtW6xzJsHB1jqMGPHrDqP0ORN/f9vOmVSFJ8O9J/CUMeYq9/ifAIwx/3tWu8nAJ8AfgT9quCt1kfPkjuTs8x/jx1tdVWlpcN111vD//Z+181iwwLoJ7rbbrKuaFiywLo+95x5rZzR/vvUtZtEiawc1apT1+cWLrW8BBQVWWCckWMOZmVbQFxRYR/+n32lQWT4+VtCLWDur4GDrG0p8vDW8d6/1rSQkxHrYXs+e1readeusnWRwMKxcCddcY51wP32+5PQ5k8BAj+48KhvuGGPK/QFGYnXFnB6/CfjHWW26AvPdw58DqeeZ10QgHUhv2bKlUUrVQ9nZxrzwgvW7rPGy2l3o8N69xjz9tDEbNhjzzTfG3HefMQsXGjNhgjFgzMiRxgwfbg33729Mr17WcJcuxnTsaA23aGFMkybWcEiIMX5+1nBVfgIDjQkONqZZM2Natzbm6qvPXd9KAtJNBbltjKnUkfso4CpjzB3u8ZuA7saYe93jPsBnwK3GmN0i8jl65K6UuhhV99vKzJkwbpx1Ke4771jnBvLyrG8b/fvDkiVWF9jIkdY5kMWLraP7wkLr6D411ere2rABXngBHrbxapmKumVEpAGwEzjh/kgMcBgYXl7Aa7grpbzOhe48omy8WkZEfLFOqA4EfsE6oTrOGLPlPO0/R4/clVKqRlQ23Ct8OIMxxgn8D7AcyADmGWO2iMgzIjK8+qUqpZTytErdqWCMWQosPWvaE+dp26/6ZSmllKqO+vlYNaWU8nIa7kop5YU03JVSygtpuCullBfScFdKKS9k2yN/RSQb+KmKH48CcjxYTl1Q39ZZ19e71bf1Bc+tcytjTHRFjWwL9+oQkfTKXMTvTerbOuv6erf6tr5Q++us3TJKKeWFNNyVUsoL1dVwn253ATaob+us6+vd6tv6Qi2vc53sc1dKKVW+unrkrpRSqhx1LtxFZJCIbBeRTBGZZHc9niYiLURkhYhkiMgWEbnfPb2RiPxXRHa4fze0u1ZPEhGHiKwTkSXu8XgRWe1e3/8TEX+7a/QkEYkQkfdFZJt7W/f05m0sIn9w//e8WUTmiEigN21jEZkpIgdFZHOpaWVuT7FMcWfYRhFJqYma6lS4u1/WPRUYDHQCxopIJ3ur8jgn8JAxpiPQA7jHvY6TgE+NMW2BT93j3uR+rEdKn/Y34BX3+h4BbrelqprzKrDMGNMB6IK17l65jUWkOXAf1us3EwEHMAbv2sZvA4POmna+7TkYaOv+mQhMq4mC6lS4A92BTGPMj8aYQmAuMMLmmjzKGLPPGLPWPXwc63/65ljr+Y672TvANfZU6HkiEgsMBd5yjwswAHjf3cTb1jccuByYAWCMKTTG5OLF2xjr8eJB7pf/BAP78KJtbIxZifUGutLOtz1HALPcr0RdBUSISFNP11TXwr05sKfUeJZ7mlcSkTisl4+vBpoYY/aBtQMAGttXmcdNBv4fUOwejwRy3S+KAe/bzpcA2UCauyvqLREJwUu3sTHmF+Al4GesUD8KrMG7tzGcf3vWSo7VtXCXMqZ55eU+IhIKzAceMMYcs7uemiIiVwMHjTFrSk8uo6k3bWdfIAWYZozpCuThJV0wZXH3NY8A4oFmQAhW18TZvGkbl6dW/vuua+GeBbQoNR4L7LWplhojIn5YwT7bGLPAPfnA6a9u7t8H7arPw3oBw0VkN1Y32wCsI/kI91d48L7tnAVkGWNWu8ffxwp7b93GVwC7jDHZxpgiYAHwG7x7G8P5t2et5FhdC/fvgbbus+z+WCdlFttck0e5+5tnABnGmJdL/WkxcIt7+BZgUW3XVhOMMX8yxsQaY+KwtudnxpgbgRXASHczr1lfAGPMfmCPiLR3TxoIbMVLtzFWd0wPEQl2//d9en29dhu7nW97LgZudl810wM4err7xqOMMXXqBxgC/ADsBP5sdz01sH69sb6ibQTWu3+GYPVDfwrscP9uZHetNbDu/YAl7uFLgO+ATOA9IMDu+jy8rslAuns7LwQaevM2Bp4GtgGbgXeBAG/axsAcrPMJRVhH5refb3tidctMdWfYJqyriDxek96hqpRSXqiudcsopZSqBA13pZTyQhruSinlhTTclVLKC2m4K6WUF9JwV0opL6ThrpRSXkjDXSmlvND/Bz+jj7G7FOJ0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.01,epochs=20,bs=512,beta1=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reach 83% in only ~100 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8395"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accuracy on test set\n",
    "y_pred=nn.predict(X_test)\n",
    "accuracy(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimizer 3: Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "act_obj = ReLU()\n",
    "layers = [X_train.shape[1],100,10]\n",
    "nn = CustomNeuralNetwork(layers,act_obj,opt=Adam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 1.0379637326109254, Val loss:0.48682836357196857\n",
      "Epoch 2. Training loss: 0.4238603192463661, Val loss:0.4227314636532378\n",
      "Epoch 3. Training loss: 0.37565871184511457, Val loss:0.4060830455430861\n",
      "Epoch 4. Training loss: 0.35011122069444095, Val loss:0.39966370521574307\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl8VNX9//HXJyEhQAIBEhYJClJcIAaIKeIDK6LWuvwEF1RwK7QV9wVEQeuKWhFxAUtr0UpttVCrX5WvVantF+GrXxWChbCJ7BoRCGENe5Lz++OGmGWSmSST3Mzk/Xw85sFM5sydz3Xa9z1z5txzzTmHiIhElxi/CxARkfBTuIuIRCGFu4hIFFK4i4hEIYW7iEgUUriLiEQhhbuISBRSuIuIRCGFu4hIFGrm1xunpKS4bt26+fX2IiIRafHixdudc6nB2vkW7t26dSM7O9uvtxcRiUhmtimUdhqWERGJQgp3EZEopHAXEYlCvo25i0jDOnLkCLm5uRw8eNDvUiQECQkJpKWlERcXV6vXK9xFmojc3FySkpLo1q0bZuZ3OVIN5xz5+fnk5ubSvXv3Wm1DwzIiTcTBgwdp3769gj0CmBnt27ev07cshbtIE6Jgjxx1/awiL9znz4ezz4a8PL8rERFptCIv3J99FubNg3vu8bsSEamB/Px8+vbtS9++fenUqRNdunQpfXz48OGQtjFq1ChWr15dbZvp06fz+uuvh6NkzjjjDJYsWRKWbTW0yPtB9cUX4fPPvR78oUPQvLnfFYlICNq3b18alI888giJiYmMGzeuXBvnHM45YmIC9ztnzpwZ9H1uvfXWuhcbBSKv5965M/z5z7BxI0yb5nc1IlJHa9euJT09nZtuuonMzEy+//57Ro8eTVZWFr1792bixImlbY/2pAsLC0lOTmbChAn06dOH008/nW3btgHwwAMP8Pzzz5e2nzBhAv379+fEE0/k//7v/wDYt28fl19+OX369GHEiBFkZWUF7aG/9tprnHLKKaSnp3P//fcDUFhYyHXXXVf692klmfTcc8/Rq1cv+vTpw7XXXhv2/2ahiLyeO8DPfgYXXQSPPQbXXw8dO/pdkUhkuesuCGW44cgR2LIFOnWCYPOt+/aFklCtqZUrVzJz5kxefPFFACZNmkS7du0oLCxk8ODBDBs2jF69epV7ze7duxk0aBCTJk1i7NixvPLKK0yYMKHStp1zLFy4kDlz5jBx4kQ+/PBDXnjhBTp16sRbb73F0qVLyczMrLa+3NxcHnjgAbKzs2nTpg3nnnsu7733HqmpqWzfvp1ly5YBsGvXLgAmT57Mpk2biI+PL/1bQ4u8nvtRzzwDBw7AAw/4XYlI9NqyBdav9/6tRz169ODHP/5x6eNZs2aRmZlJZmYmq1atYuXKlZVe06JFCy644AIATj31VDZu3Bhw25dddlmlNp988gnDhw8HoE+fPvTu3bva+r744gvOPvtsUlJSiIuL4+qrr2bBggX86Ec/YvXq1dx5553MnTuXNm3aANC7d2+uvfZaXn/99VqfhFRXkdlzBzjxRLj9dq+ncMst0K+f3xWJRI5Qe9jbt8PMmTBqFKSk1Fs5rVq1Kr2/Zs0apk6dysKFC0lOTubaa68NON87Pj6+9H5sbCyFhYUBt9285He5sm2cczWqr6r27du3Jycnhw8++IBp06bx1ltvMWPGDObOncv8+fN59913efzxx1m+fDmxsbE1es+6ityeO8BDD0H79t5XzBp+WCISgpQUb2ZaPQZ7RXv27CEpKYnWrVvz/fffM3fu3LC/xxlnnMEbb7wBwLJlywJ+MyhrwIABzJs3j/z8fAoLC5k9ezaDBg0iLy8P5xxXXHEFjz76KF9++SVFRUXk5uZy9tln8/TTT5OXl8f+/fvDvg/BBO25m9krwP8Dtjnn0gM8b8BU4EJgPzDSOfdluAsNKDnZG3e/+WZ480244ooGeVsRqT+ZmZn06tWL9PR0jj/+eAYOHBj297j99tu5/vrrycjIIDMzk/T09NIhlUDS0tKYOHEiZ511Fs45Lr74Yi666CK+/PJLfvnLX+Kcw8x46qmnKCws5Oqrr2bv3r0UFxczfvx4kpKSwr4PwViwrydmdiZQAPy5inC/ELgdL9xPA6Y6504L9sZZWVkuLBfrKCqCzEzYvRtWrYIWLeq+TZEotGrVKk4++WS/y2gUCgsLKSwsJCEhgTVr1nDeeeexZs0amjVrXCPVgT4zM1vsnMsK9tqgwzLOuQXAjmqaDMULfuec+xxINrPOwbYbNrGx3vjhpk3eCU4iIkEUFBQwcOBA+vTpw+WXX84f/vCHRhfsdRWOvekCfFvmcW7J374Pw7ZDM3gwXHopPPmk98PPMcc02FuLSORJTk5m8eLFfpdRr8Lxg2qg1W0CjvWY2Wgzyzaz7Lxwrw0zZYo3J/e++8K7XRGRCBSOcM8FupZ5nAZsDtTQOTfDOZflnMtKTQ168e6aOf54GDPGO3t14cLwbltEJMKEI9znANebZwCw2znXcEMyZf36197ZqnfeqamRItKkBQ13M5sFfAacaGa5ZvZLM7vJzG4qafI+sB5YC7wE3FJv1QaTlOSNu3/+Ofz1r76VISLit1Bmy4xwznV2zsU559Kcc390zr3onHux5HnnnLvVOdfDOXeKcy4M8xvr4Oc/h1NPhfHjYd8+X0sRkR+cddZZlU5Iev7557nllur7g4mJiQBs3ryZYcOGVbntYFOrn3/++XInE1144YVhWfflkUceYcqUKXXeTrhF9hmqgcTEeFMjv/sOJk/2uxoRKTFixAhmz55d7m+zZ89mxIgRIb3+mGOO4c0336z1+1cM9/fff5/k5ORab6+xi75wBzjjDLjqKi/cv/nG72pEBBg2bBjvvfcehw4dAmDjxo1s3ryZM844g4KCAs455xwyMzM55ZRTePfddyu9fuPGjaSne+dRHjhwgOHDh5ORkcFVV13FgQMHStvdfPPNpcsFP/zwwwBMmzaNzZs3M3jwYAYPHgxAt27d2L59OwDPPvss6enppKenly4XvHHjRk4++WRuuOEGevfuzXnnnVfufQJZsmQJAwYMICMjg0svvZSdO3eWvn+vXr3IyMgoXbBs/vz5pRcr6devH3v37q31f9tAomvWflmTJ8O773rDM7Nm+V2NSKNy14d3sWRL8CV/jxQdYUvBFjoldiIutvrVDft26svz51e9IFn79u3p378/H374IUOHDmX27NlcddVVmBkJCQm8/fbbtG7dmu3btzNgwACGDBlS5XVEf//739OyZUtycnLIyckpt2TvE088Qbt27SgqKuKcc84hJyeHO+64g2effZZ58+aRUmGdnMWLFzNz5ky++OILnHOcdtppDBo0iLZt27JmzRpmzZrFSy+9xJVXXslbb71V7frs119/PS+88AKDBg3ioYce4tFHH+X5559n0qRJbNiwgebNm5cOBU2ZMoXp06czcOBACgoKSEhIqPa/b01FZ88d4NhjvQWPZs+GTz7xuxqRiLSlYAvrd61nS0F4lvwtOzRTdkjGOcf9999PRkYG5557Lt999x1bt26tcjsLFiwoDdmMjAwyMjJKn3vjjTfIzMykX79+rFixIuiiYJ988gmXXnoprVq1IjExkcsuu4z//d//BaB79+707dsXqH5ZYfDWl9+1axeDBg0C4Oc//zkLFiworfGaa67htddeKz0TduDAgYwdO5Zp06axa9eusJ8hG709d/B67a+84q0auXChNx4vItX2sMvavn87M/8zk1H9RpHSsu4rQ15yySWMHTuWL7/8kgMHDpT2uF9//XXy8vJYvHgxcXFxdOvWLeAyv2UF6tVv2LCBKVOmsGjRItq2bcvIkSODbqe69bWal7mMZ2xsbNBhmar84x//YMGCBcyZM4fHHnuMFStWMGHCBC666CLef/99BgwYwL/+9S9OOumkWm0/kOhOu1at4KmnYPFiePVVv6sRiTgpLVO4Z+A9YQl28Ga+nHXWWfziF78o90Pq7t276dChA3FxccybN49NmzZVu50zzzyz9CLYy5cvJycnB/CWC27VqhVt2rRh69atfPDBB6WvSUpKCjiufeaZZ/LOO++wf/9+9u3bx9tvv81PfvKTGu9bmzZtaNu2bWmv/y9/+QuDBg2iuLiYb7/9lsGDBzN58mR27dpFQUEB69at45RTTmH8+PFkZWXx1Vdf1fg9qxPdPXeAq6+G3/4W7r8fhg3z5sKLiG9GjBjBZZddVm7mzDXXXMPFF19MVlYWffv2DdqDvfnmmxk1ahQZGRn07duX/v37A95Vlfr160fv3r0rLRc8evRoLrjgAjp37sy8efNK/56ZmcnIkSNLt/GrX/2Kfv36VTsEU5VXX32Vm266if3793P88cczc+ZMioqKuPbaa9m9ezfOOcaMGUNycjIPPvgg8+bNIzY2ll69epVeVSpcgi75W1/CtuRvKBYuhNNOgwkTvJOcRJogLfkbeep1yd+o0L8/XHedtyTw+vV+VyMiUu+aRriD12OPi4Nx4/yuRESk3jWdcO/SxVsO+O23ocx4m0hT4tcwrNRcXT+rphPuAGPHwnHHeVMji4r8rkakQSUkJJCfn6+AjwDOOfLz8+t0YlP0z5Ypq0ULePppuPJKePlluPFGvysSaTBpaWnk5uYS9gvlSL1ISEggLS2t1q9vGrNlynIOBg3yLqa9Zg1E8cJBIhJ9NFumKmYwdSrk58Njj/ldjYhIvWh64Q7Qrx/88pcwbRqsXu13NSIiYdc0wx3g8ce9Mfi77/a7EhGRsGu64d6xIzz4IPzjH1Dh6jAiIpGu6YY7wB13QI8eMGYMHDnidzUiImHTtMO9eXN45hlv5syLL/pdjYhI2IQU7mZ2vpmtNrO1ZjYhwPPHmdm/zSzHzD42s9pPzmxoQ4bAOefAww97M2hERKJA0HA3s1hgOnAB0AsYYWa9KjSbAvzZOZcBTAQiZ+lFM++C2rt3ewEvIhIFQum59wfWOufWO+cOA7OBoRXa9AL+XXJ/XoDnG7f0dLjpJm9oZsUKv6sREamzUMK9C/Btmce5JX8raylwecn9S4EkM2tf9/Ia0KOPehfyGDPGO4tVRCSChRLugS4/XjH9xgGDzOw/wCDgO6Cw0obMRptZtpllN7r1LVJS4JFH4KOP4L33/K5GRKROgq4tY2anA484535W8vg+AOdcwHF1M0sEvnLOVfujqm9ry1TnyBHIyPBWjFy+HOLj/a5IRKSccK4tswjoaWbdzSweGA7MqfBmKWZ2dFv3Aa/UtOBGIS7Ou1rTmjXwwgt+VyMiUmtBw905VwjcBswFVgFvOOdWmNlEMxtS0uwsYLWZfQ10BJ6op3rr3wUXwIUXwsSJsG2b39WIiNRK01vyNxSrV3szaEaNghkz/K5GRKSUlvytixNPhNtu8y7osWSJ39WIiNSYwr0qDz0E7dp5l+TT1EgRiTAK96q0betdzGP+fPiv//K7GhGRGlG4V+eGG7yx93Hj4OBBv6sREQmZwr06zZp5685s3OhNkRQRiRAK92DOOQcuuQR+8xvYvNnvakREQqJwD8WUKd7Zq/ff73clIiIhUbiHokcPb9bMq6/CokV+VyMiEpTCPVS//rV33dU779TUSBFp9BTuoWrdGp54Aj77DGbN8rsaEZFqKdxrYuRIyMyE8eNh3z6/qxERqZLCvSZiY72pkbm58PTTflcjIlIlhXtN/eQncOWVMHkyfPtt8PYiIj5QuNfG5Mnej6rjx/tdiYhIQAr32jjuOG9Jglmz4NNP/a5GRKQShXttjR8PxxzjTY0sLva7GhGRchTutZWYCE89BYsXw5//7Hc1IiLlKNzr4uqr4bTT4L77YO9ev6sRESmlcK+LmBiYOhW2bIEnn/S7GhGRUgr3ujrtNLj2Wm9J4A0b/K5GRAQIMdzN7HwzW21ma81sQoDnjzWzeWb2HzPLMbMLw19qIzZpkneC0z33+F2JiAgQQribWSwwHbgA6AWMMLNeFZo9ALzhnOsHDAd+F+5CG7UuXWDCBHjrLfj4Y7+rEREJqefeH1jrnFvvnDsMzAaGVmjjgNYl99sATe+qFuPGwbHHeksDFxX5XY2INHGhhHsXoOx59rklfyvrEeBaM8sF3gduD0t1kaRFC2+9maVL4Y9/9LsaEWniQgl3C/C3iguajwD+5JxLAy4E/mJmlbZtZqPNLNvMsvPy8mpebWN3xRXe2jMPPAC7d/tdjYg0YaGEey7QtczjNCoPu/wSeAPAOfcZkACkVNyQc26Gcy7LOZeVmppau4obMzNv1cjt2+Gxx/yuRkSasFDCfRHQ08y6m1k83g+mcyq0+QY4B8DMTsYL9yjsmocgMxNGjYJp0+Drr/2uRkSaqKDh7pwrBG4D5gKr8GbFrDCziWY2pKTZ3cANZrYUmAWMdK4JX4vuiScgIQHuvtvvSkSkiTK/MjgrK8tlZ2f78t4NYvJkb3GxuXPhvPP8rkZEooSZLXbOZQVrpzNU68udd0KPHjBmDBQW+l2NiDQxCvf60rw5TJkCK1fCiy/6XY2INDEK9/o0dCicfTY8/DDs2OF3NSLShCjc69PRqZG7dsEjj/hdjYg0IQr3+nbKKXDjjfC738GKFX5XIyJNhMK9IUycCElJMHasd2FtEZF6pnBvCCkp3rj7P/8J//iH39WISBOgcG8ot94KJ57o9d4PH/a7GhGJcgr3hhIX512tac0a+O1v/a5GRKKcwr0hXXghnH++Nwa/bZvf1YhIFFO4N7Rnn4V9++DBB/2uRESimMK9oZ18sjf+/vLL3oU9RETqgcLdDw8/DG3bepfk09RIEakHCnc/tG3rjbt//DG8/bbf1YhIFFK4+2X0aEhP9y6sffCg39WISJRRuPulWTN47jnYsMFbf0ZEJIwU7n4691xv5cjHH4fvv/e7GhGJIgp3v02Z4p2xev/9flciIlFE4e63H/3ImzXzpz9BNF92UEQalMK9MXjgAejQQVMjRSRsQgp3MzvfzFab2VozmxDg+efMbEnJ7Wsz2xX+UqNY69bwxBPw6afwt7/5XY2IRAFzQXqKZhYLfA38FMgFFgEjnHMrq2h/O9DPOfeL6rablZXlsjUM8YOiIsjKgvx8+OoraNnS74pEpBEys8XOuaxg7ULpufcH1jrn1jvnDgOzgaHVtB8BzAqtTCkVGwtTp8K338LTT/tdjYhEuFDCvQvwbZnHuSV/q8TMjgO6A/9TxfOjzSzbzLLz8vJqWmv0O/NMuOIKeOopL+RFRGoplHC3AH+raixnOPCmc64o0JPOuRnOuSznXFZqamqoNTYtkydDcTFMqPTThohIyEIJ91yga5nHacDmKtoOR0MyddOtm7ckwV//Cp995nc1IhKhQgn3RUBPM+tuZvF4AT6nYiMzOxFoCyiR6mrCBOjcGe680+vFi4jUUNBwd84VArcBc4FVwBvOuRVmNtHMhpRpOgKY7YJNv5HgEhNh0iRYtAhee83vakQkAgWdCllfNBUyiOJiOP1074fVr7/2Al9EmrxwToUUP8TEeFMjv/8ennzS72pEJMIo3BuzAQPgmmvgmWe8pYFFREKkcG/sJk3yTnC6916/KxGRCKJwb+zS0mD8eHjzTZg/3+9qRCRCKNwjwbhx0LWrt2pkUcDzw0REylG4R4KWLb31ZpYsgVde8bsaEYkACvdIceWVcMYZ8Otfw+7dflcjIo2cwj1SmHkX0t6+3bvmqohINRTukeTUU2HkSG/++5o1flcjIo2Ywj3S/OY30Ly59yOriEgVFO6RplMnb9x9zhz46CO/qxGRRkrhHonuuguOPx7GjIHCQr+rEZFGSOEeiRISYMoUWLEC/vAHv6sRkUZI4R6pLrkEBg+Ghx6CHTv8rkZEGhmFe6Q6OjVy1y549FG/qxGRRkbhHskyMuCGG2D6dFi1yu9qRKQRUbhHusce8y7kMWYM6CJYIlJC4R7pUlPh4Ydh7lx4/32/qxGRRkLhHg1uvRVOOAHGjoXDh/2uRkQagZDC3czON7PVZrbWzCZU0eZKM1tpZivM7K/hLVOqFR8Pzz7rXWt1+nS/qxGRRiBouJtZLDAduADoBYwws14V2vQE7gMGOud6A3fVQ61SnQsvhJ/9zJs5k5fndzUi4rNQeu79gbXOufXOucPAbGBohTY3ANOdczsBnHPbwlumBGXm9d4LCry57yLSpIUS7l2Ab8s8zi35W1knACeY2adm9rmZnR+uAqUGevWCW26BGTMgJ8fvakTER6GEuwX4W8U5d82AnsBZwAjgZTNLrrQhs9Fmlm1m2XkaOqgfjzwCycne+jOaGinSZIUS7rlA1zKP04DNAdq865w74pzbAKzGC/tynHMznHNZzrms1NTU2tYs1WnXDiZOhHnz4J13/K5GRHwSSrgvAnqaWXcziweGA3MqtHkHGAxgZil4wzTrw1mo1MCNN0Lv3t6a74cO+V2NiPggaLg75wqB24C5wCrgDefcCjObaGZDSprNBfLNbCUwD7jHOZdfX0VLEM2awXPPwfr13vozItLkmPNpXDYrK8tlZ2f78t5NxpAh3vDMmjXeRT5EJOKZ2WLnXFawdjpDNZo984w3LPPrX/tdiYg0MIV7NOvZE+68E2bOhMWL/a5GRBqQwj3aPfAApKR4Ia+pkSJNhsI92rVpA088AZ9+Cm+84Xc1ItJAFO5NwS9+AX36wL33woEDflcjIg1A4d4UxMbC1KnwzTfehbVFJOop3JuKQYNg2DCYNAlyc/2uRkTqmcK9KXn6aSgqggkBl+QXkSiicG9KunWDu++G11+Hzz/3uxoRqUcK96bmvvugc2dvamRxsd/ViEg9Ubg3NYmJ8OSTsHCh14MXkaikcG+KrrsOfvxjb+y9oMDvakSkHijcm6KYGG9q5ObN8NRTflcjIvVA4d5UnX46XH21N4Nm40a/qxGRMFO4N2WTJnm9+Hvv9bsSEQkzhXtT1rUrjB8Pf/87LFjgdzUiEkYK96bunnu8kL/rLu8EJxGJCgr3pq5lS+9H1f/8B/70J7+rEZEwUbgLDB8OAwfC/ffDnj1+VyMiYaBwFzDzLqS9bRs8/rjf1YhIGIQU7mZ2vpmtNrO1ZlZp1SkzG2lmeWa2pOT2q/CXKvUqKwtGjvRCfu1av6sRkToKGu5mFgtMBy4AegEjzKxXgKZ/c871Lbm9HOY6pSH85jfQvDmMG+d3JSJSR6H03PsDa51z651zh4HZwND6LUt80bmzN+7+7rvw73/7XY2I1EEo4d4F+LbM49ySv1V0uZnlmNmbZtY1LNVJwxszBrp396ZGFhb6XY2I1FIo4W4B/uYqPP5voJtzLgP4F/BqwA2ZjTazbDPLzsvLq1ml0jASErxL8S1fDi+95Hc1IlJLoYR7LlC2J54GbC7bwDmX75w7VPLwJeDUQBtyzs1wzmU557JSU1NrU680hEsvhbPOggcfhJ07/a5GRGohlHBfBPQ0s+5mFg8MB+aUbWBmncs8HAKsCl+J5eXvz2fz3s04V/HLg4TN0amRO3fCo4/6XY2I1EKzYA2cc4VmdhswF4gFXnHOrTCziUC2c24OcIeZDQEKgR3AyPoqePrC6Tw8/2HatWhH30596dOxDxkdM+jTsQ8np55MQrOE+nrrpqVPH/jVr2D6dLjpJjjpJL8rEpEaML96wFlZWS47O7vGrxv3z3E889kz9D+mPw7H8m3LOVB4AIBYi+WklJNKwz6jYwYZHTM4JukYzAL9dCDV2rYNevb0zl59/32/qxERwMwWO+eygrUL2nNvbCacMYGOrToyqt8oUlqmUFRcxNoda8nZmsPSrUvJ2ZrDp99+yqzls0pf075F+3KB36dTH3ql9lIvP5gOHeChh7x57x98ABdc4HdFIhKiiOu5h2rXwV3kbM3xQn/LUnK25bBs67JyvfwTU04sH/od+6iXX9Hhw5Ce7q37vmwZxMX5XZFIkxZqzz1qwz2QouIi1u1c54V9mZ7+pt2bStu0a9GuXNhndMygd4feTbuX/957cPHF8Nxz3vx3EfGNwr0Gdh3cxbKty0rDfunWpSzftpz9R/YDXi//hPYn0KdTHzI6eMM6GR0z6JLUpWn08p2D88+HhQthzRpISfG7IpEmS+FeR0d7+aVDOyXBv3HXxtI27Vq08360LRP4vVN70yKuhX+F15eVKyEjA0aPht/9zu9qRJoshXs92X1wN8u2LSs3tLNs27LSXn6MxXi9/ApDO2mt0yK/l3/77V6wL1kCp5zidzUiTZLCvQEVu2LW7VhXroe/dOvScr38tgltK83Yibhefn6+NzUyMxM++sg72UlEGpTCvRE42suvOGNn35F9wA+9/Iozdhp1L/+3v/V68O+8A0O1OKhIQ1O4N1LFrpj1O9eXDuvkbPOCf8OuDaVtjvbyK87YaRnX0sfKSxQWemevHjoEK1Z467+LSINRuEeYPYf2sGzrsnJDOzlbc8r18nu261lpxk7X1l0bvpf/z3/Cz37mXVj73nsb9r1FmjiFexQodsVs2Lmh3Dh+ztYc1u9cX9omOSG50oyd9A7p9d/Lv/himD/fmxrZsWP9vpeIlFK4R7E9h/awfNvySjN2Cg4XAGAYPdv3rDRj59g2x4avl//1196Zq9dfDy/rqooiDUXh3sQc7eVXnLFTtpffpnmbgDN2WsW3qt2b3n23d9bq4sXQr1+Y9kREqqNwFwD2Htrr9fIrDO1U7OVXnLETUi9/1y444QRvOeD58zU1UqQBKNylSsWumI27Npaborl0y1LW7VxX2uZoL7/srJ30DumVe/kzZsCNN8Lf/gZXXtnAeyLS9CjcpcYKDhcEnLGz9/BewOvl/6jdj8r38lPTOe6cy7Cdu+Crr6BFBJ2UJRKBFO4SFsWumE27NlUa1lm7Y21pm9axrThp0z4OxBu9D7UmpVUqia1TSWzXkcTULiR2Po7EDmkkNm9NYnxiuVtS8yRaxbUiNibWx70UiRwKd6lXBYcLys3Y+eCzV9kQt4/kwzHEOCiILeZwDS4F06JZi0rBX90tKT4paJu4WK09L9Enaq/EJI1DYnwiA9IGMCBtAADbM+5g5l/GMuoXz5LS9UQoLOTwxnXsW7uSveu/omCB3q4RAAAKnElEQVTTGgq+20DBlm8o2L6ZguKDFMTzw619LAXt4yhIjqEgsYiCFgXsjS/g+9hCCooOUHC4gILDBRwsPBhyjfGx8bU6KFR3MImPjW+8S0OIlKGeuzQ852D7dli3LvBty5by7du2hR49oEcPCnt0Z1/3LhQc24mCY1IpaNuSgiP7S8O/4HABew/vLfc42O3oWcChaBbTrPoDQVzlYadgB44WzVrogCEhC2vP3czOB6YCscDLzrlJVbQbBvwd+LFzTsktgZlBaqp3GzCg8vP79sH69ZVDPzubZm+9RZvCQtocbdu8OXTvXhr+3q2n92/37iGtfVPsitl3eF+NDggFhwsoOFLA3kPegeS7Pd9Vet4RWsfJsIC/RVR1wKj2G0bJ61rGtSTGYkL/TCTqBA13M4sFpgM/BXKBRWY2xzm3skK7JOAO4Iv6KFSakFatvPXiA60ZX1gI33xTOfjXr/fm2hcU/NDWDNLSKgR/mVtyMuCt25PUPImk5klh2wXnHAcKD1R5YDh6UAh0wDh6f9u+bazfub7ca4pcUcg1tGzWEgyS4pNo3qw5zWKaERcTR1xsXM3ux5Tcj635/Vq/Z4D7OljVTCg99/7AWufcegAzmw0MBVZWaPcYMBkYF9YKRcpq1gyOP967/fSn5Z9zDvLyAg/1vPcebN1avn27duXD/vjjf7h/zDHeRcFrycxoGdeSlnEt6dCqQ623U5ZzjkNFh0L+ZjFvwzwWfLOA9NR0Tk49mcLiQo4UH+FI0ZFK9w8UHmDPoT0B2xQWF3Kk6Eil+8WuOCz7FaoYi6nTgSbgQSdM26vp/aMH3PoUSrh3Ab4t8zgXOK1sAzPrB3R1zr1nZgp38YcZdOjg3U4/vfLzBQWBh3sWLoS//x2KyvSKExICDPeU3Lp182WpYzMjoVkCCc0SSGkZ/Dq2t/W/jZn/mcmofqNCal9Txa64NOyrO3CE835VB5raHriqul/fB65LT7qUGRfPqJfP5ahQwj3QLz2lg4lmFgM8B4wMuiGz0cBogGOPPTa0CkXCJTHRuw5sRkbl544cqTzcc/RAMG+e9zvAUWbQtWvVwz1t2lTevg9SWqZwz8B76m37MRZDfGw88bHx9fYefil74Cp7IAnHAerDtR/y9ldvc3ra6fX6+YQS7rlA1zKP04DNZR4nAenAxyW/+HcC5pjZkIo/qjrnZgAzwJstU4e6RcIrLu6HcK7IOdi2LfBwz5w53nNltW8feKinRw/o3LlOwz3SMOrzwDWs1zAG/mcgo/qNCvu2ywo6FdLMmgFfA+cA3wGLgKudcyuqaP8xMC7YbBlNhZSosXdv4OGedeu8bwMVh3sqBn7Z4Z746OsFS3iFbSqkc67QzG4D5uJNhXzFObfCzCYC2c65OXUvVySCJSV5lx7s06fyc0eOwKZNlYd61q2Df/8b9u//oW1MTPXDPa1bN9w+ScTTSUwifnHOm8FT1clceXnl25cd7ql469xZSy43EVp+QKSxM4NOnbzbwIGVn9+zJ/Bwz2efeUssF5eZ0dGiReDhnuOO8w4Kbdt6vytIk6FwF2msWreGvn29W0WHD/8w3FPxAPDRR3DgQODttWvnhf3Rf8veD/RvcjLEasXOSKRwF4lE8fHQs6d3q8g5b32exx6D3/8eLrnEuwzijh2Qn//Dvxs3ev/u3Om9JhAzL+CrOwgE+lvr1hom8pnCXSTamHlj8BMneidijRoFKdWcLFNc7F0ysWzwV/VvXp53UZYdO2D37qq3GRvrBX113woC/duqlQ4KYaJwF4lWKSlwTwgnycTE/BDENXHkSOWDQlUHhtxcyMnxHu+rZhXO+PiaHxDatdMVwAJQuItI7cTF/bC6Z00cOuQFflXfDsreP7o8RH6+97qqtGhRs2GjowezKD6vQOEuIg2reXNv2Khz55q9bv/+4MNGR/9dteqHA0VhYdXbTEys2beE9u293yCaNf7obPwViogAtGzp3dLSQn+Nc96CcaEcEHbs8M4oPvqtoriaxcOSk2s+fNSmTYMuPaFwF5HoZeadQZyU5C3vEKriYu88g1B+T8jPh7VrvX937ap6mzEx3vkGbdp4F6mZOrX6H7rrSOEuIlJRTIzXO09ODryYXFWKiryppdV9O/j0U/jrX73zF0L5wbuWFO4iIuESG+v1xqvrkW/fDjNnelNU65HCXUSkIYU6RbWOtLC0iEgUUriLiEQhhbuISBRSuIuIRCGFu4hIFFK4i4hEIYW7iEgU8u0aqmaWB2yq5ctTgO1hLMdP2pfGJ1r2A7QvjVVd9uU451zQpTh9C/e6MLPsUC4QGwm0L41PtOwHaF8aq4bYFw3LiIhEIYW7iEgUitRwn+F3AWGkfWl8omU/QPvSWNX7vkTkmLuIiFQvUnvuIiJSjUYd7mZ2vpmtNrO1ZjYhwPPNzexvJc9/YWbdGr7K0ISwLyPNLM/MlpTcfuVHncGY2Stmts3MllfxvJnZtJL9zDGzzIauMVQh7MtZZra7zGfyUEPXGAoz62pm88xslZmtMLM7A7SJiM8lxH2JlM8lwcwWmtnSkn15NECb+ssw51yjvAGxwDrgeCAeWAr0qtDmFuDFkvvDgb/5XXcd9mUk8Fu/aw1hX84EMoHlVTx/IfABYMAA4Au/a67DvpwFvOd3nSHsR2cgs+R+EvB1gP99RcTnEuK+RMrnYkBiyf044AtgQIU29ZZhjbnn3h9Y65xb75w7DMwGhlZoMxR4teT+m8A5ZmYNWGOoQtmXiOCcWwDsqKbJUODPzvM5kGxmNbzMfcMIYV8ignPue+fclyX39wKrgC4VmkXE5xLivkSEkv/WBSUP40puFX/krLcMa8zh3gX4tszjXCp/yKVtnHOFwG6gfYNUVzOh7AvA5SVfmd80s64NU1rYhbqvkeL0kq/VH5hZb7+LCabka30/vF5iWRH3uVSzLxAhn4uZxZrZEmAb8JFzrsrPJdwZ1pjDPdDRq+JRL5Q2jUEodf430M05lwH8ix+O5pEmUj6TUHyJd6p3H+AF4B2f66mWmSUCbwF3Oef2VHw6wEsa7ecSZF8i5nNxzhU55/oCaUB/M0uv0KTePpfGHO65QNneaxqwuao2ZtYMaEPj/JoddF+cc/nOuUMlD18CTm2g2sItlM8tIjjn9hz9Wu2cex+IM7NqrnzsHzOLwwvD151z/xWgScR8LsH2JZI+l6Occ7uAj4HzKzxVbxnWmMN9EdDTzLqbWTzejw1zKrSZA/y85P4w4H9cyS8TjUzQfakw/jkEb6wxEs0Bri+ZnTEA2O2c+97vomrDzDodHf80s/54/3/J97eqykpq/COwyjn3bBXNIuJzCWVfIuhzSTWz5JL7LYBzga8qNKu3DGsWjo3UB+dcoZndBszFm23yinNuhZlNBLKdc3Pw/kfwFzNbi3e0G+5fxVULcV/uMLMhQCHevoz0reBqmNksvNkKKWaWCzyM90MRzrkXgffxZmasBfYDo/ypNLgQ9mUYcLOZFQIHgOGNtPMwELgOWFYyvgtwP3AsRNznEsq+RMrn0hl41cxi8Q5Abzjn3muoDNMZqiIiUagxD8uIiEgtKdxFRKKQwl1EJAop3EVEopDCXUQkCincRUSikMJdRCQKKdxFRKLQ/wcjnyZqlu2VSgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.01,epochs=4,bs=512,beta1=0.9,beta2=0.99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 0.30495336694002445, Val loss:0.3659217473357392\n",
      "Epoch 2. Training loss: 0.2949423067010172, Val loss:0.36343557277727057\n",
      "Epoch 3. Training loss: 0.28983872290269114, Val loss:0.3612610514370151\n",
      "Epoch 4. Training loss: 0.2849694016721142, Val loss:0.3595643567342532\n",
      "Epoch 5. Training loss: 0.28008828641989314, Val loss:0.35819683898130084\n",
      "Epoch 6. Training loss: 0.2752342400254782, Val loss:0.3564860701992241\n",
      "Epoch 7. Training loss: 0.2708044361121958, Val loss:0.3550643674515842\n",
      "Epoch 8. Training loss: 0.2662767359531267, Val loss:0.35295760746345123\n",
      "Epoch 9. Training loss: 0.26209300871311625, Val loss:0.3522334579348391\n",
      "Epoch 10. Training loss: 0.2580863792794052, Val loss:0.3515822934030762\n",
      "Epoch 11. Training loss: 0.2539833758494128, Val loss:0.35094726301431717\n",
      "Epoch 12. Training loss: 0.2501648406525396, Val loss:0.3503709422611456\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xt8FPW9//HXJ3dyIVwS5W6SSq2AECAgPVi5aBW1gli8oKiglXpB23q00mqtYttj1WPRFi+0P6itFA7Vo1KlctSiaG2VgIoCIshFwkUSBARCIJt8f39MEpZkkyzJJpvdfT8fj33szOzs7CeBvGfmu9/5jjnnEBGR6BIX7gJERCT0FO4iIlFI4S4iEoUU7iIiUUjhLiIShRTuIiJRSOEuIhKFFO4iIlFI4S4iEoUSwvXBWVlZLicnJ1wfLyISkVasWFHinMtubL2whXtOTg6FhYXh+ngRkYhkZluCWU/NMiIiUUjhLiIShRTuIiJRKGxt7iLSusrLyykqKqKsrCzcpUgQUlJS6NGjB4mJiU16v8JdJEYUFRWRkZFBTk4OZhbucqQBzjl2795NUVERubm5TdqGmmVEYkRZWRmdO3dWsEcAM6Nz587NOstSuIvEEAV75Gjuv1XkhfvLL8Po0VBcHO5KRETarMgL99mzYelS+N3vwl2JiByH3bt3k5+fT35+Pl26dKF79+4180eOHAlqG1OmTGHdunUNrjNr1izmzZsXipI544wz+OCDD0KyrdYWeV+oXnopLFoEw4eHuxIROQ6dO3euCcp7772X9PR0br/99mPWcc7hnCMuLvBx59y5cxv9nJtvvrn5xUaByDtyz8/3nr/8Mrx1iEhIbNiwgX79+nHDDTcwaNAgduzYwdSpUykoKKBv377MmDGjZt3qI2mfz0eHDh2YPn06AwYM4Jvf/Ca7du0C4O6772bmzJk160+fPp2hQ4dyyimn8M477wBw8OBBvvvd7zJgwAAmTpxIQUFBo0fozzzzDKeddhr9+vXjpz/9KQA+n4+rrrqqZvljjz0GwG9+8xv69OnDgAEDmDRpUsh/Z8GIvCP36m5BGzeGtw6RSPbDH0IwzQ3l5bBzJ3TpAo31t87Ph6pQPV5r1qxh7ty5PPnkkwA88MADdOrUCZ/Px6hRo5gwYQJ9+vQ55j379u1jxIgRPPDAA9x2223MmTOH6dOn19m2c4733nuPRYsWMWPGDF555RV++9vf0qVLF5577jk+/PBDBg0a1GB9RUVF3H333RQWFpKZmcnZZ5/NSy+9RHZ2NiUlJXz00UcA7N27F4AHH3yQLVu2kJSUVLOstUXekXtqqvcfTeEu0vJ27vT+1nbubNGP+drXvsaQIUNq5ufPn8+gQYMYNGgQa9euZc2aNXXe065dO8477zwABg8ezObNmwNu++KLL66zzttvv83ll18OwIABA+jbt2+D9b377ruMHj2arKwsEhMTueKKK1i2bBknn3wy69at4wc/+AFLliwhMzMTgL59+zJp0iTmzZvX5IuQmivyjtzBO3rftCncVYhErmCPsEtKYO5cmDIFsrJarJy0tLSa6fXr1/Poo4/y3nvv0aFDByZNmhSwv3dSUlLNdHx8PD6fL+C2k5OT66zjnDuu+upbv3PnzqxatYq///3vPPbYYzz33HPMnj2bJUuW8Oabb/Liiy/yi1/8go8//pj4+Pjj+szmavTI3czmmNkuM/u4ntfNzB4zsw1mtsrMGj6/CYW8PB25i7SGrCy4444WDfbavvrqKzIyMmjfvj07duxgyZIlIf+MM844g4ULFwLw0UcfBTwz8Dds2DCWLl3K7t278fl8LFiwgBEjRlBcXIxzjksuuYT77ruPlStXUlFRQVFREaNHj+ahhx6iuLiY0tLSkP8MjQnmyP2PwO+AP9Xz+nlA76rH6cATVc8tJy8P5s/32gPDdMojIi1j0KBB9OnTh379+pGXl8fwFugZd8stt3D11VfTv39/Bg0aRL9+/WqaVALp0aMHM2bMYOTIkTjnuPDCC7ngggtYuXIl1113Hc45zIxf//rX+Hw+rrjiCvbv309lZSV33nknGRkZIf8ZGmPBnJ6YWQ7wknOuX4DXngLecM7Nr5pfB4x0zu1oaJsFBQWuyTfrmDsXrr0WPvvMC3oRadTatWs59dRTw11Gm+Dz+fD5fKSkpLB+/XrOOecc1q9fT0JC22qpDvRvZmYrnHMFjb03FD9Jd2Cr33xR1bIGw71ZqgN940aFu4gctwMHDnDWWWfh8/lwzvHUU0+1uWBvrlD8NIEGQAh4OmBmU4GpAL169Wr6J/qHu4jIcerQoQMrVqwIdxktKhRdIYuAnn7zPYDtgVZ0zs12zhU45wqysxu9v2v9unWDpCSFu4hIPUIR7ouAq6t6zQwD9jXW3t5s8fFw0knqDikiUo9Gm2XMbD4wEsgysyLg50AigHPuSWAxcD6wASgFprRUscdQd0gRkXo1Gu7OuYmNvO6A1h+pJy8Pli9v9Y8VEYkEkTf8QLXcXG/wsH37wl2JiARh5MiRdS5ImjlzJjfddFOD70tPTwdg+/btTJgwod5tN9a1eubMmcdcTHT++eeHZNyXe++9l4cffrjZ2wm1yA336h4zancXiQgTJ05kwYIFxyxbsGABEyc22DhQo1u3bjz77LNN/vza4b548WI6dOjQ5O21dZEf7mp3F4kIEyZM4KWXXuLw4cMAbN68me3bt3PGGWfU9DsfNGgQp512Gi+++GKd92/evJl+/bzrKA8dOsTll19O//79ueyyyzh06FDNejfeeGPNcME///nPAXjsscfYvn07o0aNYtSoUQDk5ORQUlICwCOPPEK/fv3o169fzXDBmzdv5tRTT+X666+nb9++nHPOOcd8TiAffPABw4YNo3///owfP549e/bUfH6fPn3o379/zYBlb775Zs3NSgYOHMj+/fub/LsNJHJ77SvcRZrsh6/8kA92Nj7kb3lFOTsP7KRLehcS4xse6iO/Sz4zx9Q/IFnnzp0ZOnQor7zyCuPGjWPBggVcdtllmBkpKSk8//zztG/fnpKSEoYNG8bYsWPrvY/oE088QWpqKqtWrWLVqlXHDNn7y1/+kk6dOlFRUcFZZ53FqlWruPXWW3nkkUdYunQpWbXGyVmxYgVz587l3XffxTnH6aefzogRI+jYsSPr169n/vz5/P73v+fSSy/lueeea3B89quvvprf/va3jBgxgnvuuYf77ruPmTNn8sADD7Bp0yaSk5NrmoIefvhhZs2axfDhwzlw4AApKSkN/n6PV+QeuWdmQseOapYRaUE7D+xk496N7DwQmiF//Ztm/JtknHP89Kc/pX///px99tls27aNL774ot7tLFu2rCZk+/fvT//+/WteW7hwIYMGDWLgwIGsXr260UHB3n77bcaPH09aWhrp6elcfPHFvPXWWwDk5uaSX3WDoIaGFQZvfPm9e/cyYsQIAK655hqWLVtWU+OVV17JM888U3Ml7PDhw7ntttt47LHH2Lt3b8ivkI3cI3dQd0iRJmroCNtfSWkJc9+fy5SBU8hKbf7IkBdddBG33XYbK1eu5NChQzVH3PPmzaO4uJgVK1aQmJhITk5OwGF+/QU6qt+0aRMPP/wwy5cvp2PHjkyePLnR7TQ0vlb1cMHgDRncWLNMfV5++WWWLVvGokWLuP/++1m9ejXTp0/nggsuYPHixQwbNozXXnuNb3zjG03afiCRe+QOCneRFpaVmsUdw+8ISbCD1/Nl5MiRXHvttcd8kbpv3z5OOOEEEhMTWbp0KVu2bGlwO2eeeWbNTbA//vhjVq1aBXjDBaelpZGZmckXX3zB3//+95r3ZGRkBGzXPvPMM3nhhRcoLS3l4MGDPP/883zrW9867p8tMzOTjh071hz1//nPf2bEiBFUVlaydetWRo0axYMPPsjevXs5cOAAn332Gaeddhp33nknBQUFfPLJJ8f9mQ2J7CP33Fx48UWorIR6bqgrIm3LxIkTufjii4/pOXPllVdy4YUXUlBQQH5+fqNHsDfeeCNTpkyhf//+5OfnM3ToUMC7q9LAgQPp27dvneGCp06dynnnnUfXrl1ZunRpzfJBgwYxefLkmm1873vfY+DAgQ02wdTn6aef5oYbbqC0tJS8vDzmzp1LRUUFkyZNYt++fTjn+NGPfkSHDh342c9+xtKlS4mPj6dPnz41d5UKlaCG/G0JzRryt9pTT8ENN8DWrdCjR2gKE4lSGvI38jRnyN/IPtxVjxkRkYAiO9xzc71nhbuIyDEiO9x79fLa2tUdUiQo4WqGlePX3H+ryA73pCTo2VNH7iJBSElJYffu3Qr4COCcY/fu3c26sCmye8uAukOKBKlHjx4UFRVRXFwc7lIkCCkpKfRoRkeRyA/33Fzw68sqIoElJiaSW/09lUS9yG6WAe/IfccO8BvtTUQk1gUV7mY2xszWmdkGM5se4PWTzOx1M1tlZm+YWet1Oq/uDtmECw5ERKJVo+FuZvHALOA8oA8w0cz61FrtYeBPzrn+wAzgv0JdaL3UHVJEpI5gjtyHAhuccxudc0eABcC4Wuv0AV6vml4a4PWWo5t2iIjUEUy4dwe2+s0XVS3z9yHw3arp8UCGmXVufnlByM6GtDQduYuI+Akm3AONll+7o+ztwAgzex8YAWwDfHU2ZDbVzArNrDBk3bHM1B1SRKSWYMK9COjpN98D2O6/gnNuu3PuYufcQOCuqmV17lztnJvtnCtwzhVkZ2c3o+xacnPVLCMi4ieYcF8O9DazXDNLAi4HFvmvYGZZZla9rZ8Ac0JbZiOqj9x15Z2ICBBEuDvnfMA0YAmwFljonFttZjPMbGzVaiOBdWb2KXAi8MsWqjewvDw4eBB05Z2ICBDkFarOucXA4lrL7vGbfhZ4NrSlHQf/7pAnnBC2MkRE2orIv0IV1B1SRKSW6Aj3nBzvWT1mRESAaAn31FTo0kXhLiJSJTrCHbymGTXLiIgA0RbuOnIXEQGiLdy3boUjR8JdiYhI2EVPuOfmQmUlfP55uCsREQm76Al3dYcUEakRfeGudncRkSgK927dIClJ4S4iQjSFe1ycdzGTmmVERKIo3EHdIUVEqijcRUSiUHSFe24u7NkDe/eGuxIRkbCKrnBXd0gRESBaw11NMyIS44IKdzMbY2brzGyDmU0P8HovM1tqZu+b2SozOz/0pQbB/6YdIiIxrNFwN7N4YBZwHtAHmGhmfWqtdjfe7fcG4t1j9fFQFxqUzEzo1EnNMiIS84I5ch8KbHDObXTOHQEWAONqreOA9lXTmcD20JV4nNRjRkQkqHDvDmz1my+qWubvXmCSmRXh3Wv1lpBU1xQKdxGRoMLdAixzteYnAn90zvUAzgf+bGZ1tm1mU82s0MwKi4uLj7/aYOTmwubNUFHRMtsXEYkAwYR7EdDTb74HdZtdrgMWAjjn/gWkAFm1N+Scm+2cK3DOFWRnZzet4sbk5UF5OWwPX8uQiEi4BRPuy4HeZpZrZkl4X5guqrXO58BZAGZ2Kl64t9CheSPUHVJEpPFwd875gGnAEmAtXq+Y1WY2w8zGVq32n8D1ZvYhMB+Y7Jyr3XTTOtQdUkSEhGBWcs4txvui1H/ZPX7Ta4DhoS2tiXr18kaIVHdIEYlh0XWFKkBiohfwOnIXkRgWfeEOXtOMwl1EYlh0hrv6uotIjIvecP/iCygtDXclIiJhEb3hDvpSVURiVnSGu7pDikiMi85w15G7iMS46Az3rCxIT9eRu4jErOgMdzN1hxSRmBad4Q7qDikiMS26w33TJgjTEDciIuEU3eFeWgq7doW7EhGRVhe94a7ukCISw6I33NUdUkRiWPSGe06O96wjdxGJQdEb7u3aQdeuCncRiUlBhbuZjTGzdWa2wcymB3j9N2b2QdXjUzPbG/pSm0DdIUUkRjV6JyYziwdmAd/Gu1n2cjNbVHX3JQCccz/yW/8WYGAL1Hr88vLgzTfDXYWISKsL5sh9KLDBObfROXcEWACMa2D9iXj3UQ2/3FzYuhWOHAl3JSIirSqYcO8ObPWbL6paVoeZnQTkAv9ofmkhkJfnXcS0ZUu4KxERaVXBhLsFWFbfZZ+XA8865yoCbshsqpkVmllhcXFxsDU2nbpDikiMCibci4CefvM9gO31rHs5DTTJOOdmO+cKnHMF2dnZwVfZVNXhri9VRSTGBBPuy4HeZpZrZkl4Ab6o9kpmdgrQEfhXaEtshq5dITlZ4S4iMafRcHfO+YBpwBJgLbDQObfazGaY2Vi/VScCC5xrQyN1xcV5FzMp3EUkxjTaFRLAObcYWFxr2T215u8NXVkhVD06pIhIDIneK1Sr6aYdIhKDoj/c8/Jg717YsyfclYiItJrYCHdQ04yIxJTYCXc1zYhIDIn+cNdNO0QkBkV/uLdvD507K9xFJKZEf7iDukOKSMyJjXBXd0gRiTGxEe55ed7IkBUBxzMTEYk6sRPu5eWwbVu4KxERaRWxE+6gphkRiRmxEe7qDikiMSY2wr1nT4iPV7iLSMyIjXBPTIRevdQdUkRiRmyEO6g7pIjElNgJ97w8hbuIxIygwt3MxpjZOjPbYGbT61nnUjNbY2arzewvoS0zBPLyYNcuOHgw3JWIiLS4RsPdzOKBWcB5QB9gopn1qbVOb+AnwHDnXF/ghy1Qa/NU95hRu7uIxIBgjtyHAhuccxudc0eABcC4WutcD8xyzu0BcM7tCm2ZIaC+7iISQ4IJ9+7AVr/5oqpl/r4OfN3M/mlm/zazMaEqMGQU7iISQ4K5QbYFWOYCbKc3MBLoAbxlZv2cc3uP2ZDZVGAqQK9evY672Gbp3BkyMtQsIyIxIZgj9yKgp998D2B7gHVedM6VO+c2Aevwwv4YzrnZzrkC51xBdnZ2U2tuGjN1hxSRmBFMuC8HeptZrpklAZcDi2qt8wIwCsDMsvCaadpeiqo7pIjEiEbD3TnnA6YBS4C1wELn3Gozm2FmY6tWWwLsNrM1wFLgDufc7pYqusmqb9rharcqiYhEl2Da3HHOLQYW11p2j9+0A26rerRdublw6BB88QV06RLuakREWkzsXKEK6jEjIjFD4S4iEoViK9xzcrxndYcUkSgXW+GekgLduunIXUSiXmyFO6g7pIjEhNgMdzXLiEiUi71wz82FoiI4fDjclYiItJjYC/e8PO8ipi1bwl2JiEiLic1wB7W7i0hUi71w1007RCQGxF64d+0Kyck6cheRqBZ74R4Xp6F/RSTqxV64g7pDikjUi81wz82Fzz7T0L8iErViM9zz8uCrr2DPnnBXIiLSImI33EHt7iIStWIz3NUdUkSiXFDhbmZjzGydmW0ws+kBXp9sZsVm9kHV43uhLzWEqsNdR+4iEqUavc2emcUDs4BvA0XAcjNb5JxbU2vV/3HOTWuBGkOvfXvIylK4i0jUCubIfSiwwTm30Tl3BFgAjGvZslqBukOKSBQLJty7A1v95ouqltX2XTNbZWbPmlnPQBsys6lmVmhmhcXFxU0oN4R0IZOIRLFgwt0CLKvdQfxvQI5zrj/wGvB0oA0552Y75wqccwXZ2dnHV2mo5eV5I0P6fOGtQ0SkBQQT7kWA/5F4D2C7/wrOud3OueoB0n8PDA5NeXWVlpeybMuy5m8oL88L9qKi5m9LRKSNCSbclwO9zSzXzJKAy4FF/iuYWVe/2bHA2tCVeKy7Xr+LkX8cyc/+8TNcc64wVXdIEYlijYa7c84HTAOW4IX2QufcajObYWZjq1a71cxWm9mHwK3A5JYqOCs1C4fjF2/9gkv+egn7D+9v2oZ0IZOIRLFGu0ICOOcWA4trLbvHb/onwE9CW1pg3y/4PonxiZSVl3HfsvtYW7KW5y97nq93/vrxbahnT4iPV7iLSFSKuCtUs1Kz+PHwH3PPyHt49apX2XVwF0N+P4RF6xY1/mZ/CQlw0klqlhGRqBRx4e5vdO5oVkxdQe9OvRm3YBw/X/pzKl1l8BtQd0gRiVIRHe4AvTJ78daUt5icP5kZy2Zw4fwL2Vu2N7g35+Up3EUkKkV8uAO0S2zHnLFzePz8x3n1s1cpmF3Ax7s+bvyNeXlQXAwHDrR8kSIirSgqwh3AzLhxyI28MfkNDpYf5PQ/nM7C1QsbfpO6Q4pIlIqacK/2Hz3/g5VTV5LfJZ/Lnr2MO/7vDnyV9VyFqu6QIhKloi7cAbpmdGXpNUu5qeAmHv7Xw4x5ZgwlpSV1V1S4i0iUispwB0iKT2LWBbOYO24ub3/+NoNnD2bF9hXHrtSpE2RkqFlGRKJO1IZ7tcn5k3n72rdxzjF8znCe/sBvTDMz9ZgRkagU9eEOUNCtgBVTVzC813AmvziZaYuncaTiiPeiwl1EolBMhDtAdlo2SyYt4fZv3s6s5bMY/fRoduzfcfSmHc0ZhExEpI2JmXAHSIhL4KFzHmLBdxfw/s73GTx7MO/0cFBWBjt3hrs8EZGQialwr3ZZv8v493X/JjUxlZFfPcbjQ6Bixr1QEqBHjYhIBLJmjYneDAUFBa6wsDAsn11tz6E9TJo/gcVb/0GiDwaXdeQ/zrqGgu5DGdJ9CF/r+DXMAt2ISkQkPMxshXOuoNH1YjncASpdJVc/eS7zdr1Gr72wK8Moi/d+Jx1TOlLQrYCCbgUM6TaEId2H0D2juwJfRMIm2HAPajz3aBZnccy8Zj4D3p/DlE2ZZE6/j9UVO1h+yXCWfyuPwi8/5qF3Hqq5yrVLepejYd9tCAXdCshOC/P9YEVEagnqyN3MxgCPAvHAH5xzD9Sz3gTgr8AQ51yDh+Vt5ci9jv374f774Te/gfR0uP9+Dl13DR/uXk3h9kKWb1/O8m3L+aTkE1zVfcJPyjyJId2Phv3groPJTMkM8w8iItEoZM0yZhYPfAp8G+9m2cuBic65NbXWywBeBpKAaREb7tXWroVbboHXX4cBA2DWLBg+vObl/Yf3s3LHSpZvX14T+hv3HO0vn5aYRvf23ema3pWs1KyaR3Zq9tHptKPTqYmp4fgpRSTChLJZZiiwwTm3sWrDC4BxwJpa690PPAjcfpy1tk2nngqvvgrPPQe33QZnnAFXXQUPPghdupCRnMGInBGMyBlR85bdpbtZsWMFj/zrEZZ8toSk+CQcjjXFaygpLWH3od313kwkNTG13p1Au4R2rP9yPTNGzeCEtBNa6zcgIhEsmCP3CcAY59z3quavAk53zk3zW2cgcLdz7rtm9gZwe6AjdzObCkwF6NWr1+AtW7aE7AdpUQcPwq9+BQ8/DMnJcN99MG0aJCYGXL2ktIS5789lysApZKVm1SyvdJXsObSHktISSkpLKC4tPjp9sJiSQ37TVcv3Hzl6A/CsdlncdeZdTM6fTIeUDi3+Y4tI2xPKZplLgHNrhftQ59wtVfNxwD+Ayc65zQ2Fu7823ywTyPr1cOut8Mor0Lcv/O53MHJki37kYd9hPt39Kb9661es/3I9K3asIDUxlStPu5KbhtxEfpf8Fv18EWlbgg33YC5iKgJ6+s33ALb7zWcA/YA3zGwzMAxYZGaNfnjE6d0bFi+GF17wjuZHjYKJE2Hbthb7yOSEZE478TTmT5hP4dRCVkxdwcR+E3lm1TMMfGogw+cM5y8f/eXoWDkiIgR35J6A94XqWcA2vC9Ur3DOra5n/TeI1iN3f4cOwa9/DQ884DXP3HMP/OAHkJTUKh+/59Ae5n4wlycKn2DDlxs4Ie0Erh90Pd8f/H16ZvZsfAMiEpFCduTunPMB04AlwFpgoXNutZnNMLOxzS81QrVrB/feC2vWwOjR8OMfe71qXnutVT6+Y7uO3PbN21g3bR2vXPkKp3c/nV+99StyHs1h/P+M57WNrxGuC9REJPxi/grVkFm82GuP/+wz+M534Otfh+nTIbv1LnDavHczTxY+yR9W/oHdh3ZzSudTuGnITVwz4Br1uxeJEhp+IBzKyuC//9s7ovf5ICsLrr0Wxo+HoUMhrnXGaSvzlfHX1X9l1vJZvLvtXVITU5l02iRuHnoz/U/s3yo1iEjLULiH0+rVcPfd8OWX8M47XtB36wbjxnlBP3Jkvd0oQ23F9hU8vvxx/vLxXyjzlXFGrzO4ecjNXHzqxSTFt873AyISOgr3tmLPHnjpJa+HzSuvQGkpdOjgNd2MHw/nngtpaS1expeHvmTu+3N5vPBxNu7ZSEpCCt3Su5GWlEZKQkq9j+T45AZfT0lIITkhmcO+w7y+6XUu+sZFnJh2IskJySTFJ5Ecn0xyQnLNc2JcogZeE2kGhXtbVFrqXfX6/PPwt795R/YpKXDOOV7QX3ghdO7coiVUukquX3Q9cz6Yw4ATB5DbMZcyX1nN47Dv8DHz/o/qsXSaKzm+Kvj9Qt//OSk+CTNj18Fd5HbIJT0pnaT4JBLjE0mMS/Smq5/9l9Warl4vMT6RMl8ZSzct5byTzyMrLYuEuAQS4hJIjEs8Oh2f2OjyeIvXzknCSuHe1vl8sGyZF/QvvABFRRAfD2ee6QX9RRdBz5bp0ljfFbQNcc5RXlleb/jvPLCTv336N0bmjCQlIYXDvsMcrjjMkYojNdO1n495rdbrm/Zu4vN9n9MlrQsd2nXgSMURyivKKa8srzNd35AOLcU/6CtdJelJ6aQkpNTZodTeCdX7WtUyX6WP1btWM7jrYDq063DMTqZ6B9OU+UBnUQlxCdpJRSiFeyRxDlas8IL++ee9QcsABg/2gn78eG+smxj6YzyeHVClq6S8oir0A4R/eUU5u0p38fza5xlz8hjSk9LxVfooryjHV+nzpivL611W3/J/bf0Xb299m9O7n06f7D51Pr96PlBtgV4r85W12o7KsJqzJ//gr70T8D/Dcs7x+Vefc0rnU0hNTCXO4oizOOItvmY6zuKIj4sP6rU4i6OsvIzl25czrMcw2ie3P+a1ht5X32P/4f28vul1zv3auXRs1zHwtiy4bfk/zKzuMuoua2jdUO1MFe6RbN26o0H/3nvesm7dvGGIL700vLVJjaacATW2vTnvz+HqAVfTIaVDzQ6l9o4n2Pnn1jzHMx89wyV9LuGs3LNqzoqOVByp9wzK/7Xa6+08sJOS0hIykzNJT0qn0lVS4SqodJXedOWfAoBfAAAK80lEQVTR6dqvtfbZVVtVHfTjThnHUxc+1aT/Nwr3aLFtmzdI2QsvePPnnusNYNavX3jrkjavJXY+zdmef9BX7wx2HdzFM6ueYWK/iXRs17HuOrV2EI09vjz0JYvWLeL83ufTPrl9nc87nm1Vukpe+vQlFq5ZyIRTJ3DuyefWed05F/B9jrrLq9d9Z+s7/GPzP3jw7Ae5Y/gdx/17VLhHk5IS+P3vobzcO3r/6iu47jqYMQO6dAl3dSJRK9Q7yFBsU+EerXbv9u4UNWuWN/zw9OneePOputmHSCwI5aiQ0pZ07gwzZ3pj2px7LvzsZ95QB08/DZVq1xQRj8I9UvXu7d0latky78vWyZOhoACWLg13ZSLSBijcI923vgX//jfMm+c12YweDWPHwiefhLsyEQkjhXs0iIuDK67wAv2//gveeMPrTTNtGhQXh7s6EQkDhXs0adfO+4J1wwaYOhWefBJOPtm7qXdZWbirE5FWpHCPRiecAI8/DqtWec02d94J3/gGLFjgXQ0rIlEvqHA3szFmts7MNpjZ9ACv32BmH5nZB2b2tpn1CX2pctz69PFGpHztNW8kyokTYdgw+Oc/w12ZiLSwYO6hGo93D9Vv490sezkw0Tm3xm+d9s65r6qmxwI3OefGNLRd9XNvZRUV8Kc/wV13wY4dXvBPmAB9+8JJJ0FOjnfEH0Pj14hEomD7uScEsa2hwAbn3MaqDS8AxgE14V4d7FXSIERjw0roxMfDlCne2DTjx3tDD8+Ycew6KSlHg7764T9/4omtdjcpEWmeYMK9O7DVb74IOL32SmZ2M3AbkASMDrQhM5sKTAXo1avX8dYqoZCWBn/5C8yd6x25HzgAmzcffWzZ4j0XFnpdK/0lJR0b9rWns7O9q2ZFJOyCaZa5BDjXOfe9qvmrgKHOuVvqWf+KqvWvaWi7apaJAAcOHA376mf/ncCuXXXfk5ICHTt6bfzVj9rz9S3LzGy12w+KRKpQNssUAf53jegBbG9g/QXAE0FsV9q69HSvTb5v38Cvl5Z6Ib9lC8ye7Q1RPHSoNxzC3r3eY9cu+PTTo/MVFY1/ZocO3nNZmXclbnb20fDPzDw6HWhZaqq+NxAhuHBfDvQ2s1xgG3A5cIX/CmbW2zm3vmr2AmA9Ev1SU72biJx6qjf0wTe/6bXrZ9Uz0p1zcPDg0aDfs+fodO3Hu+96Zwjl5d7ZwN69sG+fdwerhiQk1A38zExvG0VFXo1du9bdQfivq7MHiQKNhrtzzmdm04AlQDwwxzm32sxmAIXOuUXANDM7GygH9gANNslIFMrKgjsaGZvazDsiT0+HHj0aXrekxPtewH9n4Zx3trBv39Gw95+u/Vw9vWEDbN3qzb/9duM/S2pq4OCv/Rwf7+2ELr7Y+96h+rX0dJ09SNhpyF+JDdU7i6uv9r4YDmanUN/zkSMNf1ZcHLRvH3xTUu3p9u29q421g5AAQtnmLhL5ap9ZdOzY9G2VlXlBv2mTN2DbiBHeWUV9O4h9+7zvJaqXffVV48Mzx8d7Id/cR3q6uq/GKIW7yPFKSfEeJ57oXfF7vJzzeiLV3hHMmwfz58P550N+vrcT8H8UF8Nnnx2dLy0N7vPS0rydWXXgV58dBDNd/ZyWpjOJCKNwF2ltZpCR4T16+nVEO/10GDiw4S+l/fl8sH9/3Z1A9ePFF+Hll71t9u7tLdu3z/siu/pM4quvvC+5GxMXd/Qs4MQTvZ1FRoYX/NU/S+35+l5LS9PZRCtQm7tItAr0pXQgPp93JlEd9tU7Af8dxb593o1g/vlPbzjprl2P7lj27z/6aKyrK3g7t7S0o98tHD4MvXp5dxnLyPB2ItU7hGCmY2xnoTZ3kVgXTA8m8LqPVl9I1pDGdhbOwaFDx4Z97fCvPf/ee15TU2Ki18y0f7+3o9m/P/hmJ/ACPiPj6M7ipJOgU6djzxyCfUTJtRIKdxEJTmM7CzMvGFNTvaabYDS0w6io8JqM/HcG1cFfe7p6/r334MMPvWEwysqOfT3YnUV1E1T1zqKszOvq2qnT0a68DT2qzyz8H8nJrb7DULiLSPg0tMPw7zEUrMZ2FrV3CPXtKKofhYXedRIJCd46tR/Bio8/GvQpKd73K48+Gtx3K02kcBeR6NHYzqL6WoJgNbSzqKz0mqEChX71DiPQo7DQG7wvPz+4ZrMmUriLiNSnoZ1FXJzX1p+WFnwzFBy7w2hBCncRkdYU7BfdzRQ7/YdERGKIwl1EJAop3EVEopDCXUQkCincRUSikMJdRCQKKdxFRKJQ2EaFNLNiYEsT354FlISwnJbQ1mts6/WBagyFtl4ftP0a21p9JznnshtbKWzh3hxmVhjMkJfh1NZrbOv1gWoMhbZeH7T9Gtt6ffVRs4yISBRSuIuIRKFIDffZ4S4gCG29xrZeH6jGUGjr9UHbr7Gt1xdQRLa5i4hIwyL1yF1ERBoQceFuZmPMbJ2ZbTCz6eGux5+Z9TSzpWa21sxWm9kPwl1Tfcws3szeN7OXwl1LIGbWwcyeNbNPqn6f3wx3Tf7M7EdV/8Yfm9l8M0tpAzXNMbNdZvax37JOZvaqma2veu7YBmt8qOrfeZWZPW9mjdzMtXXr83vtdjNzZtZyt08KoYgKdzOLB2YB5wF9gIlm1ie8VR3DB/ync+5UYBhwcxurz98PgLXhLqIBjwKvOOe+AQygDdVqZt2BW4EC51w/IB64PLxVAfBHYEytZdOB151zvYHXq+bD6Y/UrfFVoJ9zrj/wKfCT1i7Kzx+pWx9m1hP4NvB5axfUVBEV7sBQYINzbqNz7giwABgX5ppqOOd2OOdWVk3vxwuk7uGtqi4z6wFcAPwh3LUEYmbtgTOB/wfgnDvinNsb3qrqSADamVkCkApsD3M9OOeWAV/WWjwOeLpq+mngolYtqpZANTrn/s8556ua/TfQo9ULO1pLoN8hwG+AHwMR8yVlpIV7d2Cr33wRbTA8AcwsBxgIvBveSgKaifcftTLchdQjDygG5lY1Hf3BzNLCXVQ159w24GG8o7gdwD7n3P+Ft6p6neic2wHewQdwQpjracy1wN/DXYQ/MxsLbHPOfRjuWo5HpIW7BVjW5vakZpYOPAf80Dn3Vbjr8Wdm3wF2OedWhLuWBiQAg4AnnHMDgYOEvzmhRlW79TggF+gGpJnZpPBWFfnM7C68ps154a6lmpmlAncB94S7luMVaeFeBPT0m+9BGzgd9mdmiXjBPs8597/hrieA4cBYM9uM16w12syeCW9JdRQBRc656rOeZ/HCvq04G9jknCt2zpUD/wv8R5hrqs8XZtYVoOp5V5jrCcjMrgG+A1zp2lb/7K/h7cQ/rPqb6QGsNLMuYa0qCJEW7suB3maWa2ZJeF9iLQpzTTXMzPDaidc65x4Jdz2BOOd+4pzr4ZzLwfv9/cM516aOOp1zO4GtZnZK1aKzgDVhLKm2z4FhZpZa9W9+Fm3oC99aFgHXVE1fA7wYxloCMrMxwJ3AWOdcabjr8eec+8g5d4JzLqfqb6YIGFT1f7RNi6hwr/rSZRqwBO+PaaFzbnV4qzrGcOAqvKPhD6oe54e7qAh1CzDPzFYB+cCvwlxPjaozimeBlcBHeH9HYb+K0czmA/8CTjGzIjO7DngA+LaZrcfr7fFAG6zxd0AG8GrV38yTbay+iKQrVEVEolBEHbmLiEhwFO4iIlFI4S4iEoUU7iIiUUjhLiIShRTuIiJRSOEuIhKFFO4iIlHo/wNeHGQH4suMIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.001,epochs=12,bs=512,beta1=0.9,beta2=0.99)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reach 87% accuracy in only 16 epochs!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8765"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accuracy on test set\n",
    "y_pred=nn.predict(X_test)\n",
    "accuracy(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since Adam results in great overfitting, let's include other regularization (l2 and dropout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adam + Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "act_obj = ReLU()\n",
    "layers = [X_train.shape[1],100,10]\n",
    "nn = CustomNeuralNetwork(layers,act_obj,opt=Adam,keep_prob=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 1.0317066498215524, Val loss:0.4802829847588148\n",
      "Epoch 2. Training loss: 0.4599599231073201, Val loss:0.4201392364568962\n",
      "Epoch 3. Training loss: 0.4065128454115538, Val loss:0.3985676814960054\n",
      "Epoch 4. Training loss: 0.3775357577247236, Val loss:0.3972471776590397\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl81NW9//HXJyEhQIBAEoQCClgQQ4gQI+IDK7hc63LFVq2CO1apWkWLqNQVcSkqIuK11qVSr1qpV2uliqW/9iJe22oNLmGXiFhStiQssgWY5Pz++E5ilkkySSb5zkzez8djHrOdzHy+jL7nzPme7/macw4REYkvCX4XICIikadwFxGJQwp3EZE4pHAXEYlDCncRkTikcBcRiUMKdxGROKRwFxGJQwp3EZE41MGvN87IyHADBgzw6+1FRGLSsmXLSpxzmY218y3cBwwYQH5+vl9vLyISk8zs63DaaVhGRCQOKdxFROKQwl1EJA75NuYuIm3r0KFDFBUVUVZW5ncpEoaUlBT69etHUlJSs/5e4S7SThQVFdG1a1cGDBiAmfldjjTAOUdpaSlFRUUMHDiwWa+hYRmRdqKsrIz09HQFewwwM9LT01v0K0vhLtKOKNhjR0s/q9gL96VL4ZRToLjY70pERKJW7IX7nDmwZAnceqvflYhIE5SWljJixAhGjBhB79696du3b9X9gwcPhvUakyZNYu3atQ22eeqpp3jllVciUTInnngin332WUReq63F3g7VX/0KPvrIC/j9+6FTJ78rEpEwpKenVwXljBkzSE1NZdq0aTXaOOdwzpGQELrfOX/+/Ebf56c//WnLi40Dsddz79MHfvtb+Ne/4LHH/K5GRFqosLCQ7Oxsrr32WnJzc9m8eTOTJ08mLy+PYcOGMXPmzKq2lT3pQCBAWloa06dP55hjjuGEE05g27ZtANx1113MnTu3qv306dMZNWoURx11FH//+98B2Lt3L+effz7HHHMMEydOJC8vr9Ee+ssvv8zw4cPJzs7mjjvuACAQCHDZZZdVPT5v3jwAHn/8cbKysjjmmGO49NJLI/5vFo7Y67mDN+Z+wQXw0ENwxRXQv7/fFYnElptvhnCGGw4dgi1boHdvaGy+9YgREAzVplq1ahXz58/nV7/6FQCzZs2iZ8+eBAIBTj75ZC644AKysrJq/M2uXbsYO3Yss2bNYurUqbzwwgtMnz69zms75/jnP//JwoULmTlzJn/605948skn6d27N2+88Qaff/45ubm5DdZXVFTEXXfdRX5+Pt27d+e0007j7bffJjMzk5KSEpYvXw7Azp07AXjkkUf4+uuvSU5OrnqsrcVez73S7Nneda2fdSISQVu2wPr13nUrOvLIIznuuOOq7r/66qvk5uaSm5vL6tWrWbVqVZ2/6dSpE2eeeSYAxx57LBs2bAj52uedd16dNh988AETJkwA4JhjjmHYsGEN1vfRRx9xyimnkJGRQVJSEhdffDHvv/8+3/3ud1m7di033XQTixcvpnv37gAMGzaMSy+9lFdeeaXZByG1VGz23AGOOAKmT4d774XrroNx4/yuSCR2hNvDLimB+fNh0iTIyGi1crp06VJ1e926dTzxxBP885//JC0tjUsvvTTkfO/k5OSq24mJiQQCgZCv3bFjxzptnHNNqq++9unp6RQUFPDuu+8yb9483njjDZ599lkWL17M0qVLeeutt3jggQdYsWIFiYmJTXrPlmq0525mL5jZNjNbUc/zZmbzzKzQzArMrOHfN5F0661eyE+ZAvV8sCLSAhkZ3v9nrRjstX3zzTd07dqVbt26sXnzZhYvXhzx9zjxxBN57bXXAFi+fHnIXwbVjR49miVLllBaWkogEGDBggWMHTuW4uJinHP86Ec/4r777uOTTz6hvLycoqIiTjnlFB599FGKi4vZt29fxLehMeH03H8D/Bfw3/U8fyYwOHg5Hng6eN36OnXypkaef743i+aGG9rkbUWk9eTm5pKVlUV2djaDBg1izJgxEX+PG2+8kcsvv5ycnBxyc3PJzs6uGlIJpV+/fsycOZNx48bhnOOcc87h7LPP5pNPPuHHP/4xzjnMjIcffphAIMDFF1/M7t27qaio4Pbbb6dr164R34bGWDg/T8xsAPC2cy47xHPPAO85514N3l8LjHPObW7oNfPy8lxETtbhHJx+OuTnw7p1bdrDEIklq1ev5uijj/a7jKgQCAQIBAKkpKSwbt06Tj/9dNatW0eHDtE1Uh3qMzOzZc65vMb+NhJb0hfYWO1+UfCxBsM9YszgiScgJwfuusvrwYuINGDPnj2ceuqpBAIBnHM888wzURfsLRWJrQm1AELInwNmNhmYDHD44YdH4K2DsrLgxhu9kJ88GRqZ1iQi7VtaWhrLli3zu4xWFYmpkEVA9Ynm/YBNoRo65551zuU55/IyMxs9v2vTzJgBmZleyDdxT7iISLyJRLgvBC4PzpoZDexqbLy9VXTvDr/4Bfz9794RrCIi7Vg4UyFfBf4BHGVmRWb2YzO71syuDTZZBKwHCoHngOtbrdrGXHklHHecN3Vr927fyhAR8VujY+7OuYmNPO+A6FipJyEBnnwSRo+GBx+EWbP8rkhExBexu/xAfY4/3uvBz5njTY0Ukagwbty4OgckzZ07l+uvb/jHfmpqKgCbNm3iggsuqPe1G5taPXfu3BoHE5111lkRWfdlxowZzK5cDiWKxF+4gzf2npLiLY4kIlFh4sSJLFiwoMZjCxYsYOLEBgcHqnznO9/h9ddfb/b71w73RYsWkZaW1uzXi3bxGe69e3uzZxYtgnfe8bsaEQEuuOAC3n77bQ4cOADAhg0b2LRpEyeeeGLVvPPc3FyGDx/OW2+9VefvN2zYQHa2dxzl/v37mTBhAjk5OVx00UXs37+/qt11111XtVzwvffeC8C8efPYtGkTJ598MieffDIAAwYMoKSkBIA5c+aQnZ1NdnZ21XLBGzZs4Oijj+aaa65h2LBhnH766TXeJ5TPPvuM0aNHk5OTww9/+EN27NhR9f5ZWVnk5ORULVi2dOnSqpOVjBw5kt0R3k8YX7P2q7vhBnjuOa/3ftppEFw8SETg5j/dzGdbGl/y91D5Ibbs2ULv1N4kJTa8uuGI3iOYe0b9C5Klp6czatQo/vSnP3HuueeyYMECLrroIsyMlJQU3nzzTbp160ZJSQmjR49m/Pjx9Z5H9Omnn6Zz584UFBRQUFBQY8neBx98kJ49e1JeXs6pp55KQUEBU6ZMYc6cOSxZsoSMWkexL1u2jPnz5/PRRx/hnOP4449n7Nix9OjRg3Xr1vHqq6/y3HPPceGFF/LGG280uD775ZdfzpNPPsnYsWO55557uO+++5g7dy6zZs3iq6++omPHjlVDQbNnz+app55izJgx7Nmzh5SUlAb/fZsqPnvuAMnJ3kFNhYXw+ON+VyMSk7bs2cL6nevZsicyS/5WH5qpPiTjnOOOO+4gJyeH0047jX//+99s3bq13td5//33q0I2JyeHnJycqudee+01cnNzGTlyJCtXrmx0UbAPPviAH/7wh3Tp0oXU1FTOO+88/u///g+AgQMHMmLECKDhZYXBW19+586djB07FoArrriC999/v6rGSy65hJdffrnqSNgxY8YwdepU5s2bx86dOyN+hGz89tzBW3PmBz+ABx6Ayy6Dvn39rkgkKjTUw66uZF8J8z+dz6SRk8jo3PJ1m37wgx8wdepUPvnkE/bv31/V437llVcoLi5m2bJlJCUlMWDAgJDL/FYXqlf/1VdfMXv2bD7++GN69OjBlVde2ejrNLS+Vsdqv/gTExMbHZapzzvvvMP777/PwoULuf/++1m5ciXTp0/n7LPPZtGiRYwePZq//OUvDB06tFmvH0r89twrPfaYtxzw7bf7XYlIzMnonMGtY26NSLCDN/Nl3LhxXHXVVTV2pO7atYtevXqRlJTEkiVL+Prrrxt8nZNOOqnqJNgrVqygoKAA8JYL7tKlC927d2fr1q28++67VX/TtWvXkOPaJ510En/4wx/Yt28fe/fu5c033+R73/tek7ete/fu9OjRo6rX/9JLLzF27FgqKirYuHEjJ598Mo888gg7d+5kz549fPnllwwfPpzbb7+dvLw81qxZ0+T3bEh899wBBg3yDmp64AG49lo48US/KxJp1yZOnMh5551XY+bMJZdcwjnnnENeXh4jRoxotAd73XXXMWnSJHJychgxYgSjRo0CvLMqjRw5kmHDhtVZLnjy5MmceeaZ9OnThyVLllQ9npuby5VXXln1GldffTUjR45scAimPi+++CLXXnst+/btY9CgQcyfP5/y8nIuvfRSdu3ahXOOn/3sZ6SlpXH33XezZMkSEhMTycrKqjqrVKSEteRva4jYkr/h2LcPhg6F9HRvaeA2PiOKSDTQkr+xpyVL/sb/sAxA587eOVc/+wyef97vakREWl37CHeAH/3IO8/qnXfC9u1+VyMi0qraT7ibwbx5sGMH3HOP39WI+MKvYVhpupZ+Vu0n3AGGD4frr4enn4bg3nWR9iIlJYXS0lIFfAxwzlFaWtqiA5vaxw7V6rZvhyFDYNgweO89r0cv0g4cOnSIoqKiRud9S3RISUmhX79+JCXVPDK4Lc+hGlt69oSHHoKf/AReew0uusjvikTaRFJSEgMHDvS7DGkj7WtYptKPfwwjR8K0abB3r9/ViIhEXPsM98RE76QeRUXe8sAiInGmfYY7wJgxcOml8Oij8OWXflcjIhJR7TfcAR5+GJKS4JZb/K5ERCSi2ne4f+c7cPfd8NZbUOv0XyIisax9hzt4J/MYPBhuugkOHvS7GhGRiAgr3M3sDDNba2aFZjY9xPNHmNlfzazAzN4zs36RL7WVdOwIc+fC2rXeTlYRkTjQaLibWSLwFHAmkAVMNLOsWs1mA//tnMsBZgKxNQXlrLPg7LPhvvtg82a/qxERabFweu6jgELn3Hrn3EFgAXBurTZZwF+Dt5eEeD76zZ0LBw7Az3/udyUiIi0WTrj3BTZWu18UfKy6z4Hzg7d/CHQ1s/SWl9eGvvtdmDoVXnwRPvzQ72pERFoknHAPtfhK7QVppgFjzexTYCzwbyBQ54XMJptZvpnlFxcXN7nYVnfnnd4MmhtvhIoKv6sREWm2cMK9COhf7X4/YFP1Bs65Tc6585xzI4E7g4/tqv1CzrlnnXN5zrm8zMzMFpTdSlJTvYOa8vNh/ny/qxERabZwwv1jYLCZDTSzZGACsLB6AzPLMLPK1/o58EJky2xDEyd6R6/+/Oewc6ff1YiINEuj4e6cCwA3AIuB1cBrzrmVZjbTzMYHm40D1prZF8BhwIOtVG/rM/OmRJaUwIwZflcjItIs7W8993Bddx089xx8/rm39ruISBTQCbJb6v77oVs378hVnblGRGKMwr0+GRlewP/1r/D73/tdjYhIkyjcG/KTn0BOjjf/fd8+v6sREQmbwr0hHTrAvHnwr395UyRFRGKEwr0xY8d651mdNQs2bPC7GhGRsCjcw/Hoo5CQ4J1zVUQkBijcw9G/P9xxB7zxhreDVUQkyincw3XLLTBoEEyZAocO+V2NiEiDFO7hSkmBxx+HVavgl7/0uxoRkQYp3JvinHPg+9+He++Fbdv8rkZEpF4K96Yw807qsXevNwYvIhKlFO5NNXSod1LtF16Ajz/2uxoRkZAU7s1x993Qq5e3c1Un9RCRKKRwb45u3eDhh73T8b30kt/ViIjUoXBvrssug9Gj4fbb4Ztv/K5GRKQGhXtzJSR4685s2+atHikiEkUU7i1x3HFw1VXeDJo1a/yuRkSkisK9pR56CLp00Uk9RCSqKNxbqlcvuO8++POf4Y9/9LsaERFA4R4Z118PWVnws59BWZnf1YiIKNwjIinJ27m6fj089pjf1YiIKNwj5tRT4fzzvTH4jRv9rkZE2rmwwt3MzjCztWZWaGbTQzx/uJktMbNPzazAzM6KfKkxYPZs74jVW2/1uxIRaecaDXczSwSeAs4EsoCJZpZVq9ldwGvOuZHABKB9rok7YABMnw6/+x0sXep3NSLSjoXTcx8FFDrn1jvnDgILgHNrtXFAt+Dt7sCmyJUYY267DY44wlt3JhDwuxoRaafCCfe+QPVB5KLgY9XNAC41syJgEXBjqBcys8lmlm9m+cXFxc0oNwZ06uTtVC0ogGee8bsaEWmnwgl3C/FY7aN1JgK/cc71A84CXjKzOq/tnHvWOZfnnMvLzMxserWx4rzzvB2sd98NJSV+VyMi7VA44V4E9K92vx91h11+DLwG4Jz7B5ACZESiwJhkBk884S0odvfdflcjIu1QOOH+MTDYzAaaWTLeDtOFtdr8CzgVwMyOxgv3OB13CdOwYXDDDd7QzKef+l2NiLQzjYa7cy4A3AAsBlbjzYpZaWYzzWx8sNktwDVm9jnwKnClc1pohRkzICMDbrxR686ISJsyvzI4Ly/P5efn+/LeberXv4arr4ZXXoGLL/a7GhGJcWa2zDmX11g7HaHa2iZNgrw878CmPXv8rkZE2gmFe2tLSIAnn4RNm+DBB/2uRkTaCYV7Wxg9Gq64wpv/vm6d39WISDugcG8rs2ZBSoq3LLCISCtTuLeV3r3h3nvhnXe8i4hIK1K4t6Ubb4SjjoKbb4YDB/yuRkTimMK9LSUne0euFhZ6J9UWEWklCve29v3vw7nnwv33ezNoRERagcLdD3PmeMsB33ab35WISJxSuPth0CCYNs07avVvf/O7GhGJQwp3v/z859Cvn7eTtbzc72pEJM4o3P3SpYt3ztVPP4Xnn/e7GhGJMwp3P114IYwdC3feCdu3+12NiMQRhbufzGDePNixwzvASUQkQhTufsvJgeuug1/+0jvvqohIBCjco8HMmdCjB0yZopN6iEhEKNyjQc+e3nLAS5fC//yP39WISBxQuEeLq6+GESPglltg716/qxGRGKdwjxaJid5JPYqKvOWBRURaQOEeTU48ES65BB59FNav97saEYlhCvdo8/DD0KEDTJ3qdyUiEsPCCnczO8PM1ppZoZlND/H842b2WfDyhZntjHyp7UTfvnDXXfDWW7B4sd/ViEiMMtfI1DszSwS+AP4DKAI+BiY651bV0/5GYKRz7qqGXjcvL8/l5+c3q+i4d+AAZGd74/AFBd468CIigJktc87lNdYunJ77KKDQObfeOXcQWACc20D7icCr4ZUpIXXs6J3MY+1abyeriEgThRPufYGN1e4XBR+rw8yOAAYC/1vP85PNLN/M8ouLi5taa/ty9tlw1llw332wZYvf1YhIjAkn3C3EY/WN5UwAXnfOhVzD1jn3rHMuzzmXl5mZGW6N7dfcuVBW5i0PLCLSBOGEexHQv9r9fkB954ebgIZkImfwYG/WzG9+Ax9+6Hc1IhJDwgn3j4HBZjbQzJLxAnxh7UZmdhTQA/hHZEts5+68E/r08U7qUVHhdzUiEiMaDXfnXAC4AVgMrAZec86tNLOZZja+WtOJwALX2PQbaZquXb2DmvLzvR68iEgYGp0K2Vo0FbIJnPOOXl23Dr74AtLS/K5IRHwSyamQ4jczb0pkSYk3e0ZEpBEK91iRmwuTJ3shvyrk8WMiIlUU7rHkgQe8MXid1ENEGqFwjyUZGXD//fDXv8Kbb/pdjYhEMYV7rLn2Whg+3Jv/vn+/39WISJRSuMeaDh1g3jz4+mt45BG/qxGRKKVwj0XjxsGFF3pnbPr6a7+rEZEopHCPVbNne1Mkp03zuxIRiUIK91jVvz/ccQe8/rq3g1VEpBqFeyybNg0GDoSbboJDh/yuRkSiiMI9lqWkwOOPw8qV8PTTflcjIlFE4R7rxo+H00+He+6Bbdv8rkZEooTCPdaZwRNPwN693vLAIiIo3OPD0KHeuPuvf+0tDSwi7Z7CPV7ccw/06qWTeogIoHCPH926eQc1ffghvPyy39WIiM8U7vHk8svh+OPh9tvhm2/8rkZEfKRwjycJCd5671u2eKtHiki7pXCPN8cdB1ddBXPnwpo1flcjIj5RuMejX/wCOneGm2/WST1E2imFezzq1cs71+rixfDHP/pdjYj4IKxwN7MzzGytmRWa2fR62lxoZqvMbKWZ/TayZUqT/fSnkJUFP/sZlJX5XY2ItLFGw93MEoGngDOBLGCimWXVajMY+Dkwxjk3DLi5FWqVpkhK8o5cXb8e5szxuxoRaWPh9NxHAYXOufXOuYPAAuDcWm2uAZ5yzu0AcM5pkZNocNppcN558OCDsHGj39WISBsKJ9z7AtWToSj4WHVDgCFm9jcz+9DMzohUgdJCjz3mHbF6221+VyIibSiccLcQj9WegtEBGAyMAyYCz5tZWp0XMptsZvlmll9cXNzUWqU5BgzwDmpasADef9/vakSkjYQT7kVA/2r3+wGbQrR5yzl3yDn3FbAWL+xrcM4965zLc87lZWZmNrdmaarbboPDD/fWnQkE/K5GRNpAOOH+MTDYzAaaWTIwAVhYq80fgJMBzCwDb5hmfSQLlRbo3NkbnikogGef9bsaEWkDjYa7cy4A3AAsBlYDrznnVprZTDMbH2y2GCg1s1XAEuBW51xpaxUtzXD++XDKKXDXXVCqj0Yk3pnz6QjGvLw8l6+1x9vWihUwYgRcc41OyycSo8xsmXMur7F2OkK1PcnO9g5ueuYZ+PRTv6sRkVakcG9v7rsP0tNhyhStOyMSxxTu7U1amrew2AcfwKuv+l2NiLQShXt7NGkSHHss3Hor7NnjdzUi0goU7u1RYqJ3Uo9Nm+Chh/yuRkRagcK9vTrhBO+0fI89BoWFflcjIhGmcG/PZs2Cjh29ZYFFJK4o3NuzPn3gnnvg7bdh0SK/qxGRCFK4t3dTpsCQId4p+Q4c8LsaEYkQhXt7l5zsndRj3TrvWkTigsJd4IwzYPx4uP9+bwaNiMQ8hbt45syBgwe9td9FJOYp3MVz5JEwbRq8/DL87W9+VyMiLaRwl2/dcQf07evtZC0v97saEWkBhbt8q0sXmD0bPvkEfv1rv6sRkRZQuEtNF10EJ53k9eJ37PC7GhFpJoW71GQG8+Z5wX7vvX5XIyLNpHCXuo45Bq69Fn75S1i+3O9qRKQZFO4S2syZ0L27TuohEqMU7hJaejo8+CC89x68/rrf1YhIEyncpX7XXOMN0dxyC+zd63c1ItIEYYW7mZ1hZmvNrNDMpod4/kozKzazz4KXqyNfqrS5ypN6bNwIDz/sdzUi0gSNhruZJQJPAWcCWcBEM8sK0fR3zrkRwcvzEa5T/PK978HFF8Mjj8BXX/ldjYiEKZye+yig0Dm33jl3EFgAnNu6ZUlUeeQR6NABpk71uxIRCVM44d4X2FjtflHwsdrON7MCM3vdzPpHpDqJDn37wp13wh/+AH/+s9/ViEgYwgl3C/FY7blxfwQGOOdygL8AL4Z8IbPJZpZvZvnFxcVNq1T8NXWqt7jYTTfBoUN+VyMijQgn3IuA6j3xfkCNRb+dc6XOucrT+DwHHBvqhZxzzzrn8pxzeZmZmc2pV/zSsSPMnQtr1ng7WUUkqoUT7h8Dg81soJklAxOAhdUbmFmfanfHA6sjV6JEjbPPhjPPhBkzYMsWv6sRkQY0Gu7OuQBwA7AYL7Rfc86tNLOZZjY+2GyKma00s8+BKcCVrVWw+MjM672XlXkLi4lI1DLn06HleXl5Lj8/35f3lha6/XZvBs2HH8Lxx/tdjUi7YmbLnHN5jbWLuSNU5/xjDhmPZPCfv/1PZrw3g9+v/j2F2wupcBV+l9Z+3HUX9OkDN94IFfp3F4lGHfwuoKlWbltJ6f5SPvr3RyxatwgXnLjTJakL2b2yyTksh5zDchjeazjDDxtOz049fa44DnXt6vXcL7sMXnwRJk3yuyIRqSXmhmVK9pUw/9P5TBo5ic5JnVm5bSXLty2nYGsBBVsL+Hzr52zfv72qfb9u/bzA75VTFfxD0oeQlJgUyc1pf5yDMWPgyy/hiy+8FSRFpNWFOywTc+HeGOccm/dsrgr7yuBfXbyaQxXe/OzkxGSOzji6KuwrL4d1OQyzUNP6JaRPPoG8PLj5Zpgzx+9qRNqFdhvu9TlYfpC1JWurQr9gm3e9afe3U/YzOmfU6eVnZWbRKalTm9UZc37yE3jhBfj8c8gKteSQiESSwj1MpftKawzrFGwtYMW2FewP7AcgwRIYkj6E4b2G1+jlH9H9CPXyAYqLYcgQrwf/5z970yVFpNUo3FugvKKc9TvW1+nlr9+xvqpNt47dagR+5Q7cbh27+Vi5T5580jtj0/DhMH48HHssHH20t1xBkvZtiESSwr0V7D6wmxXbVtTp6e86sKuqzYC0ATWGdoYfNpzBPQeTmJDoY+WtLBCAE06A2p9nhw4weDAMHeqFfeXlqKMgNdWfWkVinMK9jTjn2PjNxjo7cNeWrKXclQOQ0iGFYZnDagzrDO81nMwucbS+TkkJzJ8PP/qRN1SzerV3WbPGuy4shPLyb9v37/9t2FcP/8xMDe2INEDh7rOyQBmri1fXCf2te7dWtemd2rvODtyhGUPp2KGjj5W3koMHvYCvDPvq4b9v37ftevas29MfOhQGDICEmDvmTiTiFO5RauuerXWGdVYVr+JAubeoZoeEDgzNGFpnB27frn3jcwduRQUUFdXt6a9e7f0CqJSS4g3n1O7pDx7sPSfSTijcY0igIsC60nV1duD+a9e/qtr0SOlRY0gn57Acsntl0yW5i4+Vt7LS0tA9/Q0bvIOowOvNDxxYt6d/9NGQluZr+SKtQeEeB3aW7WTFthU1evnLty1nz8E9ABjGkT2PrBraGX6YF/qDegwiweJ4CGPfPu+o2No9/S++8IZ/KvXuXbenf/TR8J3vaFxfYpbCPU5VuAo27NzgBf3W5VW9/HWl6+pdZ6eyt9+jUw+fq29lgYDXq6/e06+8fPPNt+26dq0Z+JW3jzzSm+EjEsUU7u3MvkP7WLltZY2dt1pnJ8g57+Qiocb1N1U7qVhSUv1TN7vE8fCXxBSFu9RZZ6fysqZkjdbZqbRrF6xdW3dc/8sva07dPPzw+qduirQhhbvUK5x1djI7Z9bYedvu1tlTY2lcAAAJ10lEQVQ5cMCbulm7p79mDezf/2279PTQUzePOEJTN6VVKNylycJdZ6f2Dtx2tc5ORQVs3Fi3p796tXcgV6VOneqfutkxDo9jkDajcJeIKK8o58sdX3o7bxtYZ+eo9KM4EDjAiN4j6J3amx6depCWkkZaSho9Urzb1R9LTkz2cataSUlJ3Z7+6tXw9dfftklIgEGDQk/d1Jr4EgaFu7SqynV2KnfgLlq3iK92fkVqUioHKw5ysPxgg3/fOalzyNCveqzWc5X301LS6Nqxa2xN9dy3L/S4/hdfwKFD37br0yf01M0+fTR1U6oo3KVNVT9DVnqndMoCZewo28HOsp3s2B+8Dt4P9Vj1ttUXYgslwRLo3rF7neBv8MuiWtuoWd4hEID160P39nfv/rZdt26hp24OGqSpm+1QRMPdzM4AngASgeedc7PqaXcB8D/Acc65BpNb4S71Ka8oZ/fB3XW/AEJ9SYR4rixQ1uDrp3RIafavhu4p3Vv/V4NzsHlz6HH9zZu/bZecHHrq5pAhmroZxyIW7maWCHwB/AdQBHwMTHTOrarVrivwDpAM3KBwF7+UBcqqwj/kr4R6fjVUXipcRb2vbRjdOnZr9q+GlA4pLdv5vHOnF/S1e/rr13s7eysdcYR3UFZZGZx0krdEQ69e3uWww7zr1FQN98SgcMM9nN90o4BC59z64AsvAM4FVtVqdz/wCDCtibWKRFRKhxR6p/amd2rvJv9thatgz8E99X4hhPpS+KL0i6r7+w7ta/D1kxOTm/2rIS0ljcS0NBg92rtUd+AArFtXs6e/dKl3kNbf/17PP1TKt4Ef6lL5JdCrlzefXydeiSnhhHtfYGO1+0XA8dUbmNlIoL9z7m0zU7hLzEqwBLp17Ea3jt04giOa/PcHyw+yq2xX/b8SKr8kDni3S/eVUri9sOq5ynMA1Kdrcte6vxo69SCtY/D68DTShgygx0UjSTswkdR3/4qdeRbmwHbswEpLse07sO3bsdLt316XbsA2LPNuHzrktYea1927Y+npWHqGd8movGRimZlYRiYJmb28+2lpmCVgZhjWomtpnnDCPdS/btVYjpklAI8DVzb6QmaTgckAhx9+eHgVisSQ5MRkMrtkNutELM459h7aG/pXQj1DSet3rGfnZq/t7oO7Q7/wm0+EfrxL8NI/3Ap3BS/raz5cGry0opZ+QUTTdaAiQM5hOcz5/hwyOme02r9ZOOFeRM2Pvx9QbUEOugLZwHvBb9newEIzG1973N059yzwLHhj7i2oWyTumBmpyamkJqfSv3vYiVslUBGo8avhmfxneP7T57k853LOHXouzjkcLqLXFYFDuL17cLu/wX3zDW73bu/2nt24PXuC19VuHwrgzOsd1rjumIxLTcV16YJL7RK87ozr3AXXpbN36Ry8pHTEmUV8W2pct+Jrf7n9S14qeInhvYZz65hbI/7fUaVwwv1jYLCZDQT+DUwALq580jm3C6j6+jGz94Bpje1QFZHI6pDQgfTO6aR3TgdgwGkDGJI+hEkjJ7VqD7FJ9u6Fbdtg61bvuvZlS+XtIu9kLRUhdm4nJEBGRs19Ag1doux8vdWnDbemcKdCngXMxZsK+YJz7kEzmwnkO+cW1mr7HmGEu2bLiEiDysthx476vwhqX6ov61xd587hfQkcdpj3pRHlxw7oICYRaV/KyrzefrhfBtWPDq4uPT38WUTdurX5dNJIToUUEYl+KSnQv793aYxz3nLPjX0RLF/uXW/fHvp1kpPD+xKonE7ahovGKdxFpP0x886xm5bmrd7ZmEOHvIXhGvsyWL3aa1NWz1HS3bt7vwxGj4YnnvCGgVqJwl1EpDFJSd4Cbn36NN7WuYZ3HC9dCr/9LYwYAbf6O1tGRETCZebN0ElN9RZ3q62kBObPh0mtO1tG4S4i0pYyMlq1x14phhbFFhGRcCncRUTikMJdRCQOKdxFROKQwl1EJA4p3EVE4pDCXUQkDvm2cJiZFQNfN/PPM4CSCJbjJ21L9ImX7QBtS7RqybYc4Zxr9GwwvoV7S5hZfjirosUCbUv0iZftAG1LtGqLbdGwjIhIHFK4i4jEoVgN92f9LiCCtC3RJ162A7Qt0arVtyUmx9xFRKRhsdpzFxGRBkR1uJvZGWa21swKzWx6iOc7mtnvgs9/ZGYD2r7K8ISxLVeaWbGZfRa8XO1HnY0xsxfMbJuZrajneTOzecHtLDCz3LauMVxhbMs4M9tV7TO5p61rDIeZ9TezJWa22sxWmtlNIdrExOcS5rbEyueSYmb/NLPPg9tyX4g2rZdhzrmovACJwJfAICAZ+BzIqtXmeuBXwdsTgN/5XXcLtuVK4L/8rjWMbTkJyAVW1PP8WcC7gAGjgY/8rrkF2zIOeNvvOsPYjj5AbvB2V+CLEP99xcTnEua2xMrnYkBq8HYS8BEwulabVsuwaO65jwIKnXPrnXMHgQXAubXanAu8GLz9OnCqWRufijw84WxLTHDOvQ/Uc7ZgwNuu/3aeD4E0Mwvj3GRtL4xtiQnOuc3OuU+Ct3cDq4G+tZrFxOcS5rbEhOC/9Z7g3aTgpfZOzlbLsGgO977Axmr3i6j7IVe1cc4FgF1AeptU1zThbAvA+cGfzK+bWRincI9K4W5rrDgh+LP6XTMb5ncxjQn+rB+J10usLuY+lwa2BWLkczGzRDP7DNgG/D/nXL2fS6QzLJrDPdS3V+1vvXDaRINw6vwjMMA5lwP8hW+/zWNNrHwm4fgE71DvY4AngT/4XE+DzCwVeAO42Tn3Te2nQ/xJ1H4ujWxLzHwuzrly59wIoB8wysyyazVptc8lmsO9CKjee+0HbKqvjZl1ALoTnT+zG90W51ypc+5A8O5zwLFtVFukhfO5xQTn3DeVP6udc4uAJDPL8LmskMwsCS8MX3HO/T5Ek5j5XBrbllj6XCo553YC7wFn1Hqq1TIsmsP9Y2CwmQ00s2S8nQ0La7VZCFwRvH0B8L8uuGciyjS6LbXGP8fjjTXGooXA5cHZGaOBXc65zX4X1Rxm1rty/NPMRuH9/1Lqb1V1BWv8NbDaOTennmYx8bmEsy0x9Llkmlla8HYn4DRgTa1mrZZhHSLxIq3BORcwsxuAxXizTV5wzq00s5lAvnNuId5/BC+ZWSHet90E/yquX5jbMsXMxgMBvG250reCG2Bmr+LNVsgwsyLgXrwdRTjnfgUswpuZUQjsAyb5U2njwtiWC4DrzCwA7AcmRGnnYQxwGbA8OL4LcAdwOMTc5xLOtsTK59IHeNHMEvG+gF5zzr3dVhmmI1RFROJQNA/LiIhIMyncRUTikMJdRCQOKdxFROKQwl1EJA4p3EVE4pDCXUQkDincRUTi0P8HGz/tzIPmieIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.01,epochs=4,bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 0.32741828418452895, Val loss:0.35779257100722783\n",
      "Epoch 2. Training loss: 0.3116596966332032, Val loss:0.3545784902559664\n",
      "Epoch 3. Training loss: 0.3061841429208155, Val loss:0.35085616372123346\n",
      "Epoch 4. Training loss: 0.3004536535478716, Val loss:0.3477540070598484\n",
      "Epoch 5. Training loss: 0.296861856285127, Val loss:0.3489320359390706\n",
      "Epoch 6. Training loss: 0.29007998916082683, Val loss:0.3458671945045298\n",
      "Epoch 7. Training loss: 0.2874244908314327, Val loss:0.3439858708628181\n",
      "Epoch 8. Training loss: 0.2823114995595829, Val loss:0.3444324558715505\n",
      "Epoch 9. Training loss: 0.27808968076598034, Val loss:0.34208387505123405\n",
      "Epoch 10. Training loss: 0.27477648426464224, Val loss:0.34276998691174715\n",
      "Epoch 11. Training loss: 0.2685794579434328, Val loss:0.34050197555956063\n",
      "Epoch 12. Training loss: 0.2667708705642446, Val loss:0.3401206611421726\n",
      "Epoch 13. Training loss: 0.262950213316813, Val loss:0.34252692820436675\n",
      "Epoch 14. Training loss: 0.25890861714306956, Val loss:0.3400057587332236\n",
      "Epoch 15. Training loss: 0.25430430799999454, Val loss:0.3402890807149194\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xt8VNW99/HPL3cCIYEE5RJSQPECiBBTpEW5eAWsoIgKSFVa5dgWq/VoxR6PWjy23h6LtBytnkdq1QNSeVSqKLWFitZWAYvIRQSBYrgm3CEBclnPH3smTJJJMoFJJjPzfb9e+zX7smbPL5vht9esvdbe5pxDRERiS0KkAxARkfBTchcRiUFK7iIiMUjJXUQkBim5i4jEICV3EZEYpOQuIhKDlNxFRGKQkruISAxKitQH5+TkuG7dukXq40VEotLy5cuLnXMdGioXseTerVs3li1bFqmPFxGJSmb2r1DKqVlGRCQGKbmLiMQgJXcRkRgUsTZ3EWleZWVlFBYWcuTIkUiHIiFIS0sjNzeX5OTkE3q/krtInCgsLCQjI4Nu3bphZpEOR+rhnGP37t0UFhbSvXv3E9qHmmVE4sSRI0fIzs5WYo8CZkZ2dvZJ/cpScheJI0rs0eNk/62iL7m/9RYMGwZFRZGORESkxYq+5P788/DXv8LMmZGOREQaYffu3fTr149+/frRsWNHunTpUrV87NixkPYxadIk1q1bV2+ZmTNn8sorr4QjZC644AJWrFgRln01t+i7oHr11TB/Plx4YaQjEZFGyM7OrkqUDz30EG3atOHuu++uVsY5h3OOhITg9c5Zs2Y1+Dk/+tGPTj7YGBB9Nfc+fbzXgwcjG4eIhMWGDRvo06cPt912G/n5+Wzfvp3JkydTUFBA7969mTZtWlVZf026vLycrKwspk6dyrnnnsu3vvUtdu3aBcD999/P9OnTq8pPnTqVAQMGcOaZZ/LRRx8BcPjwYa655hrOPfdcxo8fT0FBQYM19JdffplzzjmHPn368LOf/QyA8vJyvvvd71atnzFjBgC/+tWv6NWrF+eeey4TJ04M+zELRfTV3PPyvNevv45sHCLR7M47IZTmhrIy2LEDOnaEhvpb9+sHvqTaWGvWrGHWrFk8++yzADz66KO0b9+e8vJyhg0bxtixY+nVq1e19+zfv58hQ4bw6KOPctddd/HCCy8wderUWvt2zvHJJ58wf/58pk2bxrvvvsuvf/1rOnbsyLx58/jss8/Iz8+vN77CwkLuv/9+li1bRmZmJpdccglvvfUWHTp0oLi4mM8//xyAffv2AfD444/zr3/9i5SUlKp1zS36au4dOkBqKmzZEulIRGLfjh2wcaP32oROO+00vvnNb1Ytz549m/z8fPLz81m7di1r1qyp9Z5WrVoxYsQIAM477zw2b94cdN9jxoypVebDDz9k3LhxAJx77rn07t273vg+/vhjLrroInJyckhOTmbChAksWbKE008/nXXr1nHHHXewcOFCMjMzAejduzcTJ07klVdeOeFBSCerwZq7mb0AfAfY5ZzrE2S7AU8DI4ES4Gbn3KfhDjTgA6FrVyV3kZMRag27uBhmzYJJkyAnp8nCad26ddX8+vXrefrpp/nkk0/Iyspi4sSJQft7p6SkVM0nJiZSXl4edN+pqam1yjjnGhVfXeWzs7NZuXIl77zzDjNmzGDevHk899xzLFy4kPfff58333yT//qv/2LVqlUkJiY26jNPVig1998Bw+vZPgLo6ZsmA8+cfFgNyMtTchdpDjk5cM89TZrYazpw4AAZGRm0bduW7du3s3DhwrB/xgUXXMDcuXMB+Pzzz4P+Mgg0cOBAFi9ezO7duykvL2fOnDkMGTKEoqIinHNce+21/PznP+fTTz+loqKCwsJCLrroIp544gmKioooKSkJ+9/QkAZr7s65JWbWrZ4io4HfO+/U9g8zyzKzTs657WGKsba8PHjvvSbbvYhETn5+Pr169aJPnz706NGDQYMGhf0zbr/9dm688Ub69u1Lfn4+ffr0qWpSCSY3N5dp06YxdOhQnHNceeWVXHHFFXz66ad8//vfxzmHmfHYY49RXl7OhAkTOHjwIJWVldx7771kZGSE/W9oiIXy88SX3N+qo1nmLeBR59yHvuW/APc652o9icPMJuPV7snLyzvvX/8K6Z7ztT34IDz8MBw92vBFHhEBYO3atZx99tmRDqNFKC8vp7y8nLS0NNavX89ll13G+vXrSUpqWX1Mgv2bmdly51xBQ+8Nx18SbIxs0DOGc+454DmAgoKCxjV6BcrLA+dg61bQo/pEpJEOHTrExRdfTHl5Oc45fvvb37a4xH6ywvHXFAJdA5ZzgW1h2G/duvo+bssWJXcRabSsrCyWL18e6TCaVDi6Qs4HbjTPQGB/k7a3w/G+7rqoKiISVChdIWcDQ4EcMysEHgSSAZxzzwIL8LpBbsDrCjmpqYKt4q+5ayCTiEhQofSWGd/Adgc0780cWreG7GzV3EVE6hB9I1T9NJBJRKRO0ZvcNZBJJKoMHTq01oCk6dOn88Mf/rDe97Vp0waAbdu2MXbs2Dr3vWxZrd7XtT4rcDDRyJEjw3Lfl4ceeognn3zypPcTbtGd3NXmLhI1xo8fz5w5c6qtmzNnDuPH19vyW6Vz58689tprJ/z5NZP7ggULyMrKOuH9tXTRndz37/cmEWnxxo4dy1tvvcXRo0cB2Lx5M9u2beOCCy6o6neen5/POeecw5tvvlnr/Zs3b6aP75bfpaWljBs3jr59+3L99ddTWlpaVe4HP/hB1e2CH3zwQQBmzJjBtm3bGDZsGMOGDQOgW7duFBcXA/DUU0/Rp08f+vTpU3W74M2bN3P22Wdz66230rt3by677LJqnxPMihUrGDhwIH379uXqq69m7969VZ/fq1cv+vbtW3XDsvfff7/qYSX9+/fnYJhvYx69vfYDb/1bz7BhEantznfvZMWOhm/5W1ZRxo5DO+jYpiPJifWPBu/XsR/Th9d9Q7Ls7GwGDBjAu+++y+jRo5kzZw7XX389ZkZaWhqvv/46bdu2pbi4mIEDBzJq1Kg6nyP6zDPPkJ6ezsqVK1m5cmW1W/Y+8sgjtG/fnoqKCi6++GJWrlzJj3/8Y5566ikWL15MTo375CxfvpxZs2bx8ccf45zj/PPPZ8iQIbRr147169cze/Zsnn/+ea677jrmzZtX7/3Zb7zxRn79618zZMgQHnjgAX7+858zffp0Hn30UTZt2kRqampVU9CTTz7JzJkzGTRoEIcOHSItLa3e49tY0VtzDxzIJCJNYsehHWzct5Edh8Jzy9/AppnAJhnnHD/72c/o27cvl1xyCVu3bmXnzp117mfJkiVVSbZv37707du3atvcuXPJz8+nf//+rF69usGbgn344YdcffXVtG7dmjZt2jBmzBg++OADALp3706/fv2A+m8rDN795fft28eQIUMAuOmmm1iyZElVjDfccAMvv/xy1UjYQYMGcddddzFjxgz27dsX9hGysVFzF5FGqa+GHai4pJhZ/5zFpP6TyEk/+TtDXnXVVdx11118+umnlJaWVtW4X3nlFYqKili+fDnJycl069Yt6G1+AwWr1W/atIknn3ySpUuX0q5dO26++eYG91Pf/bX8twsG75bBDTXL1OXtt99myZIlzJ8/n4cffpjVq1czdepUrrjiChYsWMDAgQP585//zFlnnXVC+w8memvunTpBYqJq7iJNKCc9h3sG3ROWxA5ez5ehQ4fyve99r9qF1P3793PKKaeQnJzM4sWLaeimgoMHD656CPaqVatYuXIl4N0uuHXr1mRmZrJz507eeeedqvdkZGQEbdcePHgwb7zxBiUlJRw+fJjXX3+dC0/gGc2ZmZm0a9euqtb/0ksvMWTIECorK/n6668ZNmwYjz/+OPv27ePQoUN89dVXnHPOOdx7770UFBTwxRdfNPoz6xO9NffERMjNVXIXiTLjx49nzJgx1XrO3HDDDVx55ZUUFBTQr1+/BmuwP/jBD5g0aRJ9+/alX79+DBgwAPCeqtS/f3969+5d63bBkydPZsSIEXTq1InFixdXrc/Pz+fmm2+u2sctt9xC//79622CqcuLL77IbbfdRklJCT169GDWrFlUVFQwceJE9u/fj3OOn/zkJ2RlZfGf//mfLF68mMTERHr16lX1VKlwCemWv02hoKDANdQvtUEXXggJCfD+++EJSiSG6Za/0edkbvkbvc0yoIFMIiJ1iP7kvnUrVFREOhIRkRYl+pN7WRnU02VKRI6LVDOsNN7J/ltFf3IHNc2IhCAtLY3du3crwUcB5xy7d+8+qYFN0dtbBqoPZBo4MLKxiLRwubm5FBYWUlRUFOlQJARpaWnk5uae8PujO7lrIJNIyJKTk+nevXukw5BmElKzjJkNN7N1ZrbBzKYG2f4NM/uLma00s7+a2YmfbhojMxMyMtQsIyJSQ4PJ3cwSgZnACKAXMN7MetUo9iTwe+dcX2Aa8MtwB1pHcOoOKSISRCg19wHABufcRufcMWAOMLpGmV7AX3zzi4Nsbzp6IpOISC2hJPcuQGCjdqFvXaDPgGt881cDGWaWffLhhUAP7RARqSWU5B7shso1+1LdDQwxs38CQ4CtQHmtHZlNNrNlZrYsbFfs8/KgqAhO8G5tIiKxKJTkXgh0DVjOBbYFFnDObXPOjXHO9Qf+w7eu1iOSnHPPOecKnHMFHTp0OImwA6jHjIhILaEk96VATzPrbmYpwDhgfmABM8sxM/++7gNeCG+Y9dBAJhGRWhpM7s65cmAKsBBYC8x1zq02s2lmNspXbCiwzsy+BE4FHmmieGvTE5lERGoJaRCTc24BsKDGugcC5l8DTvyx5CejSxevS6SaZUREqkT3vWUAUlOhY0fV3EVEAkR/cgcNZBIRqSE2krsGMomIVBMbyd0/kEm3MhURAWIpuZeWwu7dkY5ERKRFiJ3kDmqaERHxUXIXEYlBsZHc/QOZ1NddRASIleTeoYPX3101dxERIFaSux7aISJSTWwkd1ByFxEJEDvJXQOZRESqxE5yz8uD7duhrCzSkYiIRFxsJXfnYOvWSEciIhJxsZXcQU0zIiIouYuIxKTYSe4ayCQiUiWk5G5mw81snZltMLOpQbbnmdliM/unma00s5HhD7UB6emQna2au4gIISR3M0sEZgIjgF7AeDPrVaPY/XjPVu2P9wDt/w53oCFRX3cRESC0mvsAYINzbqNz7hgwBxhdo4wD2vrmM4Ft4QuxEZTcRUSA0JJ7FyCwIbvQty7QQ8BEMyvEe5D27cF2ZGaTzWyZmS0rKio6gXAb0LWr2txFRAgtuVuQdTUfeTQe+J1zLhcYCbxkZrX27Zx7zjlX4Jwr6NChQ+OjbUheHuzf700iInEslOReCHQNWM6ldrPL94G5AM65vwNpQE44AmwUf3dI1d5FJM6FktyXAj3NrLuZpeBdMJ1fo8wW4GIAMzsbL7k3QbtLA9TXXUQECCG5O+fKgSnAQmAtXq+Y1WY2zcxG+Yr9O3CrmX0GzAZudi4CT6v293VXcheROJcUSiHn3AK8C6WB6x4ImF8DDApvaCegUydITFSzjIjEvdgZoQpeYs/NVc1dROJebCV3UF93ERGU3EVEYlLsJfeuXb17uldURDoSEZGIib3knpfnPY1p585IRyIiEjGxmdxBTTMiEteU3EVEYlDsJXc9tENEJAaTe2YmZGSo5i4icS32kruZukOKSNyLveQOSu4iEveU3EVEYlBsJveuXaG4GEpLIx2JiEhExGZy10M7RCTOxXZyV9OMiMQpJXcRkRgUUnI3s+Fmts7MNpjZ1CDbf2VmK3zTl2a2L/yhNkKXLl6XSDXLiEicavBJTGaWCMwELsV7WPZSM5vve/oSAM65nwSUvx3o3wSxhi4lBTp2VM1dROJWKDX3AcAG59xG59wxYA4wup7y4/GeoxpZ6g4pInEslOTeBQhs3yj0ravFzL4BdAcWnXxoJ0nJXUTiWCjJ3YKsc3WUHQe85pwL+qQMM5tsZsvMbFlRUVGoMZ6YvDyvzd3VFaqISOwKJbkXAl0DlnOBbXWUHUc9TTLOueeccwXOuYIOHTqEHuWJ6NrVG8S0e3fTfo6ISAsUSnJfCvQ0s+5mloKXwOfXLGRmZwLtgL+HN8QTpO6QIhLHGkzuzrlyYAqwEFgLzHXOrTazaWY2KqDoeGCOcy2kHUTJXUTiWINdIQGccwuABTXWPVBj+aHwhRUGSu4iEsdic4QqQE4OpKVpIJOIxKXYTe5m3kVV1dxFJA7FbnIH9XUXkbil5C4iEoNiP7lv3w5lZZGORESkWcV2cu/a1RuhunVrpCMREWlWsZ3c1R1SROKUkruISAyK7eTe1XdLHPV1F5E4E9vJPT0dsrNVcxeRuBPbyR3UHVJE4pKSu4hIDFJyFxGJQfGR3A8cgP37Ix2JiEizif3krh4zIhKHYj+5q6+7iMShkJK7mQ03s3VmtsHMptZR5jozW2Nmq83sf8Mb5klQcheRONTgk5jMLBGYCVyK97DspWY23zm3JqBMT+A+YJBzbq+ZndJUATdax46QlKRmGRGJK6HU3AcAG5xzG51zx4A5wOgaZW4FZjrn9gI453aFN8yTkJgIXbqo5i4icSWU5N4FCKz2FvrWBToDOMPM/mZm/zCz4eEKMCzUHVJE4kwoyd2CrHM1lpOAnsBQYDzwP2aWVWtHZpPNbJmZLSsqKmpsrCdOyV1E4kwoyb0Q6BqwnAtsC1LmTedcmXNuE7AOL9lX45x7zjlX4Jwr6NChw4nG3Hh5ed493Ssqmu8zRUQiKJTkvhToaWbdzSwFGAfMr1HmDWAYgJnl4DXTbAxnoCclL897GtPOnZGORESkWTSY3J1z5cAUYCGwFpjrnFttZtPMbJSv2EJgt5mtARYD9zjndjdV0I3mH8ikphkRiRMNdoUEcM4tABbUWPdAwLwD7vJNLU9gX/eBAyMbi4hIM4j9EaqggUwiEnfiI7lnZkLbthrIJCJxIz6SO3jt7qq5i0iciJ/krr7uIhJHlNxFRGJQfCX34mIoLY10JCIiTS5+krse2iEicSR+kru6Q4pIHFFyFxGJQfGT3Lt0ATM1y4hIXIif5J6SAp06qeYuInEhfpI7aCCTiMSN+Eru6usuInEiPpO7q/kgKRGR2BJ/yf3IEdjdcm41LyLSFOIrueuhHSISJ+Iruauvu4jEiZCSu5kNN7N1ZrbBzKYG2X6zmRWZ2QrfdEv4Qw0DJXcRiRMNPmbPzBKBmcClQCGw1MzmO+fW1Cj6qnNuShPEGD45OZCWpoFMIhLzQqm5DwA2OOc2OueOAXOA0U0bVhMxU3dIEYkLoST3LkBgVbfQt66ma8xspZm9ZmZdg+3IzCab2TIzW1ZUVHQC4YaBBjKJSBwIJblbkHU1O4r/EejmnOsL/Bl4MdiOnHPPOecKnHMFHTp0aFyk4aKau4jEgVCSeyEQWBPPBbYFFnDO7XbOHfUtPg+cF57wmkBeHmzfDmVlkY5ERKTJhJLclwI9zay7maUA44D5gQXMrFPA4ihgbfhCDLO8PG+E6tatkY5ERKTJNJjcnXPlwBRgIV7SnuucW21m08xslK/Yj81stZl9BvwYuLmpAt60dxM3vXETxSXFJ7YDDWQSkTjQYFdIAOfcAmBBjXUPBMzfB9wX3tCCu2X+LSzavIjiw8W8NeEtzIJdEqiH+rqLSByIuhGqL415iX6n9mPBhgXctfAuKl1l43agmruIxIGoS+6dMzqz/N+Wc+f5dzL94+lM/H8TOVZxLPQdpKd7g5k0kElEYlhIzTItTYIl8NTlT9EpoxP3/vleikuKmXfdPDJSM0LbgbpDikiMi7qau5+Z8dNBP+V3o3/Hok2LGPbiMHYd3hXamzWQSURiXNQmd7+b+t3Em+PeZE3RGga9MIiNezc2/KbOnWHNGnj55aYPUEQkAqI+uQNcccYVLLppEXtK9/Dt//ttVuxYUf8bsrOhshK++12YNs2bFxGJITGR3AEG5g7kw0kfkpKYwuBZg1m0aVHdhe+4Ax55BK6/Hh58EK65Bg4caL5gRUSaWMwkd4CzO5zNR9//iLzMPEa8MoK5q+cGL5iTAz/7GcyeDU8/DX/8IwwcCF9+2bwBi4g0kZhK7gC5bXP5YNIHDOgygHGvjeM3n/ym7sJm8OMfw3vvwa5dMGAALFhQd3kRkSgRc8kdoF2rdvxp4p8YdeYobn/ndu5fdD/O1byRZYBhw2DZMujeHb7zHfjlL737z4iIRKmYTO4ArZJb8dp1r3Fr/q088sEj3PrHWymvLK/7Dd26wd/+BuPGeU02110Hhw41W7wiIuEUlYOYQpWUkMRvv/NbOrbpyMNLHqaopIjZ18wmPTk9+BvS0+GVV+C88+CnP4UvvoA33oDTTmvewEVETlLM1tz9zIxpw6Yxc+RM/rjuj1z20mXsKd1T3xvg3/8d3n3Xuy3wN78Jf/pT8wUsIhIGMZ/c/X74zR8y99q5LN22lAtnXUjhgcL633DppV47fG4ujBgBTz6pdngRiRpxk9wBxvYay7s3vMuW/Vvo+euejJ83nre+fIuj5UeDv6FHD/j7371+8PfcAzfcACUlzRu0iMgJiKvkDjCs+zAm50/mSPkR3lj7BlfOvpIOT3RgwrwJzFszj5KyGsm7dWt49VWvB82cOTBoEGzeHJHYRURCFVJyN7PhZrbOzDaY2dR6yo01M2dmBeELMfzuu/A+Hr/kcb664yveueEdru99Pe9tfI+xfxhLzuM5XDP3Gv738//lwFHfqFUzmDoV3n4bNm2CggJYVM8IWBGRCLN6+38DZpYIfAlcivew7KXAeOfcmhrlMoC3gRRginNuWX37LSgocMuW1VukWZVXlrPkX0uYt2Yer3/xOtsPbSclMYXLTruMa86+hlFnjqJ9q/awfj1cdZXXk+bSS+GZZ7z+8SIizcDMljvnGqxAh5LcvwU85Jy73Ld8H4Bz7pc1yk0H/gzcDdwdbck9UKWr5O9f/515a+cxb+08tuzfQlJCEsO6DeOas6/hqtxLOPXSq2DVKkhNhX/7N7j9djj99EiHLiIxLpzJfSww3Dl3i2/5u8D5zrkpAWX6A/c7564xs78S5ck9kHOO5duXM2+Nl+jX71mPYQw89TxaFe6gYG86Was3kFJWSfJZvUkedgnJZ/cmOTGF5MRkkhOS6309rd1pZKZlRvrPFJEoEWpyD2UQU7AnUFedEcwsAfgVcHMIQU0GJgPk+R9U3cKZGQWdCyjoXMAvLv4Fq3atYt7aeTy77Fl2Ju9k0SnAKf7Sq+Gr1fBV6PtPS0zjgSEPcM+ge0hKiOkxZSLSjE66WcbMMvHSmX+sfkdgDzCqvtp7tNTc61JcUsysf87ixnNvJDMtk7KKMspKDlI2by7H/ue3lH35BWXtsygbdy1l142lLDvLK1NZVvU6Z9UcXlr5EgDnnHIOvxn5GwZ/Y3CE/zIRacnC2SyThHdB9WJgK94F1QnOudV1lP8rMdQsc0Kcg/ffh+nTYf58SEz07lVzxx3enSd9ikuKeeGfL3BK61N48K8PsmX/FiacM4EnLn2CzhmdI/gHiEhLFWpyb7ArpHOuHJgCLATWAnOdc6vNbJqZjTr5UGOQGQwd6t2XZv16mDLFu2f8+efDt78Nc+dCeTk56Tn8dNBPubnfzaz90VoeGPwA89bM48zfnMkTf3uCYxXHIv2XiEiUarDm3lRiuuYezIED8LvfwYwZ8NVX3m0NpkyByZOhXbuqYhv3buQnC3/C/HXzOTP7TGaMmMFlp10WubhFpEUJW81dwqRtW+/BIOvWeU01Z5zhDYzKy/Nq+D492vXgzXFv8vaEt6lwFVz+8uWMeXUMm/dtjlzsIhJ1lNybW2IiXHkl/OUvXrI/dAjGjIGHH4aKiqpiI3uOZNUPVvGLi37Bwq8WcvbMs5n2/jRKy0ojGLyIRAs1y0RScTE8+yx89hm89po34vXll+GUU6oV+3r/19z93t3MXT2X7lndmT58OleecSVmwXqpikgsU7NMNMjJgfvv9y6wPv88fPAB9OsHS5ZUK9Y1syuvjn2VRTcuolVyK0bPGc0V/3sF63evj1DgItLSKbm3BGZwyy3w8cfQpo33TNdf/hIqK6sVG9Z9GCv+bQW/uvxX/O3rv9HnmT7c9+f72LR3E7tLdtd962IRiTtqlmlpDh70etDMmQPDh8NLL3k1/Bp2HNrB1D9P5cXPXqy2PjkhmTYpbWiT0oaM1AzvNSXj+LrAed925xwfb/2Yq8+6mrzMPNqmtiUzLZOMlAySE5Ob6y8XkRCEbRBTU1Fyr4dz8Nvfwp13eon91Ve9+8gHcee7d/L0x08z6sxRnN/lfA4dO8TBowc5VHbo+Pwx3/yxg1XrSstDuzDbKqlVVbJvm9rWm0+tPZ9gCSzfvpyRPUfSOaNz1QmkTUobWie3pk1KG1ISU3SdQOQkKbnHgn/+E6691ns4yC9+AXffDQnVW9L8t0GY1H8SOem1a/h1qais4HDZYQ4ePciW/VuYvWo2Q78xFDNj/9H9HDh6gANHD7D/iG/+WI3loweqylW6yoY/EO+B5YFJv+aUQALr96ynW1Y3khKSKKsso7yynLIK32sIy8fKj1FWWUanjE5kpWXVOsEEO+nUWp/SmtTEVNKS0khNSiUlMYUEUwumtAxK7rFi/36vPf611+CKK+DFFyE7O9JRVXHOUVJWwqZ9m/j9Z7/n8tMvJzkhuerXQmOmHYd2cPDYQdqltSM7PZvkhGSSEpJITvS9+u6k6Z+vuS0pIYk1RWtYum0pZ+ecTaeMTtX2f/jYYQ4dO8TRisZfm0hOSCY1yZfwE1NJTUqtdgIIXGcYWw5soUdWD1KSUqh0lVS6SpxzVfOVrhKHq3Pb0Yqj7Dy0k25Z3chMy6RVUitvSg7tNT05nVbJrUhOSKbSVVLhKqiorGj0/L4j+/jr5r8y4vQRdMzoSKukVqQlpVX7vLSkNFoltSIxIbEJvmGNU+kqOVJ+hJKyEkrLSiktL6W0rJSSshJ2HtrJwo0LGXPWGDpndCY9Ob3qOKUnp0fNjfuU3GOJczBzJtx1F3TmfiY3AAAM9klEQVTs6PWuGTgw0lGF3Yn+CmnsPsoqyjhcdrhawvdPs1fN5pXPX+Gqs65icN5gjlYc5Wj5UY6UH6maP1pRx7JvfuuBrRSVFJHdKpvs9GwSLAHDSLCEqsns+HKwbVsPbGXTvk3kts0lMzWzKkn5X0/kBNXUkhOSqyV7/wkg0RLZU7qHjhkdSUtKq/p7zazafOCxqLmtrKKMDXs2cGqbUymvLK+VuMNxXPzxVyX9pFa1TgCGsWnvJnq070FqYioOh3Mu6Kv/pF1z29GKo+Rm5PLYpY+d0PdcyT0WLVvm3YDs66/hscfgJz/xetpI2DTXCeZk9+GvodaV5ErLS6vuOjrxnIlcddZVJCYkkmiJJFhCyPMHjhzg9S9e5/LTLyc1MZXS8tJan3uk/Ei1z625/EXxF2zYs4HuWd3p0rZLtV8q/oQX+EumZnKsdJUUlxRTVFJE17Zd6dGuR7VfDulJ6dWXfYm55q+Yo+VHWbB+AYO7DSYlIYWSspKqqbS89Ph8WSkl5SXVl33z2w9tZ0/pHtqltaNdq3YYVnUiCnz1n6iCbdt1eBfbDm7j8Use555B9zT6uxFqcvcOZASm8847z8kJ2LvXuauvdg6cGz3auT17Ih2RtFBFh4vc4x8+7ooOF0V9HPpbjgOWuRByrGru0cg5ePppuOce6NQJhgzxBkOdeWakIxORJqYRqrHMzOsm+eGHXr/4l1+Gs86CXr28PvIvvQSbNnknARGJS9FxeViCO/98WL0aHnkE2rf3uk7+4Q/erQwAunSBCy+ECy7wXvv0qdWVUkRik5J7tOvc2etJ41dZCatWefep+fBD73XOHG9bZqY3GOrCC72poABSUyMTt4g0KbW5xzrnvEFQ/kT/wQfwxRfettRU77F/550H27bB1VfD6adDhw7elJ4e0dBFpLawdoU0s+HA00Ai8D/OuUdrbL8N+BFQgfeg7MnOuTX17VPJPYKKiuBvfzteu1+6NHj7fHr68UTf0HTKKd5Nz9Q1U6RJhfMB2Yl4D8i+FCjEe0D2+MDkbWZtnXMHfPOjgB8654bXt18l9xZkyxb47//2nvtaVga7dnkngLqm0jruS5OUBKee6vXgOeWU+qcOHSAlpVn/TJFYEGpyD6XNfQCwwTm30bfjOcBooCq5+xO7T2tA3TSiSV4ePPpow+X8Dh+ufQL4wx9gwQIvuefkwI4dsHKlV+5YHQ/6bteuesJv08Y70Xz7294Jom3b41NGRvXlVq30K0GkHqEk9y7A1wHLhcD5NQuZ2Y+Au4AU4KJgOzKzycBkgLy8vMbGKi1F69bQvbs3+X3nOzBrFkyaVP0Wxc55Dwfftav+ae1a79pASQksXtxwDImJwZN+aqr3hKuLLoKePb0eQ507e1NGhk4IEjdCaZa5FrjcOXeLb/m7wADn3O11lJ/gK39TfftVs4zUUlwML7wAEyZ4TTYHDnj9+A8cqD3VtX7TJm8/wbRufTzR15z8J4FOnXQhWVq0cDbLFAJdA5ZzgW31lJ8DPBPCfkWqy8mBn/70+HKNZ8mGpLjY+wVx7bVec9C2bcGnTz6BrVvhyJHa+0hNhaws75dAevrxqXXr6svB1rVu7V23+OADGD/e+/WQna3rC9LsQknuS4GeZtYd2AqMAyYEFjCzns45/wM9rwD0cE+JjJwc77YMfmecUXdZ57xbKgcm/dmz4d13vesQp5/uNROVlHjXGXbvPr7sX3e0nrsQPvvs8fm2bb0kn5PjTQ3N64QgJ6nB5O6cKzezKcBCvK6QLzjnVpvZNLwb2MwHppjZJUAZsBeot0lGpEUw82roWVnerRsARo4Mfu2gLhUVXu8hf7IvKfFOEq+95o0hKCvzfk0UF3snh+Ji7wL0F1948wcP1r3v1FQvyWdnexef/bEGzte13KaNRiPHOQ1iEomko0e9pO9P/P6TwBtvwMKF3iji3FzYuxf27Ts+7d9f/34TErwLyAkJ3jMA/Ak/I+P4FLhc37bMTK+bq7QI4WxzF5Gmkpp6/KJuoLFj6/8FUVHhXUD2J/uayX/vXli0CD76yNt3q1be+sJC79eCf6qoCC3O9u29/XTsWP/Urp1+MbQQqrmLxCr/xeW6ThDOeReUDx2qnvADl998E+bP925S17GjN35h507Yvj349Qb/QLbAhN+2LXz5pXdfo+xs74RWc0pJCb4+cEpKUldW9CQmEQmHuk4Q/vELO3Ycn3burL4ceCIIR55JSDh+TaFtW6+5KDOz+nzN5ZrbYuBahJplROTk1ex95Gd2PGE29JCYXbvguee87qlt2ng1fv907Fj15bqmv/zFuwZxxhnQo4d3zcE/rsE/v3+/d1fUhmRkeM1MWVle/KG8+uczM6PmTqqquYtIy9dQExN4vw4OHz6e6P2Tf/mNN+Dtt73modNO89YFXqD2vzaUE5OTj3ddbd/em9q1Oz5f13LbtmFpVlKzjIhIoFBOEJWV3jWHwGQfeAL44x+r92Las+f4tHdv3TfVA++WGf4kP3Cg96jMULrb1qDkLiISbg2dIEpLvSS/d2/txO+f/+gj+OwzePzx4E1eDVCbu4hIuNV1DcKvVStvqtm1NVDgCaIJKbmLiDSnhk4QYRLdfYJERCQoJXcRkRik5C4iEoOU3EVEYpCSu4hIDFJyFxGJQUruIiIxKGIjVM2sCPjXCb49B6jjKcgtTrTEqjjDK1rihOiJVXF6vuGc69BQoYgl95NhZstCGX7bEkRLrIozvKIlToieWBVn46hZRkQkBim5i4jEoGhN7s9FOoBGiJZYFWd4RUucED2xKs5GiMo2dxERqV+01txFRKQeLTq5m9lwM1tnZhvMbGqQ7alm9qpv+8dm1i0CMXY1s8VmttbMVpvZHUHKDDWz/Wa2wjc90NxxBsSy2cw+98VR62kp5pnhO6YrzSw/AjGeGXCsVpjZATO7s0aZiBxTM3vBzHaZ2aqAde3N7D0zW+97bVfHe2/ylVlvZjdFKNYnzOwL37/t62aWVcd76/2eNEOcD5nZ1oB/35F1vLfeHNEMcb4aEONmM1tRx3ub7XhWcc61yAlIBL4CegApwGdArxplfgg865sfB7wagTg7Afm++QzgyyBxDgXeivQx9cWyGcipZ/tI4B3AgIHAxy3ge7ADr29vxI8pMBjIB1YFrHscmOqbnwo8FuR97YGNvtd2vvl2EYj1MiDJN/9YsFhD+Z40Q5wPAXeH8N2oN0c0dZw1tv8f4IFIH0//1JJr7gOADc65jc65Y8AcYHSNMqOBF33zrwEXm4XhCbSN4Jzb7pz71Dd/EFgLdGnOGMJsNPB75/kHkGVmnSIYz8XAV865Ex3wFlbOuSXAnhqrA7+HLwJXBXnr5cB7zrk9zrm9wHvA8CYLlOCxOuf+5Jwr9y3+A8htyhhCUccxDUUoOSJs6ovTl3euA2Y31ec3VktO7l2ArwOWC6mdNKvK+L6w+4HsZokuCF+zUH/g4yCbv2Vmn5nZO2bWu1kDq84BfzKz5WY2Ocj2UI57cxpH3f9hWsoxPdU5tx28kz1wSpAyLe24AnwP71daMA19T5rDFF/z0Qt1NHW1pGN6IbDTObe+ju3NfjxbcnIPVgOv2bUnlDLNwszaAPOAO51zB2ps/hSvWeFc4NfAG80dX4BBzrl8YATwIzMbXGN7SzqmKcAo4A9BNrekYxqKFnNcAczsP4By4JU6ijT0PWlqzwCnAf2A7XhNHjW1pGM6nvpr7c1+PFtyci8EugYs5wLb6ipjZklAJif28+6kmFkyXmJ/xTn3/2pud84dcM4d8s0vAJLNLMij05uec26b73UX8DreT9tAoRz35jIC+NQ5t7PmhpZ0TIGd/qYr3+uuIGVazHH1Xcz9DnCD8zUI1xTC96RJOed2OucqnHOVwPN1fH6LOKa+3DMGeLWuMpE4ni05uS8FeppZd18Nbhwwv0aZ+YC/18FYYFFdX9am4mtr+7/AWufcU3WU6ei/FmBmA/CO++7mi7IqjtZmluGfx7u4tqpGsfnAjb5eMwOB/f4mhwioszbUUo6pT+D38CbgzSBlFgKXmVk7XxPDZb51zcrMhgP3AqOccyV1lAnle9KkalznubqOzw8lRzSHS4AvnHOFwTZG7Hg259Xbxk54PTe+xLsi/h++ddPwvpgAaXg/2TcAnwA9IhDjBXg/BVcCK3zTSOA24DZfmSnAaryr+f8Avh2h49nDF8Nnvnj8xzQwVgNm+o7550BBhGJNx0vWmQHrIn5M8U4224EyvJrj9/Gu8/wFWO97be8rWwD8T8B7v+f7rm4AJkUo1g147dT+76q/t1lnYEF935NmjvMl3/dvJV7C7lQzTt9yrRzRnHH61v/O/70MKBux4+mfNEJVRCQGteRmGREROUFK7iIiMUjJXUQkBim5i4jEICV3EZEYpOQuIhKDlNxFRGKQkruISAz6/9lPefCbI00XAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.001,epochs=15,bs=512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "88% accuracy in 19  epochs!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8818"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accuracy on test set\n",
    "y_pred=nn.predict(X_test)\n",
    "accuracy(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train some more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 0.24480102475639268, Val loss:0.3313077210549393\n",
      "Epoch 2. Training loss: 0.24516625633300557, Val loss:0.3301432893221368\n",
      "Epoch 3. Training loss: 0.24468010227850703, Val loss:0.3298356951638451\n",
      "Epoch 4. Training loss: 0.2437537268449228, Val loss:0.3298453494445676\n",
      "Epoch 5. Training loss: 0.24316273050109785, Val loss:0.329722483369439\n",
      "Epoch 6. Training loss: 0.2429608125196632, Val loss:0.32980138288948074\n",
      "Epoch 7. Training loss: 0.24232683720153375, Val loss:0.32976080993242535\n",
      "Epoch 8. Training loss: 0.24191137827398734, Val loss:0.32971262984883526\n",
      "Epoch 9. Training loss: 0.24026288037111415, Val loss:0.3300429316087054\n",
      "Epoch 10. Training loss: 0.2411287121163183, Val loss:0.3298617284136391\n",
      "Epoch 11. Training loss: 0.24094856262209272, Val loss:0.3291287933509493\n",
      "Epoch 12. Training loss: 0.24091840070194412, Val loss:0.3293980275172278\n",
      "Epoch 13. Training loss: 0.2410055118407869, Val loss:0.32967343220308915\n",
      "Epoch 14. Training loss: 0.23763759233455425, Val loss:0.32936433311825425\n",
      "Epoch 15. Training loss: 0.23709401781565312, Val loss:0.3292254586029039\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl8VPW9//HXJxsJSUgCSWQXFFwghsUUUZRFEbcrFKtVLFWpFbVVa/lV5fZ6rdLrveqlleKlWqxSWxekchWqKG2VK1orssgiIIIsEoKQIAECgWzf3x9nJkySSTJAkskM7+fjcR5nme+c+WTQz/nO95zzOeacQ0REoktMuAMQEZGmp+QuIhKFlNxFRKKQkruISBRSchcRiUJK7iIiUUjJXUQkCim5i4hEISV3EZEoFBeuD87MzHQ9evQI18eLiESk5cuXFznnshprF7bk3qNHD5YtWxaujxcRiUhmti2UdhqWERGJQkruIiJRSMldRCQKhW3MXURaVnl5Ofn5+Rw+fDjcoUgIEhMT6dq1K/Hx8cf1fiV3kZNEfn4+qamp9OjRAzMLdzjSAOcce/bsIT8/n549ex7XPjQsI3KSOHz4MB06dFBijwBmRocOHU7oV5aSu8hJRIk9cpzov1XkJfc334SLL4bdu8MdiYhIqxV5yf13v4NFi7y5iESMPXv20L9/f/r370/Hjh3p0qVL9XpZWVlI+5gwYQIbNmxosM2MGTN46aWXmiJkLrzwQlauXNkk+2ppkXdC9dJLvd77VVeFOxIROQYdOnSoTpQPP/wwKSkp/OxnP6vRxjmHc46YmOD9zlmzZjX6OT/+8Y9PPNgoEHk9927dvLnGDkWiwqZNm8jJyeGOO+5g4MCB7Ny5k4kTJ5KXl0ffvn2ZMmVKdVt/T7qiooL09HQmT55Mv379OP/889ntG6p98MEHmTZtWnX7yZMnM2jQIM4880w++ugjAA4ePMh3vvMd+vXrx7hx48jLy2u0h/7iiy9yzjnnkJOTw89//nMAKioq+P73v1+9ffr06QA8+eST9OnTh379+jF+/Pgm/85CEXk994wMb15cHN44RCLZvfdCKMMN5eXw9dfQsSM0dr11//7gS6rHat26dcyaNYtnnnkGgMcee4z27dtTUVHBiBEjuPbaa+nTp0+N9+zbt49hw4bx2GOPMWnSJJ5//nkmT55cZ9/OOT755BPmz5/PlClTeOedd3jqqafo2LEjc+fOZdWqVQwcOLDB+PLz83nwwQdZtmwZaWlpjBw5kjfffJOsrCyKiopYs2YNAMW+vPTEE0+wbds2EhISqre1tEZ77mb2vJntNrPP6nndzGy6mW0ys9Vm1vC3dKL8yX3v3mb9GBHBS+ybN3vzZnT66afzrW99q3r9lVdeYeDAgQwcOJD169ezbt26Ou9JSkriiiuuAODcc89l69atQfd9zTXX1Gnz4YcfcsMNNwDQr18/+vbt22B8S5Ys4eKLLyYzM5P4+HhuvPFGFi9eTK9evdiwYQM/+clPWLhwIWlpaQD07duX8ePH89JLLx33TUgnKpSe+x+A/wH+WM/rVwC9fdN5wNO+efNIT/fm6rmLHL9Qe9hFRTBrFkyYAJmZzRZOcnJy9fLGjRv5zW9+wyeffEJ6ejrjx48Per13QkJC9XJsbCwVFRVB992mTZs6bZxzxxRffe07dOjA6tWrefvtt5k+fTpz585l5syZLFy4kPfff5958+bxH//xH3z22WfExsYe02eeqEZ77s65xcA3DTQZA/zReT4G0s2sU1MFWId67iItJzMT7ruvWRN7bfv37yc1NZV27dqxc+dOFi5c2OSfceGFFzJnzhwA1qxZE/SXQaDBgwezaNEi9uzZQ0VFBbNnz2bYsGEUFhbinOO6667jkUceYcWKFVRWVpKfn8/FF1/Mf//3f1NYWMihQ4ea/G9oTFOMuXcBtges5/u27WyCfdeVkgIxMUruIlFq4MCB9OnTh5ycHE477TSGDBnS5J9x9913c9NNN5Gbm8vAgQPJycmpHlIJpmvXrkyZMoXhw4fjnOPqq6/mqquuYsWKFdx666045zAzHn/8cSoqKrjxxhs5cOAAVVVVPPDAA6Smpjb539AYC+XniZn1AN50zuUEee0t4L+ccx/61t8F7nfOLQ/SdiIwEaB79+7nbtsWUs35ujp0gHHj4H/+5/jeL3ISWr9+PWeffXa4w2gVKioqqKioIDExkY0bNzJq1Cg2btxIXFzrusYk2L+ZmS13zuU19t6m+EvygW4B612BgmANnXMzgZkAeXl5xzboFSgjQz13ETluJSUlXHLJJVRUVOCc43e/+12rS+wnqin+mvnAXWY2G+9E6j7nXPMMyfilp+uEqogct/T0dJYvrzO4EFUaTe5m9gowHMg0s3zgF0A8gHPuGWABcCWwCTgETGiuYKup5y4i0qBGk7tzblwjrzugZe/3TU+H/PwW/UgRkUgSeeUHwOu5a1hGRKRekZvcNSwjIlKvyEzu6elw5AjoWZAiEWP48OF1bkiaNm0aP/rRjxp8X0pKCgAFBQVce+219e572bJlDe5n2rRpNW4muvLKK5uk7svDDz/M1KlTT3g/TS0yk7vuUhWJOOPGjWP27Nk1ts2ePZtx4xo8rVetc+fOvPbaa8f9+bWT+4IFC0j3lzOJQpGZ3P3/IEruIhHj2muv5c033+TIkSMAbN26lYKCAi688MLq684HDhzIOeecw7x58+q8f+vWreTkePdRlpaWcsMNN5Cbm8v1119PaWlpdbs777yzulzwL37xCwCmT59OQUEBI0aMYMSIEQD06NGDoqIiAH7961+Tk5NDTk5OdbngrVu3cvbZZ3PbbbfRt29fRo0aVeNzglm5ciWDBw8mNzeXsWPHsteXo6ZPn06fPn3Izc2tLlj2/vvvVz+sZMCAARw4cOC4v9tgIvOqfZX9FTkh975zLyu/brzkb3llOV+XfE3HlI7ExzZc3bB/x/5Mu7z+gmQdOnRg0KBBvPPOO4wZM4bZs2dz/fXXY2YkJiby+uuv065dO4qKihg8eDCjR4+u9zmiTz/9NG3btmX16tWsXr26RsneRx99lPbt21NZWckll1zC6tWrueeee/j1r3/NokWLyKxVJ2f58uXMmjWLJUuW4JzjvPPOY9iwYWRkZLBx40ZeeeUVnn32Wb773e8yd+7cBuuz33TTTTz11FMMGzaMhx56iEceeYRp06bx2GOPsWXLFtq0aVM9FDR16lRmzJjBkCFDKCkpITExscHv91hFZs9dwzIiLeLrkq/ZXLyZr0uapuRv4NBM4JCMc46f//zn5ObmMnLkSHbs2MGuXbvq3c/ixYurk2xubi65ubnVr82ZM4eBAwcyYMAA1q5d22hRsA8//JCxY8eSnJxMSkoK11xzDR988AEAPXv2pH///kDDZYXBqy9fXFzMsGHDALj55ptZvHhxdYzf+973ePHFF6vvhB0yZAiTJk1i+vTpFBcXN/kdspHZc1fZX5ET0lAPO1DRoSJmfTqLCQMmkNn2xCtDfvvb32bSpEmsWLGC0tLS6h73Sy+9RGFhIcuXLyc+Pp4ePXoELfMbKFivfsuWLUydOpWlS5eSkZHBLbfc0uh+Gqqv5S8XDF7J4MaGZerz1ltvsXjxYubPn88vf/lL1q5dy+TJk7nqqqtYsGABgwcP5u9//ztnnXXWce0/GPXcRaRemW0zuW/IfU2S2MG78mX48OH84Ac/qHEidd++fWRnZxMfH8+iRYtorKjg0KFDqx+C/dlnn7F69WrAKxecnJxMWloau3bt4u23365+T2pqatBx7aFDh/LGG29w6NAhDh48yOuvv85FF110zH9bWloaGRkZ1b3+P/3pTwwbNoyqqiq2b9/OiBEjeOKJJyguLqakpIQvv/ySc845hwceeIC8vDw+//zzY/7MhkR2z13JXSTijBs3jmuuuabGlTPf+973uPrqq8nLy6N///6N9mDvvPNOJkyYQG5uLv3792fQoEGA91SlAQMG0Ldv3zrlgidOnMgVV1xBp06dWLRoUfX2gQMHcsstt1Tv44c//CEDBgxocAimPi+88AJ33HEHhw4d4rTTTmPWrFlUVlYyfvx49u3bh3OOn/70p6Snp/Pv//7vLFq0iNjYWPr06VP9VKmmElLJ3+aQl5fnGrsutUEpKXD77fCrXzVdUCJRTCV/I8+JlPyNzGEZ0F2qIiINiNzkrrK/IiL1itzkrp67yDEL1zCsHLsT/beK3OSenq7kLnIMEhMT2bNnjxJ8BHDOsWfPnhO6sSkyr5YBr+fuu/xJRBrXtWtX8vPzKSwsDHcoEoLExES6du163O8PKbmb2eXAb4BY4PfOucdqvX4q8DyQBXwDjHfONe/TNDQsI3JM4uPj6dmzZ7jDkBbS6LCMmcUCM4ArgD7AODPrU6vZVOCPzrlcYArwX00daB3p6bB/P1RWNvtHiYhEmlDG3AcBm5xzm51zZcBsYEytNn2Ad33Li4K83vT8d6nu29fsHyUiEmlCSe5dgO0B6/m+bYFWAd/xLY8FUs2sw4mH1wDVlxERqVcoyT1Yzc3ap9t/Bgwzs0+BYcAOoKLOjswmmtkyM1t2wid1VF9GRKReoST3fKBbwHpXoCCwgXOuwDl3jXNuAPBvvm11xkucczOdc3nOubysrKwTCBsldxGRBoSS3JcCvc2sp5klADcA8wMbmFmmmfn39a94V840Lw3LiIjUq9Hk7pyrAO4CFgLrgTnOubVmNsXMRvuaDQc2mNkXwCnAo80U71HquYuI1Cuk69ydcwuABbW2PRSw/Bpw/E+uPR7quYuI1Ctyyw8kJ0NcnHruIiJBRG5yN9NdqiIi9Yjc5A4q+ysiUo/ITu7quYuIBBXZyV09dxGRoCI7uavnLiISlJK7iEgUiuzk7h+W0ZNlRERqiOzknpEB5eVw6FC4IxERaVUiO7nrLlURkaAiO7mrvoyISFBK7iIiUSiyk7uGZUREgors5K6eu4hIUJGd3NVzFxEJKjqSu3ruIiI1hJTczexyM9tgZpvMbHKQ17ub2SIz+9TMVpvZlU0fahCxsdCunZK7iEgtjSZ3M4sFZgBXAH2AcWbWp1azB/EevzcA7xmrv23qQOul4mEiInWE0nMfBGxyzm12zpUBs4Extdo4oJ1vOQ0oaLoQG6H6MiIidYTyDNUuwPaA9XzgvFptHgb+amZ3A8nAyCaJLhTquYuI1BFKz92CbKtdqWsc8AfnXFfgSuBPZlZn32Y20cyWmdmywsLCY482GPXcRUTqCCW55wPdAta7UnfY5VZgDoBz7p9AIpBZe0fOuZnOuTznXF5WVtbxRVybkruISB2hJPelQG8z62lmCXgnTOfXavMVcAmAmZ2Nl9ybqGveCA3LiIjU0Whyd85VAHcBC4H1eFfFrDWzKWY22tfs/wG3mdkq4BXgFudaqMh6RgaUlHilf0VEBAjthCrOuQXAglrbHgpYXgcMadrQQuQvQbBvH2TWGQkSETkpRfYdqqC7VEVEgoj85K7iYSIidUR+clfxMBGROiI/uavnLiJSR/Qkd/XcRUSqRX5y1wlVEZE6Ij+5JyVBmzZK7iIiASI/uYPuUhURqSU6krvqy4iI1BA9yV09dxGRatGR3NPT1XMXEQkQHcldwzIiIjVER3LXCVURkRqiI7n7x9xbqMqwiEhrFz3JvbLSq+suIiJRktx1l6qISA0hJXczu9zMNpjZJjObHOT1J81spW/6wsxadgBcxcNERGpo9ElMZhYLzAAuxXtY9lIzm+97+hIAzrmfBrS/GxjQDLHWT2V/RURqCKXnPgjY5Jzb7JwrA2YDYxpoPw7vOaotRz13EZEaQknuXYDtAev5vm11mNmpQE/gvRMP7Rio7K+ISA2hJHcLsq2+aw5vAF5zzlUG3ZHZRDNbZmbLCgsLQ42xcTqhKiJSQyjJPR/oFrDeFSiop+0NNDAk45yb6ZzLc87lZWVlhR5lY9q1AzMldxERn1CS+1Kgt5n1NLMEvAQ+v3YjMzsTyAD+2bQhhiAmBtLSNCwjIuLTaHJ3zlUAdwELgfXAHOfcWjObYmajA5qOA2Y7F6bbRFVfRkSkWqOXQgI45xYAC2pte6jW+sNNF9ZxUNlfEZFq0XGHKqjsr4hIgOhJ7hqWERGpFj3JXWV/RUSqRU9yV89dRKRadCX30lI4ciTckYiIhF30JHcVDxMRqRY9yV3Fw0REqkVPclfPXUSkWvQkd/XcRUSqRV9yV89dRCSKkrvK/oqIVFNyFxGJQtGT3Nu0gaQkDcuIiBBNyR10l6qIiE/0JXf13EVEoiy5q+yviAgQYnI3s8vNbIOZbTKzyfW0+a6ZrTOztWb2ctOGGSINy4iIACE8icnMYoEZwKV4D8teambznXPrAtr0Bv4VGOKc22tm2c0VcIPS02Ht2rB8tIhIaxJKz30QsMk5t9k5VwbMBsbUanMbMMM5txfAObe7acMMkXruIiJAaMm9C7A9YD3fty3QGcAZZvYPM/vYzC5vqgCPSUYG7NsHVVVh+XgRkdYilAdkW5BtLsh+egPDga7AB2aW45yrcemKmU0EJgJ07979mINtVHo6OAf79x+9qUlE5CQUSs89H+gWsN4VKAjSZp5zrtw5twXYgJfsa3DOzXTO5Tnn8rKyso435vqpeJiICBBacl8K9DaznmaWANwAzK/V5g1gBICZZeIN02xuykBDorK/IiJACMndOVcB3AUsBNYDc5xza81sipmN9jVbCOwxs3XAIuA+59ye5gq6Xuq5i4gAoY2545xbACyote2hgGUHTPJN4aOyvyIiQDTeoQrquYvISS+6kruGZUREgGhL7ikpEBurYRkROelFV3I3U/EwERGiLbmDyv6KiBCNyV09dxGRKEzuKh4mIhKlyV3DMiJykou+5K5hGRGRKEzu6rmLiERhck9PhyNHoLQ03JGIiIRN9CV33aUqIhLFyV1DMyJyEou+5K7iYSIiUZjc1XMXEYnC5K6eu4hIaMndzC43sw1mtsnMJgd5/RYzKzSzlb7ph00faoh0QlVEpPEnMZlZLDADuBTvQdhLzWy+c25draavOufuaoYYj42eoyoiElLPfRCwyTm32TlXBswGxjRvWCcgLs6r666eu4icxEJJ7l2A7QHr+b5ttX3HzFab2Wtm1q1JojteuktVRE5yoSR3C7LN1Vr/C9DDOZcL/B14IeiOzCaa2TIzW1ZYWHhskR4L1ZcRkZNcKMk9HwjsiXcFCgIbOOf2OOeO+FafBc4NtiPn3EznXJ5zLi8rK+t44g2Nyv6KyEkulOS+FOhtZj3NLAG4AZgf2MDMOgWsjgbWN12Ix0HDMiJykms0uTvnKoC7gIV4SXuOc26tmU0xs9G+ZveY2VozWwXcA9zSXAFv2rOJca+No+hQUf2NNCwjIie5Ri+FBHDOLQAW1Nr2UMDyvwL/2rShBXfHW3fw7pZ3aZ/UnhlXzQjeSD13ETnJRdwdqr8f/XvS2qTx3tb3KK8sD94oPR3274fKypYNTkSklYi45N4jvQcvfPsFPi/6nGkfTwveSPVlROQkF3HJHWDMWWMYfeZoHn7/YbYVb6vbQMldRE5yEZncAaZfPh2An7zzk7ov+ksQTJ8ORQ2ceBURiVIRm9xPTT+Vh4c9zLwN85j3+byaL3byXZk5fTrMmtXywYmIhFnEJneAewffS052Dne/fTclZSVHXzj3XLjsMoiNhcGDwxegiEiYRHRyj4+N55mrnmH7/u1MeX/K0RfM4I9/hPbt4e67oawsfEGKiIRBRCd3gCHdh3DrgFt58uMnWbNrzdEXsrPh97+HVavg4YfDFp+ISDhEfHIHeHzk46S1SePOt+6kylUdfWH0aLj1Vnj8cfjww/AFKCLSwqIiuXdo24Gpo6byj+3/YNantU6gPvkknHoq3HQTHDgQngBFRFpYVCR3gJv73czQU4dy/9/vr1l3JjUV/vQn2LYNJk0KX4AiIi0oapK7mfH0VU+z/8h+7v/b/TVfHDIE7r/fG4OfPz/4DkREokjUJHeAPll9uO+C+5i1chaLty2u+eIjj0C/fnDbbbB7d3gCFBFpIVGV3AEeHPogPdJ7cNtfbuPRDx49OkSTkAAvvgj79sHEieBqP0xKRCR6RF1ybxvflhlXzuCLPV/w4HsP8tN3forzJ/KcHPjP/4R58+APfwhrnCIizSnqkjvAlb2vZN4N8+iU0okX17zIiBdGsHrXau/Fe++FESPgnntgy5bwBioi0kxCSu5mdrmZbTCzTWY2uYF215qZM7O8pgvx+Iw+czTbf7qdZ656hs92f8aA3w3gzjfvpOjwN16vPSbGuzxSNd9FJAo1mtzNLBaYAVwB9AHGmVmfIO1S8R6xt6SpgzxesTGx3J53Oxvv3shd37qLZ1c8S++nevPUznlUPPUb78amCy6AHTvCHaqISJMKpec+CNjknNvsnCsDZgNjgrT7JfAEcLgJ42sSGUkZ/OaK37DqjlXkdc7jnnfuoX/JVN4d2w8++QTOPx/y88MdpohIkwnlGapdgO0B6/nAeYENzGwA0M0596aZ/ay+HZnZRGAiQPfu3Y892hPUN7svfx3/V+ZtmMekhZMY2W8LMbmQVLad5Ke6k5SRTVJqBklxSSTFJ9E2vi0xFkN6m3RmXDWDzLaZLR6ziMjxCKXnbkG2VV9HaGYxwJPA/2tsR865mc65POdcXlZWVuhRNiEz49tnfZt1P17H2LPGUmUwoMtAxhakMeyTXZyzN57OqZ1JjEvkYNlBlhUsY866Odzx5h1hiVdE5HiE0nPPB7oFrHcFCgLWU4Ec4P/MDKAjMN/MRjvnljVVoE0tMS6RmVfP5Pyu5zNhwAQyb2rjFRn79z/D2F7wh5ehXTsKDhRwyR8vYe76ubz62atcn3N9uEMXEWmUuUZu5jGzOOAL4BJgB7AUuNE5t7ae9v8H/KyxxJ6Xl+eWLWtlud85r9DY/ffD6afD//4v9O3LofJDXPHSFXy0/SPeuP4NrjrjqnBHKiInKTNb7pxr9IrERodlnHMVwF3AQmA9MMc5t9bMppjZ6BMPtRUx84qLvfuu93Dt886DV1+lbXxb/jLuL/Q7pR/X/vla3t/6frgjFRFpUKM99+bSKnvugXbsgO9+Fz76CC69FF5+maK2MOwPw9i+bzvv3fweeZ3Dfjm/iJxkmqznftLq0gUWLYIBA+Bvf4MHHiCzbSZ/Hf9XMttmctmLl7F2d9CRKRGRsFNyb0hCglciuGNHeP112LGDLu268Peb/k6b2DZc+qdL2bx3c7ijFBGpQ8m9MV27ej34I0fge9+DykpOyziNv33/b5RVljHyjyMpOFDQ+H5ERFqQknsozjoLnn4a3n8ffvlLwLsh6p3x71B0qIhL/3Rpzac/iYiEmU6oHoubb/Ye2ffuu15lSeD9re9z2YuXkRCbwDnZ59ClXReyk7M5JfkUspOzq6dTUrz11IRUfPcDiIgcs1BPqCq5H4uSEsjLg/37YeVKyM4G4Pa/3M7MFTPpmd6ThNgEdh/czd7De4Puom1cWzq360znVG/qlNKpetm/np2cTXpiug4CIlJHqMk9lDtUxS8lBV591bv+/eab4a23ICaGRy95lF7te3l3uvrqz5RVllF0qIjdB3ezq2QXs1bO4tW1r3Ju53Pp0q4LBQcKWFawjIIDBRwqP1Tno+IsjqzkLLKSs8hOziarbRZZbb3lpPgk/rn9n+R1ziMuJo7DFYc5XHGY0orS6uXiw8VsLd7KeV3O49T0U733B+4rOYvk+GQdQESilHrux+Ppp+FHP4LHH/fuZg1B0aEiZn06q8YBAMA5x4GyAxQcKGDngZ3MXD6T2WtnM6LHCE7LOI3CQ4XsPribwoPe/EDZgaD7N4yk+CQS4xJJjEvkcMVhvin9hriYOCqqKoK+p01sGxLjEslOziYjKYPUhFRS26R684Blh2NJ/hLyOueRGJdIpauksqqyxrykrITPiz5neI/hdGvXjfTE9DpTSkJK9cHEOUdFVQXlVeWUV5bXWI6xGNIS05r94OOco7SilLiYOBJiE5rtc0SakoZlmpNzcN113uP6PvgABg9usl3XdxDwO1xxmA1FG3h17avclHsTXdp1ITEukbiYuBqJ0L+fW/rfQlJ8UvUBovBQYfX8rS/eYvFXizkn+xw6p3bmQNkBDhw5UGNeVlnWYLyGERsTi3OOStfwg09iiCEmJoaqqiqqqGr0u4i1WNIT00lLTPPmbbx5YlwiX+37itzsXFLapBBjMcRarDePia1eLi0v5aP8j+ie1p0jlUfYW7qX4sPF7D28t3q5vKoc8IbL2rdtT/uk9mQkZpCRlEH7xPZkJGXQJrYNa3avIa9zHm3j21LlqupMJWUlrN29ltvzbmfU6aNIjEts9O8TOR5K7s2tuNi7wck5+PRTyMgId0THrLEDCXjDS18Vf8XLn73MTbk3kZ2STazFVidR/wGl6FARz614jrFnjSUmJobiw8VeIvUl0eLDxby96W0WbV3EiB4juKj7RcTHxhMfE098bDxxMXHEx8Tz3pb3eG39a1zZ60r6d+xP8eFi9h3ZV2O+fd929h3ZR3yM975KV0mVq6KyqhJH3f+ek+OT6ZzamYykDNIT08lIPDpftWsVb296m4u6X8Tp7U9nb+le9h7eyzel37C31JuXVpQe0/eaEJvAeV3OY+ipQxl66lAu6HYBKQkpx/4PJBKEkntL+OQTGDIErr4a5s71atNIvUI5mJxoG+ecl+hdJbsP7ublNS/zgwE/OKHP27F/B899+hzfz/0+WclZxFhMnWnPoT38dulvOb396az8eiWLty1mxc4VVLpKYi2WgZ0GMvTUoeRk57Akfwljzx5Lx5SO1cNogVN8TLzOhUi9lNxbytSpcN998C//As8/D2GqUy+tz4EjB/hn/j9ZvG0xi7ctZsmOJY0Oc/nFWAxxFkdsTGzQgwnAkYojJMUnVa/XVuWqKC0vJSk+idiYWAzDzKrnAFVVVRyqOERyfHL1fhwOf17w/xKqrKqktKKUpDhvX34W8LgHf5u28W2rPy+QmVFZ5Z2fSYpLwsyqD8T+4a3KKu88TnlVOfEx8cTExGAYMRZTI3YezesvAAANx0lEQVR/rGWVZbRLaEdifGL1r8DAucOx88BOTkk5hbiYuOqDf+BUVllG4aFC2ie1J8Zi6rzunKO8spz9ZftJa5NGfGx80H8T5xx7SveQmZRJQlxCne/bP6+orODMzDOZOmrqcT0ASFfLtJRJk+C55+DNN6FfP+8mpxtvhKSkcEcmYZbaJpVRp49i1OmjAO98yeJti5m7bi7DewynTVwb7yqn8qNXOR2uOMyirYt4d8u7XND9Ar7V5VtBx/iXFyzn4x0f0ze7L/079g/6+Su/XsmSHUvom92X3FNycc7VSdxrdq9hWcEyerXvRU52TnVCDjwAGMbqXav5pOATzupwFrmn5Fa/P9DqXatZWrCUMzqcQU52To3X/G0/2/0ZywqWcUaHM+jfsX91YvSfJ4mxGJbvXM4HX33A+V3P59zO51bH7U+0/uUVO1ewZMcSerXvxZkdzqw+IR8437J3C9v2baPKVdE9rXvQg+WW4i0UHCggPTGdXu171XjNf2D5vOhztu7bSve07pyVeVadf4/Kqko2frOR/P35JMcnc0rKKdXfde15wYECPl71MX2z+nLfkPua6L+2IJxzYZnOPfdcFzW2b3fuO99x7uyznQPn2rd37v77nduyJdyRSQQqPFjonvjwCVd4sLBVtFFMLR93Q4BlLoQcq2GZpuQcLF4MTz0Fb7zhrV99Ndx9N1x8scbkReSEqeRvOJjBsGHw2muwZQtMngz/+AeMHOnVpxkzxrvxqbDQS/wiIs0kpJ67mV0O/AaIBX7vnHus1ut3AD8GKoESYKJzbl1D+4zKnnswhw97d7X+2795DwDxS0+H3r3hjDOOTr17e1O7duGLV0RatSa7WsbMYvGeoXop3sOylwLjApO3mbVzzu33LY8GfuScu7yh/Z40yd2vsBB+9Svo3x927YKNG+GLL7zpq69q9uRTUyE3F845x+vxn3mmN+/eHWL0Y0vkZNaUV8sMAjY55zb7djwbGANUJ3d/YvdJhiB3kpzssrLgsceCv3b4MHz5pZfon30W3n4btm+Hdetgb0ABssREr4ffowfs2weXXAK9ekGnTken1FSN7YtISMm9C7A9YD0fOK92IzP7MTAJSAAuDrYjM5sITATo3r37scYavRIToW9fb7roIpg1CyZMgA4doKgIPv/cmzZs8Ob/+Afs2ePVl6+tbVvvyVGZmVBeDqNHwwUXeL8EOnZs+b9NRMIilGGZ64DLnHM/9K1/HxjknLu7nvY3+trf3NB+T7phmaZUVOTdMHXNNVBWBjt3Hp2+/tqbL1kCm2s9AjA727sW3z/l5nrDPQkqmiUSKZpyWCYf6Baw3hVo6Llys4GnQ9ivHK/MzJrVKPv0qdumqMj7BTBmjHcid9Uqb1q92rtU88iRo23btfP22b6992uhffuay23awJo1XiXMPn007CMSAULpucfhnVC9BNiBd0L1Rufc2oA2vZ1zG33LVwO/aOzIop57GJWXe+P7jz4Kr7ziDdv06AHffONNe/Z48+LiupdsJifD6acfnXr1OrrctSvEx4flTxI5WTRZz905V2FmdwEL8S6FfN45t9bMpuDdKTUfuMvMRgLlwF6gwSEZCbP4eG98f/p0r7LlhAlez722ykrvxO2GDfD733uXaX79tXfy9/PPYcGCmr8AzOCUU7wDRbduR6fu3Y8uZ2frih+RFqA7VOX4VVV5Qz5ffgkzZng3b+XlQVqad7XPV195VwIFMvOuHOrUyTsQ+Kfs7KPLXbt6BxKdCxCpQ4XDpPnFxBztkefkwKBBNX8FOOcN8Wzf7k3PP+894OTUU70rd3bt8n4V7NpV9yAQG+td9tm3rzfO77+aSElfJCTquUvL8Z/krT0M5BwcOAC7d8OTT8Jvf+vV4klNhbVrvV8G/v9O4+LgtNO8oaXcXG8/qane821rz9u1834JdOqkKp0SNVTPXSJTsANAaanXw1+71pvmzfNu8EpL84Z5Dhzwzg80JCMDunSBzp2Pzjt39g4Ey5bBxIlw9tk6HyCtnpK7RK/aBwDnvBO7Bw5AScnR+fPPe7X2L7/c6+0XFHjnCAoKvBPDtQ8IcXFe4u/atebUrp33KMU77/QOALoUVMJIyV2kvmEg8BL77t3eL4EXX/TG9YuLIT+/5lRa6/mpiYneOYMePbwpcLlXLz2JS5qdkrvIiXLOq+2zZg288IJXwK2oCLZuPToVFdV8T+/eMGqUd85g2DDvRjCRJqTkLtISSkq8Sz5/9StvGOjMM70e/8GD3vBNv34wYoSX7IcOVTlnOWFK7iItKXAIKC0Nli6F997zpo8+8s4JxMZ6l3Pu3+8N38TEeMNDVVU152VlXpsOHbzSD3Fx3ntjY48uV1V5l5D27OmdFG7TxrtE1D9PSPD2tWaNd4lqWpr33tpTaSl88IF3AEpL82Iy8yb/ckyMF8+iRXDFFd4Ql//98fE1l/1xBk4xMTXX/X+THBcld5HW4vBh+Oc/veQ4e7ZXy99ftsGf+ALnGzd6NYD69vVOBFdWelNFxdHlbdu8XwwdO3oPfikr8w4ggfPa9w60JvHx3uWqKSleSQv/PDnZe237du98hplXLqOsrO506JB3B3V6uncw8x+UAg9OlZXevRYdOtQsjRF4Ury83Ds4+9t4T0L2XvMvV1R4n3X22d4d1/4b7jp2rLncrl2zn3DXTUwirUViotczHjEC7rmn/pO8fg2dCD7WNs89B+PHewmwouLoVF7uzXfvhj//GcaO9Xruznm/CvxJzb/8zTfeJaiXXeYl4sB9+JffeMM7eF13HVx66dEDkf8XSWWl90tmwQIYMsS78a2kxBvC8s937fIOWoWFRw9e8fFHf40kJnoJNCHBq3q6caN3d/MZZ9SM2z9t2uQNk6WkeFdCQd16SZs3e22Sk4+28Sdo//zLL70DTnm5V4CvsND7vNpiYrxfJv5fNP5fW/4pNtZrd955MG1a/f92TUA9dxFpGk15UGqsTUt/Xu02/l8EX3/tHZB27fIObG+95Z1bGTiw5sHU/8urosK7Qmv1anjiCbjvvvr/vnpoWEZEpCU15UGpAUruIiJRKNTkrnutRUSikJK7iEgUCim5m9nlZrbBzDaZ2eQgr08ys3VmttrM3jWzU5s+VBERCVWjyd3MYoEZwBVAH2CcmdV+aOenQJ5zLhd4DXiiqQMVEZHQhdJzHwRscs5tds6V4T0Ae0xgA+fcIufcId/qx3gP0RYRkTAJJbl3AbYHrOf7ttXnVuDtEwlKREROTCh3qAa7lzbo9ZNmNh7IA4bV8/pEYCJA9+7dQwxRRESOVSjJPR/oFrDeFSio3cjMRgL/Bgxzzh0JtiPn3Exgpq99oZltO+aIPZlAUaOtWp9IjRsiN3bF3bIUd/ML6YKVRm9iMrM44AvgEmAHsBS40Tm3NqDNALwTqZc75zYeb8ShMrNloVzE39pEatwQubEr7paluFuPRsfcnXMVwF3AQmA9MMc5t9bMppjZaF+z/wZSgD+b2Uozm99sEYuISKNCqgrpnFsALKi17aGA5ZFNHJeIiJyASL1DdWa4AzhOkRo3RG7sirtlKe5WImyFw0REpPlEas9dREQaEHHJvbE6N62VmW01szW+E86tttaxmT1vZrvN7LOAbe3N7G9mttE3zwhnjMHUE/fDZrbD952vNLMrwxljMGbWzcwWmdl6M1trZj/xbW/V33kDcbfq79zMEs3sEzNb5Yv7Ed/2nma2xPd9v2pmCeGO9URF1LCMr87NF8CleNffLwXGOefWhTWwEJjZVrz6O636WlozGwqUAH90zuX4tj0BfOOce8x3QM1wzj0Qzjhrqyfuh4ES59zUcMbWEDPrBHRyzq0ws1RgOfBt4BZa8XfeQNzfpRV/52ZmQLJzrsTM4oEPgZ8Ak4D/dc7NNrNngFXOuafDGeuJirSee6N1buTEOOcWA9/U2jwGeMG3/ALe/8StSj1xt3rOuZ3OuRW+5QN4lxt3oZV/5w3E3ao5T4lvNd43OeBivHt1oBV+38cj0pL7sda5aU0c8FczW+4rwxBJTnHO7QTvf2ogO8zxHIu7fKWon29tQxu1mVkPYACwhAj6zmvFDa38OzezWDNbCewG/gZ8CRT77umByMor9Yq05B5ynZtWaIhzbiBe6eQf+4YRpHk9DZwO9Ad2Ar8Kbzj1M7MUYC5wr3Nuf7jjCVWQuFv9d+6cq3TO9ccrpTIIODtYs5aNqulFWnIPqc5Na+ScK/DNdwOv4/1HFSl2+cZY/WOtu8McT0icc7t8/yNXAc/SSr9z39jvXOAl59z/+ja3+u88WNyR8p0DOOeKgf8DBgPpvlIrEEF5pSGRltyXAr19Z7YTgBuAVl/qwMySfSedMLNkYBTwWcPvalXmAzf7lm8G5oUxlpD5k6PPWFrhd+47wfccsN459+uAl1r1d15f3K39OzezLDNL9y0nASPxzhcsAq71NWt13/fxiKirZQB8l1ZNA2KB551zj4Y5pEaZ2Wl4vXXwSj683FrjNrNXgOF4VfJ2Ab8A3gDmAN2Br4DrnHOt6uRlPXEPxxsecMBW4Hb/OHZrYWYXAh8Aa4Aq3+af441ft9rvvIG4x9GKv3Mzy8U7YRqL17md45yb4vt/dDbQHu/JcuPrq24bKSIuuYuISOMibVhGRERCoOQuIhKFlNxFRKKQkruISBRSchcRiUJK7iIiUUjJXUQkCim5i4hEof8PXjV0yKsgAo4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.0001,epochs=15,bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8856"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accuracy on test set\n",
    "y_pred=nn.predict(X_test)\n",
    "accuracy(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With this, I have matched the same accuracy (and even better log loss) on validation set from Pytorch NN implementation [here](https://github.com/anhquan0412/basic_model_scratch/blob/master/NN_pytorch.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extra: combine dropout + l2 regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "act_obj = ReLU()\n",
    "layers = [X_train.shape[1],100,10]\n",
    "nn = CustomNeuralNetwork(layers,act_obj,opt=Adam,keep_prob=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 1.0317066498215524, Val loss:0.4802829847588148\n",
      "Epoch 2. Training loss: 0.4599599231073201, Val loss:0.4201392364568962\n",
      "Epoch 3. Training loss: 0.4065128454115538, Val loss:0.3985676814960054\n",
      "Epoch 4. Training loss: 0.3775357577247236, Val loss:0.3972471776590397\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl81NW9//HXJyEhQIBAEoQCClgQQ4gQI+IDK7hc63LFVq2CO1apWkWLqNQVcSkqIuK11qVSr1qpV2uliqW/9iJe22oNLmGXiFhStiQssgWY5Pz++E5ilkkySSb5zkzez8djHrOdzHy+jL7nzPme7/macw4REYkvCX4XICIikadwFxGJQwp3EZE4pHAXEYlDCncRkTikcBcRiUMKdxGROKRwFxGJQwp3EZE41MGvN87IyHADBgzw6+1FRGLSsmXLSpxzmY218y3cBwwYQH5+vl9vLyISk8zs63DaaVhGRCQOKdxFROKQwl1EJA75NuYuIm3r0KFDFBUVUVZW5ncpEoaUlBT69etHUlJSs/5e4S7SThQVFdG1a1cGDBiAmfldjjTAOUdpaSlFRUUMHDiwWa+hYRmRdqKsrIz09HQFewwwM9LT01v0K0vhLtKOKNhjR0s/q9gL96VL4ZRToLjY70pERKJW7IX7nDmwZAnceqvflYhIE5SWljJixAhGjBhB79696du3b9X9gwcPhvUakyZNYu3atQ22eeqpp3jllVciUTInnngin332WUReq63F3g7VX/0KPvrIC/j9+6FTJ78rEpEwpKenVwXljBkzSE1NZdq0aTXaOOdwzpGQELrfOX/+/Ebf56c//WnLi40Dsddz79MHfvtb+Ne/4LHH/K5GRFqosLCQ7Oxsrr32WnJzc9m8eTOTJ08mLy+PYcOGMXPmzKq2lT3pQCBAWloa06dP55hjjuGEE05g27ZtANx1113MnTu3qv306dMZNWoURx11FH//+98B2Lt3L+effz7HHHMMEydOJC8vr9Ee+ssvv8zw4cPJzs7mjjvuACAQCHDZZZdVPT5v3jwAHn/8cbKysjjmmGO49NJLI/5vFo7Y67mDN+Z+wQXw0ENwxRXQv7/fFYnElptvhnCGGw4dgi1boHdvaGy+9YgREAzVplq1ahXz58/nV7/6FQCzZs2iZ8+eBAIBTj75ZC644AKysrJq/M2uXbsYO3Yss2bNYurUqbzwwgtMnz69zms75/jnP//JwoULmTlzJn/605948skn6d27N2+88Qaff/45ubm5DdZXVFTEXXfdRX5+Pt27d+e0007j7bffJjMzk5KSEpYvXw7Azp07AXjkkUf4+uuvSU5OrnqsrcVez73S7Nneda2fdSISQVu2wPr13nUrOvLIIznuuOOq7r/66qvk5uaSm5vL6tWrWbVqVZ2/6dSpE2eeeSYAxx57LBs2bAj52uedd16dNh988AETJkwA4JhjjmHYsGEN1vfRRx9xyimnkJGRQVJSEhdffDHvv/8+3/3ud1m7di033XQTixcvpnv37gAMGzaMSy+9lFdeeaXZByG1VGz23AGOOAKmT4d774XrroNx4/yuSCR2hNvDLimB+fNh0iTIyGi1crp06VJ1e926dTzxxBP885//JC0tjUsvvTTkfO/k5OSq24mJiQQCgZCv3bFjxzptnHNNqq++9unp6RQUFPDuu+8yb9483njjDZ599lkWL17M0qVLeeutt3jggQdYsWIFiYmJTXrPlmq0525mL5jZNjNbUc/zZmbzzKzQzArMrOHfN5F0661eyE+ZAvV8sCLSAhkZ3v9nrRjstX3zzTd07dqVbt26sXnzZhYvXhzx9zjxxBN57bXXAFi+fHnIXwbVjR49miVLllBaWkogEGDBggWMHTuW4uJinHP86Ec/4r777uOTTz6hvLycoqIiTjnlFB599FGKi4vZt29fxLehMeH03H8D/Bfw3/U8fyYwOHg5Hng6eN36OnXypkaef743i+aGG9rkbUWk9eTm5pKVlUV2djaDBg1izJgxEX+PG2+8kcsvv5ycnBxyc3PJzs6uGlIJpV+/fsycOZNx48bhnOOcc87h7LPP5pNPPuHHP/4xzjnMjIcffphAIMDFF1/M7t27qaio4Pbbb6dr164R34bGWDg/T8xsAPC2cy47xHPPAO85514N3l8LjHPObW7oNfPy8lxETtbhHJx+OuTnw7p1bdrDEIklq1ev5uijj/a7jKgQCAQIBAKkpKSwbt06Tj/9dNatW0eHDtE1Uh3qMzOzZc65vMb+NhJb0hfYWO1+UfCxBsM9YszgiScgJwfuusvrwYuINGDPnj2ceuqpBAIBnHM888wzURfsLRWJrQm1AELInwNmNhmYDHD44YdH4K2DsrLgxhu9kJ88GRqZ1iQi7VtaWhrLli3zu4xWFYmpkEVA9Ynm/YBNoRo65551zuU55/IyMxs9v2vTzJgBmZleyDdxT7iISLyJRLgvBC4PzpoZDexqbLy9VXTvDr/4Bfz9794RrCIi7Vg4UyFfBf4BHGVmRWb2YzO71syuDTZZBKwHCoHngOtbrdrGXHklHHecN3Vr927fyhAR8VujY+7OuYmNPO+A6FipJyEBnnwSRo+GBx+EWbP8rkhExBexu/xAfY4/3uvBz5njTY0Ukagwbty4OgckzZ07l+uvb/jHfmpqKgCbNm3iggsuqPe1G5taPXfu3BoHE5111lkRWfdlxowZzK5cDiWKxF+4gzf2npLiLY4kIlFh4sSJLFiwoMZjCxYsYOLEBgcHqnznO9/h9ddfb/b71w73RYsWkZaW1uzXi3bxGe69e3uzZxYtgnfe8bsaEQEuuOAC3n77bQ4cOADAhg0b2LRpEyeeeGLVvPPc3FyGDx/OW2+9VefvN2zYQHa2dxzl/v37mTBhAjk5OVx00UXs37+/qt11111XtVzwvffeC8C8efPYtGkTJ598MieffDIAAwYMoKSkBIA5c+aQnZ1NdnZ21XLBGzZs4Oijj+aaa65h2LBhnH766TXeJ5TPPvuM0aNHk5OTww9/+EN27NhR9f5ZWVnk5ORULVi2dOnSqpOVjBw5kt0R3k8YX7P2q7vhBnjuOa/3ftppEFw8SETg5j/dzGdbGl/y91D5Ibbs2ULv1N4kJTa8uuGI3iOYe0b9C5Klp6czatQo/vSnP3HuueeyYMECLrroIsyMlJQU3nzzTbp160ZJSQmjR49m/Pjx9Z5H9Omnn6Zz584UFBRQUFBQY8neBx98kJ49e1JeXs6pp55KQUEBU6ZMYc6cOSxZsoSMWkexL1u2jPnz5/PRRx/hnOP4449n7Nix9OjRg3Xr1vHqq6/y3HPPceGFF/LGG280uD775ZdfzpNPPsnYsWO55557uO+++5g7dy6zZs3iq6++omPHjlVDQbNnz+app55izJgx7Nmzh5SUlAb/fZsqPnvuAMnJ3kFNhYXw+ON+VyMSk7bs2cL6nevZsicyS/5WH5qpPiTjnOOOO+4gJyeH0047jX//+99s3bq13td5//33q0I2JyeHnJycqudee+01cnNzGTlyJCtXrmx0UbAPPviAH/7wh3Tp0oXU1FTOO+88/u///g+AgQMHMmLECKDhZYXBW19+586djB07FoArrriC999/v6rGSy65hJdffrnqSNgxY8YwdepU5s2bx86dOyN+hGz89tzBW3PmBz+ABx6Ayy6Dvn39rkgkKjTUw66uZF8J8z+dz6SRk8jo3PJ1m37wgx8wdepUPvnkE/bv31/V437llVcoLi5m2bJlJCUlMWDAgJDL/FYXqlf/1VdfMXv2bD7++GN69OjBlVde2ejrNLS+Vsdqv/gTExMbHZapzzvvvMP777/PwoULuf/++1m5ciXTp0/n7LPPZtGiRYwePZq//OUvDB06tFmvH0r89twrPfaYtxzw7bf7XYlIzMnonMGtY26NSLCDN/Nl3LhxXHXVVTV2pO7atYtevXqRlJTEkiVL+Prrrxt8nZNOOqnqJNgrVqygoKAA8JYL7tKlC927d2fr1q28++67VX/TtWvXkOPaJ510En/4wx/Yt28fe/fu5c033+R73/tek7ete/fu9OjRo6rX/9JLLzF27FgqKirYuHEjJ598Mo888gg7d+5kz549fPnllwwfPpzbb7+dvLw81qxZ0+T3bEh899wBBg3yDmp64AG49lo48US/KxJp1yZOnMh5551XY+bMJZdcwjnnnENeXh4jRoxotAd73XXXMWnSJHJychgxYgSjRo0CvLMqjRw5kmHDhtVZLnjy5MmceeaZ9OnThyVLllQ9npuby5VXXln1GldffTUjR45scAimPi+++CLXXnst+/btY9CgQcyfP5/y8nIuvfRSdu3ahXOOn/3sZ6SlpXH33XezZMkSEhMTycrKqjqrVKSEteRva4jYkr/h2LcPhg6F9HRvaeA2PiOKSDTQkr+xpyVL/sb/sAxA587eOVc/+wyef97vakREWl37CHeAH/3IO8/qnXfC9u1+VyMi0qraT7ibwbx5sGMH3HOP39WI+MKvYVhpupZ+Vu0n3AGGD4frr4enn4bg3nWR9iIlJYXS0lIFfAxwzlFaWtqiA5vaxw7V6rZvhyFDYNgweO89r0cv0g4cOnSIoqKiRud9S3RISUmhX79+JCXVPDK4Lc+hGlt69oSHHoKf/AReew0uusjvikTaRFJSEgMHDvS7DGkj7WtYptKPfwwjR8K0abB3r9/ViIhEXPsM98RE76QeRUXe8sAiInGmfYY7wJgxcOml8Oij8OWXflcjIhJR7TfcAR5+GJKS4JZb/K5ERCSi2ne4f+c7cPfd8NZbUOv0XyIisax9hzt4J/MYPBhuugkOHvS7GhGRiAgr3M3sDDNba2aFZjY9xPNHmNlfzazAzN4zs36RL7WVdOwIc+fC2rXeTlYRkTjQaLibWSLwFHAmkAVMNLOsWs1mA//tnMsBZgKxNQXlrLPg7LPhvvtg82a/qxERabFweu6jgELn3Hrn3EFgAXBurTZZwF+Dt5eEeD76zZ0LBw7Az3/udyUiIi0WTrj3BTZWu18UfKy6z4Hzg7d/CHQ1s/SWl9eGvvtdmDoVXnwRPvzQ72pERFoknHAPtfhK7QVppgFjzexTYCzwbyBQ54XMJptZvpnlFxcXN7nYVnfnnd4MmhtvhIoKv6sREWm2cMK9COhf7X4/YFP1Bs65Tc6585xzI4E7g4/tqv1CzrlnnXN5zrm8zMzMFpTdSlJTvYOa8vNh/ny/qxERabZwwv1jYLCZDTSzZGACsLB6AzPLMLPK1/o58EJky2xDEyd6R6/+/Oewc6ff1YiINEuj4e6cCwA3AIuB1cBrzrmVZjbTzMYHm40D1prZF8BhwIOtVG/rM/OmRJaUwIwZflcjItIs7W8993Bddx089xx8/rm39ruISBTQCbJb6v77oVs378hVnblGRGKMwr0+GRlewP/1r/D73/tdjYhIkyjcG/KTn0BOjjf/fd8+v6sREQmbwr0hHTrAvHnwr395UyRFRGKEwr0xY8d651mdNQs2bPC7GhGRsCjcw/Hoo5CQ4J1zVUQkBijcw9G/P9xxB7zxhreDVUQkyincw3XLLTBoEEyZAocO+V2NiEiDFO7hSkmBxx+HVavgl7/0uxoRkQYp3JvinHPg+9+He++Fbdv8rkZEpF4K96Yw807qsXevNwYvIhKlFO5NNXSod1LtF16Ajz/2uxoRkZAU7s1x993Qq5e3c1Un9RCRKKRwb45u3eDhh73T8b30kt/ViIjUoXBvrssug9Gj4fbb4Ztv/K5GRKQGhXtzJSR4685s2+atHikiEkUU7i1x3HFw1VXeDJo1a/yuRkSkisK9pR56CLp00Uk9RCSqKNxbqlcvuO8++POf4Y9/9LsaERFA4R4Z118PWVnws59BWZnf1YiIKNwjIinJ27m6fj089pjf1YiIKNwj5tRT4fzzvTH4jRv9rkZE2rmwwt3MzjCztWZWaGbTQzx/uJktMbNPzazAzM6KfKkxYPZs74jVW2/1uxIRaecaDXczSwSeAs4EsoCJZpZVq9ldwGvOuZHABKB9rok7YABMnw6/+x0sXep3NSLSjoXTcx8FFDrn1jvnDgILgHNrtXFAt+Dt7sCmyJUYY267DY44wlt3JhDwuxoRaafCCfe+QPVB5KLgY9XNAC41syJgEXBjqBcys8lmlm9m+cXFxc0oNwZ06uTtVC0ogGee8bsaEWmnwgl3C/FY7aN1JgK/cc71A84CXjKzOq/tnHvWOZfnnMvLzMxserWx4rzzvB2sd98NJSV+VyMi7VA44V4E9K92vx91h11+DLwG4Jz7B5ACZESiwJhkBk884S0odvfdflcjIu1QOOH+MTDYzAaaWTLeDtOFtdr8CzgVwMyOxgv3OB13CdOwYXDDDd7QzKef+l2NiLQzjYa7cy4A3AAsBlbjzYpZaWYzzWx8sNktwDVm9jnwKnClc1pohRkzICMDbrxR686ISJsyvzI4Ly/P5efn+/LeberXv4arr4ZXXoGLL/a7GhGJcWa2zDmX11g7HaHa2iZNgrw878CmPXv8rkZE2gmFe2tLSIAnn4RNm+DBB/2uRkTaCYV7Wxg9Gq64wpv/vm6d39WISDugcG8rs2ZBSoq3LLCISCtTuLeV3r3h3nvhnXe8i4hIK1K4t6Ubb4SjjoKbb4YDB/yuRkTimMK9LSUne0euFhZ6J9UWEWklCve29v3vw7nnwv33ezNoRERagcLdD3PmeMsB33ab35WISJxSuPth0CCYNs07avVvf/O7GhGJQwp3v/z859Cvn7eTtbzc72pEJM4o3P3SpYt3ztVPP4Xnn/e7GhGJMwp3P114IYwdC3feCdu3+12NiMQRhbufzGDePNixwzvASUQkQhTufsvJgeuug1/+0jvvqohIBCjco8HMmdCjB0yZopN6iEhEKNyjQc+e3nLAS5fC//yP39WISBxQuEeLq6+GESPglltg716/qxGRGKdwjxaJid5JPYqKvOWBRURaQOEeTU48ES65BB59FNav97saEYlhCvdo8/DD0KEDTJ3qdyUiEsPCCnczO8PM1ppZoZlND/H842b2WfDyhZntjHyp7UTfvnDXXfDWW7B4sd/ViEiMMtfI1DszSwS+AP4DKAI+BiY651bV0/5GYKRz7qqGXjcvL8/l5+c3q+i4d+AAZGd74/AFBd468CIigJktc87lNdYunJ77KKDQObfeOXcQWACc20D7icCr4ZUpIXXs6J3MY+1abyeriEgThRPufYGN1e4XBR+rw8yOAAYC/1vP85PNLN/M8ouLi5taa/ty9tlw1llw332wZYvf1YhIjAkn3C3EY/WN5UwAXnfOhVzD1jn3rHMuzzmXl5mZGW6N7dfcuVBW5i0PLCLSBOGEexHQv9r9fkB954ebgIZkImfwYG/WzG9+Ax9+6Hc1IhJDwgn3j4HBZjbQzJLxAnxh7UZmdhTQA/hHZEts5+68E/r08U7qUVHhdzUiEiMaDXfnXAC4AVgMrAZec86tNLOZZja+WtOJwALX2PQbaZquXb2DmvLzvR68iEgYGp0K2Vo0FbIJnPOOXl23Dr74AtLS/K5IRHwSyamQ4jczb0pkSYk3e0ZEpBEK91iRmwuTJ3shvyrk8WMiIlUU7rHkgQe8MXid1ENEGqFwjyUZGXD//fDXv8Kbb/pdjYhEMYV7rLn2Whg+3Jv/vn+/39WISJRSuMeaDh1g3jz4+mt45BG/qxGRKKVwj0XjxsGFF3pnbPr6a7+rEZEopHCPVbNne1Mkp03zuxIRiUIK91jVvz/ccQe8/rq3g1VEpBqFeyybNg0GDoSbboJDh/yuRkSiiMI9lqWkwOOPw8qV8PTTflcjIlFE4R7rxo+H00+He+6Bbdv8rkZEooTCPdaZwRNPwN693vLAIiIo3OPD0KHeuPuvf+0tDSwi7Z7CPV7ccw/06qWTeogIoHCPH926eQc1ffghvPyy39WIiM8U7vHk8svh+OPh9tvhm2/8rkZEfKRwjycJCd5671u2eKtHiki7pXCPN8cdB1ddBXPnwpo1flcjIj5RuMejX/wCOneGm2/WST1E2imFezzq1cs71+rixfDHP/pdjYj4IKxwN7MzzGytmRWa2fR62lxoZqvMbKWZ/TayZUqT/fSnkJUFP/sZlJX5XY2ItLFGw93MEoGngDOBLGCimWXVajMY+Dkwxjk3DLi5FWqVpkhK8o5cXb8e5szxuxoRaWPh9NxHAYXOufXOuYPAAuDcWm2uAZ5yzu0AcM5pkZNocNppcN558OCDsHGj39WISBsKJ9z7AtWToSj4WHVDgCFm9jcz+9DMzohUgdJCjz3mHbF6221+VyIibSiccLcQj9WegtEBGAyMAyYCz5tZWp0XMptsZvlmll9cXNzUWqU5BgzwDmpasADef9/vakSkjYQT7kVA/2r3+wGbQrR5yzl3yDn3FbAWL+xrcM4965zLc87lZWZmNrdmaarbboPDD/fWnQkE/K5GRNpAOOH+MTDYzAaaWTIwAVhYq80fgJMBzCwDb5hmfSQLlRbo3NkbnikogGef9bsaEWkDjYa7cy4A3AAsBlYDrznnVprZTDMbH2y2GCg1s1XAEuBW51xpaxUtzXD++XDKKXDXXVCqj0Yk3pnz6QjGvLw8l6+1x9vWihUwYgRcc41OyycSo8xsmXMur7F2OkK1PcnO9g5ueuYZ+PRTv6sRkVakcG9v7rsP0tNhyhStOyMSxxTu7U1amrew2AcfwKuv+l2NiLQShXt7NGkSHHss3Hor7NnjdzUi0goU7u1RYqJ3Uo9Nm+Chh/yuRkRagcK9vTrhBO+0fI89BoWFflcjIhGmcG/PZs2Cjh29ZYFFJK4o3NuzPn3gnnvg7bdh0SK/qxGRCFK4t3dTpsCQId4p+Q4c8LsaEYkQhXt7l5zsndRj3TrvWkTigsJd4IwzYPx4uP9+bwaNiMQ8hbt45syBgwe9td9FJOYp3MVz5JEwbRq8/DL87W9+VyMiLaRwl2/dcQf07evtZC0v97saEWkBhbt8q0sXmD0bPvkEfv1rv6sRkRZQuEtNF10EJ53k9eJ37PC7GhFpJoW71GQG8+Z5wX7vvX5XIyLNpHCXuo45Bq69Fn75S1i+3O9qRKQZFO4S2syZ0L27TuohEqMU7hJaejo8+CC89x68/rrf1YhIEyncpX7XXOMN0dxyC+zd63c1ItIEYYW7mZ1hZmvNrNDMpod4/kozKzazz4KXqyNfqrS5ypN6bNwIDz/sdzUi0gSNhruZJQJPAWcCWcBEM8sK0fR3zrkRwcvzEa5T/PK978HFF8Mjj8BXX/ldjYiEKZye+yig0Dm33jl3EFgAnNu6ZUlUeeQR6NABpk71uxIRCVM44d4X2FjtflHwsdrON7MCM3vdzPpHpDqJDn37wp13wh/+AH/+s9/ViEgYwgl3C/FY7blxfwQGOOdygL8AL4Z8IbPJZpZvZvnFxcVNq1T8NXWqt7jYTTfBoUN+VyMijQgn3IuA6j3xfkCNRb+dc6XOucrT+DwHHBvqhZxzzzrn8pxzeZmZmc2pV/zSsSPMnQtr1ng7WUUkqoUT7h8Dg81soJklAxOAhdUbmFmfanfHA6sjV6JEjbPPhjPPhBkzYMsWv6sRkQY0Gu7OuQBwA7AYL7Rfc86tNLOZZjY+2GyKma00s8+BKcCVrVWw+MjM672XlXkLi4lI1DLn06HleXl5Lj8/35f3lha6/XZvBs2HH8Lxx/tdjUi7YmbLnHN5jbWLuSNU5/xjDhmPZPCfv/1PZrw3g9+v/j2F2wupcBV+l9Z+3HUX9OkDN94IFfp3F4lGHfwuoKlWbltJ6f5SPvr3RyxatwgXnLjTJakL2b2yyTksh5zDchjeazjDDxtOz049fa44DnXt6vXcL7sMXnwRJk3yuyIRqSXmhmVK9pUw/9P5TBo5ic5JnVm5bSXLty2nYGsBBVsL+Hzr52zfv72qfb9u/bzA75VTFfxD0oeQlJgUyc1pf5yDMWPgyy/hiy+8FSRFpNWFOywTc+HeGOccm/dsrgr7yuBfXbyaQxXe/OzkxGSOzji6KuwrL4d1OQyzUNP6JaRPPoG8PLj5Zpgzx+9qRNqFdhvu9TlYfpC1JWurQr9gm3e9afe3U/YzOmfU6eVnZWbRKalTm9UZc37yE3jhBfj8c8gKteSQiESSwj1MpftKawzrFGwtYMW2FewP7AcgwRIYkj6E4b2G1+jlH9H9CPXyAYqLYcgQrwf/5z970yVFpNUo3FugvKKc9TvW1+nlr9+xvqpNt47dagR+5Q7cbh27+Vi5T5580jtj0/DhMH48HHssHH20t1xBkvZtiESSwr0V7D6wmxXbVtTp6e86sKuqzYC0ATWGdoYfNpzBPQeTmJDoY+WtLBCAE06A2p9nhw4weDAMHeqFfeXlqKMgNdWfWkVinMK9jTjn2PjNxjo7cNeWrKXclQOQ0iGFYZnDagzrDO81nMwucbS+TkkJzJ8PP/qRN1SzerV3WbPGuy4shPLyb9v37/9t2FcP/8xMDe2INEDh7rOyQBmri1fXCf2te7dWtemd2rvODtyhGUPp2KGjj5W3koMHvYCvDPvq4b9v37ftevas29MfOhQGDICEmDvmTiTiFO5RauuerXWGdVYVr+JAubeoZoeEDgzNGFpnB27frn3jcwduRQUUFdXt6a9e7f0CqJSS4g3n1O7pDx7sPSfSTijcY0igIsC60nV1duD+a9e/qtr0SOlRY0gn57Acsntl0yW5i4+Vt7LS0tA9/Q0bvIOowOvNDxxYt6d/9NGQluZr+SKtQeEeB3aW7WTFthU1evnLty1nz8E9ABjGkT2PrBraGX6YF/qDegwiweJ4CGPfPu+o2No9/S++8IZ/KvXuXbenf/TR8J3vaFxfYpbCPU5VuAo27NzgBf3W5VW9/HWl6+pdZ6eyt9+jUw+fq29lgYDXq6/e06+8fPPNt+26dq0Z+JW3jzzSm+EjEsUU7u3MvkP7WLltZY2dt1pnJ8g57+Qiocb1N1U7qVhSUv1TN7vE8fCXxBSFu9RZZ6fysqZkjdbZqbRrF6xdW3dc/8sva07dPPzw+qduirQhhbvUK5x1djI7Z9bYedvu1tlTY2lcAAAJ10lEQVQ5cMCbulm7p79mDezf/2279PTQUzePOEJTN6VVKNylycJdZ6f2Dtx2tc5ORQVs3Fi3p796tXcgV6VOneqfutkxDo9jkDajcJeIKK8o58sdX3o7bxtYZ+eo9KM4EDjAiN4j6J3amx6depCWkkZaSho9Urzb1R9LTkz2cataSUlJ3Z7+6tXw9dfftklIgEGDQk/d1Jr4EgaFu7SqynV2KnfgLlq3iK92fkVqUioHKw5ysPxgg3/fOalzyNCveqzWc5X301LS6Nqxa2xN9dy3L/S4/hdfwKFD37br0yf01M0+fTR1U6oo3KVNVT9DVnqndMoCZewo28HOsp3s2B+8Dt4P9Vj1ttUXYgslwRLo3rF7neBv8MuiWtuoWd4hEID160P39nfv/rZdt26hp24OGqSpm+1QRMPdzM4AngASgeedc7PqaXcB8D/Acc65BpNb4S71Ka8oZ/fB3XW/AEJ9SYR4rixQ1uDrp3RIafavhu4p3Vv/V4NzsHlz6HH9zZu/bZecHHrq5pAhmroZxyIW7maWCHwB/AdQBHwMTHTOrarVrivwDpAM3KBwF7+UBcqqwj/kr4R6fjVUXipcRb2vbRjdOnZr9q+GlA4pLdv5vHOnF/S1e/rr13s7eysdcYR3UFZZGZx0krdEQ69e3uWww7zr1FQN98SgcMM9nN90o4BC59z64AsvAM4FVtVqdz/wCDCtibWKRFRKhxR6p/amd2rvJv9thatgz8E99X4hhPpS+KL0i6r7+w7ta/D1kxOTm/2rIS0ljcS0NBg92rtUd+AArFtXs6e/dKl3kNbf/17PP1TKt4Ef6lL5JdCrlzefXydeiSnhhHtfYGO1+0XA8dUbmNlIoL9z7m0zU7hLzEqwBLp17Ea3jt04giOa/PcHyw+yq2xX/b8SKr8kDni3S/eVUri9sOq5ynMA1Kdrcte6vxo69SCtY/D68DTShgygx0UjSTswkdR3/4qdeRbmwHbswEpLse07sO3bsdLt316XbsA2LPNuHzrktYea1927Y+npWHqGd8movGRimZlYRiYJmb28+2lpmCVgZhjWomtpnnDCPdS/btVYjpklAI8DVzb6QmaTgckAhx9+eHgVisSQ5MRkMrtkNutELM459h7aG/pXQj1DSet3rGfnZq/t7oO7Q7/wm0+EfrxL8NI/3Ap3BS/raz5cGry0opZ+QUTTdaAiQM5hOcz5/hwyOme02r9ZOOFeRM2Pvx9QbUEOugLZwHvBb9newEIzG1973N059yzwLHhj7i2oWyTumBmpyamkJqfSv3vYiVslUBGo8avhmfxneP7T57k853LOHXouzjkcLqLXFYFDuL17cLu/wX3zDW73bu/2nt24PXuC19VuHwrgzOsd1rjumIxLTcV16YJL7RK87ozr3AXXpbN36Ry8pHTEmUV8W2pct+Jrf7n9S14qeInhvYZz65hbI/7fUaVwwv1jYLCZDQT+DUwALq580jm3C6j6+jGz94Bpje1QFZHI6pDQgfTO6aR3TgdgwGkDGJI+hEkjJ7VqD7FJ9u6Fbdtg61bvuvZlS+XtIu9kLRUhdm4nJEBGRs19Ag1doux8vdWnDbemcKdCngXMxZsK+YJz7kEzmwnkO+cW1mr7HmGEu2bLiEiDysthx476vwhqX6ov61xd587hfQkcdpj3pRHlxw7oICYRaV/KyrzefrhfBtWPDq4uPT38WUTdurX5dNJIToUUEYl+KSnQv793aYxz3nLPjX0RLF/uXW/fHvp1kpPD+xKonE7ahovGKdxFpP0x886xm5bmrd7ZmEOHvIXhGvsyWL3aa1NWz1HS3bt7vwxGj4YnnvCGgVqJwl1EpDFJSd4Cbn36NN7WuYZ3HC9dCr/9LYwYAbf6O1tGRETCZebN0ElN9RZ3q62kBObPh0mtO1tG4S4i0pYyMlq1x14phhbFFhGRcCncRUTikMJdRCQOKdxFROKQwl1EJA4p3EVE4pDCXUQkDvm2cJiZFQNfN/PPM4CSCJbjJ21L9ImX7QBtS7RqybYc4Zxr9GwwvoV7S5hZfjirosUCbUv0iZftAG1LtGqLbdGwjIhIHFK4i4jEoVgN92f9LiCCtC3RJ162A7Qt0arVtyUmx9xFRKRhsdpzFxGRBkR1uJvZGWa21swKzWx6iOc7mtnvgs9/ZGYD2r7K8ISxLVeaWbGZfRa8XO1HnY0xsxfMbJuZrajneTOzecHtLDCz3LauMVxhbMs4M9tV7TO5p61rDIeZ9TezJWa22sxWmtlNIdrExOcS5rbEyueSYmb/NLPPg9tyX4g2rZdhzrmovACJwJfAICAZ+BzIqtXmeuBXwdsTgN/5XXcLtuVK4L/8rjWMbTkJyAVW1PP8WcC7gAGjgY/8rrkF2zIOeNvvOsPYjj5AbvB2V+CLEP99xcTnEua2xMrnYkBq8HYS8BEwulabVsuwaO65jwIKnXPrnXMHgQXAubXanAu8GLz9OnCqWRufijw84WxLTHDOvQ/Uc7ZgwNuu/3aeD4E0Mwvj3GRtL4xtiQnOuc3OuU+Ct3cDq4G+tZrFxOcS5rbEhOC/9Z7g3aTgpfZOzlbLsGgO977Axmr3i6j7IVe1cc4FgF1AeptU1zThbAvA+cGfzK+bWRincI9K4W5rrDgh+LP6XTMb5ncxjQn+rB+J10usLuY+lwa2BWLkczGzRDP7DNgG/D/nXL2fS6QzLJrDPdS3V+1vvXDaRINw6vwjMMA5lwP8hW+/zWNNrHwm4fgE71DvY4AngT/4XE+DzCwVeAO42Tn3Te2nQ/xJ1H4ujWxLzHwuzrly59wIoB8wysyyazVptc8lmsO9CKjee+0HbKqvjZl1ALoTnT+zG90W51ypc+5A8O5zwLFtVFukhfO5xQTn3DeVP6udc4uAJDPL8LmskMwsCS8MX3HO/T5Ek5j5XBrbllj6XCo553YC7wFn1Hqq1TIsmsP9Y2CwmQ00s2S8nQ0La7VZCFwRvH0B8L8uuGciyjS6LbXGP8fjjTXGooXA5cHZGaOBXc65zX4X1Rxm1rty/NPMRuH9/1Lqb1V1BWv8NbDaOTennmYx8bmEsy0x9Llkmlla8HYn4DRgTa1mrZZhHSLxIq3BORcwsxuAxXizTV5wzq00s5lAvnNuId5/BC+ZWSHet90E/yquX5jbMsXMxgMBvG250reCG2Bmr+LNVsgwsyLgXrwdRTjnfgUswpuZUQjsAyb5U2njwtiWC4DrzCwA7AcmRGnnYQxwGbA8OL4LcAdwOMTc5xLOtsTK59IHeNHMEvG+gF5zzr3dVhmmI1RFROJQNA/LiIhIMyncRUTikMJdRCQOKdxFROKQwl1EJA4p3EVE4pDCXUQkDincRUTi0P8HGz/tzIPmieIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.01,epochs=4,bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 0.3303902121454473, Val loss:0.3587046704807534\n",
      "Epoch 2. Training loss: 0.32510568397027445, Val loss:0.3585218106372046\n",
      "Epoch 3. Training loss: 0.3303213408327314, Val loss:0.3619360337709689\n",
      "Epoch 4. Training loss: 0.3315216302702767, Val loss:0.36253699023487734\n",
      "Epoch 5. Training loss: 0.3312165462050732, Val loss:0.3617571713215457\n",
      "Epoch 6. Training loss: 0.32735152571288756, Val loss:0.35801800676325085\n",
      "Epoch 7. Training loss: 0.3245480411498956, Val loss:0.35593411635916233\n",
      "Epoch 8. Training loss: 0.31960332516608797, Val loss:0.3548542739374896\n",
      "Epoch 9. Training loss: 0.31618157888076176, Val loss:0.355768617186535\n",
      "Epoch 10. Training loss: 0.3138719191455231, Val loss:0.34876174357725204\n",
      "Epoch 11. Training loss: 0.30795346218581066, Val loss:0.34871107310790367\n",
      "Epoch 12. Training loss: 0.3055837878880921, Val loss:0.3488325006732789\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl8VOXZ//HPlZ0sLBIQS6AkigiEADEiimURVEAFxQ1aVGgrasWNakWrVrELVX+K+Pho0YpWeUDUUqgN0FZR3KoEiigghSpI2EyiYAKBZJLr98eZCZNkkkySSSYzc71fr3nlnDNnzlyTwHfuuec+9xFVxRhjTHiJCnYBxhhjAs/C3RhjwpCFuzHGhCELd2OMCUMW7sYYE4Ys3I0xJgxZuBtjTBiycDfGmDBk4W6MMWEoJlhPnJqaqr169QrW0xtjTEhav359oap2aWi/oIV7r169yMvLC9bTG2NMSBKRXf7sZ90yxhgThizcjTEmDFm4G2NMGApan7sxpnWVl5eTn5/P0aNHg12K8UNCQgJpaWnExsY26fEW7sZEiPz8fFJSUujVqxciEuxyTD1UlaKiIvLz80lPT2/SMaxbxpgIcfToUTp37mzBHgJEhM6dOzfrU5aFuzERxII9dDT3bxV64f63v8G550JBQbArMcaYNiv0wn3BAlizBv7nf4JdiTGmEYqKihg0aBCDBg2iW7dudO/evWq9rKzMr2NMnz6dbdu21bvPU089xaJFiwJRMueccw4bN24MyLFaW+h9oXrllbBiBZx9drArMcY0QufOnauC8oEHHiA5OZk77rij2j6qiqoSFeW73blw4cIGn+emm25qfrFhIPRa7gMHOj+//Ta4dRhjAmLHjh1kZmZyww03kJ2dzb59+5gxYwY5OTn079+fOXPmVO3raUm7XC46duzI7NmzGThwIGeddRZff/01APfeey/z5s2r2n/27NkMGTKEPn368MEHHwBw+PBhLrvsMgYOHMiUKVPIyclpsIX+8ssvM2DAADIzM7nnnnsAcLlcXH311VXb58+fD8Djjz9Ov379GDhwIFOnTg3478wfoddy90w29sUXQS3DmJB2223gT3dDeTns3w/dukFD460HDQJ3qDbWli1bWLhwIc888wwAc+fO5YQTTsDlcjFq1Cguv/xy+vXrV+0xhw4dYsSIEcydO5dZs2bx/PPPM3v27FrHVlU+/vhjVqxYwZw5c1i1ahVPPvkk3bp14/XXX+eTTz4hOzu73vry8/O59957ycvLo0OHDowZM4Y33niDLl26UFhYyKeffgrAwYMHAXj44YfZtWsXcXFxVdtaW+i13JOToWtX+PLLYFdiTPjbv99pSO3f36JPc/LJJ3PGGWdUrS9evJjs7Gyys7PZunUrW7ZsqfWYdu3aMW7cOABOP/10du7c6fPYkyZNqrXPe++9x+TJkwEYOHAg/fv3r7e+jz76iHPPPZfU1FRiY2P54Q9/yNq1aznllFPYtm0bt956K6tXr6ZDhw4A9O/fn6lTp7Jo0aImn4TUXA223EXkeeAi4GtVzfRxvwBPAOOBI8A0Vd0Q6EKrSU+3cDemOfxtYRcWwsKFMH06pKa2WDlJSUlVy9u3b+eJJ57g448/pmPHjkydOtXneO+4uLiq5ejoaFwul89jx8fH19pHVRtVX137d+7cmU2bNrFy5Urmz5/P66+/zoIFC1i9ejXvvPMOy5cv59e//jWfffYZ0dHRjXrO5vKn5f4CMLae+8cBvd23GcDTzS+rARkZ1i1jTGtITYU772zRYK/pu+++IyUlhfbt27Nv3z5Wr14d8Oc455xzWLp0KQCffvqpz08G3oYOHcqaNWsoKirC5XKxZMkSRowYQUFBAarKFVdcwYMPPsiGDRuoqKggPz+fc889l0ceeYSCggKOHDkS8NfQkAZb7qq6VkR61bPLROBP6ry1/UtEOorISaq6L0A11paeDkuXgssFMaH3tYExpm7Z2dn069ePzMxMMjIyGDZsWMCf4+abb+aaa64hKyuL7OxsMjMzq7pUfElLS2POnDmMHDkSVeXiiy/mwgsvZMOGDfzkJz9BVRERfv/73+NyufjhD39IcXExlZWV3HXXXaSkpAT8NTRE/Pl44g73N+rolnkDmKuq77nX3wTuUtV6r8SRk5OjTb5Yx3PPwXXXOV0zdjUnY/yydetW+vbtG+wy2gSXy4XL5SIhIYHt27dz/vnns337dmLaWGPR199MRNarak5Djw3EK/F1jqzPdwwRmYHTdUPPnj2b/owZGc7PL76wcDfGNFpJSQmjR4/G5XKhqvzhD39oc8HeXIF4NflAD6/1NGCvrx1VdQGwAJyWe5Of0TNLmn2paoxpgo4dO7J+/fpgl9GiAjEUcgVwjTiGAodatL8doEcPiI62L1WNMaYO/gyFXAyMBFJFJB/4FRALoKrPALk4wyB34AyFnN5SxVaJiYGePa3lbowxdfBntMyUBu5XoPUnc7Cx7sYYU6fQO0PVw8a6G2NMnUI33NPT4euv4fDhYFdijPHDyJEja52QNG/ePH72s5/V+7jk5GQA9u7dy+WXX17nsRsaWj1v3rxqJxONHz8+IPO+PPDAAzz66KPNPk6ghXa4A9Qxn4Qxpm2ZMmUKS5YsqbZtyZIlTJlSb89vle9973u89tprTX7+muGem5tLx44dm3y8ti50w917rLsxps27/PLLeeONNzh27BgAO3fuZO/evZxzzjlV486zs7MZMGAAy5cvr/X4nTt3kpnpnEdZWlrK5MmTycrK4qqrrqK0tLRqvxtvvLFquuBf/epXAMyfP5+9e/cyatQoRo0aBUCvXr0oLCwE4LHHHiMzM5PMzMyq6YJ37txJ3759ue666+jfvz/nn39+tefxZePGjQwdOpSsrCwuvfRSvnVPTT5//nz69etHVlZW1YRl77zzTtXFSgYPHkxxcXGTf7e+hO6ofRvrbkyT3bbqNjbub3jK3/KKcvaX7Kdbcjdio+uf3XBQt0HMG1v3hGSdO3dmyJAhrFq1iokTJ7JkyRKuuuoqRISEhASWLVtG+/btKSwsZOjQoUyYMKHO64g+/fTTJCYmsmnTJjZt2lRtyt7f/OY3nHDCCVRUVDB69Gg2bdrELbfcwmOPPcaaNWtIrTFPzvr161m4cCEfffQRqsqZZ57JiBEj6NSpE9u3b2fx4sU8++yzXHnllbz++uv1zs9+zTXX8OSTTzJixAjuv/9+HnzwQebNm8fcuXP58ssviY+Pr+oKevTRR3nqqacYNmwYJSUlJCQk1Pv7bazQbbl36QKJiRbuxrSg/SX7+eLgF+wvCcyUv95dM95dMqrKPffcQ1ZWFmPGjGHPnj0cOHCgzuOsXbu2KmSzsrLIysqqum/p0qVkZ2czePBgNm/e3OCkYO+99x6XXnopSUlJJCcnM2nSJN59910A0tPTGTRoEFD/tMLgzC9/8OBBRowYAcC1117L2rVrq2r80Y9+xMsvv1x1JuywYcOYNWsW8+fP5+DBgwE/QzZ0W+4iNmLGmCaqr4XtrfBIIQv/vZDpg6eTmtj8mSEvueQSZs2axYYNGygtLa1qcS9atIiCggLWr19PbGwsvXr18jnNrzdfrfovv/ySRx99lHXr1tGpUyemTZvW4HHqm1/LM10wOFMGN9QtU5e//e1vrF27lhUrVvDQQw+xefNmZs+ezYUXXkhubi5Dhw7ln//8J6eddlqTju9L6Lbcwca6G9PCUhNTuXPYnQEJdnBGvowcOZIf//jH1b5IPXToEF27diU2NpY1a9awa9eueo8zfPjwqotgf/bZZ2zatAlwpgtOSkqiQ4cOHDhwgJUrV1Y9JiUlxWe/9vDhw/nLX/7CkSNHOHz4MMuWLeMHP/hBo19bhw4d6NSpU1Wr/6WXXmLEiBFUVlaye/duRo0axcMPP8zBgwcpKSnhv//9LwMGDOCuu+4iJyeHzz//vNHPWZ/QbbmDE+5vvQWqTkveGNPmTZkyhUmTJlUbOfOjH/2Iiy++mJycHAYNGtRgC/bGG29k+vTpZGVlMWjQIIYMGQI4V1UaPHgw/fv3rzVd8IwZMxg3bhwnnXQSa9asqdqenZ3NtGnTqo7x05/+lMGDB9fbBVOXF198kRtuuIEjR46QkZHBwoULqaioYOrUqRw6dAhV5fbbb6djx47cd999rFmzhujoaPr161d1ValA8WvK35bQrCl/PZ54wrkW5NdfO33wxpg62ZS/oac5U/6GfrcMWNeMMcbUENrhbmPdjTHGp9AOd8+FOqzlboxfgtUNaxqvuX+r0A735GSnr93C3ZgGJSQkUFRUZAEfAlSVoqKiZp3YFNqjZcDGuhvjp7S0NPLz8ykoKAh2KcYPCQkJpKWlNfnxoR/u6enw8cfBrsKYNi82NpZ0zyAEE/ZCu1sGnHDftQtcrmBXYowxbUboh3tGBlRUQH5+sCsxxpg2w69wF5GxIrJNRHaIyGwf939fRN4UkU0i8raINL2jqLFsrLsxxtTSYLiLSDTwFDAO6AdMEZF+NXZ7FPiTqmYBc4DfBbrQOtlYd2OMqcWflvsQYIeqfqGqZcASYGKNffoBb7qX1/i4v+X06AHR0dZyN8YYL/6Ee3dgt9d6vnubt0+Ay9zLlwIpItK5+eX5ISbGCXgLd2OMqeJPuPuabrHmWRB3ACNE5N/ACGAPUGv4iojMEJE8EckL6FhbG+tujDHV+BPu+UAPr/U0YK/3Dqq6V1Unqepg4JfubYdqHkhVF6hqjqrmdAnkLI42r7sxxlTjT7ivA3qLSLqIxAGTgRXeO4hIqoh4jnU38Hxgy2xAejocOACHD7fq0xpjTFvVYLirqguYCawGtgJLVXWziMwRkQnu3UYC20TkP8CJwG9aqF7fPCNmmjC5vjHGhCO/ph9Q1Vwgt8a2+72WXwNeC2xpjeA91r1//6CVYYwxbUXon6EKNtbdGGNqCI9w79IFEhPtS1VjjHELj3AXsREzxhjjJTzCHWysuzHGeAmfcPe03O0qM8YYE2bhXlICRUXBrsQYY4IufMLdRswYY0yV8Al3m9fdGGOqhF+4W8vdGGPCKNyTk53x7tZyN8aYMAp3sLHuxhjjFl7hbmPdjTEGCLdwT0+Hr76CiopgV2KMMUEVfuHuckF+frArMcaYoAqvcLex7sYYA4RbuNtYd2OMAcIt3Hv0gKgoa7kbYyJeeIV7bCz07Gktd2NMxAuvcAcb626MMfgZ7iIyVkS2icgOEZnt4/6eIrJGRP4tIptEZHzgS/WTjXU3xpiGw11EooGngHFAP2CKiPSrsdu9wFJVHQxMBv430IX6LT0dDhyAI0eCVoIxxgSbPy33IcAOVf1CVcuAJcDEGvso0N693AHYG7gSG8kzYmbnzqCVYIwxweZPuHcHdnut57u3eXsAmCoi+UAucLOvA4nIDBHJE5G8goKCJpTrBxvrbowxfoW7+NhW81p2U4AXVDUNGA+8JCK1jq2qC1Q1R1VzunTp0vhq/WFj3Y0xxq9wzwd6eK2nUbvb5SfAUgBV/RBIAFIDUWCjde0KiYnWcjfGRDR/wn0d0FtE0kUkDucL0xU19vkKGA0gIn1xwr2F+l0aIGLDIY0xEa/BcFdVFzATWA1sxRkVs1lE5ojIBPduPweuE5FPgMXANFWt2XXTeizcjTERLsafnVQ1F+eLUu9t93stbwGGBba0ZsjIgLffBlWnJW+MMREm/M5QBaflXlICRUXBrsQYY4IifMMdrGvGGBOxwjPcbay7MSbChWe4W8vdGBPhwjPck5MhNdXC3RgTscIz3MFmhzTGRLTwDXcb626MiWDhG+4ZGbBrF1RUBLsSY4xpdeEb7unp4HJBfn6wKzHGmFYX3uEO1jVjjIlI4RvuNtbdGBPBwjfce/SAqChruRtjIlL4hntsrBPwFu7GmAgUvuEONtbdGBOxwjvcbay7MSZChXe4Z2TA/v1w5EiwKzHGmFYV3uHuGQ65c2dQyzDGmNYWGeFuXTPGmAjjV7iLyFgR2SYiO0Rkto/7HxeRje7bf0TkYOBLbQIb626MiVANXkNVRKKBp4DzgHxgnYiscF83FQBVvd1r/5uBwS1Qa+N17QqJidZyN8ZEHH9a7kOAHar6haqWAUuAifXsPwVYHIjimk0EevWycDfGRBx/wr07sNtrPd+9rRYR+T6QDrzV/NICxMa6G2MikD/hLj62aR37TgZeU1Wf8+yKyAwRyRORvIKCAn9rbB7PWHetq2RjjAk//oR7PtDDaz0N2FvHvpOpp0tGVReoao6q5nTp0sX/KpsjPR2Ki6GoqHWezxhj2gB/wn0d0FtE0kUkDifAV9TcSUT6AJ2ADwNbYjN5RsxYv7sxJoI0GO6q6gJmAquBrcBSVd0sInNEZILXrlOAJaptrP/DxrobYyJQg0MhAVQ1F8itse3+GusPBK6sAPKEu32paoyJIOF9hipASgqkplrL3RgTUcI/3MFmhzTGRJzICHcb626MiTCREe7p6fDVV1Dhc/i9McaEncgJ9/Jy2LMn2JUYY0yriIxwt9khjTERJjLC3ca6G2MiTGSEe8+eEBVlLXdjTMSIjHCPjYUePazlboyJGJER7mBj3Y0xESVywt3GuhtjIkjkhHt6OuzfD6Wlwa7EGGNaXGSFO8DOnUEtwxhjWkPkhLuNdTfGRJDICXcb626MiSCRE+4nngjt2lnL3RgTESIn3EVsOKQxJmJETriDhbsxJmL4Fe4iMlZEtonIDhGZXcc+V4rIFhHZLCL/F9gyA8Qz1r2NXebVGGMCrcFrqIpINPAUcB6QD6wTkRWqusVrn97A3cAwVf1WRLq2VMHNkp4OxcXwzTfQuXOwqzHGmBbjT8t9CLBDVb9Q1TJgCTCxxj7XAU+p6rcAqvp1YMsMEBsxY4yJEP6Ee3dgt9d6vnubt1OBU0XkfRH5l4iMDVSBAWVj3Y0xEaLBbhlAfGyr2WkdA/QGRgJpwLsikqmqB6sdSGQGMAOgZ8+ejS622azlboyJEP603POBHl7racBeH/ssV9VyVf0S2IYT9tWo6gJVzVHVnC5dujS15qZLSYHUVGu5G2PCnj/hvg7oLSLpIhIHTAZW1NjnL8AoABFJxemmaZsJasMhjTERoMFwV1UXMBNYDWwFlqrqZhGZIyIT3LutBopEZAuwBrhTVYtaquhmsXA3xkQAf/rcUdVcILfGtvu9lhWY5b61bRkZsGwZVFRAdHSwqzHGmBYRWWeogtNyLy+HPXuCXYkxxrSYyAx3sK4ZY0xYi7xwt7HuxpgIEHnh3rMnREVZy90YE9YiL9xjYyEtzcLdGBPWIi/c4fjskMYYE6YiM9xtrLsxJsxFZrhnZMC+fVBaGuxKjDGmRURmuHuGQ+7cGdQyjDGmpUR2uFvXjDEmTEVmuNtYd2NMmIvMcD/xRGjXzlruxpiwFZnhLgK9elm4G2PCVmSGO9hYd2NMWIvccPeMddeaVww0xpjQF7nhnpEB330H33wT7EqMMSbgIjfcbTikMSaMWbhbuBtjwpBf4S4iY0Vkm4jsEJHZPu6fJiIFIrLRfftp4EsNME+425eqxpgw1OA1VEUkGngKOA/IB9aJyApV3VJj11dUdWYL1Ngy2reHzp2t5W6MCUv+tNyHADtU9QtVLQOWABNbtqxWYrNDGmPClD/h3h3Y7bWe795W02UisklEXhORHgGprqXZWHdjTJjyJ9zFx7aag8P/CvRS1Szgn8CLPg8kMkNE8kQkr6CgoHGVtoT0dNi1Cyoqgl2JMcYElD/hng94t8TTgL3eO6hqkaoec68+C5zu60CqukBVc1Q1p0uXLk2pN7AyMqC8HPbsCXYlxhgTUP6E+zqgt4iki0gcMBlY4b2DiJzktToB2Bq4EluQDYc0xoSpBsNdVV3ATGA1TmgvVdXNIjJHRCa4d7tFRDaLyCfALcC0lio4oCzcjTFhyq9x7qqaq6qnqurJqvob97b7VXWFe/luVe2vqgNVdZSqft5SBW/+ejOTXplEweEA9Nn37AlRUfalqjEm7ITcGaq3rb6NZZ8vY+yisZSWN/MaqHFxkJZmLXdjTNgJuXBfNGkR52ecz4Z9Gxj+wnB2H9rd8IPqY2PdjTFhKOTCvWtSV1ZfvZrlk5ezrXAbOc/m8N5X7zX9gDbW3RgThkIu3D0m9JnARz/9iA7xHTj3xXP5Q94fmnag9HTYtw9Km9nFY4wxbUjIhjtA3y59+fi6jznv5PO44W83cP1fr6esoqxxB/FcLHvXrsAXaIwxQRLS4Q7QMaEjKyav4O5z7mbBhgWMenEU+0v2+38Az3DIWbOgLZw1a4wxARDy4Q4QHRXNb0f/llcuf4WN+zeSsyCHdXvW+ffgnBw45RRYuRLGjIFvv23ZYo0xphWERbh7XNn/Sj748QfERsfyg4U/4MWNPqe4qS4uDt57D8aNg82bYeBAePfdli/WGGNaUFiFO8DAbgNZd906hvUcxrTl07h15a2UV5TX/6ATT4TcXPjwQ4iPh5Ej4b77nHlnjDEmBIVduAOkJqayeupqbjvzNuZ/PJ8LXr6AwiOFDT/wjDNgwwa45hr49a9h+HAbJmmMCUlhGe4AMVExPD72cV685EU+2P0BOQty2Lh/Y8MPTEmBhQthyRLYuhUGDYKXX275go0xJoDCNtw9rhl4De9OfxdXpYuz/3g2r3z2in8PvOoq+OQTpw/+6qth6lQ4dKhlizXGmAAJ+3AHOKP7GeTNyCP7pGwmvz6Z2f+cTUWlHxfo+P73Yc0amDPHackPGuT0yxtjTBsXEeEO0C25G29d+xbXn349v3//91y0+CK+LfVj2GNMjPPlqmcEzQ9+AA89ZFdvMsa0aRET7gBx0XE8c9EzPHPhM7z5xZsMeW4IWwq2+Pfgs86CjRth8mS4/34YNQq++qplCzbGmCaKqHD3uD7net669i2KjxVz5nNnsvzz5f49sEMH58vVl15ygj4rC5YubdlijTGmCSIy3AHO6XkOeTPyOC31NC555RJ6PN6DX/zjF3y852MqtbL+B0+d6oR7377OF68//jGUlLRO4cYY4wdR1aA8cU5Ojubl5QXlub2Vlpcy6ZVJrPrvqqptXRK7MK73OMafMp7zTz6fTu06+X5webnT//6b3zgTkP3f/zlj5Y0xpoWIyHpVzWlwv0gPd4DCI4Us/PdCJvSZQN7ePHJ35LJqxyq+Kf2GKIni7B5nM/6U8YzvPZ6sE7MQkeoHWLvWac3v2+eE/Z13QnR0cF6MMSasBTTcRWQs8AQQDTynqnPr2O9y4FXgDFWtN7nbUrj7UlFZwcd7PiZ3ey65O3LZsG8DAN1TujO+txP0o9NHkxKf4jzg22/h+uvh1Vdh2DA45xy44w5ITQ3iqzDGhJuAhbuIRAP/Ac4D8oF1wBRV3VJjvxTgb0AcMDPUw72mvcV7WbVjFbnbc/n7f/9OcVkxsVGxDP/+8Kqw73PCqciLLzohX17uzDb50ENwySWQkBDsl2CMCQOBDPezgAdU9QL3+t0Aqvq7GvvNA/4J3AHcEW7h7q28opz3d7/vtOq357K5YDMA6R3TnaCP7UefJxcTs30HsXv2E5PSgdgrriLm6mnEDhxMTFQM0RJdu3vHGGMaEMhwvxwYq6o/da9fDZypqjO99hkM3Kuql4nI29QR7iIyA5gB0LNnz9N3hcnVj3Yd3MXKHSvJ3Z7Lm1++yZHyI349LjYqlpioGGKj3T99rMdGxwJQqZXMu2AeozNGt+RLMca0cYEM9yuAC2qE+xBVvdm9HgW8BUxT1Z31hbu3UG651+eo6yi3rLyFZzc8yxX9rmBMxhhclS7Kiw9Svu4jXOs+orzgAK74GMr798WVlUn5SV0pr3Q5+1WU41L3z0oX5ZXlfF74OZ8Xfg7ApL6TeHDkg2R2zQzyKzXGBIO/4R7jx7HygR5e62nAXq/1FCATeNvdzdANWCEiExoK+HCUEJPAb0f/lt4n9Gb64OmkJnp9oXouoArr1sEf/wh/XAzFn0KfPvCTnzhTDZ94Yq1jFh4p5Ol1T1NcVswzec+wbOsypgyYwgMjHqB3596t9+KMMSHDn5Z7DM4XqqOBPThfqP5QVTfXsf/bRHDLvVEOH3ZG1zz3HLz/vjOPzcUXO0F/wQXOeg1FR4p45INHmP/RfMoqypg2aBr3j7ifnh16BuEFGGNam78t9wbPUFVVFzATWA1sBZaq6mYRmSMiE5pfagRLSoJp05zL/G3dCrff7oT8RRdBr15w7721LhbSObEzc8fM5Ytbv+CmM27ipU0v0fvJ3tycezP7ivcF5WUYY9oeO4mprSkvhzfecFrzq1ZBZaUzZj4jAx59FLp0qbb77kO7eWjtQzz/7+eJi45j5pCZ/GLYL6p3BxljwoadoRoO8vPhhRfgsceck6T69IGVKyE9vdauO77ZwZx35vDyppdJjkvm9qG3M+usWXRI6ND6dRtjWkzAumVMEKWlOV0zW7fCxIlO2Gdmwv/7f+ByVdv1lBNO4U+X/onPfvYZF5xyAXPWziH9iXR+9+7vOFx2OEgvwBgTLBbuoeDEE+Evf3FCfswYZ1qDIUNg/fpau/br0o9Xr3iVDTM2cHaPs7nnrXvImJ/BvH/N46jraBCKN8YEg3XLhBpV+POf4eab4cABuPVW5zKAyck+d/9w94fct+Y+3vzyTbqndOe+4fcxffB04qLjWrnwhpVXlHO4/DAlZSWUlJVwzHWMSq1s0q1CK6qtHzx6kLe/fJtR6aPo1K4T0RJNlEQRHRVNtEQTHeVe97Fcc1/PckxUDImxiaTEpZAQk2BnHJtWYX3u4e7gQbj7bnjmGedar//7vzB+fJ27r/lyDb9865d8mP8hPdv3JOd7OVzW7zI6xHdARBCcYPIElPe6r/t87VdaXlotnEvKSjhc5rVeXn295r5lFWUt87tqBdESTUp8CilxKSTHJVctV/10LyfHJVffHl/9vsTYROKj44mPiSc+Op7oKJtd1FRn4R4p3n8fZsyALVucSwDOm+fzRCgAVWXVjlVc99fr2FO8p1XKi5IokmKTSI5LrrolxR1fr3Wfe33NzjUs/mwx1wy8hsv7Xk6URDX65mmBR0kUh44eYtnny5jYZyLt49tXte6GYIKEAAALKElEQVQrKiuqWvm+lisqK6rt67381//8lVe3vMq4U8YxuNtgisuKnduxYkrKSqqWvbcdqzjWqN9fTFRMtbD3/EyISai1rdp90fFUaiXbirYxqNsg2se3J1qcTxuem+fTh2euI1/bfd3n/btxVbqq/R49P12VrlrbfN1XfKyYzws/56JTL+LUzqfSLbkb3ZK70T6+vX0SqoOFeyQpK4OHH3ZmoExMhEceca4OFeX7K5WCwwX87r3fMe6UcXRI6IDn34Di/qlabdlzn6/9POuLP13Mgg0LmDV0FjOHzKwK66Z2V3jm2K91lm8b0pQayyrKnOA/VvuN4JXPXmHplqVcfOrFDOsxjGMVxzjmOlb186jrqLPstf2o62i1fby3fXfsO45VHCNKohq+ulgbkxCTUBX03ZK70S3p+PJJKSdVLZ+YdCLxMfHBLrdVWbhHom3bnOmG33kHhg+HP/wBTjutVZ46FMK4rQv077Dm8Sq1sqrV7HLPZeRpSdfc7uu+lza9xNN5T3NDzg1MHTC1qiXv/d2Ep9Vfc1td931T+g1P5z3NmPQxHKs4xv6S/cdvh/dXWy88UujzdXZK6FQV9p0SOrGneA+npZ5Gu5h21fbz1cjwdCvWtV9peSlbC7fSN7UvCTHHp+321cCpuq+ORpBn21HXUZLjkvn1ub9u0t/Zwj1SqcLChc6ImsOH4Ze/hLvugvjIat2YwAv2G3hZRRlfH/66+htAjdvWwq18U/oNSbFJJMUlVT3WV855B3Jd+5WWl3LEdYTEmETaxbarFvx1fU/la5v340rKSvju2Hc8POZh7hx2Z6N/Dxbuke7AAWc6g8WLnQt5L1jgnOlqTBhr6U8/gdDcY1q4G8fKlXDjjbBrl9NlM3cudOwY7KqMMU1kZ6gax7hxsHkz/Pzn8OyzTiv+1Ved7htjTNjyZz53E+qSkpxJx6ZMcYZNXnkl9O7tzD6ZlgYnnODcOnc+vtypE8S14IlOqnDkCHzzje/bnj2Ql+fUGRvrTLfgfSsvr72tvvuPHYOSEmjXzplKWcT3LSqq7vtq3iornWNmZDi/u5QUaN/e+Vnfsmc9MdE5jjEtwMI9kpx+Onz0EVx6qTPz5Lx59bfgk5Nrh37Nm+e+Tp3g6NG6w/rbb2tvK6vnpKXoaKiogE8/hQ4dnECOjXV+1nVLTKx7v61bnTeLPn1gwADnddd1q6ys/37P7T//cbq7Kith/3747jsoLnZu/nwyiopyfsee4E9IcI6RkeF0nbVr57ymxMTjy7621Xd/XJy9gUQoC/dIExPjjKZZuBCuvdYZReMduEVFvsO5qMgJWs96jYnL6uR5g/Dc+vWr/QbRqVPtbYcPOzNiTp8OqQH4Iquw0HnNgTpefcf0fCrxDvvi4urrvu7btAn++1/nTbJ9e+cYpaXOzyNHnDeRxoqKct4oOnd2fs+NuaWk2BtDCLMvVE3jqTrdEd5vBC+8AIsWwc9+BjNntk7XTrip7w1I1elq8gS9d+jXtfz3vzvXBDjzTGeKim+/dW4HDx5fru8NIzraeWPwhH1SkvOYAQOcN4vk5OO3pKSG16NtKoVAsNEypnW1RMvYNE9DfxNV51ODJ+gbum3ZArt3O58qwHmDb8yniYSE2uEfF+fUecopTvdbQsLxW3x89XV/b8nJzrHC9FOHhbsxJrBqvlmoHv+i2nM7fLj+9Zrbtm1zvrfo2tUJ5aNHj99KS5s+qis62rlqWWrq8Z8NLYfIiX7+hrtffe4iMhZ4AogGnlPVuTXuvwG4CagASoAZqrql0VUbY9qu1FS40+uMSpHjreWmflprqCvK5aoe+L5ux44dX16+HF57zZl+4+STneMXFjrfZxQWOl2Idb1hJCfXDvykJOc6xmef7Wxr167uTwy+7ouPD9oniAZb7iISDfwHOA/IB9YBU7zDW0Taq+p37uUJwM9UdWx9x7WWuzEm4BrqiqqocALeE/oFBceXa64XFMDevc53Hc3h6V7yhH9MjPM9yLx5TXpTDGTLfQiwQ1W/cB94CTARqAp3T7C7JYGPSRuMMaal1fx0UZOnu6bGhebrVFjonPw3ebLTii8t9f3poTHbN250Bh8MHFh/rc3kT7h3B3Z7recDZ9bcSURuAmYBccC5vg4kIjOAGQA9e/ZsbK3GGNO6UlOdi+IEkvenixbkz/QDvjqMarXMVfUpVT0ZuAu419eBVHWBquaoak4Xf985jTEmnHg+XbTwqDJ/wj0f6OG1ngbsrWf/JcAlzSnKGGNM8/gT7uuA3iKSLiJxwGRghfcOItLba/VCYHvgSjTGGNNYDfa5q6pLRGYCq3GGQj6vqptFZA6Qp6orgJkiMgYoB74Frm3Joo0xxtTPr3HuqpoL5NbYdr/X8q0BrssYY0wz2HzuxhgThizcjTEmDFm4G2NMGAraxGEiUgDsauLDU4HCAJbTEtp6jW29PrAaA6Gt1wdtv8a2Vt/3VbXBE4WCFu7NISJ5/sytEExtvca2Xh9YjYHQ1uuDtl9jW6+vLtYtY4wxYcjC3RhjwlCohvuCYBfgh7ZeY1uvD6zGQGjr9UHbr7Gt1+dTSPa5G2OMqV+ottyNMcbUI+TCXUTGisg2EdkhIrODXY83EekhImtEZKuIbBaRNjstg4hEi8i/ReSNYNfii4h0FJHXRORz9+/zrGDX5E1Ebnf/jT8TkcUiktAGanpeRL4Wkc+8tp0gIv8Qke3un53aYI2PuP/Om0RkmYh0bEv1ed13h4ioiITEFeBDKtzdl/x7ChgH9AOmiEi/4FZVjQv4uar2BYYCN7Wx+rzdCmwNdhH1eAJYpaqnAQNpQ7WKSHfgFiBHVTNxJtSbHNyqAHgBqHl5y9nAm6raG3jTvR5ML1C7xn8AmaqahXNJzwBfHaNRXqB2fYhID5xLjX7V2gU1VUiFO16X/FPVMpy54ycGuaYqqrpPVTe4l4txAql7cKuqTUTScKZmfi7YtfgiIu2B4cAfAVS1TFUPBreqWmKAdiISAyRS/zUOWoWqrgW+qbF5IvCie/lFgnytBV81qurfVdXlXv0XzjUjgqKO3yHA48AvCKFLiIZauPu65F+bC08AEekFDAY+Cm4lPs3D+YdaGexC6pABFAAL3V1Hz4lIUrCL8lDVPcCjOK24fcAhVf17cKuq04mqug+cxgfQNcj1NOTHwMpgF+FNRCYAe1T1k2DX0hihFu5+XfIv2EQkGXgduK3GxcODTkQuAr5W1fXBrqUeMUA28LSqDgYOE/zuhCrufuuJQDrwPSBJRKYGt6rQJyK/xOnaXBTsWjxEJBH4JXB/Q/u2NaEW7o295F+rE5FYnGBfpKp/DnY9PgwDJojITpxurXNF5OXgllRLPpCvqp5PPa/hhH1bMQb4UlULVLUc+DNwdpBrqssBETkJwP3z6yDX45OIXAtcBPxI29b47JNx3sQ/cf+fSQM2iEi3oFblh1AL9wYv+RdMIiI4/cRbVfWxYNfji6rerappqtoL5/f3lqq2qVanqu4HdotIH/em0cCWIJZU01fAUBFJdP/NR9OGvvCtYQXHr4x2LbA8iLX4JCJjgbuACap6JNj1eFPVT1W1q6r2cv+fyQey3f9G27SQCnf3ly6eS/5tBZaq6ubgVlXNMOBqnNbwRvdtfLCLClE3A4tEZBMwCPhtkOup4v5E8RqwAfgU5/9R0M9iFJHFwIdAHxHJF5GfAHOB80RkO85oj7ltsMb/AVKAf7j/zzzTxuoLSXaGqjHGhKGQarkbY4zxj4W7McaEIQt3Y4wJQxbuxhgThizcjTEmDFm4G2NMGLJwN8aYMGThbowxYej/A8q7hNpGjxgvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.001,epochs=12,bs=512,l2=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.0001,epochs=15,bs=512,l2=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. Training loss: 0.27748013482975503, Val loss:0.32833993090317265\n",
      "Epoch 2. Training loss: 0.2753388743052632, Val loss:0.3281373677260899\n",
      "Epoch 3. Training loss: 0.2749341795034886, Val loss:0.3280311564041584\n",
      "Epoch 4. Training loss: 0.27629717595432346, Val loss:0.3279038519347437\n",
      "Epoch 5. Training loss: 0.2753399341957885, Val loss:0.32789766729990844\n",
      "Epoch 6. Training loss: 0.27516358246636874, Val loss:0.32786319515992557\n",
      "Epoch 7. Training loss: 0.2748811351650111, Val loss:0.3278714110469349\n",
      "Epoch 8. Training loss: 0.27488141172463326, Val loss:0.3278100096095242\n",
      "Epoch 9. Training loss: 0.27583689317370735, Val loss:0.3277322331345458\n",
      "Epoch 10. Training loss: 0.2745363724704435, Val loss:0.3277199367088059\n",
      "Epoch 11. Training loss: 0.27525607076267994, Val loss:0.3277069710313823\n",
      "Epoch 12. Training loss: 0.2759202771278635, Val loss:0.3276369788713515\n",
      "Epoch 13. Training loss: 0.2743973189452499, Val loss:0.3276622765719676\n",
      "Epoch 14. Training loss: 0.2745557499805126, Val loss:0.32755748265708134\n",
      "Epoch 15. Training loss: 0.27455638715129915, Val loss:0.3275401593097726\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl4VPXZ//H3ncm+QxIECWFRVEJkiSmiWAFXlhYVtYBalcfKhdWqD9WK1scq1qfWWotYqmIrdavUn1aliFKtVPRpiywisohAgBLCEpZAQgLJZO7fHyeZZJJJMoQJw0zu13XNNTPnnDlz54if853vOed7RFUxxhgTWaJCXYAxxpjgs3A3xpgIZOFujDERyMLdGGMikIW7McZEIAt3Y4yJQBbuxhgTgSzcjTEmAlm4G2NMBIoO1RdnZmZqr169QvX1xhgTllasWLFXVbNaWy5k4d6rVy+WL18eqq83xpiwJCLbAlnOumWMMSYCWbgbY0wEsnA3xpgIFLI+d2PMiVVdXU1RURFHjhwJdSkmAPHx8WRnZxMTE9Omz1u4G9NBFBUVkZKSQq9evRCRUJdjWqCq7Nu3j6KiInr37t2mdVi3jDEdxJEjR8jIyLBgDwMiQkZGxnH9yrJwN6YDsWAPH8f73yr8wn3BArj4Yti9O9SVGGPMSSv8wv355+Hjj2HOnFBXYow5Bvv27WPQoEEMGjSIrl270r17d+/7qqqqgNYxefJkNmzY0OIys2fP5rXXXgtGyVxwwQWsWrUqKOs60cLvgOqIEU7r/aqrQl2JMeYYZGRkeIPy4YcfJjk5mXvuucdnGVVFVYmK8t/unDt3bqvfc/vttx9/sREg/FruXbs6z3Fxoa3DGBMUmzZtIi8vj6lTp5Kfn8/OnTuZMmUKBQUF9O/fnxkzZniXrWtJu91u0tPTmT59OgMHDuS8885jz549ADz44IPMnDnTu/z06dMZMmQIZ555Jv/85z8BOHz4MFdffTUDBw5k0qRJFBQUtNpCf/XVVzn77LPJy8vjgQceAMDtdvP973/fO33WrFkA/OY3vyE3N5eBAwdyww03BH2bBSL8Wu5JSc5zeXlo6zAmnN19NwTS3VBdDbt2OY2q1s63HjQIakP1WK1bt465c+fy3HPPAfD444/TuXNn3G43I0eO5JprriE3N9fnMwcPHmT48OE8/vjjTJs2jRdffJHp06c3Wbeq8vnnnzN//nxmzJjBBx98wDPPPEPXrl156623+PLLL8nPz2+xvqKiIh588EGWL19OWloal1xyCQsWLCArK4u9e/fy1VdfAVBaWgrAE088wbZt24iNjfVOO9HCr+WenOw8Hz4c2jqM6Qh27YLCQue5HZ122ml861vf8r5//fXXyc/PJz8/n/Xr17Nu3bomn0lISGD06NEAnHPOOWzdutXvusePH99kmc8++4yJEycCMHDgQPr3799ifUuXLuWiiy4iMzOTmJgYrrvuOpYsWcLpp5/Ohg0buOuuu1i0aBFpaWkA9O/fnxtuuIHXXnutzRchHa9WW+4i8iLwHWCPqub5mS/A08AYoAK4WVVXBrtQr7pwt5a7MW0XaAt7716YOxcmT4bMzHYrJ6nuFzmwceNGnn76aT7//HPS09O54YYb/J7vHRsb633tcrlwu91+1x1X24XbcBlVPab6mls+IyOD1atX8/777zNr1izeeust5syZw6JFi/jkk0949913+fnPf86aNWtwuVzH9J3HK5CW+x+BUS3MHw30rX1MAZ49/rJaUPePwFruxrS/zEy49952DfbGDh06REpKCqmpqezcuZNFixYF/TsuuOAC3njjDQC++uorv78MGho6dCiLFy9m3759uN1u5s2bx/DhwykpKUFVufbaa3nkkUdYuXIlNTU1FBUVcdFFF/GrX/2KkpISKioqgv43tKbVlruqLhGRXi0scgXwsjq7tn+LSLqIdFPVnUGq0Ze13I2JaPn5+eTm5pKXl0efPn0YNmxY0L/jRz/6ETfeeCMDBgwgPz+fvLw8b5eKP9nZ2cyYMYMRI0agqnz3u99l7NixrFy5kltuuQVVRUT45S9/idvt5rrrrqOsrAyPx8N9991HSkpK0P+G1kggP09qw31BM90yC4DHVfWz2vd/B+5T1RbvxFFQUKBtulnHnj1wyinw29+CnfJkTMDWr19Pv379Ql3GScHtduN2u4mPj2fjxo1cdtllbNy4kejok+scE3//zURkhaoWtPbZYPwl/q6R9bvHEJEpOF035OTktO3b7ICqMeY4lZeXc/HFF+N2u1FVnn/++ZMu2I9XMP6aIqBHg/fZQLG/BVV1DjAHnJZ7m74tIQFErFvGGNNm6enprFixItRltKtgnAo5H7hRHEOBg+3W3w5OsCclWcvdGGNaEMipkK8DI4BMESkCfgbEAKjqc8BCnNMgN+GcCjm5vYr1Sk62lrsxxrQgkLNlJrUyX4ETe2QzKcnC3RhjWhB+V6iC03K3bhljjGlW+Ia7tdyNCSsjRoxockHSzJkz+eEPf9ji55Jrz5ArLi7mmmuuaXbdrZ1aPXPmTJ+LicaMGROUcV8efvhhnnzyyeNeT7CFZ7jbAVVjws6kSZOYN2+ez7R58+YxaVKLPb9ep556Km+++Wabv79xuC9cuJD09PQ2r+9kF57hbi13Y8LONddcw4IFCzh69CgAW7dupbi4mAsuuMB73nl+fj5nn3027777bpPPb926lbw85zrKyspKJk6cyIABA5gwYQKVlZXe5W677TbvcME/+9nPAJg1axbFxcWMHDmSkSNHAtCrVy/27t0LwFNPPUVeXh55eXne4YK3bt1Kv379uPXWW+nfvz+XXXaZz/f4s2rVKoYOHcqAAQO46qqrOHDggPf7c3NzGTBggHfAsk8++cR7s5LBgwdTVlbW5m3rT3ietW8HVI05Lnd/cDerdrU+5G91TTW7ynfRNbkrMa6WRzcc1HUQM0c1PyBZRkYGQ4YM4YMPPuCKK65g3rx5TJgwAREhPj6et99+m9TUVPbu3cvQoUMZN25cs/cRffbZZ0lMTGT16tWsXr3aZ8jexx57jM6dO1NTU8PFF1/M6tWrufPOO3nqqadYvHgxmY3GyVmxYgVz585l6dKlqCrnnnsuw4cPp1OnTmzcuJHXX3+dF154ge9973u89dZbLY7PfuONN/LMM88wfPhwHnroIR555BFmzpzJ448/zpYtW4iLi/N2BT355JPMnj2bYcOGUV5eTnx8fIvb91iFb8vdumWMaXe7yndRWFrIrvLgDPnbsGumYZeMqvLAAw8wYMAALrnkEnbs2MHuFu6TvGTJEm/IDhgwgAEDBnjnvfHGG+Tn5zN48GDWrl3b6qBgn332GVdddRVJSUkkJyczfvx4Pv30UwB69+7NoEGDgJaHFQZnfPnS0lKGDx8OwE033cSSJUu8NV5//fW8+uqr3ithhw0bxrRp05g1axalpaVBv0I2PFvu1i1jzHFpqYXd0N6Kvcz9Yi6TB08mM/H4R4a88sormTZtGitXrqSystLb4n7ttdcoKSlhxYoVxMTE0KtXL7/D/Dbkr1W/ZcsWnnzySZYtW0anTp24+eabW11PS+NrxTW445vL5Wq1W6Y57733HkuWLGH+/Pk8+uijrF27lunTpzN27FgWLlzI0KFD+eijjzjrrLPatH5/wrPlnpQElZVQUxPqSoyJaJmJmdw77N6gBDs4Z76MGDGC//qv//I5kHrw4EG6dOlCTEwMixcvZtu2bS2u58ILL/TeBHvNmjWsXr0acIYLTkpKIi0tjd27d/P+++97P5OSkuK3X/vCCy/knXfeoaKigsOHD/P222/z7W9/+5j/trS0NDp16uRt9b/yyisMHz4cj8fD9u3bGTlyJE888QSlpaWUl5ezefNmzj77bO677z4KCgr4+uuvj/k7WxK+LXeAigoIwVCaxpi2mzRpEuPHj/c5c+b666/nu9/9LgUFBQwaNKjVFuxtt93G5MmTGTBgAIMGDWLIkCGAc1elwYMH079//ybDBU+ZMoXRo0fTrVs3Fi9e7J2en5/PzTff7F3HD37wAwYPHtxiF0xzXnrpJaZOnUpFRQV9+vRh7ty51NTUcMMNN3Dw4EFUlf/+7/8mPT2d//mf/2Hx4sW4XC5yc3O9d5UKloCG/G0PbR7yF+C55+C226C4GLp1C25hxkQoG/I3/BzPkL/h2S1jw/4aY0yLwjvc7aCqMcb4FZ7hXncfVQt3Y45JqLphzbE73v9W4Rnu1i1jzDGLj49n3759FvBhQFXZt2/fcV3YFJ5ny1jL3Zhjlp2dTVFRESUlJaEuxQQgPj6e7OzsNn8+PMPdWu7GHLOYmBh69+4d6jLMCRLe3TLWcjfGGL8CCncRGSUiG0Rkk4hM9zO/p4j8XURWi8g/RKTtvyUCYd0yxhjTolbDXURcwGxgNJALTBKR3EaLPQm8rKoDgBnAL4JdqI+EBOdG2dYtY4wxfgXSch8CbFLVQlWtAuYBVzRaJhf4e+3rxX7mB1dUFCQmWsvdGGOaEUi4dwe2N3hfVDutoS+Bq2tfXwWkiEjG8ZfXAhv21xhjmhVIuPsbLb/xibL3AMNF5AtgOLADcDdZkcgUEVkuIsuP+3QsG/bXGGOaFUi4FwE9GrzPBoobLqCqxao6XlUHAz+tnXaw8YpUdY6qFqhqQVZW1nGUjd2NyRhjWhBIuC8D+opIbxGJBSYC8xsuICKZIlK3rvuBF4Nbph/WLWOMMc1qNdxV1Q3cASwC1gNvqOpaEZkhIuNqFxsBbBCRb4BTgMfaqd561nI3xphmBXSFqqouBBY2mvZQg9dvAm8Gt7RWJCfDruDc19EYYyJNeF6hCnZA1RhjWhC+4W7dMsYY06zwDXc7oGqMMc0K33BPSnJukO3xhLoSY4w56YRvuNeNDFlREdo6jDHmJBT+4W797sYY00T4hrsN+2uMMc0K33C3uzEZY0yzwjfcreVujDHNCt9wt5a7McY0K/zD3VruxhjTRPiGu3XLGGNMs8I33K1bxhhjmhW+4W4td2OMaVb4hntiovNsLXdjjGkifMM9KspGhjTGmGaEb7iDhbsxxjQjoHAXkVEiskFENonIdD/zc0RksYh8ISKrRWRM8Ev1w4b9NcYYv1oNdxFxAbOB0UAuMElEchst9iDOvVUH49xA+3fBLtQva7kbY4xfgbTchwCbVLVQVauAecAVjZZRILX2dRpQHLwSW2C32jPGGL8CuUF2d2B7g/dFwLmNlnkY+JuI/AhIAi4JSnWtsXA3xhi/Amm5i59p2uj9JOCPqpoNjAFeEZEm6xaRKSKyXESWl5SUHHu1jVm3jDHG+BVIuBcBPRq8z6Zpt8stwBsAqvovIB7IbLwiVZ2jqgWqWpCVldW2ihuyA6rGGONXIOG+DOgrIr1FJBbngOn8Rsv8B7gYQET64YR7EJrmrbCWuzHG+NVquKuqG7gDWASsxzkrZq2IzBCRcbWL/Ri4VUS+BF4HblbVxl03wWd97sYY41cgB1RR1YXAwkbTHmrweh0wLLilBSA52blBtsfjXLFqjDEGiIQrVMEJeGOMMV7hHe427K8xxvgV3uFuw/4aY4xf4R3udqs9Y4zxKzLC3bpljDHGR3iHu3XLGGOMX+Ed7tZyN8YYv8I73K3lbowxfoV3uNsBVWOM8Ssywt26ZYwxxkd4h3tiovNsLXdjjPER3uEeFeUEvLXcjTHGR3iHO9iwv8YY40f4h7sN+2uMMU1ERrhbt4wxxvgI/3C3bhljjGki/MPdWu7GGNNEQOEuIqNEZIOIbBKR6X7m/0ZEVtU+vhGR0uCX2gxruRtjTBOt3mZPRFzAbOBSoAhYJiLza2+tB4Cq/neD5X8EDG6HWv2zA6rGGNNEIC33IcAmVS1U1SpgHnBFC8tPwrlJ9olh3TLGGNNEIOHeHdje4H1R7bQmRKQn0Bv4+PhLC5B1yxhjTBOBhLv4mabNLDsReFNVa/yuSGSKiCwXkeUlJSWB1tiyupa7xxOc9RljTAQIJNyLgB4N3mcDxc0sO5EWumRUdY6qFqhqQVZWVuBVtqRu2N/KyuCszxhjIkAg4b4M6CsivUUkFifA5zdeSETOBDoB/wpuia2wYX+NMaaJVsNdVd3AHcAiYD3whqquFZEZIjKuwaKTgHmq2lyXTfuwYX+NMaaJVk+FBFDVhcDCRtMeavT+4eCVdQzsbkzGGNNEZFyhChbuxhjTQPiHe13L3bpljDHGK/zD3VruxhjTROSEu7XcjTHGK/zD3Q6oGmNME+Ef7tYtY4wxTYR/uCcmOs/WLWOMMV7hH+4uFyQkWMvdGGMaCP9wBxv21xhjGomMcLdhf40xxkdkhLvdjckYY3xERrgnJVm3jDHGNBAZ4W4td2OM8RE54W4td2OM8YqMcLcDqsYY4yMywt26ZYwxxkdkhLsdUDXGGB8BhbuIjBKRDSKySUSmN7PM90RknYisFZE/BbfMVtT1uZ/gO/wZY8zJqtXb7ImIC5gNXAoUActEZL6qrmuwTF/gfmCYqh4QkS7tVbBfyclOsFdW1o81Y4wxHVggLfchwCZVLVTVKmAecEWjZW4FZqvqAQBV3RPcMlthw/4aY4yPQMK9O7C9wfui2mkNnQGcISL/JyL/FpFRwSowIDbsrzHG+Gi1WwYQP9Mad25HA32BEUA28KmI5Klqqc+KRKYAUwBycnKOudhm2X1UjTHGRyAt9yKgR4P32UCxn2XeVdVqVd0CbMAJex+qOkdVC1S1ICsrq601N2Utd2OM8RFIuC8D+opIbxGJBSYC8xst8w4wEkBEMnG6aQqDWWiL7D6qxhjjo9VwV1U3cAewCFgPvKGqa0VkhoiMq11sEbBPRNYBi4F7VXVfexXdhB1QNcYYH4H0uaOqC4GFjaY91OC1AtNqHyeedcsYY4yPyLlCFaxbxhhjakVGuFvL3RhjfERGuFufuzHG+IiMcHe5ID7eumWMMaZWZIQ72LC/xhjTQOSEuw37a4wxXpET7tZyN8YYLwt3Y4yJQJET7tYtY4wxXpET7tZyN8YYr8gKd2u5G2MMEEnhnpRkLXdjjKkVOeFu3TLGGOMVOeFed0BVG98kyhhjOp7ICffkZCfYKytDXYkxxoRcZIU72EFVY4whksLdRoY0xhivgMJdREaJyAYR2SQi0/3Mv1lESkRkVe3jB8EvtRU2prsxxni1eps9EXEBs4FLgSJgmYjMV9V1jRb9s6re0Q41BsbuxmSMMV6BtNyHAJtUtVBVq4B5wBXtW1YbWMvdGGO8Agn37sD2Bu+Laqc1drWIrBaRN0WkR1CqOxZ2QNUYY7wCCXfxM63xyeR/BXqp6gDgI+AlvysSmSIiy0VkeUlJybFV2ho7oGqMMV6BhHsR0LAlng0UN1xAVfep6tHaty8A5/hbkarOUdUCVS3IyspqS73Ns24ZY4zxCiTclwF9RaS3iMQCE4H5DRcQkW4N3o4D1gevxADZAVVjjPFq9WwZVXWLyB3AIsAFvKiqa0VkBrBcVecDd4rIOMAN7Adubsea/bNuGWOM8QroPHdVXaiqZ6jqaar6WO20h2qDHVW9X1X7q+pAVR2pql+3V8Hvfv0uQ38/lJLDjfrso6MhPt5a7sYYQxheofrq6ldZumMpDy1+qOlMG/bXGGOAMAz3py5/isToRLYf2t50pg37a4wxQBiGe4+0Htw19C7e3/Q+W0u3+s60+6gaYwwQhuEOcFvBbQjC75b9zneGtdyNMQYI03DvkdaD8f3G88LKFzhc1aClHhsLGzbA3r2hK84YY04CYRnuAHedexelR0p5dfWr9RMPHYKtW+GFF0JWlzHGnAzCNtzP73E++d3ymfX5LLTu1nr33+88JyaGrjBjjDkJhG24iwh3DrmTdSXr+HjLx87ECRMgPx9++1twu0NboDHGhFDYhjvAhLwJZCVm8fTSp50JIvDgg7BpE7zxRmiLM8aYEArrcI+PjmdqwVQWfLOAzfs3OxOvuALy8uCxx8DjCW2BxhgTImEd7gBTC6biinIxe9lsZ0JUFPz0p7BuHbz9dmiLM8aYEAn7cD815VSuzb2WP3zxB8qOljkTr70W+vaFn/8ctPHQ88YYE/nCPtwB7jz3Tg4dPcTLX77sTHC54IEHYNUqWLgwtMUZY0wIRES4D80eypDuQ3jm82fwaG0/+/XXQ8+e8Oij1no3xnQ4ERHuAHcOuZMN+zbwt81/cybExMD06bB0KXz8cWiLM8aYEyxiwv3a/tfSJakLU/46pf7MmZtvhlNPdfrejTGmA4mYcI91xXJp70vZfmg75/3hPDbt3+TcvOPee+Ef/4DPPgt1icYYc8IEFO4iMkpENojIJhGZ3sJy14iIikhB8EoM3MzRM5l6zlTcHjdDXhjiXLl6662QleWc926MMR1Eq+EuIi5gNjAayAUmiUiun+VSgDuBpcEuMlCZiZk8+51nWT5lOd1SunH5q5fz/PpXYdo0+OADuOsuGzHSGNMhBNJyHwJsUtVCVa0C5gFX+FnuUeAJ4EgQ62uTPp368K9b/sWlfS5l6ntTubPfVtyJ8TBrFvzud62vwBhjwlwg4d4daHhPu6LaaV4iMhjooaoLgljbcUmNS+Wvk/7KtKHTeGbV84yZcSb/7iGs+/QvbN+zkdIjpdR4akJdpjHGtIvoAJYRP9O8J46LSBTwG+DmVlckMgWYApCTkxNYhcfBFeXi15f/mtysXKYumMp5tyjwJTx7hneZhOgEMhMz+c4Z32HGyBlkJma2e13GGNPeAmm5FwE9GrzPBoobvE8B8oB/iMhWYCgw399BVVWdo6oFqlqQlZXV9qqP0S35t/Dj838MwPXRg3lhPjx1+AIeGf4wt3/rdsqrynl2+bM8t/y5E1aTMca0p0Ba7suAviLSG9gBTASuq5upqgcBb3NXRP4B3KOqy4Nb6vG55/x7yEjIYPLgyWTu/wX86inI+A7c9zMu6n0RY/80lo37N4a6TGOMCYpWW+6q6gbuABYB64E3VHWtiMwQkXHtXWCwZCZmcu+we51ul1/9CiZNcq5gfeUVRvcdzT3n38PLX75cf4WrMcaEMdEQjbtSUFCgy5eHsHF/9CiMGQNLlsCCBRy5eDiDnx9MeVU5a25bQ1p8WuhqM8aYZojIClVt9VqiiLlC9ZjFxTnjvffvD+PHEz/9QV4a8TTFZcVMWzQt1NUZY8xx6bjhDpCaCu+/74wa+etfM2Thl0wfNp0XV73Iwo02VLAxJnx17HAH6NbN6XsH6N2bh4Y/RF6XPG79660cqDwQ2tqMMaaNLNwB7rsPevWCxx4jLiqGl658id3lu7l70d2hrswYY9okkFMhI19cHPzv/8J118Grr5J/44389Ns/ZcaSGVzd72rGndn8SUE1nhp2lO1gy4EtFB4opPBAIev3rmddyTpOST4FgKqaKu+jsrqSsqoyuiV3Izk2mfjoeOKj4wHYWb6Ti3tdzKBug+jbuS+ndz7dDuwaY9qk454t05jHA+eeC7t3w4YNVMW6yH8+n2/2fUPvTr2JiYohSqK8jxqtofhQMaVHSnGr27uaKIkiNS6V0iOl5KTl0Cu9F7GuWO9j8/7NfLXnK87MOJNuKd044j7CUfdRdhzawZ6KPU3KykrMold6L2eky1OHkJOeQ2ZiJlmJWWQlZZGZmEl6fDqJMYkkxSThinK1+GfWeGqodFeiqqTEpQR9Mxpj2legZ8tYy71OVJRz/vvIkTBrFrH33ceYvmNYW7KWOFccZ2ScgUc93sfG/RvZW7mXET1HMOnsSfTp1Ife6b3JScvh4NGDzP1irnPBVKPhDPZW7PU7r276hLwJHDxykI37N7Jx30Y27t/I4i2LKSwtZG3JWqpqqlr8M2JdsSTFJJEUm0RsVCwHjhwgxhXDUfdRKt2VPp/vl9mP0aeP5rLTLuPbPb9NYkxicLepMSZkrOXe2He/65z7vnkzexM55pBuDw2/Kzk2mb0Veyk5XOJMXzWX19e8zti+YxnSfQiHqw5TUV3B4erDrNy5ki93f8mQ7kMY2n0oCTEJJEQn8PmOz1m4aSGndzqd7Ye2c7TmKLGuWC7IuYBL+1zKkO5DiI5y9vtSO7SQiPOsqija5PnQkUOsKVnD1IKpNj6PMe0o0Ja7hXtj69bB2WfDj34EM2eGuppWtbSTae1XwuTBk0mMSeTTbZ/yYeGHfFj4Iat3rz6ueq7Pu55Xxr/i3RkYY4LLwv14TJkCf/wjrF8Pp50W6mpOqIcWP8SjSx5lSv4UJuRNoO7fh6L8ec2f+f0Xv+fW/FuZmDcRQYiSKESE7Qe385OPfkJxWTEX9b6IZ0Y/Q25Wk3u6GGOOk4X78di5E/r2hbFj4c9/DnU1J1RbfgnUqfHUMGfFHH768U8pqyrj7nPv5qHhD9mBW2OCyML9eD38MDzyCPz7385ZNCZgJYdLuP/v9/OHL/7AqSmn8ouLf0FelzxiomKIccX4PMe6YomPjichJoEoscsujGmNhfvxKi+HPn0gPh6efRZGjICkpJY/43Y7XTmffOIMa9Cvn3OKZXl5/ePAAeeXQXY2JCSAy+U83G4oLoaLLoJBg+DMM+GMMyA9/YT8ue1hadFSbl94Oyt2rgho+ZioGBJiEoiPjic2KpYKdwXpcelEu6IRBBFBEGq0hoNHDpKRmEGcKw5XlIsoicIlLmo8NZRUlNA1uStx0XHerqNqTzU7y3bSM70niTGJxETFEB0VTXRUNG6Pm80HNnNap9OIdcXiUQ+K4lEPldWVbCvdxpmZZ5Ial0qsK5Y4VxyxrliqPdWs3r2agm4FpManemuodFeyvHg55/c4n7S4NFxRLlzi1FhRXcGn//mUC3MuJDku2efAdNnRMj7b/hkje42kc0JnYlz1NR6uOsyHhR8ytu9YMhIzcInLZ73Vnmqf6ymqaqrYW7GXj7d8zFVnXUXP9J6kx6eTHp9Oalyq94C5CT8W7sEwcWJ9t0xUFOTmQkGB8/jWt5ywX7ECli93nr/4AiorfdeRnOz7OHAAtm2DHj3glFOc8K+pgV27nHPsRZyxbupkZUHv3s4y3/62E/jZ2dC9u/OcmenUdpKq8dTww/d+yJyVc7hhwA2MOX0Mbo+bak81732P4ESrAAANFElEQVTzHn/5+i+MOX0M52afS2V1JUfcR7zhuGLnCgaeMpCzMs/yOTPn671fs2bPGnKzcunTqQ8e9VDjqcGjHgoPFLL5wGb6pPchOy3bCWpVth/azn8O/ofs1GwyEjJwe9zex77KfZQeKSU9Pp1O8Z0QEe/1DAcqD1BSUUJ6fDrx0fFNAjRcJcU4DZWk2CSio6J9ruFQVcqqykiLSyM6Ktpnp3royCHSE9K91300nLe/cj+ZiZlNdhxuj5t9Fft85tUdcK+uqWZf5T4yEjKIccX4fK7hvFhXrPcz7ho3JRUldEnq4p1et/N317jZfXg3XZO7+l3f7vLm5+0q30XXlK7OOmvXV1VTxa6yXXRL6UasKxatvwkdVTVV7CzbSbeUbsS54ny2R7Wnmh2HdpCdmk2sK9bnb66qqSInNYdfXvrLNp1ZZue5B8Nvf+uE6ZlnwjffOCH+3nvOwdaGkpJg8GDnQGxBgdPiX7zYed/4jlN798LcuTB5shPMjadffz0cOuR83zffwIYN8Pe/w5YtsHKlb/ADREdD585wwQVw3nlOHYMHO9NOAq4oF49d/Bindz69SV/9uDPHMTR76DH37x/vGULB+q6SwyXMXTWX7w/4Pp0SOlHjqaFGayg5XMIrq19hQv8JpMenU6M1PvPeWvcW43PHk5GQ4RNMByoP8P/W/T+uPPNKkuOSvTuf6ppqSipKeO+b97jstMtIjk32rvPN9W/y8pcvM3nQZCbmTfReLBcTFcPhqsO8u+FdRvQaAUDpkVLv46PCj/hs+2fkdcnj7C5nO9dv4FzD8dXur9h2cBs5qTmclXVW/U615Gs2VWyiS3IXzsg4A1X1/srZuG8jxWXFpMWlkZPmewvNzfs3s6NsBylxKWSnZvsE5Ob9myk6VERybDLdU31uzUzhgUKKDhWRFJNEt5RugHMq7pYDW9h+aDvx0fH0TO/p8+tnZ9lOtpRuQUSa1LG7fDeFpYUA9Ejr4TNvV/kuCksLUdSnxu0Ht7Pt4DaqPdVkp2YD9SFddKiIraVbcXvcdE/t7rM9isuKKTpUxOHqw3RL7ubzN+8q38VHhR9xVuZZ3DvsXtqNqobkcc4552hY8nhU//Mf1e9/XxVUf/xjVbe7fb+zpET1iSdUd+1S3bFDdelS1b/8RXXWLNXhw506UlOd57pHTo7qqFHO/CefVP3b31Q3blQ9erR9azUnVMnhEn3isye05HBJ0D7X3Ly2fKY95kXqdwUKWK4BZKx1y7RVcy3wUNah6nQNffEFrFrltPhLSnyXj4qCU0+F2FgYOBBycpzuoVNOgS5dnOeePZ3XxpiTjvW5Gyf4//AHGDUKDh6EwkLnsWCBswPIyICqKigra/rZnBy48EI4/3wYNsy5qYmr5XFrjDHtL6jhLiKjgKcBF/B7VX280fypwO1ADVAOTFHVdS2t08I9hBr/6qishD17nAO6zz7rHFPIy3Na/bt3O59JSYH8fOcA8MiRznGFrl3rW/1ZWU7/vzGmXQUt3EXEBXwDXAoUAcuASQ3DW0RSVfVQ7etxwA9VdVRL67VwP0k1DP6MDOdA7j//Cf/3f/DOO85ZPf6IOK375593WvvGmHYRzHA/D3hYVS+vfX8/gKr+opnlJwE3quroltZr4R6G9u6FF1+E733POS9/9+76UzjnzYNPP3VCfvJk5yKwHj1aXaUx5tgE81TI7sD2Bu+LgCaXbIrI7cA0IBa4qJmipgBTAHJycvwtYk5mmZnwk5/Uvz/99PrX3/sePPOM073z4ovw2mtwxx1w//3OLwBjzAkVyNUv/ob3a9LcV9XZqnoacB/woL8VqeocVS1Q1YKsxud/m/CWmekM1/Dss875+ZMmwW9+4/TN33cf/PjHTvAbY06IQMK9CGj4+zobKG5h+XnAlcdTlAlzPXs6/farVzvDKTzxBDz1lDN91Ch49FHnNE1/Z+kYY4IikG6ZZUBfEekN7AAmAtc1XEBE+qrqxtq3Y4GNGNO/P7z9tnNl75NPOveq/eIL+NnPnHPyRZwWf69eznN6OqSlOY/oaFizxjkdMyPD+Wx8vPMcF+es3+32fdTUwP79ztg+w4c7wz3UKStzbsJyySXOd8XGOuuJjXUeLZ3pk5gInTo5j5iY5pcz5iQS6KmQY4CZOKdCvqiqj4nIDJwrpeaLyNPAJUA1cAC4Q1XXtrROO6DagR086Iy2+eST8NFHTt99Wpoz/eBBKC2F6upQV+lfUlJ90CcnOweZe/Z0dgAxMc7D7XbOMurd29lpuN3O3+N2Q0WFM3Bc//7OqaR16+rUybmO4JNP4OKLnVNPPR5nJ+jxONtl8WJn55Se7lyMFhXl7LQ+/BAuv9zZhiLOA5xhLD76CG66yRnELiWlfp4JW3YRkzn5NXeVryoUFcHvfw9XX+0E6tGj9Y+XXoLnnoMf/tAJruhoJxijo51Ae/ttGD/eGV+nLsz274e33nJuo5iY6Fy8VVXlrG/ePKeOm26Cq67yrfEvf4GXX4Yrr4RzznEGfqt7rF7tXBSWne38uqgL8ZISZ37nzs7fFR3thH50tHPcYft2Z56qsyM7Uf8PJiTUX5fQqZOzTc4/3/nllJXl+0hIcOqtq93lcnYmqs42KytztnXdc1ERfPABjBnj7LQSE5111D3HxjqfF6nfMTV8XTc66rEMgldT41yjUfeoqKgfuC852dmZpaQ4NTS3U/N46v8dHD1a/7rhs8fjrKfuV2Vyckh3khbuJnIFe+iHltbXlnnH8hmPxwnH0lJnR/H22zBuXP2OqS4ES0vh3XednVNamvM5j8cJ6L/+1bmxTFpa/Y6ibscxf74zkFxFRf1pq7t2wVdf+R+FtCV1gdbemVEX9lFRToBHR/uGqaqzE62pCXx9yclOyFdUOOuurnbCuy2/EKOiIDXVWefRo87ruh1f3a+t6mpniO/k5KZXdtfUOAP9PfNMm/79WrgbY5pXt5O56SYnfEpKnMeePfCnPzm/WMaOdYaZrjum8emnzoHwUaOceamp9a1jj8e5h8HllzvHMhq2pN95x1nfuHHO1c11IejxOF1NdZ8777z6IbBrauBf/6o/fjJkSH3tn3/uTL/0UqebquGvhA8+cH5pTZjgHK8pL3d+XZSVORfjLVvmfM/QofXHb+LinHkLFsA11zg70LrjMQsWOKf2Xn+9U0ddt+HBg85nVq50xmjq18/318jatc68c85xrvZuaM0aZ4jwJ56Ae499VMhAw91GhTTG+KobhbSkJLDpbV1fW+cFe30ny3cFCBsV0hhjIk+gLfeT9xY+xhhj2szC3RhjIpCFuzHGRCALd2OMiUAW7sYYE4Es3I0xJgJZuBtjTAQK2XnuIlICbGvjxzOBvUEsJ9zZ9vBl26OebQtfkbA9eqpqqzfECFm4Hw8RWR7ISfwdhW0PX7Y96tm28NWRtod1yxhjTASycDfGmAgUruE+J9QFnGRse/iy7VHPtoWvDrM9wrLP3RhjTMvCteVujDGmBWEX7iIySkQ2iMgmEZke6npONBF5UUT2iMiaBtM6i8iHIrKx9rlTKGs8UUSkh4gsFpH1IrJWRO6qnd5Rt0e8iHwuIl/Wbo9Haqf3FpGltdvjzyISG+paTxQRcYnIFyKyoPZ9h9kWYRXuIuICZgOjgVxgkojkhraqE+6PwKhG06YDf1fVvsDfa993BG7gx6raDxgK3F7776Gjbo+jwEWqOhAYBIwSkaHAL4Hf1G6PA8AtIazxRLsLWN/gfYfZFmEV7sAQYJOqFqpqFTAPuCLENZ1QqroE2N9o8hXAS7WvXwKuPKFFhYiq7lTVlbWvy3D+J+5Ox90eqqrltW9jah8KXAS8WTu9w2wPEckGxgK/r30vdKBtEW7h3h3Y3uB9Ue20ju4UVd0JTuABXUJczwknIr2AwcBSOvD2qO2GWAXsAT4ENgOlququXaQj/T8zE/gJ4Kl9n0EH2hbhFu7iZ5qd7tPBiUgy8BZwt6oeCnU9oaSqNao6CMjG+aXbz99iJ7aqE09EvgPsUdUVDSf7WTRit0V0qAs4RkVAjwbvs4HiENVyMtktIt1UdaeIdMNptXUIIhKDE+yvqepfaid32O1RR1VLReQfOMci0kUkurbF2lH+nxkGjBORMUA8kIrTku8w2yLcWu7LgL61R7xjgYnA/BDXdDKYD9xU+/om4N0Q1nLC1Pah/gFYr6pPNZjVUbdHloik175OAC7BOQ6xGLimdrEOsT1U9X5VzVbVXjg58bGqXk8H2hZhdxFT7Z54JuACXlTVx0Jc0gklIq8DI3BGt9sN/Ax4B3gDyAH+A1yrqo0PukYcEbkA+BT4ivp+1Qdw+t074vYYgHOQ0IXTcHtDVWeISB+ckw86A18AN6jq0dBVemKJyAjgHlX9TkfaFmEX7sYYY1oXbt0yxhhjAmDhbowxEcjC3RhjIpCFuzHGRCALd2OMiUAW7sYYE4Es3I0xJgJZuBtjTAT6/+IJDPrCUU7pAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit_epoch(X_train,y_train,X_test,y_test,lr=0.00001,epochs=15,bs=512,l2=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8843"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accuracy on test set\n",
    "y_pred=nn.predict(X_test)\n",
    "accuracy(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L2 reg only seems to prevent training loss from dropping too fast. There is little to no improvement on validation loss"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}