2019/s11-nn1/seminar11-nn1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><img src=\"img/logo_hse_black.jpg\"></center>\n",
    "\n",
    "<h1><center>Data Analysis</center></h1>\n",
    "<h2><center>Seminar: Neural Networks 1</center></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_moons, make_circles # import function from the library\n",
    "\n",
    "N = 1000\n",
    "\n",
    "X, y = make_moons(n_samples=N, noise=0.1, random_state=11) # generate data sample\n",
    "# X, y = make_circles(n_samples=N, noise=0.01, factor=0.6, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAF3CAYAAABDpIrRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsvX2QnMd93/ntHeyQXCwpcQcWQgHCggptnSXTVg4y7xRfJQLpRAruTvJdLJ5YAxRMWodgIftQ57o7S4WLk1KyFSUFMwZPt4IYixSCmRLMM3JllbOSTpZGlUSWYhOJZFr00YJELgTIZwm7jsglIC+w2/dHT+Pp6enX52XmeZ75faqmdmfmeenpp5/u3/N7ZZxzEARBEARB1JWpcTeAIAiCIAiiSEjYIQiCIAii1pCwQxAEQRBErSFhhyAIgiCIWkPCDkEQBEEQtYaEHYIgCIIgag0JOwRBEARB1BoSdgiCIAiCqDUk7BAEQRAEUWtI2CEIgiAIotZsG3cDRsmOHTv43r17x92MTLz66qvYvn37uJtRCqgvEqgvEqgvEqgvEqgvEurUFxcuXLjKOf8R33YTJezs3bsXzz777LibkYkvfelLeMc73jHuZpQC6osE6osE6osE6osE6ouEOvUFY2wlZDsyYxEEQRAEUWtI2CEIgiAIotaQsEMQBEEQRK2ZKJ8dgiAIgiAGuXHjBi5fvowf/vCH426Kldtvvx27d+/G9PR0qv1J2CEIgiCICeby5cu48847sXfvXjDGxt2cITjnWF1dxeXLl3HvvfemOgaZsQiCIAhigvnhD3+IVqtVSkEHABhjaLVamTRPJOwQBEEQxIRTVkFHkrV9JOwQBEEQBDFWPvvZz+JNb3oT7rvvPnzkIx/J/fgk7BAEQRAEMTY2NzfxgQ98AJ/5zGfw/PPP41Of+hSef/75XM9Bwg5BEARBEOF0u8DevcDUlPjb7WY63B/8wR/gvvvuwxvf+EY0m028733vw+/8zu/k0lQJCTsEQRAEQYTR7QJHjgArKwDn4u+RI5kEnitXruANb3jDrfe7d+/GlStX8mjtLUjYIYgqI5+wLlzI5QmLIAjCyYkTwLVrg59duyY+TwnnfOizvB2mKc8OQVQV+YQlJx75hAUA7fb42kUQRH25dCnu8wB2796N73znO7feX758Ga9//etTH88EaXYIoqoU8IRFEAThZM+euM8D+Omf/ml885vfxIsvvoiNjQ2cO3cO7373u1MfzwQJOwRRVQp4wkpFzs6KBEGUmMVFYGZm8LOZGfF5SrZt24aPfvSjeOc734kf//Efx8MPP4y3vOUtGRuqnSPXo0XCGHsKwH8D4Huc858wfN8G8Kv9t+sAFjjnX+9/9xKAVwBsArjJOX/bSBpNEGVhzx5hujJ9PirIlEYQk4W8r0+cEA9We/YIQSfj/X7gwAEcOHAghwaaGbdm55MA3uX4/kUAf5Nz/pMA/hGAJ7Xv93PO30qCDjGRFPCEFQ2Z0ghi8mi3gZdeAra2xN8KPNiMVdjhnP8bAGuO73+fc/4X/bdfBbB7JA0jiCrQbgNPPgnMz4v38/Pi/SgnnrKY0giCIByMW7MTwy8C+IzyngP4fxhjFxhjR8bUJoIYL/IJa9++8TxhFeCsSBAEkTfMFN8+0gYwthfA75p8dpRt9gNYAvBfcc5X+5+9nnP+XcbY6wB8HsAv9zVF+r5HABwBgJ07d+47d+5c/j9ihKyvr2N2dnbczSgF1BcJY+uLtTXhp7O1lXw2NSW0THNzo28PaFyoUF8kUF8k6H3xmte8Bvfdd98YWxTGxYsX8YMf/GDgs/37918IcmXhnI/1BWAvgD92fP+TAL4F4Mcc2/xDAP+L71z79u3jVafX6427CaWB+iJhrH3R6XA+P885Y+JvpzO+tnAaFyrUFwnUFwl6Xzz//PPjaUgkpnYCeJYHyBqlTirIGNsD4F8BOMQ5/1Pl8+0Apjjnr/T//9sAPjymZhLEZNNuV8JBkSCIyWWsPjuMsU8B+AqANzHGLjPGfpExdpQxdrS/ya8BaAFYYox9jTH2bP/znQD+HWPs6wD+AMC/5px/duQ/gCAIgiCIzDz22GN43eteh5/4CatHSybGHY31COf8Hs75NOd8N+f8E5zz05zz0/3v3885v5uL8PJbIeac829zzn+q/3oL53yEsbYEMWIoaR9BEDXnF37hF/DZzxans6hSNBZBTB4FVBgmCILIQhHPX3/jb/wNzBUY1EDCDkGUGUraRxBEiajq8xcJOwRRZihpH0EQJaKqz18k7BCEj3H6zFDSPoIgSkRVn79I2CEIF+PW2Zah/hVBEESfqj5/kbBDEC7GrbNV618xNp76VwRBEH2Kev565JFH8Pa3vx0vvPACdu/ejU984hPZDqhR6qSCBDF2yqCzpaR9BEGUBDkVnTghpsE9e4Sgk3WK+tSnPpW9cQ5Is0MQLmy62ampeuS9oRw+BEFEIusPb22Np/5wGkjYIQgXJp0tAGxupvPhKZNwkdYfqUy/gSAIIgASdgjChe4z02gMbxPqwzNuZ2edNP5IZfsNBEEQAZCwQxA+VJ3t1pZ5mxAfnnE6O5u0MWn8kcbtsE0QRCGIAuLlJWv7SNghiBiyxF2Oy9nZpo2xpWZ3/ZYyOGwTBJErt99+O1ZXV0sr8HDOsbq6ittvvz31MSgaiyBiWFwUgoKq3QiNu9yzRwgaps+LxKaNueMO0faY3zKu30AQRGHs3r0bly9fxve///1xN8XK7bffjt27d6fen4QdgoghS9xlFkEpCzaty+oqsLAALC+H/5Zx/QaCIApjenoa995777ibUShkxiKIWNLGXarOzoBwdpb+LkU6+LoqCZ85IwSV0N/SbgOHDyeO2o2GeJ819pQivAiCKBASdghilLTbSTj75qb4LE1Eky4crK2la0+sc3G3KwQk2fbNTfE+i3Bi8ik6dEhEv5VV8CHhjCAqBQk7BJGWtAte1ogmk3CwsmI/v08QinEutrX94MGkD2L7xXRM6SgpBZ9jx8LbWDQUfk8QlYOEHYJIQ5YFL2tEk0k42NqyC0s+5+GpqfCF2tXGlRXgsceARx+N6xff7+YcOH26PMIEhd8TROUgYYeoB6M2K9gWvOPH/e3IWjbYJSyZ+sGWBVqyuRkuqPnauLEB3Lgx+JlPEAj53ZyXR5ig8HuCqBwk7BDVJ1TLkqdAZAq/BkSEk68drrLBso2MAdu2mf1WbMLB3Jy5H4BBx2gToZoJn+BkwyUIhB6zLMJEVmGVIIiRQ8IOUX1CzAp5+ll0u0IICcEkROglKObnxXsgaSMw6MCsOuweODAsHDAmfHNs/jQnTgihwiXwhAgTekRZKC5BIPSYZREmXMIqQRClhIQdovqEmBXycAqWWqHDhxMH2rTtM4Wvm9ooUR12n3xyUNianR3cxsTKihB6bBopIFyYkG3vdMI0Moz5BQF5TM5F7h9dmCyTMGETVqtQ+pkgJhQSdojqE2JWyOJnoWuFpMYla/vStAUQ53/11eT9q6/GCV8mGBMaoxgzn77o2+A8XBDodkWSQ86TXD5lFCbS5loiCGIskLBDVJ8Qs0IWPwuXxkWl1Upv3uh2hZCRhjwEnQcfFPlyYs186qJvM0OFmrxUoRIQQp3af5TXhiCIlJCwQ1SfELNCFj+LEI0LY8DDD4dlSNYdpY8dE4t8rMYoK7Kvzp4FLl7MHk594EA285Mrwo3y2hAEkQGqjUXUg3bbbUrIUtPKVvxShXPgYx8DnnlGCD1nziQLtxoVBQzWllpZETlkiqg23GyKUHAT8/NCIyM5dMi8XahpTWZWVn8HY3GlJFw1vHSkIEbmI4IgAiDNDjE5hPhZmEK/19eF4KBi81FZXRXCi01L4soWrMOYcAKeng74cQaeekqY1nRM2pas4dS237W8HLZ/zLkkPgGUIAiiDwk7RDXpdoHnnsvXh8PkMwIIAYZzIThI049LE2P77tKluFwxe/YIgezppxPTWGjI+/y82PfqVSEw+SKH0pj5VHOcTfCI+b22Nth8maQDs96W554jExdBEAOMVdhhjD3FGPseY+yPLd8zxtgTjLGLjLE/Yoz958p3hxlj3+y/Do+u1cTYkULJxkY+OXPUkHKbI/KNGyLEW2qFYvPMAEJ4sWkvXL4uUiPlE7JM+6r7uzRaseHUeoSajRhtja0NW1vm7aUwqrdlY4N8egiCGGDcmp1PAniX4/u/A+BH+68jAD4GAIyxOQD/AMB/AeABAP+AMXZ3oS0lykNetYliQ8pXVhJNkimxn4pNeLFpL44e9QsaLi2JNLllCdOOCacOiVALya9jasPiohCSLl0S5zGZ4uTxu12qVUUQhJexOihzzv8NY2yvY5P3APiXnHMO4KuMsdcyxu4B8A4An+ecrwEAY+zzEELTp4ptMVEK8qpNFBpSriI1SWfOCE3Q2bPCp0fnwQdFhNOlS6KMAyCcgPfsEfstL+frKH3zpmjLqBx2Q/o6Jr+ORAqgqgO3zWdJ1suiWlUEQXgYt2bHxy4A31HeX+5/ZvucqCKxNatsphHOzbWkbGRZDK9dEwKLTetw8aLQjpw9C1y/nvj9SEHpwIFB7UVIew8ccH9ftOlGvU4hOYHSmPpMAqheWFRFCowmylJegiCIscN4ESGvMQ0Qmp3f5Zz/hOG7fw3gn3DO/13//RcA/G8AHgRwG+f8H/c///sArnHOf91wjCMQJjDs3Llz37lz5wr6JaNhfX0ds7I8QB1YWxMCgOqXMTUlFkqpEbHss/7612P28mXzNq5jrK0BV67Yw7LzYt8+4Swbcp6pKSE4/eAHYvtmE9i1a7D9jmOt794t+qLZBO6/P6cfoGC6Ti5819DGhQtx28t+Utq2vns3Zr/73XTnN6GOF9N1KTG1my8yQH2RUKe+2L9//wXO+du8G3LOx/oCsBfAH1u++ziAR5T3LwC4B8AjAD5u28722rdvH686vV5v3E3Il/l5zoXOY/A1O8t5oyH+bzQ4X1gY3K/T4b2TJ837ytf8/NA+vNVy7zMzI7YLaeP8vP27RoNzxtzn0l/69npbHMe71ReMiX3m58X/8/PDv0f2hW0b03e238mYeLVa4mU7Z0ibXH3dbLr7Rzl+74kn7MePpdMR5/GNkZJSu/kiA9QXCXXqCwDP8hBZI2SjIl8eYee/BvAZAAzAfwngD/qfzwF4EcDd/deLAOZ85yJhp4TECASawOMVdhhLNjYtWibhxSYY6PsyJtoTctwsL1VgswkCqrCzfbt/cXYt4LbvXG30LfwxAkOnw/n0dFjfbN9uPEau94hL0K0AtZsvMkB9kVCnvggVdsYdev4pAF8B8CbG2GXG2C8yxo4yxo72N1kG8G0AFwH8CwDHAIALx+R/BOAP+68P9z8jqkaMX8WTTw6+1xP9uY7tc0ZmzB2WffjwYIQV58L3RrZLRlKpuV/yYGUl8cMxRXLpXL/uj0xyRS/ZvnP9Ll/UU0y0VLsN3Hab+3iSV18t3k+JnJ8JohaMOxrrEc/3HMAHLN89BeCpItpFjJDFxcHoGxd6aPiuXWLxN+2r55rxLU4+HwxZiVtFLtiqkBRazJOxwePp71UOHQK+/GVgaUm8P3HCHpVl86lRf3+aBdwVlu/LZBx7PlN0m42iy0bYIuDI+ZkgKkXZo7GIumNKJGfLEqxrF+bmhgtvAuZcM1kXp9AFO/Q8nA/+5qNH7VobzkUJim53MMGgCZuwpbbLFb1k+25+PiyTse/cIZ/HIq+BjBa7cMEfkRcaAZilgCxBEKWBhB1i/OjJ7I4eNW+nFtPU9+Vc5JrhXCxEJ04MLmQ+E9CaxwoaumCHmJoAITyoyfOWl4WpzAbng2afxUVz/hnGhs17+uLsWsBt3x044M5k7BIaYgWG0JIYkj17hkt9uLJq68kkbduqCQtVQe+OO+LaRxDE+Alx7KnLixyUK0KnI5xPpTPo1BTnDz00FM1j7Auf8+3UVDqHU5+TrRptpEYmtVrDDrczM8K52XQ8V7SY6nDN+cC2A87arVa+0VimtrraaHL21iPhWi27Y3OMA7e8Booj8UBfmK5riNOxz/G8IhFZEzFfBEJ9kVCnvkAVHJQJYgj51P3qq8lnjQbwb//t8JO4ro3pds31rVRn2G0GN7Xpab9ZwlU7StcUrK4KR+GzZ0Uhzve/PzH1NBpJBmVTOwG7ZkPXItm0UWtr4WUfbL9V3d/UVhu873dk0pZcv578v7pq17z4khFu3z58DWL8gkK29Tm0UzkKgqgWIRJRXV6k2akAjvBq/dV74olkP9+TuNRSmL5rtYpps9RuNJuDn+vv9XYuLAyH5MvPLef1ajN0zZNJ22TTVMTmCzL1gcyZFKJR63Tcx2w0nNcgF81OyG/WUxuE5BEaMbWfLyKgvkioU1+ANDtEJfFF9qhsbCR+Ir4ncekbY8Lnr+PDpSk4fnw46/HGhtuReGlJ+C3poe6nTwPHjiWfhfrCmDRPegkGl6YiS7ZgqeGxRXOZ+q7dtpfhAMzHcvWF7oxsKuKq91uI87TcxuQDdPAgsGMHVV4niJJAwg5RHrrdeOdUuZi6hCS5kIU4GcfW6dL31z9fXTV/t7XlXnBNoe5S4Ol2BwU8VxQaEF7wtIjcMY2GXwg1ceqU+5g6qpkRSPoCGBZEZBFXV5V5n6O5eq1s/esy1REEMVJI2CHcpFn803LixPACLzH52khcSe8ajWQh82lCQqN0dEzHZcyvpbL5AAF2wYNzoS1SBTyp6ThwwOyfEyrE2ASPtJqvmRl3fh7AXty03QYeesj8nSkqz4YtoeHystuvSSaSlONqasrsKwS4+5d8ewiiFJCwQ9hJu/inxbVovOY1bsfVzU2zIHPmTLIo2ZyMASHIHTxoXhiPH08Evh07xEsV/nStgitBoKTVGnYEDs0LtLpq1iRIrY9OiEnGFQoesv/0NLCwMNy3Pmfj5eXkf12wfvRRcUzVuXthIUmuqHLsmEi+qIee2wTOlRXRTpsA3+2KsSOFta0tcU3Pno27VgBlWyaIMhDi2FOXFzkoRzLqukA+52TO7eHW0ik0JLxZd9Z1OQz7Xrpjb4iD9fR0WLsi2nGrL2wOvy7n7UbD7VAbWv8rzbmlk69tO1eIunoOxaF4YFzYHKNd19B1HdP8xjHW0aqTI2pWqC8S6tQXqEoh0FG+JlbYSRspYotI0fO95IW2aA2dc2FhIIro1qLWaCTCgaky9sKCOxIp6ys0iseX90ZfMG05gQznGKh6butbkwAVmi9GHUM+wcW0ry8ayyUk+tqo7TtUIDaNoBY79rP2b0HUaVHLCvVFQp36IlTYITNW3cliiio6zb9Ou23Pnsy5MIvoUUSAMDVIR2DOB7+7dg342MfckUhZUc0Utr7xlVQw+ZZsbZmjtvTfqGI7f7stcv50Om7HXBuqyc0WKeU695kzbn8pn9/L8eP27137qqZKFysrgyat2LGftX8JgigUEnbqTkzFaZ1x1AUy+WNIfM6ueTM/7w6BlqgLoC2KZ3PTLWzaFmxbiQYTIdfG5ScUQrcLvPLK8Oe+xIyupIyAX4BeXbUL6LZ9GRNtCv2N6rVJO/az9i9BEIVAwk7dSVPhWuJboIrC59RaNDMz4gn9pZdECHRoCDIw3GcmjY5J2EyrLZPnaTbDrk3W6LoTJ4bzBgHAXXcl57adwyUIhNQUswnotmi4o0eTc4QIrYC4NocPC2fnO+4Q+5GWhiCqT4itqy6vifTZGbWTcR74fHdsvhlZXlNTdp8aW92rEP+nUB+eUCdgyzUMGhe++l4h+HxZspzD5vfi85eR+/avUe+JJ8zXMI0jekVqYNmok29GVqgvEurUFyCfHQLAeExRWWm37X4pnCfajDzh3J1zRWokrl4Vr1AzhUtjw/tmrUOHgC9/2a8RUmHMnqPGRhaTpsTny5LlHNLvJdYnSO4rr9H995uv4VNPDWopQ7Q9sf0TqjkbZf4qgiBI2Kk9eZqiRjlBu5IEykXt3nv9po9QinK6DjHPcC5y5ADJbzM59Or7nDmTXIO1Nf+1yWLSlPiE5zzOYTId5iGg62Y0n4lSEtr20GCAUeevIgiCzFhVY2zqxzxMIDG4TAt9er3eoIkpJKdKGlNF1iKPIWHbJtNiyG/rt6f3+OP+35SXSdPVH6M4h4eoeyR0/IS0IfS3j9C0XCdzRVaoLxLq1BegPDsk7OTKqH1/bL4bLj8Vm0C2sDD8uRQ89EVMX2RN+2YR8tL6pLh8Zebnzf5LJuGpaIF11EKxel6Xz45j+2C/KdPviBFk5b4jzF9Vp0UtK9QXCXXqCxJ2SNjJlzQLdFpczqQLC7c2M/aFTSMQoikwLXa2hSlUyNPPu327e0G04RI2GTMLO6Zrk6V/Qok9Vh6aM+W69U6eHBZMdCdzPbGk3F5uF3KNYp3K5TlIszMWqC8S6tQXJOwYXiTspMQVHZV1gjYtdIHZdHPvi5BSDzFCXuxiGFuyQVk8nZodnzAxLm1MXue2ZVBWf39sFuUQ7UvMeFHPMcL+rtOilhXqi4Q69QUJO4YXCTspcU3qMWHYOrZJP3BRyr0vfKaIWCEvZjFstfzHc2hlrD47IQvrONMT5HFu7boN1UwLrTMWIsiElgbxnSNPTZqDOi1qWaG+SKhTX4QKOxSNRfhxRaOsroppPE1EiS1M2Rd2XVQVaVcmXpXQyKDQds7MiMggH7akfO22iLIzRdyFhILnEUGVlrTnViMDTSU1JCsrSSkRH75M2Op173bd5/WdgzItE8RIIWGH8BMalh2bk8S2oG1uukOCbe3JGhpvW+COHk0Xum9rZ6uVf1bquTnz4hkiTIy6BlrIOdTP9et67Nhg6HYeZURMmbAPHx4UaKTQ2+0Cjz4af96ZGZEbifLrEMToCVH/1OVFZqwA0kSp2EwBPlymgoAq0gN9kZcfRN6OuqPyzTh/3tzuEHNMmX12YpzG+6HjQZm1m023CbbTGXZilvvZnMwZEw70tmzbeUf2BVAnc0VWqC8S6tQXIJ8dEnaicS08uhAQEBo+dOwQIcoWRaMsZrdyy6h9UdayGKPwzcjqszOqdjraH523x/ayOWurQpIv3YDPSd71cv2eMYzROi1qWaG+SKhTX5CwQ8JOPDGTcYw2IEaIsu2vP2VPTwtthmSEuUtKR9ZorDIT4wTcaJgFP/WlO4KndZK3vVxjfQxjtE6LWlaoLxLq1Behws5YfXYYY+9ijL3AGLvIGPug4ft/zhj7Wv/1p4yx/6R8t6l89+nRtrymxDiLxpShyFqT6fhx4MaNwc9u3AC+853k/Tj9TsaN77pV2Rk25vptbibO2jZWVwf9ZNI4ydsck1st91if5DFKEGNmbMIOY6wB4P8E8HcAvBnAI4yxN6vbcM7/Z875WznnbwXwfwD4V8rX1+V3nPN3j6zhdSZ2Mg5dRFdW7J+H1AiyRdPcvJn8X8WCp3lR50XUdF1tRWBVIcdVKDYkEm1zE5ieHv682QT+3t8Tf/XPT51yC56TPEYJYsyMU7PzAICLnPNvc843AJwD8B7H9o8A+NRIWjapFDEZd7v2hafRyF6FW5JnwdOqsbg4rG2oyyJquq5Hj7rH6ZUrQni2oQrfrnQD73//YGX0VktUTl9aGq6g/tRToq1zc+bjzc1N9hgliDEzTmFnFwDFDoHL/c+GYIzNA7gXwBeVj29njD3LGPsqY+znimvmBJF1MjaFfp84YV54GLOH7q6sDIblqguOyrZtw+0vwlwzymrvaXDl2akD+nVdWnKP040N9/EYS67h4qJZGOccWF4Grl5NvGuuXh3MbZRmrFXZpEgQFYZx1xNQkSdm7L0A3sk5f3///SEAD3DOf9mw7a8C2K1+xxh7Pef8u4yxN0IIQQ9xzr9l2PcIgCMAsHPnzn3nzp0r5geNiPX1dczOzo63EWtr4ul5Y0Oo73f1ZdSVFTGJS6amBt/HMjWVmCZeemlQaGIM6298I2Zf+9p07bU9gZv2Nf2u+fnwY4yAUoyLkrD+ve9hVvXnMtFsAvffL/6/cMG+3b59cSfP81g5QOMigfoioU59sX///guc87d5NwzxYi7iBeDtAD6nvP8QgA9Ztv2PAP6641ifBPDzvnNSNJYDV8SOr4ji9DTnU1P2CBlbhFdgKLGtfUF9kTUSrKwh7Rp1iq7ISu/8eX9EVWxpiFB8RURt0YYFRcvRuEigvkioU1+gAtFYfwjgRxlj9zLGmgDeB2Aoqoox9iYAdwP4ivLZ3Yyx2/r/7wDwMwCeH0mr60i3a3cU1r9bXTVHRtk0OKZsyNK/wlcWAkhMWkA69b8tOub48TDn6HGWUiDSMTeXmLlsxJSGiMF0LIlpjLnuPYIgcmNswg7n/CaAXwLwOQB/AuAZzvk3GGMfZoyp0VWPADjXl+AkPw7gWcbY1wH0AHyEc07CTlpc4bKm72KQ/hQm/4ojR8KOkWUBsAklq6thztF1jnSqM9I3ptMZFj6mp4H19cQHC8jPcVj1ezOhj7GsaRkIgghim3+T4uCcLwNY1j77Ne39PzTs9/sA7i+0cZOETSCwhYyHIp+O223zwrG0JP4++aS/zpBcAGIXoD174n6H3heLi0LQUhekukQ6TQJyvJw4Ia7t3BzwyitJOgMpSD/5pBCORoE6xlz33t69yf1DEEQmqBAo4dZSuPKVuGg0wp6Ol5ZEvpwQR/k0pqPFxeGcKC70vqBw4eqxtjYYPQckJtDZ2eForTw1KapZyoY6xlz3Hpm0CCI3SNgh3H4GnLsFnu3bh4WJmRngzJl4gcDlYwEMLwz6omZaFNpt4M47zcfTf5dNY0PhwuVGTQ2wY4e4RjYfmKJ9sHxmX8YGx5jr3gPIpEUQOUHCDpFoL2y4tC47dgwnWEur+ThwwC5Y6YLIsWPAiy+GOXaurZmPyTlpbKqOyYFeH6/XrgGHDwthyFbqQX4XmkfJlnvJJzRxPjjGfD4+IcckCMILCTuTiGmi9tUUsnHp0rDmA4hPwtftCm2QSbDSBZFuFzh9eng721OwzVTQaAgBijQ21SXUgX5zU4wtm2+Y/D7EdOSKoPI5rpvuMXn/2O4/ecyyJ7ckiBJDwk5VSTvxuSbqWP8WYHhyNx3/sceEBsjVVtuiNT8/LIhYhT1uAAAgAElEQVTYsjID5qdgm6lgc5N8IqpOWq1HoyE0eqb0Bz7TkSuCymWW8jm2u0LgKUSdIDJBwk4VyTLx+UJdfVFRKqbJ23T8jY3EvLCyAhw8KIQftb0xvhSuBc70ZC1NBWkWNqLcpE0BsLWVvEy4HIxdY1U3S8kxF2ImdTnDU4g6QWSChJ0q4pr4fBofVwXyEyfChR3b5B36pL26OiigxeSzcRVvtD05t9v2hY18IqqLrSp6q2XX3ADJGHKV/GDMfA/5xqo0S3GeRBqGmkltzvCU3JIgMkHCThVx5ebwaXxsk//UVNjEyZhI1GabvGOetNUnU9Oi1WwOJn9TizeaFrijR90LCiUIrB8mbcjevaJo59aW8APLkh1Zv4e6XTEmdYrOvURjlyAyQcJOFXE53PpU3TbNzdZWWGFLPZpEJ9bvRyZPO3QIuOOO5Im81RqMrlEXHbnANZvJAnf2bJKk0NW2vMoCEOVB14ao49iXJ8kWqaeiak2PHEkSEkpareIj+WLHrtTwXrhAzswEARJ2qolt4rMJMqrGxhdx5RNUGg2/U3RIgkAJY4Nhw9evC8Fldna4BpcquLXbomp1TCQVJQicLOSCf+iQeH/27PBYCdWMXLpkd6KfnfWPoSwBBaaHAdfY1RMbkjMzQYyv6vk4XrWqem6qlBxSvbnTcVeC7nREZfOQiuSyeriKrQ2mquiM2dtr+06pVl2nyr1Zob5I6PV67mr3Kp0O59PT/rHeaLjvGxehbQnZDxD3p2tf5R7snTxpngcmELpHEurUF6hA1XMiCyZHxhBVd7stng5N7Nkjvr96VUyPnU6iBQmNZLL5/ejHm593h4+TjwKRhdDopXYbePpp+z0hcTnu+8ZkbCSV1OYcPGjWJOnO/TrkzEwQQ5CwUydCzTSnToXZ/1WBKjRE1zbxz80NC2iuJGrkX0P4cJmGYhZ8VcAPEfJVbE70vnPaPg+prQW4hSV6UCCIIUjYqRtSoDh6FLh8WTwdTk2J+lBqYcRY3xVXuLc6wdsclF9+eXghcAk05F9DuPDlmsqy4IcI+YC4n/QcUiaNS0xbQjNCA4lzf0i0oulBgTIyE5NEiK2rLq9a+ey4WFhw+x80m35/AR2Xr4/uC2Dz+TH5DJh8jwKpk905KxPXFw7/tCifnbTnsfmUyXGujutWS9xzIf5DIb5ytjaox+yfv3fypPm+yqt/KsTE3SMO6tQXIJ+dCcZV1BMQT6PHjw9/nvZJT1fH28J5TU+iVFE8dybigd1nGspLM2jL6cQt/mZAkiVcL07qiqSSmqpQTG24dk3c12oE2r33mu8ryshMTBgk7NSRkCzIeq4Qk1ng0CFRXRxwT4K6Ot5l8qLaPoUyMSWUQkxDeQjSJqHJJejYuHFDhKjb2uIyX83MAAsLYW1YXR28+Csr2f2ICKIGkLBTVVyP7z6nShOmyZZzUV2823VPgrovQOjTcGiJCyKYiXlgH6UDe6hjvQ/1HtLHvMsh+cknRcLMNG3Y2jJffHJiJiYMEnaqiO/xPUQdrofa2iZbzsVkaZsEW63hp1T9aVhmQzYRUuKCCGZiHtjH6cDuqmzuQmZ2Nt2/jJn3mZ83/6aYNpgu/oED5m1tnxNExSFhp4r4Ht+XloTa26bhmZ4W4efy6dI20UouXbI/SZ86Zd5HPg2fPSuyItsIKXFBBDNRD+zj8vfSK5vHYtOi6vehS1NlEvZc+bN0lpfN29o+J4iKQ8JOFVBV3s89Z9fCqE9wS0tJxWU9md/TT4ttQvJ5AEmyQXWCl0KKNEPZ8PkihJS4IIKh9EQjQgpaMQKPdNx3Jd60aapMpl5d2Hv44WGBaWrKfPEnRgU4OZA3gIeQkK26vCoZeq6FiPZOnnSXWQjFFlKrv/Rw1NiQVVeIbmiJCwt1Cp/MitoXIdH8GSL+S89Ix0Wn4x7jpjEdO+ZD7jnTNozx3pkz5mNmuO+qSp3ni9jKIrF9Ueb5AoGh52MXQEb5qqSwo01Kt2rduHJsmNBHa8jk3GiE176yTZK+7TPk+6jz5BVLTF/UPcXKyMdF7END7AUIuecs2/SeeMJ8zLoPAgN1ni9cU7rpstZpvggVdsiMVXbSqLx1TA6RPmZmgDNnho8Zq/722VUoU/LImZiIrTxx2QhspqxGwzymY8d8yD1n22Zjw/w53Xe1wmV9zHpv12W+IGGn7Ng8S+fnw50zQ1PQS3u/a+KL9YClSbV0kLtGJL7oR5tAf+SIuC8uXRr2bYtxrg6551we6DbnDUroWRt8AQhZ7u26zBck7JSdPDxOXaNSffo8e1ZM5q6JL017XJPqxGTBKw8TFbGVB75HW5NAf/gw8IlPDI7rxx5LN65D7rnFRXtUZdUewYlofJkIstzbdZkvSNgpO/pE2mzGa0Zco1IWOzQJOLYIEJemJjYkoC460oIoIsKCIrYiCXm01QX6Z54ZNiHZyrT4kPecGlp+xx3D23DubydRatLe76YhIsl6b9dlviBhp2wUtbrZnvpsgpBL42LT1KTR0tRFR1oARSm9yLIYSZpHW70ci+/zENR8Vaurw4MhJs8OUTqy3u/tNnD16nCmkaz3dm3mixAv5qJeAN4F4AUAFwF80PD9LwD4PoCv9V/vV747DOCb/dfhkPOVPhrL5PbebHI+PZ1EV5w8aQ479cUFPvTQsJu+jOgy7ZMmNHVU+/Spc3QF53Fdo/dFmUNFiyb3cZEmHMUVmRVyPv3ihUQ1KvPErfni1399si6+g7LPF6PMBlD2vogBZQ89B9AA8C0AbwTQBPB1AG/WtvkFAB817DsH4Nv9v3f3/7/bd87SCzsBIeG3Qs9DQ7c7HZFsISQ8Vk3KYMsdwpi9/TH7qBN4bBh9nzrdsCZi1ks9z06ZQ0WLJtdxoY7TRiO593wPG7Z7bvt2//lMF882EOS9ZQs9/43fyK8vKk7Z54s0U25ayt4XMYQKO+M0Yz0A4CLn/Nuc8w0A5wC8J3DfdwL4POd8jXP+FwA+D6ElqjYxphu5rcvnRepFQ1Xnqmo8jeo+dB9VXwuIezokEqzixFoobdU+fHVeyQ0qJ/RxurmZOCv4TLcPPywutM6NG/EZx69ds190eW/Z5o6bN+3nIkpFXRyBy8o4hZ1dAL6jvL/c/0zn7zLG/ogx9tuMsTdE7lstYka1b5KT4a4hIecqclVM45UWuo+tNpAMp6+poBNrj7dV0rB9LiE3qJwIkRpt2ywvA3ffPXzMjQ231GnLgbW56a6dZZs7mk37uYhSURdH4LLChBZoDCdm7L0A3sk5f3///SEAD3DOf1nZpgVgnXP+l4yxowAe5pw/yBj7XwHcxjn/x/3t/j6Aa5zzXzec5wiAIwCwc+fOfefOnSv8t6VibU1MdFtbg5/LCa5/ndZ378bsd78rBIO5OVEry5Q4rNm0JxQLYd8+sTp+//vJZz/yI36BbG0NuHJFnLvZBHbtSqo9Sy5ccJ83kPX1dczOzgZvP0r0btjaMj9kN5vA/febj+G6tPo+al/E7FdHchsXIePUtY0L2zgPPZ5+b5nmj6kprN97L2Zf+9p0bawZZZ4vJCHTZ5btJVXoi1D2799/gXP+Nu+GIbauIl4A3g7gc8r7DwH4kGP7BoAf9P9/BMDHle8+DuAR3zlL67PT6QhHZJPBdmFhwCeg98QT4bWqQstCmHx3TD4HeTl+5OSJV1a7s61OTaw93lLuyOg2cv5875bbSKs17KtKPjspyFCmgc/PpxvnIYPGVUNL8x0q6z0yDurWF1l88+rUF6iAg/I2CMfie5E4KL9F2+Ye5f//DsBX+//PAXgRwjn57v7/c75zllbYcQkl2sRmHKS20JuYVVe+tOivrAKJkZw8aMt6w8bImL7uDPHj7nQ4f/zx3tBlbLUoGisTaQtwqhcmdpyHDp7Ai1rWe2Qc1K0vsjwz1qkvSi/siDbiAIA/hYjKOtH/7MMA3t3//58A+EZfEOoB+M+UfR+DCFm/CODRkPONVdhxxQK7qiZrj/7Rg1RdLaem3BNoo+GP3MorNCCH2Oiy3rChRbBD5Dufgk4qEE6e7Bm/m0QKicZKW0I+dpzHPKAEDKCy3iPjoG59kSV6q059UQlhZ9SvsQk7vie8rJodeQ7XhBsygTLmX6lLtIKW9Ya1Xc5WK/91T14yk7BTRMhqFfL3lHVcBLOwEC4xNxrOi1D5vsiRuvUFaXYEocIOZVAeBb6ojsVFc9TE9HSYK74v1Cc0KmvPHrcDMoUGBGGLqjh1Kq7uYshlc12yvENWqYyZQhGZziXLy6KDQ9jczP8iFPnbiNzII3proi51iERUl9fYNDsh+kY9EZma4E/BKJH7RPyQp0SXn4GjPeOkzE8neWhAfJfN5bNThEPyKDO8ZqHwcZFX1kbbIAnV6gRchN7589nViTXxbi/zfJGWtPNMr9erzaUGmbFKJOzkuEoYb1ifMGU7f6OR3CULC8ldI6Oxymyr4PWcvFR81k1bNFboJYudKEeZ4TULhY+LPO7nvKMoLVnKe48/HreaVUWiTUFd5wvTfey7t3u9XtClroLZmoSdMgk7OYrQqTQ7vvObfAQqIOJXdfIKnUBihk1sX5hKKU1P12MdLHxc5CH1uTrTViPPpfHRL0Knw3mjkZSXCb1gVZFoU1D2+SKNYBHqjqlXEHriiZ5Xbq6K5oeEHcOrtNFYERhv2BBhxRWebpvgyraKaZR98jIRO4GEDhtbX9j2twXdtVr5tX1cFFIIVO1EW+eF3i+djn1VUlca9Zzbt4etZHLf/oUyCjsuwaUqEm0KyjxfpL23YtNcyPOYAhr0S51lKIxSI0TCjuFV2jw7EQzdsLbMcwsLYQd03S0lf5or8+RlI+0EEqKWNu1j0964JsUs7SgDuYeem7QsabI26n55rhVJ72TXPvp5lUEWrdmpikSbgjLPF7HzQhprpxxOgF3YUS91WiXfqIcQCTuGV+HCTp4rgeVYQzds1iexGNV4ySjz5GXD192hmQP0ycPUFy7tTVphpwrkOi5s99fUVJxfW4i9YWZGPKTo27mSfJoumDLIhoSdmOROqj9fQPBE2SnzfBEjWMSkYjLNLzZhRx/GMUuLOmQajdEuJyTsGF6FCjt5irOOYw3dsFlt7LYRzVjpJ7QyT142XN1tGzohk46pL1wTXxozVlXIdVy4pNOY+zvkMTzt47rumaqsNgPCjicnj5FOx1zKxufgVULKPF/ECBZp/dflEDAJOzYBJmRJCxW+ijIUkLBjeBUq7ORp63YcK3fNTlYz2Bgp8+Rlw9bdrksYIs/GCjsLC/bPq85INDux91loss40YefyZTCv3RJ20j54+UICK0SZ54uYZ+XQISK3M22vCjsxPoNq0K5qcQ1pz7g1O5RUMC8uXYr7PK9jZc0s1W4DTz4pqqgzJv6ePQssLcW1mQjC1N2cm7ddWRFJvmKTBspEYTZaLZG3zsQzz9j3m0hM95dK6P0dmqwzSybIjQ3gxo3hzxsNMeh8WSxNuH5fmrmNMGKaF2yXzDZEWq3hadw1v/jOI9slE6EuLgJnzgwnFV1Z8f++MuSjJWEnL/JMY2vbZ24OeO65wXSX8i5ptZLt7rgj7nzqiA5J7UtkQu/u+Xn7tkeOAAcOhMuzapZjE82myORsW6dWV2ueRTUWeX81GubvQ+9vm9DUag2uNj7hKg2bmyIdd5o0ua7fl3eK7gkndBoOzdAOhAkiEl82ZVshANut0WiECVQjI0T9U5dXZXx2bDaGbdsGbfBZqivXgDKrpWPw2bxtAToqsi9cKmU1V6TNibCC1okhChkXedxjMQmWfBco0JZhjMaKaTf57JQS31DyzSm6Geuhh/zZS3zua+NafkA+OyMWdjjPLxrLsWINTV7z8/bt1QzJFZuYQqjb5OVat3yXT/aFa0IyrVm2V5WHTWHjIq/7O+Q4rgvpGyw+YSdWotVD5ikaK4pxpGvw+dG48uzYhokvD+a4UlKQsGN4VSbPjmOis05eIatXHbxPFeok7HDunqB8T0ohmp00ryoqCEszLkwrQKiGyDcYfPkDfPNFyXNoFcE4xsW4lO4+xV+osKOXbyyjASFU2CGfnTJShC2cc+D06UFD7ESVvC0/LneNa9eEzTzLMdIQel5Cw1Yi/vhxs+OD3sm+wQAMfz89DWzbFtY+8rcZCTY/l6Lvqbwu79xc8n+ME3UZIWGnjJgmuulp4V2aBc6Tu+zYMeDQoeHJmASesSEnExuhwS+x/uly8sp6XkLBtsqtrpq31zvZNxjW1gZXnlZLzA83bw5vq88bZQiNmRDyDNKNIeahx3Xv61Q5loWEnTJiEqGffhp46qnsAo+MZz59Wgg5KvQYP3babXt0lu9pTSoTbOupiVYrmbzSnpcwEBMGA5g72TcY5Mpz9ixw/Trw6qvmbe+8M+xxnDS9uZNnkG4MviBClQcftH+3ulqfoUDCzrixTTAmEbrdBu6/3x2r7KPREAKNLuhI6DF+7KRNnWRSJriYnhbhqq7zAsD6ej0mu5HR7dofl1utuIsbMhh8F35tzf84bjO70YXPRJY0aFllz3Zb5MXxaXguXhzMXKJTm6EQ4thTl1fpHJRTeHz1er24NLw2r7MQ9/uSUxpH1AKIjW7o9XrRQ0D3V3fVqSyDI2IoYx8XvhIssRdX3V7NHyD39QU0hNzTNa52LqlSNFbI0hCTyUC/r1UHZTksfSUfxh11ZQMUjVUBYSfFBHPrhjXl8Q4tUGJb0ULqYZVotI99USsRvV4vNEDHOIEtLIRXNCg7Yx8Xro7Mgm0FdFz43uOPh92jWWvsVYCxj4sIfEtD7HNypyNq15qEHfWYvmhOisYi0pHFe003cy0tCSPtlOeSci7+6rpNxoCjR90eZ6Tqrh0rK8BjjwEf+1gyNGyQhTMQm0NGFvMzYHd6Buy2Ctd8oNpJbNuRw9ZY8C0NMVFectre2hr+TjWpySXFNUzHEVmWFyTsjJO8vdfabeDuu/3b6ZEcofWwxhVHOSFktdGvraU778ZG2Ha07gWStV6dDdsKqN7PwKC/0M2b5gcS/cFlc3P4uBS1NTZs95qUS23+76YhYnPpspVMi01fUZWHIKewwxi7izH2Vw2f/2RxTZogipgUQ1a8ubl0MYTjiqOcAPJQmrkKBGbNvdNs0roXjB5N2WqJfACHDvmlWJfE63o4Uh/LdRWd6YHEtQJWMYlKzbAJHJubbg3s3Nzw8LFNz1tb5surDt8QqvIQZBV2GGMPA/h/AZxnjH2DMfbTytefLLphE0ERWZpCRt7qqkg+xlicCkHNMBV7TsJJHkozV4HAmMnLxJ130roXhR4WvrqaSLGHDpnvPZ/EG/JwZFvZZMoJ33ZbW9VMolIz9KUhJIS82QRefnlw+Bw8aA8MdE3bcvj6cvBUSvlnc+YB8DUA9/T/fwBC8Pnv++//Y4hDUNlepXNQToHXyS7Erd7kdRbimFyygoBVcjj0kdU/VPaFyW/d58febIrL6PNrrwojGxchzvohHp9yv5CABds5Ld6lxsLBoeeqGVWeL1w+73IoxAQoPP54L2jadg3f2kRjAfhj7f09AC4A+J8A/IeQg5ftNRHCDudhbvWxk5zteK1W9h+VkipPXjpZ1x5TX9giNnQBSI2GTjs8ysRIxkVoOExIPgDZuWklXscDjrFwcEz7a0SV5wvXvSkzEfiGmVoX+vz5XtB5qzBMQoUdl8/Oy6q/Duf8zwC8A8B7ALwlV/USkS9SB+nKFKXj87txOUcSmTFZKBgTqui0GUxtprHlZXO+ypdeAjods6XkwAFKrjtAqN0xxMQr7620AQumelu+c1W90NGEsbhoNylJC6kP1UJp80jQqdMwcQk7vwpgoHs5568AeBeAqljp6sfamn3V0Z0bY+oG+CbUceU9nxB0p0DGkgksbYR/TMSGqR1ycjt8WGRipYwDCqHO+iGhLfIeShOw0O3G3ecynGfvXvG+qoWOJox2O0ygcZElyLcOw8Ql7JwB8HcZY7fK6DLGdgJ4GsB/m8fJGWPvYoy9wBi7yBj7oOH7X2GMPc8Y+yPG2BcYY/PKd5uMsa/1X5/Ooz2lptsFduwAXnzRvOqYnBtDK7yFeJkVFU5L3CI2oMaFq2KBb9LTJ7flZco4MESo8G+SYlX0RCexj9GxF0GG82SVWKmO1sjJEmBAUzWcPjt3AzgN4DkADwI4DmAFwAcATIXYyFwvAA0A3wLwRgBNAF8H8GZtm/0AZvr/LwD4LeW79dhzFuKzM4qMworhdMgGL+3wrhT1pveNRrJvaJtLlD2Z82rb4F2kcd3Q+8JXsSCP9gDRP20klMpnx7RfnveQx1nDOF9kdcaqgiOHgSrNF6ZhEhp7YivroJaNOHlSZFwv+SULAnmVi+gLOVsALgPYHXLQoBMDbwfwOeX9hwB8yLH9XwPwZeX9+IWdUd30ysplnLwY89e7soXnuPKLl0iwMVGlySuGNM7K58/3Bi6XazLMqz3AcH2tMlCqaKyiz+O52F5hJ02YXUUjuaoyX7iWFb1Mmh4ga1t+9GBatVxE1YWezMIOgNcC+DhECPrfBvAbUssTcmDviYGfB/CbyvtDAD7q2P6jAP535f1NAM8C+CqAnws5Z+7CTh43vRoGY9O2KIJMtGZHD1sNEc4q8uRWlckrljR1bx5/vGe8/Hk9yLu0TSUbFvUaF77BEFIINO8BUdE6WlUZF7apvNEwT9W25UMVjOR3JmGnpNN7MKHCDhPbDsMY+zaAJQC/wTm/2f/srf3PVjjnj4SayizHfy+Ad3LO399/fwjAA5zzXzZsexDALwH4m5zzv+x/9nrO+XcZY28E8EUAD3HOv2XY9wiAIwCwc+fOfefOncvS7EEuXLB/t2+ff/+1NWE7NxUtmZoSRtq5OeC5527l9F/fvRuzly8Pb7e+Dnz/+8PH+ZEfSfwIlOMM0GwC99+fvA/dbsysr69jdnZ23M0ohLU14MoVcRmaTWDXLnsExXPPAa973TouX3b3hRwqQPixJa6hXrJhUa9xYbsXt20TF9RT62NovlBR55g82lS2gaBRlXHhutdMl8y0jEjXMMvyjt27h+eLkl8+K/v377/AOX+bd0ObFASHyQrA/xgiSbleCDRjAfhZAH8C4HWOY30SwM/7zlk6zY7P3mDIiTHwpKbqH0PaEvpEVpEnt3E9qZXJwtfpmJ/U1KdBn91fXm7Xb3EN1ZINi8o8wQcRkkDF8bJqdkxqglAqovnVqcq4CF0WQrc3vUzzRdnu41CQNc8O59zyOABwzv9FiMTl4Q8B/Chj7F7GWBPA+wAMRFUxxv4ahCnt3Zzz7ymf380Yu63//w4APwPg+RzaFEfWCCVfbhuZZAUYjOiYnxcJUa5eTSI1QkJhQyNIKMzcSpkKv8u2uNCz/5vSw/D+05/rt7jyfNCwKJC0nTszI+aIe+81z1Fnzoj/00RU1Sn5SgnxZStQp/Ru155iIpba38chElFRLwAHAPwpRFTWif5nH4YQbgDg9wD8OYTf0NcAfLr/+V+H8B/6ev/vL4acr3TRWKEiufLo3Tt/Pu5Y5LOTK2XyzVTbYtPs6O2KSeirs7AwvH8Jh0VlnuCDiC3/os1DvV4vPLTHdzHLpNJMQZXGxcKC//5MUxnINl+U0fcuFOQVjVWnV+nKRaQYrb3HH7e724fYJ0InrApMbOOYvMpk4VPbYhJ2TGtXiHzt+i0VGBaVWtSC0DvdVgTJIKVa+yJWaq/IA5CLKo2LkNQRrnvZVO/ON19UlVBhx5VUkCgaqQ4OKWkr2doStgg9qRcQloI3NB1mXdJm5kyZLHyuc9osCzEJfU3QsBgDeqefOmU3n+vzgq2cS2gGaEloeQwiF2yXgXO/5wIAPPUU8PTTg5bGo0ftpugsCQurAgk746bdFvZz3wqkIoUX1XHk0CHg4EHxfauVCDoSmphyoUyJpG1t6XTsgkhMQl+ipNh8ZoDheWFlxeyLEyu1xwpHRCZsl0Etd2jbZn5+sN6dlJGXlswCz6Tc8yTsjAJfanXfCqTTaLi9TG21cmhiykyZfDP1YRPaFjkJcg6cPVuO30L0CS3DYFKxmbQvUhOsEyu1l0mlOQEsLopQcJ2XX06GRJoHr6Ulcc9v25Z8dscd2dtbCUJsXXV5jcVnJ4sjoG5olT47aTzSXPb4ilIlG3yemPxmYvoiq99N2f12KjsusvrFGBzKeidP2p2wYi4k+eyMDHX6903jae5FUxLSil3KAUAOyiURdrKG72ijuXf+fLrEClUezRaqMnnlhVrbRr+058/3go+RZc2qwppX2XGRZa7odIbT5EphJ+vqmMe+JaAK4yI0ZiXLJZifNzsoV/VZOFTYITNW0WS1devq6rm5MC9THbJPVA7VorFjB/Doo2YL5bVrIiNyCLF+prpV5fhx8lMtjLRzhUy4tLk5/N3UVGLXyJokirzTC8d0f5oIuXw2i+ikul+RsFM0IbbuUDu9JNbHR3qsEZVBX5dWV4EbN+zbe6oG3CJmojOtjeQOViBp/WJsK2SjMXjvU0RV6Ym9j/TLJ5cSxkTMih7DwphYZkzU3f2KhJ2i8XmRpX3aUr1Mjx61h69Piqt9zQh9wlMJkZNtZZBMn8e0oe4TpZPYhxUbaUP9bCvk1tbghZ3UR/oKkeY+kpdPXUqA4YBc+d6kAJyEZYKEnaLxhe/YnraOHw+bQLtdEbpuGsGMAYcPk1angqRZf/IuXRHThvX18ZTMGDt51g8JDfVTH9+3bRte1SRUBqZyuNJJ2HLhyMsX+4DUaExYFGaIY09dXqXLoMx5eKG/vhfokJOdz1m51aq0U6GLKjgcpiXWB111OJSX2XTZYzJA29rQatkdpcswvEY6LkZdPyTUg9U0X1TBu7xAqjJf6PfuwoI1OHfg8sXUjJXzRR0AOSiXGFXtbTOg6ths677H79XVclStJKJI44MuMeWclJc95uHe9pR56hQwOzu8/US6f+RhGooxgyliizIAACAASURBVIU8vtse1fNIEpWXyY6wovqBLy4Kxb3NNKXmyIlV0MUk7q8FIRJRXV6l0Oxkqd7GWLxmZ5RPnSOmKk9qafHl27BpdgwRyLcue2wxzzw0RKOmUpqdWG2L7/FduwC59kXFNUNVnC9C7n95CVzlEUM0O1XNLADS7JQUV+SEj9DHbx/kkFgJ5BNeTN2amRmz+xYgng7PnBl8OvS5ddmijcn9o0/W+iGxEVK+Di7yAlA018gJmarlJTAp7mSGdBOMJYq5PF3PygoJO6PGFTnhWtWmp4UX6IULg+pjPQxdCk3z84OFVFQmbkWqNjHyrDoUdGxVRpaXBz/TLRXHjg1bLspUI2ysZDUN+cxg+sW47z57qgnXBcjD/ETRXCMndKqWZi7Tw8nionnIcJ7IqRMhx4aof+ryKoUZy6X2dpm4tm3jHP2MqKHqY5des0p6SgtVVEunRVUxm8xUJ0/2bllObNaGEMtHiJW12UwclGVbyjScKjUu0s4Htu01er1efuanUTtj50ylxkWf0CEwNeU2QbnM3py7raP6cctm7gKViyipsOObeGw1AfqvW8JO6CTjqLNVJXu7iSpOXmlRJ5hWSwgc6qV8/PHewKU0TUgha1UdKpFUaly45oPQi+GYB3q9XryQYlvNyGdnZOj3e6sVF22lXxZ9CEhhhzH3UNPPOT09PPeMewiQsGN4lULY4dwvGjsmuQFhR47GEPG64k9lJqo0eWXBtMZMTycT4Px8WG2skLUqZkIt6zCq3LiI9QI3rUgWer1e3H4hD2NleqyPYBTjIo/uyUP+1e/JTmdwOJlSVYQ6N5ft/idhx/AaibCTx2h3jLIhYSdUvC5z+ExKKreopSRETg3tiwxytvdVFmozLvLQ7Jw/b7/3TfvZztloVEqwMVH0uCjaWthqiWPpmpVQWVb9ThV25Hb63BBz749zGQkVdshBOU/ycmlP40Ds8yaj8JnKkqdfqK+WY9r8PhOXsyMvXI7DIRfD5JSsHvPFF8VcpMOY2ZnZNqg2N+sXnpMzeTn52i7B6irw5S+bL6cJfWq3xavIiiL63BATBVqFZYSEnTzJa7TbJjlTJjcV1+pH4TOVpSg51bTOmoKLFhaS9zZs4e6EA9/Dke9imCK/9GPa4NwcMeYaVLULz8mXvB5KXJfg4x93FwSWmKb2H/7QvK3tc9OSMT0NNJv+c5WSEPVPXV6Fm7HyNBVZsr/1zp9P739TYXu7idqYKzyEqMdj+8KmDpeqchtVcP2qzLgoojO1Y1rN3i7n5NDQvYpR9LjI63J2OvbuD3mZ7mH9mKoZC3C3RS9docbP+OaLUQAyY42BPB/Bl5eHn8yuXQOuXLFraQ4ccOfS8NkwiFKSR5Z/nePHgY2N4c9XV93WClIQ5kgReWtC9nVdMDnYbHbJKtgrxkRe90ba+3p+XhQMvXrVrOxLg6l0xepq8v316+mOOw5I2MmLblck/dNJuxLYJq2NDTECDx9OJqRGA3j724Hf/M1Blfijj5KNvSaEyKkxeePUCUvn2jUhDJmOZRp6rgzMhIMi7JMh+6oFlUy022JVI6k2ijwfSrZvD99WVkW3zQu+cmo2X56Q41TJsknCTh5I0VlfQVqt8NGur1TSa0yn2RTbnjmTOEpsbgJf+MKwMffGDbFqEbVnbS3fdO+2+rGmoXfmDMnUqShCTRbi1OxT3wHFqBMngLyU57ffHradr9wL4Fb2NZuisG8IVU+gTcJOHthE59nZcEFHX6leeUV4g6nMzAC7doVVPpa4HuGJ2nDliv2py6TxCX2a049V9ae7UlGEQCGP6bvA166JVdIn8JDZeyysrYVtxznwzDPubWzKvkYDeOqp8Mta9YBeEnbyIKvIe/z48AqysQHcddfwRDg3Vx1RmhgZJv8bINHK6Fqahx8elqV9rKxU/+mudKQRKHz2ynZbOG50OsOhMyoUTl5aYgSI1VX3JTQp+6amhEY2Rn6tur8eCTt5YDM57dnjn5i6Xbv2ZW0truS0idhHeKKS2NY0xsyamOVl4Omn4+rHMmYf6pzH1ZfMoy5l5cjjR8fk8mq3gfvvdydMIbVcKYnNd+W6hCYF4vx8vKKu8pbNkJCtol4A3gXgBQAXAXzQ8P1tAH6r//2/B7BX+e5D/c9fAPDOkPMVEnre6Yjc/Xr8X7Mp4vR8McOuVJWGmMVbhf1M5zS1YdxxgQVSmRDjEXD+fC9oSIREEOup5fWwVldkctr6tHnW1ynluBhTMU5rIdCahJPHMM5xkSbrhynsO69LWMp7JCUoe7kIAA0A3wLwRgBNAF8H8GZtm2MATvf/fx+A3+r//+b+9rcBuLd/nIbvnIUIO67c3iETk6sAia2KMef2YqGNRm3y6Pio0w2blV6v56ofG7o23sI1qfpq9PiOXXSunlKOi7x+dEx5aq70RaeTlKgvquNLzjjGha2us0/OVe8xednm5znfvt19CUOFqlLeIykJFXbGacZ6AMBFzvm3OecbAM4BeI+2zXsAnOn//9sAHmKMsf7n5zjnf8k5fxFCw/PAiNo9iM1ZYW0tzMHBZpJqtdz6QZsH29YWORROKKFOjSF2dpvlY8+exM3EllHZ578zkX4/RafXZcxt2qJw8pFjC9IF3NZD1VIJJJGPKysi27EpbmVxMb9qRXVlnMLOLgDfUd5f7n9m3IZzfhPADwC0AvcdDS4X9RD3dZvXly8esOqu8YSVtK4dLrlZ9cPxpVkBwpwRXa5qKqFZFWo9dPO6X00XhjGxuqmYVtPKO11UC1/QrCrnqvfI4cP2/TY3BzOMqNlNQiIl5XkuXJggX7k+jOs3yahOzNh7IXxt3t9/fwjAA5zzX1a2+UZ/m8v999+C0OB8GMBXOOed/uefALDMOT9vOM8RAEcAYOfOnfvOnTuX7w9ZWxMi9NZW8tnUVPJobPtOnfHX1kTs8MaG8DTdtcu6Iqyvr2N2dtZ9XttqUjNu9UWNSHtZ19fXsbExa9y31RJPl7HHdA1L6TuvTx+MDQo0pt9jIs+hW8pxkef9ql8YWygegPU3val8fTEmRj0uLlwI227bNiHEpFmK1SHkOl+zCbzmNck8sHv3Oi5fnq3FkrF///4LnPO3eTcMsXUV8QLwdgCfU95/CMCHtG0+B+Dt/f+3AbgKgOnbqtu5XoXVxnIZSnOuRzVga61ZratY6mR3lqR17ZB9YbL121w1Wq3BY4QMJ5+/jn5M17aqH9DCQorO8vRF6cj7fg1wniptX4yBovrCdllDxn4eLzk3+M6nunuptbGq7rKFCjgobwPwbQgHY+mg/BZtmw9g0EH5mf7/b8Ggg/K3MS4H5RFDk1dCHfsibS1ZtS98wTfqS07MIcFCIcfV2+nypy3KR7aO42II38XoX7yJ6ItAiugL131ju0RTU/kJOvJla4vtpQo7VQ/GCxV2xuazw4UPzi9BaGX+BEKQ+QZj7MOMsXf3N/sEgBZj7CKAXwHwwf6+3wDwDIDnAXwWwAc455uj/g2lYSKTltSTPFw7YhJsyxpYBw/67f0hx9XbGdpu6YxJBOK6GOSLMxK6XbN/jbxvTC5SnY4QMfJmxw7xV54vhlr7yimMNakg53yZc/5jnPO/yjlf7H/2a5zzT/f//yHn/L2c8/s45w9wzr+t7LvY3+9NnPPPjOs3jB1ywa8VeWQpjQnwkTWwQo4VctwDBwbfhyZHsxXZJiy4LsbiIgk6BSOn3U3LI7a8PDJy8exZ8f7QIfFM6sOV+NqELHcGiPPZBB49gnKSgvEog3LVoWJFtcL0NHj4sLicoYq7PJ/U1GOFHHd5efC9/nts2BYNQkHV4LpWTHrYKRyfllO9V7pd4LHHkufRkLHOefr6dYD9oeno0UQQmjQFIAk7VWcik5bUG7Vc0uKiSI8So7iLTTVvQ3/qC3kCNA079ffYnjjn58ka60TX4LpWzNiHHer4aFzTq37fHD9uDpibmhIPACatpgwvN2UZcCG1tLYsA0tL4l7ct2/yUrGRsFN1KN9OrUmjuFMnurSYnvrabcAXuSuVDrY10/bEeeAAWWOdxDhiAeEPO2QGT4Wrkrh+39hKH8r8r7a0DGtrwwLL0aNuE5cqOFHR+kFI2Kk6VS9FSzhJq7iTE12n438aVJmZEfvIyVF/6Pc5V8p8IbY10/bEubxM1lgneWVa1iEzeCps025sJXHAfqnm5oYFlqUl4M477ccic7AdEnaqxtra4OoDmFcPgFTTNSCr4q7d9gsoUhjStTmmh/5XXw07L2BfM01PnGSN9eBKj53lYYc6PhUxyahtvjfy88XF4RIQAPDKK+Zp21UWhszBdkjYqRLdrlhxdJUzMLh6AKSargl5KO5c5qz5eREpwvmwqjvWcmIidM0ka6wHV1mZLCUgqONvESskhJqJTp0aFmamp5OKQO02cNddw/ttbJgfFlzl0cgc7CAkGU9dXpVPKjg/z3snTw5niNIzshVdVrokTErCtJDEu66+CEkYaCI0IeDMjL0as55ROaaNzabYPzbhcG3HRYoMzN6+SDs4KkgR94jpOKZL5Lt0MclETW2VWchDp/463SMoewblcbxKI+ykTRvPmFnYCU1bW/VUmRp1umFjMA0fX1+kGXK2ibPVGj5Wq2Xedvv28POqbWy1OJ+eTrf4TOq4MBHUFxNSdsbVFzFj3UYWgcl2/9geFmyXLHTqr9M9QsJOWYWdLHcEaXYGqNMNG4pt+Jw/3yvkXM3m4LmaTfNQjdECySdQ1wLiq/PjWngmcVzYoL5IcPVFzPi1jbss026ssGODNDv2F/nsjJos0Q+Li8PJxEwOHBShVVtsw+fKleFt83BU5Hzw/cYG8OUvD28X6uJx7Rpw+rTfp8Dn67OyAjz6KPkiENmQ94g+zm24pmrbmF1ZMY9T9f60hae7nJFN0NRvh4SdUZMl+qHdFg6IPmfEmFABolLYhometCyP9CknTiTJzVROnx4+TkwiQ31hMS0gIcLTjRsiYRsBCsFJgXqPxGC7B+fm7Pvo955+f9qI9ROnqd8OCTujJmv0w9xcWAgAZZSqJbZhoicayyN9im1S5zw5jlxjDx0C7rhDhNPKSTYm3b1+rlDhyfZEPFH4JFsShIy4og1bLfv4lfeg2q07dgA/+IH9XGmK6gLDteZCoKnfDAk7o2aUekaa5GqHbfjs2jX4WR7pU1zy96VLw2vs6ipw/boIZX/pJRFaG5ruXj+XfEKlAqEBuCTbtTWKRbbguheuXwceftg+VZvG/s2b7vOtrCRTcag2Sa01R9N5NkjYGTWj0jNSGvhaYhs+ugo9j/QpLvl7bs6vPTK19ehRswBkeoJtt0VGWpeGJ7ZYYi1xSbZXrlCGZAuue+HaNSFoHD6cCNyNhnjfbqfPQSWn4tCs5isrQrA5dqyY6XyiBKgQL+a6vEoRjZWRYC/6CYjIqlNEQVb0vnDl4nChh7Q2GuZhtH17+gwHCwvD+7qiXGyh7bbIsIkbF4573Ri9GXKRakjIPWKKvjKN09DorbxetvOlnc57vV5tUiyBorEmHEoDP9G02+IpVH2C5FxoSmxPbyZloK3WzquvptceLS+HOSmrv+XqVVGzS9USPfUU+SMAcJvGbVUja5AhOatWwmcqbTTsSrFRd59+v0iyTOeTVhaNhJ26QmngJx6bUHHwoHlxiFXNh7qf6YuSzV9BVdmbFjHpeHn2rHh/6NAEqN5DcJnGd+2qZSxyWiu9PhYBs6l0ZsYu6F+6FBd9aCMPf7Qs0/nEPQ+HqH/q8pooM1ZddJQOJs5c4cDUFz5VuzocOp141fr8vD9BoM2cFnMevZ2+YU3jIqHX69UyQ3IaK/358z3r2DF1ke8cnY7dzBvy2r59OGnnzIw9waDrvoil1+vVxtMBlEF5woUdzms5yanQopZg6gtfFmI5sYX4LqSdcG1tiBV45AQcMkHTuEgoS1/kPRWl8Rd74ole1OIeIlhn9d2Znh6u/xZyPzYa2fqQfHaIekEJFyaaEFX7pUtu89XMDLCwYK+c7rPx20xWnLursZvaqf61fU+UjyICQ9NY6fXEmxLb2AkJnM3qFXDjBjA7OzhFq+e1sbWVfTqftASEJOwQRE0JmTT37HELCk8+CSwtiYnYFi5rEmikb4SNRkMcM1TgkYsKuaJVjyIcYdOkK9u2zfy5a+z4nhdN7QgNK5eY7j95Xtv9kdd4n6TnYRJ2CKLGyMms07EvDraJc34+7CmWMXsqfBvS+TNE+6QuYqYAo2azWH/bicpFUgBFaONitRLdrtnheHpa5HhKe31N7eA87re4BBeqdZUfJOwQxATgWhxCJ9TFRfNTK+fxqfDlE2tI+O+TT4r/9+4VkWS6OSJ2cXGhCzZFJXObJIrSxsVoJU6cMI+T224T0VhZrq/ejhjzrE9wmTRTU5GQsEMQNUetXwUk5RzkhBk6obbbdsFCfUr3PbHrE7wtU/LMjPgccGuKbtzIJzeIybfk9OkS5CKpuGqpDNoJ25hcXw+/vqbLYPpscVFojHxIQd4nuEySqalQQryY6/KauGismkN9kWDriywRF2nCcTl3R4HZInH0LMmtVrJdSFSZGoWTdlyEnCck6idXMobMlOUeGXdg6Pw85ydP9lJfX9NlaDZFNJXp0tiyfo9lDBkoy7jIA1A0FkEQNufQ48fdygJbBM2BA/6ndNuTfKdjfjKV51IrmF+/nvwf4tuRh8NmXkVSc6UmaW7HrZ1YXBRjXWVmxl/ZXGK6DBsbQquoIi+NzPrNHVGHvjFUcYVe6SBhhyBKTNYJz7aAr64OCjKHDgkTljyHbY1dXvabvGL9DHzruW9RyMsk4nLALuJ8QVCsfS6022Ic6mPy1KkwE1tMd+vbpjHjUR3nAghR/9TlRWaselH3voixYNj6IsY0o57Dp3oPMUuEmi58CeJcWZhNx007Lmz97csSXSgZ09zW/R6JwWXq9V3fmPvIdGlizXhFZzeu07hAmTMoA5gD8HkA3+z/vduwzVsBfAXANwD8EYD/QfnukwBeBPC1/uutIeclYade1L0vYia8GJ+dkJctDb4t47IuhMUIajbfBvV3xiwWWcbFuH1LjA2qgc8O5+Pv2yxCsGmMunx2fMfz9UOaDNExlGlcZKXsws4/A/DB/v8fBPBPDdv8GIAf7f//egB/BuC1PBF2fj72vCTs1Iu690XMhOfqC31yDa29o79kavuQJ9pQQa3T4XzbNvO5stT9qRUZpISy9EXRpQlCuihNX9geFqQDfeylCe0H0uyEU3Zh5wUA9/T/vwfACwH7fF0RfkjYIWrfF3lodkyk0faYiha6hLBQQc0mPM3Ohv8WfbGp+7gIQfbLyZO9Umiosi7eLqEiVIBIMy7yFjpiHgKKFA7rdI+ECjvjclDeyTn/MwDo/32da2PG2AMAmgC+pXy8yBj7I8bYP2eM3VZcUwliPBSVn0QvIxGS3v6HP7TXFpKoDr6uRHKq07UagaWyvu5vk82Jc23Nv2+d0TNYl8G51ebg68qyLfE56xYZsJa3f3jo8XQn/1YLuOMOEUhAkVnpYEIwKuDAjP0egL9i+OoEgDOc89cq2/4F5/xuy3HuAfAlAIc5519VPvv/IASgJwF8i3P+Ycv+RwAcAYCdO3fuO3fuXOrfVAbW19cxOzs77maUgknoi7U14MoVIWg0m8CuXcDc3PB2WfpCPUdapqbExCzbtrYmFqWtrcFtWi0h4Kif29i3z/39c8+Z2/yGN6zjz/98dqDPgLB+rBqm8aFey92713H5shgXzSZw//3jaaftWgHAvfe6r4VtX/l7Llyw76uOoTT3iO/csaQ5nu1eUu83F6Yx0mzWZ+7cv3//Bc7527wbhqh/8n4h0IwF4C4A/wHAex3HegeA3w05L5mx6gX1RUJefWFTs9sclqUKPjQaKzSqpdXyt9VmKtOTx01PD5vg8jQJjAubqcPWF+NMYtfp2K+XzyTkM4m6xmxW86bLnJTGlSqNeSqLKc12vvPne0G/vwqg5GasTwM43P//MIDf0TdgjDUB/N8A/iXn/P/Svrun/5cB+DkAf1xoawliQrCZzo4ciUsUCJgTyYWo/6enRf4TH6GJ/W7cGH6armBeviFs5htbnbEiEiFKkyRjoqq4mqtJJbTUiAmfSdRm8tzcNJs3Y3JXtdvA4cNJnzYa4j2QLg9OmlpXWUxptjFy5Yp/39oRIhHl/QLQAvAFiNDzLwCY63/+NgC/2f//IIAbSMLLb4WYA/gigOcghJwOgNmQ85Jmp15QXyTk2Re2J9YsocM+rY7+FO47htQ0mZ76Y8oCVBmbxkPV8Mi+KEKT5XJ0zzPCyJX/yHT+qSnzeZ54ohetWbFtH5IuIS+yaHZc2s+6gDJHY43rRcJOvaC+SChzX/iiv0Jzk+g5TWyvUGGHsdGYsorKL+NaBEcRjeUzSeYZYaT34cKC27RqGxchgoN6rthzhJoKY8ZEln6z/d4nnuiFNbQCkLBjeJGwUy+oLxLK3BdpCoPqxOQGMvnshC7IkrwElDSahDwXwSLHhUuzZFv48+jXtIkyXUKwK1t3zCuLH43vWqfpN/LZIWGnspR5URs11BcJZe6LPLLBplnU1IUhdkHOK8dJjCZBtkc/r6tchW8RLHJc+DQ7IaZJiclEadvPJ/i2WmaBxSXsyOsRUxbCdK2yaFuKMIFxXv9cVCTsGF4k7NQL6ouEMvdFHgnlYoUd/dgxbchzMUpT9yuvRbXTEeaKosozxGhBYv1ibPv5xkKzmURK6SYom7Cj1lmLGWf6vqHaOtexRkWZ54tYQoUdqnpOEESh+JIjuqJjZEK5GBgbTrwYk6Axz0RyrkgiwBwto8P54PuQSDLZbxsbYv8iEgvqySnViCUdV5tdfaDv5/vdd96ZRDZtbrq3BcRYkf3rSnBoi3DjXPx+W0SiKSGiLYlnEdFyRAIJOwRBFIor3DZNdlwfjcbwwmMLIVa3k0KXLlxI0ixGPiEr70y8kiKzCgNJXx06JN53OsDNm6LvbAkj0wqR6ve+bdfW4gRk27VWmZkBzpyxCymuNpmuA+fDx8ojMzrhhoQdgiAKx5RzB/AvymmEgZs3B993u8COHcDHPpY87W9uAqdPA8eOJduoJRZ0pqfDFiNdSwW486r4BCjbAjs1ZdfSdLv235FWuNKPrwuohw4lOXZsWX19Wi4bIWVI1O/TCMg2GEvKNExZVktXm2z9LTVCobl2iOyQsEMQxNjwPe2nVe1LQeDYMbFQmWpwcS4Enm7Xv0DeuAEcPOhOQmfTUgFmQQ8wa36kgDM/Dxw9Ovw9IIQ1k1nKp9XIw1Ri01YA4je/8ooQDlVcmgtTH9j2O3DA3i65bYhANzMjSpf44FyMHc7NZjGfRsbW39L0ZRoTRDGQsEMQxNjwPe27FkIXR44IQef0abepgnOxeIdqPFZWhPnLJGQcPpzOdHTHHcn/rRZw9qxo10svAUtL4qk/1A/GJbTlZSrx9dXGBnDXXeGaC933R2IyNS4vm4/RaCTncAl0antOnUo3thqNcI1MUcV8iRSEeDHX5UXRWPWC+iKhbH0RmhckJMxbP9ZDD7lzvMiom9CEcL4oGdNr+3b3bwiJsokJcQ8N39e3UyOQik5mqL9iCemPkPPF5JZxhf3nFTVVVFLJLJRtvsgCKBqLIIhx4HM6VgmpFaT6+ywuAl/5SphjaUg0DiA0AbEapFdfTf73mcBsmobjx8M1QaH+Li6zSV6mkpC+skUvufD5b/3sz9r3ZSwZX7YxZfIlkmOLc6FRU/exmbliTYE2fzVitJCwQxBErsRGAsUsBjHOpyELLmPCD0SP1gpBLq4us47NZNHtmv2IbMcLNYeMwmxiMzupSEFTd9g+dsyeZsDlv3XsGPCFL9jPx/ng+EojYOj7PPyweTuX3xBRXkjYIQgiV/LMU6Mvlq5cKCqyUnuz6d6OcxFWfOyY+Ktqg2yRUBLpu2N70lf9SCTy9xw8aD+u6Xih1bL17ZpNext8Aogr/5EUDGwCz/y8WcP3sY8Nvn/sMREpNzXljnZ68kl7f0nyiDQDRH9s2ybaasLmN0SUnBBbV11e5LNTL6gvEsrUF74MxFn8eWx+Fa1WcswnnujxhYW4elo2/57t29372apvm3xvQjMO5+nToY+LkDZk/U1yu5j+d7UlNIO2L8N1yD2ysOA/zygzHRdFmeaLrIB8dgiCGAcuU0qMP09MQrZTpxITxK5dQktjMxOZsPn3+Exm166JJ/0QrUuICa7VKtanI6QN166J9oeYIm0aJyCu/1VktFOrleS48RFrqrNprUI0SKrmTWqBGBN/Zd4mooSESER1eZFmp15QXySUrS9s2puYulOu6BhT0UhZMNNV8DFWsxMSeRT6pO+L9rFpTkK1YKbt9HERGnGU9bemqTOlnmNhIbytoRFOsi9sGsMQrY56jWzbLyyE9dE4Kdt8kQVQIVASduoO9UVCVfoipgK6SzBymWN8wo6tsrjNHONbuEMLhLqOY1qwQ0PTXdudP98bEIKympZkO30CWBahymc6BIRwGitUyHvEdh0Y43xqyn7O2dnB32oTkBuNuHaNg6rMFyGECjtkxiIIYmTElAxwmcPSlgSYmRFZiXWzi0zeZzJFxWT4dWH7PZ2OOWIoNKrNtt3x48JMqJoMTdmNY5ClIXxmSNt13r7dHcXVaAyG9evIwp03b4prFkq3Czz3nDBb2ZzcOR9M8Kijt8tm+gxNeUCMFhJ2CIIYGTGh0a4IpDSRN61WItiYwpJt4cq26t4hGXRV35ATJ0QEV2hm4dCoNtvivbo6XJRTZjdOkwdHwvnge5MAZrvOH/+4+M5VRdx3blfJDhN6BXgXLkGL88HfafsNWfqWKA4SdgiCKIzYwpg6NgEkJrFbowEsLABXr6Z3/lWTz8nq3r78LSZn7DNnxGIfkv8lRAvW7fpD5HXW1uyVydOiC1wux+UjR+zaj5B2uZzaTeRZGFQVnXXMYQAAFCFJREFUNG01yDY34wUyonhI2CEIohDSFMYMJSbj8eamEDJGvfjEJlfUBcP77jNvd/Vqss3x42ZthYxmMrFnTz4FQVVM2gyToJqX4BFSc0ySV/4dYLDflpaEEG367bECGVE8JOwQBFEIsYt9DFJzoC7os7P2Bf7aNX/V8qyEJkA0Lb4mwfCLXzTv/+qryTa28G4pAJnC9BcXxculEYrVFoX6qeQpeKys2JMeAsn1sJmuWi23/5COydy6tCQ0fabj5DXWiXwgYYcgiELIM5OyjevXk//X1wffmyjqidskrNgwaVVsOYWysLoqjiEzE6tVxNtt+/EZG6wT1Wr5M1GbFntTLptYjRJjQoi1fWdzlFavhwk1N1OIwCP9vXQtpPyNMYItMR5I2CEIohBiIq/SYNMc+Yh94naVTZCYinqamJ42O2MXuShKPxjdnGdb5PfsGTRBXb0KPPVUsr1NW6RiM2EeOBBXcJVz4LbbzPu4HKVd5jLdTyzUJGoSdFwCFZC/uZBIDwk7BEEUQtFFKX0FONPuqxKS8dlV1FPHZh6yLYqx5iQfqkAQGxlnqw4emi3alG06RGu0tjZssrQhr6vt+jI27CcWUth0dXVYyPX5H+VdgJXIBgk7BEEUQmjxyrS4npqvXXOHAIc+cYf4HcVoiTY2zNvbBA81J1CrlS1HjkQKAkVeH5cJ06U1MiE1TTZzlsrcnNtPx3bdfYVNgeHr5hKY8x7rRHZI2CEIojBsoeN54DM/bG4K4UDXHMQ8cYf4HcWaoEzb64JHqyX+yirhc3PCx+Tpp8M0HC7UBT/m+nS7okL5wYPpkwraKrovLpp/l3qtfP3cbAIvv2w3K01N+a+763v9/LbfOD+f/1gnskPCDkEQlUQXEEyanBs3gDvvTK+9CFm0Y/0y5ubMn0vB4+xZsWirCe5WV4HHHhP/X70qQp6nUszezaZw5Hb5H5mQ5jyTuS4mqaAUJlQ/qB07gEcfHT627hTs6uf5eXGdb9ywfz8/77/u7bY7ZF+laDMtkS9jEXYYY3OMsc8zxr7Z/3u3ZbtNxtjX+q9PK5/fyxj79/39f4sx5rH6EgQxbkIcfWNRNRO2hHRra+m1SyELWkzOH0AIMq7ffuKEedGWJrBuVzgaxyYGbLWENkZGacVEpvn8U3Sth8tEpvtBra6af+/s7OC18pXbWFszt0366diETJ1Tp8KEmKLNtETOhBTQyvsF4J8B+GD//w8C+KeW7dYtnz8D4H39/08DWAg5LxUCrRfUFwll74vQopZZkEUu9UKgoYU6bYQUvtS3CSmoacNVRNN1fFmYUt3/5Mmet6BpSP/4CnvG9HFoRXRTcVjXtfD9vph7JLTafFUp+3wRA0peCPQ9AM70/z8D4OdCd2SMMQAPAvjtNPsTBDF6ikwwCAhtwcsvD3/ebGY3K4T4tejb+HK3uPxPXOaaPXvs+25tDUdLNZv+emIhPkeuNsWabkJ9nGz+PbZrYdL8yFw8e/faNT8mivQ1I8bDuISdnZzzPwOA/t/XWba7nTH2LGPsq4wxKdC0APwnzvnN/vvLAHYV21yCILJQdIJBm+nnzjuLX6iOHQO2bRML67Zt4v3iojtyyiU8HDhg/nzbNnFcnx+RulDff7/f50X/3GRutJnqbMn2XIT4OKXxfdFDyGWFdCCp/k7lGyYXxm0xelkPzNjvAfgrhq9OADjDOX+tsu1fcM6H/HYYY6/nnH+XMfZGAF8E8BCAlwF8hXN+X3+bNwBY5pzfb2nHEQBHAGDnzp37zp07l/GXjZf19XXMhsRgTgDUFwll74vnnhM+JzrNpliQs3LhQvL/7t3ruHw56Yt9+8KOsbYGXLki2tlsArv6j1D6Z6rvx6VLwPe/P3ysO+8UjsCm6XVqSizI6nHUc9vYtg34qZ8S266sDPrsmI4JJONibQ34zndEaQNXW1zH9vVFKKZzSAfzmzezHVtiGm+7d6/je9+bzWW8VZ2yzxcx7N+//wLn/G3eDUNsXXm/ALwA4J7+//cAeCFgn08C+HkADMBVANv6n78dwOdCzks+O/WC+iKh7H1RtM+O6q+h+uyE+JJ0Oom/j/pqNjmfnna3WfrJhL4ajeHfbOobnw9LqE9Jr9ezHr/VGt4vi19PDEX7xJh8jE6e7Bn9gCaRss8XMaDkPjufBnC4//9hAL+jb8AYu5sxdlv//x0AfgbA8/0f14MQfKz7EwRRHoqOXEkbBuwKqd7YGDaN6X5GoQUwJVtbYdmGTaiajhifEtvx9WgnIJ25MU2Unan9eUbrFV2qhKge4xJ2PgLgbzHGvgngb/XfgzH2NsbYb/a3+XEAzzLGvg4h3HyEc/58/7tfBfArjLGLED48nxhp6wmCiKZIp0/dXyNUmAoVNFTUhd+VpdmEabEtulhkjAATKySElNMIwXScQ4eE/1MaTMJvSFJBor6MRdjhnK9yzh/inP9o/+9a//NnOefv7///+5zz+znnP9X/+wll/29zzh/gnN/HOX8v5/wvx/E7CIIoD1KY2rdP/AX8moI0goa68B85Yt7moYfCNE3dbnhywJhoIpUYASZWQ5ZXlJ2t6vvp0+k0PCZNYkhSQaK+UAZlgiBqR6jGwWXWaDaHI6r0hX9pSWQzlhqeRkO8/73fAw4fHvz88OHBxVa2MdQUltYEE1vwUy9bcccdQssiBUbV3GQrzZBHCQ1AXLu06Ql0TWIWh2ei+pCwQxBE7QjVOLhCqp96StSi8vkZLS2JKCLOxd+lpSTLsRRkNjfF+25XmGYaDVFjymRCm5ryC1kxxPpLqWUrrl8fzLj86KOibIUUIm24zF4mbZtLkCvazEdMBiTsEARRO0L9VEyCQKcj6k+1224/I5dDrU3YOnpUFPd0lXrgPEzIiiGNv5TpN9y44Q6PB+yCmUvbtrgofquJPJ2KiyhZQlSDbeNuAEEQRN7s2WM2sdiy8gJicb90KdH++CqAHzmSCANy4Zb72YSt9fWwtktBa5yk0ai0WqK2lKntLm3bSy8BX/6y8NFRNUZ5FtZcW3NfM6LekGaHIIjaEeOnkiaiyLZwHz4stAZpKpK72ijb6dNK6NtIp+Y0Go00GhVTOLvEp21bWhosdZF3eoIrV4otWUKUGxJ2CIKoHTF+KmkiimwL9+amEJhi8+8Awo/H1sYQgcy0zcqK8BHSP3/sMWDHDrfw4yt5YSJNzS/18yLTE9jMb+QTNBmQsEMQRC0JXTjTJNIL1Xo0GkkpBB9nzgy3UWpkTM7MukBmEtq2toSPkP75xsag47FJk9VuA3fd5W+3iqtf0iZ+zItm0/w5JRqcDEjYIQhiokmTbdcWxaWztZW8XCwsmAUdqZGxoQpkWTQUNk2WK7ePTXCxmcyKzqLtY9eu8QpbxHghYYcgiIkmVuPQ7SZaFDWPjgkpMNkEp0ZDRH8tLQ1/F5LdWT1uVg1FTEZlKajogguQT0blIpibG6+wRYwXEnYIgphoYjQOurZlc1MIRkeOuAUmm0BlMl1JfJoaXSAL1TbZiM2obDITuvyf8iotkYUifYKIckPCDkEQE0/oImhbzJeX3QJTGhOOS1Nj2l+ew6Zlkv5DrVZ40sLYdrv8n/IqLUEQaSBhhyAIIhDXYu6r5H3ihBAoQrUKNq1Kp2Pfv90W2iKbFmlrSyRMjElaGKMNcfk/pXEEJ4i8IGGHIAgikBhnZpvZ5tixsJw3aR16QyrAF2XOcZm90jiC61AGZCItlEGZIAgikMXFwSy8gN0EZDPbqFmCfVl802ZSlvt96UtJBfhRoGej3rMn8e8BwvvOhC9rNUG4IM0OQRBEIDHaFlclb5Vr14Djx/Nv67iwaY2yhp6Tzw+RBdLsEARBRBCqbbHV5zKxuio0F3XXUGSp+UU+P0QWSLNDEAThIY2vyIEDw5W8bZW9AdJQ+MjD54eYXEjYIQiCcJAmP0y3K6KfVJMVY8CDD9r3IQ2Fm3GXmyCqDQk7BEEQDtL4ipj24Ry4eFHkuTFBGgo34y43QVQbEnYIgiAcpPEVce1z6hRpKNJCGZCJtJCwQxAE4SCNr4hrH9JQEMToIWGHIAjCQRpfEd8+pKEgiNFCwg5BEISDNJoY0t4QxP/f3t3FSnWVYRz/P+kHXpBYkPZAkbYaSWz14qikwdTEAxRbuABUauiFUlMkNRrFaBNiE02IJhg/SDS2ilhB00C1fhS1rVJabG/aiA0ClTTQ1lLCsVRasYTaFvt6Mft0j4eZMwOHWXtmzfNLJjN79pqdd969zsx71t6zV3fxdXbMzFo4k+vDjOeaMmZ2dnlkx8ysAp7nySwdj+yYmSXmeZ7M0vLIjplZYp7nySwtFztmZol5nieztCopdiRNlrRN0v7iflKDNnMk7aq7/UfSkmLdRklP160bTP8uzMzOjOd5MkurqpGd1cD2iJgJbC+W/09EPBgRgxExCMwFTgB/rGty88j6iNiVJGozs7PA8zyZpVVVsbMY2FQ83gQsadF+KXBvRJxo0c7MrOv5OjxmaVVV7AxExDBAcX9Ri/bLgM2jnvu6pN2S1kma0Ikgzcw6xVdRNktHEdGZDUv3A1MbrLoF2BQRF9S1fTEiTjlvp1g3DdgNXBwRr9U99w/gfGA98GRErGny+pXASoCBgYH3bdmy5czfVBc4fvw4EydOrDqMruBclJyLknNRci5KzkUpp1zMmTPnLxExq1W7jl1nJyKubrZO0nOSpkXEcFG4HBljUx8Dfj1S6BTbHi4eviLpJ8CXxohjPbWCiFmzZsXQ0NBpvIvus2PHDnr9PZwtzkXJuSg5FyXnouRclPoxF1UdxtoKLC8eLwfuHqPt9Yw6hFUUSEgStfN99nYgRjMzM8tAVcXOWmC+pP3A/GIZSbMkbRhpJOkyYAbwp1Gvv0PSHmAPMAX4WoKYzczMrAdVMl1ERBwF5jV4fiewom7578D0Bu3mdjI+MzMzy4evoGxmZmZZc7FjZmZmWXOxY2ZmZllzsWNmZmZZc7FjZmZmWXOxY2ZmZllzsWNmZmZZ69jcWN1I0vPAM1XHMU5TgH9WHUSXcC5KzkXJuSg5FyXnopRTLi6NiAtbNeqrYicHkna2M+lZP3AuSs5FybkoORcl56LUj7nwYSwzMzPLmosdMzMzy5qLnd6zvuoAuohzUXIuSs5FybkoORelvsuFz9kxMzOzrHlkx8zMzLLmYqfLSbpO0uOSXpfU9Ox5SddKekLSAUmrU8aYiqTJkrZJ2l/cT2rS7r+SdhW3ranj7KRW+1nSBEl3FusflXRZ+ijTaCMXN0h6vq4vrKgizhQk3S7piKS9TdZL0neLXO2W9N7UMabSRi6GJB2r6xdfSR1jCpJmSHpQ0r7iO+TzDdr0Tb9wsdP99gIfAR5q1kDSOcD3gQXAFcD1kq5IE15Sq4HtETET2F4sN/JyRAwWt0XpwuusNvfzjcCLEfEOYB3wjbRRpnEaff7Our6wIWmQaW0Erh1j/QJgZnFbCdyWIKaqbGTsXAA8XNcv1iSIqQongS9GxOXAbOAzDf5G+qZfuNjpchGxLyKeaNHsSuBARDwVEa8CW4DFnY8uucXApuLxJmBJhbFUoZ39XJ+ju4B5kpQwxlT6pc+3JSIeAl4Yo8li4KdR8whwgaRpaaJLq41c9IWIGI6Ix4rHLwH7gOmjmvVNv3Cxk4fpwLN1y4c4tVPnYCAihqH2hwxc1KTdmyTtlPSIpJwKonb28xttIuIkcAx4S5Lo0mq3z3+0GJ6/S9KMNKF1pX75jGjX+yX9VdK9kt5VdTCdVhzOfg/w6KhVfdMvzq06AANJ9wNTG6y6JSLubmcTDZ7ryZ/ZjZWL09jMJRFxWNLbgQck7YmIJ89OhJVqZz9n0xdaaOd9/hbYHBGvSLqJ2ojX3I5H1p36pV+04zFqUwwcl7QQ+A21wzhZkjQR+CWwKiL+PXp1g5dk2S9c7HSBiLh6nJs4BNT/1/pW4PA4t1mJsXIh6TlJ0yJiuBhqPdJkG4eL+6ck7aD2H00OxU47+3mkzSFJ5wJvJs8h/Za5iIijdYs/ItPzl9qUzWfEeNV/4UfEPZJulTQlInKZK+oNks6jVujcERG/atCkb/qFD2Pl4c/ATElvk3Q+sAzI6ldIha3A8uLxcuCUUS9JkyRNKB5PAa4C/pYsws5qZz/X52gp8EDkeTGtlrkYde7BImrnLPSrrcAnil/fzAaOjRwS7jeSpo6cxybpSmrfg0fHflXvKd7jj4F9EfGdJs36pl94ZKfLSfow8D3gQuD3knZFxDWSLgY2RMTCiDgp6bPAH4BzgNsj4vEKw+6UtcDPJd0IHASuA1DtJ/k3RcQK4HLgh5Jep/YhtjYisih2mu1nSWuAnRGxldqH288kHaA2orOsuog7p81cfE7SImq/SnkBuKGygDtM0mZgCJgi6RDwVeA8gIj4AXAPsBA4AJwAPllNpJ3XRi6WAp+WdBJ4GViW6T8EVwEfB/ZI2lU892XgEujDfpHnPjYzMzOr8WEsMzMzy5qLHTMzM8uaix0zMzPLmosdMzMzy5qLHTMzM8uaix0z61nFzM5PS5pcLE8qli+VdJ+kf0n6XdVxmlm1XOyYWc+KiGepzdS8tnhqLbA+Ip4BvkntOiNm1udc7JhZr1sHzJa0CvgA8G2AiNgOvFRlYGbWHXwFZTPraRHxmqSbgfuAD0XEq1XHZGbdxSM7ZpaDBcAw8O6qAzGz7uNix8x6mqRBYD4wG/jCqAlAzcxc7JhZ7ypmdr4NWBURB6mdlPytaqMys27jYsfMetmngIMRsa1YvhV4p6QPSnoY+AUwT9IhSddUFqWZVcqznpuZmVnWPLJjZmZmWXOxY2ZmZllzsWNmZmZZc7FjZmZmWXOxY2ZmZllzsWNmZmZZc7FjZmZmWXOxY2ZmZln7H7WzHjGQwP5bAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create an figure with a custom size\n",
    "plt.figure(figsize=(9, 6))\n",
    "\n",
    "# Plot all objects with y == 0 (class 0)\n",
    "plt.scatter(X[y == 0, 0],     # selects all objects with y == 0 and the 1st column of X\n",
    "            X[y == 0, 1],     # selects all objects with y == 0 and the 2nd column of X\n",
    "            color='r',        # points color\n",
    "            label='0')        # label for the plot legend\n",
    "\n",
    "\n",
    "# Plot all objects with y == 1 (class 1)\n",
    "plt.scatter(X[y == 1, 0],     # selects all objects with y == 1 and the 1st column of X\n",
    "            X[y == 1, 1],     # selects all objects with y == 1 and the 2nd column of X\n",
    "            color='b',        # points color\n",
    "            label='1')        # label for the plot legend\n",
    "\n",
    "plt.xlabel('X1') # set up X-axis label\n",
    "plt.ylabel('X2') # set up Y-axis label\n",
    "\n",
    "plt.legend(loc='best') # create the plot legend and set up it position\n",
    "plt.grid(b=1) # create grid on the plot\n",
    "\n",
    "plt.show() # display the plot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tarain / Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(a, num_classes):\n",
    "    return np.squeeze(np.eye(num_classes)[a.reshape(-1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import train_test_split function to split the sample\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
    "                                                    test_size=0.5,    \n",
    "                                                    random_state=123) \n",
    "\n",
    "y_train_hot = one_hot(y_train, 2)\n",
    "y_test_hot = one_hot(y_test, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 1, 1])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0.],\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       [0., 1.],\n",
       "       [0., 1.]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_hot[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network Implementation\n",
    "\n",
    "<center><img src='img/nn.png'></center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Definitions\n",
    "\n",
    "* Denote $w_{ij}$ as weight of edge, connecting $i$-th and $j$-th neuron\n",
    "* Define $\\delta_j = \\frac{\\partial L}{\\partial I_j} =  \\frac{\\partial L}{\\partial O_j}\\frac{\\partial O_j}{\\partial I_j}$\n",
    "* Since $L$ depends on $w_{ij}$ through the following functional relationship $L(w_{ij}) = L\\left(O_j\\left(I_j(w_{ij})\\right)\\right)$, using the chain rule we get:\n",
    "$$ \\frac{\\partial L}{\\partial w_{ij}} = \\frac{\\partial L}{\\partial I_j}\\frac{\\partial I_j}{\\partial w_{ij}} = \\delta_j O_i$$\n",
    "because $\\frac{\\partial I_j}{\\partial w_{ij}} = \\frac{\\partial}{\\partial w_{ij}} \\left(\\sum\\limits_{k\\in inc(j)} w_{kj} O_k\\right) = O_i$, where $inc(j)$ is a set of neurons with outgoing edges to neuron $j$\n",
    "* $\\frac{\\partial L}{\\partial I_j} = \\frac{\\partial L}{\\partial O_j}\\frac{\\partial O_j}{\\partial I_j} = \\frac{\\partial L}{\\partial O_j} f'(I_j) $"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Backprop Algorithm:\n",
    "\n",
    "1. Forward propagate $x_n$ to the neural network, store all inputs $I_j$ and outputs $O_j$ for each neuron\n",
    "2. Calculate $\\delta_i$ for all $i \\in$ output layer using:\n",
    "\n",
    "$$ \\delta_j= \\frac{\\partial L}{\\partial O_j}\\frac{\\partial O_j}{\\partial I_j} = \\frac{\\partial L}{\\partial O_j} f'(I_j) \\qquad (1)$$\n",
    "\n",
    "\n",
    "3. Propagate $\\delta_i$ from final layer back layer by layer:\n",
    "\n",
    "$$ \\delta_j = \\frac{\\partial L}{\\partial I_j} =  \\frac{\\partial L}{\\partial O_j}\\frac{\\partial O_j}{\\partial I_j} = \\sum\\limits_{k\\in out(j)} \\left(\\delta_k w_{jk}\\right) f'(I_j) \\qquad (2)$$\n",
    "\n",
    "\n",
    "4. Using calculated deltas and outputs calculate $\\frac{\\partial L}{\\partial w_{ij}}$ with:\n",
    "\n",
    "\n",
    "$$ \\frac{\\partial L}{\\partial w_{ij}} = \\frac{\\partial L}{\\partial I_j}\\frac{\\partial I_j}{\\partial w_{ij}} = \\delta_jO_j \\qquad (3)$$\n",
    "\n",
    "More details in this [notebook](https://github.com/shestakoff/hse_se_ml/blob/master/2018/seminar-nn1/seminar-nn1-proc.ipynb\n",
    ")."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Example\n",
    "\n",
    "Consider a neural network with 1 input layer, 1 hidden layer and 1 output layer with ReLU activation function. The network output values will be calculated uwing SoftMax function:\n",
    "\n",
    "\n",
    "$$z_l = \\frac{\\exp(net_l)}{\\sum_l \\exp(net_l)}$$\n",
    "\n",
    "So, the network have to return probabilities for each class.\n",
    "\n",
    "We will use cross-entropy loss function:\n",
    "$$L(\\hat{y}(t)) = - \\sum\\limits_{c \\in classes} y_c\\log(\\hat{y}_c)$$\n",
    "\n",
    "\n",
    "Total error is calculated with regularization:\n",
    "$$L = \\frac{1}{N}\\sum\\limits_{t=1}^N L(\\hat{y}(t)) + \\lambda\\frac{1}{2}\\|\\mathbf{W}\\|^2$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyNN(object):\n",
    "    \n",
    "    def __init__(self, n_input, n_hidden, n_output, learning_rate=0.01, C=0.01, n_iters=100):\n",
    "        \n",
    "        # Initialize number of neurons in each layer\n",
    "        self.n_input = n_input\n",
    "        self.n_hidden = n_hidden\n",
    "        self.n_output = n_output\n",
    "        \n",
    "        self.learning_rate = learning_rate\n",
    "        self.C = C\n",
    "        self.n_iters = n_iters\n",
    "        \n",
    "        # Initialize weigths of the network\n",
    "        self.W1 = 0.01 * np.random.randn(self.n_input, self.n_hidden) \n",
    "        self.b1 = np.zeros((1, self.n_hidden))\n",
    "\n",
    "        self.W2 = 0.01 * np.random.randn(self.n_hidden, self.n_output) \n",
    "        self.b2 = np.zeros((1, self.n_output))\n",
    "        \n",
    "        \n",
    "    def activation(self, Z):\n",
    "        return np.maximum(0, Z)\n",
    "    \n",
    "    def softmax(self, A):\n",
    "        probs = np.exp(A)/np.exp(A).sum(axis=1, keepdims=True)\n",
    "        return probs\n",
    "        \n",
    "        \n",
    "    def forward(self, X):\n",
    "        \n",
    "        # The first layer of the NN\n",
    "        Z1 = np.dot(X, self.W1) + self.b1\n",
    "        A1 = self.activation(Z1)\n",
    "        \n",
    "        # The second layer of the NN\n",
    "        Z2 = np.dot(A1, self.W2) + self.b2\n",
    "        A2 = self.activation(Z2)\n",
    "        \n",
    "        # The NN output values (predictied y)\n",
    "        y_hat = self.softmax(A2)\n",
    "        \n",
    "        return Z1, A1, Z2, A2, y_hat\n",
    "    \n",
    "    \n",
    "    def backpropagation(self, X, y, Z1, A1, Z2, A2, y_hat):\n",
    "        \n",
    "        dy = y_hat - y\n",
    "        \n",
    "        dW2 = np.dot(A1.T, dy) + self.C * self.W2 # regularization \n",
    "        db2 = np.sum(dy, axis=0, keepdims=True)\n",
    "        \n",
    "        dA1 = np.dot(dy, self.W2.T)\n",
    "        dA1[A1 <= 0] = 0\n",
    "        \n",
    "        dW1 = np.dot(X.T, dA1) + self.C * self.W1\n",
    "        db1 = np.sum(dA1, axis=0, keepdims=True)\n",
    "        \n",
    "        \n",
    "        # Update weights\n",
    "        self.W1 = self.W1 - self.learning_rate * dW1\n",
    "        self.b1 = self.b1 - self.learning_rate * db1\n",
    "        \n",
    "        self.W2 = self.W2 - self.learning_rate * dW2\n",
    "        self.b2 = self.b2 - self.learning_rate * db2\n",
    "        \n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        for i in range(self.n_iters):\n",
    "            Z1, A1, Z2, A2, y_hat = self.forward(X)\n",
    "            self.backpropagation(X, y, Z1, A1, Z2, A2, y_hat)\n",
    "            \n",
    "    \n",
    "    def predict_proba(self, X):\n",
    "        Z1, A1, Z2, A2, y_hat = self.forward(X)\n",
    "        return y_hat\n",
    "    \n",
    "    \n",
    "    def predict(self, X):\n",
    "        y_hat = self.predict_proba(X)\n",
    "        labels = y_hat.argmax(axis=1)\n",
    "        return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = MyNN(n_input=2, n_hidden=5, n_output=2, learning_rate=0.01, C=0.01, n_iters=10000)\n",
    "nn.fit(X_train, y_train_hot)\n",
    "y_test_hat = nn.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_decision_boundary(clf, X_test, y_test):\n",
    "\n",
    "    # Plot all objects with y == 0 (class 0)\n",
    "    plt.scatter(X_test[y_test == 0, 0],     # selects all objects with y == 0 and the 1st column of X\n",
    "                X_test[y_test == 0, 1],     # selects all objects with y == 0 and the 2nd column of X\n",
    "                color='r',                  # points color\n",
    "                label='0')                  # label for the plot legend\n",
    "\n",
    "\n",
    "    # Plot all objects with y == 1 (class 1)\n",
    "    plt.scatter(X_test[y_test == 1, 0],     # selects all objects with y == 1 and the 1st column of X\n",
    "                X_test[y_test == 1, 1],     # selects all objects with y == 1 and the 2nd column of X\n",
    "                color='b',                  # points color\n",
    "                label='1')                  # label for the plot legend\n",
    "\n",
    "\n",
    "    ### THIS IS JUST MAGIC :)\n",
    "\n",
    "    # plot decision boundary\n",
    "    h=0.1\n",
    "    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5\n",
    "    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n",
    "    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                         np.arange(y_min, y_max, h))\n",
    "\n",
    "    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])\n",
    "    Z = Z.reshape(xx.shape)\n",
    "    plt.contourf(xx, yy, Z, cmap=plt.cm.RdBu, alpha=.6, levels=1)\n",
    "    \n",
    "    #Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]\n",
    "    #Z = Z.reshape(xx.shape)\n",
    "    #plt.contourf(xx, yy, Z, cmap=plt.cm.RdBu, alpha=.6, levels=5)\n",
    "    #plt.colorbar()\n",
    "\n",
    "    ### THE END OF THE MAGIC\n",
    "\n",
    "\n",
    "    plt.xlabel('X1') # set up X-axis label\n",
    "    plt.ylabel('X2') # set up Y-axis label\n",
    "\n",
    "    plt.legend(loc='best') # create the plot legend and set up it position\n",
    "    plt.grid(b=1) # create grid on the plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'NN')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAGDCAYAAAAh5Mk5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X2UHGd9J/rvr6dHrfGMbCzLlizZE2PGjiXH4UX2mFhCayvgGLPXZgMmxjdg7gV0uHdZ2Nyze+JsiHOuN+fEkL27mz0JCXLCxmEz5l0gWBM2IGmF5UtkxAUMlhBtBw96xW/YGlnTmlY/94/qR11T/TxPPVVd3V3d9f2co6OZ7prqqq7u+j2vv0eUUiAiIqLhV+r3ARAREVFvMOgTEREVBIM+ERFRQTDoExERFQSDPhERUUEw6BMRERUEgz4REVFBMOgTkTcR+amIHBeR8dBj7xORXc2flYg8LiKl0PN/JCJ/0/ujJaIoBn0iSqoM4MOO51cDuLNHx0JECTDoE1FSfwLg34jIKyzPfwzA/y0i5R4eExF5YNAnoqS+A2AXgH9jef6LAF4C8J4eHQ8ReWLQJ6I07gXwr0TkQsNzCsAfALhXRCq9PSwicmHQJ6LElFI/BPBVAPdYnn8YwCyALb08LiJyY9AnorT+EMD7AayxPP8RAL8P4JyeHREROTHoE1EqSqkqgM8A+JDl+V0AHgdwdw8Pi4gcGPSJqBP3ARh3PP8RAMt7dCxEFEOUUv0+BiIiIuoB1vSJiIgKgkGfiIioIBj0iYiICoJBn4iIqCAY9ImIiApi6BbEuOC8ZWryIlNm0HROKcGYDM8MB55PvvF88o3nk29FPp/vVf/pWaVUbPAbuqA/edGF2Pmf/31m+9tXH8P68qnM9tdvPJ984/nkG88n34p8Puf/899+2mc7Nu8TEREVBIM+ERFRQTDoExERFcTQ9ekTERElVS+N4PnzL8HCaAUQ6ffhAACWK8Hh8EA+pTC6UMPyFw6h3DiTap8M+kREVHjPn38JXrF6Dc5ftgySk6D/Mko4B42zvyul8PxLJ/A8gIue8xq314bN+0REVHgLo5VcBXwTEcHyc5cFrREpMegTERGJ5DrgayLSUfcDgz4REVFOfP0b38TV112Pq153Hf7jf/rPme+fQZ+IiCgHzpw5gw/929/FVz73Gfzg23vwhS98AU8c+HGmr8GgT0RElJB87vMYueY1GFl+IUaueQ3kc5/veJ97930Xr7r8lbj8ssuwZMkS/OZv/ia+8vDXMjjaFgZ9IiKiBORzn0fpw78DOXQIohTk0KHg9w4D/5GjR3HJmtVnf1+zejUOHz3a6eEuwqBPRESUQOm+P4KcWpwTX06dQum+P+pov0q1L66T9eBCBn0iIqIkDh9O9rinNatX49DhI63dHTmC1atWdbTPKAZ9IiKiJNasSfa4p+te91pUn3wK//T00zh9+jS++MUv4p+/+ZaO9hnFoE9ERJRA496PQI2NLXpMjY2hce9HOtpvuVzGn37sfrzlbXfgmutvwFvf+lZcvfaqjvbZ9hqZ7o2IiGjIqTvejgaCvn0cPgysWRMUBO54e8f7fvPNb8Kbb34TgCANL0JpeLPAoE9ERJSQuuPtOJNBkO81Nu8TEREVBIM+ERFRQTDoExERFQSDPhERUUEw6BMRERUEgz4REVFOvO+DH8LqK67Ca35tY1f2z6BPRESUE3e/80589fOf6dr+GfSJiIgSeuhzS/Cqa16BJcuX41XXvAIPfW5JJvt9w4YbsPz88zPZl0lfg76IfFJEfi4iP7Q8f6OIvCgi32v+u7fXx0hERBT20OeW4AMfnsDsoREoJZg9NIIPfHgis8DfTf2u6f8NgLjVBL6llHpN8999PTgmIiIiq4/cdw5ePrV4yduXTwk+ct85fToif30N+kqp3QCe7+cxEBERJfGzw+bQaXs8T/J/hMCvicj3ReRrInJ1vw+GiIiK7dI15kVwbI/nSd4X3PkugF9SSs2JyK0AvgTgiuhGIrIFwBYAWHnhCuyrj0U3Se1llDLdX7/xfPKN55NvPJ986+R8litprmoX7w/uPYUPfXgcp0JN/GNjCn9w7ynvfdj87+99Hx7ZswfPPfcc1l59NX7vnnvw7ne9a9E2p5WkPs9cB32l1Euhnx8WkY+LyAql1LOR7bYC2AoAr73icrW+fCqzY9hXH0OW++s3nk++8XzyjeeTb52cz2FROMdzGdv33FFDBQofue8c/OxwCZeuaeCP7n0Z77zjdKrXDvv0X289+/PLKDWPafFxLRGV+jxzHfRFZBWA40opJSLTCLojnuvzYRERUcG9847TmQT5Xutr0BeRhwDcCGCFiBwC8IcARgFAKfWXAN4O4P8QkTqAUwDuVEqpPh0uERHRQOtr0FdKvTPm+T8D8Gc9OhwiIqKhNgij94mIiLpLKQxCQ7JSCujgOBn0iYio8EYXanj+pRO5DvxKKTz/0gmMLtRS7yPXA/mIiIh6YfkLh/A8gGefrQAisdv3wmklWCKhQohSGF2oYfkLh1Lvk0GfiIgKr9w4g4uee7rfh7FIN6ZUsnmfiIioIBj0iYiICoJBn4iIqCAY9ImIiAqCQZ+IiKggGPSJiIgKgkGfiIioIBj0iYiICoJBn4iIqCAY9ImIiAqCQZ+IiKggGPSJiIgKgkGfiIioIBj0iYiICoJL6xIVSbUK7N0LzM0BExPA9DQwNdXvoyKiHmHQJyqKahXYvRuo14Pf5+aC3wEGfqKCYPM+UVHs3dsK+Fq9HjxORIXAoE9UFHNzyR4noqHDoE9UFBMTyR4noqHDoE9UFNPTQDkyjKdcDh4nokLgQD6iotCD9Th6n6iwGPSJimRqikGeqMDYvE9ERFQQDPpEREQFwaBPRERUEAz6REREBcGBfETDjvn2iaiJQZ8oC3kNrMy3T0QhDPpEnep2YO2kQOHKt8+gT1Q47NMn6lQ3F7LRBQqdH18XKKpVv79nvn0iCmHQJ+pUNwOrrUCxc6df4Ge+fSIKYfM+UacmJswBPovAais4KOXXhTA9vbjrAUiebz+v4xWIKDHW9Ik61c2FbFwFB58uhKkpYNOm1n4mJoLffYN2p90LRJQrrOkTufjUcru5kI2pph7m04XQSb59DgQkGioM+jQcutEEnWRUfpYL2UTP5corgf37gyb9qG73zXMgINFQYfM+Db5uNUF3c1S+Ta3Wfi4HDwJr13avC8GFAwGJhgpr+jT4sm6CDte0TTqt5Yb3v7QSPDZfC36++e3mc5mdDfriez2gLouBgESUGwz6NPiyDM7RJn2TTmq50f3P11rPzdcAGJrwgeBcbF0I3Rxd383xCkTUcwz6NPiynDJnajUI07Vcn0Br2iZu/za2c0mTDTBpISHL8QpE1Ffs06fBl+WUOVfrgEgwqA6IH0NgG2eQpvXBdS5Jxx1wCh5RobGmT4OvkyboaK13aWVxk3uYUsGguqeejB9DYAvGIuZR+DYi7nn1Sbs2OAWPqNAY9GlwVavAuWuAL30qfaAPm5sDSqXgX6Nh/tt63W/OvCuTXrns18RfLscn0vHt2uj24ETba731XcBLh1mgIMqJvjbvi8gnReTnIvJDy/MiIv9FRKoi8gMReV2vj5FySjdTN84Ev/s2U0ebt6MajWQ18bBwoHVNddu0KajBG8ni7eKCpU/XRtw5u443qehrNc6w+4AoR/rdp/83AG5xPP9mAFc0/20B8Bc9OCYaBGnn0PsMpEsb9OsLreDmCsZTU47XUMDmzcBdd/nVjl1pdqtVYGYG2LEj/pyzmoLXj9wGROStr837SqndInKZY5PbAfytUkoB+LaIvEJELlZKHe3JAVJ+pW2m7mYmufla+8j5R/e0xgiUR1rbuvr2fRbSCTONrveZehjdRxaYwY8o10SlrdVkdQBB0P+qUupXDM99FcD9SqlHmr9/E8DvKqW+E9luC4KWAKy8cMX6v3vgE5kd38so4RxY+ncH0ECeT60GnDwZNBWXRoDx8bO/v3zeBTjnxeda25ZGgOXL7X8PgXUufFb0MdRqwIkTkdcTYNky4MRLxj89ez5SCgoG4XOuVPyP4fnnW10fvscbx3QdoscUed2z5+P7Gjk3kN8fB55PviU5nze+9W37lFLXxm2X94F8po7Ptju2UmorgK0A8NorLlfry6cyO4B99TFkub9+G7jzMdVYy+Vg6tzBg9h36zuwfvtM6/FNm4CLxtx/b2OrfScdcQ8AW7YAn50x13AdMwT23XZX63zC4gb1RWch+NasTe+Zbf+m6xA9ppcOL9pu3213Yf3Dn/V7jQEwcN+fGDyffOvG+fS7Tz/OIQCXhn6/BMCRPh0L9YOtj1inpS01m8xtA99sffgSGjC3eXMQpG+6ydwPb8p776JT69oC73wt2f6A5HPvXSThYEHAfh0e3bP4segYg9JIsqV8iair8l7T3w7ggyLyaQDXA3iR/fkF4+ojnpoC6mNBwE7690q1/51rvv+qVe4pbyauGreIe2qgSZK59yY+UwCTvvZ8LSh0dLrGAdP7EvVEX4O+iDwE4EYAK0TkEIA/BDAKAEqpvwTwMIBbAVQBvAzgf+vPkVLf2JrCfaeYJU3Ra0s5qx+fsTTZh83X4rdbWADWrbMvmWtiO2afgojI4taCpDn8XQWYcGKfaDeAnrIXfU2f9MEsFBBlrq/N+0qpdyqlLlZKjSqlLlFK/bVS6i+bAR8q8C+VUq9SSl0THcBHQ65aBU4vmJ+bmwsCa82SPU/LMkWvbX+244szO2vuUtAJgsJcx+xTANIFC1s+g7j0vK73K3yuvlP24rZjumCirsh78z4V2d697ubvublgdPzRo63to7XCrFeJi+6vE7qLIry/0ghw443uY47WgCcng/TAvtPzTGl349LzTk0tnn4YpZv4fafsxW3HdMFEXcGgT/nlFVRVEIzqZ+xNxVmvEhfeX9pR80Crhh7eX32sNcrdNPc+Gnjn5oKAf+WVQcuB7+snDcIAcMOGINGPiQ7Gvt0pcdv5HA+b/4kSy/vofSoy3377+Vr/ssBNTQXZ87ZsCf73PeZwc73OnLd1azDP3dSErZu7TTVtPZtBH4c1xW+IKQjHbecKqDoY+3anxG0Xdzxs/idKhTV9yq/p6WRZ5aJctd64WmLaWqTpmHVeAV0TD+/Pd+Bb3Aj98LnGDQw0BeHJSeCJJ9q3nZxc/HtcDX1qCjh2LDRAsbkccfS9i+t2sb2P+rjZ/E+UCoM+5Zcpla2JbdW6crk1it4VaKPdAT4jy+OOWQczPWp+dtZccPANXnHN9tHFfmzb2wows7Pm7aOPxwXjajXobjhb8GguR7xqVeu99SlMxRUKmO6XKBUGfcq/ekw6WVsNuF5vb/4F4gNtklqkK4j5FBx8g5crkEdr7rbA7Jqj73scccE4blR+9D3ZtatVqIvuyzUWI+lUTCICwKBPeeebeMZHvQ7s3Glv/o7LaBd93NUi4Ftw8A1etq6OpZVggF14n2lmLCQJoq5g7HrvTO9Jo9FqxUnSohLX4kBERgz6lG9ZN9e6+rvDy9P6BEBXYPctOPgGr2hfuUiQHnjjRvPrJJ2xkFUQdb13PteyXg9mCOzd6y6oZD0Vk6ggGPQp35JOg0srHOB8A2CafuVowcE0T9/UDB/tK1eRvvJOZRVEXe9dktwGPrX+rKdiEhUAgz7lW6cj+H2Y+pKB+ACYtEBiqznb5umH9WK0ehZBNK4Qk+RacjQ+UeYY9CnfTM3aq1cDL74YX6P2zUmv+5v16+n/0/Qru46nk+bnYRitHi0QLK0EaZbjsi6mXdCHiNow6FO+mZq1jx9v1R5PlloBPhpYfRbHieakB/wDTJKUvHfd5bdPm0EZrR6XdyBamArPfrCxXRdm5CNKjBn5KN/ipoBVKosz4oVv+q7FcUxZ69Jk8dMZ+VzBN4vAnPXCQd3iu+COpt+/zZvt18r098zIR5QKgz7lWyfN2lNTQYtAeFT+5s1BASFu2l5WxwlkE5hN5+Kad98vaa+XPj/f/SYtXBARADbvU94lada2NfeaAmPWzeW2/S2t5GugXbd18r7qxEi299GnK2CQxjgQ9QFr+pRvvs3aSZt7s24ut+3vhg3p9jeoTO9DqQTUF4IFhWZm2q9JeMGh+oK566V2OkisFBfUl1Y6O36iIcegT/kWbdZeWgHKI0ECl5kZoNbM5pamLznL5vJBaX7vtuj7IM1bTDTrng780cLafM3c9aJU/EJCQDAbgP36RFZs3qf8083aprS3J04AR4+m60vOurl8EJrfeyH8Phx/uX1KXtw6B51oNDi3n8iBQZ/6J+mUK2OAUMHjgzKlrUiqVWBilfm5uHUOOsF+fSIrNu9Tf6SZcuWqzQ/KlLYicY2k133v3SiUsaBHZMWaPvVGtFZfX0ieVtZVm+cCLPnjqnHrvvfp6WB8holI0I8/MQFMTgIHDizuKigJAFn8GAt6RE4M+pScqVkeMAfcarW1XrqWdsqVMe2ttF7flO1NZ+VjIaD3XDVu3fd+1132oK9UkFNBW7XK/3NHREYM+pSMaTDdrl3Bz7rGpZvqjx0LUugmGagVDhThwoWu9enR+/O1YNtly4BV437HmTTNLnVmehqAYfqdNjcXTNPT1zbKtCKh6drxehJ5Y9CnZEyD6UwLptTrrUVyfIWbZqNBW+9nvhZst3lzcLOvVwCc8jtOrtrWW3ptBFtQ10zPdaOZnrn6iRj0KaEkI6PjAv7SClAeNd+EXVO5wsG7VgM+a2jCH4ZV6YZBpQLcdJPfaoThPvysAzJbfogAMOhTUknWkHfV8HS2OtsNN+419JKrExe3zwBwHSdHdndXtDZ9+2/7r0YY7cPPElt+iABwyh4lZUyzaui3LZeBtWvNK6ctrcRnq4sLzksrQVpWRAoV+kbOKXy9Z5qGeeJE+zRMU5pd/Xi3sumx5YcIAGv6xZW2fzNaaxMBGoba/JVXAhs3mkdc+7yOcaR+U6kUTPlyrZTHKXy9Z0ue9OgeoH6mfXxGlFLda3Jnyw8RAAb9Ykrav2kqILiCMgDMzrb257qBx62MFx29r+f4h6cARukbOdPi9pat1uy6VlH1ejCFT7fWZHX9TJ9XtvxQATHoF5Gtf3PnzuCGG51nbyoglEfcA7N8mk3jCh+2oL11q32fvJH3T5LxHnGyHmjHlh8iAAz6xWS7Metm1/AN11ZAiBuJbWs2Nc29j+7bVPiI7tt0DiLFXNkuL2zJk5ZWzLX9uKl8WQ+0Cxci9efQ9TnTwp/Zt74LeOkwP2M0sBj0i8inRqZvuGlrbqbatm3ufZSp8BG+ydqaahnw+8tUm1661LxtuRyM+3jiCfc+dQIf35q5z1gVUwvTjh3B2IPojJLoto0znOpHA41Bv4ji+uM1feM0BX5b7U0/Z7ohpllG1VTbCwcXgE21eRKtTc/Pt39OllZawfWpJ/36/H2a+33Hqtg+h/O19u051Y+GDKfsFdHUVFAr1k3wrilUk5PmqW83bADWrWv/G/2cSdpWA9PfTU0FedtXXBj8zxtw/uzdi7YplUCQkElfrxs2mKd1mtTrQW18Ziao/c/MLJ7i5wrQYa7PYXR7TvWjIcOaflFFa2Smmr9SQfPr6GirZh8dYR+dkjc5ae8rdfXFK+Wfg50Gg0/AtM3SsJmvAai19hOumce9nm76T3LcPlP9mN6XBgiDPrVuUDt3mm+4CwuACuW7t61eF9e8GtcXbyp8FHA0/k+qwGOhGHLdNHDFIMYQW2HNZyEd/fmKE57iZ+tympiwF2zjji9uqh/T+9KAYfM+Baam/EZSm7Ku7dgBPPJIfPNqtFthYmLx4Lu45wugVgO+FXl7v7U7KAgMHNMqe76FOFNGRZe5uSBhUylyS9Ov5zueJHp80c9kaWTxZ9K3S4EoJ1jTp5a4Uf1zc/abp2sUdrQ5V9fqdTdAOBFLwRPqnDxpjiGP7R3A2r5eZU9/rjrJ/OiTlKnRCLoHTF1RO3bY/y7u+MKfyfoYcNFY6zn2+dOAYdCnlrhR/SLum5lvnzybRK1MqxQDAxxDKpVgoGUa0QKgTxO9UkHKX90Vpbn65tMeX9x+iXKIzftFoPvgt24Fnn/evqiJbspcWjE/H7dUrlLmJtnJycW/s0nUKto6rTGGoL2p3cb0WbItwDQ5aZ8N4IMLO9GAYU1/2CVNLhJufn90T7K86Xr0frSp/+DBYJS/fj02iVqNjwcxIzpu7LpmDBmaQX5phT+frlp/9LNk6i6YnAw+m520ODG978Ap+neIQX/YdZJcpH7G/3XCA6ba9hN5PTaJWlUqwBs2mW9KP6kGg/rCMepbzRhVpJsWgPgZJ6bPUrS7YGYmm8Q7BR+HMkj4HWLQH35pa9U+o53DK9/FDZgKvx5XPHO6Ysp8A3rMUn4byEF+WdCBNu1niS1OhcPvEIP+8Etbq4678dly3fu8HptEU2GMMujks+RKMU1Did8hBv3hl7ZW7Zq+57qx+r4em0QTY6+IRdrP0vQ0sGtX+5SJ0wvBmAF+PodON79D206v7nwnEefKHLadPj/TffY16IvILQD+FMAIgL9SSt0fef49AP4EwOHmQ3+mlPqrnh7koIvWhKLJRWxMN8RSCbjxRvffshafCdNgo+umF/dHAosH+RVaklS40eWdoxoNLqgzpLr1HdIBf8lktp8ZOX4AS1Zmu8++BX0RGQHw5wDeBOAQgMdEZLtSKprl5TNKqQ/2/ACHiSu5iIkeuW+bNJ7k9ZJgDnMA9sFGb9hkH+RXaEnyPvgu71yk9t4C0d+VLL9D3Qr43dLPmv40gKpS6ikAEJFPA7gdQMwC29RVrqlQ3awBDXDCnqynALkGG911F4N8myQzVHzT8Ra+z2R42QbKpjFoAR8ARMUlXOnWC4u8HcAtSqn3NX9/F4Drw7X6ZvP+HwN4BsBBAL+jlPqZYV9bAGwBgJUXrlj/dw98IrPjfBklnIOUNd4cij2f558P5vK7rLgw24NyvW5pBFi+3Ppn/b4+tRpw4kT748uWBdPvknoZJbz8rP18VqxIvs9+6sn1efYZ+3PRz6pr27PEegH7/XnLGs8nvV+oUQCALFnatdco1efRKPvt/21vuXmfUurauO36WdM3LeIeLYF8BcBDSqmaiHwAwIMANrf9kVJbAWwFgNdecblaXz6V2UHuq48hy/31W+z5fOlT7h2IADfdlH3t2/W6W7ZYn+r39Zn5bLbZXR89OYbZr5yyTj3/jQ4yxvZDT67Pju3+F8G2rSYCrF0LbNwIoP24+/15yxrPJ51e1fDHjh/AqZVXZbrPfgb9QwAuDf1+CYAj4Q2UUs+Ffn0AwEd7cFyDoVv933GL7ijVnWb3AR2anmQKUFw3wE+qwIkJczczB+w5JJmhYhuxrynVnkGSKGQQm/TD+hn0HwNwhYi8EsHo/DsBLCqWi8jFSqmjzV9vA7C/t4eYU93s/45bdAdIl7UszesOQMIe37KKTyawx/YCF7S1YwWVzzdsYl++taCb9YyRbny+qWe6mWZ30AM+0Megr5Sqi8gHAXwdwZS9TyqlfiQi9wH4jlJqO4APichtAOoAngfwnn4db650klo3Tlx6Uy3r0c0DOtXPdwqQTyawuTngAsNrKMWAH1vQtc0YiRYU6gt+s1JMn+9qFTh3TdAV5fp8chZK39Rq3UuzOwwBH+jzPH2l1MMAHo48dm/o598D8Hu9Pq7c63Zaqbj1x4Gg+rl1a7Y3tQFM2OM7Bcjnktl6MnLew9EbaQq6poKCL9ty0Le+o7UvU+vaAM9C6aUktfEk2548aS9cA+lbAIYl4APMyDeYetH/7dO3D/CmhtYUIH1z2rkj+D98U/G5ZNdNA09Fni+VgIV6q3wVXnxn6Ofrh2vMNrbnqtX41iqbUqm9W8m30NHNVrghkWTRG9O2O3cAx481x1pG2Bpx9GukaQEYpoAPMOgPpl70f9v69vUiO2G8qcXeyEzdAHq7mZlW0H7mZKuAUFkKLJwGavOL93n8WPuKsEO3Uljc0rmaqaCr/zbL6ci+rWsFT+7uk4q2ehpYWN/++JdOA1Ong59ffAl45ufAwgIAw7a7AVSfBc47d/HjlYk5PHu9IW2toH1uWOQ1XYYl4AMM+oPB1Ee4aVN3+w1tfew+q+gVkK3PftfOoGYyMQFceSUwO2uOEzpoVy5rzTKbmWkF/PA+9+83l7uGaqUwnyQ6toKubwIeG1MSKt/WNdt23egOyxnfGvEjf29/bt3mZpnN4xJOPNc+I1NmD2Df4am2+pBrX+sMg2eHGYN+3tn6CDdtSjcRPAlTH7utubXgHc+2Mk+4F+TgwcVpdMN00P7ly/z36XsMAynuZFzBM4s3QjfB6NfQLV9hpkKHrYWsIN1hPjXiuPKTb5nNtI9KxVwf4m2rhUE/7/LWRzigU+u6LW4IBNAK7L4twK5Koy15z9BwRYa4wm7SN87GFKCl1Hq+PNL+N9EWsoJ0h207vdq7CTzuFuJbZrN93m3jgU1lsWjZrghK8ZtQX3W7j7BaDT71W7cG/1er7u2npoKitP7GTUz4rdo35K6bDm5ccXTtwyT6uGmf5TJwsaXbdHIy/vUHxrTl5H0Kl7a/vemm5CUjHaC1cACfrwWRJPqdmZoKCiZbthSiWSZJwAfibyE+lyj6UdC3sWeeNd/Goq8Zpst2cbe+YcGaft51c6R+2ulFAzi1rtuiU/dctXHX3P6XHPvUf/vYXhjNzmZyKvnQSd6GqSng2LHW4AeRYECF/tu46ahR+vu3dy+w+bbFz8XV2gc006QvW8CPS1XguoWYWgJKJWB0NJiHH92f6Ta2axewZ0/79lNTQaHA1L02ZI0vVgz6edfN5nRb18HOncGNcYgHHXVDePWu6Gh+bW4uCNrhQX3hKXf7UvZluh4fWJ0s0XzwYKvUFU2tmzTo6wCd5o0f4u4wV8DvJFVB0vKe6TbWaAQBX7/+jh1BOXDjxgJ9fywY9POum5nqfEafhQcvMcuYt2gtPSw8qM802j48Bz/6d9/aHUzli47qB4am8ti5uHEwPgMwwnTH71LLsomuN35AM03GcU3Ny2IYUpLynu+lfOKJoNw35I0vsRj0B0G3mtN9R589ugeon2GWsYR0zd+wXXWsAAAgAElEQVTWnGiaYhdNIxpVrwMj5fZpSFyQJySuKueaegqYvxdzc0Ebc3RxUFPnsmt9gCEQNzWv1zXpJGW4vXuHuvHFCwfyDYukA/IA84Ank/mavehOsZLcBE1pRKNq80ErQXggFBfkCYkbKTk15a6133WXeR+NRjA+wDYCTbdr6ws7hCPEfObiJ00nnebWFeZ7GwOCS1L0scis6Q+DTgbkAenTlRalE6xDSZoTfdaCmZhoHz+g0/8ObUreJHyqcjdscM/hsnZ9NexTBvM2vTZjvsl3ktSkXbcuwK9XJNyDAgRz9XV/ftTEBNdDYtAfBp3cbNIMbNJsKVCL/I0y8F2JDwgqki7RXPznngccOdx6fihT8ibl048e3SbMVZgtGebmG/5u2/W/tfg5j/S0/XCuzGHbaUPaWgufqXlJhjHYbl179gBnYnoUTbeaC5cBd98NPPJI0IcfVi4H01qLvh4Sg/4w6LQTLa5TTLedxRXducKYke9KfHFMufhNl23oUvKm4dOP7prDZVIuA+PjACzNMc3vkQ74S54+2no8p7nb5fgBLFmZ/bH5DmOwve2mmnq4HmO71Vz5xuD3jRuDQXumzHxD3BjjxRn0ReRcABcqpZ6MPP6rSqkfdPXIyF+nw1HjBjY1GsBVVy2eY2YquvMbZRVujndx9bKMls2j9k3Y85KA683S3y39ma9UAJwybzs9jW2nVgKNRivgF2mEWAppJlIA9lvNyZPAWPN3U8GDS4c4BvKJyDsAHADwBRH5kYhcF3r6b7p9YJRAJ9nLAPfAJiAI+k88Ed9WV/QJsBkoOYbWJnkbizL9yCrJ6DDbmyWSqJtq2+Qm4OJVWPLcidZ+izRCLIUkg/CA+JQJZ2LGxCQdZDiMXG/3vwOwXil1VESmAXxKRP6dUuqLaJu3Qn2VxVxg28CmKFeTfdEnwGZgfDx+VTAfhZ6+l7SbKckiOZddY3zJs4PcfmU98CuGtWDJyDW0wkSnmrbdakZi5qMVfboe4A76ZaXUUQBQSu0VkZsAfFVELoFxZWLqq07nAicZyW9rsuc3yiqccMfVp1+p2Ffi87VuXcH785N2MyVZJOeya9pGkG27+f3AucO15novJRlaoRMr2m414+PWERdnXwso9lhjV9B/SURepfvzmzX+GwF8CcDVvTg4ypC+UW2+DdixvT15tf4WLK0Apxfi546Zvp38RhlFU/LGjbDX/f9bt7r3OzER1HxM6XwLLU03U7jQbHvj5+aCEWahaLPt6rcAR49hydLzOjhgAuKHFgGtspeeNRm91VQq1hEXZw1ZrqTEXEH/dxFpxldKnRCRWwD8XlePirIVNxk2/Nx8LehYjluGNOm6lgX2mKXiGTfC3nYJRID3vz/+dX1bF4ZOp91Mrr8PZU86O0r/nw4Dz7zo97nnlFYr39nD+tLoW41+S3fsANa8ATh8gm+pi6sH5EEAbxORswUDEVkJ4L8C+F+6fWCUIVdzp221isoS+wgb3WTfaSqtgkg7vtFW5vLJo/STKvA/dy1ODvc/dwWPD71OB7ba1iienAQaZwCgfVqeT19MATL2dcqnXBbeJvqWnmkEwf/BB/m22rhq+usB/DGA/09EPgzgGgD/F4CPAXh3D46N0jDVJNJEnfla0DkczXABBEvEAZyT7ylJxbNWA2Y+a+9atv1d1KOPtvfQNBrB40Nf20/bzRT+7pjMzgJXrm8P+IDfReGUVqu4tz7svPNa/f+274juhQEK/9a2sQZ9pdQLAD7QDPjfAHAEwOuVUod6dXCUkKkZP25hkZMn7W3ItgXaZ2eDf7yBefHNyPeTKnAiVEAwXZZoRj5bk71tPr/vPP+Bl7SbKfrdMZmbwy/OXQEcPxY06Wu+rQic0mrkeutNM1kOh956V6sXb0dm1qAvIq8A8FEA1wO4BcCtAL4mIh9WSqXM20pdZapJ2Ogbla1QoFS6m1TBb2Amvhn5HtsLXLC5/e91bcaUkS/vKXd/oUYTpXntm9MvA+vf5t5mdBTnjoxgySt/OejDT9ovzymtRnv2mG9bExPAwkJn01d5O2rnat7/LoCPA/iXSqk6gP8hIq8B8HEReVop9c6eHCH5S/IJv/LK4EZla1MTCfr15w35MF0ZMgp+A7Pxycg3NwdcYHhcKWDLlqBJM1pTtw0IrCw11+orSxMddke2nV6N8yr1rqR5zdzfx9RjymVg0ybIkjpwacrBqpzS2qZatS+Ok0XA5u2onWsg3yal1H9oBnwAgFLqe0qpGwCwpp9HST7h+/cHbcT1BXMaOKWCqXvR5/RNqtPBUtQmLltYkoaXG25ov3SlUvB4L2w7vXqw5q27vjtZZdYr+pquBq7VueMWn/LB21E7V5++te9eKfVAdw6HvNim/dgyi5nozjBTTV5rNIJ5++VRe1Mmpx9l5rpp4KnIY+G+/yStw1kt8pPGtpyuKOecLmerhWcdlDmldRFXbd53tW9duI0OXF23jm+1SYKsx5QLPilGH93jDuZJzNeALXebn+MNLFNXTAHPnFy8xks4UCdZolfvr9d9/b5rrvdc3PeGiaV6rlq1j76vVIDRUXOhYHQ0eD58mYBWqwEvnRuD/qCJm/YTzVjRaccYO8UyE06Wo/vWa/OLg3ul0so2FtXP2ruP3AZ8wG+6XDj4hzO+MIpkTpfBbDNUAPOtq1wGrriifWKRvnRjx+3fHwow6A8a345d0w1sbg6J1kpiH31moql4w4PswqPwcZl7P/2ovfvIdcAHkg2IcLUKLOvO4RWNbaKR7sc3De7TaacPHmR6kE7ErElEuZNmbcipqaD4u2ULsGyZey1L/a3jIKNMmVLxhtXrwK6dwLPPBqP0BylzXu4DPpDse+NqFaBM2MpgSpmX/ZiYCG5hrvQg5Ic1/UHjM+0nXLPXnWa6ifKya4Jgbur378bAJQLg18sSXsk17/PvtYEI+ED898anO4yTvjNjG5RqE85cbHt+Zgb47Y2dH9uwY01/0ESn/SytAOWRoO9xZgZ45JHFyaija4LXasE+3n03sHkzpw/1SNKhEXr+fZ4NTMDXAb1eN7dkRRO423B8S2ZsM34rFfP2+nHXJZibA06cYM79OKzpD6LwYL1o36MpV75Wrwdpd8fHFu+Hus408j5ONAbladW8gQr44e+IUq0afnjEftyFOdsq0EF6ODrLNlkCAHbtam/iX1gILmXcrGQFpt6Nw6A/yJKk3dWaq4RR72w7vRqYBEq3AS/+PLiBjYwADQWoyM1tzfgcnr0+SFs7OgpsOx08/uJLwNFTgLo6+P1ZAE+fAi5+Fjjv3N6dS1juAz4Q3z/vO8NFtwocP8DlcTNiq3Ps2dM+kK/RCN5yPTLfddnYC+PGoD/I0ny6SyP+2/Lm1rFwjfhCABeGntMrhYW96pID+Menp84Or1jSXOX1HwzbAsDEM8GNkJfKwhUZfBNZTUy03szaPLD7uxw+3kVxaXl1YeGBB+xrhZEdg/4gSzoaplwGxscBGIbHRvkkASKnuCZw16WLDq9wxS5eKgfbd0TEL+BHB8nOnYyf788SWEdcl6xabb2Vtox9Ppn8inyJGPQHQZK0u+VysJjO7Kx59H6lAlQfj//Ec+3vjvj0ebvS6kbfYte2vi3YRbu5AbB/R3wCvsji0le12t4fo+mLwxJYx2z99kotfivTLlpY9EvEoJ93Pp/QJHf1kzW/Tzw7zFLzHeRmurkJzPmQXDPObKsjR1uwi3ZzA2D/jvj05SvVvs7Eqy81b6sjDQvLHdNv086d7bX28FtpKxycd557/0W/RAz6eeebdtfXSY/mSYBrf6eUZFS7KR4tWwaMT/ptq8t3rtWRi3xzO8v2HYnr049+1l2FBF1SY2E5tWiDpq2ZPty3/+MfA4cPL37+8OFg5vJGy5z9ol8iBv28y/oTahu9PzcXLLWrownX/k4szTS2aDyqHAdOeW6rJW3BLsrNzSlaiooyfdZtBd7K0tb+WFhOxdSgaRN+K48cMW+zf7896Bf9EjE5T96lSbvrEjd6f24umCgLcO3vFDqdxlarBaP6t24N/vdJNGJbpj3rj87QCaen9klUNT2NtrUrymVgww2LtzFlnWFh2apaDZryk46rrFbTDeYr+iViTT/vsq5xL1kSv02jEaTpfffdDPKetp1e3XHAr1aBi2uLU4769sH7tmAX6eaWiE832dQUMDu3eO3j6BgaLtGbiGu1vajoW+nKt++atlf0S8Sgn3dZf0JPn/bbLpqXn6yyCPhAcIlve/Xix+r1IFmJz+U3TfLYtKm4N7euqCyNX7uVmS69+eYXMy057eoCWLvWvb8iXyIG/UGQ5Trffc7Ip/u9s3KuzGHb6fMz3WdSWWWms93EarVWwhJb7d82yWPTptbNsjBLxKedhF3kydt94ju+pFZbPEcfsPfNl8v2/nzqc9AXkVsA/CmAEQB/pZS6P/J8BcDfAlgP4DkAv6WU+mmvjzM3sphg6puRb6ll5YsOdCNfuxw/gCUrh+PG7NvXbhqBHzfJozBzk9OeqOnvduwoQAmpv5LkF4t+5m1TXjdtyvQQh07fgr6IjAD4cwBvAnAIwGMisl0pFV4x5r0AXlBKTYnInQA+CuC3en+0OZHFBNPx8fhtSiXghg3Jj89hYBZo6aPpaQCevSrRG2XcJI/CzE1Oe6KuduZwwWFZNoc57HwbTeIW0AnTy+e6uq9sU16ppZ+j96cBVJVSTymlTgP4NIDbI9vcDuDB5s+fB/DrIgXOrJzF9D3b2pXaxARw442ZRgIGfD9J3vJoq0DcSP24NL5JZwzkVtrvSNzz4RSH5BRdqViXmUyfq6kpYOVK/32bBrlOT7daDE6eHPDPbw/0s3l/DYCfhX4/BOB62zZKqbqIvAjgAgSLjBVPVhNMXfuJG6SUEAN+MiOexfDJSG0mbpKH7ZJXKkPWqp32O+LTzswEB21MNXpbY8vOnebPlW2ufRw9yPXMmdbrnWkMabdVhkT5zJXoxguL3AHgN5RS72v+/i4A00qpfxXa5kfNbQ41f3+yuc1zkX1tAbAFAFZeuGL93z3wicyO82WUcI7PAjW9UKsBJ04gWDVak6BNK64G3/QySjindqrj/fj4hRoN9rxkaWb7jCrV59Eod2//PXd6Hs++tBRx38qRErB8+eLHarWgpnOmETyve3L0Y1GCYGpTw/Figs4+Fj2/PrV54MQc2j/bE8HI+0R/F1EqoXTu+FB93jq5Pvp2FHmnYz+7ejv9uXompgo3UjJ/fk0uGJ/HcyeXGr8fgyjJ9XnbW27ep5S6Nm67ftb0DwEIJ7K+BEC0zKe3OSQiZQDnAXg+uiOl1FYAWwHgtVdcrtaXbTnNkttXH0OW++tIGcDRo+1F61XjsOdxW2xffQzrxxsd7ydOr2r4Y8cP4NTKq7r6Gr00dvwAjlau8koNv2WL4e9DPx+s2vtL9SW35e2Pbpu2Aagv18dU/fT5HIb/Lqq51vFYuT50n7e052NaGhporfEVR3+uHtpuXyJ3fHzxumFx7tpwADN7gvMxfT8GTTe+P/0M+o8BuEJEXgngMIA7AURvLdsB3A3g/wXwdgA7VL+aJvIiqwmmXZyoyib9zoQvzYMP2tcXn5lxN7/bxqaFg7hP4WLgWrWTfrajhYTNm4PHTSPRjh/ozjEPINvnQim/hQz1wLzVq9vz50dfw3TXL5eBkRHz94NZJ+36FvSbffQfBPB1BFP2PqmU+pGI3AfgO0qp7QD+GsCnRKSKoIZ/Z7+Ol/ww4PdO3Gw0nzFtPqOnh/oG6pPggIxcwyfCCxm6aulzc8D8PLBmTdC3r1Sw/chI/GdSj1dh1slk+jpPXyn1MICHI4/dG/p5HsAdvT4uSocBP3u2Wr7mmo1WqZj/vlxePPXpyiuB2Vn/dWeGSmHmMmbPNXg0mk/MVbCs14EXXwTe//7WY1u32l938+b2S6MLGCMlLhEShxn5KBMM+N3h05eZtPm9Xl889engwdaNsnBJ6Yq+zmoHfDOExy1oCLQ/7ppMES2PhQsYY8eBsQRTAIuIQb8Psk5Fm0Q309Yy4GfPd0CUSVwrgRau2BYuJ3nR11k1SFLw8/286O1sg/+ib7drkCnLY51h0O+xfteIhyltbRHETR93Nb8nSXFa2Btp1qtYDrhup2u2jSGJ5p2Ymgrm4HOQXvYY9HuoKwG/cO2xwyV6+X47slCIa6Bd3OVOkuI0w/QMg6Xo66xGJE2sk9TUFHDsGPDEE4sfP3gQWLVq8X43bGB5rBsY9HukawG/EKuoDCfT5TtxAjgaWk0sTUwKFyQqlda0pomJYKS0TyGgUArXp2Hnmoann3fdYlx1EFcaBNPYSZbHuoNBv4cyb9LnyOOBZrp8Cu6BSnGiBYlaLagdrVsXjNC3BXzf/v/oaw31Ddl0gkO+4I5Pl5DtFuOqgwDxrU6m12V5LHsM+j2w7fTq7vThc+TxQOvG5bOVA6PNqVFJ+0mHvpHJdoL/7BrgmzNDW9Lx7RLSCzWFy0QLC/Y6iP7ZpVIpQEEyBxj0u6xrAR/ozshjfut6phuXL02BIdpP6vMRGPpGJtsJzs+bl3obipNub1K3TRk1LdRk4/uZPHMmm4Ikb2FuDPpd1NWAD2Q/8njoq2/5Yrp8guQBOCzJiH29fbTf1ecjMPSNTL4nMlQlnUBcYp1yM2r4jg2JW95ZM+0v6dtbq/EWFsdzIU9KqusBHwg+xZs2tb5VExOdpaNyVd8oc6bLt2xZ+w03XLHcsSPIx29bM3x6unVTjqNz8Ic/Lr4fAVtrxNBMp0pyIkNT0mlnu8UkGQNy3nmdjbhP8vaePMlbWBzW9Lugp8l3shzpMvTVt/yJXr7K8dY6h7YFc3RtRv99dH/6b3XrwORkMCXKp0HI9yMw9NPbk8x3HJqSjpnpFuOzUJN25AjwlrfY593HSfL22pbg5S2shUE/Y/1OvtMRZifLFdeNytXsabpJr1rl103g+xEY+ulUttJTVBdKOoPQJ52kTKTHBJjm3QPuVNNJ394RS9s1b2EtQxf0X1Cjmda006StHciADxSg+jZY4vrnk9ReTAUBU3BJ8hEY+ulUphOc/V7rwnQhIvdyWE21CqxZAD715eSnYioTnTxpDt4i9r9xpdsFkvdWjo+3L+vLW9hiQxf0S0sqmQbdQqWtHfrq22CJq0351l5MwR2wryi7aVP6ZEBZfmRyWeOtLO3qkru9mhWhCxfvuD74PU3hIlomeuQR89TQtWvtfwPYuwomJpKfc6WS/PNbNEMX9KlDQ199Gxz6Mpj6Qn1rL7aao2m9ch1cooP7XPuOHlsnNdNoJsGFBaDR6Hy/3i+agwjRq2E1tsLFnj3p346NzRTS+/cHNX6R4HP2xBNBYijbvrJuYOQtzI1BnyjH9A0sbWyy3dxtrQe+wcW1Rnqamqkpk2AW+030ojmY39WrYTW261yrtd5709sR/RxOTgYBPfy53Lgx2VvLBsbeYtAnGgDR4L9jR/B/9OYYvSknrSH6BhfbzAIt6evG7S/tfhO/aJ/n3fdqWI3vZyP8dpgCebg5PxzYk7y1OWtsGXoM+kQ91MkNznTT3bGjtfpZdGqe66ZeqQQZ0NIGl7iAkbRm6hvMM63x5nCKaq9qvbpw4UO/HT4FMx3Yfd/aHDa2DD0GfaIe6fQG57rpRmtdLuVyMH1K79MVXGyFFFdNMU3N1KfmmXmNN6dTVKOBXyeWyTII6n2NLAT/69z5tvXrq1X/spCrlUkk2Ff4HHPW2DL0GPSJeqTTG1wnFVDbLLO4Ufm2QoptZkGlEhQokt6wTfsrlYDR0daywJnXeHM6RTXL2q+rZWlqChg7DmzZYn5dIHg7Jif9WwWA1uuYPh9KLT6XHDa2DD0GfaIe6fQGl6aPXv9dmllmrkKK3p9pUFd4vME1nkvR9mUwV4IX1cHztlcD27/Z3WNLUzhMMi0TsCd10q8f3o/veAugVWbS+9q5s33ufvhcctrYMtQY9Il6pNMbXJIsaFqaims4gJjox10Ls8zNAbt2AatvAP7bl+0jvcPBpy9TrTxetNf9zkkLh2mmZdqO2/R2uJLnrFtnv6ZTU/a/1edi+0zPzQEzM+7Clamg41vILDIuuEPUI6bFcJIE5ejiJ1HlcnAT7mT9pegiPyam1zfVBhsNoNGs5ekxB+HFg3bvti8clCe9Xocq6WJGtuOz5bnPakZHpeIuxLn+Vj/u+ky7PiOmxah2706X279oWNMn6pEsmrCjteusm8PjmnKTLtTjMigDtrLud467bkmHGnQSxGu1oEbt+gzZxluEB/7ZWj98zkV/pvVxhNk+I7aCzsmTwFj8W1BoDPpEPZRlE3aW+4pr0gfMQUFn5UtrEAZsZdnv7NNVkLRwmGSsRzjgVqvAxbX22nL4GGzHYxrpbwrQpqySIyPmY0tSuLJta1tlj1oY9IkKzpVdTzMNBqxWg377Rgc3WlPgTNuC0a0kL1kO8vcdpJekQOc71kNkcXePHphoOxbX+7l1q/k1rMH4TOtn29LQSQpXtm1tq+xRC4M+UcGladKvVs0js5Ow7TfNoDnX3wGdd6nofQCdFShctdm4ZnbNFIzDi8zYKLV4n65jsb2fx44F/fg2vuM9TAWdJIUr27bj4wAr+24M+kQFl6ZJf/fuZAHfZ/Q+kD6XgWsBmXDmwX5nfHM1xcc1swP2YLxpU6sl5sEHzQPayuVWwaJSsR9jpWKfaudKABXtOogrhESfS9KtYdu2UgFO2V+SwKBPVHiuZlXT/P4k87aj+wkHg3CmOd9pgja257NYuCfLKXu+TfFJB7D5nE+93nqfXKPcFxaSt+CEA7RPd5H+m6gk3RrGbY/7/W2RMegTFVw3R4sLFtf+TGsH/PjHwPHj8WMKXJImLkqybZapYqM1VBF7gE0ygC38eKfT1tKM0QgXDn0KhTlIfFhYDPpEBdfN0eIirZ9tweDwYfc+bH3/0WyA4cWG9N+NjNjzyfvKesqefl99Bk+aHosb7JY2c2Na0eOMe22upNdfDPpEBZV2tHuSzICNUK71tIGoXg/6mI8ds6/VfvAgcOWV7WMGjh0z90NPTvq/fjdSxabNh2B77/X5VKtB83warkKSCLB2rblgFT3OpN1F1FsM+kQF1Ek/dbRlII5uCu+kBqpUK3jPzpqb22dn24OKLYeAawR6VDfW5Uk6eFKbmjIXZA4eBI4eBV54of1vKhXgVa9qD9ilElCSxa8JmM9VT/VbtSq+oJjTdYyoiUGfqIA66af2aVqPmpsDNm9253H3sX+/Xx+4ThqURSratFP2XC0pSWvD4X2Fu0y0et0c8IFgpcKNG80B+4JlrVX2wlwr8yUtFLI5P18Y9IkKKO18cZ+mdRORIOCXy+2Fg3K5vWneth+lgpqrq5/eN9lQEjrYjR2PD8qmglC0JcWV2nbrVvdo+KQj600LJJ1lGO2ut9PnFF41saOR9ZQLDPpEBZR2vrithUA3rdsCrg5U9XoQ3EZHg8BtqwU+8IA9uC0sBPsIjzIPNx/7jB7PsqnZVBAyjSMIt6REa8OVij2XfdIpklFxBRzbsrzhbItzc8HvAIP5oGPQJyqgtPPF47K46QClp6IZWqLRaARB/+677a+7dq09EUyjEQTJ0VFzi0Rc032lki5wVavAmgXgU19e/JpJgnL42MLBf2bGnsu+05H4rgKOTokbnUY5MtI+da/RAL71LQb9QcegT1RApn5Xn6lptu0qlfYm6HIZsLVExwWyjRuD/22Bv1azFxpc51IuAxs2uF/bRNfm33F98Hu4Jp4kKNtq3VlPC9TWrDEHaV1Au+3V5gJLOFd+2MJC8LcM/IOLQZ+ooKL9rqalTYHFgco2MhswN/vb+PSpb9xoHyfg+ntbK0alEgT8uKWJTY+7Bj4mmZVgq3V3Y279unWtwlOYb8Y8m+hgz24tdETdwTWJiAquWrUHfNPa55s2tYLuxETwe5IscKWSf5/69HSrUGE7pijTMW7eHLQMhAP+7t3t4xceecT8uKsmbjpG17H5nmccV/78cjkYrW/S6RiB6CwJ0/tVrabfP3UXa/pEBeaq9dlqbaaR2Un6nkdHk40CD+8/rkbuO7XMVnM3TQmMa7HwTa3rap1Ikp5XMw1oDB/zzp1B/3zSMQ9xwueRZYpi6g0GfaICs9X6kmZPm5x0r8AW5tsqEA3qmze319SjA9BMQc7ENSXQV7jFIlzIMBWkfJLThPdhW68+TA9otL2f+lyiszA66UqIttJ0aywCdQ+b94kKLKubdpIMdz79+XHNxq4map8m5k5S6MaxdYEkqfn6Hp+e9hhH174Bc1dCqdTqLjAl/zHp1/tLnWFNn6jAssor71tI8E3HGtdsHPd6cU3MtgGJSfq6Gw37a5i6F5IMePOdUqn347Pt3Fwr8Y9OhhTeh08rQ/icdSHCdvyUT6zpExVYmoFyJrZCQknS1XjjWiB8CiWugoGtNt6twk7SAW/R46tUgtp4mL5Oetskx3zwYPC3F64IunHC1yTuPQifgw378/OLNX2iAssqT7qt5px2ZbW4Fgif2m1cM7VtsF/S6WwzM/HvWZoBb9Hjixu4mGQwpX7ta369/bm49zZcUMp69UHqvr4EfRFZDuAzAC4D8FMA71BKtS0XISJnADze/HVWKXVbr46RqCiyyJNuSisLAC+dALZ7BMWouJXafFb6S5qjPrxf12I9UT4rFGYxdiLuOiVZ8tj12q73IHwNuJreYOpXTf8eAN9USt0vIvc0f/9dw3anlFKv6e2hEVEa4YVa0i7bG94X4DdVL24Bnk7Owzf4x9Xae1Ertr1ntoJR3BRCn2mRptdj036+9Svo3w7gxubPDwLYBXPQJ6IBk8XcbVe2vGiBolRyL8Dj2p+LKfCNOEZBuWrtvaoV+3ZZ+L52XOsCV9MbPKLStIF1+qIiv1BKvSL0+wtKqfMN29UBfA9AHcD9SqkvWfa3BcAWAFhx0cr1n1E3siYAAA19SURBVPivn8rsWEv1eTTKSzPbX7/xfPJtGM7nmWdbP18wPo/nTrbO58IV8X9fqwEnTizO2y8Ali0DTp4EzhiS0ZQk6MM/0wgC8/h4q4vBtT9XVjuj0/N45iXz9RkpAcuXu89LH3/0GLvN9trD8HkLK/L5vO0tN+9TSl0bt13Xavoi8g0ApkSQv59gN5NKqSMicjmAHSLyuFLqyehGSqmtALYCwOXrflWdWnlVqmM2GTt+AFnur994Pvk2DOez/ZutWu9dGw5gZk9wPr6D+lxrALhq01u2tH5uADjlsb+kgwxP7W+dT9TmzcDYSvffj4V+Dh9jL5heexg+b2E8n3hdC/pKqTfanhOR4yJysVLqqIhcDODnln0caf7/lIjsAvBaAG1Bn4jywzagbG7Ob6S7a9Bbmr7xLLPGmVoZtF42c3ORG0qrX3362wHcDeD+5v9fjm4gIucDeFkpVRORFQA2APhYT4+SiBILD/CK8hnU5wrspgJFqRTkodeJZ3RfddwUtjSD6Gx9+qZ9dSswZzFQkoqrX0H/fgCfFZH3ApgFcAcAiMi1AD6glHofgLUAPiEiDQRJhO5XSnlm9yaiftIDvE7tb38ubba8cNAMTw1cWGiNsJ+bA3btCqbrxQ1XSjOIbny8PXOfaVBcFoHZVmjIcpGbWq3V/cEWg2LoS9BXSj0HoC0thFLqOwDe1/z5UQDX9PjQiChDtubwaA08GuB0mthwMAIWB6jNm4O/iU6pM606Z5ImuFUqQfa7uBp8p4HZVWjIqruiWgUurrVnCQQY+IcZM/IRUdf4NIebAtzBg4tT9tpW1Uur21njOg3MrkJDVnP+9+4Fbnu1+TUY9IcXc+8TUdfo5vCwaHO4K8C5tkmrk/nxtZpfDn1XAHatTqe5Cg1ZrZfAZXGLiTV9Iuoan+Zwn+CTVSByZfaz9WeHt/tfN/g1209P21sifGrSrtp8VpnwbAUT5s4fbgz6RNRVcVnbfJqrbal2k9i82d1dYOrPjm5nGxsYPf6pKXvQ9ynATE8HAxLD4xNKpcVrD3TaBD89DYgjtz4NJzbvE1FfxTVXV6vBCP04ExP2WmqlsjhI+nQp2LazvbbPY67He21qKshKmGbpYxpcrOkTUV/FNVfv3Rs/Il/P1Te1BpTLwIYNix/z7c/2qZXbased5Ns3nXOjkf0gu0ol3dLHNLgY9Imo71zN1XGBNzpXP2pkpP0x3xHwtu1EgjwArv70Tvre0w6yY6Y+isOgT0S55grQd90VzN139ffrEfdAKwD61sJt2/k2g6fte08zLY+Z+sgH+/SJKNfi+vx9muCj/fVTU0HgjuvPjm43UupNv3eaaXm+4xSo2FjTJ6JcCjdVVypBM32t1t5sHbf6nmYaYZ+0tj52PH4lvSyk6RrgvHvywaBPRLkTbaqu1YKabnjanTY5CTzhsSpHXkbN+0raNZBVpj4abmzeJ6LcSdJUPTsbv78izD/PKlMfDTfW9Ikod5I0Vcc1XxdlFHtWmfpouDHoE1HuJGmqjhvdXyRZZOqj4cagT0S5kySxjWtbzlsnWoxBn4hyJ0lTtW1bgPPWiaIY9Ikol5I0VZu2nZnxWxGPqEg4ep+IhhLnrRO1Y9AnoqGU91XuiPqBQZ+IhhLnrRO1Y58+EQ0lzlsnasegT0RDi/PWiRZj8z4REVFBMOgTEREVBIM+ERFRQTDoExERFQSDPhERUUEw6BMRERUEgz4REVFBMOgTEREVBIM+ERFRQTDoExERFQSDPhERUUEw6BMRERUEgz4REVFBMOgTEREVBIM+ERFRQTDoExERFQSDPhERUUEw6BMRERUEgz4REVFBMOgTEREVBIM+ERFRQTDoExERFQSDPhERUUH0JeiLyB0i8iMRaYjItY7tbhGRH4tIVUTu6eUxEhERDZt+1fR/COA3Aey2bSAiIwD+HMCbAawD8E4RWdebwyMiIho+5X68qFJqPwCIiGuzaQBVpdRTzW0/DeB2AE90/QCJiIiGUJ779NcA+Fno90PNx4iIiCiFrtX0ReQbAFYZnvp9pdSXfXZheExZXmsLgC0AsOKilRg7fsD7OOOU6vOZ7q/feD75xvPJN55PvvF84nUt6Cul3tjhLg4BuDT0+yUAjlheayuArQBw+bpfVadWXtXhS7eMHT+ALPfXbzyffOP55BvPJ994PvHy3Lz/GIArROSVIrIEwJ0Atvf5mIiIiAZWv6bs/QsROQTg1wD8dxH5evPx1SLyMAAopeoAPgjg6wD2A/isUupH/TheIiKiYdCv0fvbAGwzPH4EwK2h3x8G8HAPD42IiGho5bl5n4iIiDLEoE9ERFQQDPpEREQFwaBPRERUEAz6REREBcGgT0REVBAM+kRERAXBoE9ERFQQDPpEREQFwaBPRERUEAz6REREBcGgT0REVBAM+kRERAXBoE9ERFQQDPpEREQFwaBPRERUEAz6REREBcGgT0REVBCilOr3MWRKRJ4B8HSGu1wB4NkM99dvPJ984/nkG88n34p8Pr+klLowbqOhC/pZE5HvKKWu7fdxZIXnk288n3zj+eQbzycem/eJiIgKgkGfiIioIBj0423t9wFkjOeTbzyffOP55BvPJwb79ImIiAqCNX0iIqKCYNCPEJE7RORHItIQEeuoSRH5qYg8LiLfE5Hv9PIYk0hwPreIyI9FpCoi9/TyGJMQkeUi8g8i8pPm/+dbtjvTvDbfE5HtvT7OOHHvt4hUROQzzef/UUQu6/1R+vM4n/eIyDOha/K+fhynDxH5pIj8XER+aHleROS/NM/1ByLyul4fYxIe53OjiLwYujb39voYkxCRS0Vkp4jsb97bPmzYZiCukee5ZHt9lFL8F/oHYC2AXwawC8C1ju1+CmBFv483i/MBMALgSQCXA1gC4PsA1vX72C3H+jEA9zR/vgfARy3bzfX7WB3nEPt+A/g/Afxl8+c7AXym38fd4fm8B8Cf9ftYPc9nE4DXAfih5flbAXwNgAB4PYB/7Pcxd3g+NwL4ar+PM8H5XAzgdc2flwE4aPi8DcQ18jyXTK8Pa/oRSqn9Sqkf9/s4suJ5PtMAqkqpp5RSpwF8GsDt3T+6VG4H8GDz5wcBvLWPx5KWz/sdPs/PA/h1EZEeHmMSg/T5iaWU2g3geccmtwP4WxX4NoBXiMjFvTm65DzOZ6AopY4qpb7b/PkEgP0A1kQ2G4hr5HkumWLQT08B+B8isk9EtvT7YDq0BsDPQr8fQpc/eB1YqZQ6CgRfGAAXWbZbKiLfEZFvi0jeCgY+7/fZbZRSdQAvArigJ0eXnO/n523NptbPi8ilvTm0rhik74uvXxOR74vI10Tk6n4fjK9mt9drAfxj5KmBu0aOcwEyvD7lTv54UInINwCsMjz1+0qpL3vuZoNS6oiIXATgH0TkQLNE3XMZnI+pBtm3aR2u80mwm8nm9bkcwA4ReVwp9WQ2R9gxn/c7V9ckhs+xfgXAQ0qpmoh8AEErxuauH1l3DNK18fFdBClc50TkVgBfAnBFn48plohMAPgCgH+tlHop+rThT3J7jWLOJdPrU8igr5R6Ywb7ONL8/+cisg1BE2dfgn4G53MIQLjmdQmAIx3uMzXX+YjIcRG5WCl1tNlc93PLPvT1eUpEdiEoQecl6Pu833qbQyJSBnAe8ttEG3s+SqnnQr8+AOCjPTiubsnV96VT4SCjlHpYRD4uIiuUUrnNYS8iowiC5N8ppb5o2GRgrlHcuWR9fdi8n4KIjIvIMv0zgJsBGEfGDojHAFwhIq8UkSUIBo7lbsR703YAdzd/vhtAW0uGiJwvIpXmzysAbADwRM+OMJ7P+x0+z7cD2KGao3pyKPZ8Iv2ptyHouxxU2wG8uzlC/PUAXtRdToNIRFbp8SIiMo0gLjzn/qv+aR7rXwPYr5T6j5bNBuIa+ZxL5ten36MX8/YPwL9AUEqsATgO4OvNx1cDeLj58+UIRih/H8CPEDSj9/3Y055P8/dbEYwcfTLn53MBgG8C+Enz/+XNx68F8FfNn28A8Hjz+jwO4L39Pm7DebS93wDuA3Bb8+elAD4HoApgL4DL+33MHZ7PHze/K98HsBPAVf0+Zse5PATgKICF5nfnvQA+AOADzecFwJ83z/VxOGb55OGfx/l8MHRtvg3ghn4fc8z5bETQVP8DAN9r/rt1EK+R57lken2YkY+IiKgg2LxPRERUEAz6REREBcGgT0REVBAM+kRERAXBoE9ERFQQDPpElEhzZbB/EpHlzd/Pb/7+SyLy9yLyCxH5ar+Pk4jaMegTUSJKqZ8B+AsA9zcfuh/AVqXU0wD+BMC7+nVsROTGoE9EafwnAK8XkX+NIMHI/wMASqlvAjjRzwMjIrtC5t4nos4opRZE5N8C+HsAN6tgSV0iyjnW9IkorTcjSO/6K/0+ECLyw6BPRImJyGsAvAnA6wH8TmRBHSLKKQZ9IkqkueLXXyBY+3sWweC9/9DfoyIiHwz6RJTU+wHMKqX+ofn7xwFcJSL/TES+hWB1wF8XkUMi8ht9O0oiasNV9oiIiAqCNX0iIqKCYNAnIiIqCAZ9IiKigmDQJyIiKggGfSIiooJg0CciIioIBn0iIqKCYNAnIiIqiP8fp2j/n4D/aH8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 6))\n",
    "plot_decision_boundary(nn, X_test, y_test)\n",
    "plt.title('NN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NN in Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use the data set [Pima Indians](https://www.kaggle.com/uciml/pima-indians-diabetes-database/data) with the following features:\n",
    "\n",
    "\n",
    "1. Number of times pregnant\n",
    "2. Plasma glucose concentration a 2 hours in an oral glucose tolerance test\n",
    "3. Diastolic blood pressure (mm Hg)\n",
    "4. Triceps skin fold thickness (mm)\n",
    "5. 2-Hour serum insulin (mu U/ml)\n",
    "6. Body mass index (weight in kg/(height in m)^2)\n",
    "7. Diabetes pedigree function\n",
    "8. Age (years)\n",
    "9. Class variable (0 or 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/diabetes.csv',\n",
    "                 sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train / Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = df.iloc[:, :8].values, df.iloc[:, 8].values\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, \n",
    "                                                    random_state=11)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create NN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(768, 8)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(12, input_dim=8, init='normal', activation='sigmoid'))\n",
    "model.add(Dense(8, init='normal', activation='sigmoid'))\n",
    "model.add(Dense(1, init='uniform', activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to define [error function](https://keras.io/objectives/), [optimizer](https://keras.io/optimizers/) and [quality metric](https://keras.io/metrics/):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile model\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'], )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(614, 8)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 491 samples, validate on 123 samples\n",
      "Epoch 1/150\n",
      "491/491 [==============================] - 0s 61us/step - loss: 0.6710 - acc: 0.6741 - val_loss: 0.6856 - val_acc: 0.5610\n",
      "Epoch 2/150\n",
      "491/491 [==============================] - 0s 79us/step - loss: 0.6494 - acc: 0.6741 - val_loss: 0.6888 - val_acc: 0.5610\n",
      "Epoch 3/150\n",
      "491/491 [==============================] - 0s 84us/step - loss: 0.6372 - acc: 0.6741 - val_loss: 0.6973 - val_acc: 0.5610\n",
      "Epoch 4/150\n",
      "491/491 [==============================] - 0s 63us/step - loss: 0.6305 - acc: 0.6741 - val_loss: 0.7043 - val_acc: 0.5610\n",
      "Epoch 5/150\n",
      "491/491 [==============================] - 0s 102us/step - loss: 0.6293 - acc: 0.6741 - val_loss: 0.7049 - val_acc: 0.5610\n",
      "Epoch 6/150\n",
      "491/491 [==============================] - 0s 94us/step - loss: 0.6280 - acc: 0.6741 - val_loss: 0.7051 - val_acc: 0.5610\n",
      "Epoch 7/150\n",
      "491/491 [==============================] - 0s 80us/step - loss: 0.6270 - acc: 0.6741 - val_loss: 0.7065 - val_acc: 0.5610\n",
      "Epoch 8/150\n",
      "491/491 [==============================] - 0s 78us/step - loss: 0.6265 - acc: 0.6741 - val_loss: 0.7062 - val_acc: 0.5610\n",
      "Epoch 9/150\n",
      "491/491 [==============================] - 0s 76us/step - loss: 0.6252 - acc: 0.6741 - val_loss: 0.7049 - val_acc: 0.5610\n",
      "Epoch 10/150\n",
      "491/491 [==============================] - 0s 91us/step - loss: 0.6241 - acc: 0.6741 - val_loss: 0.7116 - val_acc: 0.5610\n",
      "Epoch 11/150\n",
      "491/491 [==============================] - 0s 59us/step - loss: 0.6240 - acc: 0.6741 - val_loss: 0.7075 - val_acc: 0.5610\n",
      "Epoch 12/150\n",
      "491/491 [==============================] - 0s 65us/step - loss: 0.6229 - acc: 0.6741 - val_loss: 0.7037 - val_acc: 0.5610\n",
      "Epoch 13/150\n",
      "491/491 [==============================] - 0s 68us/step - loss: 0.6232 - acc: 0.6741 - val_loss: 0.7054 - val_acc: 0.5610\n",
      "Epoch 14/150\n",
      "491/491 [==============================] - 0s 68us/step - loss: 0.6213 - acc: 0.6741 - val_loss: 0.7004 - val_acc: 0.5610\n",
      "Epoch 15/150\n",
      "491/491 [==============================] - 0s 69us/step - loss: 0.6207 - acc: 0.6741 - val_loss: 0.6982 - val_acc: 0.5610\n",
      "Epoch 16/150\n",
      "491/491 [==============================] - 0s 63us/step - loss: 0.6199 - acc: 0.6741 - val_loss: 0.7022 - val_acc: 0.5610\n",
      "Epoch 17/150\n",
      "491/491 [==============================] - 0s 62us/step - loss: 0.6190 - acc: 0.6741 - val_loss: 0.6972 - val_acc: 0.5610\n",
      "Epoch 18/150\n",
      "491/491 [==============================] - 0s 70us/step - loss: 0.6188 - acc: 0.6741 - val_loss: 0.7051 - val_acc: 0.5610\n",
      "Epoch 19/150\n",
      "491/491 [==============================] - 0s 67us/step - loss: 0.6171 - acc: 0.6741 - val_loss: 0.6991 - val_acc: 0.5610\n",
      "Epoch 20/150\n",
      "491/491 [==============================] - 0s 72us/step - loss: 0.6156 - acc: 0.6741 - val_loss: 0.6973 - val_acc: 0.5610\n",
      "Epoch 21/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.6150 - acc: 0.6741 - val_loss: 0.6984 - val_acc: 0.5610\n",
      "Epoch 22/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.6141 - acc: 0.6741 - val_loss: 0.6956 - val_acc: 0.5610\n",
      "Epoch 23/150\n",
      "491/491 [==============================] - 0s 69us/step - loss: 0.6137 - acc: 0.6741 - val_loss: 0.6965 - val_acc: 0.5610\n",
      "Epoch 24/150\n",
      "491/491 [==============================] - 0s 72us/step - loss: 0.6118 - acc: 0.6741 - val_loss: 0.6927 - val_acc: 0.5610\n",
      "Epoch 25/150\n",
      "491/491 [==============================] - 0s 74us/step - loss: 0.6102 - acc: 0.6741 - val_loss: 0.6905 - val_acc: 0.5610\n",
      "Epoch 26/150\n",
      "491/491 [==============================] - 0s 75us/step - loss: 0.6082 - acc: 0.6741 - val_loss: 0.6941 - val_acc: 0.5610\n",
      "Epoch 27/150\n",
      "491/491 [==============================] - 0s 103us/step - loss: 0.6067 - acc: 0.6741 - val_loss: 0.6936 - val_acc: 0.5610\n",
      "Epoch 28/150\n",
      "491/491 [==============================] - 0s 127us/step - loss: 0.6065 - acc: 0.6741 - val_loss: 0.6844 - val_acc: 0.5610\n",
      "Epoch 29/150\n",
      "491/491 [==============================] - 0s 154us/step - loss: 0.6065 - acc: 0.6741 - val_loss: 0.6915 - val_acc: 0.5610\n",
      "Epoch 30/150\n",
      "491/491 [==============================] - 0s 226us/step - loss: 0.6051 - acc: 0.6741 - val_loss: 0.6785 - val_acc: 0.5610\n",
      "Epoch 31/150\n",
      "491/491 [==============================] - 0s 107us/step - loss: 0.6003 - acc: 0.6741 - val_loss: 0.6873 - val_acc: 0.5610\n",
      "Epoch 32/150\n",
      "491/491 [==============================] - 0s 60us/step - loss: 0.5986 - acc: 0.6741 - val_loss: 0.6774 - val_acc: 0.5610\n",
      "Epoch 33/150\n",
      "491/491 [==============================] - 0s 82us/step - loss: 0.5989 - acc: 0.6741 - val_loss: 0.6769 - val_acc: 0.5610\n",
      "Epoch 34/150\n",
      "491/491 [==============================] - 0s 74us/step - loss: 0.5952 - acc: 0.6741 - val_loss: 0.6727 - val_acc: 0.5610\n",
      "Epoch 35/150\n",
      "491/491 [==============================] - 0s 81us/step - loss: 0.5930 - acc: 0.6741 - val_loss: 0.6898 - val_acc: 0.5610\n",
      "Epoch 36/150\n",
      "491/491 [==============================] - 0s 71us/step - loss: 0.5933 - acc: 0.6741 - val_loss: 0.6755 - val_acc: 0.5610\n",
      "Epoch 37/150\n",
      "491/491 [==============================] - 0s 85us/step - loss: 0.5952 - acc: 0.6741 - val_loss: 0.6733 - val_acc: 0.5610\n",
      "Epoch 38/150\n",
      "491/491 [==============================] - 0s 81us/step - loss: 0.5901 - acc: 0.6741 - val_loss: 0.6814 - val_acc: 0.5610\n",
      "Epoch 39/150\n",
      "491/491 [==============================] - 0s 107us/step - loss: 0.5887 - acc: 0.6741 - val_loss: 0.6686 - val_acc: 0.5610\n",
      "Epoch 40/150\n",
      "491/491 [==============================] - 0s 88us/step - loss: 0.5857 - acc: 0.6741 - val_loss: 0.6577 - val_acc: 0.5610\n",
      "Epoch 41/150\n",
      "491/491 [==============================] - 0s 80us/step - loss: 0.5848 - acc: 0.6741 - val_loss: 0.6596 - val_acc: 0.5610\n",
      "Epoch 42/150\n",
      "491/491 [==============================] - 0s 90us/step - loss: 0.5831 - acc: 0.6741 - val_loss: 0.6575 - val_acc: 0.5610\n",
      "Epoch 43/150\n",
      "491/491 [==============================] - 0s 77us/step - loss: 0.5820 - acc: 0.6741 - val_loss: 0.6476 - val_acc: 0.5610\n",
      "Epoch 44/150\n",
      "491/491 [==============================] - 0s 79us/step - loss: 0.5864 - acc: 0.6741 - val_loss: 0.6469 - val_acc: 0.5610\n",
      "Epoch 45/150\n",
      "491/491 [==============================] - 0s 77us/step - loss: 0.5810 - acc: 0.6741 - val_loss: 0.6662 - val_acc: 0.5610\n",
      "Epoch 46/150\n",
      "491/491 [==============================] - 0s 86us/step - loss: 0.5793 - acc: 0.6741 - val_loss: 0.6440 - val_acc: 0.5610\n",
      "Epoch 47/150\n",
      "491/491 [==============================] - 0s 90us/step - loss: 0.5757 - acc: 0.6741 - val_loss: 0.6627 - val_acc: 0.5610\n",
      "Epoch 48/150\n",
      "491/491 [==============================] - 0s 78us/step - loss: 0.5749 - acc: 0.6741 - val_loss: 0.6524 - val_acc: 0.5610\n",
      "Epoch 49/150\n",
      "491/491 [==============================] - 0s 84us/step - loss: 0.5745 - acc: 0.6741 - val_loss: 0.6546 - val_acc: 0.5610\n",
      "Epoch 50/150\n",
      "491/491 [==============================] - 0s 61us/step - loss: 0.5740 - acc: 0.6741 - val_loss: 0.6520 - val_acc: 0.5610\n",
      "Epoch 51/150\n",
      "491/491 [==============================] - 0s 66us/step - loss: 0.5715 - acc: 0.6741 - val_loss: 0.6382 - val_acc: 0.5610\n",
      "Epoch 52/150\n",
      "491/491 [==============================] - 0s 75us/step - loss: 0.5741 - acc: 0.6741 - val_loss: 0.6447 - val_acc: 0.5610\n",
      "Epoch 53/150\n",
      "491/491 [==============================] - 0s 72us/step - loss: 0.5756 - acc: 0.6741 - val_loss: 0.6466 - val_acc: 0.5610\n",
      "Epoch 54/150\n",
      "491/491 [==============================] - 0s 71us/step - loss: 0.5702 - acc: 0.6741 - val_loss: 0.6538 - val_acc: 0.5610\n",
      "Epoch 55/150\n",
      "491/491 [==============================] - 0s 81us/step - loss: 0.5702 - acc: 0.6741 - val_loss: 0.6447 - val_acc: 0.5610\n",
      "Epoch 56/150\n",
      "491/491 [==============================] - 0s 76us/step - loss: 0.5735 - acc: 0.6741 - val_loss: 0.6228 - val_acc: 0.5610\n",
      "Epoch 57/150\n",
      "491/491 [==============================] - 0s 86us/step - loss: 0.5704 - acc: 0.6741 - val_loss: 0.6337 - val_acc: 0.5610\n",
      "Epoch 58/150\n",
      "491/491 [==============================] - 0s 82us/step - loss: 0.5665 - acc: 0.6741 - val_loss: 0.6332 - val_acc: 0.5610\n",
      "Epoch 59/150\n",
      "491/491 [==============================] - 0s 78us/step - loss: 0.5667 - acc: 0.6741 - val_loss: 0.6323 - val_acc: 0.5610\n",
      "Epoch 60/150\n",
      "491/491 [==============================] - 0s 65us/step - loss: 0.5657 - acc: 0.6741 - val_loss: 0.6414 - val_acc: 0.5610\n",
      "Epoch 61/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "491/491 [==============================] - 0s 69us/step - loss: 0.5645 - acc: 0.6741 - val_loss: 0.6478 - val_acc: 0.5610\n",
      "Epoch 62/150\n",
      "491/491 [==============================] - 0s 82us/step - loss: 0.5660 - acc: 0.6741 - val_loss: 0.6487 - val_acc: 0.5610\n",
      "Epoch 63/150\n",
      "491/491 [==============================] - 0s 58us/step - loss: 0.5625 - acc: 0.6741 - val_loss: 0.6502 - val_acc: 0.5610\n",
      "Epoch 64/150\n",
      "491/491 [==============================] - 0s 124us/step - loss: 0.5613 - acc: 0.6741 - val_loss: 0.6332 - val_acc: 0.5610\n",
      "Epoch 65/150\n",
      "491/491 [==============================] - 0s 67us/step - loss: 0.5606 - acc: 0.6741 - val_loss: 0.6342 - val_acc: 0.5610\n",
      "Epoch 66/150\n",
      "491/491 [==============================] - 0s 74us/step - loss: 0.5606 - acc: 0.6741 - val_loss: 0.6245 - val_acc: 0.5610\n",
      "Epoch 67/150\n",
      "491/491 [==============================] - 0s 81us/step - loss: 0.5596 - acc: 0.6741 - val_loss: 0.6239 - val_acc: 0.5610\n",
      "Epoch 68/150\n",
      "491/491 [==============================] - 0s 81us/step - loss: 0.5605 - acc: 0.6741 - val_loss: 0.6306 - val_acc: 0.5610\n",
      "Epoch 69/150\n",
      "491/491 [==============================] - 0s 62us/step - loss: 0.5606 - acc: 0.6741 - val_loss: 0.6375 - val_acc: 0.5610\n",
      "Epoch 70/150\n",
      "491/491 [==============================] - 0s 64us/step - loss: 0.5581 - acc: 0.6782 - val_loss: 0.6224 - val_acc: 0.6016\n",
      "Epoch 71/150\n",
      "491/491 [==============================] - 0s 60us/step - loss: 0.5580 - acc: 0.6986 - val_loss: 0.6301 - val_acc: 0.5854\n",
      "Epoch 72/150\n",
      "491/491 [==============================] - 0s 72us/step - loss: 0.5564 - acc: 0.7067 - val_loss: 0.6418 - val_acc: 0.6016\n",
      "Epoch 73/150\n",
      "491/491 [==============================] - 0s 77us/step - loss: 0.5595 - acc: 0.6965 - val_loss: 0.6364 - val_acc: 0.6016\n",
      "Epoch 74/150\n",
      "491/491 [==============================] - 0s 63us/step - loss: 0.5554 - acc: 0.6904 - val_loss: 0.6159 - val_acc: 0.6098\n",
      "Epoch 75/150\n",
      "491/491 [==============================] - 0s 70us/step - loss: 0.5541 - acc: 0.7026 - val_loss: 0.6152 - val_acc: 0.6098\n",
      "Epoch 76/150\n",
      "491/491 [==============================] - 0s 56us/step - loss: 0.5540 - acc: 0.7026 - val_loss: 0.6169 - val_acc: 0.6098\n",
      "Epoch 77/150\n",
      "491/491 [==============================] - 0s 57us/step - loss: 0.5540 - acc: 0.7149 - val_loss: 0.6250 - val_acc: 0.6098\n",
      "Epoch 78/150\n",
      "491/491 [==============================] - 0s 66us/step - loss: 0.5515 - acc: 0.7067 - val_loss: 0.6098 - val_acc: 0.6098\n",
      "Epoch 79/150\n",
      "491/491 [==============================] - 0s 94us/step - loss: 0.5553 - acc: 0.7047 - val_loss: 0.6010 - val_acc: 0.6260\n",
      "Epoch 80/150\n",
      "491/491 [==============================] - 0s 64us/step - loss: 0.5661 - acc: 0.6864 - val_loss: 0.6068 - val_acc: 0.6016\n",
      "Epoch 81/150\n",
      "491/491 [==============================] - 0s 62us/step - loss: 0.5584 - acc: 0.6762 - val_loss: 0.6195 - val_acc: 0.5610\n",
      "Epoch 82/150\n",
      "491/491 [==============================] - 0s 64us/step - loss: 0.5519 - acc: 0.6823 - val_loss: 0.6092 - val_acc: 0.6098\n",
      "Epoch 83/150\n",
      "491/491 [==============================] - 0s 59us/step - loss: 0.5507 - acc: 0.7047 - val_loss: 0.5975 - val_acc: 0.6179\n",
      "Epoch 84/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.5600 - acc: 0.6884 - val_loss: 0.6106 - val_acc: 0.6098\n",
      "Epoch 85/150\n",
      "491/491 [==============================] - 0s 66us/step - loss: 0.5508 - acc: 0.6945 - val_loss: 0.6126 - val_acc: 0.6098\n",
      "Epoch 86/150\n",
      "491/491 [==============================] - 0s 66us/step - loss: 0.5497 - acc: 0.7006 - val_loss: 0.6276 - val_acc: 0.6098\n",
      "Epoch 87/150\n",
      "491/491 [==============================] - 0s 61us/step - loss: 0.5553 - acc: 0.6904 - val_loss: 0.6149 - val_acc: 0.6098\n",
      "Epoch 88/150\n",
      "491/491 [==============================] - 0s 78us/step - loss: 0.5458 - acc: 0.7026 - val_loss: 0.6181 - val_acc: 0.6098\n",
      "Epoch 89/150\n",
      "491/491 [==============================] - 0s 108us/step - loss: 0.5455 - acc: 0.6986 - val_loss: 0.6089 - val_acc: 0.6098\n",
      "Epoch 90/150\n",
      "491/491 [==============================] - 0s 84us/step - loss: 0.5447 - acc: 0.7088 - val_loss: 0.6284 - val_acc: 0.6098\n",
      "Epoch 91/150\n",
      "491/491 [==============================] - 0s 85us/step - loss: 0.5460 - acc: 0.7108 - val_loss: 0.6006 - val_acc: 0.6179\n",
      "Epoch 92/150\n",
      "491/491 [==============================] - 0s 78us/step - loss: 0.5502 - acc: 0.7006 - val_loss: 0.6054 - val_acc: 0.6098\n",
      "Epoch 93/150\n",
      "491/491 [==============================] - 0s 80us/step - loss: 0.5450 - acc: 0.7149 - val_loss: 0.6150 - val_acc: 0.6098\n",
      "Epoch 94/150\n",
      "491/491 [==============================] - 0s 74us/step - loss: 0.5413 - acc: 0.7312 - val_loss: 0.6359 - val_acc: 0.6098\n",
      "Epoch 95/150\n",
      "491/491 [==============================] - 0s 71us/step - loss: 0.5462 - acc: 0.7312 - val_loss: 0.6122 - val_acc: 0.6098\n",
      "Epoch 96/150\n",
      "491/491 [==============================] - 0s 76us/step - loss: 0.5412 - acc: 0.7230 - val_loss: 0.6062 - val_acc: 0.6423\n",
      "Epoch 97/150\n",
      "491/491 [==============================] - 0s 87us/step - loss: 0.5421 - acc: 0.7149 - val_loss: 0.6054 - val_acc: 0.6341\n",
      "Epoch 98/150\n",
      "491/491 [==============================] - 0s 63us/step - loss: 0.5456 - acc: 0.7189 - val_loss: 0.6340 - val_acc: 0.6098\n",
      "Epoch 99/150\n",
      "491/491 [==============================] - 0s 65us/step - loss: 0.5425 - acc: 0.7128 - val_loss: 0.6043 - val_acc: 0.6260\n",
      "Epoch 100/150\n",
      "491/491 [==============================] - 0s 81us/step - loss: 0.5397 - acc: 0.7271 - val_loss: 0.6043 - val_acc: 0.6504\n",
      "Epoch 101/150\n",
      "491/491 [==============================] - 0s 79us/step - loss: 0.5390 - acc: 0.7413 - val_loss: 0.6237 - val_acc: 0.6423\n",
      "Epoch 102/150\n",
      "491/491 [==============================] - 0s 65us/step - loss: 0.5395 - acc: 0.7230 - val_loss: 0.6084 - val_acc: 0.6504\n",
      "Epoch 103/150\n",
      "491/491 [==============================] - 0s 61us/step - loss: 0.5356 - acc: 0.7291 - val_loss: 0.6043 - val_acc: 0.6585\n",
      "Epoch 104/150\n",
      "491/491 [==============================] - 0s 72us/step - loss: 0.5371 - acc: 0.7413 - val_loss: 0.6136 - val_acc: 0.6667\n",
      "Epoch 105/150\n",
      "491/491 [==============================] - 0s 70us/step - loss: 0.5372 - acc: 0.7556 - val_loss: 0.6441 - val_acc: 0.6504\n",
      "Epoch 106/150\n",
      "491/491 [==============================] - 0s 69us/step - loss: 0.5488 - acc: 0.7373 - val_loss: 0.6020 - val_acc: 0.6829\n",
      "Epoch 107/150\n",
      "491/491 [==============================] - 0s 83us/step - loss: 0.5395 - acc: 0.7332 - val_loss: 0.6046 - val_acc: 0.6585\n",
      "Epoch 108/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.5345 - acc: 0.7434 - val_loss: 0.6368 - val_acc: 0.6423\n",
      "Epoch 109/150\n",
      "491/491 [==============================] - 0s 61us/step - loss: 0.5388 - acc: 0.7312 - val_loss: 0.6168 - val_acc: 0.6341\n",
      "Epoch 110/150\n",
      "491/491 [==============================] - 0s 60us/step - loss: 0.5372 - acc: 0.7413 - val_loss: 0.6110 - val_acc: 0.6341\n",
      "Epoch 111/150\n",
      "491/491 [==============================] - 0s 76us/step - loss: 0.5349 - acc: 0.7475 - val_loss: 0.6094 - val_acc: 0.6260\n",
      "Epoch 112/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.5339 - acc: 0.7291 - val_loss: 0.6079 - val_acc: 0.6423\n",
      "Epoch 113/150\n",
      "491/491 [==============================] - 0s 86us/step - loss: 0.5335 - acc: 0.7352 - val_loss: 0.6047 - val_acc: 0.6504\n",
      "Epoch 114/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.5313 - acc: 0.7536 - val_loss: 0.6161 - val_acc: 0.6260\n",
      "Epoch 115/150\n",
      "491/491 [==============================] - 0s 70us/step - loss: 0.5316 - acc: 0.7475 - val_loss: 0.6119 - val_acc: 0.6667\n",
      "Epoch 116/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.5334 - acc: 0.7393 - val_loss: 0.6033 - val_acc: 0.6585\n",
      "Epoch 117/150\n",
      "491/491 [==============================] - 0s 83us/step - loss: 0.5313 - acc: 0.7332 - val_loss: 0.5934 - val_acc: 0.6829\n",
      "Epoch 118/150\n",
      "491/491 [==============================] - 0s 127us/step - loss: 0.5304 - acc: 0.7556 - val_loss: 0.5994 - val_acc: 0.6911\n",
      "Epoch 119/150\n",
      "491/491 [==============================] - 0s 97us/step - loss: 0.5370 - acc: 0.7434 - val_loss: 0.5866 - val_acc: 0.6911\n",
      "Epoch 120/150\n",
      "491/491 [==============================] - 0s 74us/step - loss: 0.5319 - acc: 0.7393 - val_loss: 0.5885 - val_acc: 0.6992\n",
      "Epoch 121/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "491/491 [==============================] - 0s 76us/step - loss: 0.5295 - acc: 0.7556 - val_loss: 0.5917 - val_acc: 0.7073\n",
      "Epoch 122/150\n",
      "491/491 [==============================] - 0s 89us/step - loss: 0.5252 - acc: 0.7576 - val_loss: 0.6217 - val_acc: 0.6667\n",
      "Epoch 123/150\n",
      "491/491 [==============================] - 0s 146us/step - loss: 0.5347 - acc: 0.7597 - val_loss: 0.6070 - val_acc: 0.6585\n",
      "Epoch 124/150\n",
      "491/491 [==============================] - 0s 147us/step - loss: 0.5302 - acc: 0.7373 - val_loss: 0.6034 - val_acc: 0.6504\n",
      "Epoch 125/150\n",
      "491/491 [==============================] - 0s 134us/step - loss: 0.5287 - acc: 0.7536 - val_loss: 0.5972 - val_acc: 0.6748\n",
      "Epoch 126/150\n",
      "491/491 [==============================] - 0s 131us/step - loss: 0.5245 - acc: 0.7454 - val_loss: 0.6024 - val_acc: 0.6585\n",
      "Epoch 127/150\n",
      "491/491 [==============================] - 0s 91us/step - loss: 0.5289 - acc: 0.7373 - val_loss: 0.5983 - val_acc: 0.7236\n",
      "Epoch 128/150\n",
      "491/491 [==============================] - 0s 82us/step - loss: 0.5231 - acc: 0.7495 - val_loss: 0.5955 - val_acc: 0.6748\n",
      "Epoch 129/150\n",
      "491/491 [==============================] - 0s 75us/step - loss: 0.5292 - acc: 0.7312 - val_loss: 0.6017 - val_acc: 0.6667\n",
      "Epoch 130/150\n",
      "491/491 [==============================] - 0s 74us/step - loss: 0.5246 - acc: 0.7454 - val_loss: 0.6038 - val_acc: 0.6585\n",
      "Epoch 131/150\n",
      "491/491 [==============================] - 0s 68us/step - loss: 0.5290 - acc: 0.7536 - val_loss: 0.5925 - val_acc: 0.6992\n",
      "Epoch 132/150\n",
      "491/491 [==============================] - 0s 71us/step - loss: 0.5205 - acc: 0.7434 - val_loss: 0.6115 - val_acc: 0.6667\n",
      "Epoch 133/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.5329 - acc: 0.7373 - val_loss: 0.6125 - val_acc: 0.6585\n",
      "Epoch 134/150\n",
      "491/491 [==============================] - 0s 86us/step - loss: 0.5231 - acc: 0.7434 - val_loss: 0.6210 - val_acc: 0.6341\n",
      "Epoch 135/150\n",
      "491/491 [==============================] - 0s 83us/step - loss: 0.5206 - acc: 0.7454 - val_loss: 0.5899 - val_acc: 0.7480\n",
      "Epoch 136/150\n",
      "491/491 [==============================] - 0s 79us/step - loss: 0.5237 - acc: 0.7312 - val_loss: 0.5975 - val_acc: 0.6992\n",
      "Epoch 137/150\n",
      "491/491 [==============================] - 0s 77us/step - loss: 0.5227 - acc: 0.7413 - val_loss: 0.6012 - val_acc: 0.6748\n",
      "Epoch 138/150\n",
      "491/491 [==============================] - 0s 79us/step - loss: 0.5213 - acc: 0.7475 - val_loss: 0.5895 - val_acc: 0.7236\n",
      "Epoch 139/150\n",
      "491/491 [==============================] - 0s 99us/step - loss: 0.5196 - acc: 0.7454 - val_loss: 0.6036 - val_acc: 0.6504\n",
      "Epoch 140/150\n",
      "491/491 [==============================] - 0s 125us/step - loss: 0.5153 - acc: 0.7556 - val_loss: 0.6106 - val_acc: 0.6423\n",
      "Epoch 141/150\n",
      "491/491 [==============================] - 0s 111us/step - loss: 0.5140 - acc: 0.7475 - val_loss: 0.5929 - val_acc: 0.7073\n",
      "Epoch 142/150\n",
      "491/491 [==============================] - 0s 77us/step - loss: 0.5144 - acc: 0.7556 - val_loss: 0.6069 - val_acc: 0.6504\n",
      "Epoch 143/150\n",
      "491/491 [==============================] - 0s 78us/step - loss: 0.5203 - acc: 0.7393 - val_loss: 0.6005 - val_acc: 0.6667\n",
      "Epoch 144/150\n",
      "491/491 [==============================] - 0s 106us/step - loss: 0.5157 - acc: 0.7515 - val_loss: 0.5897 - val_acc: 0.7236\n",
      "Epoch 145/150\n",
      "491/491 [==============================] - 0s 69us/step - loss: 0.5431 - acc: 0.7271 - val_loss: 0.5928 - val_acc: 0.6911\n",
      "Epoch 146/150\n",
      "491/491 [==============================] - 0s 64us/step - loss: 0.5264 - acc: 0.7454 - val_loss: 0.6095 - val_acc: 0.6911\n",
      "Epoch 147/150\n",
      "491/491 [==============================] - 0s 83us/step - loss: 0.5167 - acc: 0.7352 - val_loss: 0.5944 - val_acc: 0.7398\n",
      "Epoch 148/150\n",
      "491/491 [==============================] - 0s 90us/step - loss: 0.5140 - acc: 0.7556 - val_loss: 0.6043 - val_acc: 0.6748\n",
      "Epoch 149/150\n",
      "491/491 [==============================] - 0s 73us/step - loss: 0.5232 - acc: 0.7230 - val_loss: 0.5986 - val_acc: 0.6748\n",
      "Epoch 150/150\n",
      "491/491 [==============================] - 0s 62us/step - loss: 0.5108 - acc: 0.7495 - val_loss: 0.6154 - val_acc: 0.6667\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1c1a837e10>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the model\n",
    "model.fit(X_train, y_train, nb_epoch=150, batch_size=10, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "154/154 [==============================] - 0s 23us/step\n",
      "\n",
      "acc: 70.13%\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(X_test, y_test)\n",
    "print(\"\\n%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = model.predict(X_test).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "nav_menu": {},
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "217px",
    "width": "253px"
   },
   "navigate_menu": true,
   "number_sections": false,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}