NeuralNetwork.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Sepal_length  Sepal_width  Petal_length  Petal_width             Type\n",
      "69            5.6          2.5           3.9          1.1  Iris-versicolor\n",
      "136           6.3          3.4           5.6          2.4   Iris-virginica\n",
      "122           7.7          2.8           6.7          2.0   Iris-virginica\n",
      "79            5.7          2.6           3.5          1.0  Iris-versicolor\n",
      "134           6.1          2.6           5.6          1.4   Iris-virginica\n",
      "['Iris-versicolor' 'Iris-virginica']\n",
      "(100, 5)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XuYXFWZ7/HvDwgSwiVgIBMhEC88jEgUMCIODrYCThAVZg76wCDIqAdvKJwnOsMwM97nGM4ZvCAODAomKqIYRRHxgkCDDIrD/RYVDgYIxASIISQwaPA9f6zVUqlUd+2urqq9a/fv8zz1dFXtvave3r3qrdVrr4siAjMzG3yblR2AmZl1hxO6mVlNOKGbmdWEE7qZWU04oZuZ1YQTuplZTTih94GkOZJC0hZt9huW9I5+xdX03iHpBWW8t5l1x6RP6JKWSXpS0jpJKyV9SdI2bY45QdK1/Yqx28r84rDJpWhlZhyvd6ekoVG2DUla3q9YqmjSJ/TsDRGxDbAf8DLgn0uOx6znBrEyExEviojhIvvm3++QHodUKU7oDSLiQeAHwN6Stpd0nqQVkh6U9AlJm0t6IXAO8Ir8QVgDIOlwSTdLWivpAUkfmWg8kt4maamk30n6kaTdG7aFpHdJujtv/7wk5W2bSzpD0iOSfiPppJGaiaR/Bf4SOCvHf1bDWx7S6vWs1lyZqREn9AaSZgOvA24GFgMbgBcA+wKvBd4REUuBdwE/i4htImJ6Pnw9cDwwHTgceLekIycQy5HAacDfADsBPwUubNrt9aQP4UuANwN/lZ//n8BhwD6kD+qf4oiIf8qvdVKO/6QCr2c1V2ZlRtKrJd3e8Pgnkn7R8Pjakc9SY61b0lRJi3IF5C5S2R055ivAbsD3cqx/3/CWx0q6P1d4/mmcp6raImJS34BlwDpgDXAf8O/A7sBTwNSG/Y4Brsr3TwCubfO6nwE+ne/PAQLYos0xw6QvDUgfrrc3bNsMeALYPT8O4JUN2y8CTs33rwTe2bDtkMb3b3yfhn1GfT3f6nnLZf+QfH82cCfwceA7wH8A04CdgV+MlKdWZR8YAubmMvpiYCVwZN7WtuwDWwFPAjOALYDfAg8B2wJT87Znt4h5IalysmOO/w5geavfrymWL+TXfUn+nL+w7L9Ft261vTgwTkdGxE9GHkjaH5gCrGhoddgMeGC0F5D0clIB2xvYEngW8M0JxLQ78FlJZzS+DbAL6YsHUsEf8QQw0v75nKZYR427yWivZ/X1HUkbgMeA7wNfBH4NTI+IJ4H1kj4NnEhK8puIjdu0b5N0IfAq0hdDWxHx35JuAA4iJfLbSBWsA0kJ9+6IeLTFoW8G3hMRq4HVks4EPlTgLT+af7dbJd1KSuxLi8RadU7orT1AKkgzImJDi+2tpqj8GnAWcFguoJ8h1TgmEsO/RsQFHRy7Ati14fHspu2eYtNGVKUyczWppr883/8d6Uvhqfy4leaKy32j7NesthUXt6G3EBErgB8DZ0jaTtJmkp4v6VV5l5XArpK2bDhsW2B1Tub7A387wTDOAf5R0osAcrvmmwoeexFwsqRdJE0H/qFp+0rgeROMz+qpsTIzPd+2i4gX5e2jVWYuAWZHxPaksjveC+ojCf2gfP9qUkJ/FaMn9BVsXFnZrWn7pKu4OKGP7nhSbeMuUm1hCTArb7uS1N74W0mP5OfeA3xM0uOkf/sumsibR8TFwOnA1yWtJbUPHlbw8C+QvpBuI13gvYx0gffpvP2zwFH5YtKZE4nT6qXEysx1wJ7A/sAvIuJOUrPjy4FrRjnmIlKlZwdJuwLva9o++SouZTfi+9b7G+mL4L6y4/CtWjeaLho2PL89cDap+eMxUqXg6LxtS1Jb+2rgkfzcUaTmjseBS0lNj1/N2+ZQoENA3vdn5I4H+fESYOloMQNbA18mtbffBXyQjS+KHgHcn7d/oFUstOggMMg35V/KakTSVODVpJrWTOBbwM8j4pRSAzOznnJC7zNJ60bZdFhE/LRL77E1qd3xz0ldvr4PnBwRa7vx+mZWTU7oZtYX/ajMTHZO6GZmNdHXfugzZsyIOXPm9PMtx2X9+vVMmzat7DAqo6rn48Ybb3wkInYqO44ixirzVT2/o3G8vTVWvEXLfF8T+pw5c7jhhhv6+ZbjMjw8zNDQUNlhVEZVz4ekogNISjdWma/q+R2N4+2tseItWubdD93MrCac0M3MasIJ3cysJjw5VwFzTv3+uI9ZtvDwHkRiNn6dlF9wGR5ErqGbNZE0W9JVebWoOyWdnJ/fUdLleVWnyyXtUHasZo2c0M02tQFYEBEvBA4A3itpL+BU4IqI2AO4Ij82qwwndLMmEbEiIm7K9x8nLX6wC2myp8V5t8U0LO1nVgVuQzcbg6Q5pDVlrwdmRppelohYIWnnUY45kbTCDzNnzmR4eLjla69bt27Ubd20YG6rNVraa46tX/F2y2SM1wm9R3whavBJ2oY0U+UpEbG2YQWfMUXEucC5APPmzYvRBov0a+DLCZ2WxWOHNnpcp4E6VdSNeN3kYtaCpCmkZH5BRHw7P71S0qy8fRawqqz4zFpxQjdrolQVP4+0uMKnGjZdArw1338r8N1+x2Y2Fje5mG3qQOA44HZJt+TnTiMthHyRpLeTVsIpusarWV84oZs1iYhrGX2R44P7GYvZeDihV4xHpZpZp9yGbmZWE20TuqTzJa2SdEfDcx+R9KCkW/Ltdb0N08zM2ilSQ18EzG/x/KcjYp98u6y7YZmZ2Xi1TegRcQ2wug+xmJnZBEzkouhJko4HbiBNZPS7VjsVHQZdBaMNve106HS/9OqcDtrQabPJrtOEfjbwcSDyzzOAt7Xasegw6CoYbehtp0On+6V5iHa3DNrQabPJrqNeLhGxMiKejog/Al8A9u9uWGZmNl4dJfSR+SyyvwbuGG1fMzPrj7ZNLpIuBIaAGZKWAx8GhiTtQ2pyWQa8s4cxmplZAW0TekQc0+Lp83oQi3XIo0vNDDxS1MysNpzQzcxqwgndzKwmnNDNzGrCCd3MrCac0M3MasIJ3cysJpzQzcxqwgndzKwmJt2aomONqlwwd0PlZ1Y0MxuNa+hmZjXhhG5mVhNO6GZmNeGEbmZWE07oZmY14YRuZlYTbRO6pPMlrZJ0R8NzO0q6XNLd+ecOvQ3TzMzaKdIPfRFwFvDlhudOBa6IiIWSTs2P/6H74ZlZWZrHbBQZp9HJSlhecat72tbQI+IaYHXT00cAi/P9xcCRXY7LzMzGqdORojMjYgVARKyQtPNoO0o6ETgRYObMmQwPD3f4lt2xYO6GUbfNnDr29jr53AXfbbvPzKkb7zd3l+17GZKZTVDPh/5HxLnAuQDz5s2LoaGhXr/lmMb6l3HB3A2ccfukmw1hVM3nY9mxQ+UFY2ZtddrLZaWkWQD556ruhWRmZp3otDp6CfBWYGH+2f7/dzOzLilyIbX5Im6nF1IH6aJtkW6LFwI/A/aUtFzS20mJ/FBJdwOH5sdmteHuujaIivRyOSYiZkXElIjYNSLOi4hHI+LgiNgj/2zuBWM26BYB85ueG+muuwdwRX5sVhkeKWrWgrvr2iBylw6z4gp11y3aVXfV6scKdR9t1EnX0W51xS3SrbeTbsm96ircHG+nXaY7ia+T91q3bt2Eu3U7oZt1WdGuup+74Lvj7ibbSdfRbq3CVaRbb5nxNetWt9tO4uvkvYaHh5lot243uZgV5+66VmlO6GbFjXTXBXfXtQpyQjdrwd11bRC5Dd2shYg4ZpRNB/c1ELNxcA3dzKwmnNDNzGrCCd3MrCac0M3MasIJ3cysJpzQzcxqwgndzKwmnNDNzGrCCd3MrCYmNFJU0jLgceBpYENEzOtGUGZmNn7dGPr/6oh4pAuvY2ZmE+C5XMysazpZULlf+hlbJ++1aP60Cb/vRBN6AD+WFMB/5In9N1J09ZZ+GWv1kSIrskwm3Vrxxcz6Y6IJ/cCIeCgvxXW5pF/mtRj/pOjqLf0y1uojRVZkmUy6teKLmfXHhHq5RMRD+ecq4GJg/24EZWZm49dxdVTSNGCziHg8338t8LGuRWZmm6hyG7WVbyLtCzOBiyWNvM7XIuKHXYnKzMzGreOEHhH3Ai/pYizj5trKYOjk77Rs4eE9iMSs3jxS1MysJpzQzcxqwgndzKwmnNDNzGqiMqNofIHTzGxiXEM3M6sJJ3Qzs5pwQjczqwkndDOzmqjMRVGrPl+4Nqs219DNzGrCCd3MrCac0M3MasIJ3cysJpzQzcxqwgndzKwmJpTQJc2X9CtJ90g6tVtBmVWZy71VVccJXdLmwOeBw4C9gGMk7dWtwMyqyOXeqmwiNfT9gXsi4t6I+D3wdeCI7oRlVlku91ZZExkpugvwQMPj5cDLm3eSdCJwYn64TtKvJvCePfV+mAE8UnYcVVHm+dDpY27evU9htNK23I+jzA9UeRu0z8egxfvq08eMt1CZn0hCV4vnYpMnIs4Fzp3A+/SNpBsiYl7ZcVSFz0dLbct90TI/aOfX8fZWN+KdSJPLcmB2w+NdgYcmEozZAHC5t8qaSEL/L2APSc+VtCVwNHBJd8IyqyyXe6usjptcImKDpJOAHwGbA+dHxJ1di6wcA9E01Ec+H026XO4H7fw63t6acLyK2KTZ28zMBpBHipqZ1YQTuplZTTihN5C0uaSbJV1adixlkzRd0hJJv5S0VNIryo5pEEk6X9IqSXeMsl2SzszTCNwmab9+x9gUT7t4hyQ9JumWfPtQv2Nsime2pKtyGb1T0skt9qnMOS4Yb8fn2EvQbexkYCmwXdmBVMBngR9GxFG5N8fWZQc0oBYBZwFfHmX7YcAe+fZy4GxaDNDro0WMHS/ATyPi9f0Jp60NwIKIuEnStsCNki6PiLsa9qnSOS4SL3R4jl1DzyTtChwOfLHsWMomaTvgIOA8gIj4fUSsKTeqwRQR1wCrx9jlCODLkfwcmC5pVn+i21SBeCslIlZExE35/uOkCtkuTbtV5hwXjLdjTujP+Azw98Afyw6kAp4HPAx8KTdBfVHStLKDqqlWUwl07QPeI6+QdKukH0h6UdnBjJA0B9gXuL5pUyXP8RjxQofn2AkdkPR6YFVE3Fh2LBWxBbAfcHZE7AusBzxNbG8UmkKjQm4Cdo+IlwCfA75TcjwASNoG+BZwSkSsbd7c4pBSz3GbeDs+x07oyYHAGyUtI82e9xpJX+3FG0laJumQNvt8pFfv346kYVJzy/KIGKk5LCEleOu+gZpKICLWRsS6fP8yYIqkGUWOLVL2i5J0jqR/yfenkJLjBRHx7fxcSHpB3r3VOf5Ot2IZr1bxNprIOZ7UCV3SKyVdB7yHdNHvQeDDwJUR8ZZSg+uDMb441gIPSNozPz4YaL5oY91xCXB87olxAPBYRKzo9ZuOlP3cm2K1pP+U9LICx/2ZJOX7+5NyyKO9jrdZRLwrIj6eYzkPWBoRnxpl9+cDH288x8DT/Yq1UZF4J3KOJ20vl3zh71Lg3cBFwJbAX5K+vQ3eB1yQe7jcC/xdyfEMJEkXAkPADEnLSRWGKQARcQ5wGfA64B7gCfpwnsco+08ViPco4N2SNgBPAkdHucPNDwSOA26XdEt+7rT88xjg46Qa+nQ2PsdL+hzniNHi3Q26cI4jYlLegHnAmjG2v410Bfp3pHk7dm/YFsD7SYnuEeD/Apvlbc8HriR9oz4CXABMbzh2GXBIm9g+Any14fEBwHXAGuBWYKhh2zCp0P4n8DjwY2BGw/bjgftyPP8y8v7AfOD3wB+AdcCtRV7Pt8G/VbXsA1uREtiM/PifSd38tsuPPwF8Jt9fBHyi4dgPAitIzVVvy3G+gDQv/R9yWV8HfK8hlg8At5Fq7N8Atir7bzPR22Rucvk18LSkxZIOk7TDyAZJR5K+Nf8G2An4KXBh0/F/Tfpg7EfqFvW2kcOBTwLPAV5Iarv7SKdBStoF+D6pMO9IKoTfkrRTw25/S6p17EyqbX0gH7sX8O/AscAsYHvy1f2I+CHwv4FvRMQ2kS7AjPl6VhuVLPsR8d+k2SxflZ86iFQZObDh8dXNx0maTyqjh5L6mv+pbTzS3PQXAP8nl/M3NBz6ZlLF5rnAi4ETisZaVZM2oUe6svxK0jf5F4CHJV0iaSbwTuCTEbE0IjaQEt8+khpXDTk9IlZHxP2kLo/H5Ne9JyIuj4inIuJh4FM8U0A78Rbgsoi4LCL+GBGXAzeQ/k0f8aWI+HVEPEn6F3qf/PxRpBrJtZGWS/sQxa7uj/Z6VgMVL/tXA6+StAUpyZ6ZH28FvIz0BdPszaQye0dErKf4l8iZEfFQRKwGvkcNyvmkTegAudCeEBG7AnuTahafIS339FlJayStIQ20EBv3XW3s13pfPhZJO0v6uqQHJa0FvkpaCqtTuwNvGoklx/NKUo17xG8b7j8BbJPvP6cxzoh4gmIXV0Z7PauJCpf9q0lt+PsBtwOXk74UDiCt5dpqibaNynmOqYjalfNJndAbRcQvSe1ye5MKxzsjYnrDbWpEXNdwSGM3qN14pqvZJ0k1nxdHxHakGnarfrBFPQB8pSmWaRGxsMCxK2i4yCtpKvDshu1V7u9sfVKxsn8dsCepWefqSEPidyON4t6kuSVb0SKmjX7FccYwsCZtQpf055IW5CH/SJpN+tfx58A5wD+OjNCStL2kNzW9xAcl7ZCPO5l0UQVgW9LFlzW5/fuDEwz1q8AbJP2V0uRhWylN3lOkN86SfOxf5N4qH2XjD9hKYI6kSVsOJqMql/38X+SNwHt5JoFfR2oKGi2hXwScIGkvSVuTeuY0Wkka/Vx7k/mD/Dhpgp7rJa0nFeY7SBPnXAycDnw9/+t4B2mCn0bfJRW8W0gXLc/Lz3+U9O/iY/n5TQYOjEdEPEC68HQaaTj+A6QPStu/XaSVdN5HGiy1gvQ7rwKeyrt8M/98VNJNE4nTBkrVy/7VpK6Sv2h4vC1wTaudI+IHpOaiK0ldE69s2uU8YK/cjFSJka294hWLOiApgD0i4p6yYxkPpeHGa0ix/6bseGzwDGrZnywmcw19UpD0BklbK02u9W+kC03Lyo3KzHrBCb0kSrOorWtxO6390eNyBOmi1UOkPrplj+yzSa6PZX/ScZOLWZPc5/ka4Fmk6TGWRMSHJT2XdD1iR9KMeMfl/v1mleAautmmngJek0fP7gPMz5M6nQ58OiL2IA2Lf3uJMZptoq+Tc82YMSPmzJlTaN/169czbdrgrKngeHurMd4bb7zxkYjYqc0hHctNUuvywyn5FsBrSNMiACwmjUg8e6zXGqvMV/FvULWYqhYPlBNT4TLfz4ljXvrSl0ZRV111VeF9q8Dx9lZjvMAN0fsJrDYndctbR6qZzyCNVBzZPhu4o93rjFXmq/g3qFpMVYsnopyYipb5STt9rtlYIuJp0hwm04GLSZNNbbJbq2MlnUia5Y+ZM2cyPDzc8j3WrVs36rayVC2mqsUD1YxphBO62RgiYo3SKk4HkBYX3iLSpFWjriwUaYa/cwHmzZsXQ0NDLV97eHiY0baVpWoxVS0eqGZMI3xR1KyJpJ1yzXxk/ptDSPODX0WawRLgraQRk2aVMelq6HNO/f64j1m28PAeRGIVNgtYLGlzUqXnooi4VNJdpCHxnwBu5pkh7zYBY30mF8zdwAkttvsz2dqkS+hm7UTEbcC+LZ6/F9i//xGZFeMmFzOzmnBCNzOrCSd0M7OacEI3M6sJJ3Qzs5pwQjczq4m2CV3SbElXSVoq6U5JJ+fnd5R0uaS7888deh+umZmNpkgNfQNprcEXkoY/v1fSXsCpwBWRphK9Ij82M7OSFFloeEVE3JTvP04aAr0LaSWcxXm3xcCRvQrSzMzaG1cbuqQ5pBF01wMzI2IFpKQP7Nzt4MzMrLjCQ//zivHfAk6JiLWSih5XaCrRZr2aonLB3A3jPqZIHFWeUrMVx2tWP4USuqQppGR+QUR8Oz+9UtKsiFghaRawqtWxRacSbdarKSpbTfTTzrJj28dR5Sk1W3G8ZvVTpJeLSLPKLY2ITzVsuoQ0hSh4KlEzs9IVqaEfCBwH3C7plvzcacBC4CJJbwfuB97UmxDNzKyItgk9Iq4FRmswP7i74ZiZWac8UtTMrCac0M3MasIJ3cysJpzQzcxqwgndzKwmnNDNzGrCCd3MrCac0M3MasIJ3ayJF3WxQeWEbrYpL+piA8kJ3ayJF3WxQVV4PnSzyWisRV0ktVzUpegaAFWc472MmMZao2Dm1NbbyzxvVfy7jXBCNxtFp4u6FF0DoIpzvJcR01hrFCyYu4Ezbt80TRVZo6BXqvh3G+EmF7MWxlrUJW8fdVEXs7K4hl7AnAKrHC2Yu2GjmsayhYf3MqSNFImv2aL503oQST0UWNRlIV7UxSrICd1sU17UxQaSE7pZEy/qYoPKbehmZjXhGnrFdNIebmYGrqGbmdWGE7qZWU04oZuZ1YTb0HvEbeFm1m+uoZuZ1YQTuplZTTihm5nVhNvQzWzgdHKNqp/zK5XFNXQzs5pwQjczqwk3uZhZS+56O3ja1tAlnS9plaQ7Gp7z6udmZhVTpMllETC/6Tmvfm5mVjFtE3pEXAOsbnraq5+bmVVMpxdFN1r9HGi5+rmZmfVPzy+KSjoROBFg5syZDA8PFzpu3bp1hfcdjwVzN3T9NQFmTu3da/dCr85vrwxavGZl6DShr5Q0KyJWtFv9PCLOBc4FmDdvXgwNDRV6g+HhYYruOx4n9OjK/YK5Gzjj9sHpNLRo/rSenN9e6VV5MKuTTptcRlY/B69+bmZWCUW6LV4I/AzYU9LyvOL5QuBQSXcDh+bHZmZWorZtBBFxzCibvPq5mVmFDE6jr1kfSTofeD2wKiL2zs/tCHwDmAMsA94cEb8rK8aixjPic8HcDT27zmS957lczFpbhAfU2YBxQjdrwQPqbBC5ycWsuI0G1ElqOaCu6NiLfvWtH8/4iKqNp+hmPN0611UeE+GEbtZlRcde9Ktv/XjaxKs2nqKb8Sw7dqgrr1PlMRFucjErbmUeSEe7AXVmZXBCNyvOA+qs0pzQzVrwgDobRNVpLDOrEA+os0HkGrqZWU24hj5J3f7gY+MeEbhs4eEdvVcna1N2+l5mk5lr6GZmNeEauplNCp38pwiD9d+ia+hmZjVRmRp687dnkVnfBumb08ys1yqT0M2svU6bDWxycJOLmVlNOKGbmdWEE7qZWU04oZuZ1YQTuplZTTihm5nVhBO6mVlNOKGbmdWEE7qZWU04oZuZ1cRAD/33MGgzs2e4hm5mVhMDXUM3M6uislbpcg3dzKwmXEO3wnzNwqzaJpTQJc0HPgtsDnwxIhZ2JSqzCutWue9koW7rv04W3ylLx00ukjYHPg8cBuwFHCNpr24FZlZFLvdWZRNpQ98fuCci7o2I3wNfB47oTlhmleVyb5WliOjsQOkoYH5EvCM/Pg54eUSc1LTficCJ+eGewK8KvsUM4JGOgiuH4+2txnh3j4idygiiSLkfR5mv4t+gajFVLR4oJ6ZCZX4ibehq8dwm3w4RcS5w7rhfXLohIuZ1ElgZHG9vVSjetuW+aJmv0O/0J1WLqWrxQDVjGjGRJpflwOyGx7sCD00sHLPKc7m3yppIQv8vYA9Jz5W0JXA0cEl3wjKrLJd7q6yOm1wiYoOkk4AfkbpvnR8Rd3Ytsg6aaUrmeHurEvF2udxX4ndqUrWYqhYPVDMmYAIXRc3MrFo89N/MrCac0M3MaqKSCV3SMkm3S7pF0g1lx9OOpOmSlkj6paSlkl5RdkyjkbRnPq8jt7WSTik7rrFI+l+S7pR0h6QLJW1VdkxFSZot6apcLu6UdHKLfSTpTEn3SLpN0n4lxzMk6bGGMvKhXsWT328rSb+QdGuO6aMt9nmWpG/kc3S9pDkViOkESQ83nKd39DKmQiKicjdgGTCj7DjGEe9i4B35/pbA9LJjKhj35sBvSYMWSo9nlBh3AX4DTM2PLwJOKDuuccQ/C9gv398W+DWwV9M+rwN+QOrjfgBwfcnxDAGX9vEcCdgm358CXA8c0LTPe4Bz8v2jgW9UIKYTgLPKLmONt0rW0AeJpO2Ag4DzACLi9xGxptyoCjsY+H8RcV/ZgbSxBTBV0hbA1gxQv++IWBERN+X7jwNLSV9SjY4AvhzJz4HpkmaVGE9f5d97XX44Jd+ae2scQao4ASwBDpbUapBXP2OqnKom9AB+LOnGPIy6yp4HPAx8SdLNkr4oaVrZQRV0NHBh2UGMJSIeBP4NuB9YATwWET8uN6rO5GaCfUm1vUa7AA80PF5OH5LsGPEAvCI3N/xA0ov6EMvmkm4BVgGXR8So5ygiNgCPAc8uOSaA/5GbyZZImt1ie19VNaEfGBH7kWa0e6+kg8oOaAxbAPsBZ0fEvsB64NRyQ2ovD4p5I/DNsmMZi6QdSLWz5wLPAaZJeku5UY2fpG2AbwGnRMTa5s0tDulpbbBNPDeRmuFeAnwO+E4vYwGIiKcjYh/SyNv9Je3dHHKrw0qO6XvAnIh4MfATnvkPojSVTOgR8VD+uQq4mDTDXVUtB5Y3fHsvISX4qjsMuCkiVpYdSBuHAL+JiIcj4g/At4G/KDmmcZE0hZQ8L4iIb7fYpa/TCbSLJyLWjjQ3RMRlwBRJM3oVT9N7rwGGgflNm/50jnLT2/bA6jJjiohHI+Kp/PALwEv7Ec9YKpfQJU2TtO3IfeC1wB3lRjW6iPgt8ICkPfNTBwN3lRhSUcdQ8eaW7H7gAElb5zbTg0ntvgMhx3wesDQiPjXKbpcAx+feLgeQmpVWlBWPpD8baZ+WtD8pTzzai3jye+wkaXq+P5X0Jf7Lpt0uAd6a7x8FXBn5ymRZMTVd53gjFSiXVVyCbiZwcS5PWwBfi4gflhtSW+8DLsjNGPcCf1dyPGOStDVwKPDOsmNpJyKul7SE1AywAbiZCg+9buFA4Djg9tweC3AasBtARJwDXEbq6XIP8AS9LT9F4jkKeLekDcCTwNG9TJ6knjeLlRYP2Qy4KCIulfQx4IaIuIT0JfQVSfeQauZH9zCeojG9X9IbSeVyNanXS6k89N/MrCYq1+RiZmadcUI3M6sJJ3Qzs5pwQjfaswJJAAAAGUlEQVQzqwkndDOzmnBCNzOrCSd0M7Oa+P+1teEw+70ATAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1650cb93208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "# Read in dataset\n",
    "iris = pandas.read_csv(\"Iris.csv\")\n",
    "iris = iris[(iris.Type == 'Iris-versicolor') | (iris.Type == 'Iris-virginica')]\n",
    "\n",
    "# shuffle rows\n",
    "shuffled_rows = np.random.permutation(iris.index)\n",
    "iris = iris.loc[shuffled_rows,:]\n",
    "\n",
    "print(iris.head())\n",
    "\n",
    "# There are 3 species\n",
    "print(iris.Type.unique())\n",
    "iris.hist()\n",
    "print(iris.shape)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.  5.6 2.5 3.9 1.1]\n",
      "(5,)\n",
      "y (100,)\n"
     ]
    }
   ],
   "source": [
    "iris[\"ones\"] = np.ones(iris.shape[0])\n",
    "X = iris[['ones', 'Sepal_length', 'Sepal_width', 'Petal_length', 'Petal_width']].values\n",
    "y = (iris.Type == 'Iris-versicolor').values.astype(int)\n",
    "\n",
    "# The first observation\n",
    "x0 = X[0]\n",
    "print(X[0])\n",
    "print(X[0].shape)\n",
    "print('y',y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use a class for this model, it's good practice and condenses the code\n",
    "class NNet3:\n",
    "    def __init__(self, learning_rate=0.5, maxepochs=1e4, convergence_thres=1e-5, hidden_layer=4):\n",
    "        self.learning_rate = learning_rate\n",
    "        self.maxepochs = int(maxepochs)\n",
    "        self.convergence_thres = 1e-5\n",
    "        self.hidden_layer = int(hidden_layer)\n",
    "        \n",
    "    def _multiplecost(self, X, y):\n",
    "        # feed through network\n",
    "        l1, l2 = self._feedforward(X) \n",
    "        # compute error\n",
    "        inner = y * np.log(l2) + (1-y) * np.log(1-l2)\n",
    "        # negative of average error\n",
    "        return -np.mean(inner)\n",
    "    \n",
    "    def _feedforward(self, X):\n",
    "        # feedforward to the first layer\n",
    "        l1 = self.sigmoid_activation(X.T, self.theta0).T\n",
    "        # add a column of ones for bias term\n",
    "        l1 = np.column_stack([np.ones(l1.shape[0]), l1])\n",
    "        # activation units are then inputted to the output layer\n",
    "        l2 = self.sigmoid_activation(l1.T, self.theta1)\n",
    "        return l1, l2\n",
    "    \n",
    "    def predict(self, X):\n",
    "        _, y = self._feedforward(X)\n",
    "        return y\n",
    "    \n",
    "    def sigmoid_activation(self,x,theta):\n",
    "        x = np.asarray(x)\n",
    "        theta = np.asarray(theta)\n",
    "        return 1 /(1 + np.exp(-np.dot(theta.T,x)))\n",
    "    \n",
    "    def learn(self, X, y):\n",
    "        nobs, ncols = X.shape\n",
    "        self.theta0 = np.random.normal(0,0.01,size=(ncols,self.hidden_layer))\n",
    "        self.theta1 = np.random.normal(0,0.01,size=(self.hidden_layer+1,1))\n",
    "        \n",
    "        self.costs = []\n",
    "        cost = self._multiplecost(X, y)\n",
    "        self.costs.append(cost)\n",
    "        costprev = cost + self.convergence_thres+1  # set an inital costprev to past while loop\n",
    "        counter = 0  # intialize a counter\n",
    "\n",
    "        # Loop through until convergence\n",
    "        for counter in range(self.maxepochs):\n",
    "            # feedforward through network\n",
    "            l1, l2 = self._feedforward(X)\n",
    "\n",
    "            # Start Backpropagation\n",
    "            # Compute gradients\n",
    "            l2_delta = (y-l2) * l2 * (1-l2)\n",
    "            l1_delta = l2_delta.T.dot(self.theta1.T) * l1 * (1-l1)\n",
    "\n",
    "            # Update parameters by averaging gradients and multiplying by the learning rate\n",
    "            self.theta1 += l1.T.dot(l2_delta.T) / nobs * self.learning_rate\n",
    "            self.theta0 += X.T.dot(l1_delta)[:,1:] / nobs * self.learning_rate\n",
    "            \n",
    "            # Store costs and check for convergence\n",
    "            counter += 1  # Count\n",
    "            costprev = cost  # Store prev cost\n",
    "            cost = self._multiplecost(X, y)  # get next cost\n",
    "            self.costs.append(cost)\n",
    "            if np.abs(costprev-cost) < self.convergence_thres and counter > 500:\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEWCAYAAACT7WsrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8XXWd//HXJzf7njRJt6R7SylQWkjLjoAgBUdwQQXGn+IyOCo4yowOqMPPwZnfuI/ODI6io7ggiyiKihRGqEhlaUpLoRuE0iUtbdMlXZJm//z+OCflNmRvcs+9yfv5eNzHPfec7z33c0/uve+c7XvM3REREelJWtQFiIhI8lJIiIhIrxQSIiLSK4WEiIj0SiEhIiK9UkiIiEivFBIig2CBH5nZfjN7doDPudPM/mWkaxsLzGytmV0QdR1jiUJiFDGza82sxswOm9lrZvYHMzs36rpGmXOBS4BKd1/cfaKZXWdmT45kAWZ2qZk9YWaHzKzezP5kZlcc5zyXmdlH+pg+zcw8/Gx13Z4/ntccQE1vCFd3P8ndl43k68qxFBKjhJndBHwL+H/AeGAK8B3gyijrimdm6VHXMAymApvdvTGKFzezq4BfAD8BKgn+1rcCb0tQCcXunh/eTk3Qa0qU3F23FL8BRcBh4N19tMkiCJEd4e1bQFY47QKgDvh7YDfwGvDBcNqZwE4gFjevdwBrwuE04GbgFWAvcB9QGk6bBjjwYWAr8EQ4/v3AlrD9PwGbgYsHMb8PhPPbA3w+rq4Y8LnwuYeAlUBVOG0u8CiwD9gIvKePZTUJeDBsWwv8TTj+w0Az0BEu73/u9rwTu01vCMffCdwO/D6s6xlgZtzzBlQbYOH7/kwftacBXwiX726CMCkKp2UDPwuXawOwgiBk/jWsuTms+796mG/Xsk/vYdoXgZ/11hZYBnwJWB6+/0eAsrj25wJ/CWvaBlwHXA+0Aa1hTb8N28Z/Vob0mdZtkL8vUReg2zD8EWEJ0N7TFziuzW3A00AFUB5+Kb8UTrsgfP5tQAZwOdAElITTXwEuiZvXL4Cbw+FPhfOtDL+03wPuDqd1/Vj8BMgDcoB54Zf+XCAT+Hr4Y3DxIOb3/XBepwItwInh9M8ALwAnEPygngqMC197G/BBIB04jSBgTuplWf2JYC0sG1gA1ANvDqddBzzZx3J+w3SCkNgHLA5f/y7gnnDagGsjCBMHpvfx+h8iCLYZQD7wK+Cn4bSPAr8FcgkC9XSgMJy2DPhIH/PtWvZDDYlXgDnh320Z8OVw2hSC4LiG4LM3DlgQt9z+pdtrbY77rAz5M63bIH5foi5At2H4I8JfAzv7afMKcHnc40sJNpt0faGOxP8AEPz3dWY4/C/AD8PhAqARmBo+Xt/1Axo+nkjwo58e92MxI276rYQ/+uHjXIL/Fi8exPwq46Y/C1wdDm8Eruzhvb8X+HO3cd8D/m8PbasI/qsuiBv3b8Cd4fB1DC0kfhD3+HJgwxBqOyd8/9l9vP4fgY/HPT4hbvl9KPwhnd/D85YxsJBoiLv9Qzjti/QfEl+Im/5x4OFw+BbggV5e8076Dokhf6Z1G/htNGwjlmDzQZmZpbt7ey9tJhFsguiyJRx3dB7dnttE8J8owM+Bv5jZx4B3As+5e9e8pgIPmFln3HM7CDZjdNnWrY6jj929ycz2xk0fyPx29lJnFcEPR3dTgTPMrCFuXDrw0x7aTgL2ufuhuHFbgOoe2g5GbzUPprau5TQReLWX1+np75xOsPx+SrCM7jGzYoJNT59397ZBvI+yPj5jfRns32wgjuczLQOkHdejw1ME25Pf3kebHQQ/SF2mhOP65e7rCL6AlwHXEoRGl23AZe5eHHfLdvft8bOIG36NYFMSAGaWQ7CJYTDz6802YGYv4//UbZ757v6xHtruAErNrCBu3BRgIK8Px77XgRhMbRvD9u/qY349/Z3bgV3u3ubu/+zu84Czgb8i2D80lLrjNRKsEXaZMIjn9vY3g/5rGvJnWgZOITEKuPsBgs04t5vZ280s18wyzOwyM/tq2Oxu4AtmVm5mZWH7nw3iZX4OfBI4n2CfRJfvAv9qZlMBwvn3dUTV/cDbzOxsM8sE/plg/8FQ5xfvB8CXzGx2eD7DfDMbB/wOmGNm/ydcLhlmtsjMTuw+A3ffRrBJ5t/MLNvM5hPssL5rgDXsAirD9zYQg6nNgZuAfzKzD5pZoZmlmdm5ZnZH2Oxu4NNmNt3M8gmOdrvX3dvN7EIzO8XMYsBBgs1QHXF1zxhgzd2tBs43sylmVkSwCWmg7gIuNrP3mFm6mY0zswUDrOl4P9MyAAqJUcLdv0nwA/IFgh2t24AbgF+HTf4FqAHWEOzcfS4cN1B3E2znfczd98SN/zbBkUCPmNkhgh2JZ/RR51rgRuAegrWKQwTbiluGMr9uvklwNNQjBD+C/wPkhJuO3gJcTfCf5k7gKwQ7xntyDcF29R3AAwT7Bx4dYA2PAWuBnWa2p7/Gg63N3e8n2I/xobD9LoK/42/CJj8k2Kz0BMEmqWaC5Q3Bf/j3Eyyb9QQ76Lt+VL8NXBWeJPgfA3yvXTU9CtxL8NlaSRB8A33uVoJ9NH9PsHN/NcEBBxD8/eaZWYOZ/bqHpx/vZ1oGwMIdOiKRCP/bbQBmu3tv29lFJCJak5CEM7O3hZvE8ggOgX2B4KgVEUkyCgmJwpW8fgLUbIJDWLVKK5KEtLlJRER6pTUJERHpVcqfTFdWVubTpk2LugwRkZSycuXKPe5e3l+7lA+JadOmUVNTE3UZIiIpxcy29N9Km5tERKQPCgkREemVQkJERHqV0JAwsyVmttHMas3s5h6m/7uZrQ5vL3XrGVNERBIsYTuuw07Fbie4PnAdsMLMHgx7GAXA3T8d1/5GYGGi6hMRkTdK5JrEYqDW3Te5eytBB2999e55DUGnciIiEpFEhsRkjr34TF047g3CbqKnE/So2dP0682sxsxq6uvrh71QEREJJPI8CethXG99glwN3O/uHT1NdPc7gDsAqqurh9SvyIrN+/jzy3tITzPSY0ZGWhqxNCMjZqTH4obT0sjLilGSmxnc8jIpzE7HrKe3IyIyuiQyJOoILlXYpZLeryJ1NfCJkSzmuS37+Y8/vjyk5xZkpTO9PI9p4/I4cWIhp08tYX5lEdkZsWGuUkQkWgnr4M/M0oGXgDcTXApyBXBteBGa+HYnAEuB6QPpGbS6utqHesa1u9PR6bR33To6w3unvbPz6H1jSwf7mlrZ39jKvsZWtu1rYtOeRjbVN7K94QgA6WnGwinFXDR3PBefWMGsinytbYhI0jKzle7e77XbE7YmEV4+8QaCAIgBP3T3tWZ2G1Dj7g+GTa8B7klE19Fmwaam9ONYAdh7uIVVWxuo2bKfP79cz1ce3sBXHt7AzPI83l1dxTtPm0xFQfbwFS0ikkAp31X48axJjITXDhzhf9fv5jertlOzZT+xNOOv5k/khgtnMXt8QdTliYgAA1+TUEiMoFfqD3P3M1v5+bNbOdLWwZWnTuJzl59IRaHWLEQkWgqJJLKvsZUf/HkTP/jzq2Slp/G5t57I1YuqtM9CRCIz0JBQ300JUJqXyWeXzGXpp8/nlMoibvnVC3z63tU0tbZHXZqISJ8UEgk0vSyPn374DG66ZA6/eX4H137/GRqaWqMuS0SkVwqJBIulGZ9882y++77TWbfjIO/93tPsb1RQiEhyUkhE5NKTJvCjDy7i1b2NfOQnNTS39XhyuYhIpBQSETpnVhnfeu8Cntu6n8/ev4ZUP4hAREYfhUTELj9lIjddPIcHn9/BvSu29f8EEZEEUkgkgY9fOItzZ5Xxxd+u5dU9jVGXIyJylEIiCcTSjG++51Qy0tL4wq9f0GYnEUkaCokkUVGYzWcvm8vy2r38ZnVvneOKiCSWQiKJ/PXiKcyvLOJrSzfqaCcRSQoKiSSSlmbcvGQu2xuO8LOnt0RdjoiIQiLZnD2rjPNml3H747XqtkNEIqeQSEKfung2+5va+EVNXdSliMgYp5BIQqdPLeW0KcX84MlNdHTqSCcRiY5CIkldf/4Mtu07wsMv7oy6FBEZwxQSSeqSeROoKs3RDmwRiZRCIknF0oz3Vlfx1Ka9bNmrs7BFJBoKiSR21elVpBncV6M+nUQkGgqJJDahKJs3zSnn/pV1tHd0Rl2OiIxBCokk9+7qKnYdbOGZV/dFXYqIjEEKiSR34QkV5GbG+N2a16IuRUTGIIVEksvJjHHxieN5+MXXtMlJRBIuoSFhZkvMbKOZ1ZrZzb20eY+ZrTOztWb280TWl6zeOn8i+5vaeGrT3qhLEZExJmEhYWYx4HbgMmAecI2ZzevWZjZwC3COu58EfCpR9SWzN80pJz8rnd9rk5OIJFgi1yQWA7XuvsndW4F7gCu7tfkb4HZ33w/g7rsTWF/Sys6IccEJ5fzv+t10qpsOEUmgRIbEZCD+gP+6cFy8OcAcM1tuZk+b2ZKEVZfkLppbwZ7DLby440DUpYjIGJLIkLAexnX/tzgdmA1cAFwD/MDMit8wI7PrzazGzGrq6+uHvdBkdMEJFZjBH9dr5UpEEieRIVEHVMU9rgS6X6ezDviNu7e5+6vARoLQOIa73+Hu1e5eXV5ePmIFJ5PSvEwWVhXz+EaFhIgkTiJDYgUw28ymm1kmcDXwYLc2vwYuBDCzMoLNT5sSWGNSu2huBWvqDrD7UHPUpYjIGJGwkHD3duAGYCmwHrjP3dea2W1mdkXYbCmw18zWAY8Dn3F3HfcZunBuBQDLNo6NTWwiEr30RL6Yuz8EPNRt3K1xww7cFN6km3kTCynLz+QvtXt4T3VV/08QETlOOuM6hZgZZ80sY/krewnyVERkZCkkUszZM8dRf6iFV+oPR12KiIwBCokUc87MMgCW12pXjYiMPIVEipkyLpfKkhz+8sqeqEsRkTFAIZGCzp45jqde2UuHuugQkRGmkEhB58wq42BzO2vVRYeIjDCFRAo6c8Y4AJ7V1epEZIQpJFLQ+MJsqkpzqNm8P+pSRGSUU0ikqEVTS6nZsk/nS4jIiFJIpKjqaaXsOdzK5r1NUZciIqOYQiJFLZpWAsCKzdovISIjRyGRomaW51Ocm0GNQkJERpBCIkWlpRnVU0u081pERpRCIoVVTytl055G9hxuiboUERmlFBIprGu/hNYmRGSkKCRS2MmTi8hMT2PlFu2XEJGRoZBIYVnpMU6aVMjqbQ1RlyIio5RCIsUtrCphTd0B2jo6oy5FREYhhUSKWzClmJb2TjbuPBR1KSIyCikkUtzCqmIAVmmTk4iMAIVEiqssyaEsP5NVW3WEk4gMP4VEijMzFlSVaOe1iIwIhcQosHBKMZvqGznQ1BZ1KSIyyigkRoGu/RKr67Q2ISLDSyExCpxSWYQZ2i8hIsMuoSFhZkvMbKOZ1ZrZzT1Mv87M6s1sdXj7SCLrS1UF2RnMqSjQfgkRGXbpiXohM4sBtwOXAHXACjN70N3XdWt6r7vfkKi6RosFVcUsXbcTd8fMoi5HREaJRK5JLAZq3X2Tu7cC9wBXJvD1R7UFU4ppaGrTlepEZFglMiQmA9viHteF47p7l5mtMbP7zayqpxmZ2fVmVmNmNfX19SNRa8pZOCXceb1N+yVEZPgkMiR62gbi3R7/Fpjm7vOB/wV+3NOM3P0Od6929+ry8vJhLjM1za4oIC8zxqqt2i8hIsMnkSFRB8SvGVQCO+IbuPted++6gs73gdMTVFvKi6UZ8yuLtfNaRIZVIkNiBTDbzKabWSZwNfBgfAMzmxj38ApgfQLrS3kLpxSzbsdBmts6oi5FREaJhIWEu7cDNwBLCX7873P3tWZ2m5ldETb7pJmtNbPngU8C1yWqvtFgQVUx7Z3O2h0Hoi5FREaJhB0CC+DuDwEPdRt3a9zwLcAtiaxpNFkQ7rxetbWB06eWRlyNiIwGOuN6FKkoyKayJEc7r0Vk2CgkRpkFVdp5LSLDRyExyiycUsL2hiPsOtgcdSkiMgooJEaZBVWv75cQETleColR5qRJhWTETJucRGRYKCRGmeyMGPMmFanbcBEZFgqJUWhhVTEvbD9Ae0dn1KWISIpTSIxCC6cU09TawUu7DkddioikOIXEKLSwqgSAVeoRVkSOk0JiFKoqzaE0L5PVOsJJRI6TQmIUMjMWVhWzSkc4ichxUkiMUgunFFO7+zAHjrRFXYqIpDCFxCi1INwvsaZOaxMiMnQKiVFqflURZjrzWkSOj0JilCrMzmBWeb5OqhOR46KQGMUWTgl6hHXvfilxEZGBUUiMYgunlLC/qY0te5uiLkVEUpRCYhQ72iOsTqoTkSFSSIxic8YXUJCVzorNCgkRGRqFxCgWSzNOn1bCilf3RV2KiKSoQYeEmeWZWWwkipHht2haKS/vPsy+xtaoSxGRFNRvSJhZmplda2a/N7PdwAbgNTNba2ZfM7PZI1+mDNXi6aUArNistQkRGbyBrEk8DswEbgEmuHuVu1cA5wFPA182s/eNYI1yHOZXFpGZnqZNTiIyJOkDaHOxu7+hAyB33wf8EvilmWUMe2UyLLLSYyyoKtaahIgMSb9rEl0BYWbZZnaymZ1kZtk9tZHktHhaKS/uOEhjS3vUpYhIihnIPol0M/sqUAf8GPgZsM3MvmxmA1kTiZ/XEjPbaGa1ZnZzH+2uMjM3s+rBzF96tmh6KR2dznPqokNEBmkg+yS+BpQA04HfuftCgn0UZcDXB/pC4RFRtwOXAfOAa8xsXg/tCoBPAs8MdN7St9OmFJNmaL+EiAzaQELir4Dr3f0Q8DYAdz8IfDScNlCLgVp33+TurcA9wJU9tPsS8FWgeRDzlj4UZGcwb1IhzygkRGSQBhIS7q/3EGdxIzuAzkG81mRgW9zjunDcUWa2EKhy99/1NSMzu97Masyspr6+fhAljF2Lp41j9bYGmts6oi5FRFLIQEJivZm9Pxze0TUyPOx1/SBey3oYd7R7UjNLA/4d+Pv+ZuTud7h7tbtXl5eXD6KEseucWeNoae/kuS3aLyEiAzeQkPgEcKOZLQM2mNnXzexPwI3AxwbxWnVAVdzjSuJCBygATgaWmdlm4EzgQe28Hh5nzBhHLM14snZP1KWISArp9+gkd68DFpnZmwl2OBvwkLs/NsjXWgHMNrPpwHbgauDauNc5QLAzHIAwlP7B3WsG+TrSg/ysdBZWFbNcISEig9BvSJiZeeCPwB/7atPXfNy93cxuAJYCMeCH7r7WzG4Datz9wSHUL4Nwzqwy/uOxlznQ1EZRrs5/FJH+DahbDjO70cymxI80s0wzu8jMfgx8YCAv5u4Pufscd5/p7v8ajru1p4Bw9wu0FjG8zp1dhjs8tUlrEyIyMAMJiSVAB3C3me0ws3Vm9irwMnAN8O/ufucI1ijDZEFVMXmZMe2XEJEBG8g+iWbgO8B3wj6ayoAj7t4w0sXJ8MqIpXHGjHEsr90bdSkikiIGdT0Jd29z99cUEKnrnFllvLqnkbr9uu61iPRvIH03HTKzg73c6s3s6fDIJ0kB580ODiB74iVtchKR/g2kF9gCdy/s6QZMIOie49sjXqkMi9kV+VSW5PDYhl1RlyIiKeC4rnHt7h3u/jzwn8NUj4wwM+OiuRU8WbtHXXSISL+OKyS6uPv3hmM+khgXza2gua2TpzZpB7aI9G1YQkJSy5kzxpGTEeOx9bujLkVEkpxCYgzKzohx7uwyHtuwm35OlBeRMU4hMUZdNLeC7Q1H2LjrUNSliEgSU0iMURfNrQDgkbU6yklEeqeQGKPGF2Zz+tQSHnrhtahLEZEkppAYwy4/ZSIbdh5iU/3hqEsRkSSlkBjDLj9lAoDWJkSkVwqJMWxiUQ6nTSnm9y/sjLoUEUlSCokx7vJTJrL+tYO8uqcx6lJEJAkpJMa4y0+ZCMDvnt/RT0sRGYsUEmPcpOIcFk8v5YFV23VinYi8gUJCuOr0SjbtaeS5rbpMiIgcSyEhXH7KRHIyYvzyubqoSxGRJKOQEPKz0rns5An89vkd6j5cRI6hkBAA3nV6JYea23l0nbrpEJHXKSQEgLNmjGNycQ73rtgWdSkikkQUEgJAWppx7RlTeLJ2D7W71U2HiAQSGhJmtsTMNppZrZnd3MP0vzWzF8xstZk9aWbzElnfWPfeRVVkxtL42dNboi5FRJJEwkLCzGLA7cBlwDzgmh5C4Ofufoq7LwC+CnwzUfUJlOVn8db5E7l/ZR2HW9qjLkdEkkAi1yQWA7XuvsndW4F7gCvjG7j7wbiHeYDO7kqw9581lcMt7TywanvUpYhIEkhkSEwG4veK1oXjjmFmnzCzVwjWJD7Z04zM7HozqzGzmvr6+hEpdqxaUFXM/MoifrT8VTo7ldEiY10iQ8J6GPeGXyF3v93dZwL/CHyhpxm5+x3uXu3u1eXl5cNc5thmZvzNeTPYVN/II+vUO6zIWJfIkKgDquIeVwJ99Sp3D/D2Ea1IenT5KROZNi6X7yx7Rf05iYxxiQyJFcBsM5tuZpnA1cCD8Q3MbHbcw7cCLyewPgnF0oyPvmkma+oOsLx2b9TliEiEEhYS7t4O3AAsBdYD97n7WjO7zcyuCJvdYGZrzWw1cBPwgUTVJ8d652mTqSjI4vbHa6MuRUQilJ7IF3P3h4CHuo27NW747xJZj/QuKz3GR980ky/9bh1/qd3D2bPKoi5JRCKgM66lV399xhQmFWXzlaUbtW9CZIxSSEivsjNifOqSOTy/rYGla9Xxn8hYpJCQPr1z4WRmVeTztaUbaO/ojLocEUkwhYT0KT2WxmcvPYFX6hvVp5PIGKSQkH5dMm88580u4xuPvkT9oZaoyxGRBFJISL/MjC9ecRLNbR189eENUZcjIgmkkJABmVmez4fPncEvVtaxcsu+qMsRkQRRSMiA3XjRLCYX5/CZ+9foWtgiY4RCQgYsLyudr7xrPpvqG/n60o1RlyMiCaCQkEE5d3YZ7ztzCv+z/FVWbNZmJ5HRTiEhg3bLZScyuTiHm+5bzYEjbVGXIyIjSCEhg5aXlc63r17Iaw3NfPb+59Vlh8goppCQITl9agn/uGQuS9fu4kfLN0ddjoiMEIWEDNlHzpvOxSeO59/+sJ4a7Z8QGZUUEjJkZsY33n0qlSW5XP/TlWzd2xR1SSIyzBQSclyKcjP44XWL6Oh0PvTjFdqRLTLKKCTkuE0vy+O77zudLXsb+fhdK3WincgoopCQYXHWzHF85V3zWV67lxvvXkWbuhUXGRUUEjJs3nlaJbddeRKPrtvF39/3PB2dOjRWJNUl9BrXMvq9/6xpNLZ08JWHN5CeZnz1qvmkx/S/iEiqUkjIsPvYBTNp7+jkG4++xOGWdv7z2oVkpceiLktEhkD/4smIuPHNs/ni2+bxyLpdfPjOGhpb2qMuSUSGQCEhI+a6c6bzjXefylOb9vKu//4L2xuORF2SiAySQkJG1LtOr+RH1y1ie8MRrvyvJ1m5ZX/UJYnIICgkZMSdP6ecBz5+NnlZ6Vzz/af5+TNb1SmgSIpIaEiY2RIz22hmtWZ2cw/TbzKzdWa2xsz+aGZTE1mfjJxZFQX8+uPncMb0Uj73wAt88p7VHGrW2dkiyS5hIWFmMeB24DJgHnCNmc3r1mwVUO3u84H7ga8mqj4ZeSV5mfz4g4v5zKUn8Ps1O3jbfz7JC3UHoi5LRPqQyDWJxUCtu29y91bgHuDK+Abu/ri7d/US9zRQmcD6JAHS0oxPXDiLe64/i+a2Tt7+neV8felGWtrVlYdIMkpkSEwGtsU9rgvH9ebDwB96mmBm15tZjZnV1NfXD2OJkiiLp5ey9FPn846Fk/mvx2t5238+yfPbGqIuS0S6SWRIWA/jetx7aWbvA6qBr/U03d3vcPdqd68uLy8fxhIlkYpyM/j6u0/lRx9cxMEj7bz9O8v53AMvsL+xNerSRCSUyJCoA6riHlcCO7o3MrOLgc8DV7h7S4JqkwhdeEIFj9x0Ph88ezr3rtjGBV9fxk+f2qy+n0SSQCJDYgUw28ymm1kmcDXwYHwDM1sIfI8gIHYnsDaJWGF2Bre+bR5/+LvzOGlSIf/0m7Us+dYTPPziTh0uKxKhhIWEu7cDNwBLgfXAfe6+1sxuM7MrwmZfA/KBX5jZajN7sJfZySg1Z3wBd33kDL77vtPocOdvf7aSt9++nCdf3qOwEImApfoXr7q62mtqaqIuQ0ZAe0cnv1q1nW//78tsbzhC9dQS/vZNM7lobgVpaT3t4hKRgTKzle5e3W87hYQku5b2Du55dht3PLGJ7Q1HmDM+n4+eP5MrFkwiQ92QiwyJQkJGnbaOTn63Zgff+9MmNuw8REVBFtcsnsI1i6cwoSg76vJEUopCQkYtd2fZxnp+/NRm/vRSPWlmXHLieP7PWVM5a8Y4bYoSGYCBhoQuOiQpx8y4cG4FF86tYOveJu56dgv3rdjGw2t3UlmSwzsWTuYdCyczozw/6lJFUp7WJGRUaG7r4OEXd/LL5+pYXruHTodTq4p5x4JJvHX+JMoLsqIuUSSpaHOTjFm7Djbzm9Xb+dVz29mw8xBmsGhqKZeePIElJ09gcnFO1CWKRE4hIQJs2HmQP7ywk6Vrd7Jh5yEA5lcWcelJE7jghHLmTSzETPswZOxRSIh0s3lPI0vX7uThtTtZtTXoTLC8IIvzZpfxpjnlnDe7nNK8zIirFEkMhYRIH3YfaubPL+3hTy/V88TL9TQ0tWEG8ycXcebMcZw5YxzVU0soyM6IulSREaGQEBmgjk7nhe0HWLZxN39+eQ/Pb2ugvdNJMzh5chFnTC/ljOnjWDStlKJchYaMDgoJkSFqam3nuS0NPPPqXp7ZtI/V2xpo7egEYGZ5HguqSlgwpZiFVcWcMKFAZ31LStJ5EiJDlJuZzrmzyzh3dhkQHF67amsDNZuDwFi2cTe/fK4OgOyMNE6eVMSCqmJOqSxi3sRCppflka7gkFFCISHSj+yMGGfNHMdZM8cBwRnfdfuPsGpbA6u3NrB6235+8vQWWtuDtY2s9DTmTihg3qRC5k0sZN6kQuZOKCQvS183ST3a3CQyDFrbO3ml/jDrdhxk3WsHj94fONIGgBlMKc1lVnle0K9oAAAMaElEQVQ+s8bnM7uigNkV+cysyCdf4SER0OYmkQTKTE/jxImFnDixkHeF49yd1w40s27HQdbuOMhLuw5Ru/swT7xcT1vH6/+cTSrKZtb4giBAKvKZVpbL1HF5TCzMVj9UEjmFhMgIMTMmFecwqTiHi+eNPzq+vaOTLfuaqN19+Ojt5d2HuPvVfRxp6zjaLjM9jSmluUwbF4TG6/d5TCrO1n4PSQiFhEiCpcfSmFmez8zyfC496fXxnZ3OjgNH2Lq3ic17m9iyt5HNexvZsreJJ2v30NzW+fo80owJRdlMLs5hckkOleH95OJcJpfkMLEom+yMWATvTkYbhYRIkkhLMypLcqksyeXsWcdOc3d2H2ph854gNDbvbWR7wxG27z/CU6/sZdfBZjq77V4sL8g6GiKTi3MYX5jNhMJsxhdmMb4wm4rCLLLSFSTSN4WESAowM8YXZjO+MJszZox7w/S2jk52Hmimbv+Ro+GxvaGJ7Q1HWLv9AI+u23X06Kt4pXmZVBRkMaEom/EF2YwvOjZIyguyKM3L1LkgY5hCQmQUyIilUVWaS1Vpbo/T3Z2GpjZ2HWpm54Fmdh9sYefBZnYdvbWwdsdB9hxuoacDHotzMxiXl0lZfhZlBVmUhcPj8rMoy88Mx2VRVpBJbqZ+VkYT/TVFxgAzoyQvk5K8TOZOKOy1XVtHJ3sOt7DrYAs7DzSz53ALew+3BveNLew51Mr6MEwONrf3OI+cjBhlBZmU5GZSnJtJaW4GxbnB45K8jHBcJsW5GUFNuRnkZMTUG2+SUkiIyFEZsTQmFuUwsSgHqvpu29Lewd7DrUdDJLiFgXK4hf1NbexvauXVPYdpaGzjUEvPoQLBkVwluRlBkMSFSVFOBkU5GRRmZ1CYkx7ed41LpyA7g8x0bQobSQoJERmSrPTY0UN8B6Kto5OGpjYamlrZ19jK/nD49ftgeH9jKxt3HqKhqY0DR9po775HvpucjBiFOelxYRIESGG3gCnIziA/K5387PTgPiudvPA+pvNReqWQEJGEyIilUV6QNahLybo7zW2dHDjSxsHmNg4evW/nYHMbB5qOfXywuY3dh5qp3d1+tH0/GQMEQZOfnU5BXHDkZaVTkJ1OXlaM/KyMYDgzRn52BvnhuLysGAXZ6eRmppObGSM3M33UrdkkNCTMbAnwbSAG/MDdv9xt+vnAt4D5wNXufn8i6xOR5GJm5GTGyMmMMaEoe9DPd3caWzuCkDnSRmNLO4fDW2NLO4eaXx8OxndwuLmNwy3tbG84wuGWNhpbOjjU3HbMWfJ9SU8Las7rCo6sGLkZ6cG4rBg5GceOPzochkxw//pw17xyMmKRnIGfsJAwsxhwO3AJUAesMLMH3X1dXLOtwHXAPySqLhEZvczs6Kal4722eUt7B40tHRxubudQGB6HW9o41NzOkdYOGls7ONLaTlNrR3g7drihqZXtDR0cCR83tnb0eFhyX7Iz0sjJiJGTESM7M8anLp7DFadOOq731Z9ErkksBmrdfROAmd0DXAkcDQl33xxOG9ySExEZYVnpMbLSY8N6idv2jk6OtHX0GCxHWttpbOmgqe314ea2Do60hUHT1kFJAi6ClciQmAxsi3tcB5yRwNcXEUkq6bE0CmJpSX2Z3ETuYelpY9qQ+ik3s+vNrMbMaurr64+zLBER6U0iQ6KOY4+8rgR2DGVG7n6Hu1e7e3V5efmwFCciIm+UyJBYAcw2s+lmlglcDTyYwNcXEZFBSlhIuHs7cAOwFFgP3Ofua83sNjO7AsDMFplZHfBu4HtmtjZR9YmIyBsl9DwJd38IeKjbuFvjhlcQbIYSEZEkMLpODRQRkWGlkBARkV4pJEREpFfmPV1hJIWYWT2wZYhPLwP2DGM5IylVak2VOkG1joRUqRNSp9aRqnOqu/d7DkHKh8TxMLMad6+Ouo6BSJVaU6VOUK0jIVXqhNSpNeo6tblJRER6pZAQEZFejfWQuCPqAgYhVWpNlTpBtY6EVKkTUqfWSOsc0/skRESkb2N9TUJERPqgkBARkV6N2ZAwsyVmttHMas3s5iSoZ7OZvWBmq82sJhxXamaPmtnL4X1JON7M7D/C2teY2WkjXNsPzWy3mb0YN27QtZnZB8L2L5vZBxJU5xfNbHu4XFeb2eVx024J69xoZpfGjR/xz4aZVZnZ42a23szWmtnfheOTarn2UWfSLVczyzazZ83s+bDWfw7HTzezZ8Llc2/YCzVmlhU+rg2nT+vvPYxwnXea2atxy3RBOD6y7xQQXCh8rN2AGPAKMAPIBJ4H5kVc02agrNu4rwI3h8M3A18Jhy8H/kBwIaczgWdGuLbzgdOAF4daG1AKbArvS8LhkgTU+UXgH3poOy/8u2cB08PPQyxRnw1gInBaOFwAvBTWlFTLtY86k265hssmPxzOAJ4Jl9V9wNXh+O8CHwuHPw58Nxy+Gri3r/eQgDrvBK7qoX1k3yl3H7NrEkevt+3urUDX9baTzZXAj8PhHwNvjxv/Ew88DRSb2cSRKsLdnwD2HWdtlwKPuvs+d98PPAosSUCdvbkSuMfdW9z9VaCW4HORkM+Gu7/m7s+Fw4cIus+fTJIt1z7q7E1kyzVcNofDhxnhzYGLgPvD8d2Xadeyvh94s5lZH+9hpOvsTWTfKRi7m5t6ut52Xx/8RHDgETNbaWbXh+PGu/trEHxZgYpwfDLUP9jaoqz5hnA1/Yddm2/6qCfhdYabORYS/EeZtMu1W52QhMvVzGJmthrYTfCj+QrQ4MH1bLq/7tGawukHgHGJqLV7ne7etUz/NVym/25mWd3r7FZPQpbpWA2JYbve9jA6x91PAy4DPmFm5/fRNhnr79JbbVHV/N/ATGAB8BrwjXB8UtRpZvnAL4FPufvBvpr2MC5h9fZQZ1IuV3fvcPcFBNelWQyc2MfrRlZr9zrN7GTgFmAusIhgE9I/Rl0njN2QGLbrbQ8Xd98R3u8GHiD4gO/q2owU3u8OmydD/YOtLZKa3X1X+IXsBL7P65sNIq/TzDIIfnjvcvdfhaOTbrn2VGcyL9ewvgZgGcE2/GIz67rAWvzrHq0pnF5EsLkyYbXG1bkk3LTn7t4C/IgkWaZjNSSS6nrbZpZnZgVdw8BbgBfDmrqOWPgA8Jtw+EHg/eFRD2cCB7o2USTQYGtbCrzFzErCTRNvCceNqG77at5BsFy76rw6PMJlOjAbeJYEfTbCbd//A6x392/GTUqq5dpbncm4XM2s3MyKw+Ec4GKCfSiPA1eFzbov065lfRXwmAd7hHt7DyNZ54a4fw6MYL9J/DKN7js13HvCU+VGcMTASwTbLD8fcS0zCI6meB5Y21UPwfbRPwIvh/el/vrREbeHtb8AVI9wfXcTbFJoI/jv5cNDqQ34EMFOwFrggwmq86dhHWsIvmwT49p/PqxzI3BZIj8bwLkEmwbWAKvD2+XJtlz7qDPpliswH1gV1vQicGvc9+vZcPn8AsgKx2eHj2vD6TP6ew8jXOdj4TJ9EfgZrx8BFdl3yt3VLYeIiPRurG5uEhGRAVBIiIhIrxQSIiLSK4WEiIj0SiEhIiK9UkiIhMzscHg/zcyuHeZ5f67b478M5/xFRopCQuSNpgGDCgkzi/XT5JiQcPezB1mTSCQUEiJv9GXgvLBP/0+HnbF9zcxWhJ2vfRTAzC6w4FoLPyc4yQkz+3XYSeParo4azezLQE44v7vCcV1rLRbO+0ULrify3rh5LzOz+81sg5ndFZ6JK5JQ6f03ERlzbia4VsJfAYQ/9gfcfVHYM+dyM3skbLsYONmDLqUBPuTu+8LuFlaY2S/d/WYzu8GDDt26eydBJ3mnAmXhc54Ipy0ETiLoj2c5cA7w5PC/XZHeaU1CpH9vIeg7ZzVBN9njCPrzAXg2LiAAPmlmzwNPE3S+Npu+nQvc7UFnebuAPxH0Ato17zoPOtFbTbAZTCShtCYh0j8DbnT3YzpPM7MLgMZujy8GznL3JjNbRtA/UH/z7k1L3HAH+r5KBLQmIfJGhwgu1dllKfCxsMtszGxO2Ftvd0XA/jAg5hJ0U92lrev53TwBvDfc71FOcAnWYetxVOR46T8TkTdaA7SHm43uBL5NsKnnuXDncT2vXwIz3sPA35rZGoLeQ5+Om3YHsMbMnnP3v44b/wBwFkEPwA581t13hiEjEjn1AisiIr3S5iYREemVQkJERHqlkBARkV4pJEREpFcKCRER6ZVCQkREeqWQEBGRXv1/DKHhDvepCJkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1650a95b9b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set a learning rate\n",
    "learning_rate = 0.5\n",
    "# Maximum number of iterations for gradient descent\n",
    "maxepochs = 10000       \n",
    "# Costs convergence threshold, ie. (prevcost - cost) > convergence_thres\n",
    "convergence_thres = 0.00001  \n",
    "# Number of hidden units\n",
    "hidden_units = 4\n",
    "\n",
    "# Initialize model \n",
    "model = NNet3(learning_rate=learning_rate, maxepochs=maxepochs,\n",
    "              convergence_thres=convergence_thres, hidden_layer=hidden_units)\n",
    "# Train model\n",
    "model.learn(X, y)\n",
    "\n",
    "# Plot costs\n",
    "plt.plot(model.costs)\n",
    "plt.title(\"Convergence of the Cost Function\")\n",
    "plt.ylabel(\"J($\\Theta$)\")\n",
    "plt.xlabel(\"Iteration\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X[:70]\n",
    "y_train = y[:70]\n",
    "\n",
    "X_test = X[-30:]\n",
    "y_test = y[-30:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "# Set a learning rate\n",
    "learning_rate = 0.5\n",
    "# Maximum number of iterations for gradient descent\n",
    "maxepochs = 10000       \n",
    "# Costs convergence threshold, ie. (prevcost - cost) > convergence_thres\n",
    "convergence_thres = 0.00001  \n",
    "# Number of hidden units\n",
    "hidden_units = 4\n",
    "\n",
    "# Initialize model \n",
    "model = NNet3(learning_rate=learning_rate, maxepochs=maxepochs,\n",
    "              convergence_thres=convergence_thres, hidden_layer=hidden_units)\n",
    "model.learn(X_train,y_train)\n",
    "predictions = model.predict(X_test)[0]\n",
    "#print(predictions)\n",
    "auc = roc_auc_score(y_test, predictions)\n",
    "print(auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}