code/ch02/ex02.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Asssignment 2 - linear perceptron\n",
    "\n",
    "A very simple classifier that also illustrates main aspects of machine learning."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Decision boundary for linear perceptron\n",
    "\n",
    "Below are the code and data for linear classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class Perceptron(object):\n",
    "    \"\"\"Perceptron classifier.\n",
    "\n",
    "    Parameters\n",
    "    ------------\n",
    "    eta : float\n",
    "        Learning rate (between 0.0 and 1.0)\n",
    "    n_iter : int\n",
    "        Passes over the training dataset.\n",
    "\n",
    "    Attributes\n",
    "    -----------\n",
    "    w_ : 1d-array\n",
    "        Weights after fitting.\n",
    "    errors_ : list\n",
    "        Number of misclassifications in every epoch.\n",
    "\n",
    "    \"\"\"\n",
    "    def __init__(self, eta=0.01, n_iter=10):\n",
    "        self.eta = eta\n",
    "        self.n_iter = n_iter\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        \"\"\"Fit training data.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : {array-like}, shape = [n_samples, n_features]\n",
    "            Training vectors, where n_samples is the number of samples and\n",
    "            n_features is the number of features.\n",
    "        y : array-like, shape = [n_samples]\n",
    "            Target values.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        self : object\n",
    "\n",
    "        \"\"\"\n",
    "        self.w_ = np.zeros(1 + X.shape[1])\n",
    "        self.errors_ = []\n",
    "\n",
    "        for _ in range(self.n_iter):\n",
    "            errors = 0\n",
    "            for xi, target in zip(X, y):\n",
    "                update = self.eta * (target - self.predict(xi))\n",
    "                self.w_[1:] += update * xi\n",
    "                self.w_[0] += update\n",
    "                errors += int(update != 0.0)\n",
    "            self.errors_.append(errors)\n",
    "        return self\n",
    "\n",
    "    def net_input(self, X):\n",
    "        \"\"\"Calculate net input\"\"\"\n",
    "        return np.dot(X, self.w_[1:]) + self.w_[0]\n",
    "\n",
    "    def predict(self, X):\n",
    "        \"\"\"Return class label after unit step\"\"\"\n",
    "        return np.where(self.net_input(X) >= 0.0, 1, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>6.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>6.3</td>\n",
       "      <td>2.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.9</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>5.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.1</td>\n",
       "      <td>1.8</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0    1    2    3               4\n",
       "145  6.7  3.0  5.2  2.3  Iris-virginica\n",
       "146  6.3  2.5  5.0  1.9  Iris-virginica\n",
       "147  6.5  3.0  5.2  2.0  Iris-virginica\n",
       "148  6.2  3.4  5.4  2.3  Iris-virginica\n",
       "149  5.9  3.0  5.1  1.8  Iris-virginica"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#data_src = '../datasets/iris/iris.data'\n",
    "data_src = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "\n",
    "df = pd.read_csv(data_src, header=None)\n",
    "\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAEbCAYAAABgLnslAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuYFNWd//H3d7g5SFARJCAICpqAl6gbNQiGMXjNrsYb\nChrvTzYGTSTmlyiiYYxZTdCfumT9JcuzxltW3JiNBneDokhr1ESNaEBFTEAmCojgFRBmYOb7+6N6\nZnrGuVT1dHVX9Xxez9NPT1VXV51TBfOdOt8655i7IyIikkQVpS6AiIhIexSkREQksRSkREQksRSk\nREQksRSkREQksRSkREQksWIPUma2i5k9YGbLzexVMzsi7mOKiEh56FmEY/wr8Ht3n2xmPYG+RTim\niIiUAYuzM6+Z9QdecvdRsR1ERETKVtzNfXsDG83sTjNbYmZzzawy5mOKiEiZiDtI9QQOBW5390OB\nT4CrYj6miIiUibhzUm8Db7n7n7PLvwGubL2RmWkAQRGRMufuFvU7sd5Juft64C0z2y+7ahLwWjvb\nlt1r1qxZJS+D6qV6qV7pepVrvfJVjKf7vgP8p5n1AlYBFxbhmCIiUgZiD1Lu/hfgsLiPIyIi5Ucj\nTsSoqqqq1EWIheqVLqpXupRrvfIVaz+p0IUw8ySUQ0RE4mFmeB4PThQjJ5W3kSNHUlNTU+piSI4R\nI0awevXqUhdDRLqJRN9JZSNvCUok7dE1EZF85HsnpZyUiIgkloKUiIgkloKUiIgkloKUiIgkloJU\nEdXU1FBRUUFDQ0OpiyIikgrlG6TWrIEXXoCPPip1SZq4u56OExGJIJ1Byh1Wr4aamuDn1mbPhtGj\n4ZhjYNgwyGRiKcZPf/pThg0bRv/+/RkzZgyLFy/G3fnJT37C6NGjGTRoEFOmTOHDDz8EYOLEiQDs\nuuuu9O/fn+eeew5358c//jEjR47ks5/9LBdccAEff/wxALW1tZx77rkMHDiQ3XbbjSOOOIINGzYA\ncNdddzF27Fj69+/P6NGjmTt3bix1FBEpqVKPjJu9q/C2tLl+0yb3cePcKyvdd9rJfdIk961bmz//\ny1/c+/Z1D8JX8NplF/cdO1ruZ9489zPOcJ82zf2tt9o8fkdWrFjhw4cP93feecfd3WtqanzVqlV+\n2223+bhx43zt2rVeV1fnl1xyiU+dOtXd3VevXu0VFRXe0NDQtJ877rjD9913X1+9erVv2bLFTzvt\nND/vvPPc3f3f//3f/eSTT/Zt27Z5Q0ODL1myxDdt2uTu7r///e/9zTffdHf3p556yvv27esvvfRS\n5HpE1d61EhHpSPZ3R/T4kM+XCv2KFKSmTXPv06c5AFVWus+Y0fz5r3/t3r9/yyDVp4/7u+82b3Pz\nzc2BrEcP9913d88Gm7D+9re/+eDBg/3xxx/37du3N60fM2aMP/HEE03La9eu9V69enl9fb2/+eab\nXlFR4fX19U2fT5o0yX/+8583La9YscJ79+7t9fX1/stf/tLHjx/vS5cu7bQ8p5xyis+ZMydSHfKh\nICUi+cg3SKWvue/556G2tnl561Z47rnm5c99DnbsaPmdnXaCAQOal//lX+CTT4Kf6+th82aYNy9S\nMUaNGsVtt91GdXU1e+yxB2effTbr1q2jpqaGU089lQEDBjBgwADGjh1Lr169WL9+PWaf7my9du1a\nRowY0bQ8YsQItm/fzvr16zn33HM5/vjjmTJlCsOGDeOqq66ivr4egAULFjBu3Dh23313dtttNxYs\nWMDGjRsj1UFEJOnSF6TGjoVevZqX+/SB/fdvXj7oIJg1KwhM/ftDv37w0EPQo0fzNq2DWEMD1NVF\nLsqUKVP4wx/+wN///ncArrzySvbaay8WLFjA+++/z/vvv88HH3zAli1bGDJkSJtBaujQoS3GJ6yp\nqaFXr14MHjyYnj17cu211/Lqq6/y7LPP8vDDD3PPPfdQV1fHGWecwQ9+8AM2bNjABx98wIknnqgH\nMkSk7KQvSN1yC4wcCZ/5TBCA9t0Xfvzjltv84AewciUsWgRvvw2th74//3zo27d5uXdvOPXUSMV4\n4403WLx4MXV1dfTu3ZvKykp69OjBJZdcwtVXX90UuDZs2MD8+fMBGDRoEBUVFaxcubJpP1OnTuXW\nW29l9erVbN68mZkzZzJlyhQqKirIZDK88sorNDQ00K9fP3r16kWPHj2oq6ujrq6OgQMHUlFRwYIF\nC1i4cGGk8ouIpEGiR0Fv0+67w7Jl8OKLYAZf/GLLO6tGQ4cGr7bceivsuiv89rfB/m65JQh2EdTW\n1nLVVVfx+uuv06tXL4488kjmzp3L4MGDcXeOO+441q1bxx577MFZZ53FySefTGVlJTNnzmT8+PHs\n2LGDRx55hIsuuoh169bx5S9/mdraWk444QTmzJkDwDvvvMMll1zCmjVr6NevH1OmTOHrX/86FRUV\nzJkzh8mTJ1NXV8dJJ53E1772tahnUkQk8TQKukSiayIi+dAo6CIiUnYUpEREJLEUpEREJLEUpERE\nJLEUpEREJLEUpEREJLEUpEREJLEUpEREJLEUpBLsM5/5DKtXr+7SPvbee2+eeOKJwhRIRKTI0jcs\nUjeyadOmUhdBRKSkyvJOKjubRbvLSVGf1IKR7LJJOGn5fyDSkVQGqdyh41oPI7d5MxxyCLzySrA8\nbx6cfnphjz979mwmT57cYt3ll1/O9OnT+fjjj7n44osZOnQow4cP59prr20a6+7uu+9mwoQJXHHF\nFQwcOJDrrruOlStXUlVVxa677soee+zB1KlTm/ZZUVHBqlWrANi2bRvf+973GDlyJLvttlvTgLQA\n8+fP54ADDmDAgAF85Stf4fXXX2+z3HV1dUyfPp0999yTYcOG8d3vfpft27cD8OSTTzJ8+HBmz57N\nkCFDuOiiiwp70qSo6uthwgTIZILlTCZYVqCS1MlnpsQoL2A18BfgJeD5drbpaCbHFu6/3/3884PZ\n4Ovq3M880/3hh1tuc9997p/9rPvMmcH7smUtP3/5Zff165uXn3jCPWdy3U7V1NT4zjvv7Js3b3Z3\n9/r6eh8yZIg/99xzfuqpp/q3vvUt37p1q2/YsMGPOOIInzt3rru733XXXd6zZ0+//fbbvb6+3rdu\n3epTp071G264wd3da2tr/Zlnnmk6TkVFha9cudLd3adNm+ZHH320r1u3zhsaGvyPf/yj19XV+YoV\nK3znnXf2RYsW+Y4dO3z27Nk+evToptmCR44c6YsWLXJ392uvvdbHjRvnGzdu9I0bN/qRRx7pP/zh\nD93dPZPJeM+ePX3GjBleV1fn27Zta7Pu7V0rSZ7Fi90HDnSfNSt4X7y4xAWSbo2kTh8PrAJ262Sb\njirVwubN7kcf7X7OOe6nneb+1a+6b9366e+edFJQu/vv//RnN9zgfsABQaC66y73oUPdV69uswjt\nOuqoo/zee+91d/eFCxf66NGjff369d6nT58Wv+DnzZvnRx99tLsHQWrEiBEt9nPeeef5N7/5TX/7\n7bc/dQwz85UrV3pDQ4NXVlb6stbR1t2vv/56P+uss5qWGxoafM899/Qnn3zS3VsGqVGjRvkjjzzS\ntO2jjz7qe++9t7sHQapPnz5eV1fXYb0VpNJl1qzg/8GsWaUuiXR3+QapYjw4YRSwWXHnneHBB4Pp\noAC2bAkm4c01bx688ALMnAnTpwcT9x5wQPPnV10VzDo/eHAwFdXSpZAzg3soU6dOZd68eXz9619n\n3rx5nH322dTU1LB9+3aGDBkCNP8BsNdeezV9b/jw4S32c9NNN3HNNddw+OGHM2DAAK644gouvPDC\nFtts3LiR2tpa9tlnn0+Vo/X082bG8OHDWbNmTZvb5pZlxIgRrF27tml50KBB9Gprbi5JpUwGbr89\nmKj69tuDuT9bz/8pgfr6lpN3t16W0ilGTsqBx8zsBTP7Rld3tn07fOMbMHEiHHkkTJvWsp1982a4\n8UZ47LFgwt5bboFrrmm5DzMYNSpbOIcBA6KXY/LkyWQyGdasWcODDz7IOeecw/Dhw9lpp5147733\nmqaO//DDD1m6dGnOsVtOp7LHHnswd+5c1qxZwy9+8QumTZvWlIdqNHDgQHbaaacWM/o2aj39PMBb\nb73FsGHDOt22pqaGoTkTQ7Y1vb2kU309zJgBDzwA1dXB+4wZykm1Rfm7hMvn9ivKCxiSfR8EvAxM\naGObjm4PW7j33uYmvsamvwcfbLnNjh0dLzc28S1f7n7ttUHT37vvtlmEDp144ol+7LHH+qGHHtq0\n7pRTTvHLL7/cP/74Y29oaPCVK1c2Nb3dddddftRRR7XYxwMPPNDU1PfKK6943759/c0333T35uY+\nd/dLL73UjznmGF+7dq3X19e3yEn169fPn3jiCd++fbvfdNNNPmrUqDZzUtdcc42PHz/eN2zY4Bs2\nbPAJEya0yEkNHz680zq3d60keTr7fyDNlL+LH0lt7nP3ddn3DWb2IHA48HTr7aqrq5t+rqqqoqqd\ndolzzoGzzmqeMf6RR6B375bbtL5Nb708aBAsWgSf/zxcdx3stRf06ROlVoGzzz6b888/n5tuuqlp\n3T333MOVV17J2LFj2bx5M/vssw9XXnllu/t44YUXmp4KHDx4MHPmzGHkyJFAyzubm2++mauvvprD\nDjuMLVu28IUvfIFHH32U/fbbj1/96ldcdtllrF27loMPPpiHH36Ynj17fmof11xzDZs2beKggw7C\nzDjzzDOZOXNm9IpLKnT2/0CaVVXBpZcGvw9mzVKzaCFkMhkyjbenXRDr9PFm1heocPfNZrYzsBC4\nzt0XttrO2yqHpipPHl0TCStNeZ5MBiZPDgLV7bcHzaOFCFRpOgdxS+r08YOBp83sJeBPwMOtA5SI\nlJ805Xniyt+l6RwkWax3UqELoTup1NA1kbDiujuJQ1x3PGk6B3FL6p2UiHRTuXmeSy9N9i/nuPJ3\naToHSaUgJZIiaRqPr3U/rQLk0BMhyjUo13NQTApSIimRphxHufbTinINyvUcFJtyUhKJrklppSnH\nUa5PtkW5BuV6DvKhnJRIN5CmHEe59tOKcg3K9RwUU6KD1IgRIzAzvRL0GhF1kEMpqDhyHHV1HS/n\nK035syTkmdJ0vooqn2EqCv1CQ+2IdGrHDvcvfal5yJ7Fi4Plrgx3VFvr3q+f+623Bsu33hos19Ym\nr6xxiVLWuOqVpvOVL5I6VUeoQihIiYQSx3h8t97qbuY+cWLw3hiwuipN4+FFKWtcYyKm6XzlQ0FK\nRPI2cWLw22DixMLuN03zWSWhrEkoQ1zyDVKJzkmJSPxuuQWeeiqY/uapp4LlQih17iYJeaYoklCG\nRMonshX6he6kRErik0/cKyrcL700WL700mD5k0+6tt9S526SkGeKIglliBt53kklup+UiMTvscfg\n7LOb+/3cdx8ce2zX91vq8fDS1p8pCWWIk/pJiUhejj22Zb+fQgQoKP14eGnrz5SEMiSRgpSUpTj6\nnETZZxL6vITt/xQlFxJXveLIHynHUybyaSMs9AvlpKSA4mjfT1uOI2z/pyTUK44yJOEaSEvoEXSR\nZnH0OYmyzyT0eQnb/ylKv5+46hVHP6W4+jNJfhSkRFqJo89JlH0moc9LHP2f4qpXEs6XxEdBSiRH\nHH+Zl+udVBRxjcwQx/mKcnzddcVPQUokq9Q5jiTkQ+IYky+uc6AcYvegICWSI46/4tP2l3nrgNTV\nQWPd47s7iuN8pe3Ot9wpSIl0gfIh4UQNJmHPa1xBPW05xHKWb5BSPynp9tSfJpyo09eHPa9R9xtW\nlOuqfwMJlk9kK/QL3UlJiSgfEU2UptEo57XQzW3KSSUPGrtPJD/lPmZaoVVXB0MNzZoV/NyeqOc1\n7H7DinJ8/RuIn8buE8lT2DHTogzdU65TskdpFosyFl0czW1Rjq9x85JLQUokhCh5k7o62H13uO22\nYPm224LlrgaquHI3UY4/Y0Ywmnh1dfA+Y0bXjx/XfqU8qLlPJKQoUz/cdhtccQV8+cvNEwlOn17c\nMsQhrmYxNbeVv3yb+xSkRCKIkjepqoInnwxmvC3k02KFzt2IFINyUiIxW7SoZd5k0aL2t40yJXva\npjkXKap8Hgks9As9gi4JF2WYoShTsutRaekuyPMR9J4ljpEiqdC7Nzz0EEyZAh9+GNzFPPRQsL61\nykp45JFgSvaBA+G//itYrqz89LY9esCNN346z9RWPqZHD3j66ebPqqpaLouUo6I095lZhZktMbP5\nxTieSBwmTWo5HfmkSe1vG2VK9rRNcy5STMXKSV0OvFakY4lEEsc061G2jZLriiJsrqvUfa9EOtRe\nOyAwP8Trrs7aE4FhwGNAFTC/nW1ibw8VaUupp1mPY0qNKGVQnkuKhUKPgg78FZjYwasKeLXTA8AD\nwMHZ7yhISeLEMc16lG0ff7zluHWPPx6m1J0LOx6epqmQYsg3SHX04MRMd3+yo7swM7uuk8//EVjv\n7i+bWRXQ7jPy1TkdPqqqqqgqZg9F6damTw8egmjs09Rep9u4htnJzXXNmtVxriuK3FzXrFnt57rC\nbicSRSaTIVOIPhL5RLawL+AG4O/AKmAdsBm4p43tYordknRJmBwwjjupKHQnJd0BcU16CHwReBBY\nAiwFlgFLIx9IzX3SShLyIXHkpOI4flTKSUnS5BukOh0WycxWAN/PBqeGnDuwmih3bGY2Efieu5/c\nxmfeWTmkPJV6LDoInubL7e/UerlRXGVdtCjof9W43/vvL0yTX9jx8DRunhRDbGP3mdnT7j4h75KF\nKYSCVLeWprHo4iprms6BSD7iHLtvlpn9h5lNNbPTGl95lFHkU5IwFl3YfkJxlTUJ50AksTprDwR+\nBfwZuBu4M/v6ZT5tix0co7CNn5IKSciHlDp3k4RzIFIMxJmTcvfPxRko1dzXfSUhHxI216S5lETy\nF2dz37NmNjaPMol0Kglj0YUdOy+usibhHIgkVZg7qeXAKOBNoJagQ667+0EFK4TupKSE4nq6TkSa\n5XsnFWaqjhPyKI9IKtTVwSmnwPXXByNN7LprsPzee20/hi4ixRUmSA0hGKNvE4CZ9QfGAJH6SYkk\nUZR5okSk+MLkpH5OMJxRo83ZdSJlIco8USJSXGGCVIuEkbs3EO4OTCQVwvZT0rxLIsUXJkitMrPv\nmFmv7OtyggFjRVKvvh5mzAgeO6+uDt5nzGg7IE2Y0BzAMplgWYFKJF5hnu7bA5gDfAVwYBEw3d3f\nLVgh9HSflFDYfkpJGGdQJK1iG7uvGBSkJC00xp5IfgremdfM/jnEQTvdRspHd8/JRBljr7ufK5FC\n6SgndVXugLJtvE4HLi9WQaW0untOJmzuqnHb7nyuRAqp3eY+M7szxPc/cvd2JtuOUAg196VCd8/J\nRBljr7ufK5HWlJOSolBOJjydK5FmcQ4wKwJo3qModK5ECkN3UhJKY57lxhuDZqtMJsjJPP20Ru1u\nTedK5NPU3Cex07xH4elcibQUW5Aysz7A6cBIcoZDcvcfRT1YB8dQkBIRKWNxTtXxO+Aj4EWC+aSk\njET5i193ByJSbGEenBjm7me5+2x3/7+Nr9hLJrGL0p9HfX9EpBTC3Ek9a2YHuvuy2EsjRdWjR5Dc\nb92fp627oyjbiogUSrtBysyWEQwo2xO40MxWEdP08VI6VVXNcynNmtVxh9Mo24qIFEJHI06M6OiL\n7l6wmXn14ETpRBkZQaMoiEi+4ny67153P7ezdV2hIFUaUfrzqO+PiHRFnEFqibsfmrPcA1jm7mOj\nF7PdYyhIlYie7hORYohjqo4ZZrYJOMjMPs6+NgHvEjyWLmWgdZDpKOhE2VZEpBDC3End6O4zYi2E\n7qRERMpanM19h7ax+iOgxt13RD1gO8dQkBIRKWNxBqk/AYcCSwkePz8QeAXYBfiWuy/s4Lt9gKeA\n3gSPsv/G3a9rYzsFKRGRMhbnVB1rgUPc/Yvu/g/AwcAq4FhgdkdfdPda4Gh3PyT7vRPN7PCohZR0\n0dTpIlIoYYLUfu7+auOCu78GfN7dV4U5gLt/kv2xD8HdlG6ZypiGTxKRQgozLNKrZvZz4P7s8lnA\na9mmvO2dfdnMKggGpx0F3O7uL+RbWEk+DZ8kIoUUJkhdAEwDpmeXnwH+D0GAOrqzL7t7A3CImfUH\nHjKzsdm7sRaqc+bXrqqqokpDGaSWhk8SkUwmQ6YAU1IXddJDM7sW2OLut7RarwcnyoiGTxKR1mJ7\ncMLMxpvZY2b2hpmtanyFLNRAM9sl+3MlwcMWr0ctpKRHfX0wXNIDD0B1dfA+Y4ZyUiKSnzCPoL8O\nfJcgr9T0q8bd3+t052YHAncTBMMK4L/c/V/a2E53UmVEwyeJSGtx9pN6zt2PyLtkYQqhICUiUtbi\nDFI/AXoAvyVn+nh3XxL1YB0cQ0FKRKSMxRmkFrex2t39K1EP1sExFKRERMpYbEGqGBSkRETKW5xP\n9w02szvMbEF2eayZXZxPIUVERKIIMyzSXcCjwNDs8hs0d+wVERGJTZggNdDdfw00AGSn51CvFxER\niV2YILXFzHYnOzCsmX2JYD4pERGRWIUZu+8KYD4wysyeAQYBZ8RaKhEREUI+3WdmPYHPEUx6uMLd\nOx39PFIh9HSfiEhZK/gj6GZ2WkdfdPffRj1Yu4VQkBIRKWv5BqmOmvtO6uAzJxiBQkREJDbqzCsi\nIrGLrTOviIhIqShIiYhIYilIiYhIYrX74EQxn+4TERFpi57uExGRxNLTfSIiErs4+knl7vwfgf2B\nnRrXufuPoh5MREQkijDzSf0COAv4NsGwSJOBETGXS0REJNT08Uvd/aCc937AAnc/qmCFUHOfiEhZ\ni7Mz79bs+ydmNhTYDgyJeiAREZGowuSk/sfMdgVuApYQPNn3H7GWSkREhHDNfX3cvbbxZ4KHJ7Y1\nritIIdTcJyJS1uJs7vtj4w/uXuvuH+WuExERiUtHI058FtgTqDSzQwie7APoD/QtQtlERKSb6ygn\ndTxwATAMuCVn/cfA1TGWSUREBAiXkzrd3f871kIoJyUiUtbizEk9Y2Z3mNmC7IHGmtnFkUsoIiIS\nUZggdSfwKDA0u/wGMD22EomIiGSFCVID3f3XQAOAu+8A6sPs3MyGmdkTZvaqmS0zs+90oawiItLN\nhOnMu8XMdifoxIuZfQn4KOT+dwBXuPvL2eGUXjSzhe7+en7FFRGR7iRMkLoCmA+MMrNngEHAGWF2\n7u7vAO9kf95sZssJHmtXkBIRkU6Fmk/KzHoCnyPoK7XC3bdHPpDZSCADHODum1t9pqf7RETKWGzz\nSZnZTsA0YAJBk98fzOwX7r4tQuH6Ab8BLm8doERERNoTprnvHmAT8LPs8tnAvQTzSnUqexf2G+Be\nd/9de9tVV1c3/VxVVUVVVVWY3Ut7NmyA1ath5EgYNKj7HV9ESiqTyZDJZLq8nzCdeV9z97Gdrevg\n+/cAG939ig62UXNfIc2bBxdfDL17Q10d3HEHTJ3afY4vIomTb3NfmCD1K+Df3P1P2eUjgEvd/bwQ\nhRoPPAUsI2gqdOBqd3+k1XYKUoWyYQOMGAFbtzavq6yEmpri3NGU+vgikkix5aSAfwCeNbO/Z5f3\nAlaY2TLA3f2g9r7o7s8APaIWSrpg9ergDiY3SPTqFawvRpAo9fFFpKyECVInxF4KKZyRI4Mmtlzb\ntwfru8PxRaSsdDrihLvXdPQqRiElgkGDghxQZSX07x+833FH8e5iSn18ESkrofpJxV4I5aQKr9RP\n15X6+CKSKLE9OFEMClIiIuUtzqk6RKLbsAFeeCF4L5Xly+Huu4P3UkjCORBJOQUpKbx584LH0I89\nNnifN6/4Zfj2t2HsWLjgguD9298u7vGTcA5EyoCa+6SwktBPavnyIDC19tprMGZM/MdPwjkQSRg1\n90kyNPaTytXYT6pYnn8+2vpCS8I5ECkTClJpEiXHEVc+5plnYNas4L0tSegndfjh0dYXWhLOgUiZ\nUJBKiyg5jrjyMccdBxMmwI9+FLwff/yntxk0KBi3L9fFFxe3mWvMGLjsspbrLrusOE19oL5iIgWk\nnFQaRMlxxJWPeeaZIDC19vTTMH58fmWN2/LlQRPf4YcXL0DlUl8xkSbKSZWzKDmOuPIxCxeGW5+k\nfMyYMXD++aUJUBAEpsMOU4AS6QIFqVILk2eKkuPIJx/TWZ4Jgqa+MOvzycdEyZ8lIS8XlvpJiXSd\nu5f8FRSjG7rvPvfKSvdddgne77uv82379+9828suc4fm12WXtb/tsce23Pa449rf9sADW2574IFt\nb3fAAeG2i1rWKOcryn7jEKWsIt1A9vd85PignFSp5JO7iZLjCJOPCZtnilLeKPuMkj9LQl4urCTl\n5UQSQjmptMkndxMlxxEmHxM2zwThyxtln1HyZ0nIy4WVpLycSMopSEVRyBxD3H1pwuRjwuaZIHx5\no+wzSv4s7rxcmq6tSHeSTxthoV+kIScVR44hSj4oiij5mLB5pijlHT685XbDh7e/z+OOC38O4srL\nxXFto5RVpBtAOakYxZFjiCtvUuo8T1zHz/1OIfNyceaP1E9KpIlyUnGKI8cQV96k1HmeuI7fqNB5\nuTjzR+onJdJlClJh5JNj6CzHEVfepNR5nriOn49C90ETkeLLp42w0C/SkJOKkj8Km+NIQh+hKLmT\nsPmjuI4fRZr6VIl0AygnFaM48yxx5U2ijFsXJncSR72iHD+KKGVVnyaRosg3J9UzjsKUnY7yLK1/\nATfmOHJ/6TXmONr6pTdmTOe/xKPuM+x+Gw0a1Pkv5DjqFeX4UUQpaz7nVkSKRjmpMEqdZ0lC3iQJ\nZQgrSlmTUi+N8yfSJgWpMKLMTxTHXEJJmJ8oCfNEhRXlfCXh3EaZK0ykm1FOKopS5lni2meUY6ct\ndxPlfJXq3KbxvIrkQTmpYihlniWufYaVxtxNlPNVqnObxvMqUkRq7lMuIJyk5G7Kjc6rSIe6d5BS\nLiC8JORuypHOq0iHYs1JmdkdwD8B6939oA62K35OSrmA/Gg8unjovEqZS2pO6k7gZ8A9MR8nOuUC\n8lPKvFhrk4mhAAAJHUlEQVQ503kVaVOszX3u/jTwQZzHyFtScgHKiYmItKv75qSSkAtQTkxEpEOx\n95MysxHAw4nLSTVS/xgRkdglNScVWnV1ddPPVVVVVFVVFefA6h8jIlJwmUyGTCbT5f0U405qJMGd\n1IEdbJOOEScKSXdSItKNJHJmXjO7D3gW2M/M/m5mF8Z5vFRJQk5MRCThNHZfqal/jIh0A/neSSlI\niYhI7BLZ3CciItIVClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIi\nIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJY\nClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIiIpJYClIi\nIpJYClIiIpJYsQcpMzvBzF43szfM7Mq4jyciIuUj1iBlZhXAvwHHA/sDU83s83EeM0kymUypixAL\n1StdVK90Kdd65SvuO6nDgb+6e427bwfuB74W8zETo1z/sale6aJ6pUu51itfcQepPYG3cpbfzq4T\nERHplB6cEBGRxDJ3j2/nZl8Cqt39hOzyVYC7+09bbRdfIUREJBHc3aJ+J+4g1QNYAUwC1gHPA1Pd\nfXlsBxURkbLRM86du3u9mV0GLCRoWrxDAUpERMKK9U5KRESkK4r64ISZVZjZEjOb387nc8zsr2b2\nspkdXMyydUVH9TKziWb2YfbzJWZ2TSnKGJWZrTazv5jZS2b2fDvbpO56dVavFF+vXczsATNbbmav\nmtkRbWyTxuvVYb3SeL3MbL/sv78l2fePzOw7bWyXqusVpl75XK9Ym/vacDnwGtC/9QdmdiIwyt33\nzf5D/AXwpSKXL1/t1ivrKXc/uYjlKYQGoMrdP2jrwxRfrw7rlZXG6/WvwO/dfbKZ9QT65n6Y4uvV\nYb2yUnW93P0N4BBoGvDgbeDB3G3SeL3C1Csr0vUq2p2UmQ0Dvgr8RzubfA24B8DdnwN2MbPBRSpe\n3kLUCyDyEy0JYHT87yOV14vO69W4TWqYWX/gKHe/E8Ddd7j7x602S931ClkvSNn1auUYYKW7v9Vq\nfequVyvt1QsiXq9iNvfdCnwfaC8J1rrj7xrS0fG3s3oBjMvesv+vmY0tUrm6yoHHzOwFM/tGG5+n\n9Xp1Vi9I3/XaG9hoZndmm1Dmmlllq23SeL3C1AvSd71ynQXMa2N9Gq9XrvbqBRGvV1GClJn9I7De\n3V8miKJp/sunSch6vQjs5e4HE4xj+FARi9gV4939UIK7xEvNbEKpC1QgndUrjderJ3AocHu2bp8A\nV5W2SAURpl5pvF4AmFkv4GTggVKXpZA6qVfk61WsO6nxwMlmtooguh5tZve02mYNMDxneVh2XZJ1\nWi933+zun2R/XgD0MrMBxS9qNO6+Lvu+gaBd+fBWm6TxenVar5Rer7eBt9z9z9nl3xD8cs+VxuvV\nab1Ser0anQi8mP232Foar1ejduuVz/UqSpBy96vdfS933weYAjzh7ue12mw+cB40jVTxobuvL0b5\n8hWmXrntyGZ2OMFj/+8XuaiRmFlfM+uX/Xln4DjglVabpe56halXGq9X9ry/ZWb7ZVdNIniQJ1fq\nrleYeqXxeuWYSvtNYqm7XjnarVc+16vYT/e1YGbfJBgmaa67/97MvmpmfwO2ABeWsmxdkVsv4Awz\n+xawHdhK0FabdIOBBy0Yrqon8J/uvrAMrlen9SKd1wvgO8B/ZptaVgEXlsH1gk7qRUqvl5n1JXi4\n4J9z1qX+enVWL/K4XurMKyIiiaVR0EVEJLEUpEREJLEUpEREJLEUpEREJLEUpEREJLEUpEREJLEU\npERCyE4x8HDY9QU43tfM7PM5y4vNrPUoEm2V5UMz+58CHH+n7HQL21I0goOUIQUpkfDa61QYR2fD\nU4D98/jeU+7+T109uLtvc/dDgLVd3ZdIVyhISVnIDnn0P9m//pea2eTs+kPNLJMd9XxB47As2TuT\n23K2/2J2/WFm9qyZvWhmT5vZvhHLcIeZ/Sn7/ZOy6883s//OHn+Fmf005zsXZ9f9KTvK98/MbBzB\nAJ2zs6N/75Pd/Ewze87MXjez8SHLdGW2fi+Z2Q05db8le05eNbMvZsu3wsyub72LsPUXiUNJh0US\nKaATgDWNdxFm9hkLJsn7GXCyu79nZmcCNwAXZ79T6e6HmNlRwJ3AgcByYIK7N5jZJOBG4IyQZZgJ\nLHL3i81sF+B5M3s8+9kXgIMJhoNZYWZzCCZgvCa7fjOwGHjZ3f9owSzPD7v7b7P1Aejh7kdYMCFe\nNXBsR4UxsxOAk4DD3L3WzHbN+bjW3Q+zYObU3xFMVvchsNLMbulkUkiRolGQknKxDLjZzG4E/tfd\nnzaz/YEDCOaPapzsMLf5ah6Au/8hG9T6E8yufE/2DqpxjL+wjgNOMrPvZ5d7A3tlf17k7psBzOxV\nYAQwCMi4+0fZ9Q8AHd25/Tb7/mL2+505BrjT3WsB3P3DnM/mZ9+XAa+4+7vZMqwkGH1bQUoSQUFK\nyoK7/zX7YMFXgevNbBHBXDWvuHt7TWOtc0kOXE8wmv1pZjaC4O4mLANOd/e/tlgZjGJdm7Oqgeb/\ne1Ga0xr3UU/X/+827quBlmWLGphFYqWclJQFMxsCbHX3+4CbCeYdWgEMygYJzKyntZwJ9Kzs+gnA\nR+6+CdiF5nl7oo48/SjBqN2NZTq4k+1fAL5sZrtkmyZPz/lsE8FdXXvCBLfHCEYNr8yWZ7cQ3xFJ\nFAUpKRcHEuSAXgJ+CPzY3bcT5JN+amYvAy8B43K+s83MlgD/D7gou2428BMze5Ho/z+uJ5jEbamZ\nvQL8qJ3tHMDd1xLkyJ4H/gC8CXyU3eZ+4PvZBzD2oe27vg65+6MEzXp/ztbzeyG+q2kRJFE0VYd0\nS2a2GPieuy8pcTl2dvctZtaDYKbgO9z9d3nuayLwf9z9pAKW703gH1I0kaCUGd1JSXeVlL/OqrN3\nf8uAVfkGqKw6YP9CduYFehDkrURKQndSIiKSWLqTEhGRxFKQEhGRxFKQEhGRxFKQEhGRxFKQEhGR\nxFKQEhGRxPr/hYKrtRphSmEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x272faefb780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# select setosa and versicolor\n",
    "y = df.iloc[0:100, 4].values\n",
    "y = np.where(y == 'Iris-setosa', -1, 1)\n",
    "\n",
    "# extract sepal length and petal length\n",
    "X = df.iloc[0:100, [0, 2]].values\n",
    "\n",
    "# plot data\n",
    "plt.scatter(X[:50, 0], X[:50, 1],\n",
    "            color='red', marker='o', label='setosa')\n",
    "plt.scatter(X[50:100, 0], X[50:100, 1],\n",
    "            color='blue', marker='x', label='versicolor')\n",
    "\n",
    "plt.xlabel('sepal length [cm]')\n",
    "plt.ylabel('petal length [cm]')\n",
    "plt.legend(loc='upper left')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "ppn = Perceptron(eta=0.1, n_iter=10)\n",
    "\n",
    "ppn = ppn.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "def plot_decision_regions(X, y, classifier, resolution=0.01):\n",
    "\n",
    "    # setup marker generator and color map\n",
    "    markers = ('s', 'x', 'o', '^', 'v')\n",
    "    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')\n",
    "    cmap = ListedColormap(colors[:len(np.unique(y))])\n",
    "\n",
    "    # plot the decision surface\n",
    "    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1\n",
    "    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1\n",
    "    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),\n",
    "                           np.arange(x2_min, x2_max, resolution))\n",
    "    Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)\n",
    "    Z = Z.reshape(xx1.shape)\n",
    "    plt.contourf(xx1, xx2, Z, alpha=0.4, cmap=cmap)\n",
    "    plt.xlim(xx1.min(), xx1.max())\n",
    "    plt.ylim(xx2.min(), xx2.max())\n",
    "\n",
    "    # plot class samples\n",
    "    for idx, cl in enumerate(np.unique(y)):\n",
    "        plt.scatter(x=X[y == cl, 0], y=X[y == cl, 1],\n",
    "                    alpha=0.8, c=cmap(idx),\n",
    "                    marker=markers[idx], label=cl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "## Question (50 points)\n",
    "\n",
    "As shown in function <font color='blue'>plot_decision_regions</font>, the decision regions can be visualized by dense sampling via meshgrid.\n",
    "However, if the grid resolution is not enough, as artificially set below, the boundary will appear inaccurate.\n",
    "\n",
    "Implement function <font color='blue'>plot_decision_boundary</font> below to analytically compute and plot the decision boundary.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "def plot_decision_boundary(X, y, classifier):    \n",
    "    \n",
    "    # replace the two lines below with your code\n",
    "    x1_interval = [X[:, 0].min() - 1, X[:, 0].max() + 1]\n",
    "    x2_interval = [X[:, 1].min() - 1, X[:, 1].max() + 1]\n",
    "    \n",
    "    plt.plot(x1_interval, x2_interval, color='green', linewidth=4, label='boundary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAEbCAYAAABgLnslAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VGX2wPHvmUkvE7p0sACKDbHBKghiRxCxANnV1WXX\n/QliZ1WQJvbeQNeydoogiqirWEBFZbGhIApKF9FQM+nJzLy/P2YCSWaSuUmm53yehyeZOTNzzwyQ\nk3vPW8QYg1JKKRWLbNFOQCmllKqNFimllFIxS4uUUkqpmKVFSimlVMzSIqWUUipmaZFSSikVs8Je\npEQkR0TmiciPIvKDiJwY7mMqpZRKDEkROMYjwDvGmItEJAnIiMAxlVJKJQAJ52ReEXEA3xpjDg7b\nQZRSSiWscF/uOxDYKSLPicg3IvKUiKSH+ZhKKaUSRLiLVBLQG5hhjOkNFAM3h/mYSimlEkS4e1K/\nAluNMV/5bs8Hbqr5IBHRBQSVUqqJM8ZIzfvCWqSMMX+IyFYR6W6MWQcMAtYEeuy//534dWrRoqkM\nGTI12mkkDP08Q0s/z9DSzzOwDeXLmZU/hq2ub6sHpgZ+fCRG910NvCIiycAG4PIIHFMppVQMKfTs\n5HXnLSwreaZezwt7kTLGfAccH+7jKKWUij0e4+Gzkmd53XkzRWa3XzxVshiadRvzuD7g8yNxJqV8\nuncfEO0UEop+nqGln2do6ecJWyq+YVb+GDZW/C9g/Pi0kVzoeIBm9va1FqmwzpOySkRMU+hJKaVU\nU1Ds2cvCglv5uPgJDB6/+AH2HozKmcFhqYP23ffPf0rkB0401qRJXcnL2xztNJRPmzZdmD59U7TT\nUErFKGMMy0te4rWC8RR48vziKZLB4KxJnJZ5PUmSYuk1Y7pI5eVtJhbO9JSXiN8vOUopBcC2ilXM\nco7ll/JPA8Z7pZ7PiJyHaWHvXK/XjekipZRSKraVegpYVDiVj4oewYPbL97afjAjHY9xRNrZDXp9\nLVJKKaXqzRjDV6WvMt95PXs9v/nFk0jlrKxbOCvrJpIlrcHH0SKllFKqXn53rWVO/lX8WP5BwPgR\nqWcz0vEYrZMav7a4bnrYQAceeCAfffRRVHOYNm0al1xySVRzUEo1HWWeIl53TuC2HUcGLFAt7J25\nsvnrXNX87ZAUKNAzqbingxmUUuFmjOG7soXMdV7DbvcWv7idZE7PvJFzsiaSassM6bG1SDVRHo8H\nm01PpJVSddvh2sAc5zhWl70TMN4j5VRyc2bQNunQsBw/Ln9KyTQJ+Z+GWLFiBYcffjgtW7Zk9OjR\nlJeXA/D000/TrVs3WrVqxbBhw9i+fTsAmzdvxmaz4fHsn9w2cOBA/vOf/wDwwgsv0K9fP8aPH0+L\nFi04+OCDeffdd/c9dtOmTQwYMICcnBzOPPNMdu7cWS2fiy++mHbt2tG8eXMGDBjAmjX71/K9/PLL\nGTNmDIMHDyY7O5sHH3yQtm3bVhviv2DBAnr16tWgz0IplVgqTCmLCqYxdUfPgAUqx9aOvzebzXUt\nPghbgYI4LVKxYtasWbz//vusX7+etWvXcvvtt7NkyRImTJjA/Pnz2b59O507d2bkyJH7nhPs8tyK\nFSs47LDD2LVrF+PHj2f06NH7Yrm5uRx//PHs3LmTW2+9lRdeeKHac8855xzWr19PXl4evXv35s9/\n/nO1+OzZs5k0aRIFBQWMGzeOVq1asXjx4n3xl19+mcsuu6wRn4hSKhGsLv0v03YcwVuFU3FRVi1m\nw86gzOuY1vonjk8fGfaWg17ua4Rx48bRvn17ACZOnMi4ceP47bffGD16NEcffTQAd911F82bN2fL\nFv/ruIF06dKFv/3tbwD89a9/ZcyYMeTl5VFWVsZXX33Fhx9+SHJyMv369WPIkCHVnlu1wEyePJmH\nH36YgoICsrOzATjvvPPo06cPAKmpqVx66aW89NJLnHnmmezevZv33nuPJ554olGfiVIqfu12b2Fu\n/rWsLHs9YPyQ5JPJzZlJh+QjI5aTFqlG6Nix477vu3Tpwm+//cb27ds59thj992fmZlJy5Yt2bZt\n276CVpe2bdvu+z49PR2AwsJCduzYQfPmzffdV3nMX3/9FfD2mCrP4Hbu3ImIICLs3LlzX5Hq1KlT\ntWP95S9/oWfPnpSUlPDqq6/Sv39/DjjggAZ8EkqpeOYy5XxQ9CBvF06n3BT7xbNtrbkg+376pF8S\n8cFacVmkzJTYWCpp69at+77fsmULHTp0oH379mzatGnf/UVFRezatYuOHTvuKzDFxcVkZWUB8Pvv\nv1s6Vrt27dizZw8lJSX7XmfLli37Bj+88sorLFq0iI8++ojOnTuTn59P8+bNq/Wcav7jat++PX37\n9uW1117j5ZdfZsyYMfX/EJRSce2nso+YnT+W390/+cUEoX/GlZyXfTuZtuZRyE57Uo0yY8YMtm3b\nxu7du7njjjsYOXIkI0eO5Pnnn+f777+nrKyMCRMm0KdPHzp16kSrVq3o0KEDL7/8Mh6Ph//85z+s\nX7/e0rE6d+7Mcccdx5QpU6ioqGDZsmUsWrRoX7ywsJDU1FSaN29OUVERt9xyi6XfeC655BLuvfde\nVq9ezfDhwxv8WSil4ste9288s2cUD+0eFLBAdU0+gVtafUluzoyoFSjQItVgIkJubi5nnHEGhxxy\nCN26dWPixIkMGjSI6dOnM3z4cDp06MDGjRuZM2fOvuc9/fTT3HvvvbRq1Yoff/yRk046KehxKr3y\nyissX76cli1bMn36dP7617/ui1166aV07tyZDh06cMQRR/CnP/3J0vs4//zz2bx5M8OHDyctreFL\nlyil4oPbuPig6GGm7DiUL0vn+MUzpQV/zvk3N7X8gi7JxwZ4hciK6f2kfPuLRCGjpuWQQw7hqaee\n4tRTT63zcSKC7vulVPz6pfwzZueP4VfX9wHjJ6WPZrjjbrJsrSKcWZzuJ6XC77XXXsNmswUtUEqp\n+OV057Gg4F98UfJCwHinpF6MypnJwSl9I5xZcFqkmrCBAwfy448/8vLLL0c7FaVUGHiMm0+K/83C\ngokUm71+8TRxcF727ZyScSV2ic1yEJtZqYhYsmRJtFNQSoXJpvIvecV5JVsqvg4YPzH9L1yQfR85\n9rYB47FCi5RSSiWQIs9u3iiYwKfFT2Hw7yG3TzqcUY4ZdE89JQrZ1Z8WKaWUSgAe4+GLkudZUHAT\nhZ6dfvFUyeTcrKkMyrwGuyRHIcOG0SKllFJxbmvFSmblj2FDxRcB48emXcRFjgdpbu8YMB7LtEgp\npVScKvHks7BgEkuLZ2Dw+MXb2LsxKudxeqaeEYXsQkOLlFJKxRljDCtKZzHfeQNOzx9+8WTSOSdr\nIqdn3UiypEYhw9DRIqWUUnHkt4o1zHaOZV350oDxo1OHcrHjEVoldY1oXuGiyyKFyA8//MBZZ51F\n69atsdvt0U5HKZVgSj2FvOb8F9N3Hh2wQLW0d2Vs80WMabEwYQoUaJEKmeTkZEaMGLFvl12l4oXH\nU/dtFV3GGL4umc/UHYexuOg+PLiqxZNIYXDWJKa2XsNRaedGKcvwScjLfacefzzOvLxq9znatOGj\nL78M2zG7d+9O9+7dLa9qrlQs8HjgxRdh4EDo0gU2b4YlS+DSS8Gmv8JG3R+un5mTfxVryhcHjPdM\nPZORjsc4IKlbhDOLnLAXKRHZBOQDHqDCGHNCY1+zZhGqWYCceXl81bp1teccV6NoKaW8hWjgQFiw\nAI49Fr7+GoYP1wIVbeWmmP8W3sXiwntxUe4Xb27ryMWOhzkmbXjENyGMtEicSXmAAcaYPaF6wZpF\nqCEFKFihU6qp6NLFW6A+/RT69fPeVtHzXeki5jqvZpd7k1/MRhKnZV7P4KxJpNmyIp9cFETi9yWJ\n0HHqpbLQVf6peXkwmFmzZpGdnY3D4WDw4MFhylKp8Nu82XsG1a+f9+vmzdHOKHxiuf+207WRGbuH\nMnPP0IAFqnvKACa1+o4LHPc0mQIFkSkeBnhfRL4UkX9E4HgRkZubS0FBAU6nk7fffjva6SjVIB6P\ntwc1fDj07+/9umRJbP3wDpXK/ltlEd682Xs72u+1wpTxdsHtTN3Rk+/LFvnFHba2jG72Cte3+Ij2\nyT2jkGF0ReJy30nGmO0i0hpvsfrRGLMsnAd0tGnjdwnQ0aZNOA8JQFlZGWVlZRhjKCsrQ0RISUkJ\n+3GVaiibrfogiS5dEnfQRCz239aULWZ2/lXkuX/2iwk2BmaMY2j2NNJtOVHILjaEvUgZY7b7vu4Q\nkdeBEwC/IrVo0dR933fvPoAePQbU+po1i1DNAmSltxTsNepr8+bNHHjggYgIIkJ6ejpdu3Zlw4YN\njXpdpcKt5g/pRCxQlWKl/7bH/SuvOq/jm9L5AeMHJfclN2cmnZJ7RTizyFm7dinr1i0N+riwbh8v\nIhmAzRhTKCKZwGJgmjFmcY3H6fbxcUC3j1cN5fFUL341b0fK5s3+Z1L1LVSNeS9uU8EHRQ/zduE0\nykyRXzzL1orh2ffSN/2v2CSBf1sIoLbt48P9KRwALBORb4HlwKKaBUopldhipRcUiv5bY97LurKP\nmb6zFwsK/uVXoAShf8b/cVvrtZyUcXmTK1B1CevlPmPMRiBxz1eVUkHFSi8oFP23hryXfPfvzHfe\nyIrSVwLGOycfy58dT9A15fh6vJumIyFXnFBKxZZY6QWFov9m9b24jYuPi2eysGASpcbpF8+QZgzL\nvpN+GVdgE13vszZapJRKcLHQD6o5F6tLl9icNGzls7LyXtaXf8Hs/DFsda0MeJy+6ZcxPPseHPbw\njzqOd3rhU6kEFgv9oHiZi2Xlswr2Xgo9O3lx72ju3fWngAWqY9JRjG+5jMuaPacFyqKwju6znISO\n7osLOrovPoViRFtjxcLZnBVWPqtA7wXxsKz4aV4vuIXiACvApUk2Q7OnMyBjLHbRC1iB1Da6Tz8t\npRJcLPSD4mUulpXPqmbuW91fMyv/SjZVBJ6feUJaLhc67ifH3i4MGSc+LVJKJbhQ9INcLkhKqv12\nMLFyJhUsj/p8VkWePSwsmMgnxU9i8L/C0NZ+KKNyZtA9+dSYLcrxQD86pRJYKPpBLhdMnQrLl3tv\nL1/uve1y1fWs6jlEuy9mJQ+rn5Uxhi+KX2DKjh58XPyEX4GyuTIY6LqbSa2/I/33U2NifcB4pmdS\nDTRjxgyef/55Vq1aRW5uru7Iq2JSKOYGJSXBuefC3Lnwww+wejWMGGH9TCqW5knVlYeVz+rXiu+Z\nnT+WXyoCLz/aO+0C+jof5MMFnfk8RtYHjHcJWaQicWmhQ4cOTJo0iffee4+SkpLQvrhSIRSKflCf\nPt4CtXIl9OrlvV0fsdAXs5JHbZ9VicfJosIpLCl6DA9uv9dtbT+YkY7HOSLtLGgOe2PgvSaKuKzv\nVQf81Rz8V1wMublQuYv7e+/B+PGhz2HYsGEMHTqUFi1ahP7FlYoxX3zhPYPq1cv79Ysv6vf8UOxZ\nFWwvKCt7RdU3D2MMK0pmM2XHoXxY9LBfgUomjSFZ05jSerW3QDXgGKpucVekFi+GadO8/wBdLpgw\nwfsbS6WMDLj8chgzBmbOhAcf9H5f1bp1sHv3/ttffQVu/1+OlFJAeTnMng2DBsHo0d6vs2d777ci\nEmvmhWKOU03bK37kod2n8ezeXPI92/3iR6YOZkrrHzg3ezLJkhay96qqi7t5UiUlcN110Lo1lJV5\n/9x3H9Tctun66+GTT+DOO+GMM6rHnnsO3n0XnngCPv8cZsyA//wH2jVghOikSZPYtm1bk+hJ6Typ\npmvDBli4cH8v57zz4KCDrD8/FJfgg81haugcp5p5lHmKeKfwdt4vegA3FX55tLR34WLHIxydOhQR\nv2k9MTOSMd4kzDyp9HS4/34YMMB7e9ky/wL13nve6+d/+xs88AAcfLD3T6XLLvMWtzPO8DZ/58xp\nWIFSqqk46KDqvZz6FCiIzJp5DZnjVPW2MYaVZW/wqvNadru3+D3XTjJnZI7nnOyJpEhGrXnGy5yw\neBF3H5/LBbffDr17w1FHwd13Vz+VLi72ninNnOm9zHf99d7vqxKBjh293xsDDkfk8leJw0oPpLGv\nEYpjBFNzKHmgoeXB+iyR+CyC5dCYXlCe6xce3zOYJ/cMD1igDks5jcmtVzHMcUedBUqFXtwVqcWL\nvZf8Hn/ce5nu99+9l/UqZWTArFn7z5zOPNN7ObCqt97yPnf+fG//6sorYY//SiZ1crvdlJaW4na7\ncblclJWV4dbGVpMRirk/oeizNJaVOVDB+iyR+iyC5dCQXlC5KeHNgilM23EEq8v+6xdvZmvPP5rN\n5ZoWi2mb1MP6G1IhE3c9KWO8gxwq52hUVEBycv2O99ln0KEDdO3qfb2FC+H00yEz0/prTJs2jWnT\nplW7Jj1lyhQmT55cv2TiiPakqgvFmnih6LM01vLl3jlQRxyxfw5UzSHmVlZqCPdnESyH+vaCVpW+\nwxznOHa6N/jFbNgZlHkt52ZNIc2WXb83ohqktp5U3BUpFT1apPx98sn+Hkj//uF5jVAcI5hnn90/\nB2r06Ia9RiQ+i1DY5drMq85rWVn2RsD4ISn9yHXMpEPyEeFJQAUUre3jlUpYoejThLPPYtXy5dXn\nQFVe+quPSHwWwQQ7hsuU807BXUzZcVjAApVta8PlOS9yY4uPtUDFEC1SSjVAKPo04eqz1IfL5e3R\njhjhPYMaMcJ72+q6fFbfR6jnMAXKoa5j/Fj2AbftOIqFhROooPoKMYKNARlXcVvrtfTJuCTgsHIV\nPXq5T1mml/uqC0WfJtR9loZo7ArngfIKx2cRTKBjODpuY77zBr4qnRvwOe09J3J5m5l0Tu5t/UAq\nLBJmnpRSsSLYfJjGztuxcjsUahak+hYoKyKxdl/VY5zUr4J1LR9j0Y4plJlCv8faSltwfP49XHb0\n37CJXlCKZfq3o1SYNJU13KxczrPSs2rsMPbKY/Q4/VMWtO7N/IIbAhaonPX/YPi2deQv+Ttbt+iP\nwFinZ1JKhUHVHkvlxnlLltR/m4x4EGwLDCufRWO38/B44L3P/iD5on/xvv3FgI/plHQMOcueYGjv\nE+nSBbp3Sty/k0SiPSllmfak6qepreFW1/Bxq59FQ4age4ybj4ufZGHBREpMvl88XXI4L/sOTsn4\nPzD2JvV3Ek/isifVpk0XHWkTQ9q00Y1x6sNKPynYD+9IDGoIhWDbrlv5LBqyzf3G8v8xK38MW1zf\nBIz3Sb+UC7LvxWE/wHtHjR8nWqBiX0wXqenTN0U7BaXCprIPM3Cg94fx5s3VLz9VLll07rneFSCW\nL/cOD5861XqhCnaMUL2Pxl7arO9rFHp28brzFj4recZv+3aA9klHMMoxg+6pYZoRrCImpi/3KZXo\ngg3NtrJkUWOPEQqhOFuz8hoe4+Hzkv+wwHkzRWaX32ukShZDsqZxauY47FLP9dJUVMXl5T6lEl2w\nodmN3bbdyjFCIRRD5YO9xpaKb5mVP4aNFYGXxDgubQQXOh6gub1D/Q+uYpZekVUqijZurN6H2bix\nejzYtu2RWG4o2oo9e5mdP447dx4XsEAdYO/BtS3e5x/N52iBSkBapJSKEpfLu7Br377ekWx9+3pv\nVy5JFGzb9kgsNxRNxhiWF7/ElB2HsrT4cQzVk04mnWHZdzKp9XcclnpalLJU4aY9KaWiaNMmeP31\n/f2i88/3biFTKdi27ZFYbigatlWsZrZzLD+XfxIw3it1GBc5HqJVUtfIJqbCJqo9KRGxAV8Bvxpj\nhkbimErFg65dq/eLqhYoCL5teyiWXoolpZ4C3iqcxodFD+PBfxPRVvaDGOl4lCPTBkchOxUNkRo4\ncQ2wBtCN2lWTYWWOU7C5QcHimzZVj3fu7F/ogrE0qi7MZ2PGGL4uncc85/Xs9WzziyeRyllZN3Nm\n1k2kSHroDqxiXq3/zETkTQt/ng92ABHpCJwDPBPCvJWKaaHalr2uuMsFzzxTvaf1zDP132bDSl8r\nnNvY/+5ayyO7z+TpvSMCFqjDU89iSuvVDMmeqgWqCaq1JyUiPwN/r+u5wAxjzOF1HkBkHnAHkAPc\nEOhyn/akVCIKxbbswW5v3AhvvLG/JzVsGBx4YP3ytNLXCsdcq3JTzDsFd7C46D7cVPjFm9s6cbHj\nYY5JO19XnmkCGtKTmmiM+biuFxWRaUHig4E/jDErRWQAfouS7Ldo0dR933fvPoAePQbU9dJKxTwr\nc5wau1XHgQdW70nVt0CBtb5WqOdafVf6JnOdV7PL7T8e3kYSp2fewOCsSaTaMht3IBWz1q5dyrp1\nS4M+Lqyj+0TkTuAvgAtIB7KBBcaYS2s8Ts+kVERFYsRbKM6kgom3M6kdrg3MdV7DqrK3AsZ7pJzK\nKMfjtEs+rP4vruJabWdSQf87iMhxIvK6iHwjIt+LyCoR+d7KQY0xE4wxnY0xBwEjgY9qFiilIi3c\nPRawti17Y/MINs/KCivzqEIx16rClPJWwW1M23F4wAKVY2vH35vN5roWH2iBUtUEPZMSkbXAeGAV\n7J9NZ4yp17x1ETkF7UmpGBGJ9eysju5rTB7B5llZEe7RfatL32Wucxx57l/8YjbsDMwcx5CsaaTb\ndPBvU9aYeVI7jDFvNjYBX3+rzh6XUpESifXsrGzL3tg8gs2zssLKPKqGzLXa7d7Kq85r+bZ0QcD4\nwcknkZszk47JR1nMVDVFVorUFBF5BvgQKKu80xgT+F+eUnGgIXsX1ZeVs4/G5hGJ91FfLlPOB0UP\n8XbhbZSbYr94tq01w7PvpU/6pdgkhmcWq5hgpUhdDhwKJLP/cp8BtEipuBSJrd2t7OPU2DxicYv6\ntWVLmO0cy3bXj34xQeif8X+cl30HmbbmUchOxSNLPSljTI+wJqE9KRVhkdqtNtzr6sXKunz57u3M\nc97Al6WzA8a7Jh/PKMdMuqYcF+HMVLxoTE/qcxHpaYxZE4a8lIqKSKxnF4l19aK9Lp/buFhS/DiL\nCiZTagr84hnSnPOz7+LkjL9jE3tkk1MJwUqR6gOsFJGNeHtSAhhjjHY7lapDKNbVi2W/lH/G7Pwx\n/OoKPCPlpPS/cX723WTbW0c4M5VIrBSps8KehVIJpnJdvXPP9U7gTUnx3p46NfAov3hS4N7BgoKb\n+LzkuYDxjklHk5szk4NT/hThzFQisvLfpR3wgzHec3kRcQCHAXG2v6dSkZOU5J3E+8Yb3k0Kv/7a\nezueC5THuPm0+GneKJhAsdnjF08TB+dlT+eUjDHYJY7fqIopVv4lPQH0rnK7MMB9SqkaQrGuXqzY\nVP4ls5xj2FzxVcD4CWl/5kLHfeTY20U4M5XorBQpMVWGABpjPCL6a5JSwViZwxQro/NqU+TZzRsF\nE/m0+N8Y/EfgtkvqySjHDHqkDoh8cqpJsPLfYYOIXC0iyb4/1wAbwp2YUvHM6pp44V5DsKE8xsNn\nxc8xeUcPPil+0q9ApUomw7PvZVKrlVqgVFhZmSfVBngUOBXvJN4PgWuNMXkhS0LnSakEZHXFiXCv\nIVhfWyu+Y3b+GNZXfB4w3jvtQi5yPEgLe6cIZ6YSWYPnSfmK0ciwZKVUArMyhykSawhaVeLJ582C\nKSwpfgyD/+lcG3s3RuY8xuGpZ0YhO9VU1bV9/BXBnmzlMUqFWs3LYbFweayhavatNtcYMxuJ92qM\nYUXJLCbvOJSPih/xK1DJpDE0azqTW6/SAqUirq4zqZtFZGcdcQGuAZ4KbUpK1c7KmnjxItjae5F4\nr79VrGGO8yrWli8JGD8qdQgjHI/QKimOhyaquFZrT0pEAs/Uqy7fGHNto5PQnpSqh1js4zRUsL5V\nuN5rqaeQtwun80HRg3jw3yWxpb0rIxyPcnTakMYfTCkL6t2TMsZcHt6UlGqYWOrjNFawvlWo36sx\nhm9LF/Cq81r2eH71iyeRwhlZ/+LsrFtIkYzGHUypEND5TiruxOIeSuESyvf6h+tn5jjHsabsvYDx\nnilnMDLnMQ5I6t6IjJUKLS1SKq7E4h5K4RKq91puSni38C7eK7wHF+V+8Wa2DlzseJjeaRcg4ne1\nRamoCjpPKiJJaE9K1UOsr9IQSo19r9+XvsVc59XsdG/0i9lI4rTM6xicNZk0W1YIslWq4Ro8T0pE\nUoELgK5VH2+MuS2UCSplVbT3UIqkhr7Xna5NvOq8hu/K3gwY755yCqMcM2iffHgjM1QqvKxc7lsI\n5ANf491PSqmwCnb20JTOpOqrwpTxfuH9vFN4BxWU+MUdtgO40PEAJ6Tl6qU9FResFKmOxhjdU0pF\nRLC5QYk0TyrU1pS9z5z8q/jDvc4vJtgYmHEVQ7NvI92WE4XslGoYq9vHH2mMWRX2bFSTZ7N5C1DN\nuUGVBShYvCna4/6Vec7r+bp0XsD4Qcl9yc2ZSafkXhHOTKnGq7VIicgqvAvKJgGXi8gGdPt4FQHB\n5gYl0jypxnCbCj4seoS3CqdSZor84pnSkuGOe/hT+uXYpAlXcRXX6jqTOjdiWShVRbC5QU1pnlRt\n1pV9wmznGH5z/eAXE4R+GVdwXvYdZNlaRiE7pUKnrhUnNgOIyEvGmEuqxkTkJeCSgE9UqhGsrGfX\nVOZJBZLv/p3XCsbzv5KXA8Y7Jx9LrmMmB6acEOHMlAoPK/tJfWOM6V3lth1YZYzpGbIkdJ6UqkJH\n9/lzGxcfFz/BwoJbKTVOv3iGNGNY9p30y7gCm9ijkKFSjVPveVIicgswAUgXkcr/FQKUoyufqzAK\nNjeoKc2TAthQvpxZ+Vey1bUyYLxv+l8Znn0vDnubCGemVN0++QR+/hnyGrFFbl2X++4C7hKRu4wx\ntzT8EEqphij07GSB82Y+K3k2YLxD0pHk5szkkJSTI5yZUnWrLE4A3VhHN/Lo32Ztnc/556bA91sZ\ngj5PRHrXuC8f2GyM8V/jXynVKB7jYVnJM7zhvIUis9svnibZDMmaxsDMcdhFl99UsaHqWVObNt7i\n1J9PuYKnwMJJ/j9rud/Kv/CZQG/ge7yX+44EVgM5InKlMWZxbU/0Lan0CZDiO9Z8Y8w0C8dUqkna\nXPE1s/JCHNe6AAAfZUlEQVTHsKliRcD48WkjudDxAM3s7SOcmUpEn3wSmtfZV5z4g5P4mf55n3JF\nmze8wW7doH//4C+yIvC/eStF6jdgtDHmBwAR6QncBvwLWADUWqSMMWUiMtAYU+wbcPGZiPzXGBM4\nG6VCJN4GVxR59rCw4FY+KX4Cg/8gorb2QxmZ8ziHpQ6KQnYq0dQ862m0PF9x2nfm1MZ6cQrCSpHq\nXlmgAIwxa0TkUGPMBitrfxljin3fpvqOp8P4VFjF09JJxhiWl7zIawXjKfDs8IunSAaDsyZxWub1\nJElKFDJUiaRacao866HuXpE1eVWK00khKU6VrBSpH0TkCWCO7/YIYI3vUl5FsCeLiA3v4rQHAzOM\nMV82NFmlrIiXpZO2VaxiVv4YfqlYFjDeK/V8RuQ8TAt75whnphJN4OLkO+ux0jAKpg0hL06VrBSp\ny4AxwLW+258BN+ItUAODPdkY4wGOEREH8IaI9DTGrKn5uEWLpu77vnv3AfToMcBCakoFFstLJ5V6\nClhUOJWPih7Bg9sv3tp+MCMdj3FE2tlRyE7FirD1i6pdkgtPYbFi6dq1LF3nvxhyTRHd9FBEJgFF\nxpgHa9yvk3lVSG3e7H8mFe1CZYzhq9K5zHfewF7Pb37xJFI5K+sWzsq6iWRJi0KGKhaEo1/ULUz9\nolCSf/6zwZsengRMBbpQfdPDgyw8txVQYYzJF5F04HTg7nrkrVS9xeLSSb+7fmJ2/lX8VP5hwPgR\nqecw0vEorZMOjnBmKlbEY78oEqwsi/QTcB3evtK+axPGmF1BX1zkSOAFwOb7M9cYc0eAx+mZlAqp\nWBndV+Yp4p3CO3i/6H7cAVq4LeydGeF4hKNTz9NNCJuomsXJ76wnFGLwzKmmBp9JAfnGmP825KC+\nPahqTgRWKuyivXSSMYbvyhYy13kNu91b/OJ2kjk980bOyZpIqi0zssmpsLPaT/JblSFG+kWxxEqR\nWiIi9+GdE7Vv+3hjzDdhy0qpOLbDtZ45zqtZXfZOwPihKYMYlfM4bZMOjXBmKtzq208KuCqDFqdq\nrBSpE31fj6tynwFODX06SsWvClPKu4X38G7hXbj2/z63TzNbey50PMhxaRfrpb0E06B+Ul5e/Vdl\naIKCFiljTNBh5ko1datL/8sc5zh2uNf7xWzYOTXzGoZkTSXNlh2F7FS4NGr+URu0OFlgZXTfAcCd\nQHtjzNm+ZZH6GmMCL82sVBOy272FufnXsrLs9YDxQ1L6keuYQYfkIyOcmapNKOcfgfaTws3K5b7n\ngeeAib7b64C5gBYp1WS5TDkfFD3I24XTKd+38td+2bY2XJB9H33SL9FLezEi1POPqvWTyNPiFCZW\nilQrY8yrvk0QMca4RMR/mrxSTcRPZR8xO38sv7t/8osJNk7JuJLzsm8nw9YsCtmpqirPmqqe9Yxu\n82lIXvuKyr1fu3WD0aND8prKn5UiVSQiLfEtDCsiffDuJ6VUk7LX/RvznTfwZemcgPGuySfw55wn\n6Jyssy6irc69jUJF+0kRYaVIXQ+8CRwsIp8BrYELw5qVUjHEbVwsKXqMRYVTKDUFfvFMacH5jrs5\nKX00NomxVWzjSEztbaRihpXRfd+IyClAD7ybHq41xgRd/VypRPBL+TJm5Y9hm2tVwPjJ6X/nfMdd\nZNlaRTizxBFPexupyKu1SInI8FpC3UUEY8yCMOWkVNQ53XksKPgXX5S8EDDeKekYcnNmclBKnwhn\nljh0rTplRV1nUkPqiBm8K1AolVA8xs0nxf9mYcFEis1ev3i65HBe9u2cknElNrFHIcP4F897G6nI\nq7VIGWMuj2QiSkXbxvIVzHKOYUvF1wHjfdIv4YLs+3DYD4hwZrGv5ii6usTi3kYqdlkZOKFUQiv0\n7OKNggksK34ag/9q/O2TDmeUYybdU/UHaE2BRtEFoxNfVX1okVJNlsd4+LzkORY4b6IowM4zqZLJ\nuVlTGZR5DXZJjkKGsSvgJbuqo+jqpBNflXVapFSTtLViJbPyx7Ch4ouA8WPTLuYixwM0t3eMcGax\nrc5+UuUoumB0pJ2qh4aM7gPQ0X0qLpV48llYMImlxTMwePziB9i7MzLncXqmnh6F7MKjsrCEgvaT\nVKTp6D7VJBhj+F/JK7xWcCNOzx9+8WTSOSf7Vk7PvIFkSY1ChqHnt+Nrm8YvFKP9JBVpOrpPJbzf\nKn5gtnMs68o/Dhg/OnUoFzseoVVS18gmFia1X5Kz0i8KRvtJKrIs9aREZDBwOJBWeZ8x5rZwJaVU\nKJR6CnmrcBofFj2MB5dfvJX9QEY4HuWotHOjkF3ohaRfFIz2k1SEWdlP6kkgAxgIPIN33b4VYc5L\nqQYzxvBN6XxedV7HXs82v3gSKZyZdTNnZd1MiqRHIcP9QtkvAt3bSCUeK2dSfzLGHCUi3xtjponI\nA8B/w52YUg3xh2sdc/LHsaZ8ccD44alnMdLxGG2SDolwZtVVK055oekX7S9Mvju0OKkEYKVIlfi+\nFotIe2AX0C58KSlVf+WmmHcK7+T9wvtwUe4Xb27ryMWORzgm7fyobUIYaG+j/nzKFW10+wilamOl\nSL0lIs2A+4Bv8I7seyasWSlVD9+Vvslc5zXscm/yi9lI4rTM6xmcNYk0W1bkkyPwKLtqexuFolcE\nWpxUQrJSpO41xpQBr4nIW3gHT5SGNy2lgtvp2sgc59WsKnsrYLx7ygBGOWbQPrlnWI5vtZ/kN5Ah\nT7ePUMoqK0XqC6A3gK9YlYnIN5X3KRVpFaaMxYX38d/CO6gI8PuSw9aWixwPcHzaqLBc2qvv/CMd\nyKBUw9W14kRboAOQLiLH4N3wEMCBd7SfUhH3Q9l7zMm/ijz3L34xwcbAjHEMzZ5Gui0n5MeueubU\njXWMbvMp5OVZmH+kc4uUaqi6zqTOBC4DOgIPVrnfCUwIY05K+dnt3so853V8U/pawPjByX9iVM5M\nOiUfHdLjBlrlu1o/6aRuQJCekl7WU6rB6lpx4gXgBRG5wBgT+CeDUmHmNhV8UPQwbxdOo8wU+cWz\nbK0Ynn0vfdP/ik1sQBjXqqu6yrcWHqUiwkpP6jMReRZob4w5W0R6An2NMc+GOTfVxK0tW8ps51i2\nu9b4xQShX8Y/GZZ9B5m2FkCk1qrT4qRUJFkpUs/5/kz03V4HzAW0SKmwyHf/znznjawofSVgvEvy\nceQ6ZtI15XhA16pTKpFZKVKtjDGvisgtAMYYl4i4rby4iHQEXgQOADzA08aYRxucrUpobuNiafEM\n3iyYTKlx+sUzpDnDsu+kX8Y/sIld16pTqgmwUqSKRKQl3km8iEgfwOp1FBdwvTFmpYhkAV+LyGJj\nzE8NS1fFu9r6RQXNvmDzEVdS7Pgu4PNabb2cTmvv4Zfy1lSO69O9jZRKfFaK1PXAm8DBIvIZ0Brv\nIrNBGWN+B373fV8oIj/iHdauRaqJqa1fVJqym5U97md9p8Bjc5o5e3D8D1Nos+dYYI/vj5fOP1Iq\n8QUtUsaYb0TkFKAH3rlSa40xFfU9kIh0BXoB/6vvc1X8qu2S3Ghe55lOO7ilx1b2pPhfPc6usDH9\n546M3ewgyTxcy6trv0ipRGdlq440YAxwMt5Lfp+KyJPGGMtLI/ku9c0HrjHGFDY0WRUZlQuhNlZt\n/aKvumXQ99g/+DJtV8Dn5Tq7cv/O3rRLToe6FisfPTo0iSqlYpaVy30vAgXAY77bucBLwEVWDiAi\nSXgL1EvGmIW1PW7Roqn7vu/efQA9egyw8vIqhGpOXG2s6qsyPMWe9i0Yc0IyTzb7EeNtcVZzWFI7\nZjhGMbBdD+95u1IqYS1du5al69YFfZwY4//DotoDRNYYY3oGu6+O578I7DTGXF/HY8y//113Hip8\n/PY24mf6t1nb6NetXJXB0+0QXszewL/a/8AOT4Hf4zIkhSlZ53Jt5iBSxNJm0UGdeuedOAv2H8uR\nnc1HE3ShFKVilfzznxhj/BbbtPIT4RsR6WOMWQ4gIicCX1k6qMhJwJ+BVSLyLd7LhROMMe9aT12F\nQ81RdtX2NsrLY//OeY3QrRvf9z2IMfmz+KxivXcSQg3Nfknm+z9No5O9ReOPV4WzoICvsvZvzXFc\ngX9xVErFPitF6ljgcxHZ4rvdGVgrIqsAY4w5qrYnGmM+A+yNT1NZZaWfVG2Une+sqdreRiHo9Tg9\nJUwpXMRjO2fjDlCdDslP4rHlzbl1rZtO/UJboJRSicNKkTor7FmoRqtXPymvxtwiQrfcjzGGOaVf\ncoNzPts9/tPpxAXTvsth/GoHaW7hVnQcjVKqdlaGoG+ORCKqYQIP8Q7WT8oLy9yiHyu2c5VzNh+V\nBz7+ualHsnPeLhZuL2Uh3sViHdnZITl2TY7s7GqX+MJ1HKVUeAUdOBGRJHTgRL1UXtKrurfRvrMi\nq0PzQrjcT5GnjOmFb/Ng0QdU4D/nqYu9JY86RjA0LbTbaCilEkdjBk6oEAnF/KOaqzZU29uoW7eI\nrjVnjOH10m+51vkqWz17/OLJ2BmfeQYTs88hQ1IikpNSKrFokQqzUO5tBFTvJ+VFb/uIX1x5jHPO\n4d2yHwLGT085jMdzRtE96YCI5qWUSixapMLEb626EM09Clc/yaoSU87dhe9yT+F7lOHyi3ewNeMh\nx8VcmNYbEb8z94iJxDypDuPGgavKZ5CUxLbHHqv9CQ2g871UU6dFKsTq3D4iFHOP2hC1tereLl3F\n1c45bHDv9IslYePazEFMzjqXbFtaxHOrKSLzpFwutiXt/y/UweVftBtL53uppk6LFOFfqy7eF0Hd\n7NrFNc65LCwLvI1G/5RuzHTkcnhy+whnppRKdE26SIV6rTq/+UdxXpzKTAUPFL3P7QXvUIL/wvdt\nbNncn30hf0k/MaqX9pRSiatJFqmGzS2yIrr9olD6oOxHxubPZp37D7+YDWFsxgBuyx5KM1tGtViw\nHkoo+jgt/+//SK4ydaJChF1PPlntMdsLC+mwd2+144RcUlL1S3xhOIbO91JNXZMqUoncLwqVbe49\nXO+cx6ulXweM90k+kJk5uRyT3DlgPGgPJQR9nGRj+L3KmVvbAHP92mVlVc+jMPQrW4R6kEQgOkhC\nNXVxX6Ss9pMStV8UKhXGzaNFHzG1cBGFpswv3lIyuccxnMvT/4RNbFHIUCnVFMVtkapvP6nm3kZa\nnPb7pGwdY52zWe36zS8mCH9PP4m7HOfT0pYV4NlKKRU+cVekGtpP2rcqw0ndAC1OAH+4nYwveI2X\nSpYHjPdO6szMnFxOTDlw333Bek5BeygW+jjBek5lVL/E53/eB+t376bt7t21PiZYb8zK/CSdJ6VU\n+MVNkWp0PykKqzLEKrfx8GTxx0wsWEi+KfGL50g6d2YP458Z/bHXuLQXrOcU7AeolR/iwXpOB7do\nEbTflAp1962C9MYszU/SeVJKhV3MFKm6ekvaTwqd5eUbGJM/i29dWwPGL03vw73ZF3CA3RHhzJRS\nyl/MFKm61rfTflLj7fIUcrPzdZ4pWRYwfkRSe2bm5NIvpVuEM1NKqdrFTJEazbO1xrSf1HAe4+E/\nJZ9zk3MBu02RXzxLUpmWNYRxmaeSLME3Ud64Z0+1Xk9FPSfxWunjBOs5Wckh2GuUGkPb8vJaX8PS\n/KQg/bVQ9JN0npRq6mKmSO0rRIFoP6lBvq3Ywpj8WSyv2Bgw3nxdCqtOnkYHe3PLr5kmwrbk5H23\n692HsdDHCdZzspJDhs1W53EObN68zmNYKSbB+muh6CfpIAnV1MVMkWL06GhnkDD2eoqZVLCQmcUf\n48F/omuPvUnM+KIFN/3sosMp1guUUkpFWuwUKdVoxhheKlnO+ILXyPP4/9YuFXDHdzlcv9pBqkeA\n0K/CoJRSoaRFKkGsrtjGWOdsPikPPAJlWGovljy8kkf25vMI+YB/H8ZSD6WxfRgL86SC9WGC9ZOs\nHCcUvZ5g7zUi6wcqleD0f02cK/CUMq3wLR4u+hA3Hr/4QfZWPOYYyTlpR9LBeSXbUho3N6ixfRgr\n86SC9WGC9ZOsHCcUvZ5g7zUS6wcqlei0SMUpYwzzSr/mOuc8fvPs9YunksTNWWdxU9aZpEtKFDJU\nSqnG0yIVh9a6fueq/Dl8UP5jwPjZqUfwqGMEhySFYpMspZSKHi1ScaTYlHNHwTvcV7SYCtx+8U62\n5jySM4Jhqb0Cb0IYgT5NJOb1xEqvJ9h7jcRnoWv7qUSnRSoOGGN4s+w7rnG+ymb3Lr94MnZuyDyd\nW7POIdOWWuvrRKJPE4kfkLHS6wn2XiPxWejafirRaZGKcRtcO7jaOZe3y1YFjJ+a0oMZObkcmtQ2\nwpkppVT4aZGKUaWmgnsL3+OuwncppcIv3s6Ww4OOixiRdlzgS3tKKZUAtEjFoHdLV3OVcw7r3Tv8\nYnZsXJ05kKlZQ3DY0vfd35R6E7qe3X76WahEF9YiJSLPAucCfxhjjgrnsRLBVvdurnW+yoLSbwPG\nT04+hBk5ozgquaNfrCn1JhK1+DaEfhYq0YX7TOo54DHgxTAfJ66VGxcPFX3AbYVvU2zK/eKtbdnc\nl30Bl6b30Ut7SqkmJaxFyhizTES6hPMY8W5J2VrGOmfzo2u7X0wQrszoz+3Z59HclhmF7JRSKrq0\nJxUl29353OCcx+zSLwPGT0juysycXI5NtlbjI9WbaEq9L6VU9GmRijCXcfN48RImFyyiwJT6xZtL\nBnc7zufv6SdjE5vl141UoWhKvS+lVPTFTJGaumjRvu8HdO/OgB49ophNeHxW/gtj8mfzvevXgPHR\n6Sdxt2M4rWxZAeNKKZUolq5dy9J164I+LhJFSnx/6jR1yJAIpBIdeW4nNxUs4PmSLwLGeyV1YmbO\nKPqmHBzhzJRSKjoG9OhR7WRk2ltvBXxcuIegzwIGAC1FZAswxRjzXDiPGUvcxsNTxZ8yoeAN9ppi\nv7hD0piefR5jMk4hSexRyLD+dF6OUiqSwj26Lzecrx/LvizfxBjnLL6q2Bww/pf0E7kv+wLa2nMi\nnFnj6CAJpVQkxUxPKlHs9hQxoeANnir+FIPxi/dMasdMRy6npHaPQnZKKRVftEiFiMd4eL7kC24q\nWMBOj/+q3JmSytSsc7kmcxDJcXJpTymlok2LVAisrNjK2PzZfF6xPmD8wrTePOS4mI725hHOTCml\n4psWqUbI95QwueBNHi9egifApb1u9jY8njOKM1J7RiE7pZSKf1qkGsAYw6zSFdzonM/vHqdfPI1k\nbs06hxuzTidVkqOQoVJKJQYtUvW0puI3xjpns7Q88CS0IalH8YhjBAcmtYpwZkoplXi0SFlU6Cnl\ntsK3eajoA1x4/OJd7S151DGCIWlHRyE7pZRKTFqkgjDGsKD0W651vsqvnj1+8RSS+FfWGdySdTYZ\nkhKFDJVSKnFpkarDz64/GOecw3tlawLGz0jpyeM5I+mWdECEM1NKqaZBi1QAJaacuwrf5Z7C9yjH\n5RfvYGvGw46LuSCtt25CqJRSYaRFqoZFpd9xtXMum9y7/GJJ2Lgu8zQmZw0my5YWheyUUqpp0SLl\ns8m1k2ucr/Jm2XcB46ekdGeGYxSHJ7ePcGZKKdV0NfkiVWYquL/wfe4ofIcSKvziB9gcPOC4kNy0\nE/TSnlJKRViTLlLvl63hqvw5rHP/4RezIVyVMZDbsoeSY0uPQnZKKaWaZJH61b2H653zmFf6dcB4\n3+SDmJmTS6/kThHOTCmlVFVNqkhVGDePFH3I1MK3KDJlfvFWtizuyR7OZel9sYktChkqpZSqqskU\nqY/L1jHWOZsfXL/5xQThiox+3Jk9jBa2zChkp5RSKpCEL1K/u/MZX/AaL5f8L2D82OTOPOH4M8en\ndI1sYkoppYJK2CLlMm6eKP6YWwsW4jSlfvFmksGd2cO4IqMfdr20p5RSMSkhi9QX5esZkz+bla6t\nAeOXpfflnuzhtLE7IpyZUkqp+kioIrXTU8jNzgU8W/JZwPhRSR2ZkTOKk1MOiXBmSimlGiIhipTH\neHimZBm3ON9gtynyi2dLGrdlD+GqjIEkiT0KGSqllGqIuC9SX1dsZkz+LFZUbAoYH5V2PPc7LqS9\nvVlkE1NKKdVocVuk9niKuLVgIU8Uf4LB+MUPtbdlRs4oTk09NArZKaWUCoW4K1LGGF4sWc74gtfY\n4Snwi2dICpOzBnNd5mmkSNy9PaWUUlXE1U/xVRXbGJM/i2UVvwSMD087hoccF9PZ3iLCmSmllAqH\nuChSTk8JUwsX8WjREtx4/OIH21vzuGMkZ6UdEYXslFJKhUtMFyljDHNLv+J65zy2e/L94qkkMSHr\nbP6VdSZpkhyFDJVSSoVTzBapn1y/MzZ/Fh+Vrw0YH5x6JI86RnBQUusIZ6aUUipSYq5IFXnKuL3w\nHR4oep8K3H7xLvaWPOK4mKGpR+smhEopleDCXqRE5CzgYcAGPGuMuSfQ44wxvFG2kmudr7LFvdsv\nnoyd8ZlnMDH7HDIkJbxJK6WUiglhXVlVRGzA48CZwOHAKBEJOHFp8J7HGb7nyYAFalDKoXzfejJ3\nOIbFdYFaujbwpUvVMPp5hpZ+nqGln2dohHv57xOAn40xm40xFcAc4LxAD/xv2Wq/+9rbmjGn2d95\nv8W1HJrUNryZRsDSdeuinUJC0c8ztPTzDC39PEMj3Jf7OgBVlyL/FW/hqpMdG9dmDmJK1rlk29LC\nlpxSSqnYFnMDJ/qlHMIMRy5HJneIdipKKaWiTIzxX/cuZC8u0geYaow5y3f7ZsDUHDwhIuFLQiml\nVFwwxvgN2Q53kbIDa4FBwHZgBTDKGPNj2A6qlFIqYYT1cp8xxi0iVwGL2T8EXQuUUkopS8J6JqWU\nUko1RriHoKsqRMQmIt+IyJvRziXeicgmEflORL4VkRXRzifeiUiOiMwTkR9F5AcROTHaOcUrEenu\n+3f5je9rvohcHe284lXMje5LcNcAawBHtBNJAB5ggDFmT7QTSRCPAO8YYy4SkSQgI9oJxStjzDrg\nGNi3oMGvwOtRTSqO6ZlUhIhIR+Ac4Jlo55IgBP33GxIi4gD6GWOeAzDGuIwxziinlShOA9YbY7YG\nfaQKSP+TR85DwHgIsNe9aggDvC8iX4rIP6KdTJw7ENgpIs/5LlE9JSLp0U4qQYwAZkc7iXimRSoC\nRGQw8IcxZiXeMwBdvr3xTjLG9MZ7djpWRE6OdkJxLAnoDczwfabFwM3RTSn+iUgyMBSYF+1c4pkW\nqcg4CRgqIhvw/lY1UERejHJOcc0Ys933dQfe6/1Bl9tStfoV2GqM+cp3ez7eoqUa52zga9+/UdVA\nWqQiwBgzwRjT2RhzEDAS+MgYc2m084pXIpIhIlm+7zOBMwD/FYqVJcaYP4CtItLdd9cgvAN8VOOM\nQi/1NZqO7lPx6ADgdd9yWknAK8aYxVHOKd5dDbziu0S1Abg8yvnENRHJwDto4opo5xLvdDKvUkqp\nmKWX+5RSSsUsLVJKKaVilhYppZRSMUuLlFJKqZilRUoppVTM0iKllFIqZmmRUqoBROQUEVlk9f4Q\nHO88ETm0yu0lIlLnqhC+XPaKyFshOH6ab9uJUhFp0djXU8oqLVJKNVxtkwzDMflwGHB4A573iTHm\n3MYe3BhTaow5Bvitsa+lVH1okVIJybd00lu+3/6/F5GLfPf3FpGlvtXT/ysiB/juXyIiD1d5/HG+\n+48Xkc9F5GsRWSYi3eqZw7Mistz3/CG++/8qIq/5jr9WRO6p8pzRvvuW+1Yjf0xE+uJdqPRe3yrl\nB/kefrGI/E9EfhKRkyzmdJPv/X0rIndWee8P+j6TH0TkOF9+a0Vkes2XsPr+lQoFXRZJJaqzgG2V\nZxEiku3bzO8xYKgxZpeIXAzcCYz2PSfdGHOMiPQDngOOBH4ETjbGeERkEHAXcKHFHCYCHxpjRotI\nDrBCRD7wxY4GegEVwFoReRTvRo63+u4vBJYAK40xX/h2c15kjFngez8AdmPMiSJyNjAVOL2uZETk\nLGAIcLwxpkxEmlUJlxljjvftILsQ76Z9e4H1IvKgbi6pokWLlEpUq4D7ReQu4G1jzDIRORw4Au8+\nVJWbJla9fDUbwBjzqa+oOfDuovyi7wyqcq1Aq84AhojIeN/tFKCz7/sPjTGFACLyA9AFaA0sNcbk\n++6fB9R15rbA9/Vr3/ODOQ14zhhTBmCM2Vsl9qbv6ypgtTEmz5fDeqAToEVKRYUWKZWQjDE/+wYW\nnANMF5EPgTfw/gCu7dJYzV6SAabjXbV+uIh0wXt2Y5UAFxhjfq52p0gfoKzKXR72/1+sz+W0ytdw\n0/j/y5Wv5aF6bvUtzEqFlPakVEISkXZAiTFmFnA/3v2R1gKtfUUCEUkSkZ5VnjbCd//JQL4xpgDI\nAbb54vVdGfw9vKuLV+bUK8jjvwT6i0iO79LkBVViBXjP6mpjpbi9D1xeueuuiDS38BylokqLlEpU\nR+LtAX0LTAZuN8ZU4O0n3SMiK4Fvgb5VnlMqIt8AM4G/+e67F7hbRL6m/v9fpgPJvoEKq4Hbanmc\nATDG/Ia3R7YC+BTYCOT7HjMHGO8bgHEQgc/66mSMeQ/vZb2vfO/zBgvP1W0SVFTpVh1K4R3hBtxg\njPkmynlkGmOKRMSOd8fhZ40xCxv4WqcANxpjhoQwv43AscaY3aF6TaXqomdSSnnFym9rU31nf6uA\nDQ0tUD7lwOGhnMwL2PH2rZSKCD2TUkopFbP0TEoppVTM0iKllFIqZmmRUkopFbO0SCmllIpZWqSU\nUkrFLC1SSimlYtb/A0mH89kx9mOYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x272f8469128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "low_res = 0.1 # intentional for this exercise\n",
    "plot_decision_regions(X, y, classifier=ppn, resolution=low_res)\n",
    "plot_decision_boundary(X, y, classifier=ppn)\n",
    "\n",
    "plt.xlabel('sepal length [cm]')\n",
    "plt.ylabel('petal length [cm]')\n",
    "plt.legend(loc='upper left')\n",
    "\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./perceptron_2.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Stochastic gradient descent\n",
    "\n",
    "Below is the AdalineSGD code.\n",
    "Notice that it does not compute costs for <font color='blue'>partial_fit</font>.\n",
    "We are going to do that for this exercise to study the behavior of stochastic gradient descent."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "from numpy.random import seed\n",
    "\n",
    "class AdalineSGD(object):\n",
    "    \"\"\"ADAptive LInear NEuron classifier.\n",
    "\n",
    "    Parameters\n",
    "    ------------\n",
    "    eta : float\n",
    "        Learning rate (between 0.0 and 1.0)\n",
    "    n_iter : int\n",
    "        Passes over the training dataset.\n",
    "\n",
    "    Attributes\n",
    "    -----------\n",
    "    w_ : 1d-array\n",
    "        Weights after fitting.\n",
    "    errors_ : list\n",
    "        Number of misclassifications in every epoch.\n",
    "    shuffle : bool (default: True)\n",
    "        Shuffles training data every epoch if True to prevent cycles.\n",
    "    random_state : int (default: None)\n",
    "        Set random state for shuffling and initializing the weights.\n",
    "        \n",
    "    \"\"\"\n",
    "    def __init__(self, eta=0.01, n_iter=10, shuffle=True, random_state=None):\n",
    "        self.eta = eta\n",
    "        self.n_iter = n_iter\n",
    "        self.w_initialized = False\n",
    "        self.shuffle = shuffle\n",
    "        if random_state:\n",
    "            seed(random_state)\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        \"\"\" Fit training data.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : {array-like}, shape = [n_samples, n_features]\n",
    "            Training vectors, where n_samples is the number of samples and\n",
    "            n_features is the number of features.\n",
    "        y : array-like, shape = [n_samples]\n",
    "            Target values.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        self : object\n",
    "\n",
    "        \"\"\"\n",
    "        self._initialize_weights(X.shape[1])\n",
    "        self.cost_ = []\n",
    "        for i in range(self.n_iter):\n",
    "            if self.shuffle:\n",
    "                X, y = self._shuffle(X, y)\n",
    "            cost = []\n",
    "            for xi, target in zip(X, y):\n",
    "                cost.append(self._update_weights(xi, target))\n",
    "            avg_cost = sum(cost) / len(y)\n",
    "            self.cost_.append(avg_cost)\n",
    "        return self\n",
    "\n",
    "    def partial_fit(self, X, y):\n",
    "        \"\"\"Fit training data without reinitializing the weights\"\"\"\n",
    "        if not self.w_initialized:\n",
    "            self._initialize_weights(X.shape[1])\n",
    "        if y.ravel().shape[0] > 1:\n",
    "            for xi, target in zip(X, y):\n",
    "                self._update_weights(xi, target)   \n",
    "        else:\n",
    "            self._update_weights(X, y)\n",
    "        return self\n",
    "\n",
    "    def _shuffle(self, X, y):\n",
    "        \"\"\"Shuffle training data\"\"\"\n",
    "        r = np.random.permutation(len(y))\n",
    "        return X[r], y[r]\n",
    "    \n",
    "    def _initialize_weights(self, m):\n",
    "        \"\"\"Initialize weights to zeros\"\"\"\n",
    "        self.w_ = np.zeros(1 + m)\n",
    "        self.w_initialized = True\n",
    "        \n",
    "    def _update_weights(self, xi, target):\n",
    "        \"\"\"Apply Adaline learning rule to update the weights\"\"\"\n",
    "        output = self.net_input(xi)\n",
    "        error = (target - output)\n",
    "        self.w_[1:] += self.eta * xi.dot(error)\n",
    "        self.w_[0] += self.eta * error\n",
    "        cost = 0.5 * error**2\n",
    "        return cost\n",
    "    \n",
    "    def net_input(self, X):\n",
    "        \"\"\"Calculate net input\"\"\"\n",
    "        return np.dot(X, self.w_[1:]) + self.w_[0]\n",
    "\n",
    "    def activation(self, X):\n",
    "        \"\"\"Compute linear activation\"\"\"\n",
    "        return self.net_input(X)\n",
    "\n",
    "    def predict(self, X):\n",
    "        \"\"\"Return class label after unit step\"\"\"\n",
    "        return np.where(self.activation(X) >= 0.0, 1, -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "## Question (50 points)\n",
    "\n",
    "Modify the code above so that we can visualize the costs with different batch sizes.\n",
    "Note that each batch size should have a separate plot for average cost (y axis) versus number of trained samples (x axis).\n",
    "What you find out from your experiments?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "# standardize features\n",
    "X_std = np.copy(X)\n",
    "X_std[:, 0] = (X[:, 0] - X[:, 0].mean()) / X[:, 0].std()\n",
    "X_std[:, 1] = (X[:, 1] - X[:, 1].mean()) / X[:, 1].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "iterations = 15 # i.e. epochs\n",
    "batch_sizes = [1, 5, 10, 20, 50, 100]\n",
    "\n",
    "for batch_size in batch_sizes:\n",
    "    ada = AdalineSGD(n_iter=0, eta=0.01, random_state=1)\n",
    "    for iteration in range(iterations):\n",
    "        num_batches = np.ceil(y.ravel().shape[0]/batch_size).astype(int)\n",
    "        for batch_index in range(num_batches):\n",
    "            start_index = batch_index*batch_size\n",
    "            end_index = start_index + batch_size\n",
    "            ada.partial_fit(X_std[start_index:end_index, :], y[start_index:end_index])\n",
    "    plt.plot([batch_size * x for x in range(1, len(ada.cost_) + 1)], ada.cost_, label=str(batch_size))\n",
    "\n",
    "if len(batch_sizes) > 0: \n",
    "    plt.xlabel('# trained samples')\n",
    "    plt.ylabel('Average Cost')\n",
    "    plt.legend(loc='best')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    # plt.savefig('./adaline_5.png', dpi=300)\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}