black_box.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Opening the Black Box"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook provides Python code that corresponds with the chapter in [Introduction to Machine Learning](https://m-clark.github.io/introduction-to-machine-learning/).  There will be some differences in the implementations, but results should be similar."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "wine = pd.read_csv('data/wine.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>color</th>\n",
       "      <th>white</th>\n",
       "      <th>fixed.acidity</th>\n",
       "      <th>volatile.acidity</th>\n",
       "      <th>citric.acid</th>\n",
       "      <th>residual.sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free.sulfur.dioxide</th>\n",
       "      <th>total.sulfur.dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "      <th>good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>red</td>\n",
       "      <td>0</td>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>5</td>\n",
       "      <td>Bad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>red</td>\n",
       "      <td>0</td>\n",
       "      <td>7.8</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0.098</td>\n",
       "      <td>25.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.9968</td>\n",
       "      <td>3.20</td>\n",
       "      <td>0.68</td>\n",
       "      <td>9.8</td>\n",
       "      <td>5</td>\n",
       "      <td>Bad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>red</td>\n",
       "      <td>0</td>\n",
       "      <td>7.8</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.04</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0.092</td>\n",
       "      <td>15.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>3.26</td>\n",
       "      <td>0.65</td>\n",
       "      <td>9.8</td>\n",
       "      <td>5</td>\n",
       "      <td>Bad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>red</td>\n",
       "      <td>0</td>\n",
       "      <td>11.2</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.56</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.075</td>\n",
       "      <td>17.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.9980</td>\n",
       "      <td>3.16</td>\n",
       "      <td>0.58</td>\n",
       "      <td>9.8</td>\n",
       "      <td>6</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>red</td>\n",
       "      <td>0</td>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>5</td>\n",
       "      <td>Bad</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  color  white  fixed.acidity  volatile.acidity  citric.acid  residual.sugar  \\\n",
       "0   red      0            7.4              0.70         0.00             1.9   \n",
       "1   red      0            7.8              0.88         0.00             2.6   \n",
       "2   red      0            7.8              0.76         0.04             2.3   \n",
       "3   red      0           11.2              0.28         0.56             1.9   \n",
       "4   red      0            7.4              0.70         0.00             1.9   \n",
       "\n",
       "   chlorides  free.sulfur.dioxide  total.sulfur.dioxide  density    pH  \\\n",
       "0      0.076                 11.0                  34.0   0.9978  3.51   \n",
       "1      0.098                 25.0                  67.0   0.9968  3.20   \n",
       "2      0.092                 15.0                  54.0   0.9970  3.26   \n",
       "3      0.075                 17.0                  60.0   0.9980  3.16   \n",
       "4      0.076                 11.0                  34.0   0.9978  3.51   \n",
       "\n",
       "   sulphates  alcohol  quality  good  \n",
       "0       0.56      9.4        5   Bad  \n",
       "1       0.68      9.8        5   Bad  \n",
       "2       0.65      9.8        5   Bad  \n",
       "3       0.58      9.8        6  Good  \n",
       "4       0.56      9.4        5   Bad  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wine.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>white</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>0.753886</td>\n",
       "      <td>0.430779</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fixed.acidity</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>7.215307</td>\n",
       "      <td>1.296434</td>\n",
       "      <td>3.80000</td>\n",
       "      <td>6.40000</td>\n",
       "      <td>7.00000</td>\n",
       "      <td>7.70000</td>\n",
       "      <td>15.90000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>volatile.acidity</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>0.339666</td>\n",
       "      <td>0.164636</td>\n",
       "      <td>0.08000</td>\n",
       "      <td>0.23000</td>\n",
       "      <td>0.29000</td>\n",
       "      <td>0.40000</td>\n",
       "      <td>1.58000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>citric.acid</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>0.318633</td>\n",
       "      <td>0.145318</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.25000</td>\n",
       "      <td>0.31000</td>\n",
       "      <td>0.39000</td>\n",
       "      <td>1.66000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>residual.sugar</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>5.443235</td>\n",
       "      <td>4.757804</td>\n",
       "      <td>0.60000</td>\n",
       "      <td>1.80000</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>8.10000</td>\n",
       "      <td>65.80000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chlorides</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>0.056034</td>\n",
       "      <td>0.035034</td>\n",
       "      <td>0.00900</td>\n",
       "      <td>0.03800</td>\n",
       "      <td>0.04700</td>\n",
       "      <td>0.06500</td>\n",
       "      <td>0.61100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>free.sulfur.dioxide</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>30.525319</td>\n",
       "      <td>17.749400</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>17.00000</td>\n",
       "      <td>29.00000</td>\n",
       "      <td>41.00000</td>\n",
       "      <td>289.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total.sulfur.dioxide</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>115.744574</td>\n",
       "      <td>56.521855</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>77.00000</td>\n",
       "      <td>118.00000</td>\n",
       "      <td>156.00000</td>\n",
       "      <td>440.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>density</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>0.994697</td>\n",
       "      <td>0.002999</td>\n",
       "      <td>0.98711</td>\n",
       "      <td>0.99234</td>\n",
       "      <td>0.99489</td>\n",
       "      <td>0.99699</td>\n",
       "      <td>1.03898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pH</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>3.218501</td>\n",
       "      <td>0.160787</td>\n",
       "      <td>2.72000</td>\n",
       "      <td>3.11000</td>\n",
       "      <td>3.21000</td>\n",
       "      <td>3.32000</td>\n",
       "      <td>4.01000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sulphates</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>0.531268</td>\n",
       "      <td>0.148806</td>\n",
       "      <td>0.22000</td>\n",
       "      <td>0.43000</td>\n",
       "      <td>0.51000</td>\n",
       "      <td>0.60000</td>\n",
       "      <td>2.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alcohol</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>10.491801</td>\n",
       "      <td>1.192712</td>\n",
       "      <td>8.00000</td>\n",
       "      <td>9.50000</td>\n",
       "      <td>10.30000</td>\n",
       "      <td>11.30000</td>\n",
       "      <td>14.90000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quality</th>\n",
       "      <td>6497.0</td>\n",
       "      <td>5.818378</td>\n",
       "      <td>0.873255</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>9.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       count        mean        std      min       25%  \\\n",
       "white                 6497.0    0.753886   0.430779  0.00000   1.00000   \n",
       "fixed.acidity         6497.0    7.215307   1.296434  3.80000   6.40000   \n",
       "volatile.acidity      6497.0    0.339666   0.164636  0.08000   0.23000   \n",
       "citric.acid           6497.0    0.318633   0.145318  0.00000   0.25000   \n",
       "residual.sugar        6497.0    5.443235   4.757804  0.60000   1.80000   \n",
       "chlorides             6497.0    0.056034   0.035034  0.00900   0.03800   \n",
       "free.sulfur.dioxide   6497.0   30.525319  17.749400  1.00000  17.00000   \n",
       "total.sulfur.dioxide  6497.0  115.744574  56.521855  6.00000  77.00000   \n",
       "density               6497.0    0.994697   0.002999  0.98711   0.99234   \n",
       "pH                    6497.0    3.218501   0.160787  2.72000   3.11000   \n",
       "sulphates             6497.0    0.531268   0.148806  0.22000   0.43000   \n",
       "alcohol               6497.0   10.491801   1.192712  8.00000   9.50000   \n",
       "quality               6497.0    5.818378   0.873255  3.00000   5.00000   \n",
       "\n",
       "                            50%        75%        max  \n",
       "white                   1.00000    1.00000    1.00000  \n",
       "fixed.acidity           7.00000    7.70000   15.90000  \n",
       "volatile.acidity        0.29000    0.40000    1.58000  \n",
       "citric.acid             0.31000    0.39000    1.66000  \n",
       "residual.sugar          3.00000    8.10000   65.80000  \n",
       "chlorides               0.04700    0.06500    0.61100  \n",
       "free.sulfur.dioxide    29.00000   41.00000  289.00000  \n",
       "total.sulfur.dioxide  118.00000  156.00000  440.00000  \n",
       "density                 0.99489    0.99699    1.03898  \n",
       "pH                      3.21000    3.32000    4.01000  \n",
       "sulphates               0.51000    0.60000    2.00000  \n",
       "alcohol                10.30000   11.30000   14.90000  \n",
       "quality                 6.00000    6.00000    9.00000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wine.describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a160ddef0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAFQCAYAAACyDHmtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzsnXmYHFXV/z/fhEgCWdjDJgQRyMuSBJKgIRACREREkF1EIIAiCrIIKr64RP3xCiIqiwIB2QIiAgIB2UMWCFsSskMAhajsOySsWc7vj3s7qenpma6q7p7p6Tmf56lnqm7de+pWd0+dutv5ysxwHMdxOi9d2rsCjuM4TvvijsBxHKeT447AcRynk+OOwHEcp5PjjsBxHKeT447AcRynk+OOwHEcp5PjjsBxHKeT447AcRynk7NKe1fAyYUvB3ec2qL2rkBb4o6gA/LWuBsqKr/WEYdy2rjxucufd8S+AHzxrEty27jnzOMBWDx5am4bPXcdXtFnsdYRhwJw0pV/z23jgqMPACr7TtY64tCqfKfn3TEpd/nT9hkJwBWTHs9t45iROwZbFf62lr76eu7yAKv0XZcP587PXb7HdttUdP2OiHcNOY7jdHLcETiO43Ry3BE4juN0ctwRVBFJi1tIP17SkXF/tKQN27ZmjuM4LeODxW2AmSVHVUcD84CX2qc2juM4TXFHkAFJPwQ+MrMLJP0eGGhmu0vaAzg65jkL2Af4ENjPzF6VNAZYDCwEhgDXSfoQGAZsDfwO6Am8AYw2s5fb9s4cx+nMeNdQNqYAu8T9IUBPSd2AnYEHgdWBR81sYMz7rWRhM7sJmA4cbmaDgKXAhcBBZjYYuAI4q9SFJR0nabqk6VdPvL/6d+Y4TqfFWwTZmAEMltQL+Bh4guAQdgFOAj4B7kjk/UIZe1sB2wL3SQLoCpRsDZjZWGAswFvjbvAFZY7jVA13BBkwsyWSFhK6gR4G5gC7AZsDTwFLbKUI9DLKf74C5pvZsNrU2HEcpzzeNZSdKcDp8e+DwPHArIQDKMcioFfcfxpYV9IwAEndJHW+ZY2O47Qr7giy8yCwAfCImb0KfBTT0nIVcImkWYSuoIOAcyTNBmYBO1W3uo7jOK3jXUMZMbMJQLfE8ZaJ/Z6J/ZuAm+L+mET6zcDNCZOzgBG1q7HjOE7reIvAcRynk+OOwHEcp5Oj9GOcTh3hX5rj1BbXI3Dqm0rivUOI+V6NOP7v3T0ht43ee+0BVB7H/+lX3shdfqv116lKHQBufGxObhsHf25ARToAELQAqnEf42c8mdvGvoO3Bir/PP/12tu5ywNsvt6a/Oetd3OX32StPhVdvyPiXUOO4zidHHcEjuM4nRx3BI7jOJ0cdwSO4zidnA7hCCSdJOkpSW9LOqNKNkuKyGS0MUTSBS2cWyhpnbj/cPzbT9LXK72u4zhONekos4a+C3zJzJ5v74okMbPphLDS5fIVwkb0A74O/KWG1XIcx8lE3bcIJF0CfAYYL+lUSRfF9NsS8o/flnRd3N9c0t2SZkh6UFL/mL6ZpEckTZP0q1aud3GM+z9f0i8S6UMlPSxptqTHJfWSNFLSHfH82pLulTRT0qUk5iEnWh9nA7tImhXv5UFJgxL5pkoaUKWPznEcJxV17wjM7HiCrONuQHKC8XHAzyTtApwGfC+mjwW+F4VeTgf+FNPPBy42s6HAK61c8kwzGwIMAHaVNEDSp4AbgJOj6MwoggJZkp8DD5nZ9sB4YJMSts8AHjSzQWb2e+BygnQlkrYEVjWzkhPSk8I0cybe00r1HcdxslH3jqAlYuTPnwETgdPM7C1JPQnRO2+M0T0vJUQKBRgOXB/3x7Vi+hBJTwAzgW0IUpJbAS+b2bR47ffMbGlRuRHAtfH8P2jqtFriRmCfqHJ2DCEyaUv3O9bMhpjZkAG7fTGFacdxnHR0lDGCltgOeBPYMB53Ad6JMpClaDU0g6TNCK2IoWb2tqSrgO6Ebp40YR0yhX4wsw8k3QfsBxxCUDtzHMdpUzpsi0DSjsCXgO2B0yVtZmbvAc9LOjjmkaSBschU4Gtx//AWzPYG3gfeldQ32gdYAGwoaWi020tSsROdUrAr6UvAmiXsJ0VpClwOXABMM7O3yty24zhO1emQjkDSqsBlwDFm9hJhjOAKBeHfw4Fjo9DLfMLbNsDJwAmSpgF9iuzNAjCz2YQuofkEIfmpMf0T4FDgwmj3PkJLIckvgBGxW2lP4D8lqj4HWBoHnE+NtmcA7wFX5vw4HMdxKqJDdA2ZWb+4exUr+9EHJs6PJwzQAjwP7FXCxvNAUhv47MS5QYn90S3UYRrw+aLkSXHDzN4kOIACpybK9ox/lwB7JA1I2pDgkO8tdV3HcZxa0yFbBI1CnP76GGGm0vL2ro/jOJ2TDtEiaFTM7Brgmvauh+M4nRsXpumY+JfmOLXFhWmc+uaLZ11SUfl7zjy+LkRlAM6/68HcNk7+0i5MWZA/6siI/psB1bmPS+9/JLeNb48axq9vzf99APz4q3vwzt9uyV1+jUP2B+Cdm/OLHq1x4L4AvHdn/uGu3nvvyZvvf5S7PMDaq3fnlunzcpfff8i2FV2/I+JjBI7jOJ0cdwSO4zidHHcEjuM4nRx3BI7jOJ2cunYEUcil1VGfYrGXpFiMpNGFsNU1qt/xhVDYJeo0r0R9RkraqTi/4zhOe9IIs4b6kRB7SSsWUw3MrOz0naL6jAQWAw/XsFqO4ziZaPMWgaRzJH03cTxG0mmSzpU0T9JcSYeWKNcvCrk8EbfCm3Wx2MsKsZii8utKujkK00yTNLxEnp6SJkT7cyXtlzh3pKQ5MU7QuETdT4/7g+O5R4ATEuVGSrpDUj/geODUWNddJD0fQ1AjqbeCvGW3XB+s4zhOTtqja+ivhABuBQ4B3gAGEeIHjQLOlbRBUbnXgC+Y2Q6xfEEruFjspSXOB34fhWkOJET9LOYjYP94jd2A82IE022AM4HdozDNySXKXgmcZGbDSpzDzBYCl8Q6DDKzBwlxir4cs3wNuDnGI2pGUpjmhWn55947juMU0+aOwMxmAutJ2jCGiH6b4ASuN7NlUXBmMjC0qGg34DJJcwmCLltnvPQo4KIYaXQ80FtScUhoAf8naQ5wP7AR0BfYHbjJzN6I99AkXLSkPsAaZjY5JrUmfJPkcuDouH80rUQgTQrTbDx0l5TmHcdxytNeYwQ3AQcB6xNaCJunKHMq8Cqh1dCF8PaehS7AMDMrlphMcjiwLjDYzJZIWkg6YZq0wjVNMLOpsctrV6CrmeVfDuk4jpOT9po19FdCV8hBBKcwBThUUldJ6xJkHx8vKtOHIBe5HDgC6BrTS4m9lOJe4MTCQVI0vugar0UnsBuwaUyfQJCwXDuWXStZyMzeIYjZ7ByTWhK+KVXXawgSmq5H4DhOu9AujsDM5hMeiC+a2cvALQTRltnAA8APzaxYYP5PwFGSHgW2JCiJQQmxlxY4CRgSB3yfJAzcFqZ3FsYLrot5phMe5gsS9T0LmByFaX5Xwv7RwB/jYHFLrY7bgf0Lg8WJa67JSj1lx3GcNqXdpo+a2XaJfQN+ELdknoXAtnH/WWBA4vSPY3ozsRdWisVcRRSyif37zWYjxemd30zkaWmw92rg6qK0MYn9GSTEcoAxMX1Soj7PFN0DwM6E8Yd3Sl3XcRyn1jTCOoIOi6QLCbrIe7d3XRzH6by4HkHHxL80x6ktnUqPoK5DTDiO4zi1x7uGOiCLJ0+tqHzPXYfXhagMVC4K88G0J3KXX23oDgAseeXV3Da6rd8XgA9nzc1to8eg7Vj8wJTc5QF67j6CjxY8k7t89/5bAnDFpOLJeuk5ZuSOACx54cXcNrptvBGLFi3KXR6gV69efPzsv3KXX3WLNLPZGwtvETiO43Ry3BE4juN0crxryHEcpwa8Ne6GZpM61jri0LochO40LYKkdkDUKdiwlby/lDSqRvXYV9IZLZxbXItrOo7jtEanaREUaQeMBuYBLxXnk9TVzH5Ww3qMJwS9cxyngVHXjvOe3bCOIL79n06Ycz8H+BdBFGYhMAS4TtKHhJXETwFXAHsSIpTuBdxhZjdJGkoIYb068DGwh5ktKrrWrcCnCQHqzjezsTF9L+D/CHGR3jCzPSSNBoaY2YmSNiMI6qwC3F2rz8JxnLZHXbuWz1QnNKQjSOgHDDezN2KQuJMA4sP9ROD0GF4CSQAfmdnO8Xiv+PdTwA3AoWY2TVJvSscROsbM3pLUA5gm6WZCt9tlwAgze744UF3kfOBiM7tG0gklzjuO01FRXQ4HlKTjtF2y0ap+QAuUmtC+FSHi6bRo5z0zW1oi30kxGN2jhJbBFsDngSlm9nwrdRjOymBzrWoYJIVprrj9thS34zhOu9K1a/OtTmnIFgH59AHeL5FW1o6kkQTRm2Fm9oGkSaTTMCiQqp6xu2kswOLJUz3EhOPUOR1pjKDj1DQbreoHkF7DYAGwYRwnQFIvScXOsw/wdnQC/QktAYBHgF3jOECpOgBMJegyQMsaBo7jdES6dGm+1Sn1W7MKSKEfcBVwSdQF6NGKnU8IoasvjHbuA7pHmc07Y7a7gVWivOWvCN1DmNnrwHHA32PZUl1PJwMnSJpGcCiO4zQIWmWVZluqctJekp6W9M9SU80lfV/Sk1FbZYKkTUvZyUKjdg2V1A9InLsZuDmR1K/o/OjE/jRWvuUXWEwMHW1mHxNCSZe6zl3AXUVpV7FSI+F5muofnF3yZhzH6XjkGCyW1BX4I/AF4AXC5JPxZvZkIttMwszDDyR9B/gNJbRWstCQLQLHcZx2R12ab+XZEfinmT0XeyT+CuyXzGBmE83sg3j4KLBxpVV1R+A4jlMD1G2V5lti9l/cjisqthHw38TxCzGtJY6lqNchV11dmKZD4l+a49SWihcBLH5gSrP/0567j2jVrqSDgS+a2Tfj8RHAjmb2vRJ5vwGcCOwau6hz07BjBI7jOO1KvnUDLxDWIhXYmNKhcEYRFs1W7ATAHUGHpBIxFwiCLk+/8kbu8lutvw4AUxY8n9vGiP6bAVQsLFMNgZ15L+QXptl24yBMU2k9qvGdVuOz+Pu0/AI7BwzdDoCpz/w7t43hW27KH++tTHjphD2H8+b7H+Uuv/bq3Su6foGcISamAVvEaecvEqaXf72JXWl74FJgLzN7rdJ6go8ROI7j1Aap+VaGGLngROAeQgy0v5nZ/BgRed+Y7VygJ3BjnAJfcRBLbxE4juPUgpwhJczsTuDOorSfJfarHiLfHYHjOE4N8BATbUBSaKYovZ+keRXYnSRpSGW1cxyn05Oja6i9qJsWgUIsaJnZ8jT5i4RmGgpJq7QQ5dRxnA5C2pAS9UC7tgji2/tTkv4EPAEcIekRSU9IulFSz5jv7ERsjd/GtDGSTo/7gyXNlvQIcELC/mhJFyWO74jRQpF0cVzQMV/SL1LU9aREHf5aXId4PE9Sv7j/U0kLJN0n6fpEXb8laVqs782SVovpV0n6naSJwDkVfKyO49QDXbs03+qUeqjZVsA1hNgaxwKjzGwHYDrw/Ri1c39gGzMbAPy/EjauBE4ys2ElzrXEmWY2BBhAiBI6oEz+M4DtYx2Oby1j7Fo6ENgeOICgiFbg72Y21MwGEmYFHJs4tyXh/k8rYXPFisSrJ95f7t4cx2lnpC7NtnqlHmr2bzN7lBDYbWtgqqRZwFHApsB7wEfA5ZIOAD5IFpbUB1jDzCbHpFYFXhIcIukJQgCnbeK1W2MOQd7yG0C5bpudgdvM7MMoa3l74ty2kh6UNJcQenqbxLkbzWxZKYNmNtbMhpjZkKN2q/qkAcdxqk0HahHUQydWQRBGwH1mdlhxBkk7AnsQFlecSFAgI1GupZALS2nq7LpHe5sR9IyHmtnbkq4qnGuFLwMjgH2BnyrIYZa0T+vL068CvmpmsxX0i0cmzpUSx3EcpwOiLvWrSFZMPbmoR4Hhkj4LIGk1SVvGcYI+cW7tKcCgZCEzewd4V9LOMSkp8LIQGCSpi6RPEyL7AfQmPHTfldSXFsJIF1Bo033azCYCPwTWICzoWAjsEPPsAGwWizwEfEVS91j/LyfM9QJeltQNF6NxnMali5pvdUo9tAiAIOQS35Cvl7RqTP4JQU3sNkkF+cdTSxQ/GrhC0geEFXkFpgLPA3OBeYQBaeLb+ExgPvBczNcMSZcDlwCzgWtjN5SA35vZOwoi9UfGrqxpwDPR/rS42m828G/CeMe70exPgcdi+lzSKaU5jtPByBliol1oV0dgZguBbRPHDwBDS2TdsTjBzMYk9mcAAxOnx8R0o4W37qT4TFH6yMT+NxOndi6R90Ngz1J2gN+a2Zg4K2gKcF4sczFwcdr6OI7TQXFH4ABjJW1NGDe42szyR1dzHKfjUccLyIpxR1AjzOzr5XM5jtOodKSuIRem6Zj4l+Y4taXi1/klr7za7P+02/p967KZ4C2CDshJV/69ovIXHH1AVWLXV8PGklfyawF0W79vXWgJQOXaDDc+Nid3eYCDPzegOt/HS6/kttFtw/WByj/P8TOeLJ+xFfYdvDWTnnoud/mR//OZiq5fIO8CMkl7AecDXYHLzezsovOrEhbhDgbeBA6N4625cUfgOI5TAz5crfnSpHJTBCV1Bf5IiLTwAjBN0ngzS3rHY4G3zeyzkr5GCElzaCV1rad1BI7jOJ2dHYF/mtlzZvYJ8Fdgv6I8+wFXx/2bgD1i0M7cuCNwHMdpI5Ixw+J2XFGWjYD/Jo5fiGkl88Qoxe8Ca1dSL+8achzHaSPMbCwwtpUspd7siwed0+TJRKdtEcSwzweVSM8sbCNpQ0k3tXDOhW4cx0nLC8CnE8cbAy+1lEfSKkAf4K1KLtppHUG1iCIyL5lZM6fiOI6TkWnAFpI2k/QpQqDNYnH68YTozAAHAQ9YhesAOo0jkHRkFJWZLakQqnqEpIclPddC66C7pCslzZU0U9JuMX10FM65Hbg32YqQ1EPSX+O1bgB6JOztqZTCO47jdGw+6dKt2VaO2Od/IiFm2lPA38xsvqRfSto3ZvszsLakfwLfJ2ilVESnGCOIIaPPBIab2RtR7OZ3wAaEGEL9CV62uHvnBAAz205Sf8JDf8t4bhgwwMzeKqiSRb4DfGBmA6LYzROxDusQguiNMrP3Jf2IILxzEUF4p7+ZmaQ1WriH44DjAHY78ttsO7KlEEeO49QDy3O+pMdIy3cWpf0ssf8RcHBFlSuis7QIdgduMrM3AMys0J92q5ktj3N0+5YotzNR6MbMFhAihhYcwX0JO0lGANfGMnMIgjaQU3inQFKYxp2A49Q/y5cvb7bVK52iRUDL4jUfF+UpVa4lWhORKXWtvMI7juN0QJYt7ziRYDpLi2ACQZpybYDYNZSGKcQw1rFLaBPg6QxltiVoIkNO4R3HcTomZtZsq1c6RYsgDracBUyWtIygU5yGPwGXRH3hpcBoM/u4zCK+i4ErJc0BZgGPxzpUIrzjOE4HY5nVb1dQMZ3CEQCY2dWsXJZd6nzP+HchUSwnDsqMLpH3KoL2MCXKfEjo4il1jdTCO47jdGyWLnNH4DiO06mp456gZrgjcBzHqQEdqWvIhWk6Jv6lOU5tqVhAZsHLrzf7P+2/wbouTONUh0qEPyCIf1QihHLw58JEqEvvfyS3jW+PGgbAh7Pm5rbRY9B2dSEqA5WLsVTjO120aFHu8r16hUj5M/9dHNYmPdtvuiEAU5/5d24bw7fclEee/U/u8gDDttiEJS+8mLt8t42Lg33mowPNHnVH4DiOUwvqeQFZMe4IHMdxasBSdwSO4zidm440/pppZbGkkyQ9Jem6WlUoRR2a6AVIuj5G7azKQqwYWfSiuH+8pCNz2rlc0tat2Xccp3FZttyabfVK1hbBd4EvmdmK0bUYj39pdauVDknrAzuZ2aYZyqSur5ldkrduZvbNvGUdx+n4LKty11AMjXMD0A9YCBxiZm8X5RlEiG7QG1gGnGVmZWcipG4RSLoE+AwwXtK7ksZKuhe4RlJXSedKmhbfzr+dKPeDRPovWrDdLB5/sYKYpMUlit4LrCdplqRdkmpgktaRtDDuN9EPKHH9oyU9I2kyMDyRPkbS6XF/kKRHYx1vkbSmpFXivY2MeX4dQ1lQVJeW7K8r6eZoY5qk4TiO0xDUINbQGcAEM9uCED+tlA7BB8CRZrYNsBfwh5ZC2ydJ3SIws+Ml7QXsRoiQ+RVgZzP7MMbKf9fMhsY4OlOjk9gibjsS5uWOlzTCzKYU7EYvVzYefwvsC9xhZoOirdbyrtAPSCZK2gD4BTCYIAI9kdKxiK4BvmdmkyX9Evi5mZ0S4wfdJOkkwgf/uQz2zwd+b2YPSdqEIEbxPynv3XGcOqYGXUH7ASPj/tXAJOBHyQxm9kxi/yVJrwHrAu+0ZriS6KPjY1wdgD2BI2Oc/ceAtQkOYM+4zSQItPSP6UlSxeOvAi3pB3wOmGRmr5vZJ4SmVxMk9QHWMLPJMelqgu4AZjafoFlwO3BMtJHW/ijgovi5jQd6S+pVqvKSjpM0XdL0qyfen/aeHcdpJ5YtX95sS/4fx+24DCb7mtnLAPHveq1ljuHtPwX8q5zhSmYNJePxi/C2fE9RRb4I/NrMLm3JiJktbSEe/1Kio1J41f9UijqtKAN0b6W+zaqRwnZrbEfwuKXEbVqz3wUYlnCoLWJmY4GxAG+Nu6F+R50cxwFKjxEk/49LIel+YP0Sp87Mcu3YEzEOOMqsfKyLaukR3AN8R1K3WIktJa0e04/RSm3ejSQ18WJqOR7/QkJ3CoQmUXnBz6Zl0orJPwaMlLR2rH8zCTgzexd4W9IuMekIYHKs/wGEFtAI4IISXVut2b+X4PiItlyLwHEahDyzhsxslJltW2K7DXg1PuALD/rXStmQ1Bv4B/ATM3s0TV2r5QguB54EnohTOy8FVjGze4G/AI8oxPS/CegVK3unpA3j8R0K8fsnszIe/2XArpIeJ3SvtPZGX+C3BIf0MLBOaxljd0yhiTUGeAS4n6gxXIKjgHNjPQcBv1TQIT4bODb2zV1E6PdfQRn7JwFD4gD0k8DxKe7RcZwOQA0Gi8cTnkPEv7cVZ5D0KeAW4BozuzGt4UxdQ2bWL+6OKUpfDvxv3IrLnE/RwzGm7504bBaP38xeJej8FvhxTF/Iytj/K/bj8QJWKoJBEH5pph8Q0wYl9q8ErixRhzGJ/VlF9SmwZSLPBYn9kSnsvwEcWsKm4zgdnBpEHz0b+JukY4H/EHsX4uzE4+OU9UMIvRNrx4ksEAS1ZrVm2FcWO47j1IBqzxoyszcJY6nF6dOBb8b9a4Frs9p2R+A4jlMDltfxSuJiXI+gY+JfmuPUlop1A26bPr/Z/+l+Q7ZxPQLHcZzOQkdSKHNH0AGphojJFZMez13+mJFhbP/Xt07IbePHXw1dnYsfmFImZ8v03H1EVYRpqiHSUw/CNEtffT13+VX6rgvAXbMX5LbxpYH9Abj58fxiQwfuuB23TJ9XPmMr7D9kW15fXHZpTous27NHRdcv0JE6W9wROI7j1IBqB52rJe4IHMdxaoA7AsdxnE7O8g7UN9TqymJJa0j6bpk8/SR9vdyFVCQokxc1FY5ZV9JjkmYmwj9Uan9F+Gu1IC6T0s7D5ew7jtO4dCRhmnIhJtYgiNG0Rj+grCOoEXsAC8xsezN7ME0BSVlCb3/TzJ7MUzEz2ylPOcdxGoPly5c32+qVco7gbGDzKPxybtzmSZor6dBEnl1inlPjm/+Dkp6IW6sPREmrS/qHpNnR9qExfWGM5YOkIZImFZUbBPwG2Dteu4cS4jWSDpJ0Vdy/StLvJE0EzimyI0kXKQjj/INEaFc1FZc5LN73PEnnxLRNJT2rIILTJd73nvHc4hT2B0uaLGmGpHsKAaUcx+n4NFKL4AzgXzEuz6OEYGsDCXH0z40PrjOAB81skJn9nhAR7wtmtgMhjs4FpU2vYC/gJTMbaGbbAnenqXiMnfEz4IZ47XLzxbYERpnZaUXp+wNbEUJJfwto5rhicLxzCOGxBwFDJX3VzP4d0y8BTgOejIH2ytqPkUgvBA4ys8HAFcBZLVVerkfgOB2KGgSdqxlZBot3Bq43s2WEcKiTgaEEYZkk3QhiK4MImplb0jpzgd/Gt+w70nbx5ODGWPdiRrDyvl6S9ECJPEOJ4jIAkq6L5W41s8slHUyIHFoqjHRL9rciBMy7T0FZrSvwckuVdz0Cx+lYNOqsobRLo08FXiW0HLoQ1MdaxMyekTQY2Bv4taR7zeyXtC4y06K5xH41hWlavHdJqwEbx8OewKKU9gXMN7NhZa7tOE4HZGkHcgTluoYWEfUDgCnAoQpC9esS3nQfL8oD0Ad4OYamPoLwptsisdvlgxg177fADvHUQlaKzByY6m5CS+V/JHUhdMmkYQrwtXhfGxA0mYt5jKCNsI6krsBhRGEaQtfQdYRuqssy2H8aWFfSMAhdRZK2SVlnx3HqnGp3DUlaS9J9cVzyPklrtpK3t6QXCzMsy9GqI4hhT6fGaZ/DgDnAbOAB4Idm9kpMWxoHe08F/gQcJelRQrdQszdxSRtKujMebgc8riAUcybw/2L6L4DzJT1I6GJKwxnAHbF+LXazxMHny+PhLcCzhC6qi1n5gE9+Di8T9BAmxvt/wsxuk7QrodvoHDO7DvhE0tFFxUvaj/rFBwHnSJoNzKLE+ITjOB2TGgwWnwFMMLMtgAnxuCV+RYlnWUuU7Roys+KpoT8oOr+E5jGyk+IwpQRlXiJ0BRF1ju8pKk8cK2g2vpAUmSkWnDGzmwgqaMVlRhcdJ+N3Gwm5yKJ8IxP7fyGorSXPTyYhVmNmByT2e6awP4vQsnIcp8GowXTR/YCRcf9qYBLwo+JMsau9L2HizZA0hqslVek4juMkWG7WbEvO/ovbcRlM9o29E4VeivWKM8Ru8fMoemEvh4eYcBzHqQHLSowJJGf/lULS/cD6JU6dmfKy3wXuNLP/xtmIqXBhmo6Jf2mOU1sqFpD59a0Tmv2f/vire+S2K+lpYKSZvRwnnkwys62K8lwH7AIsJ8xi/BTwJzNrbTzBWwSO4zi1oAYv2eOBowjRHI4CbitxzcML+wri9UO53tBJAAAgAElEQVTKOQFwR9AhOe+OSRWVP22fkVURdHnnb7fktrHGIWF270cLnslto3v/LatyH9WwsWhRqeUj6ejVq1dFojIQhGXq4T4A3rl5fG4baxy4LyPHpJrx2CKTxpzItQ/NyF3+GzsPLp8pBaW6hirkbOBvko4F/gMcDGEWJHC8mX0zr2F3BI7jODWg2iuL43T+4hmaTWZBFqVfRWJWZWu4I3Acx6kBjRpiwnEcx0lJR5qH4+sISiBpjKTTq2jvTgWRn7JCP47jNAbLli9vttUr7gjaADPb28zeIZ3Qj+M4DYA7gg6IpDMlPR0XdGwV0zaXdHcUjnlQUv+YfpWkCyQ9LOk5rZS23EDSFAWhnHmK8plaKbJTLPQzTtJ+iTpcJ2nfNr95x3GqTiMJ03QKYmyOrwHbAwcQAslBWAH4vSgcczohoF6BDQgaDfsQHvAQJDvviUI+AwmB5JKsEPoxsx8AlwNHxzr0IQSdu5MSJJemP3r37ZXcruM4bUCjCtM0MrsAt5jZBwCSxhP0DHYCbkws1V41UebWGGr7SUl9Y9o04IqoPnZrDCrXImY2WdIfJa1HcEA3m9nSFvKuWJp+3h2T6vcX5TgO4LOGOirFD9cuwDvx7b4UHyf2BWBmUySNAL4MjJN0rpldU+a644DDCS2SY7JX23GceqSeu4KK8a6hwBRgf0k9JPUCvgJ8ADwfZSgLIvQDWzMiaVPgNTO7DPgzK0V2ChSL+EBY8HEKgJnNr/RGHMepD5bb8mZbveKOADCzJ4AbCH36NwMF3eTDgWOjcMx8Qjzw1hgJzJI0k6Cqdn7RdVYI/Ug6N6a9CjwFXFmdu3Ecpx5YvtyabfWKdw1FzOws4KwSp/YqkXd00XFBhOZqgmBEcf5+if0mQj9R83gL4Poc1XYcp07pSGME3iJoRySNAhYAF5rZu+1dH8dxqofPGnJSYWb3A5u0dz0cx6k+HWmw2IVpOib+pTlObalYmObIP17X7P/0mhMOr0SYZi3CWGY/YCFwiJm9XSLfJoQ1Sp8mPCv2jprxLeItgg7IFZMer6j8MSN3ZPyMJ3OX33fw1kDlceehsns5ZuSO/H3a3NzlDxi6HQBLXnolt41uGwZVwZn/fim3je033ZC7Zi/IXR7gSwP7V0VLoBqaBosfmJLbRs/dR/Dh3Momz/XYbpuqfBaVUoPB4TOACWZ2tqQz4nEz8XrgGuAsM7tPUk+CWlmr+BiB4zhODahBiIn9WDkZ5Wrgq8UZJG0NrGJm9wGY2eLCQtnWcEfgOI5TA2qwjqCvmb0MEP+uVyLPlsA7kv4uaWaMada1nGHvGnIcx6kBy0uMv0o6DjgukTQ2ho8pnL8fWL+EuTNTXnYVQsic7QlyljcAowkLXFst5DiO41SZpcuatwCSMcNKYWajWjon6VVJG5jZy5I2AF4rke0FYKaZPRfL3Ap8njKOwLuG2hFJk6LwdOG4n6R57Vknx3GqQw3WEYwHjor7RwG3lcgzDVhT0rrxeHeg7MwQdwSO4zg1YLlZs61Czga+IOlZ4AvxGElDJF0OYGbLCCHzJ0iaS5gGe1k5w9411AZI6gfcDTxG6Lt7BjiyHavkOE6NKdU1VAkxVtkeJdKnA99MHN8HDMhi21sEbcdWhIGhAcB7rJSsvC4qls2iBVEaaCpMM/n2W9qguo7jVEINWgQ1wx1B2/FfM5sa968lqJsBHB4VywYBe7dU2MzGmtkQMxuy61f2r3VdHcepEI815JSi+FdQv78Kx3EqptpdQ7XEWwRtxyaShsX9w4CH2rMyjuPUlo7UNeQtgrbjKeAoSZcCzwIXE5TQHMdpQCb+/ISKA9e1Fe4I2o7lZnZ8UdrI5EGMELhtW1XIcRwHvGvIcRyn0+N6BB0T/9Icp7Z0mG6dauAtAsdxnE6OjxF0QE4bl18QBuC8I/atigDJe3fem9tG7733BGDJCy/mttFt442Y+sy/c5cfvuWmQHXEWCqtx82P5xfYAThwx+2qIhRUqagMVP55ViIqA0FYZskrr+Yu3239vhVdvyPiLQLHcZxOjjsCx3GcTo47AsdxnE6OOwJA0mhJF1Wap0SZUyStVlntHMdxaos7gtpyCuCOwHGcuqZhHYGk1SX9Q9JsSfMkHSppoaR14vkhkiaVKHeVpEskPSjpGUn7JE5vKOluSc9K+k2izMUxRPR8Sb+IaScBGwITJU2MaXtKekTSE5JulNQzpp8t6UlJcyT9tnafiuM4TnMaefroXsBLZvZlAEl9gHNSlu0H7ApsTniQfzamDyIIy3wMPC3pQjP7L3Cmmb0lqStBGWiAmV0g6fvAbmb2RnRAPwFGmdn7kn4EfD92N+0P9Dczk7RGVe7ecRwnJQ3bIgDmAqMknSNpFzN7N0PZv5nZcjN7FngO6B/TJ5jZu2b2EUEHdNOYfoikJ4CZwDbA1iVsfj6mT40iNEfF8u8BHwGXSzoA+KBUhZLCNHMm3pPhVhzHcVqnYVsEZvaMpMEEsZdfS7oXWMpK59e9teItHH+cSFsGrCJpM4JG6FAze1vSVS3YFnCfmR3W7IS0I0GC7mvAiQTB6eL7GQuMBTht3HgPMeE4TtVo2BaBpA2BD8zsWuC3wA7AQmBwzHJgK8UPltRF0ubAZ4CnW8nbG3gfeFdSX+BLiXOLgF5x/1FgeKGbSdJqkraM4wR9zOxOwuDyoAy36TiOUzEN2yIAtgPOlbQcWAJ8B+gB/FnS/xKE5FviaWAy0Bc43sw+kkrHoDKz2ZJmAvMJ3UhTE6fHAndJetnMdpM0Grhe0qrx/E8IzuI2Sd0JrYZTc92t4zhOThrWEZjZPUCpzvQtS+S9CrgqkTTVzE5tLY+Z7ZPYH91CHS4ELkwcPwAMLZF1x1LlHcdx2oKG7RpyHMdx0tGwLYK8tPR27ziO06i4ME3HxL80x6ktLkzjOI7jdB68a6gDsvTV1ysqv0rfdfnXa2/nLr/5emsC8Ob7H+W2sfbqYalFJSIkvXr14o/3Ti2fsQVO2HM4AONnPJnbxr6Dw9rBR579T24bw7bYhFumz8tdHmD/IdsyckymmIhNmDTmRAA+nDs/t40e220DVP6dViJsA0HcphpiQ50JbxE4juN0ctwROI7jdHLcETiO43Ry3BGUIRm6OmO5qyQdlCF/P0mVdRQ7juPkwB2B4zhOJ8cdQQJJt0qaEQVmjitx/sgoHjNb0riYtqmkCTF9gqRNEkVGSHpY0nOF1oEC50axnLmSOt8UBcdx6gp3BE05xswGA0OAkyStXTghaRvgTGB3MxsInBxPXQRcY2YDgOuACxL2NgB2BvYBzo5pBxAijA4ERhEC421QrmJJPYLLxl1TyT06juM0wdcRNOUkSfvH/U8DWyTO7Q7cZGZvAJjZWzF9GOHhDjAO+E2izK1mthx4MoaohuAYrjezZcCrkiYTAtHNaa1iST2Cpa++7iuLHcepGu4IIpJGEt7Qh5nZB1HPOCkwI9KFdkjmSQrZqOiv4zhOXeBdQyvpA7wdnUB/grRkkgkEScq1ASStFdMfJiiLARwOPFTmOlOAQyV1lbQuMAJ4vBo34DiOkwdvEazkbuB4SXMIwjSPJk+a2XxJZwGTJS0j6BOPBk4CrpD0A+B14Ogy17mF0J00m9B6+KGZvSKpX/VuxXEcJz3uCCJm9jFNZSYL9EvkuRq4uqjcQkprDI8uOu4Z/xrwg7gV29k2R9Udx3EqwruGHMdxOjnuCBzHcTo5LkzTMfEvzXFqS6ea3edjBB2QSmLGQ4gb/5+33s1dfpO1+gBUFEN//yFhOOTjZ/+V28aqW2xeFU2ESU89l9vGyP/5DABLXngxt41uG2/E64s/zF0eYN2ePbj2oRm5y39j58FA5VoCAEteeTW3jW7r93U9gnbAu4Ycx3E6Oe4IHMdxOjnuCBzHcTo57ggcx3E6Oe4IakRSaEbSEEkXxP2RknZq39o5juOsxGcNtQFmNh2YHg9HAosJMYocx3HaHW8RlEDSmZKelnS/pOslnS5pkqQh8fw6khbG/X6SHpT0RNyave3HVsAdMZ7Q8cCpkmZJ2kXS85K6xXy9ozRmtza7WcdxOj3uCIqQNJgQTXR7gs7A0DJFXgO+YGY7AIfSVJimCTGe0CXA781skJk9CEwCvhyzfA242cyWlKjXCmGaP990Y7abchzHaQXvGmrOLsAtZvYBgKTxZfJ3Ay6SNAhYBmyZ8XqXAz8EbiVELv1WqUxJYZoP5873lcWO41QNdwSlKfWgXcrKFlRSsOZU4FWC9GQXINNSVzObGruXdgW6mln+5bqO4zg58K6h5kwB9pfUQ1Iv4CsxfSEwOO4flMjfB3g5SlIeAXQtY38R0Kso7RrgeuDKCurtOI6TC3cERZjZE8ANwCzgZuDBeOq3wHckPQyskyjyJ+AoSY8SuoXeL3OJ2wmOZpakXWLadcCaBGfgOI7TpnjXUAnM7CzgLABJY2LaAmBAIttPYvqzRek/jukLiUIzZjaJMCiMmT1TlB+CoP1NZvZONe/DcRwnDe4I2hlJFxKU0fZu77o4jtM5cUdQBjMbU2P736ulfcdxnLKYmW8NtgHHtbeNeqhDvdiohzr4fVTfRiNtPljcmBxXBzbqoQ71YqMe6lANG/VQh3qy0TC4I3Acx+nkuCNwHMfp5LgjaEzG1oGNeqhDvdiohzpUw0Y91KGebDQMigMnjuM4TifFWwSO4zidHHcEjuM4nRx3BI7jOJ0cdwSO4zidHHcEDgCStq2CjXIhuGuOpK6Szm3vehSQtHo7XnsfSf4/Tn38NusZ/5E0AJL6SvqzpLvi8daSjs1o5hJJj0v6rqQ1clbln5LOlbR11oKSFkl6r6UtrR0zWwYMlqSsdUjU5TdRP7qbpAmS3pD0jYw2dpL0JPBUPB4o6U8ZylfjO/0a8Gy8n//JUlDSXElzSmxzJc3JaCv351nFeuT+bXYGfPpoAxAfFlcCZ5rZQEmrADPNbLuMdrYAjgEOBh4HrjSz+zKU70V4+BxNeMm4AvirmaV+kEv6JfAKMA4QcDjQy8x+k8HGecAWwI0k9CHM7O8py88ys0GS9ge+SlChm2hmAzPU4TGCgNF4M9s+ps0zs1Qtryp+p72BwwjfiUWb15vZojLlNi3sAv+gKDqumf07Qx1yf56JepQkbT2q8dtsaNo72JFvlW/AtPh3ZiJtVk5bXYEDgRcJb7MLgANy2BkRbbwPXA18NmW5x9KklbFxZYntigzl58e/lwF7xf3ZGevwWInvJLWNKn+n6wCnEFT27gKeBb6XofwTeX+b1fo8Y5m+wD5xW6+C+uT6bTby5mGoG4P3Ja1N1FqW9Hng3SwGJA0gvC19GbgP+IqZPSFpQ+ARoOzbdOyH/XK00w84j6C+tgtwJ0HBrRzLJB0O/DXez2HAsiz3YmZHZ8lfgtslLQA+BL4raV0yalED/5W0E2CSPgWcROwmSkk1vtN9Cd/F5oQW1o5m9pqk1WJdLsxirwIq/jwlHQKcSxB4EnChpB+Y2U0py1fjt9m4tLcn8q3yDdgBmEp4UEwFngEGZrQxhaC53KPEuSNS2ngO+DOwU4lzF6S00Q+4DXgDeB24FeiX8V66AycQZESvKGwZbawJdI37qwPrZyy/DuFB8yrwGnAtsHaF3+mAjHW4BhjRwrk9Uly/sD1VdLxDjt9opZ/nbBKtAGBdsrWwKv5tNvLmYwQNgKRVCW/NWxHelp4GupjZxxlsnGJmfyhKO9nMzs9gY2cze6gobbiZTU1roxpIupHQpfV14JeEcYanzOzklOVXA74PbGJmx8Wxk63M7I5a1bmFeqxC4js1syUZy59jZj8ql9ZC2YlFSYUHhQAzs90z1KPiz1PSXEuMj8TZULMt5ZhJvfw26xV3BA2ApCfMbIdyaTlszLQ40Fnrekj6oZn9RkG6s9mP0sxOylCPmWa2vaQ5ZjZAUjfgnrQPL0k3ADOAI81sW0k9gEfMbFCKsiXrX6DcfUg6oLXzlnLAO9oq9X3MMbNizezWbPQAvkvQ1TbgQeBiM0vdtVPJ55mwcS5B6/v6mHQoMCeNU4vlK/4faWR8jKADI2l9YCOgh6TtCW9rAL2B1VLaOIzw5ryZpPGJU72AN1PaGAbsBKwr6fuJU70Jg89pKPSfT0+ZvzUKb87vKKyPeIXQ5ZSWzc3s0PjZYGYfSqmnoxbqPxzYGrghHh9MeBiW4yvx73qEz/SBeLwboX88zVjNdwgP782Lplj2InQzZeFq4D3ggnh8GKHL6ZAMNir5PIllfiDpQMLnKmCsmd1SrlyVfpsNjzuCjs0XgdHAxsDvEumLgP9NaeNh4GVCn/Z5RTbSztP+FNCT8HvqlUh/jzCFsixmdnv8e3XKa7bGWElrAj8Fxse6/SxD+U/iW2thoHZzIFU3W6H+kkYDuxW6cyRdAtybovzRMf8dwNZm9nI83gD4Y8r6/4UwO+jXwBmJ9EVm9lZKGwW2sqbTPCdKmp3RRu7PM4mZ3QzcnLFYxb/NTkF7D1L4VvkGHNjedYj12LQKNu4D1kgcr0no1mnL+/gCMJkwWH0dYdrlyIw2ngbWKrqPpzOUn1d03KU4rZWyvePftUptGe/jKuDziePPAX/KaGPPEp/nbhltHECY9vou4SG+CHivLX+bjbz5GEEHRtI3zOxaSadRul/9dyWKFdt4yMx2lrSoyEZhULB3Cht/MLNTJN3eQj32LWcjYWuWFfUd5xir+H6J5HeBGWY2K6WNtYHPEz6HR83sjbTXj+WPBsYAhUHXXYExlrLFI+kiwqK46wmf6deAf5rZ91KUvcPM9pH0fCyb7IYxM/tMhvt4ijBg/Z+YtAmhG295tJVqvKEKn+c/CVOas0zBrepvs5HxrqGOTSGOTc+8Bsxs5/i3V7m8rTAu/v1tBTYKLJO0iZn9B1asLM36tjIkbrfH4y8D04DjJd1oLaxSllQ8cPhy/LtJrNMTaStgZlfG1cGfi0lnmNkrGcqfGAeOd4lJqfrEY9l94t/N0l6vFfaq1ICkCWa2B2GFcnFaWl7N6gQi1fxtNizeIujkSFqrtfOWvU+5IiTtRZARnByTRgDHmdk9GWzcQ+guWxyPewI3AfsTWgUl480kpkx2JziS2YQ32AGElcI7p7h2fzNbUMKpAJDFmVSKpOGE1cjvK8T22QH4Q8HJtsH1uxMmLUwERtJ0MsNdZlY2/lFiFtWuwPqEdSUrxhcswywqp2W8RdAAxJWa3yLMjFnxnZrZMSmKz2Bl98EmwNtxfw1Cd0DZt0pJc2l9ymTq6Ypmdnd8iBa6EU7N2o1AuI9PEsdLCH3EH0pqcZDSzHYDkPRXgvOZG4+3BU5Pee3vA8fRdOB9xSWAVqewVqOrLsHFwEBJA4EfEhZUjSM8VNuCbxNCW2xI+J0VHMF7pB/4/kpi/wPCeEMBo8wsqmr+NhsZdwSNwW2E+d33kz0cw2awYlbLeDO7Mx5/CRiV0sw+8e8J8W+hOX444Z83K8sIq3G7A1tLwsymZCj/F+BRSbfF468A1yuEhH4yRfn+BScAYGbzJKWa825mx8W/u2Wob7J8NbrqCiw1M5O0H3C+mf1Z0lFVsJsKC4sRz5f0PTPLFc7CKg8Xsk/5LI53DTUApQZYc9iYYWaDi9Kmm9mQDDammtnwcmllbHwTOJkwJXYWoWXwiGVYyRrtDCYsghLwkJmlXp8g6XpCQLJrCW+T3wB6mtlhGWz8ijA4vCwe9yY8jFM92CSNMrP7i9KOSjvYHPNPBu4mxNcZQZi1M8syRjCtBrFVtTXBuQNgZtdkKL8xITbScMJ38hBwspm9UOWqdkpcj6AxuEPS3uWztcobkn4iqZ+kTSWdScoFZQlWl7SiH10h6FpWYZaTgaHAv+Nb9faEB1hqJG0Sy9xC6Dp4Laal5WhgfqzLKYRWRNY301WAxyUNkLQnYbA6zYKyAj+TdLGk1RW0CW6naTdJGg4l9KcfGweqNyIEbmtTJP2c8BC/kLAw7jdA1tk6VxLWhGxIuI/bY1raOnxe0jRJiyV9ImmZMuhcNDztPX/Vt/wbcS513JYTojsuIuMc62hrLeB8YGbczif7nPPBhAHWhXGbRcYAZawMvzwLWLWwn9HGXMJiuDmEuedLiaGQ2/j7GRW/k5fIGOqY0JI5Pdb/WeCw9v69VfA5zCW8dM6Ox32B2zPaaPYbyPK7IKz4/mz8bXclOPaz2vuzqZfNxwg6MBb7kSWNI4wRPGj5pthhYXZQqqBsrdiYQRic7E3odswUNjnygoJC2q3AfZLeJjxIs9SjSddHHHz+drlykv5mZoe0NMBo2WL0jCA4018C2wEXSTrGzNLey5qEqaf/InSTbSpJFp9qKetwAHAOIVyFyDfgXA0+NLPlkpbG38ZrQOq1DJGCqlkh1tBhZGyxmtk/JXW10F13paSHM9ahYfExggZA0u6E/vBdCP9gMwlOoWzk0GosuEksbCu1kAtLsbCtBbu7An2Au83sk3L5y9gqG2BM0gZm9rJaUMWybKpcjwOjzezJeHwA8H9m1j9l+WeAs83sihie4RxgiJntlKEOuRZhVRsFic7/JSyKOw1YTHibT93dFrv2LgKGEX6nDxPGCNIqlE0htNAuJ8Seepnw/aRWnWtk3BE0CArCG0MJfbDHE97Cyj50JA02sxnxodsMM5tcKr3IxrfN7NLYF1zKxi/K2agmRQ6pC6HLai0z+2KKsl0JIS3Szphq0U5880ymrW1maQP5rVhUl0gbYRlmT2UdqG8LJPUjhMDIpHtchetuSmiJdCNIZfYhhMr4Z1vWo15xR9AASJpAGJR9hNBF9JCZvZbRxurEJnw87kroo88z/bOqKIZMyJA/6ZCWEsYrbraUoZMVorAekbNrq2CjDyHExAjCG+xk4JdZbCoojI2Ih5MtBubLUP586mQRlqSNgE1pus4li1O7mtACeCcerwmcZ+nWyjhl8DGCxmAO4a13W0JMnXckPWJmH2awMYHQdF4cj3sQomVm6Yqo1T/rt7JkTrZAFARMeqZ1ApGPgLmS7iNMIy3YTa2JQFBFm0cIPw1B/e1KQvC0skg6m9DCuy4mnSRpJzP7cYY69CbHIqxqI+kcwgymJ1m5zsUIqnhpGVD4XQGY2dsKodfT1qEQd6kJliHuUiPjjqABMLNTYUUohaMJD5z1gVUzmOluMSRDtLlYQVkqCxX9s0LJlkkXsmv1/oXQPbaMMGWzj6TfmVnaqZP/IBEXJyebm9mBieNfSEoV8C6yNzAo8TlcTRj7Se0IsvTB15ivEsJZZw49naCLpDXN7G1YERoly/MruR6mO8FBtxpepTPh6wgaAEknKqhAzSL8010BfCmjmfeViI8TF2RlaVFA/GdN2Mj6zwqhZZJ0QKsRVkxnYWsze4/wWdxJCDlxRNrCFhZtXU9wIjOAv1h2nYQPi9ZUDCf757lGYr9PxrJI2lLSBEnz4vEAST/JaqcKPEfom6+E84CHJf0qLtZ7mLAeIRVm9mZie9GCLGumRYqNjLcIGoMeBGGaGWa2NKeNU4AbJRWmN25AaM5nofDPelM8Phg4K6ONarRMuinIU34VuMjMlkjKMu1yJEGZayFhyuWnFVb1ZunK+A5wdRwrEPAWQUQoLb8GZioEwhNhrCBLtxDAZcAPgEsBzGxObC39v4x2cqGVsp0fALPiWFZyrCJ1V5uZXSNpOuHhLeCAwoyslHVJzhjrQmghVCOMR0PgjqAByNDl0ZqNaZL6s1IsfYFlFEuP/6wzCDOXMv+zRt6XtIPFKJ05WyaXEh7is4EpccZIllWk5wF7mtnTsQ5bEloIg1stlcCC7kFhTQWxhZIaM7te0iTCOIGAH1mGMNaR1czscTVVhcz7opCHQliPGYRVwZlR0+i4rxDiSK04Z+mj457HyjGCwgSCg1vM3cnwWUPOClRhPJiEnfWKbKQOeyxpKPBXVi4i2wA4NC5Wy4XCk7Br2taSSgi8l0proWxFaypUxTDWCnoIJwI3mtkOkg4ihJvI2m1YMZI+BfQnPIyfTrsuRM3FdQoPrMLiuFSDvVop3lRsJxzkXOvSKHiLwAFWTLkcSXAEdxLGGB4iCJWntbEv4c1rQ8Kc7U0JalbbpLVRjZZJCZsmaQCQ9iE6XVIhZDOEKKppHVEhtlLebofTCLOkcoWxLuIEgrZDf0kvAs8T7qVNUYiDdSlhlbSAzeLak7vKlbWEuE5sHWxB4iUjA4MJravbYh2+Qpi19N8cthoObxE4AIW47QOBmWY2UFJf4HIzSx3oTEHUfHfgfjPbXtJuhBg5x6Uou7uZPaCVQiRNqHTuu6TLzCzVNFRJqxIeooXopVMIi48qmfXSZpRojfQg9Iu/D23/9itpAbBPYfGWgnj9P9Kuso5lSkWlfdhSqpxJupcgVrQoHvcitJQqVmBrBLxF4BSoRjyYJWb2pqQukrqY2cQ4hzwNuwIPUDrCZsVz39M6gZj3Y8Lge+YHpqQLythudYC0JUeYKJ/mcyi0Rrai6VvwEWSbu18tXitawfsc4feVhUJU2kfNbLfYasyyYr1YrOgTgpCTgzsCZyXTFYK9XUboBlkMPJ7RxjtxLcMU4DpJr5FycNLMfh7XDNxlZn/LeF2g2cyQUtdotWtI1VGzKnQhDSd0s90Qjw8mXfdSwRGuR1jM90A83g2YRAqHWFhQF9+Cd0i8BY8BbkxRh2ozX9KdwN8In+/BwLSC00vp3D4ys48kIWnVOI6yVYY6jCOEBb8l1mF/wswwB+8ackqgnPFgCovBCN0QhxPmvl9nKePrRBtTzGxE+Zwly05s5bRZGXEbtRBsLmEgS9C5iYSZR0vicTfgXkupXCbpDuBbZvZyPN4A+KOZpVqZHMssAAYWurRil9fsLF0y1UBSa7oBlmbleXyAH02Y5rw7QVK1m5ml1uGILwq7xMMpZjYzbdlGxx2BU1dI+inBmdxA0/AOaacJ1gWSngaGFeodF9o9amap3u6Yh14AAAv2SURBVGIlzTOzbRPHXYA5ybQUNs4EDiEI9BTegm8ws1+nv5P6Q1WMSusE3BE4LaIUoZtT2BibZrA4kf/5Esmppwkm7OSeCqsqxPGXdDQh6FyhlbIrQboyVXeEpIsIM2SuJzzEvwb808y+l7YO0U67vQUnFpSVJMuCMqe2uCNwaopimOsM+btbUYC4UmllbJScCmtmB6UsX5U4/pLWJ4jLADyWdUGYpP1ZGX10ipndUkl92hpJR7V2Pq1TdGqPOwKn6kha3czeL5+zZNlmrZCsLZNKp8KqDuP4O04t8VlDnRxJi2i9+Z6lO2QnggJUT2ATSQOBb5vZd1OUXZ8gSt5DIWJpYQVob5oGoUtDrqmwiamb0xWC+FU1jn+lXW1Zu9nqhThwXioEtAd9qxPcEXRybKXu8S8JsVzGER7Ch5N9dezvgS8S48qY2WwF7d40fJEQlG1jms7fX0SQOcxC3qmwhRZDIVBaVeP4VzreQgwe1wE5PbHfHTiQto155JTBu4YcACQ9ZmafK5eWxoakmWa2fUybbRl0YSUdaGY3p695WXv9yDgVVhUK7KhCuctY/mwz+0Ge8h0BSZPNrKQ8qtP2eIvAKbBM0uGEgG8GHMZKNam0/Dd2D1kMMnYSIdZQWRSDtQH9SoRIyBQWoVQrRNn0fisS2DGzZZI+kNTHcshdxvKDJcka4E1NTSOIFkJAr99O1XFK4I7AKfB14Py4GTA1pmXh+Fh+I+AFgtTlCSnLFoK19SxxLuvDMPkm3R3YkdBFlLZPulI1LKhc7nImcJukG4vKt7necBWYwcrIn0sIIaCPbc8KOU1xR+AAYGYLgf0qtPEGOaNbmlmh//szlOiWyWiryewgSZ8mg5oVTQV2jLAoK6vATqVyl2sBb9LUebW53nCV+BFh8dd7ccHgDoQxGKdO8DECB1ghvnIx0NfMtlUI27yvmZVVs6rmwqHk+EJraVmQJMKq3O0ylNmalWpYEyy7wI4TUdRyUJDu/D+Co/3fLONPTm3xFoFToBJZw+nls6Sm4m6ZIsfUBRhEUCtLTXzw5374a6WgSrHdtEIqV7ZQPtWAdZ1RGGv6MnCJmd0WA+A5dYI7AqdAblnDKq8QrUa3TNIxLQWuN7OpVapfWoYk9rsTIm6u1ULeUtxRVH5/Vqq2dTRelHQpMAo4Jwa/69LOdXISeNeQA1QmayjpD2Z2iqTbKf0Wu2/GujRkt4ykh8xs55xluxAEfzrcIixJqwF7AXPN7NkYSXU7M7u3navmRNwROABI+gxB1nAnQojf54FvxEHkcmUHm9mMGBWyGWY2uZp1baUe1dATqFZdkovHClMmv5NlTUWRva0Iql6frUb9HCeJdw05AJjZc8CoqCnQxaKYScqyhaByg8zs/OQ5SScDbeIIgH3i38KU1aTmcFvPUknOdFpKcKyHpC1cIvTHK4TZN45TdbxF4AAgaRxwYmEBlIJIyxWWUhM2likVMK6iGT95KBU0rq0CyUk62czOl7SzmT2Uo/xwM5uaNeKq41SCD9g4BR4CHpO0t6RvAfcBf0hTUNJhcXxgM0njE9tEwlz4tmb1OFWxUL+dWLlgrdYcHf+2ql3cCoVyD1ehLo6TCu8acoCwoEvSfIKQyhvA9hni5z8MvAysQ9MukUVAJrnLKnEscIWkPvH4HaCtpl0+JWkhsK6k5L0XxG3KjVMsiVNHN5bUzJm4mItTC7xryAFA0hHAT4GfAwMI0UCPNrNM8+/riRiCWnni/VR43fWBe4Bms6WsjO6xpHWI0yyBn5Uo72IuTtVxR+Dw/9u7+1A96zqO4+/P1gZW8yFYmmGjrRz9kytbHLGIKA2jP6ZgZuTcPw2mQQ/gA5nEIKhOUdCqiRVLIso2etLU0SQ17EFF1oIVtShDsoYbcZYu18anP37Xve7dHXeu+9zXdd3H3Z/XP2fXufa7ri9j5/6e3+MXQNIPgY2291fXbwZut71miGdMAVuA1wFLgcXAM8PUNBhF7+C62Q6tg+EOrhs3SRe8kJNwvLBkaCgAsL1u4PqRKhkM48uU2rrbKcsl1wNdLnfszQMMW0ehMSdZwlpraKh/V/TA5j4gQ0PRjiSCCSfpRtvTJzkvaKgPHtv7JC22fQzYJqmzSc/ewXW2N3f1zlm8Z+6/clJNHtcRUUsSQdxEOZnzT5SNZKN4tqpDsFvSNGUCuavVOsdV7/4UcBi4j1K/+CNVvYNWzTUHUKN95gCic5kjmHCS9gKXUcpLvn3wvu2DQzxrBaU+8BLgo8AZwFdt72sm2tpx7La9RtLlwLoqlp/Nd1fvPGPo3xC2lPJvUnu+RKnzGx1KjyC2Un5rXsmJwxKifBDVOi0TTvht+DAwzuGZJdXXd1MOnDs423h7m3q1oHskraMUyKkrdX6jM+kRBACSttreNM+2C+aMHwBJn6H0BA5TPnzPBO4e9/n3kn5le2qE9qnzG61IIoiRVUNCz2vUcfP5qCqbzVT1f18CLBtig1wT77+i77J36NzbbF9Us/1gnd8LgS/ZXt1clBFFhoZiZP0f9JLOBtZWl4/09iV0qTr2+HrgVcBG4FxgNSee8d+2/nKZRyl1eocpBdpf57d3aF3q/EYr0iOIxkh6L/A54AHKB9hbgRts7+g4jjspH6Trq7KbpwG/HGZzXMQkyaFz0aRbgLW2r7W9njI+f+sY4lhlexr4D4Dtw5TE1BlJ05JOl7RE0v2Snpb0gSHaXylpWfXnT0j6/kCNg4jGJBFEkxYNDAUdYDz/x45UvYDeDt1VwHMdx3Cp7RnKBrMngfMpNaHrutX2oeoU1XcBd1BWeEU0LokgmnSvpJ2SNkjaAPwEuKfLAFTWid5GWRJ7nqRvA/cDN3YZB7MsYR2yfX/B9622f0TZjxDRuEwWR5P+DvwWWEMZirnd9g+6DMC2q6polwJTVRwftv10l3EAd0n6PWUJ63WSlgPDFJpJwffoTCaLozGSPkkpx3gQ+C6ww/Y/xhDHV4Bv2n6063cPxDHvJawp+B5dSiKIxkl6PXAVZTfsk7bf2fH791LG5J8AnqF+UZi24zqny70MEXVlaCjasJ8yTHQAePkY3n/ZGN5ZxzcoY/7zIulu26Oebhrxf9IjiMZI2kTpCSwHdgB32t473qhOHZJeYfupcccRp570CKJJKyjHPe8edyALQbX087W2t1WTxS+1/ef5Pi9JINqSHkFEC6qJ8zcBq22fL+lcYLvti+doN1KFs4j5SI8goh2XA28AHgew/bfeTuE5ZA4gOpdEENGOI9Weht7u5lqV2sZxUmtENqhEtON71YawMyV9ENgFfK1uY0lTkh6V9C9JRyQdkzTTWrQx0TJHENESSZdQdjgL2Gn7p0O0fQx4H7CdMtewHniN7VvaiDUmW4aGItrzB8oE7y5JL5a0zPahuo1t75O02PYxYJukX7QXakyyJIKIFlTDQRuBlwGrgFdSDsN7R81HPCtpKbBb0jTwFFBrniFiWJkjiGjH9cDFwAyA7T8y3C7rayg/nx+iHJNxHnDFSVtEzFMSQUQ7nrN9pHch6UXMvj/g+ayz/W/bM7Y32/4YWVoaLUkiiGjHg5I+DpxWTRpvB+4aov21s3xvQxOBRQzKqqGIFkhaRCk2f3zVEPB1z/EDJ+lq4P3AW4Cf9906HTja9UmuMRmSCCIaJmkxcIft2jWK+9quAF4NfBq4ue/WIWCP7aPNRBnxP1k1FNGwqhDNcklL++cJarZ9glJH4SJJZwNrq1u/SxKItiQRRLTjL8DDkn5MWfUDgO0v1Gks6Urg88ADlKGlLZJusL2j+VBj0mVoKKJBkr5l+xpJ/wS+OHjf9uaaz/kNcInt/dX1cmCX7QsaDTiC9AgimnZhNc7/V2DLCM9Z1EsClQNklV+0JIkgolm3AfdRJnwf6/u+KPsIVtZ8zr2SdgLfqa6vAu5pKsiIfhkaimiBpK22N43Q/rPArynLSAU8BEzZvqmhECOOSyKIWIAkPW77jQPf25MKZdGGDA1FLCCSNgHXASsl7em7tQx4eDxRxakuPYKIBUTSGcBZzLKhzPbB8UQVp7okgoiICZflaBEREy6JICJiwiURRERMuCSCiIgJ918Jt59201pDUwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a16076b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cmap = sb.diverging_palette(220, 10, as_cmap=True)\n",
    "\n",
    "sb.heatmap(wine.corr(), cmap=cmap, vmax=.3, center=0,\n",
    "            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note that we won't have the formula option with sklearn\n",
    "wine_X = wine.drop(['free.sulfur.dioxide', 'density', 'quality', 'color', 'white', 'good'], axis=1)\n",
    "wine_y = wine['good']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standardize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import scale\n",
    "\n",
    "wine_X = scale(wine_X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Data Partition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.32319841, -1.09137315, -0.6787934 , ...,  0.13372214,\n",
       "        -1.15103964, -0.91546416],\n",
       "       [-0.55179227, -0.54467214,  0.90406569, ..., -0.11507303,\n",
       "        -0.00852369, -1.16701119],\n",
       "       [-0.24322981, -0.6054167 ,  0.35350601, ..., -0.17727182,\n",
       "        -1.28545329,  1.60000607],\n",
       "       ..., \n",
       "       [ 0.21961388,  1.76362102, -0.33469359, ...,  0.25811972,\n",
       "         0.32751041, -0.58006813],\n",
       "       [-0.08894858, -0.1802048 , -0.12823371, ..., -1.73224161,\n",
       "         7.31701977, -0.83161516],\n",
       "       [-0.01180796, -1.21286227,  0.21586609, ...,  1.50209555,\n",
       "        -0.94941918, -0.49621913]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "wine_train, wine_test, y_train, y_test = train_test_split(wine_X, wine_y, test_size=.2, random_state=1234)\n",
    "wine_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regularized Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this we'll use LogisticRegression from sklearn.  A key difference from the R elasticnet version is that it does lasso *or* ridge style penalty, and by default the latter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logreg = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "regreg_opts = {'C': np.logspace(-4, 1, 50)}\n",
    "regreg_estimator = GridSearchCV(logreg, cv=10, param_grid=regreg_opts)\n",
    "results_regreg = regreg_estimator.fit(wine_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'C': array([  1.00000e-04,   1.26486e-04,   1.59986e-04,   2.02359e-04,\n",
       "         2.55955e-04,   3.23746e-04,   4.09492e-04,   5.17947e-04,\n",
       "         6.55129e-04,   8.28643e-04,   1.04811e-03,   1.32571e-03,\n",
       "         1.67683e-03,   2.12095e-03,   2.68270e-03,   3.39322e-03,\n",
       "         4.2919...    3.08884e+00,   3.90694e+00,   4.94171e+00,   6.25055e+00,\n",
       "         7.90604e+00,   1.00000e+01])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_regreg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get training prediction and plot over the values of C."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.73426977102174329"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_regreg.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAE/xJREFUeJzt3X9s3Hd9x/HXK45TXCqWVPEQcX4yRQG6smY9hUIFGrDSTBtNBNqIga1sk7IJmkE0MSWoIl1AgimwMmnVtIx1YwtKVtLMi6AiVIT9EEqqOCQiJCUhZLSx01JDcAcjXRP3vT/unFzsO9/Hl7PP+X6eD8mKv5/v5+77/uROr/v6c98fjggBAPIwq90FAACmD6EPABkh9AEgI4Q+AGSE0AeAjBD6AJARQh8AMkLoA0BGkkLf9mrbJ22ftr2pxvoHbR+t/JyyPVy1bqRq3d5WFg8AmBw3OiPXdoekU5LukjQg6ZCk3og4Uaf/BkkrI+IPKss/i4ibUguaP39+LF26NLU7AEDS4cOHfxQR3Y36zU54rlWSTkfEGUmyvUvSGkk1Q19Sr6QtqYWOtXTpUvX39zf7cADIku2nUvqlTO/0SDpbtTxQaau10SWSlknaX9X8Mtv9tg/aXlvncesrffqHhoZS6gYANCEl9F2jrd6c0DpJuyNipKptcUSUJL1X0uds/9K4J4vYHhGliCh1dzf86wQA0KSU0B+QtKhqeaGkc3X6rpO0s7ohIs5V/j0j6d8lrZx0lQCAlkgJ/UOSltteZnuOysE+7igc2yskzZN0oKptnu0bKr/Pl3Sn6n8XAACYYg2/yI2IS7bvk7RPUoekhyPiuO2tkvojYvQDoFfSrrj6cKDXSvpb2y+p/AHz6XpH/QAApl7DQzanW6lUCo7eAYDJsX248v3phDgjFwAyQugDQEYIfQDICKEPABkh9AEgI4Q+AGSE0AeAjBD6AJARQh8AMkLoA0BGCH0AyAihDwAZIfQBICOEPgBkhNAHgIwQ+gCQEUIfADJC6ANARgh9AMgIoQ8AGSH0ASAjhD4AZITQB4CMEPoAkBFCHwAyQugDQEYIfQDICKEPABkh9AEgI4Q+AGSE0AeAjBD6AJCRpNC3vdr2SdunbW+qsf5B20crP6dsD49Z/wrbg7b/ulWFAwAmb3ajDrY7JD0k6S5JA5IO2d4bESdG+0TExqr+GyStHPM0n5D0Hy2pGADQtJQ9/VWSTkfEmYh4UdIuSWsm6N8raefogu3bJb1S0teupVAAwLVLCf0eSWerlgcqbePYXiJpmaT9leVZkj4r6aMTbcD2etv9tvuHhoZS6gYANCEl9F2jLer0XSdpd0SMVJY/KOmxiDhbp3/5ySK2R0QpIkrd3d0JJQEAmtFwTl/lPftFVcsLJZ2r03edpA9VLb9R0pttf1DSTZLm2P5ZRIz7MhgAMPVSQv+QpOW2l0kaVDnY3zu2k+0VkuZJOjDaFhHvq1r/AUklAh8A2qfh9E5EXJJ0n6R9kp6U9EhEHLe91fY9VV17Je2KiHpTPwCANvNMy+hSqRT9/f3tLgMAriu2D0dEqVE/zsgFgIwQ+gCQEUIfADJC6ANARgh9AMgIoQ8AGSH0ASAjhD4AZITQB4CMEPoAkBFCHwAyQugDQEYIfQDICKEPABkh9AEgIyl3zgLq6jsyqG37Turc8AUtmNulj969QmtX9lx325iJ204x0+tLVZRxNGs6x1+o0L+/75h2PnFWIxHqsNX7hkX65Npbx/Wr/g/+ha5O2dLwzy9qwdwuvfU13frGd4eu+s+XVPMFafRCpbyQzbzY1/IGaeWbq+/IoDbvOaYLF0ckSYPDF7R5zzFJatkbdjq2MRO3nWKm15eqKONo1nSPvzB3zrq/75h2HHx6XPv771h8Ofj7jgzqgb3HNXzhYvLzds6yZOniyJX/p67ODr379h49enjw8gs12v6pd916+QOh+oUcu360nkZ9xmrmMa14bC13fnq/BocvjGvvmdulb25626Sfr13bmInbTjHT60tVlHE0q1Xjz+7OWbUCX5K+ePBp9R0Z1G1//jV95F+OTirwJeniS3FV4EvShYsj2vnE2avCc7R9276Tksp/GUy0PrXPWM08phWPreVcjTfqRO0zdRszcdspZnp9qYoyjmZN9/gLEfrv+7sDddeFpI1NhH0jI3X+Qhp9oVJeyGZe7Gt5g7T6zbVgbtek2mfqNmbitlPM9PpSFWUczZru8Rci9L/5/fMTrp+KCawOu2b76AuV8kI282Jfyxuk1W+uj969Ql2dHVe1dXV2XP4epBWmYxszcdspZnp9qYoyjmZN9/gLEfpTqXOW1dlxdcB3dXao9w2LJnyhUl7IZl7sa3mDtPrNtXZljz71rlvVM7dLVnkOstnvB9q5jZm47RQzvb5URRlHs6Z7/IX4Infppq80vb0bO2fphs4Ojt4BcF1L/SK3EKG//GNf0cWXJredeTd2ass7byHwABRCaugX4jj9VctubjivX21uV6eOfPwdU1gRAMxMhZjTP3jmJ8l9uzo79MA9t0xhNQAwcxViT7/e4ZNj9TCHDSBzhQj9DnvC4K8+KxcAclaI6Z07Xj2vZvsNs2fpc++5jcAHgIpChP4Pflz7jNL5N93AVA4AVClE6Od+7Q4ASFWI0M/92h0AkCop9G2vtn3S9mnbm2qsf9D20crPKdvDlfYltg9X2o/b/uNWD0Di2h0AkKrh0Tu2OyQ9JOkuSQOSDtneGxEnRvtExMaq/hskrawsPiPpTRHxf7ZvkvSdymPPtXIQo/P2XF4AACaWcsjmKkmnI+KMJNneJWmNpBN1+vdK2iJJEfFiVfsNmsLppLUrewh5AGggJYR7JJ2tWh6otI1je4mkZZL2V7Utsv3tynP8Rav38gEA6VL29GtdOL7emVDrJO2OiMu3Z4qIs5Jeb3uBpD7buyPih1dtwF4vab0kLV68OKnwsbh6JAA0lrKnPyBpUdXyQkn19tbXSdpZa0VlD/+4pDfXWLc9IkoRUeru7k4o6Wqj934dHL6g0JUbC/cdGZz0cwFAkaWE/iFJy20vsz1H5WDfO7aT7RWS5kk6UNW20HZX5fd5ku6U1NwNWSfQ6nu/AkBRNZzeiYhLtu+TtE9Sh6SHI+K47a2S+iNi9AOgV9KuuPoC/a+V9FnbofI00Wci4lhrh8DJWQCQKumCaxHxmKTHxrR9fMzyAzUe97ik119DfUkWzO3SYI2A5+QsALhaIc7I5eQsAEhTiNBfu7JH7769Rx0uH2jUYevdt3PcPgCMVYjQ7zsyqEcPD16+pv5IhB49PMjROwAwRiFCn6N3ACBNIUKfo3cAIE0hQp9LKwNAmkKEPkfvAECaQtwYnUsrA0CaQuzpS1L/U+f17PMvKCQ9+/wL6n/qfLtLAoAZpxB7+vf3HdOOg09fXh6JuLz8ybW3tqssAJhxCrGnv/OJs5NqB4BcFSL0R6L25f3rtQNArgoR+qOXX0htB4BcFSL0e9+waFLtAJCrQoR+acnN4wYyq9IOALiiEKG/bd9JvTSm7aVKOwDgikKEPtfeAYA0hQh9rr0DAGkKEfpcewcA0hTijFyuvQMAaQoR+lI5+Al5AJhYIaZ3AABpCH0AyAihDwAZIfQBICOEPgBkpDBH7/QdGeSQTQBooBCh33dkUJv3HNOFiyOSpMHhC9q855gkEfwAUKUQ0zvb9p28HPijLlwc4YJrADBGIUKfC64BQJpChD4XXAOANIUI/be+pntS7QCQq6TQt73a9knbp21vqrH+QdtHKz+nbA9X2m+zfcD2cdvftv2eVg9Akr7x3aFJtQNArhoevWO7Q9JDku6SNCDpkO29EXFitE9EbKzqv0HSysrizyX9XkR8z/YCSYdt74uI4VYOgjl9AEiTsqe/StLpiDgTES9K2iVpzQT9eyXtlKSIOBUR36v8fk7Sc5JaPufCnD4ApEkJ/R5JZ6uWBypt49heImmZpP011q2SNEfS9ydf5sS4iQoApEk5Ocs12qJO33WSdkfEVQfN236VpH+WdG9EjL2HuWyvl7RekhYvXpxQ0tW4iQoApEnZ0x+QtKhqeaGkc3X6rlNlameU7VdI+oqk+yPiYK0HRcT2iChFRKm7u7nZn/6nzuvZ519QSHr2+RfU/9T5pp4HAIosJfQPSVpue5ntOSoH+96xnWyvkDRP0oGqtjmS/lXSP0XEl1pT8nj39x3TjoNPayTKf4CMRGjHwad1f9+xqdokAFyXGoZ+RFySdJ+kfZKelPRIRBy3vdX2PVVdeyXtiojqqZ/fkfQWSR+oOqTzthbWL0na+cTZSbUDQK6SLrgWEY9JemxM28fHLD9Q43E7JO24hvqSjETtrxjqtQNArgpxRm6Ha33XXL8dAHJViNC/49XzJtUOALkqROj/4Me1z7yt1w4AuSpE6HMZBgBIU4jQn3tj56TaASBXhQj9F8bcNatROwDkqhChf+HiuCs7TNgOALkqROgDANIUIvTn1Zm7r9cOALkqROhveect6uy4+kSszg5ryztvaVNFADAzJV2GYabj0soAkKYQoS+Vg5+QB4CJFWJ6BwCQhtAHgIwUZnqn78ggc/oA0EAhQr/vyKA27zmmC5UzcAeHL2jznvJdswh+ALiiENM72/advBz4oy5cHNG2fSfbVBEAzEyFCP3BOlfTrNcOALkqROgDANIQ+gCQEUIfADJSiNDv6qw9jHrtAJCrQqTip971+nEDmVVpBwBcUYjj9LngGgCkKUToS1xwDQBSFGJ6BwCQhtAHgIwQ+gCQEUIfADJC6ANARgh9AMgIoQ8AGUkKfdurbZ+0fdr2phrrH7R9tPJzyvZw1bqv2h62/eVWFg4AmLyGJ2fZ7pD0kKS7JA1IOmR7b0ScGO0TERur+m+QtLLqKbZJulHSH7WqaABAc1L29FdJOh0RZyLiRUm7JK2ZoH+vpJ2jCxHxdUk/vaYqAQAtkRL6PZLOVi0PVNrGsb1E0jJJ+6+9NABAq6WEvmu0RZ2+6yTtjoiROutrb8Beb7vfdv/Q0NBkHgoAmISU0B+QtKhqeaGkc3X6rlPV1E6qiNgeEaWIKHV3d0/24QCARCmhf0jSctvLbM9ROdj3ju1ke4WkeZIOtLZEAECrNAz9iLgk6T5J+yQ9KemRiDhue6vte6q69kraFRFXTf3Y/i9JX5L0dtsDtu9uXfkAgMnwmIxuu1KpFP39/e0uAwCuK7YPR0SpUT/OyAWAjBD6AJARQh8AMkLoA0BGCH0AyAihDwAZIfQBICOEPgBkhNAHgIwQ+gCQEUIfADJC6ANARgh9AMgIoQ8AGSH0ASAjhD4AZITQB4CMEPoAkBFCHwAyQugDQEYIfQDICKEPABkh9AEgI4Q+AGSE0AeAjBD6AJARQh8AMkLoA0BGCH0AyAihDwAZIfQBICOEPgBkhNAHgIwkhb7t1bZP2j5te1ON9Q/aPlr5OWV7uGrdvba/V/m5t5XFAwAmZ3ajDrY7JD0k6S5JA5IO2d4bESdG+0TExqr+GyStrPx+s6QtkkqSQtLhymN/0tJRAACSpOzpr5J0OiLORMSLknZJWjNB/15JOyu/3y3p8Yg4Xwn6xyWtvpaCAQDNSwn9Hklnq5YHKm3j2F4iaZmk/ZN5rO31tvtt9w8NDaXUDQBoQkrou0Zb1Om7TtLuiBiZzGMjYntElCKi1N3dnVASAKAZKaE/IGlR1fJCSefq9F2nK1M7k30sAGCKpYT+IUnLbS+zPUflYN87tpPtFZLmSTpQ1bxP0jtsz7M9T9I7Km0AgDZoePRORFyyfZ/KYd0h6eGIOG57q6T+iBj9AOiVtCsiouqx521/QuUPDknaGhHnWzsEAEAqV2X0jFAqlaK/v7/dZQDAdcX24YgoNerHGbkAkBFCHwAyQugDQEYIfQDIyIz7Itf2kKSnruEp5kv6UYvKuV7kNubcxisx5lxcy5iXRETDs1tnXOhfK9v9Kd9gF0luY85tvBJjzsV0jJnpHQDICKEPABkpYuhvb3cBbZDbmHMbr8SYczHlYy7cnD4AoL4i7ukDAOooTOg3uo9v0dheZPsbtp+0fdz2h9td03Sx3WH7iO0vt7uW6WB7ru3dtr9beb3f2O6appLtjZX39Hds77T9snbXNBVsP2z7OdvfqWq72fbjlXuKP165OnFLFSL0q+7j+xuSXiep1/br2lvVlLsk6U8j4rWS7pD0oQzGPOrDkp5sdxHT6K8kfTUiXiPpV1TgsdvukfQnkkoR8csqX9l3XXurmjL/qPG3j90k6esRsVzS1yvLLVWI0Nfk7+N73YuIZyLiW5Xff6pyENS8jWWR2F4o6Tclfb7dtUwH26+Q9BZJfy9JEfFiRAy3t6opN1tSl+3Zkm5UQW+8FBH/KWnspebXSPpC5fcvSFrb6u0WJfST7+NbRLaXSlop6Yn2VjItPifpzyS91O5CpsmrJQ1J+ofKlNbnbb+83UVNlYgYlPQZSU9LekbS8xHxtfZWNa1eGRHPSOUdO0m/2OoNFCX0J3Mf30KxfZOkRyV9JCL+p931TCXbvyXpuYg43O5aptFsSb8q6W8iYqWk/9UU/Mk/U1TmsNdIWiZpgaSX235/e6sqlqKEfpb34rXdqXLgfzEi9rS7nmlwp6R7bP9A5Sm8t9ne0d6SptyApIGIGP0rbrfKHwJF9euS/jsihiLioqQ9kt7U5pqm0w9tv0qSKv8+1+oNFCX0k+7jWyS2rfI875MR8Zftrmc6RMTmiFgYEUtVfo33R0Sh9wIj4llJZyv3oJakt0s60caSptrTku6wfWPlPf52FfiL6xr2Srq38vu9kv6t1RtoeI/c60G9+/i2uaypdqek35V0zPbRStvHIuKxNtaEqbFB0hcrOzRnJP1+m+uZMhHxhO3dkr6l8hFqR1TQM3Nt75T0a5Lm2x6QtEXSpyU9YvsPVf4A/O2Wb5czcgEgH0WZ3gEAJCD0ASAjhD4AZITQB4CMEPoAkBFCHwAyQugDQEYIfQDIyP8DLEeq51C2RRIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1f7d27b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.scatter(results_regreg.cv_results_['param_C'],\n",
    "            results_regreg.cv_results_['mean_test_score'])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Examine the best value of C."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 3.088843596477485}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_regreg.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.21769231  0.14      ]\n",
      " [ 0.09769231  0.54461538]]\n"
     ]
    }
   ],
   "source": [
    "import sklearn.metrics as metrics\n",
    "\n",
    "regreg_predict = results_regreg.predict(wine_test)\n",
    "\n",
    "# create confusion matrix, and accuracy\n",
    "cm = metrics.confusion_matrix(y_test, regreg_predict)\n",
    "cm_prob = cm / np.sum(cm)  # as probs\n",
    "print(cm_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.762308</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy\n",
       "0  0.762308"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = metrics.accuracy_score(y_test, regreg_predict)\n",
    "acc = pd.DataFrame(np.array([acc]), columns=['Accuracy'])\n",
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76230769230769235"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_regreg.score(wine_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the following, to compare with the document, recall of the positive class is also known as 'sensitivity', while recall of the negative class is 'specificity'.  Likelise precision goes with positive/negative predictive value.  F1 is the harmonic mean of the two."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        Bad       0.69      0.61      0.65       465\n",
      "       Good       0.80      0.85      0.82       835\n",
      "\n",
      "avg / total       0.76      0.76      0.76      1300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(metrics.classification_report(y_test, regreg_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## k-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not much different from the R implementation in the document."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = [np.arange(3,11,2).tolist(), [25, 51, 101]]\n",
    "grid = sum(grid, [])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "knn = KNeighborsClassifier()\n",
    "\n",
    "knn_opts = {'n_neighbors':  grid}\n",
    "knn_estimator = GridSearchCV(knn, cv=10, param_grid=knn_opts)\n",
    "results_knn = knn_estimator.fit(wine_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75428131614392924"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_knn.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAGCVJREFUeJzt3X+QVtdh3vHv40XIxBoN2FprJH5pNSFETJWC/YbIVtVJlWqEGo9gMk7LRrXl1B3GcVBTuaZCnekvUk+kkR0a2xq12JIszThgDaGIaeVix8huZ4o1vBQiBCp4g5OyixKtpt5ErqkQ+Okf71n7srzL3v3FsrvPZ+ad3XPuec+eo4vus/fc++6VbSIiIt4x1QOIiIgrQwIhIiKABEJERBQJhIiIABIIERFRJBAiIgKoGQiS1kg6LqlH0uY227dKOlxeJyQNlPq/U6k/LOn/SVpXtnVJeknS9yR9TdLciZ1aRESMhkb6HIKkDuAEcBfQCxwAum0fG6b9A8Aq2/9oSP27gR5gke0fSXoO2GV7h6T/APyJ7SfGPaOIiBiTOmcIq4Ee2ydtnwV2AGsv0b4b2N6m/sPA10sYCLgT2Fm2PQOsqz/siIiYaHNqtFkInKqUe4FfatdQ0lKgC9jXZvN64PfL9+8BBmyfq/S5cKSBXHfddb7ppptqDDkiIgYdPHjwDdudI7WrEwhqUzfcOtN6YKft8xd0IN0A3ArsHW2fkjYAGwCWLFlCs9msMeSIiBgk6c/rtKuzZNQLLK6UFwGnh2m7nvbLRX8f+E+23y7lN4D5kgYDadg+bW+z3bDd6OwcMeAiImKM6gTCAWBZuStoLq2D/p6hjSQtBxYA+9v0ccF1BbeuZL9I67oCwP3A86MbekRETKQRA6Gs82+ktdzzKvCc7aOStki6t9K0G9jhIbctSbqJ1hnGd4Z0/RDwKUk9tK4pPDnWSURExPiNeNvplaTRaDjXECIiRkfSQduNkdrlk8oREQEkECIiokggREQEkECIiIgigRAREUACISIiigRCREQACYSIiCgSCBERASQQIiKiSCBERASQQIiIiCKBEBERQAIhIiKKBEJERAAJhIiIKGoFgqQ1ko5L6pG0uc32rZIOl9cJSQOVbUskfUPSq5KOlSeoIekrkr5fed/KiZpURESM3pyRGkjqAB4H7gJ6gQOS9tg+NtjG9oOV9g8AqypdPAt8xvY3JV0D/LiybZPtneOcQ0RETIA6ZwirgR7bJ22fBXYAay/RvhvYDiBpBTDH9jcBbP/Q9o/GOeaIiJgEdQJhIXCqUu4tdReRtBToAvaVqp8DBiTtknRI0mPljGPQZyS9XJacrh6mzw2SmpKa/f39NYYbERFjUScQ1KbOw7RdD+y0fb6U5wB3AJ8GfhG4GfhY2fYw8POl/t3AQ+06tL3NdsN2o7Ozs8ZwIyJiLOoEQi+wuFJeBJwepu16ynJR5b2HynLTOWA38D4A26+55S3gaVpLUxERMUXqBMIBYJmkLklzaR309wxtJGk5sADYP+S9CyQN/mp/J3CstL+hfBWwDnhlrJOIiIjxG/EuI9vnJG0E9gIdwFO2j0raAjRtD4ZDN7DDtivvPS/p08C3yoH/IPClsvmrJSgEHAY+MWGzioiIUVPl+H3FazQabjabUz2MiIhpRdJB242R2uWTyhERASQQIiKiSCBERASQQIiIiCKBEBERQAIhIiKKBEJERAAJhIiIKBIIEREBJBAiIqJIIEREBJBAiIiIIoEQERFAAiEiIooEQkREADUDQdIaSccl9Uja3Gb7VkmHy+uEpIHKtiWSviHpVUnHJN1U6rskvSTpe5K+Vp7GFhERU2TEQJDUATwO3AOsALolrai2sf2g7ZW2VwJfAHZVNj8LPGb7FlrPTX691D8KbLW9DPgB8PHxTiYiIsauzhnCaqDH9knbZ4EdwNpLtO8GtgOU4Jhj+5sAtn9o+0flcZp3AjvLe56h9VzliIiYInUCYSFwqlLuLXUXkbQU6AL2laqfAwYk7ZJ0SNJj5YzjPcCA7XMj9RkREZdHnUBQm7rhHsS8Hthp+3wpzwHuAD4N/CJwM/Cx0fQpaYOkpqRmf39/jeFGRMRY1AmEXmBxpbwIOD1M2/WU5aLKew+V5aZzwG7gfcAbwHxJc0bq0/Y22w3bjc7OzhrDjYiIsagTCAeAZeWuoLm0Dvp7hjaStBxYAOwf8t4FkgaP5HcCx2wbeBH4cKm/H3h+bFOIiIiJMGIglN/sNwJ7gVeB52wflbRF0r2Vpt3AjnKwH3zveVrLRd+SdITWUtGXyuaHgE9J6qF1TeHJiZhQRESMjSrH7yteo9Fws9mc6mFEREwrkg7abozULp9UjogIIIEQERFFAiEiIoAEQkREFAmEiIgAEggREVEkECIiAkggREREkUCIiAgggRAREUUCISIigARCREQUCYSIiAASCBERUSQQIiICSCBERERRKxAkrZF0XFKPpM1ttm+VdLi8TkgaqGw7X9m2p1L/FUnfr2xbOTFTioiIsZgzUgNJHcDjwF1AL3BA0h7bxwbb2H6w0v4BYFWlizO2hzvYb7K9c0wjj4iICVXnDGE10GP7pO2zwA5g7SXadwPbJ2JwERFx+dQJhIXAqUq5t9RdRNJSoAvYV6l+p6SmpO9KWjfkLZ+R9HJZcrp6NAOPiIiJNeKSEaA2dR6m7Xpgp+3zlboltk9LuhnYJ+mI7T8FHgb+ApgLbAMeArZc9MOlDcAGgCVLltQY7oV2H+rjsb3HOT1whhvnz2PT3ctZt6ptnkVEzGp1zhB6gcWV8iLg9DBt1zNkucj26fL1JPBtyvUF26+55S3gaVpLUxexvc12w3ajs7OzxnB/avehPh7edYS+gTMY6Bs4w8O7jrD7UN+o+omImA3qBMIBYJmkLklzaR309wxtJGk5sADYX6lbMLgUJOk64HbgWCnfUL4KWAe8Mr6pXOyxvcc58/b5C+rOvH2ex/Yen+gfFREx7Y24ZGT7nKSNwF6gA3jK9lFJW4Cm7cFw6AZ22K4uJ90C/EdJP6YVPo9U7k76qqROWktSh4FPTMyUfur0wJlR1UdEzGZ1riFg+wXghSF1/2pI+d+0ed//AG4dps87a49yjG6cP4++Ngf/G+fPm+wfHREx7czoTypvuns5867quKBu3lUdbLp7+RSNKCLiylXrDGG6GrybKHcZRUSMbEYHArRCIQEQETGyGb1kFBER9SUQIiICSCBERESRQIiICCCBEBERRQIhIiKABEJERBQJhIiIABIIERFRJBAiIgJIIERERJFAiIgIIIEQERFFrUCQtEbScUk9kja32b5V0uHyOiFpoLLtfGXbnkp9l6SXJH1P0tfK4zkjImKKjBgIkjqAx4F7gBVAt6QV1Ta2H7S90vZK4AvArsrmM4PbbN9bqX8U2Gp7GfAD4OPjnEtERIxDnTOE1UCP7ZO2zwI7gLWXaN8NbL9Uh5IE3AnsLFXPAOtqjCUiIiZJnUBYCJyqlHtL3UUkLQW6gH2V6ndKakr6rqTBg/57gAHb52r0uaG8v9nf319juBERMRZ1npimNnUepu16YKft85W6JbZPS7oZ2CfpCPDXdfu0vQ3YBtBoNIb7uRERMU51zhB6gcWV8iLg9DBt1zNkucj26fL1JPBtYBXwBjBf0mAgXarPiIi4DOoEwgFgWbkraC6tg/6eoY0kLQcWAPsrdQskXV2+vw64HThm28CLwIdL0/uB58czkYiIGJ8RA6Gs828E9gKvAs/ZPippi6TqXUPdwI5ysB90C9CU9Ce0AuAR28fKtoeAT0nqoXVN4cnxTyciIsZKFx6/r2yNRsPNZnOqhzGt7T7Ux2N7j3N64Aw3zp/HpruXs25V2+v5ETFDSDpouzFSuzoXlWOG2H2oj4d3HeHM261r/n0DZ3h41xGAhEJE5E9XzCaP7T3+kzAYdObt8zy29/gUjSgiriQJhFnk9MCZUdVHxOySJaNRmO7r7zfOn0dfm4P/jfPnTcFoIuJKM2vPEHYf6uP2R/bRtfm/cPsj+9h9qG/E9g/vOkLfwBnMT9ffR3rflWTT3cuZd1XHBXXzrupg093Lp2hEEXElmZWBMJaD+0xYf1+3aiG/92u3snD+PAQsnD+P3/u1W6fVWU5ETJ5ZuWR0qYP7cAfHmbL+vm7VwgRARLQ1K88QxnJwH26dPevvETFTzMpAGMvBPevvETHTzcpAGMvBPevvETHTzcprCIMH8dHeQpr194iYyWZlIMDwB/fp/lmDiIixmrWB0E7+1k9EzGaz8hrCcGbCZw0iIsYqgVAxUz5rEBExFrUCQdIaSccl9Uja3Gb7VkmHy+uEpIEh26+V1Cfpi5W6b5c+B9/33vFPZ3zyWYOImM1GDARJHcDjwD3ACqBb0opqG9sP2l5peyXwBWDXkG5+F/hOm+7vG3yf7dfHNIMJlM8aRMRsVucMYTXQY/uk7bPADmDtJdp3A9sHC5LeD1wPfGM8A70c8lmDiJjN6txltBA4VSn3Ar/UrqGkpUAXsK+U3wF8DvgI8Ctt3vK0pPPAHwH/zlfA8zzzWYOImK3qnCGoTd1wB+71wE7bg7fqfBJ4wfapNm3vs30rcEd5faTtD5c2SGpKavb399cYbkREjEWdQOgFFlfKi4DTw7RdT2W5CPgAsFHSnwGfBT4q6REA233l65vAH9JamrqI7W22G7YbnZ2dNYYbERFjUWfJ6ACwTFIX0EfroP8bQxtJWg4sAPYP1tm+r7L9Y0DD9mZJc4D5tt+QdBXwIeCPxzORiIgYnxEDwfY5SRuBvUAH8JTto5K2AE3be0rTbmBHzesAVwN7Sxh00AqDL41pBhERMSF0BVzHra3RaLjZbE71MCIiphVJB203RmqXTypHRASQQIiIiCKBEBERQAIhIiKKBEJERAAJhIiIKPLEtIhRymNWY6ZKIESMQh6zGjNZAiFiFC71mNUEQky0y302mkCIGIU8ZjUul6k4G81F5YhRyGNW43K51NnoZEkgRIxCHrMal8tUnI0mECJGIY9ZjctlKs5Gcw0hYpTymNW4HDbdvfyCawgw+WejCYSIiCvQ4C8ducsoIiIu+9lorWsIktZIOi6pR9LmNtu3SjpcXickDQzZfq2kPklfrNS9X9KR0ufnJWn804mIiLEaMRAkdQCPA/cAK4BuSSuqbWw/aHul7ZXAF4BdQ7r5XeA7Q+qeADYAy8przZhmEBERE6LOGcJqoMf2SdtngR3A2ku07wa2DxYkvR+4HvhGpe4G4Frb+8szmJ8F1o1h/BERMUHqBMJC4FSl3FvqLiJpKdAF7CvldwCfAza16bO3Tp8REXF51AmEdmv7HqbtemCn7cH7pD4JvGD71JB2tfuUtEFSU1Kzv7+/xnAjImIs6txl1AssrpQXAaeHabse+O1K+QPAHZI+CVwDzJX0Q+APSj8j9ml7G7ANoNFoDBdEERExTnUC4QCwTFIX0EfroP8bQxtJWg4sAPYP1tm+r7L9Y0DD9uZSflPSbcBLwEdpXYyOiIgpMuKSke1zwEZgL/Aq8Jzto5K2SLq30rQb2FEuEtfxW8CXgR7gT4Gvj2rkERExoVT/+D31Go2Gm83mVA8jImJakXTQdmOkdvnjdhERASQQIiKiSCBERASQQIiIiCKBEBERQAIhIiKKBEJERAAJhIiIKBIIEREBJBAiIqJIIEREBJBAiIiIIoEQERFAAiEiIooEQkREAAmEiIgoagWCpDWSjkvqkbS5zfatkg6X1wlJA6V+qaSDpf6opE9U3vPt0ufg+947cdOKiIjRGvGZypI6gMeBu4Be4ICkPbaPDbax/WCl/QPAqlJ8Dfig7bckXQO8Ut57umy/z3YegRYRcQWoc4awGuixfdL2WWAHsPYS7buB7QC2z9p+q9RfXfPnRUTEFKhzgF4InKqUe0vdRSQtBbqAfZW6xZJeLn08Wjk7AHi6LBf9S0kaps8NkpqSmv39/TWGGxERY1EnENodqD1M2/XATtvnf9LQPmX7F4CfBe6XdH3ZdJ/tW4E7yusj7Tq0vc12w3ajs7OzxnAjImIs6gRCL7C4Ul4EnB6m7XrKctFQ5czgKK2DP7b7ytc3gT+ktTQVERFTpE4gHACWSeqSNJfWQX/P0EaSlgMLgP2VukWS5pXvFwC3A8clzZF0Xam/CvgQ8Mp4JxMREWM34l1Gts9J2gjsBTqAp2wflbQFaNoeDIduYIft6nLSLcDnJJnW0tNnbR+R9C5gbwmDDuCPgS9N3LQiImK0dOHx+8rWaDTcbOYu1YiI0ZB00HZjpHa5DTQiIoAEQkREFAmEiIgAEggREVEkECIiAkggREREkUCIiAgggRAREUUCISIigARCREQUCYSIiAASCBERUSQQIiICSCBERESRQIiICKBmIEhaI+m4pB5Jm9ts3yrpcHmdkDRQ6pdKOljqj0r6ROU975d0pPT5eUntnt0cERGXyYhPTJPUATwO3EXr+coHJO2xfWywje0HK+0fAFaV4mvAB22/Jeka4JXy3tPAE8AG4LvAC8Aa4OsTM62IiBitOmcIq4Ee2ydtnwV2AGsv0b4b2A5g+6ztt0r91YM/T9INwLW295dHbj4LrBvjHCIiYgLUCYSFwKlKubfUXUTSUqAL2FepWyzp5dLHo+XsYGHpZ8Q+IyLi8qgTCO3W9od7EPN6YKft8z9paJ+y/QvAzwL3S7p+NH1K2iCpKanZ399fY7gRETEWdQKhF1hcKS8CTg/Tdj1luWiocmZwFLij9LmoTp+2t9lu2G50dnbWGG5ERIxFnUA4ACyT1CVpLq2D/p6hjSQtBxYA+yt1iyTNK98vAG4Hjtt+DXhT0m3l7qKPAs+PezYRETFmI95lZPucpI3AXqADeMr2UUlbgKbtwXDoBnaUi8SDbgE+J8m0lok+a/tI2fZbwFeAebTuLsodRhERU0gXHr+vbI1Gw81mc6qHERExrUg6aLsxUrt8UjkiIoAEQkREFAmEiIgAEggREVEkECIiAkggREREkUCIiAgggRAREUUCISIigARCREQUCYSIiAASCBERUSQQIiICSCBERESRQIiICGCaPQ9BUj/w5202XQe8cZmHcyXIvGeXzHt2mch5L7U94jOIp1UgDEdSs87DH2aazHt2ybxnl6mYd5aMIiICSCBEREQxUwJh21QPYIpk3rNL5j27XPZ5z4hrCBERMX4z5QwhIiLGaVoHgqQ1ko5L6pG0earHM1kkLZb0oqRXJR2V9Dul/t2Svinpe+Xrgqke62SQ1CHpkKT/XMpdkl4q8/6apLlTPcbJIGm+pJ2S/lfZ9x+YDftc0oPl3/krkrZLeudM3OeSnpL0uqRXKnVt969aPl+OdS9Let9kjGnaBoKkDuBx4B5gBdAtacXUjmrSnAP+me1bgNuA3y5z3Qx8y/Yy4FulPBP9DvBqpfwosLXM+wfAx6dkVJPvD4D/avvngb9J67/BjN7nkhYC/wRo2P4bQAewnpm5z78CrBlSN9z+vQdYVl4bgCcmY0DTNhCA1UCP7ZO2zwI7gLVTPKZJYfs12/+zfP8mrQPDQlrzfaY0ewZYNzUjnDySFgG/Cny5lAXcCewsTWbqvK8F/jbwJIDts7YHmAX7HJgDzJM0B/gZ4DVm4D63/d+A/zOkerj9uxZ41i3fBeZLumGixzSdA2EhcKpS7i11M5qkm4BVwEvA9bZfg1ZoAO+dupFNmn8P/HPgx6X8HmDA9rlSnqn7/WagH3i6LJd9WdK7mOH73HYf8Fngf9MKgr8CDjI79jkMv38vy/FuOgeC2tTN6FumJF0D/BHwT23/9VSPZ7JJ+hDwuu2D1eo2TWfifp8DvA94wvYq4P8yw5aH2ilr5muBLuBG4F20lkuGmon7/FIuy7/76RwIvcDiSnkRcHqKxjLpJF1FKwy+antXqf7LwdPG8vX1qRrfJLkduFfSn9FaEryT1hnD/LKcADN3v/cCvbZfKuWdtAJipu/zvwt833a/7beBXcAHmR37HIbfv5fleDedA+EAsKzcfTCX1oWnPVM8pklR1s2fBF61/fuVTXuA+8v39wPPX+6xTSbbD9teZPsmWvt3n+37gBeBD5dmM27eALb/AjglaXmp+hXgGDN8n9NaKrpN0s+Uf/eD857x+7wYbv/uAT5a7ja6DfirwaWliTStP5gm6e/R+o2xA3jK9memeEiTQtLfAv47cISfrqX/C1rXEZ4DltD6H+nXbQ+9SDUjSPpl4NO2PyTpZlpnDO8GDgH/0PZbUzm+ySBpJa2L6XOBk8Bv0volbkbvc0n/FvgHtO6uOwT8Y1rr5TNqn0vaDvwyrb9q+pfAvwZ202b/lnD8Iq27kn4E/Kbt5oSPaToHQkRETJzpvGQUERETKIEQERFAAiEiIooEQkREAAmEiIgoEggREQEkECIiokggREQEAP8fNmpW89flkDEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1f502908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(results_knn.cv_results_['param_n_neighbors'],\n",
    "            results_knn.cv_results_['mean_test_score'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 3}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_knn.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.23076923  0.12692308]\n",
      " [ 0.12384615  0.51846154]]\n"
     ]
    }
   ],
   "source": [
    "knn_predict = results_knn.predict(wine_test)\n",
    "\n",
    "# create confusion matrix, and accuracy\n",
    "cm = metrics.confusion_matrix(y_test,knn_predict)\n",
    "cm_prob = cm / np.sum(cm)  # as probs\n",
    "print(cm_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.749231</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy\n",
       "0  0.749231"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = metrics.accuracy_score(y_test, knn_predict)\n",
    "acc = pd.DataFrame(np.array([acc]), columns=['Accuracy'])\n",
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        Bad       0.65      0.65      0.65       465\n",
      "       Good       0.80      0.81      0.81       835\n",
      "\n",
      "avg / total       0.75      0.75      0.75      1300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(metrics.classification_report(y_test, knn_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Net"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the following, we set things up so they are similar to that in the document, which uses the nnet under the hood.  As with that demonstration, I would not recommend these settings in practice, as `MLPClassifier` has an adaptive learning rate and can do multiple layers also.  Unfortunately, it appears `n_jobs` is ignored here so this could take a long time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = MLPClassifier(activation='logistic', learning_rate='constant')\n",
    "nn_opts = {\n",
    "    'hidden_layer_sizes': [(3,), (5,), (7,), (9,)],\n",
    "    'alpha': np.linspace(.0001, 1, 10) \n",
    "}\n",
    "\n",
    "nn_estimator = GridSearchCV(nn, cv=10, param_grid=nn_opts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_nn = nn_estimator.fit(wine_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.73638637675582064"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_nn.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3,  7, 11, 15, 19, 23, 27, 31, 35, 39])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.arange(3,40, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"e4281d43-1fb0-4a9f-8edd-b86fb1c93175\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id !== undefined) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var element_id = msg.content.text.trim();\n",
       "            Bokeh.index[element_id].model.document.clear();\n",
       "            delete Bokeh.index[element_id];\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[0].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[0].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[0]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"e4281d43-1fb0-4a9f-8edd-b86fb1c93175\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        root._bokeh_is_loading--;\n",
       "        if (root._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"e4281d43-1fb0-4a9f-8edd-b86fb1c93175\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'e4281d43-1fb0-4a9f-8edd-b86fb1c93175' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.14.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"e4281d43-1fb0-4a9f-8edd-b86fb1c93175\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"e4281d43-1fb0-4a9f-8edd-b86fb1c93175\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n    }\n    finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.info(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(js_urls, callback) {\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = js_urls.length;\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var s = document.createElement('script');\n      s.src = url;\n      s.async = false;\n      s.onreadystatechange = s.onload = function() {\n        root._bokeh_is_loading--;\n        if (root._bokeh_is_loading === 0) {\n          console.log(\"Bokeh: all BokehJS libraries loaded\");\n          run_callbacks()\n        }\n      };\n      s.onerror = function() {\n        console.warn(\"failed to load library \" + url);\n      };\n      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.getElementsByTagName(\"head\")[0].appendChild(s);\n    }\n  };var element = document.getElementById(\"e4281d43-1fb0-4a9f-8edd-b86fb1c93175\");\n  if (element == null) {\n    console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'e4281d43-1fb0-4a9f-8edd-b86fb1c93175' but no matching script tag was found. \")\n    return false;\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.14.min.js\"];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.css\");\n    }\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"e4281d43-1fb0-4a9f-8edd-b86fb1c93175\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(js_urls, function() {\n      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<div class=\"bk-root\">\n",
       "    <div class=\"bk-plotdiv\" id=\"9a446ddd-2dfe-4970-88b7-285b12b31320\"></div>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"ec9ba0c8-7ee7-4d9e-945b-012310adea94\":{\"roots\":{\"references\":[{\"attributes\":{},\"id\":\"5f71debc-3600-487b-a600-2a639f113cad\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"ad1efbcb-4105-4a4e-a1ef-22b80de8434a\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null},\"id\":\"88f4c469-3904-44ae-9757-83eeb22ad224\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"204a61d7-c63d-4043-bfce-b2c4aac83503\",\"type\":\"LinearScale\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"x\",\"y\"],\"data\":{\"x\":[0.0001,0.0001,0.0001,0.0001,0.11120000000000001,0.11120000000000001,0.11120000000000001,0.11120000000000001,0.2223,0.2223,0.2223,0.2223,0.33340000000000003,0.33340000000000003,0.33340000000000003,0.33340000000000003,0.4445,0.4445,0.4445,0.4445,0.5556,0.5556,0.5556,0.5556,0.6667000000000001,0.6667000000000001,0.6667000000000001,0.6667000000000001,0.7778,0.7778,0.7778,0.7778,0.8889,0.8889,0.8889,0.8889,1.0,1.0,1.0,1.0],\"y\":{\"__ndarray__\":\"fVqfUSN/5z8VFaRxcYXnP59brCl6kOc/b+ai6d2D5z8VFaRxcYXnPz9xmalBd+c/W1mSecxt5z/zE5eZGnTnPzH9nEH8e+c/D/yPaaVq5z+1KpHxOGznP4vOm7loeuc/HXCM0epl5z+ntpSJ83DnP7UqkfE4bOc/p7aUifNw5z9pzY7hEWnnPwGIkwFgb+c/p7aUifNw5z9bWZJ5zG3nP2nNjuERaec/D/yPaaVq5z+Lzpu5aHrnP6e2lInzcOc/0RKKwcNi5z/ln5ox1XjnPz9xmalBd+c/p7aUifNw5z+1KpHxOGznP+WfmjHVeOc/8xOXmRp05z/zE5eZGnTnP1tZknnMbec/fVqfUSN/5z8/cZmpQXfnP31an1Ejf+c/8xOXmRp05z+Lzpu5aHrnP7UqkfE4bOc/TeWVEYdy5z8=\",\"dtype\":\"float64\",\"shape\":[40]}}},\"id\":\"399e9946-72ac-4187-a725-0f156188b865\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"d94ecf94-9a88-48e5-a339-9dd2e2fa74dd\",\"type\":\"Circle\"},{\"attributes\":{\"formatter\":{\"id\":\"89649557-7010-4e0f-aee3-9033fc06965b\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"f567f8d3-d627-44b4-92ef-835810f52c82\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"09ba0ddc-61ed-4a07-9fbf-cea6549ad32f\",\"type\":\"BasicTicker\"}},\"id\":\"893f0fa6-f63e-4972-bb7e-b48060c7df6d\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"09ba0ddc-61ed-4a07-9fbf-cea6549ad32f\",\"type\":\"BasicTicker\"},{\"attributes\":{\"plot\":{\"id\":\"f567f8d3-d627-44b4-92ef-835810f52c82\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"09ba0ddc-61ed-4a07-9fbf-cea6549ad32f\",\"type\":\"BasicTicker\"}},\"id\":\"a9276f58-e18e-4476-a760-d09dc1c7e813\",\"type\":\"Grid\"},{\"attributes\":{\"formatter\":{\"id\":\"ad1efbcb-4105-4a4e-a1ef-22b80de8434a\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"f567f8d3-d627-44b4-92ef-835810f52c82\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"c70a9d6d-55bd-4e06-a7ed-593a53fed6fa\",\"type\":\"BasicTicker\"}},\"id\":\"39e0e70c-f0be-478d-a061-0ec50fbbd114\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"c70a9d6d-55bd-4e06-a7ed-593a53fed6fa\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"f567f8d3-d627-44b4-92ef-835810f52c82\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"c70a9d6d-55bd-4e06-a7ed-593a53fed6fa\",\"type\":\"BasicTicker\"}},\"id\":\"e68ae3f5-3417-4379-9c60-2a5b3399b891\",\"type\":\"Grid\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"b98af2d7-8e64-45a3-a882-6033aa79659e\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"data_source\":{\"id\":\"399e9946-72ac-4187-a725-0f156188b865\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"5bf7338e-66e3-43fb-aee6-bc181adedb44\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"d94ecf94-9a88-48e5-a339-9dd2e2fa74dd\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"9f02dfba-a92f-4065-955c-a8a14b78f62d\",\"type\":\"CDSView\"}},\"id\":\"954f7b1d-a63d-45a6-892a-31cd8546e9fe\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"89649557-7010-4e0f-aee3-9033fc06965b\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"6d313d87-68d4-4618-b174-ac429c4124d7\",\"type\":\"PanTool\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"6ef51d9e-9b92-4015-be71-5f5c53962af3\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"22edb430-f962-4a52-a6d7-6759b3875089\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"overlay\":{\"id\":\"b98af2d7-8e64-45a3-a882-6033aa79659e\",\"type\":\"BoxAnnotation\"}},\"id\":\"6186281d-b08c-4420-8703-02c34d1836a3\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"659d2e40-5806-45b9-aa3a-479493a5dd49\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1a1986cd-1e58-4f1a-b99a-64ba5f198c12\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"a4470db9-4676-4ae4-b47f-26d119cda913\",\"type\":\"HelpTool\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"xs\":{\"field\":\"xs\"},\"ys\":{\"field\":\"ys\"}},\"id\":\"6a491d12-8bc0-414c-83e2-50d706297868\",\"type\":\"MultiLine\"},{\"attributes\":{\"below\":[{\"id\":\"893f0fa6-f63e-4972-bb7e-b48060c7df6d\",\"type\":\"LinearAxis\"}],\"left\":[{\"id\":\"39e0e70c-f0be-478d-a061-0ec50fbbd114\",\"type\":\"LinearAxis\"}],\"plot_height\":400,\"plot_width\":400,\"renderers\":[{\"id\":\"893f0fa6-f63e-4972-bb7e-b48060c7df6d\",\"type\":\"LinearAxis\"},{\"id\":\"a9276f58-e18e-4476-a760-d09dc1c7e813\",\"type\":\"Grid\"},{\"id\":\"39e0e70c-f0be-478d-a061-0ec50fbbd114\",\"type\":\"LinearAxis\"},{\"id\":\"e68ae3f5-3417-4379-9c60-2a5b3399b891\",\"type\":\"Grid\"},{\"id\":\"b98af2d7-8e64-45a3-a882-6033aa79659e\",\"type\":\"BoxAnnotation\"},{\"id\":\"954f7b1d-a63d-45a6-892a-31cd8546e9fe\",\"type\":\"GlyphRenderer\"},{\"id\":\"9f5831da-418f-42dd-a0c7-c72bf13b641b\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"6ef51d9e-9b92-4015-be71-5f5c53962af3\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"d4239dab-6c4e-482c-90ef-56af668bdc65\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"b844c4e0-e0a1-48b9-a542-5bf3d23bb86a\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"204a61d7-c63d-4043-bfce-b2c4aac83503\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"88f4c469-3904-44ae-9757-83eeb22ad224\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"5f71debc-3600-487b-a600-2a639f113cad\",\"type\":\"LinearScale\"}},\"id\":\"f567f8d3-d627-44b4-92ef-835810f52c82\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"data_source\":{\"id\":\"64189507-43c4-493a-af96-c4ddb40b877d\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"6a230a1a-ed51-4124-b7cc-5d8a66067fa8\",\"type\":\"MultiLine\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"6a491d12-8bc0-414c-83e2-50d706297868\",\"type\":\"MultiLine\"},\"selection_glyph\":null,\"view\":{\"id\":\"23071187-5676-4011-8049-b58a3bb15e93\",\"type\":\"CDSView\"}},\"id\":\"9f5831da-418f-42dd-a0c7-c72bf13b641b\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"callback\":null},\"id\":\"b844c4e0-e0a1-48b9-a542-5bf3d23bb86a\",\"type\":\"DataRange1d\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"5bf7338e-66e3-43fb-aee6-bc181adedb44\",\"type\":\"Circle\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"xs\",\"ys\",\"line_color\"],\"data\":{\"line_color\":[\"darkred\",\"dodgerblue\",\"orangered\",\"green\"],\"xs\":[[0.0001,0.11120000000000001,0.2223,0.33340000000000003,0.4445,0.5556,0.6667000000000001,0.7778,0.8889,1.0],[0.0001,0.11120000000000001,0.2223,0.33340000000000003,0.4445,0.5556,0.6667000000000001,0.7778,0.8889,1.0],[0.0001,0.11120000000000001,0.2223,0.33340000000000003,0.4445,0.5556,0.6667000000000001,0.7778,0.8889,1.0],[0.0001,0.11120000000000001,0.2223,0.33340000000000003,0.4445,0.5556,0.6667000000000001,0.7778,0.8889,1.0]],\"ys\":[{\"__ndarray__\":\"fVqfUSN/5z8VFaRxcYXnPzH9nEH8e+c/HXCM0epl5z9pzY7hEWnnP2nNjuERaec/0RKKwcNi5z+1KpHxOGznP1tZknnMbec/8xOXmRp05z8=\",\"dtype\":\"float64\",\"shape\":[10]},{\"__ndarray__\":\"FRWkcXGF5z8/cZmpQXfnPw/8j2mlauc/p7aUifNw5z8BiJMBYG/nPw/8j2mlauc/5Z+aMdV45z/ln5ox1XjnP31an1Ejf+c/i86buWh65z8=\",\"dtype\":\"float64\",\"shape\":[10]},{\"__ndarray__\":\"n1usKXqQ5z9bWZJ5zG3nP7UqkfE4bOc/tSqR8Ths5z+ntpSJ83DnP4vOm7loeuc/P3GZqUF35z/zE5eZGnTnPz9xmalBd+c/tSqR8Ths5z8=\",\"dtype\":\"float64\",\"shape\":[10]},{\"__ndarray__\":\"b+ai6d2D5z/zE5eZGnTnP4vOm7loeuc/p7aUifNw5z9bWZJ5zG3nP6e2lInzcOc/p7aUifNw5z/zE5eZGnTnP31an1Ejf+c/TeWVEYdy5z8=\",\"dtype\":\"float64\",\"shape\":[10]}]}},\"id\":\"64189507-43c4-493a-af96-c4ddb40b877d\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"line_color\":{\"field\":\"line_color\"},\"xs\":{\"field\":\"xs\"},\"ys\":{\"field\":\"ys\"}},\"id\":\"6a230a1a-ed51-4124-b7cc-5d8a66067fa8\",\"type\":\"MultiLine\"},{\"attributes\":{\"source\":{\"id\":\"64189507-43c4-493a-af96-c4ddb40b877d\",\"type\":\"ColumnDataSource\"}},\"id\":\"23071187-5676-4011-8049-b58a3bb15e93\",\"type\":\"CDSView\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"6d313d87-68d4-4618-b174-ac429c4124d7\",\"type\":\"PanTool\"},{\"id\":\"22edb430-f962-4a52-a6d7-6759b3875089\",\"type\":\"WheelZoomTool\"},{\"id\":\"6186281d-b08c-4420-8703-02c34d1836a3\",\"type\":\"BoxZoomTool\"},{\"id\":\"659d2e40-5806-45b9-aa3a-479493a5dd49\",\"type\":\"SaveTool\"},{\"id\":\"1a1986cd-1e58-4f1a-b99a-64ba5f198c12\",\"type\":\"ResetTool\"},{\"id\":\"a4470db9-4676-4ae4-b47f-26d119cda913\",\"type\":\"HelpTool\"}]},\"id\":\"d4239dab-6c4e-482c-90ef-56af668bdc65\",\"type\":\"Toolbar\"},{\"attributes\":{\"source\":{\"id\":\"399e9946-72ac-4187-a725-0f156188b865\",\"type\":\"ColumnDataSource\"}},\"id\":\"9f02dfba-a92f-4065-955c-a8a14b78f62d\",\"type\":\"CDSView\"}],\"root_ids\":[\"f567f8d3-d627-44b4-92ef-835810f52c82\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.14\"}};\n",
       "  var render_items = [{\"docid\":\"ec9ba0c8-7ee7-4d9e-945b-012310adea94\",\"elementid\":\"9a446ddd-2dfe-4970-88b7-285b12b31320\",\"modelid\":\"f567f8d3-d627-44b4-92ef-835810f52c82\"}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "f567f8d3-d627-44b4-92ef-835810f52c82"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from bokeh.plotting import figure, output_notebook, show\n",
    "\n",
    "output_notebook()\n",
    "\n",
    "p = figure(plot_width=400, plot_height=400)\n",
    "\n",
    "colors = ['darkred', 'dodgerblue', 'orangered', 'green']#*10\n",
    "\n",
    "grid_par = results_nn.cv_results_['param_alpha']\n",
    "grid_res = results_nn.cv_results_['mean_test_score']\n",
    "\n",
    "\n",
    "# add a line renderer\n",
    "p.scatter(results_nn.cv_results_['param_alpha'],\n",
    "            results_nn.cv_results_['mean_test_score'])\n",
    "\n",
    "\n",
    "p.multi_line([grid_par[np.arange(0,40, 4)],\n",
    "              grid_par[np.arange(1,40, 4)],\n",
    "              grid_par[np.arange(2,40, 4)],\n",
    "              grid_par[np.arange(3,40, 4)]],\n",
    "             [grid_res[np.arange(0,40, 4)],\n",
    "              grid_res[np.arange(1,40, 4)],\n",
    "              grid_res[np.arange(2,40, 4)],\n",
    "              grid_res[np.arange(3,40, 4)],],\n",
    "      line_color=colors)\n",
    "\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'alpha': 0.0001, 'hidden_layer_sizes': (7,)}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_nn.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.23        0.12769231]\n",
      " [ 0.10923077  0.53307692]]\n"
     ]
    }
   ],
   "source": [
    "nn_predict = results_nn.predict(wine_test)\n",
    "\n",
    "# create confusion matrix, and accuracy\n",
    "cm = metrics.confusion_matrix(y_test,nn_predict)\n",
    "cm_prob = cm / np.sum(cm)  # as probs\n",
    "print(cm_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.763077</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy\n",
       "0  0.763077"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = metrics.accuracy_score(y_test, nn_predict)\n",
    "acc = pd.DataFrame(np.array([acc]), columns=['Accuracy'])\n",
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        Bad       0.68      0.64      0.66       465\n",
      "       Good       0.81      0.83      0.82       835\n",
      "\n",
      "avg / total       0.76      0.76      0.76      1300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(metrics.classification_report(y_test, nn_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The random forest implementation for scikit-learn won't be too much different. The corresponding `mtry` from `randomForest` in R is `max_features`. Note that this model is run in parallel to speed things up (`n_jobs=4`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(n_estimators=1000)\n",
    "\n",
    "rf_opts = {'max_features': np.arange(2,7)}\n",
    "rf_estimator = GridSearchCV(rf, cv=10, param_grid=rf_opts, n_jobs=4)\n",
    "results_rf = rf_estimator.fit(wine_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.82297479314989419"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_rf.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAFUlJREFUeJzt3W+wXPV93/H3xxJCDLHQjHXdYgksMqEMf9wWvJGdIfEkwdiy6iAS54EYY0qGRPEfaEqpO6IDHcqTOOOZTpuakgEbY9MQonEMkW0cOR7wg2QI0SrinyDqyJTARe74kgwOxsRY+NsH+5O8Xl+4517p3r26vF8zO3fP2d85+z1H2v3s+Z2zv01VIUnSG8ZdgCRpcTAQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpWT7uAmZjzZo1tX79+nGXIUnHjDVr1rBz586dVbVxprbHVCCsX7+efr8/7jIk6ZiSZE2XdnYZSZIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVLTKRCSbEyyL8n+JNumefzUJPcn2ZPkkSSb2vwNSR5qt4eT/OrQMk8lebQ95q/eSNKYzfiLaUmWATcBFwKTwK4kO6rq8aFm1wHbq+rmJGcB9wLrgceAXlUdTHIy8HCSL1XVwbbcL1XVc0dxeyRJc9TlCGEDsL+qnqyql4G7gM0jbQpY1e6fBBwAqKrvDb35r2ztJEmLUJdAWAs8MzQ92eYNuwG4NMkkg6ODqw49kOQdSfYCjwIfHgqIAr6WZHeSrXOsX5J0lHQJhEwzb/ST/iXA7VW1DtgE3JHkDQBV9WBVnQ38LHBtkpVtmfOr6jzgfcDHkrxr2idPtibpJ+lPTU11KFeSNBddAmESOGVoeh2tS2jIFcB2gKp6gEH30JrhBlX1BPAicE6bPtSt9G3gbgZdUz+hqm6pql5V9SYmJjqUK0maiy6BsAs4PclpSVYAW4AdI22eBi4ASHImg0CYasssb/PfCpwBPJXkxCRvbPNPBN7D4AS0JGlMZrzKqF0hdCWwE1gG3FZVe5PcCPSragdwDXBrkqsZdCddXlWV5OeBbUl+APwQ+GhVPZfkp4G7kxyq4c6q+rN52UJJUiepOnYu/On1etXv+5UFSZqNJLurqjdTO7+pLEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJ6BgISTYm2Zdkf5Jt0zx+apL7k+xJ8kiSTW3+hiQPtdvDSX616zolSQtr+UwNkiwDbgIuBCaBXUl2VNXjQ82uA7ZX1c1JzgLuBdYDjwG9qjqY5GTg4SRfAqrDOiVJC6jLEcIGYH9VPVlVLwN3AZtH2hSwqt0/CTgAUFXfq6qDbf7K1q7rOiVJC6hLIKwFnhmanmzzht0AXJpkksHRwVWHHkjyjiR7gUeBD7eA6LJOSdIC6hIImWZejUxfAtxeVeuATcAdSd4AUFUPVtXZwM8C1yZZ2XGdgydPtibpJ+lPTU11KFeSNBddAmESOGVoeh2tS2jIFcB2gKp6gEH30JrhBlX1BPAicE7HdR5a7paq6lVVb2JiokO5kqS56BIIu4DTk5yWZAWwBdgx0uZp4AKAJGcyCISptszyNv+twBnAUx3XKUlaQDNeZdSuELoS2AksA26rqr1JbgT6VbUDuAa4NcnVDLp+Lq+qSvLzwLYkPwB+CHy0qp4DmG6d87GBkqRuUjVt1/2i1Ov1qt/vj7sMSTqmJNldVb2Z2vlNZUkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJQMdASLIxyb4k+5Nsm+bxU5Pcn2RPkkeSbGrzL0yyO8mj7e8vDy3zjbbOh9rtzUdvsyRJs7V8pgZJlgE3ARcCk8CuJDuq6vGhZtcB26vq5iRnAfcC64HngF+pqgNJzgF2AmuHlvtgVfWPzqZIko5ElyOEDcD+qnqyql4G7gI2j7QpYFW7fxJwAKCq9lTVgTZ/L7AyyfFHXrYk6WjrEghrgWeGpif58U/5ADcAlyaZZHB0cNU06/kAsKeqvj8077Otu+j6JOletiTpaOsSCNO9UdfI9CXA7VW1DtgE3JHk8LqTnA38HvDbQ8t8sKreBvxCu31o2idPtibpJ+lPTU11KFeSNBddAmESOGVoeh2tS2jIFcB2gKp6AFgJrAFIsg64G7isqr55aIGqerb9fQG4k0HX1E+oqluqqldVvYmJiS7bJEmagy6BsAs4PclpSVYAW4AdI22eBi4ASHImg0CYSrIa+ApwbVX95aHGSZYnORQYxwHvBx470o2RJM3djIFQVQeBKxlcIfQEg6uJ9ia5MclFrdk1wG8leRj4I+Dyqqq23M8A149cXno8sDPJI8BDwLPArUd74yRJ3WXwvn1s6PV61e97laokzUaS3VXVm6md31SWJAEGgiSpMRAkSYCBIElqDARJEtBhcDu9vtyz51k+uXMfB55/ibesPoGPv/cMLj53dKQSSUuRgaDD7tnzLNd+8VFe+sErADz7/Etc+8VHAQwF6XXALiMd9smd+w6HwSEv/eAVPrlz35gqkrSQDAQdduD5l2Y1X9LSYiDosLesPmFW8yUtLQaCDvv4e8/ghOOW/di8E45bxsffe8aYKlr87tnzLOd/4j5O2/YVzv/Efdyz59lxlyTNmSeVddihE8deZdSNJ+G11BgI+jEXn7vWN7OOXuskvPtQxyK7jKQ58iS8lhoDQZojT8JrqTEQpDnyJLyWmtfFOQSHY9B88CS8lpolHwheCaL55El4LSVLvsvI4RgkqZslHwheCSJJ3Sz5QPBKEEnqZskHgleCSFI3S/6ksleCSFI3nQIhyUbgfwDLgE9X1SdGHj8V+BywurXZVlX3JrkQ+ASwAngZ+HhV3deWeTtwO3ACcC/wO1VVR2OjRnkliCTNbMZASLIMuAm4EJgEdiXZUVWPDzW7DtheVTcnOYvBG/x64DngV6rqQJJzgJ3AoXfmm4GtwF+19huBrx6VrZKkJWChv0PV5RzCBmB/VT1ZVS8DdwGbR9oUsKrdPwk4AFBVe6rqQJu/F1iZ5PgkJwOrquqBdlTweeDiI9wWSVoyDn2H6tnnX6L40Xeo5nOI9S5dRmuBZ4amJ4F3jLS5AfhakquAE4F3T7OeDwB7qur7Sda29Qyv0z4daYlz1IDuxjGabpcjhEwzb7Sv/xLg9qpaB2wC7khyeN1JzgZ+D/jtWazz0LJbk/ST9KempjqUK2kxGscn3mPZOL5D1SUQJoFThqbX0bqEhlwBbAeoqgeAlcAagCTrgLuBy6rqm0PrXDfDOmnru6WqelXVm5iY6FCupMXIUQNmZxzfoeoSCLuA05OclmQFsAXYMdLmaeACgCRnMgiEqSSrga8A11bVXx5qXFXfAl5I8s4kAS4D/vSIt0bSouWoAbMzju9QzRgIVXUQuJLBFUJPMLiaaG+SG5Nc1JpdA/xWkoeBPwIubyeLrwR+Brg+yUPt9ua2zEeATwP7gW/iFUbSkuaoAbNz8blr+d1fextrV59AgLWrT+B3f+1t83rOJfN06f+86PV61e/3x12GpDkYHXkYBp945/tNTpBkd1X1Zmq35L+pLGlxcNSAxc9AkLRgHDVgcVvyg9tJkroxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJKaToGQZGOSfUn2J9k2zeOnJrk/yZ4kjyTZ1Oa/qc3/bpJPjSzzjbbOh9rtzUdnkyRJc7F8pgZJlgE3ARcCk8CuJDuq6vGhZtcB26vq5iRnAfcC64F/Aq4Hzmm3UR+sqv6RbYIk6WjocoSwAdhfVU9W1cvAXcDmkTYFrGr3TwIOAFTVi1X1FwyCQZK0iHUJhLXAM0PTk23esBuAS5NMMjg6uKrj83+2dRddnyQdl5EkzYMugTDdG3WNTF8C3F5V64BNwB1JZlr3B6vqbcAvtNuHpn3yZGuSfpL+1NRUh3IlSXPRJRAmgVOGptfRuoSGXAFsB6iqB4CVwJrXWmlVPdv+vgDcyaBrarp2t1RVr6p6ExMTHcqVJM1Fl0DYBZye5LQkK4AtwI6RNk8DFwAkOZNBILzqx/kky5OsafePA94PPDb78iVJR8uMVxlV1cEkVwI7gWXAbVW1N8mNQL+qdgDXALcmuZpBd9LlVVUASZ5icMJ5RZKLgfcAfwfsbGGwDPg6cOtR3zpJUmdp79vHhF6vV/2+V6lK0mwk2V1VvZna+U1lSRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJElAx0BIsjHJviT7k2yb5vFTk9yfZE+SR5JsavPf1OZ/N8mnRpZ5e5JH2zp/P0mOziZJkuZixkBIsgy4CXgfcBZwSZKzRppdB2yvqnOBLcD/avP/Cbge+I/TrPpmYCtwerttnMsGSJKOji5HCBuA/VX1ZFW9DNwFbB5pU8Cqdv8k4ABAVb1YVX/BIBgOS3IysKqqHqiqAj4PXDz3zZAkHanlHdqsBZ4Zmp4E3jHS5gbga0muAk4E3t1hnZMj61zboRZJ0jzpcoQwXd9+jUxfAtxeVeuATcAdSV5r3V3WOWiYbE3ST9KfmprqUK4kaS66BMIkcMrQ9Dpal9CQK4DtAFX1ALASWDPDOtfNsE7a+m6pql5V9SYmJjqUK0maiy6BsAs4PclpSVYwOGm8Y6TN08AFAEnOZBAIr/pxvqq+BbyQ5J3t6qLLgD+dQ/2SpKNkxnMIVXUwyZXATmAZcFtV7U1yI9Cvqh3ANcCtSa5m0PVzeTtZTJKnGJxwXpHkYuA9VfU48BHgduAE4KvtJkkak7T37WNCr9erfr8/7jIk6ZiSZHdV9WZq5zeVJUmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBHQMhycYk+5LsT7JtmsdPTXJ/kj1JHkmyaeixa9ty+5K8d2j+U0keTfJQkv7R2RxJ0lwtn6lBkmXATcCFwCSwK8mOqnp8qNl1wPaqujnJWcC9wPp2fwtwNvAW4OtJ/kVVvdKW+6Wqeu4obo8kaY66HCFsAPZX1ZNV9TJwF7B5pE0Bq9r9k4AD7f5m4K6q+n5V/V9gf1ufJGmR6RIIa4FnhqYn27xhNwCXJplkcHRwVYdlC/hakt1Jts6ybknSUdYlEDLNvBqZvgS4varWAZuAO5K8YYZlz6+q84D3AR9L8q5pnzzZmqSfpD81NdWhXEnSXHQJhEnglKHpdfyoS+iQK4DtAFX1ALASWPNay1bVob/fBu7mVbqSquqWqupVVW9iYqJDuZKkuegSCLuA05OclmQFg5PEO0baPA1cAJDkTAaBMNXabUlyfJLTgNOBv05yYpI3tvYnAu8BHjsaGyRJmpsZrzKqqoNJrgR2AsuA26pqb5IbgX5V7QCuAW5NcjWDLqHLq6qAvUm2A48DB4GPVdUrSf4ZcHeSQzXcWVV/Nh8bKEnqJoP37WNDking745gFWuAxXaZ62KsCaxrtqxrdqxrdo6krucAqmrjTA2PqUA4Ukn6VdUbdx3DFmNNYF2zZV2zY12zs1B1OXSFJAkwECRJzestEG4ZdwHTWIw1gXXNlnXNjnXNzoLU9bo6hyBJenWvtyMESdKrWFKBkOSUNgz3E0n2Jvmdadokye+3IbkfSXLeIqnrF5N8pw0H/lCS/7IAda1M8tdJHm51/ddp2hyf5I/b/nowyfpFUtflSaaG9tdvznddQ8+9rA31/uVpHlvw/dWhpnHuq9cc5n4cr8eOdY3j9bg6yReS/G17r/i5kcfnf19V1ZK5AScD57X7bwT+D3DWSJtNwFcZjLP0TuDBRVLXLwJfXuD9FeCn2v3jgAeBd460+SjwB+3+FuCPF0ldlwOfGtP/s/8A3Dndv9c49leHmsa5r54C1rzG4wv+euxY1zhej58DfrPdXwGsXuh9taSOEKrqW1X1N+3+C8AT/OTIrJuBz9fAXwGrk5y8COpacG0ffLdNHtduoyeVNjP4jwrwBeCCtK+Yj7musUiyDvg3wKdfpcmC768ONS1mC/56XIySrALeBXwGoKperqrnR5rN+75aUoEwrB2qn8vg0+WwLsN5z5vXqAvg51o3yVeTnL1A9SxL8hDwbeDPq+pV91dVHQS+A7xpEdQF8IF26PyFJKdM8/h8+O/AfwJ++CqPj2N/zVQTjGdfwczD3I/r9dhl+P2FfD3+NIPx3z7buv4+ncE4b8PmfV8tyUBI8lPAnwD/vqr+cfThaRZZkE+fM9T1N8Bbq+pfAf8TuGchaqqqV6rqXzMYiXZDknNGmoxlf3Wo60vA+qr6l8DX+dGn8nmT5P3At6tq92s1m2bevO2vjjUt+L4aMtMw9+N6Pc5U10K/HpcD5wE3V9W5wIvA6M8Vz/u+WnKBkOQ4Bm+6f1hVX5ymSZfhvBe8rqr6x0PdJFV1L3BckjXzXdfQ8z8PfAMYHe/k8P5KspzBL+L9w7jrqqq/r6rvt8lbgbcvQDnnAxcleYrBLwf+cpL/PdJmoffXjDWNaV8deu6Zhrkfy+txprrG8HqcBCaHjoS/wCAgRtvM675aUoHQ+mo/AzxRVf/tVZrtAC5rZ+zfCXynqr417rqS/PNDfc1JNjD4t/n7ea5rIsnqdv8E4N3A34402wH823b/14H7qp3hGmddI32nFzE4LzOvquraqlpXVesZnDC+r6ouHWm2oPurS03j2FftebsMcz+O1+OMdS3067Gq/h/wTJIz2qwLGIwSPWze99WMw18fY84HPgQ82vqfAf4zcCpAVf0Bg5/43MTg952/B/zGIqnr14GPJDkIvARsme83XgZXP30uyTIG/+G3V9WX8+NDm3+GwS/g7WfwSXfLPNfUta5/l+QiBsOq/wODK2nGYhHsr5lqGte+mnaY+yQfhrG+HrvUNY7X41XAH2bwuzNPAr+x0PvKbypLkoAl1mUkSZo7A0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSAP8flLa52SWsKDoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a20864d68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(results_rf.cv_results_['param_max_features'],\n",
    "            results_rf.cv_results_['mean_test_score'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_features': 3}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_rf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.26615385  0.09153846]\n",
      " [ 0.07846154  0.56384615]]\n"
     ]
    }
   ],
   "source": [
    "rf_predict = results_rf.predict(wine_test)\n",
    "\n",
    "# create confusion matrix, and accuracy\n",
    "cm = metrics.confusion_matrix(y_test,rf_predict)\n",
    "cm_prob = cm / np.sum(cm)  # as probs\n",
    "print(cm_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.83</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy\n",
       "0      0.83"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = metrics.accuracy_score(y_test, rf_predict)\n",
    "acc = pd.DataFrame(np.array([acc]), columns=['Accuracy'])\n",
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        Bad       0.77      0.74      0.76       465\n",
      "       Good       0.86      0.88      0.87       835\n",
      "\n",
      "avg / total       0.83      0.83      0.83      1300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(metrics.classification_report(y_test, rf_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support Vector Machines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As with some of the others, this implementation is fairly similar to the one used in the R code. `C` is our penalty parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "svm = LinearSVC()\n",
    "\n",
    "svm_opts = {'C': [.25, .5, 1, 2, 4]}\n",
    "svm_estimator = GridSearchCV(svm, cv=10, param_grid=svm_opts)\n",
    "results_svm = svm_estimator.fit(wine_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.73638637675582064"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_svm.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAFH5JREFUeJzt3X+sX3d93/HnK3acWssye+SWBdtxXNXySAdzyHdWaASq2BCmRUnUodYmI0nVyQIatgUpI0z9saWaWgTUHRC1CiUUaImhXgoeDXKHDN0/JvN1MUmczM4l6+RrR8KhNQQ1Smrz3h/fz6Xf3Nzre3x/fa/t50P6Kvd8zvt7/D4nuufl8/me73GqCkmSLhl2A5KkpcFAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkZvmwGzgXV155ZV1zzTXDbkOSzisHDx58tqpGZqo7rwLhmmuuYXR0dNhtSNJ5Jcn/61LnlJEkCTAQJEmNgSBJAgwESVJjIEiSAANBktR0CoQkW5McSTKW5J4p1u9Mcqi9jiY5NbDuzMC6PQPjG5I8kuSpJJ9PsmJ+dkmSNBszBkKSZcB9wFuBa4HtSa4drKmqu6pqc1VtBj4GPDSw+vmJdVV108D4B4GdVbUR+Bvgl+e4L5KkOehyhbAFGKuqp6vqRWAXcPNZ6rcDD55tg0kCvAnY3YY+DdzSoRdJ0gLpEghrgGMDy+Nt7GWSrAc2APsGhn8syWiSbySZOOm/AjhVVadn2qYkaXF0eXRFphiraWq3Abur6szA2NVVdSLJTwD7kjwGfL/rNpPsAHYAXH311R3alSTNRpcrhHFg3cDyWuDENLXbmDRdVFUn2n+fBr4OXAc8C6xKMhFI026zqu6vql5V9UZGZnw2kyRplroEwgFgY7sraAX9k/6eyUVJNgGrgf0DY6uTXNZ+vhK4EXiiqgr4GvD2Vno78KW57IgkaW5mDIQ2z38nsBd4EvhCVR1Ocm+SwbuGtgO72sl+wquB0STfoh8Av11VT7R17wfel2SM/mcKn5z77kiSZisvPX8vbb1er3z8tSSdmyQHq6o3U53fVJYkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAR0DIcnWJEeSjCW5Z4r1O5Mcaq+jSU5NWn9FkuNJPj4w9vW2zYn3/fjcd0eSNFvLZypIsgy4D3gzMA4cSLKnqp6YqKmquwbq3wtcN2kzvwn8xRSbv7WqRmfTuCRpfnW5QtgCjFXV01X1IrALuPks9duBBycWklwPvBL487k0KklaWF0CYQ1wbGB5vI29TJL1wAZgX1u+BPgIcPc02/5Umy76tSTp3LUkad51CYSpTtQ1Te02YHdVnWnL7wEerqpjU9TeWlWvAd7QXu+c8g9PdiQZTTJ68uTJDu1KkmajSyCMA+sGltcCJ6ap3cbAdBHweuDOJH8FfBi4LclvA1TV8fbf54DP0Z+aepmqur+qelXVGxkZ6dCuJGk2ZvxQGTgAbEyyAThO/6T/jslFSTYBq4H9E2NVdevA+juAXlXdk2Q5sKqqnk1yKfA24Ktz2RFJ0tzMGAhVdTrJncBeYBnwQFUdTnIvMFpVe1rpdmBXVU03nTToMmBvC4Nl9MPgE7PaA0nSvEi38/fS0Ov1anTUu1Ql6VwkOVhVvZnq/KayJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJTadASLI1yZEkY0numWL9ziSH2utoklOT1l+R5HiSjw+MXZ/ksbbNjybJ3HdHkjRbMwZCkmXAfcBbgWuB7UmuHaypqruqanNVbQY+Bjw0aTO/CfzFpLHfA3YAG9tr66z2QJI0L7pcIWwBxqrq6ap6EdgF3HyW+u3AgxMLSa4HXgn8+cDYVcAVVbW/qgr4DHDLLPqXJM2TLoGwBjg2sDzexl4myXpgA7CvLV8CfAS4e4ptjnfc5o4ko0lGT5482aFdSdJsdAmEqeb2a5rabcDuqjrTlt8DPFxVxybVdd5mVd1fVb2q6o2MjHRoV5I0G8s71IwD6waW1wInpqndBvzKwPLrgTckeQ9wObAiyQ+A/9a202WbkqRF0CUQDgAbk2wAjtM/6b9jclGSTcBqYP/EWFXdOrD+DqBXVfe05eeS3AA8AtxG/8NoSdKQzDhlVFWngTuBvcCTwBeq6nCSe5PcNFC6HdjVPiTu4t3AHwBjwLeBr5xT55KkeZXu5+/h6/V6NTo6Ouw2JOm8kuRgVfVmqvObypIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUdPknNM97X/zmcT609wgnTj3Pq1at5O63bOKW69YMu60Fc7Htr6T5ccEHwhe/eZwPPPQYz//dGQCOn3qeDzz0GMAFeZK82PZ3oRiquhhd8FNGH9p75EcnxwnP/90ZPrT3yJA6WlgX2/4uhIlQPX7qeYq/D9UvfvP4sFuTFtQFHwgnTj1/TuPnu4ttfxeCoaqL1QU/ZfSqVSs5PsXJ8FWrVs56m0t5OmEh9vdiY6jqYnXBXyHc/ZZNrLx02UvGVl66jLvfsmlW21vq0wnzvb8Xo+nC01DVhe6CD4RbrlvDb/38a1izaiUB1qxayW/9/Gtm/Tf6pT6dMN/7ezEyVHWxuuCnjKB/kpyvE+L5MJ0wn/t7MZo4dkt1WlBaKBdFIMwn5+gvDoaqloLF/ryy05RRkq1JjiQZS3LPFOt3JjnUXkeTnGrj65McbOOHk7xr4D1fb9uceN+Pz99uLRynEyQthmF8XjnjFUKSZcB9wJuBceBAkj1V9cRETVXdNVD/XuC6tvgM8NNV9UKSy4HH23tPtPW3VtXoPO3LonA6QdJiONvnlQt1vukyZbQFGKuqpwGS7AJuBp6Ypn478BsAVfXiwPhlXCAfYjudIGmhDePzyi4n6DXAsYHl8Tb2MknWAxuAfQNj65I82rbxwYGrA4BPtemiX0uSc+5eki5Qw7j9uUsgTHWirmlqtwG7q+pH1zlVdayqXgv8JHB7kle2VbdW1WuAN7TXO6f8w5MdSUaTjJ48ebJDu5J0/hvG55VdAmEcWDewvBY4MU3tNuDBqVa0K4PD9E/+VNXx9t/ngM/Rn5qa6n33V1WvqnojIyMd2pWk898wvlPU5TOEA8DGJBuA4/RP+u+YXJRkE7Aa2D8wthb4blU9n2Q1cCPwO0mWA6uq6tkklwJvA746572RpAvIYn9eOWMgVNXpJHcCe4FlwANVdTjJvcBoVe1ppduBXVU1OJ30auAjSYr+1NOHq+qxJP8A2NvCYBn9MPjE/O2WJOlc5aXn76Wt1+vV6Oh5dZeqJA1dkoNV1Zup7oK4DVSSNHcGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDWdAiHJ1iRHkowluWeK9TuTHGqvo0lOtfH1SQ628cNJ3jXwnuuTPNa2+dEkmb/dkiSdq+UzFSRZBtwHvBkYBw4k2VNVT0zUVNVdA/XvBa5ri88AP11VLyS5HHi8vfcE8HvADuAbwMPAVuAr87NbkqRz1eUKYQswVlVPV9WLwC7g5rPUbwceBKiqF6vqhTZ+2cSfl+Qq4Iqq2l9VBXwGuGWW+yBJmgddAmENcGxgebyNvUyS9cAGYN/A2Lokj7ZtfLBdHaxp25lxm5KkxdElEKaa269parcBu6vqzI8Kq45V1WuBnwRuT/LKc9lmkh1JRpOMnjx5skO7kqTZ6BII48C6geW1wIlparfRposma1cGh4E3tG2u7bLNqrq/qnpV1RsZGenQriRpNroEwgFgY5INSVbQP+nvmVyUZBOwGtg/MLY2ycr282rgRuBIVT0DPJfkhnZ30W3Al+a8N5KkWZvxLqOqOp3kTmAvsAx4oKoOJ7kXGK2qiXDYDuxqHxJPeDXwkSRFf5row1X1WFv3buAPgZX07y7yDiNJGqK89Py9tPV6vRodHR12G5J0XklysKp6M9X5TWVJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJKaToGQZGuSI0nGktwzxfqdSQ6119Ekp9r45iT7kxxO8miSXxx4zx8m+b8D79s8f7slSTpXy2cqSLIMuA94MzAOHEiyp6qemKipqrsG6t8LXNcW/xa4raqeSvIq4GCSvVV1qq2/u6p2z9O+SJLmoMsVwhZgrKqerqoXgV3AzWep3w48CFBVR6vqqfbzCeA7wMjcWpYkLYQugbAGODawPN7GXibJemADsG+KdVuAFcC3B4b/a5tK2pnksmm2uSPJaJLRkydPdmhXkjQbXQIhU4zVNLXbgN1VdeYlG0iuAj4L/FJV/bANfwD4p8C/AP4x8P6pNlhV91dVr6p6IyNeXEjSQukSCOPAuoHltcCJaWq30aaLJiS5Avgz4Fer6hsT41X1TPW9AHyK/tSUJGlIugTCAWBjkg1JVtA/6e+ZXJRkE7Aa2D8wtgL4U+AzVfUnk+qvav8NcAvw+Gx3QpI0dzPeZVRVp5PcCewFlgEPVNXhJPcCo1U1EQ7bgV1VNTid9AvAG4FXJLmjjd1RVYeAP04yQn9K6hDwrnnZI0nSrOSl5++lrdfr1ejo6LDbkKTzSpKDVdWbqc5vKkuSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSgI6BkGRrkiNJxpLcM8X6nUkOtdfRJKfa+OYk+5McTvJokl8ceM+GJI8keSrJ55OsmL/dkiSdqxkDIcky4D7grcC1wPYk1w7WVNVdVbW5qjYDHwMeaqv+Fritqn4K2Ar8bpJVbd0HgZ1VtRH4G+CX52OHJEmz0+UKYQswVlVPV9WLwC7g5rPUbwceBKiqo1X1VPv5BPAdYCRJgDcBu9t7Pg3cMrtdkCTNhy6BsAY4NrA83sZeJsl6YAOwb4p1W4AVwLeBVwCnqur0TNuUJC2O5R1qMsVYTVO7DdhdVWdesoHkKuCzwO1V9cN2hdBpm0l2ADva4g+SHOnQ82K5Enh22E2cxVLvD5Z+j0u9P1j6Pdrf3M21x/VdiroEwjiwbmB5LXBimtptwK8MDiS5Avgz4Fer6htt+FlgVZLl7Sph2m1W1f3A/R36XHRJRquqN+w+prPU+4Ol3+NS7w+Wfo/2N3eL1WOXKaMDwMZ2V9AK+if9PZOLkmwCVgP7B8ZWAH8KfKaq/mRivKoK+Brw9jZ0O/Cl2e6EJGnuZgyE9jf4O4G9wJPAF6rqcJJ7k9w0ULod2NVO9hN+AXgjcMfAbamb27r3A+9LMkb/M4VPzsP+SJJmqcuUEVX1MPDwpLFfn7T8n6d43x8BfzTNNp+mfwfT+WxJTmUNWOr9wdLvcan3B0u/R/ubu0XpMS/9C70k6WLloyskSYCB0EmHR3fckeTkwOck/3aR+3sgyXeSPD7N+iT5aOv/0SSvW2L9/UyS7w0cv1+fqm4B+1uX5GtJnmyPWfn3U9QM7Rh27G/Yx/DHkvzvJN9qPf6XKWoua4+pGWuPrblmifU31N/j1sOyJN9M8uUp1i388asqX2d5Acvof5nuJ+h/se5bwLWTau4APj7EHt8IvA54fJr1Pwt8hf53Sm4AHlli/f0M8OUhHr+rgNe1n/8hcHSK/8dDO4Yd+xv2MQxwefv5UuAR4IZJNe8Bfr/9vA34/BLrb6i/x62H9wGfm+r/5WIcP68QZnauj+5YdFX1v4C/PkvJzfRv/a3qfxdkVfuy4KLo0N9QVdUzVfWX7efn6N9NN/mb80M7hh37G6p2XH7QFi9tr8kfUN5M/zE10H9szb+c5kuqw+pvqJKsBX4O+INpShb8+BkIM+v66I5/3aYSdidZN8X6Yer8+JEhen27nP9Kkp8aVhPtMvw6+n+DHLQkjuFZ+oMhH8M23XGI/jPL/mdVTXsMq387+/fo33K+VPqD4f4e/y7wH4EfTrN+wY+fgTCzLo/Z+B/ANVX1WuCr/H2KLxXn8viRYfhLYH1V/XP6T8v94jCaSHI58N+B/1BV35+8eoq3LOoxnKG/oR/DqjpT/ScerwW2JPlnk0qGegw79De03+MkbwO+U1UHz1Y2xdi8Hj8DYWYzPrqjqr5bVS+0xU8A1y9Sb12dy+NHFl1VfX/icr7633m5NMmVi9lDkkvpn2z/uKoemqJkqMdwpv6WwjEc6OUU8HX6j7wf9KNjmGQ58I8YwlTidP0N+ff4RuCmJH9Ff1r6TUkmf4drwY+fgTCzGR/dMWku+Sb6c7xLyR7gtnanzA3A96rqmWE3NSHJP5mYC03/qbiXAN9dxD8/9L8p/2RV/c40ZUM7hl36WwLHcCTt3zpJshL4V8D/mVS2h/5jaqD/2Jp91T4hXQr9DfP3uKo+UFVrq+oa+ueYfVX1byaVLfjx6/RN5YtZVZ1OMvHojmXAA9Ue3QGMVtUe4N+l/xiP0/QT+47F7DHJg/TvMrkyyTjwG/Q/NKOqfp/+t8x/Fhij/48W/dIS6+/twLuTnAaeB7Yt1omiuRF4J/BYm2MG+E/A1QM9DvMYdulv2MfwKuDT6f+DWpfQf8TNlyf9nnwS+Gz6j6v5a/onvqXU31B/j6ey2MfPbypLkgCnjCRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCYD/D+ITIQg/f/c4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a214238d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(results_svm.cv_results_['param_C'],\n",
    "            results_svm.cv_results_['mean_test_score'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 0.25}"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_svm.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.21538462  0.14230769]\n",
      " [ 0.09384615  0.54846154]]\n"
     ]
    }
   ],
   "source": [
    "svm_predict = results_svm.predict(wine_test)\n",
    "\n",
    "# create confusion matrix, and accuracy\n",
    "cm = metrics.confusion_matrix(y_test,svm_predict)\n",
    "cm_prob = cm / np.sum(cm)  # as probs\n",
    "print(cm_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.763846</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy\n",
       "0  0.763846"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = metrics.accuracy_score(y_test, svm_predict)\n",
    "acc = pd.DataFrame(np.array([acc]), columns=['Accuracy'])\n",
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        Bad       0.70      0.60      0.65       465\n",
      "       Good       0.79      0.85      0.82       835\n",
      "\n",
      "avg / total       0.76      0.76      0.76      1300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(metrics.classification_report(y_test, svm_predict))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}