3-mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_by_target(mnist):\n",
    "    reorder_train = np.array(sorted([(target, i) for i, target in enumerate(mnist.target[:60000])]))[:, 1]\n",
    "    reorder_test = np.array(sorted([(target, i) for i, target in enumerate(mnist.target[60000:])]))[:, 1]\n",
    "    mnist.data[:60000] = mnist.data[reorder_train]\n",
    "    mnist.target[:60000] = mnist.target[reorder_train]\n",
    "    mnist.data[60000:] = mnist.data[reorder_test + 60000]\n",
    "    mnist.target[60000:] = mnist.target[reorder_test + 60000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 데이터셋 다운로드\n",
    "from sklearn.datasets import fetch_openml\n",
    "import numpy as np\n",
    "\n",
    "mnist = fetch_openml('mnist_784', version=1, cache=True)\n",
    "mnist.target = mnist.target.astype(np.int8)\n",
    "sort_by_target(mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mnist['data'], mnist['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-28-18cf3a53a5e0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0msome_digit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m36000\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0msome_digit_image\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msome_digit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'X' is not defined"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "some_digit = X[36000]\n",
    "some_digit_image = some_digit.reshape(28, 28)\n",
    "\n",
    "plt.imshow(some_digit_image, cmap=matplotlib.cm.binary, interpolation='nearest')\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[36000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 테스트셋 분리\n",
    "\n",
    "X_train, X_test, y_train, y_test = X[:60000], X [60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "shuffle_index = np.random.permutation(60000)\n",
    "X_train, y_train = X_train[shuffle_index], y_train[shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# binary classifier\n",
    "\n",
    "y_train_5 = (y_train == 5)\n",
    "y_test_5 = (y_test == 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
       "       early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
       "       l1_ratio=0.15, learning_rate='optimal', loss='hinge', max_iter=5,\n",
       "       n_iter=None, n_iter_no_change=5, n_jobs=None, penalty='l2',\n",
       "       power_t=0.5, random_state=42, shuffle=True, tol=None,\n",
       "       validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stochastic gradient descent\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf = SGDClassifier(max_iter=5, random_state=42)\n",
    "sgd_clf.fit(X_train, y_train_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 교차검증"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.93515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.93485\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.95895\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.base import clone\n",
    "\n",
    "skfolds = StratifiedKFold(n_splits=3, random_state=42)\n",
    "\n",
    "for train_index, test_index in skfolds.split(X_train, y_train_5):\n",
    "    clone_clf = clone(sgd_clf)\n",
    "    X_train_folds = X_train[train_index]\n",
    "    y_train_folds = y_train_5[train_index]\n",
    "    X_test_fold = X_train[test_index]\n",
    "    y_test_fold = y_train_5[test_index]\n",
    "    \n",
    "    clone_clf.fit(X_train_folds, y_train_folds)\n",
    "    y_pred = clone_clf.predict(X_test_fold)\n",
    "    n_correct = sum(y_pred == y_test_fold)\n",
    "    print(n_correct / len(y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.93515, 0.93485, 0.95895])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "cross_val_score(sgd_clf, X_train, y_train_5, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "\n",
    "# 무조건 5가 아님으로 분류하는 분류기\n",
    "class Never5Classfier(BaseEstimator):\n",
    "    def fit(self, X, y=None):\n",
    "        pass\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return np.zeros((len(X), 1), dtype=bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.90995, 0.90875, 0.91025])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "never_5_clf = Never5Classfier()\n",
    "cross_val_score(never_5_clf, X_train, y_train_5, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53419,  1160],\n",
       "       [ 2261,  3160]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "precision = $$ TP \\over TP + FP$$ \n",
    "recall = $$ TP \\over TP + FN$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7314814814814815\n",
      "0.5829182807600074\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "print(precision_score(y_train_5, y_train_pred))\n",
    "print(recall_score(y_train_5, y_train_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " $$F_1 = {2 \\over { {1 \\over precision} + {1 \\over recall} }} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6488040242274921"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-334529.73985706]\n",
      "[False]\n"
     ]
    }
   ],
   "source": [
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "print(y_scores)\n",
    "threshold = 0\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "print(y_some_digit_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3, method='decision_function')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEKCAYAAAAcgp5RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4VFX6wPHvm4QkEEILoQYh9B4IoSugFAEBQQFhxa6sIHbdZVfFusV1LevPgr0gS3VhESkiEEVYaYr0EiBg6E1aenJ+f9xJmISUSZjJnZm8n+eZ594599x73juZvDm55VwxxqCUUsp/BdgdgFJKKc/SRK+UUn5OE71SSvk5TfRKKeXnNNErpZSf00SvlFJ+rthELyIfi8hxEdlayHIRkTdFJEFENotIrPvDVEopVVqu9Og/BQYWsXwQ0MzxGg+8e+VhKaWUcpdiE70x5nvgdBFVbgQ+N5YfgWoiUtddASqllLoyQW7YRn3gV6f3SY6yI/krish4rF4/YWFhnVq2bFnixo5eOMqhc4dKF2l5JiCIY1Yuex8gAYgUslyEQAlERHLr5tR3fu+8bu48FFgeKIEEBAQQFBBkzUtAbhvO21aqKFlZcPo0BAZCUBBUqAAVK+ats2ULpKdDhw5WvaQkOHbs8m3FxoIIHDwIJ07kXSZiLQfYv99q01lQEMTEWPN798Jvv+VdHhwM7dpZ8ydPQs2apdtfgI0bN540xkSWZB13JHqXGWPeB94HiIuLMxs2bCjxNpIzkjlx8QRpWWk527y0fUyhZaWta9f66VnppGelY4wh22STbbIxWPOFlWWZLJIzksnMziQzO5OMrAwyszO5mHGRjKwMsk02WSYrd92s7KzcsgvpF8gyWXnKcubTs9K5mHExN6bzaedJzUwlIzuDjKwMUjNTSclMwd1CAkMICgiiQmAFggKCqBJShdCgUIIDg4msFElUlSiqh1anWmg1alSsQXT1aGJqx1C/Sn23x6Lsd+ECzJ0L//wnbNpkJdfWra3EnKN69UtJ+MknYccO2LgRJkyAt96CgAA4exbOn798+/XrWwn9zBm4ePHy5VFR1vT0aUhOzrssIADq1bPmT56E1NS8ywMDoa7jOMfFixAWVvL9zyEiB0q6jjsS/SGggdP7KEeZR1SqUImG1Rp6avPqChhjcv/w5P4hMvneY3L/WJxPP8/RC0etPxQZKZxNO5v7x+Rc2jnSMtNy/2ilZ6VzLt0qS8lM4fD5w2w5voVzaedIzsj7W1ejYg1qhdWibuW61K9SnzphdQgPCSeiYgSRYZE0qNKARtUaUbty7dz/RJR3MMZKxFu3Wgm3XTto1AhatoRduy7VW7wYhg6FypWhbVvrD8CxY3kTdHAwJCZafxD69LGSMUDVqtarMNWrW6/C1KhhvQpTXG/9SpJ8aYkrg5qJSCNgoTGmbQHLbgAmAYOBrsCbxpguxW2ztD16pfJLy0zjVMopdpzYweZjm9l1ahcnk09y+PxhDp0/xNELR0nPSr9svTqV69C7YW+ub3I9bWq1oUOdDgQHBtuwBwqsxJ4/gb76qtUbr14d0tKgb194/nno2tVK4K4wxuqp+wsR2WiMiSvJOsV+VCIyA+gD1BSRJOBZoAKAMWYqsAgryScAycBdJQtbqSsTEhRCvfB61AuvR9/GfQusk5GVwcnkk5xIPkHC6QQOnj1IfGI8SxKWMGvbrNx64cHhtIpsRcc6Hekb3ZfejXpTK6xWWe1KuXP0qHVIY9QoeOMNuPNOWLDA6oH37AndulnH3C9csA5/lCZh+1OSLy2XevSeoD165Q2ysrPYfWo3G49sZM+pPZxOOc3WE1tZd2hd7iGhtrXa0iOqB01rNKVrVFeuvupqPeRTSvv3W8ew27WzjonnHE+vWtXq0WtSLp5HevRK+bPAgEBaRbaiVWSrPOWpmamsPriaxQmL2XR0E19s+SI38UdWimRk65E81v0xmtZoakfYPmnuXKvnDpCSYiV2gGuuge++0yTvSdqjV8oFxhh+S/2NRXsWMX3LdJbvX056Vjo3t7qZJ3s8SZf6XXIvT1WX++YbuP56a759e+uqmb17oan+nSyx0vTo9f9PpVwgIlSvWJ1b29/KolsXsWvSLh7o/ABf7f6Kbh91o8VbLfhg4wdkZWfZHapX6NPH6qGLwAcfwPr1VvlTT8Evv1jlmuTLjvbolboCxy8eZ/rm6UzdOJXdp3bTvnZ73r3hXXo06GF3aLaJj4drr730vkUL+M9/rGvXu3a1LSy/oT16pcpYrbBaPNr9UXY8sIP3h7zP4fOH6flxTzpM7cDUDVPJyMqwO8Qy9ec/X7pZKCEBtm+HzZutG5s0ydtHE71SbhAgAdzX6T52PLCDv/f9O1kmiwlfT2DQ9EGcSj5ld3hl4rXX4G9/g/nzrWvXmzSBVq2sG5eUvTTRK+VGNSvV5I9X/5HN92/mtQGvEZ8YT8f3OrJ833K7Q/OY7GyoUwcef9x6f9tt9sajLqeJXikPEBEe7f4oy29fTpbJot+0foz9cuxlwzX4OmOsa+FzBgl76SXrRiflXTTRK+VBvRv1ZvvE7Tze/XFmbZ1F1w+7su/MPrvDcouoKGv8mKlTrbFnUlOtq2qU99FEr5SHVQ2tyj8H/JP5Y+az9/Rev+jZd+4MhxxDF27eDAMHQkiIvTGpwmmiV6qMDGsxjA+Hfci6Q+sYPnO4z16RY8yloYH37YPZs+2NRxVPE71SZeh37X7H24PfZtm+Zdw0+yafTfYjRljDFkRH2x2JcoWOdaNUGZvYeSKnU07zzMpn6D+tP/NumUf1ikUMgO5FckZ5SEyEhvpYCJ+hPXqlbPB0r6d5vs/zfHfgO/pN61fgePnexnkon6uusi8OVXKa6JWyyZTeU/h42Mf8dOQnHl/6uN3hFGnjxkvzJ07oSJO+RhO9Uja6q+Nd3N3hbt5a/xZTN0y1O5xCdexoXU65b9+VPdha2UOP0Stls6lDprLvt31M/HoiXet3pWPdjnaHlMdnn1lPd/r1V7sjUaWlPXqlbFYhsAKzRs6iSkgVnvvuObvDySMz03q835QpdkeiroQmeqW8QK2wWjzU9SEW7FrA6oOr7Q4n1+LF1vSll+yNQ10ZTfRKeYknezxJRMUInln5DHY9JyK/YcOsac4jAJVv0kSvlJcIDwnnmV7PsDJxJX9e/me7w2HOHGtavTpUqGBvLOrKaKJXyos81PUhxrQdwz/W/IONhzcWv4IHNWliTXfvtjUM5Qaa6JXyIiLCW4PeIrJSJE+vfNq2OC5etK6dz8zUyyn9gSZ6pbxMRKUIJsRNYEnCErYc22JLDAMGwPjxsHevLc0rN9NEr5QXmth5IpUqVOKlVfZc7rJmjTXNOXyjfJsmeqW8UGRYJLe1v43Z22aX+bH6fY7novzud9aNUsr3aaJXyku9cO0LhAeH84dv/1Cml1vm9OIf9+7hd1QJaKJXykvVCqvFs72fZcX+FczeVnZP95g82Zp29K6RGNQV0ESvlBd7uNvDNK7emOe/e55sk+3RtoyBBx6wHiqSna0jVPoTTfRKebGggCCe7/M8O07uYObWmR5t64034J134G9/0yTvb8SuW63j4uLMhg0bbGlbKV+SlZ1Fm3faEFEpgtV3e24cnJzkvns3NGvmsWbUFRKRjcaYuJKsoz16pbxcYEAg98bey5pf17By/0qPtJHT3wsO1iTvjzTRK+UDJsRNIKJiBC+vftkj2//+e2v6zDMe2byymSZ6pXxAWHAYEztPZOnepXx/4Hu3b//qq+Grr+DRR92+aeUFNNEr5SMe7/44tcNq88iSR9y63fnzYd48GDIEwsLcumnlJVxK9CIyUER2iUiCiEwuYPlVIrJSRH4Wkc0iMtj9oSpVvlUNrcoTPZ7g56M/u/Vu2REjdLx5f1dsoheRQOBtYBDQGhgrIq3zVXsamG2M6QiMAd5xd6BKKbgj5g4EYe72uW7Z3pIl1vTqq92yOeWlXOnRdwESjDH7jDHpwEzgxnx1DFDFMV8VOOy+EJVSOSLDIrk2+lq+3PGlW4ZFGDfOmn766RVvSnkxVxJ9fcD5+e9JjjJnzwHjRCQJWAQ8WNCGRGS8iGwQkQ0nTpwoRbhKqXHtxrHn9B7iE+OvaDvGwKlT1ryOUunf3HUydizwqTEmChgMTBORy7ZtjHnfGBNnjImLjIx0U9NKlS83t76ZKiFV+Nfaf13RdpKTrQT/hz+4KTDltYJcqHMIaOD0PspR5uweYCCAMeZ/IhIK1ASOuyNIpdQlVUKqMKnzJP76w185fP4w9cLrlWo7YWGQkODm4JRXcqVHvx5oJiLRIhKMdbJ1Qb46B4G+ACLSCggF9NiMUh4ypu0YgCsa1fLVV2H5cndFpLxZsYneGJMJTAKWAjuwrq7ZJiIviMgwR7XHgftE5BdgBnCnsWsQHaXKgba12tKhTgc+2fRJqdbPzoYnnoB+/dwcmPJKrhy6wRizCOskq3PZFKf57UBP94amlCqMiHB3h7t5aMlD/HTkJ2LrxpZo/VWrrGlsyVZTPkrvjFXKR41tN5bgwGDeXf9uidddt86aLlpUdD3lHzTRK+WjalaqycjWI5m5bSZpmWklWnfNGmjaFGrX9lBwyqtoolfKh41sNZIL6RdYd2hdidaLiYGhQz0UlPI6Lh2jV0p5pz6N+hAUEMTC3Qu5puE1Lq/34INQsaIHA1NeRXv0Svmw6hWr06dRH+bvmu/ykAgbN1rjzwcHezg45TU00Svl425udTO7T+1my/EtLtV/7DG46SYI0N/+ckN/1Er5uOEthxMgAczYMqPYusZcepqUJvryQ3/USvm4OpXrMLDpQD775TMysjKKrLtzpzV94IEyCEx5DU30SvmB8bHjOXLhCN8d+K7Iejm9+Ufc+5Aq5eU00SvlB/o27kulCpWKPXwTHg6DBumwxOWNJnql/EDl4MoMbT6UJXuXFHn1ze9+Z90NK1KGwSnbaaJXyk9c2+haDp8/zOZjmwtcnpICe/aUcVDKK2iiV8pPjGw9kpDAEN7b+F6BywcPhubNYdeuMg5M2U4TvVJ+IqJSBDe1uolpm6dxMf3iZcvj461pixZlG5eynyZ6pfzI7TG3cyH9AksSluQpP3zYmt5/vw1BKdtpolfKj/Rr3I964fX4fPPnecr//W9rOmGCDUEp22miV8qPBAUEMaLlCJYkLOF82vnc8n85niPepo1NgSlbaaJXys/c2u5W0rPSmbVtVm7Z/v2wfTsEBtoYmLKNJnql/Ey3qG40qd6EOdvn5JYFBUGrVjYGpWyliV4pPyMijGo9iuX7lnM65TRLlsAdd8CpU3ZHpuyiiV4pPzS42WCyTBbf7P2GCRPg88+halW7o1J20USvlB/q3qA7tcNq88XmL0hMtMqC9Hly5ZYmeqX8UFBAELe0uYXl+5ZD8HnuvdfuiJSdNNEr5acGNxtMalYqXLWaUaPsjkbZSRO9Un6qW1Q3KkgIV127jG7d7I5G2UmP2inlp6qGViW2XgcqNFhHlSp2R6PspD16pfxUZiZ0rN6H//36P45eOGp3OMpGmuiV8lO//AJTHxxFlsli0Z5FdoejbKSJXik/9dhjwJFYaoTUZPn+5XaHo2ykiV4pP7V+PYBwbePerD64ushHDCr/poleKT+UnW09OrBePejfuD8Hzh5g+4ntdoelbKKJXik/tHWrNe3fHwY2HQjA0r1LbYxI2UkTvVJ+qF07+PBD+PvfoWG1hrSObM30LdPtDkvZRBO9Un7om29g9GioU8d6f2/He/npyE96+KaccinRi8hAEdklIgkiMrmQOqNFZLuIbBORf7s3TKWUq5KTYeBAGDv2UtmoNtYYCEsT9PBNeVRsoheRQOBtYBDQGhgrIq3z1WkG/AnoaYxpAzzigViVUi7YsMGa9ulzqSyqShTNI5qzInGFLTEpe7nSo+8CJBhj9hlj0oGZwI356twHvG2MOQNgjDnu3jCVUq564AFrOmZM3vJrG13L9we+JzM7s+yDUrZyJdHXB351ep/kKHPWHGguIqtF5EcRGVjQhkRkvIhsEJENJ06cKF3ESqkiHXd0s+rn+y29Lvo6zqWd46cjP5V9UMpW7joZGwQ0A/oAY4EPRKRa/krGmPeNMXHGmLjIyEg3Na2UymEMBATArbeCSN5lfRr1AWDFfj18U964kugPAQ2c3kc5ypwlAQuMMRnGmP3AbqzEr5QqQ5mZ8NRTcNddly+rFVaLtrXasjJxZdkHpmzlSqJfDzQTkWgRCQbGAAvy1ZmP1ZtHRGpiHcrZ58Y4lVIuqFABJk2Cvn0LXt7rql6sOrCK1MzUsg1M2arYRG+MyQQmAUuBHcBsY8w2EXlBRIY5qi0FTonIdmAl8KQxRp85r1QZ+8tfYM6cwpcPbTGUlMwUFu9ZXHZBKduJXQMdxcXFmQ0514Eppdwi57h8Yb/WmdmZ1H21LoObDeaz4Z+VXWDKbURkozEmriTreNUTpjIyMkhKSiI1Vf+tLI3Q0FCioqKoUKGC3aEoG2zebE1r1Ci8TlBAEL0b9mbF/hUYY5D8Z2yVX/KqRJ+UlER4eDiNGjXSL2AJGWM4deoUSUlJREdH2x2OssG771rTyQXeu37JddHX8eWOLzlw9gCNqjXyeFzKfl411k1qaioRERGa5EtBRIiIiND/hsqxzz+3phMmFF2ve1R3AFYfXO3hiJS38KpED2iSvwL62ZVvycnWtHLlouu1q92OWmG1mLO9iLO2yq94XaL3Vz169Chy+eDBg/ntt9/KKBrlb5KSrOmrrxZfNyggiJta3sSK/SvIyMrwbGDKK2iiL4WsrKwSr7NmzZoily9atIhq1S67mVgpl3zxhTUdMMC1+tc3vZ7z6ef5/sD3ngtKeQ1N9PkkJibSsmVLbr31Vlq1asXIkSNJTk6mUaNG/PGPfyQ2NpY5c+awd+9eBg4cSKdOnbjmmmvYuXMnAMeOHWPEiBHExMQQExOTm+ArO/6fPnLkCL169aJDhw60bduWVatWAdCoUSNOnjwJwGuvvUbbtm1p27Ytb7zxRm5crVq14r777qNNmzYMGDCAlJSUsv54lJf68Udr2rata/X7Ne5HUECQPjS8nPCqq27ycx5mNcfo0TBxonU8cvDgy5ffeaf1OnkSRo7Muyw+3rV2d+3axUcffUTPnj25++67eeeddwCIiIjgp5+sAaH69u3L1KlTadasGWvXrmXixImsWLGChx56iN69ezNv3jyysrK4cOFCnm3/+9//5vrrr+epp54iKyuL5JwDqw4bN27kk08+Ye3atRhj6Nq1K71796Z69ers2bOHGTNm8MEHHzB69Gi+/PJLxo0b59pOKb/Wsyd06+Z6/crBlekW1Y2Fuxfy175/9Vxgyit4daK3S4MGDejZsycA48aN48033wTglltuAeDChQusWbOGUaNG5a6TlpYGwIoVK/jccflDYGAgVatWzbPtzp07c/fdd5ORkcHw4cPp0KFDnuU//PADI0aMICwsDICbbrqJVatWMWzYMKKjo3Prd+rUicTERDfvufJVTz5Z8nWGtxjOE8ue4ODZg1xV9Sr3B6W8hlcn+qJ64JUqFb28Zk3Xe/D55b96Jed9TvLNzs6mWrVqbNq0qcTb7tWrF99//z1ff/01d955J4899hi33367S+uGhITkzgcGBuqhGwVAYiJERoLj6+mywc0G88SyJ1i4eyETO0/0SGzKO+gx+gIcPHiQ//3vf4B1qOXqq6/Os7xKlSpER0czxzGoiDGGX375BbAO6bzruHMlKyuLs2fP5ln3wIED1K5dm/vuu497770391BQjmuuuYb58+eTnJzMxYsXmTdvHtdcc41H9lP5h/vvtw7dlFTLmi1pHdmaf2/RJ3/6O030BWjRogVvv/02rVq14syZM0wo4A6U6dOn89FHHxETE0ObNm3473//C8C//vUvVq5cSbt27ejUqRPbt+d9GHN8fDwxMTF07NiRWbNm8fDDD+dZHhsby5133kmXLl3o2rUr9957Lx07dvTcziqflpkJS5dC06YlX1dEGN16NGt+XcOpZB2D0J951aBmO3bsoFWrVrbEkyMxMZEhQ4awdetWW+MoLW/4DFXZmT8fRoyARx6B118v+frrDq2j64ddmTZiGuPa64l9X1CaQc20R6+UD3vhBWv64oulW79T3U7UqFhDL7P0c5ro82nUqJHP9uZV+XP2LISHFz/sQWECAwK5Lvo6vt33LXb9d688TxO9Uj7sp5/AcR1AqfWL7kfSuST2nN7jnqCU19FEr5QPq1oVrnRU6n6N+wHw7b5v3RCR8kaa6JXyUc88A1OmXPl2GldvTMOqDYlPjL/yjSmvpIleKR+UnQ0vvQSO0TmuiIjQvUF3Vh1cRbbJvvINKq+jib4MJCYm0tYx2lR8fDxDhgyxOSLl6xYssKa33eae7Q1uOpijF46y8fBG92xQeRVN9EUwxpCdrT0c5X3eftuaPvaYe7Y3uNlgAiSABbsWuGeDyqtoos8nMTGRFi1acPvtt9O2bVumTZtG9+7diY2NZdSoUbmjUa5fv54ePXoQExNDly5dOH/+PImJiVxzzTXExsYSGxtb7Bj0SpXWt47zpg0auGd7EZUiuPqqq1mwWxO9P/LaQc0eWfIIm46WfNCwonSo04E3Br5RbL09e/bw2Wef0bRpU2666Sa+/fZbwsLCePnll3nttdeYPHkyt9xyC7NmzaJz586cO3eOihUrUqtWLZYtW0ZoaCh79uxh7Nix5L/7Vyl3ePZZqF/fvdsc3HQwk5dP5tC5Q9Sv4uaNK1t5baK3U8OGDenWrRsLFy5k+/btuUMWp6en0717d3bt2kXdunXp3LkzYA1yBnDx4kUmTZrEpk2bCAwMZPfu3bbtg/Jvzz3n/m1e3/R6Ji+fzNd7vmZ8p/Hub0DZxmsTvSs9b0/JGY7YGEP//v2ZMWNGnuVbtmwpcL3XX3+d2rVr88svv5CdnU1oaKjHY1Xlz44d1iGb0t4NW5iY2jHUC6/H0r1LNdH7GT1GX4Ru3bqxevVqEhISAKvHvnv3blq0aMGRI0dYv349AOfPnyczM5OzZ89St25dAgICmDZtWqmeLatUccaMgeHD3b9dEWFIsyEs27uMtMw09zegbKOJvgiRkZF8+umnjB07lvbt29O9e3d27txJcHAws2bN4sEHHyQmJob+/fuTmprKxIkT+eyzz4iJiWHnzp25/xko5S6//gqbN8PAgZ7Z/rAWwziffp5v9n7jmQaULXSYYj+jn6F/mzoVJkyA7dvBEz/m9Kx0av6jJje1uolPh3/q/gbUFdNhipXycwsXQuPG0LKlZ7YfHBjMkOZDWLRnkY5m6Uc00SvlI1JSYMUKGDIE8j3W2K36N+7PieQTbDyid8n6C030SvmI0FD48Ud46CHPtnNjyxsJCghi1tZZnm1IlRmvS/T672Lp6Wfn30SgfXto0sSz7dSoWIMBTQYwe/ts/U75Ca9K9KGhoZw6dUq/XKVgjOHUqVN67b6fysiA+++3HjRSFsa0GcPBswf5MenHsmlQeZRX3TAVFRVFUlISJ06csDsUnxQaGkpUVJTdYSgPiI+H996DQYMgNtbz7d3Y8kZCAkOYuXUm3Rt093yDyqO8KtFXqFCB6Ct9XI5Sfug//4FKlWDAgLJpr0pIFW5ofgOzt8/mtetfIzAgsGwaVh7h0qEbERkoIrtEJEFEJhdR72YRMSJSoms8lVKFy8qC+fNh8GCoWLHs2h3bdixHLxxlZeLKsmtUeUSxiV5EAoG3gUFAa2CsiLQuoF448DCw1t1BKlWerVoFR4/CzTeXbbtDmg+hcnBl5m6fW7YNK7dzpUffBUgwxuwzxqQDM4EbC6j3IvAykOrG+JQq906cgNatrevny1JoUCgDmw5kzvY5pGbqr7UvcyXR1wd+dXqf5CjLJSKxQANjzNdFbUhExovIBhHZoCdclXLNqFGwbZv7R6t0xT0d7+F0ymm+3fdt2Teu3OaKL68UkQDgNeDx4uoaY943xsQZY+IiIyOvtGml/N6ZM9Yxertc2+hawoPD9RGDPs6VRH8IcH5gWZSjLEc40BaIF5FEoBuwQE/IKnXlHn0U2rQBu24tCQkKYVCzQXy1+yuyjT4/2Ve5kujXA81EJFpEgoExQO6fd2PMWWNMTWNMI2NMI+BHYJgxRp+hp9QVSE2FefOgRw/Pjm1TnBEtR3D0wlGW71tuXxDqihSb6I0xmcAkYCmwA5htjNkmIi+IyDBPB6hUebV4MZw7B2PH2hvHiJYjqBZajU82fWJvIKrUXDpGb4xZZIxpboxpYoz5i6NsijHmsgN3xpg+2ptX6srNmAGRkXDttfbGERIUwm3tb2Pu9rmcTjltbzCqVLxqrBullOX8eWvs+VGjIMgL7l//XbvfkZGdoU+e8lGa6JXyQpUqwVdfwYMP2h2JpXO9zkRUjGBxwmK7Q1Gl4AV9BaVUfoGB0Lev3VFcEhgQyIAmA1iSsIRsk02AaB/Rl+hPSykvc/o0PPkkJCbaHUleg5oO4vjF4/x85Ge7Q1ElpIleKS/z5Zfwz39aCd+bDGhiDZ25aM8imyNRJaWJXikvM3MmNGsGHTvaHUletSvXpmeDnszZPsfuUFQJaaJXyoscPWo9ZGT0aHtvkirM6Daj2XJ8CztP7rQ7FFUCmuiV8iLTpkF2NowbZ3ckBRvZeiSC6IPDfYwmeqW8yMWL1lOkWra0O5KC1QuvR7eobizcs9DuUFQJaKJXyos89xwsWWJ3FEUb1HQQGw5v0LtkfYgmeqW8xIED1tQbj807uzbaGpNBx6j3HZrolfICv/0GrVrBCy/YHUnxukd1J6JiBF/vKfI5Q8qLaKJXygs8/jikpMANN9gdSfECAwIZ3GwwX+36ivSsdLvDUS7QRK+UzYyBjz+25jt1sjcWV41pO4YzqWdYtneZ3aEoF2iiV8pm331nTf/xD3vjKInroq+jakhVZm3Tyyx9gSZ6pWz2xRcQEgITJ9odietCg0IZ2mIoixMWk5mdaXc4qhia6JWy2TvvwP/+B2FhdkdSMsNbDOdk8kl9xKAP0ESvlI2MgeBg7xvXxhVDmg+hVlgtXlnzit2hqGKyP2BVAAAVDElEQVRoolfKJmfOWJdULvLRwSBDgkJ4tNujLN+/nG3Ht9kdjiqCJnqlbPLee7BrF9Svb3ckpXdv7L0EBQQxbfM0u0NRRdBEr5QNUlLgT3+C666DmBi7oym9mpVqcl30dczcOpNsk213OKoQmuiVskHOdfN/+IO9cbjDuHbjOHD2AOsPrbc7FFUITfRKlbGUFHjxRejRwxqp0tcNbjYYQViS4OWjsZVjmuiVKmMVK8L06fDWW94/gJkrIipF0Ll+ZxYnLLY7FFUITfRK2aBvX9+8pLIwQ5sPZd2hdRw6d8juUFQBNNErVYYeeggmT7Y7Cvcb0XIEBqNDIngpTfRKlZEdO+Dtt61j9P6mTa029GjQg/c3vo8xxu5wVD6a6JUqA8bAY49B5crwzDN2R+MZd3W4i12ndvHz0Z/tDkXlo4leqTIwY4b1iMAXXoCaNe2OxjNGtBxBcGAwn//yud2hqHw00SvlYdnZ8Oij1nAHDzxgdzSeE1Epghtb3Mi0zdNIy0yzOxzlRBO9Uh4WEGCNZ/PqqxAUZHc0njW+03hOp5zWk7JeRhO9Uh50+rR1fL5TJxg0yO5oPK9vdF8aV2/MjK0z7A5FOdFEr5SHpKVB794wYYLdkZQdEWFkq5F8u+9bTqectjsc5aCJXikPmTwZtm6FG2+0O5KyNbrNaDKzM5m/c77doSgHlxK9iAwUkV0ikiAil93uISKPich2EdksIstFpKH7Q1XKd3z1FbzxBkyaVD4O2TiLrRtL84jmvPHjG3pNvZcoNtGLSCDwNjAIaA2MFZHW+ar9DMQZY9oDcwEfesyxUu516BDceac1xME//2l3NGVPRHiyx5NsOb6FdYfW2R2OwrUefRcgwRizzxiTDswE8vwzaoxZaYxJdrz9EYhyb5hK+Y4jR6wbo2bMsB76XR6NbjOaShUq8dHPH9kdisK1RF8f+NXpfZKjrDD3AAUOYyci40Vkg4hsOHHihOtRKuVD4uJg925o0cLuSOxTJaQKo1qPYsbWGVxMv2h3OOWeW0/Gisg4IA4o8GnBxpj3jTFxxpi4yMhIdzatlK2MsR4i8uyz1nx57ck7+32n33Mh/QKv//i63aGUe64k+kNAA6f3UY6yPESkH/AUMMwYo7fFqXIjO9salfKVV+DkSbuj8R7dG3RnZOuR/GXVXziZrB+MnVxJ9OuBZiISLSLBwBhggXMFEekIvIeV5I+7P0ylvFNWFvz+99ZDRB5/3H8eJuIuU3pNITUzlc82fWZ3KOVasYneGJMJTAKWAjuA2caYbSLygogMc1R7BagMzBGRTSKyoJDNKeVX7rkHPvwQnn7a6tFrks+rXe129GzQk/c2vqcPD7eRSyNvGGMWAYvylU1xmu/n5riU8gn9+kGzZvDUU3ZH4r0mxE1g3LxxfLP3GwY2HWh3OOWS3hmrVAkdOwbvv2+ddB03TpN8cW5ufTN1K9dlysopegOVTTTRK1UCa9ZYN0L9/vdwUa8adEloUCjP9n6W9YfXM2f7HLvDKZc00SvlgqwseOkl6NULKlaEn3+2bopSrrmr4110qNOBSYsm8Vvqb3aHU+5ooleqGFlZcN111iMAR42Cn36CDh3sjsq3BAcG896Q9ziZfJIXvnvB7nDKHU30ShUiPd2aBgZaPfkvvrCGNaha1d64fFWX+l24L/Y+3lz7JluObbE7nHJFE71S+WRkwGefQdOm8P33VtmLL8Ktt9oblz/4a9+/UqNiDcbNG6eXW5YhTfRKOaSkwP/9HzRvbo0+GRlpHY9X7hNRKYJXB7zK5mObWb5vud3hlBua6JXCulSya1drKIN69WDBAtiwATp3tjsy/zOy9UjqVK6jY+CUIU30qlxKTYXZs63DMZmZ1h2tTz8N8fGwejUMHap3uXpKxQoVmRA3gcUJi5mxRZ8tWxbErhsY4uLizIYNG2xpW5VPWVnWdfBffAGzZsHZs1C/PqxYYR2uUWUnNTOV3p/2Zt+ZfeyatIsaFWvYHZLPEJGNxpi4kqyjPXrl106fhqNHrfkffrh09cywYbBsGRw4oEneDqFBobx7w7ucTT3LqDmjyMjKsDskv6aJXvmVM2dg8WLrwdydO0PNmvAPx4Mtr77aOlxz9Ch8/rk1Tk1goL3xlmexdWP5cNiHrNi/gpe+f8nucPyaHrpRPislBbZuhXPnoG9f64Rq3brWWDRBQdC9u5XMhw61hi1Q3mnErBF8vftrNozfQPva7e0Ox+uV5tCNS6NXKmWn5GSoVMmaf+89+OYb2LYN9uyxHvrRrJn16D4RePNNqxffpYsOUeAr3hr0FvGJ8Tz+zeN8M+4bRM+Cu50memWrzEyrB16vnpWoly+/dOx8/37rlZxs9dpFrOEHtm2Dli3hllsgJiZvb330aPv2RZVO/Sr1eb7P8zy85GE+/vlj7om9x+6Q/I4meuVWZ8/CwYPw22/W69Qp63XPPVCtGsydC+++ayX348etR+8ZY9WpUcO6Aua116BBA2jc2Dpp2rSpNRxBSIjVo1f+Z2LniczcOpN7v7qXsOAwxrQdY3dIfkUTvY/KzrYuF8zIgLQ0CA217uJMS4O9e61pWpqVINPSoFUriIqCEyesk5XOy9LTYfhwq5e8cye8/XbeZWlp1jXmHTta15n/6U9W2fnzl14rVlg3HM2dC/fee3m8AwZYiT4jw7qGvXlz6+RorVpWb75CBaveM89Yww0E6GUC5UpQQBDLb1/OwOkDuWP+HTSPaE5s3Vi7w/IbPpnoX37ZGibWGOsF1vXQrztutHv6adi+/dLynOO4r75qLX/wQUhIuLTMGGs0wldesZbfdhskJV1aZoyVlP72N2v5DTdYPVDn5YMGWQkKrJOAycmXlhljHWZ45hkrObdsmbft7GwYP956gMXZs1asWVnWKyehP/20lWB//RWuuuryz+T11+GRR6z9atv28uUffmj1qvfvhzvuuHx5dLQV17FjMH261XsODr40vXDBqlehAoSHW8fBw8OtV+XK1nABYI3yOGeOldSrVoWICOtVpYq1fOxY61WY0NDClyn/VrFCRWaNnEWn9zsx8IuB/HL/L9QNr2t3WH7BJxP93r2waZM1L2K9nB8C8euv1ok6EatnKHIp0YB1Cd6pU5eWiVi9zByZmVaCdV7f+TK88HAr+easK5J3RMMGDayesPPy2rWtZQEB1mV/OdsNCLBeTZtay4ODYeRIqyww0HoFBECc4xx71aowZcql5RUqWMm4d+9Lbc+adXmibtbMWt6+vfX5OS/LmYK1ndOnC//se/a0ToYWJjraeilVGnUq12HxrYvp/EFnfr/w9/x3zH/15Kwb6OWVSimv88aPb/Do0kd5ptczPN/neU32TvTySqWUX3i468NsOLyBF79/kWyTzYvXvqjJ/gpooldKeR0R4fMRnxMgAfxl1V+IqhLF/XH32x2Wz9JEr5TySgESwCc3fsKxi8eYtGgStcJqcVOrm+wOyyfpRWxKKa8VGBDInFFzaFOrDWPmjmHZ3mV2h+STNNErpbxalZAqxN8RT4uaLRg9dzSbj222OySfo4leKeX1qleszoIxCwirEMbVH1/NnG1z7A7Jp2iiV0r5hOjq0fxw9w+0qdWG0XNH88iSR0jPSrc7LJ+giV4p5TMaVWvEittXcEfMHfxr7b+IfS+WtUlr7Q7L62miV0r5lIoVKvLJjZ8we+RszqSeodtH3fjz8j9zJuWM3aF5LU30SimfIyKMajOKHQ/s4NZ2t/K3H/5Glw+7sOXYFrtD80qa6JVSPqtKSBWmjZjGot8t4sj5I7Sf2p5b/3Mru0/ttjs0r6KJXinl00SEQc0GseOBHTzR/Qnm7ZhH+3fbM2XlFBJ/S7Q7PK+gg5oppfzKoXOHuGfBPSzduxSAng16MqbtGIY2H0rDag1tju7KlWZQM030Sim/tP/Mfv695d/M2DqDbSe2ARBXL47b29/OXR3vonKwbz5UWBO9UkoVYM+pPczaNouFuxey9tBaBKFFzRb0b9yfoc2H0qFOByLDIu0O0yUeS/QiMhD4FxAIfGiM+Xu+5SHA50An4BRwizEmsahtaqJXSpU1YwyrDq4iPjGetYfWsjRhKVkmC4CWNVvSpX4XGlRpQM1KNakXXo/YurE0rNqQCoEVbI78Eo+MRy8igcDbQH8gCVgvIguMMdudqt0DnDHGNBWRMcDLwC0lCUQppTxNROjVsBe9GvYC4HzaeX5M+pEfk35k3eF1LNu7jOMXj+cmf7CeZ9usRjOa1GhC42qNaVy9MdHVo6keWp2w4DDCKoQRFhxGlZAqBAcGIwgiUujUDq4MU9wFSDDG7AMQkZnAjYBzor8ReM4xPxd4S0TE2HVcSCmlXBAeEk7/Jv3p36R/blm2yea31N/Yf2Y/m45uYufJnew9s5d9Z/YRnxjPhfQLV9zuuze8W6bj67uS6OsDvzq9TwK6FlbHGJMpImeBCOCkcyURGQ+Md7y9ICK7ShO0m9QkX3w+RuO3ly/H78uxgx/EP+G5CScnMKG065f40qEyffCIMeZ94P2ybLMwIrKhpMe5vInGby9fjt+XYweNvzRcuWHqENDA6X2Uo6zAOiISBFTFOimrlFLKZq4k+vVAMxGJFpFgYAywIF+dBcAdjvmRwAo9Pq+UUt6h2EM3jmPuk4ClWJdXfmyM2SYiLwAbjDELgI+AaSKSAJzG+mPg7bziENIV0Pjt5cvx+3LsoPGXmG03TCmllCobOqiZUkr5OU30Sinl53wu0YvIKyKyU0Q2i8g8EanmtOxPIpIgIrtE5Hqn8oGOsgQRmexUHi0iax3lsxwnmxGREMf7BMfyRqVto4D4R4nINhHJFpE4p/JGIpIiIpscr6lOyzqJyBbHtt8Ux+11IlJDRJaJyB7HtLqjXBz1EhyfU6zTtu5w1N8jIncU14ar8fvK558v3udE5JDTZz7YG/bFk0rzObm5/UTH92yTiGxwlHn8e1xYGy7E+7GIHBeRrU5ltsVbVBtFMsb41AsYAAQ55l8GXnbMtwZ+AUKAaGAv1snjQMd8YyDYUae1Y53ZwBjH/FRggmN+IjDVMT8GmFXaNgqIvxXQAogH4pzKGwFbC1lnHdANEGAxMMhR/g9gsmN+stNnMdhRTxzrrXWU1wD2OabVHfPVi2qjBPH7xOefb1+eA54ooNy2ffHw706pPic3x5AI1MxX5vHvcWFtuBBvLyAWp99NO+MtrI1i96Msf8ge+NKMAKY75v8E/Mlp2VKgu+O11Kn8T46XYN1dl/NHI7dezrqO+SBHPSlpG8XEHo8LiR6oC+x0ej8WeM8xvwuo61Rvl2P+PWCs0zq7HMtz13WuV1QbJYjfpz5/R73nKDjR27YvHv59KdXn5OYYErk80Xv8e1xYGy7G3Ii8id62eAtro7h98LlDN/ncjfXXDQoeqqF+EeURwG/GmMx85Xm25VieM6RDSdsoqWgR+VlEvhORa5xiSSpk27WNMUcc80eB2vnjdzHOotpwla9+/pMc/wJ/7PTvvJ374kl2tJmfAb4RkY1iDYkCZfM9LqyN0rAz3lL9DMt0CARXici3QJ0CFj1ljPmvo85TQCYwvSxjc9Efgboi0idfeW78BTgCXGWMOSUinYD5ItLG1QaNMUZE3HWt7OtAc+fjkg5Fxe+VivouAe8CL2IlnxeBV7E6D8pzrjbGHBKRWsAyEdnpvNDN3+MCubMNX4nXKxO9MaZfUctF5E5gCNDXOP5/oeihGgoqPwVUE5EgR0/LuX7OtpIk75AOrrYx3bEffytyR50YY9KANMf8RhHZCzR3bD+qkDaPiUhdY8wREakLHM8Xf/51DgF98pXHF9DGq0AfY8zvXY2/iDYppNyTn39ueXHfpRwi8gGw0Ev2xVPsaDMPY0zOz+W4iMzDGh3XU99jV35XSsPOeEv3MyzL43PueAEDsYZIjsxX3oa8J7f2YZ18CnLMR3PpBFQbxzpzyHsCbaJj/gHynkCbXdo2itiPePIe447EcTIO62TZIaCG433+EzaDHeWvkPeEzT8c8zeQ94TNOkd5DWA/1gmh6o75ItsoQfw+9fk7tlfXaf5RYKbd++Lh351SfU5ubD8MCHeaX4P1++zx73FhbbgYdyPyHqO3Ld7C2ih2H8rqh+zGL0sC1jGqTY7XVKdlT2FdVbALp6tGsM5U73Yse8qpvLHjg05w/KKGOMpDHe8THMsbl7aNAuIfgXVcLQ04xqWTdjcD2xz79BMw1GmdOGCrY9tvcemO5ghgObAH+NbpyyNYD4vZC2whb0K+27FfCcBdxbXhavy+8vnn25dpjs9nM9Z4TXW9YV88/PtT4s/JjW03xvrj8ovju/5UWX2PC2vDhZhnYB1WzXB87++xM96i2ijqpUMgKKWUn/P1q26UUkoVQxO9Ukr5OU30Sinl5zTRK6WUn9NEr5RSfk4TvfJJIlJNRCY65vuIyMLi1ilFG3eKyFslXCdRRGoWUP6ciDzhvuiUcp0meuWrqmGNDOkyEQn0UCxKeTVN9MpX/R1oIiKbsO4irCwic8V6VsF0p3G9E0XkZRH5CRglIk1EZIljUK1VItLSUW+UiGwVkV9E5Hunduo56u8RkX/kFIrIWMc44ltF5OWCAhSRp0Rkt4j8gDW0s1K28MqxbpRywWSgrTGmg2PwuP9iDStwGFgN9AR+cNQ9ZYyJBRCR5cD9xpg9ItIVeAe4DpgCXG+sAbeqObXTAeiIdSfwLhH5PyAL61kInYAzWKMxDjfGzM9ZyTEw3RjH+kFYdztvdP/HoFTxNNErf7HOGJME4OjlN+JSop/lKK8M9ADmyKUHaIU4pquBT0VkNvAfp+0uN8acday/HWiIdXt6vDHmhKN8OtYDKuY7rXcNMM8Yk+yos8Bte6pUCWmiV/4izWk+i7zf7YuOaQDWuPEd8q9sjLnf0cO/Adjo6JEXt12lfIIeo1e+6jwQXpIVjDHngP0iMgpyn78Z45hvYoxZa4yZApwg71Cw+a0DeotITccJ3rHAd/nqfA8MF5GKIhIODC1JrEq5k/ZOlE8y1gNaVjsejpKCNZKmK24F3hWRp4EKwEys0RRfEZFmWKMDLneUXdbzd7R9RKwHa6901P/a5HsgizHmJxGZ5djOcWB9SfdRKXfR0SuVUsrP6aEbpZTyc5rolVLKz2miV0opP6eJXiml/JwmeqWU8nOa6JVSys9poldKKT/3/3oJn5nqBtNxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label='precision')\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label='recall')\n",
    "    plt.xlabel('threshold')\n",
    "    plt.legend(loc='center left')\n",
    "    plt.ylim([0, 1])\n",
    "\n",
    "plot_precision_recall_vs_threshold(precisions, recalls, thresholds)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### roc (receiver operation characteristic) curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8VHW+//HXZyaNJITeDFVFTegYQECaQaQpKh2F0MTG1auwKrv7u66uV1dX0atXmoC67LosYgMXhasICEqVXqVIEpASCCU9M/P9/TFDyLIQAmTmzJl8no8HD6acnPl4DOc95/s953PEGINSSil1KQ6rC1BKKRXcNCiUUkqVSINCKaVUiTQolFJKlUiDQimlVIk0KJRSSpXIb0EhIrNF5JiIbLvE+yIib4vIXhHZIiKt/VWLUkqpq+fPI4oPgJ4lvN8LaOz7Mw6Y6sdalFJKXSW/BYUxZgVwsoRF+gF/MV6rgcoiUsdf9SillLo6YRZ+djyQVux5uu+1Xy9cUETG4T3qICYm5tZbbrklIAUqpezNGDAY39/nX/P4XvC+Zrx/+xY4t5zL40EQ77u+Zd0eg8cYRARjvD/n8RjcHu/jnHw3EWHnvn9ffL0X1pZb4CYyzInB4PGA23g/o6y4Th/Dk58NHneGMabG1azDyqAoNWPMDGAGQFJSklm/fr3FFSmlyoIx3p1sRlYB+S43hW7v8xPZ+eS7PBw/k09uoZtCt4fMnALO5LrYcug0NWIjcHsM+S4PP6VmckONWNy+HbbbY0g9mYPL47/2ROfWLIDT9wcg8irWVekSr4c5BJfHUCsukjCHg3CncPBkDrfUjiMuKowwp5CemUvr+lVwOoTM7AISr4sDoMDlIb5KBcKdDr799K+czTzBvBmTD15Fed5arvYHy8AhoF6x53V9rymlLGKMocDtIbfAzd5jWew7nkWYw4HbY3B5jG9nXUhUuJOcAhd7jmZRp1KUbwfvYffRLCpGhRHhdJzfcRvDLxnZAMRGhXEyq4Cz+S4inA4K3J4yqXv74TMlvl8xKgynQ3CK4HQIDhGOnMnjplqxOEQQEZwOih47xBsC+zOyudW3Iz7351ROIZUqhFOjYiThTsHpcFAlOpzYqDDCnQ6y811cV7kCgnd9DgcIgsi59Z//+9wRS1S4kzjfz8dFhRNXIRynQ65qWxw6dIhHH32UwYMHM/SBBxja9rcAzJsx+arWB9YGxQJgvIjMBdoBp40x/zbspJQqWXa+i4ysfApcHjzm/PDIub/TM3PJzClAgAK3YdevZ4gMd3DwRA5xFcLZ8EsmtSpFsTntFCLeIRF/OZFdUPT4XEg4BDwGqsZEULlCOGFO7w7yVE4hbRpWpWpMBBFhDiLDHFSPjSQm0klOgZv6VaMJdzpwOoTIMAcxkWHnd+gihIc5qFUxkjBn+bgKwBjDzJkzmThxIoWFhfTp06fM1u23oBCRvwNdgeoikg48D4QDGGOmAYuA3sBeIAcY5a9alAo23mGT80Mt+S43J7K839aPZ+WTdjKHQ6dyqRAehsvj4VROIXuOnuXY2XwATmYXUDEqjLN5rjKp58iZPOB8SMRGhpGV76JKdDgNqsXQuGYsYU7vTjgrz0WlCuHUjIvyDR1BncpRRd+u8wrc1KoURVixb+FOh2B8YRAR5h1GiYsKL9rRq2uzb98+HnroIb777ju6devGe++9xw033FBm6/dbUBhjhl7mfQM87q/PV8qfjO+b+u4jZ3F5DOmZORgDu46cxeB9r3KFcNweQ6HHsOGXkzSsHkNWvouDJ3LKpIaLhUTjmrFFQytOh+BweIdR0k7m0LJeZepWiSYizMGpnAJurBlLhXAnteKiEBGuqxxF/arRVIwKL5P6VOBs3bqVDRs2MGPGDMaOHYtI2YavLSazlQoEYwzHz+Zz+HQep3ML2Xcsi19OZLP/eDbHzuax52gWVWMiKHR5OJt/5d/kLzaOXjEyzPdN3UFcVBhVYiLIzndRLTaCcKeD2nFRJNSJI9zpQASiI5zUqxpNlegIqkZHEBnuHZIp6x2DCn7btm3jp59+YsSIEdx7773s37+fatWq+eWzNChUSMsrdJOemUtWvovjZ/PJd7k5cDybX8/kecfsXR5+2HeCw6dzSzU2f7LYGDtA9dgIHCIkNazCoVN53HZ9VbLyXDSuGUtMZBhVoiNwOoUw37f8arERRDgd1KlUgQoRzkt8ilKXVlBQwMsvv8zLL79MrVq1GDRoEFFRUX4LCdCgUDZmjCGnwM3RM3lsST/NvuNZnM1zcfhULr+eziMjK59fT+dd8Xqjwh3UqxLN9TViOJPr4oaaMTSsFkNCnTgaVo8hNjKMyDD9Jq8Cb82aNYwZM4bt27fz4IMP8uabbxIVFeX3z9WgUEHHGO/58SezCziZXUBuoZu0kzkUuDys3JtBuNPB9z9nkJGVX+p13lK7InFR4VSvGEFkmJMT2QU0rBZNk+viiAxzEhsZRpuGVakUrePzKjgdOnSITp06UatWLb788ssyPavpcjQolCUysvL59Kd0DmXm4jaGH/adQIC8Qg/HzuZR6C79OZoOgaoxkeS73PRuWodW9StTv2o0tSpFUS0mgsrREf77D1HKz/bs2cNNN91EfHw8//jHP0hOTiYuLi6gNWhQKL/yHgUcZ/X+k2w/fJo9R7M4fvbyRwLhTqHQbWhYLZq0zFw63FCNo2fy6HJTDaIjwmhyXRw31oylYbUYHHp6pQpBp06d4plnnmHmzJksW7aMzp07c99991lSiwaFuiYnswtIPZlDXqGbNftPsuvIGQ5kZLPryNnL/mydSlFEhDno1bQO9atGU6dSFI2qx1C9YiSxkfqrqcqvBQsW8Oijj3LkyBF+85vf0KZNG0vr0X+NqlSy812kZ+by474MNqef5pudR6/oYq/YyDB6Nq1NQp042jSsUnTKp1LqX40dO5ZZs2bRrFkzvvjiC5KSkqwuSYNCXZzHY/h4Qxo/7jvB4u1HyS10l7h89dhIEupU5ExuIf1vrUvjmhWpW6UC8ZUr6NCQUpdhfOdmiwhJSUk0aNCAZ599loiI4Jhf06BQABS6PSzffZyVezP4csthMrIKLrpcdISTelWieeC2+rSoW5km18WVm146SvlDWloajzzyCEOGDGH48OE88sgjVpf0bzQoyim3x7DryBlW7c1g1soDHD1z8QnmuKgw3h7ais6Na+iRgVJlyOPxMH36dJ599lncbrdlE9WloUFRTnivQM7gx30nmL5i/yWXq1ulAt0TanF/63iaxVfSC8qU8oOff/6ZsWPHsmLFCrp3786MGTNo1KiR1WVdkgZFCDqZXcCqvRl8s/MoWw+d5vCpXPIKL933f2SHhnRPqMXtjasHsEqlyq8dO3awZcsWZs+ezciRI4P+C5kGRYjId7n5aE0qX2w6zKa0U5dcLtwpvHBPU/o0q6NXISsVQJs3b2bTpk2kpKTQr18/9u/fT5UqVawuq1Q0KGzsl4xs/rE+jaU7j7H76L9ft3BTrVjubRVPm4ZVia9cgTqVooL+m4tSoSY/P5+XXnqJP/3pT9SpU4fBgwcTFRVlm5AADQrb+W73MT5en8airUcu+n6r+pUZe/v19GleJ8CVKaUu9OOPPzJmzBh27tzJiBEjmDx5ckCa+JU1DQqbWPfLSV5ZtJOfUv91WCkyzEHnm2rQv3U8dzWprUcMSgWJQ4cO0aVLF2rXrs2iRYvo1auX1SVdNQ2KIGaM4YMffuHlRTv/pUlei7qV6Ncynt7N6lC7kv2+nSgVynbu3ElCQgLx8fHMmzeP5ORkKlasaHVZ10SDIkgdyMim2+vL/uW1OpWimPdwe+pVjbamKKXUJWVmZjJhwgTef/99VqxYQadOnbj33nutLqtMaFAEkfTMHL7bdYwFmw+z7pfMotfrVqnAn+5vrqevKhWkPvvsMx577DGOHz/OpEmTLG/iV9Y0KCxmjGHuujQmfbr1397rdnMNnkhuTKv69jk7QqnyZvTo0bz//vu0bNmSf/7zn7Ru3drqksqcBoWF3l91gBcW7vi31+9qUosnk28i8brA3pxEKVU6xZv43XbbbTRu3JiJEycSHh6a1yZpUFjg19O5jHp/3b/cs2FE+wZMuPNmvQhOqSB38OBBHn74YYYNG8aIESMYN26c1SX5nQZFgP1jXSrPfnJ+mKl3s9q8PrAF0RH6v0KpYObxeJg6dSrPPfccxhgGDhxodUkBo3unAPn56FlSZq/l8Ok8AJwO4a9j2tH+hmoWV6aUupzdu3czduxYVq5cSY8ePZg+fToNGza0uqyA0aDws7N5hTz1j818s/No0WsPtKvPS/c21YvjlLKJ3bt3s337dj744ANGjBhR7v7talD4Sb7LzVvf/MzUZfuKXqtUIZypD7Smw416mqtSwW7jxo1s2rSJUaNGcc8997B//34qV65sdVmW0KDwg+2HT9Pn7ZX/8trkQS24v3VdiypSSpVWXl4eL774Iq+99hrx8fEMHTqUqKiochsSoEFRpowxjJuzgf/bcX6Y6f7W8bzWv7neLlQpG1i1ahVjxoxh9+7djBo1ijfeeMOWTfzKmgZFGTHGMGTGatYcOAl47/uw7nfdqRwdHDdHV0qV7NChQ3Tr1o34+HgWL15Mjx49rC4paGhQlIHDp3IZMXste49lAd47xj1/d2K5m/BSyo527NhBYmIi8fHxfPLJJ3Tr1o3Y2FirywoqOh5yjdb9cpIOf1paFBKvD2zBH+5poiGhVJA7efIkI0eOpEmTJqxYsQKAu+++W0PiIvSI4ioZY5j06VbmrksDoFpMBH8d246EOtp2Q6lg98knn/D4449z4sQJfve739G2bVurSwpqGhRXYVPaKfpP/QG3x9vvpXtCLd4Z2ooKEU6LK1NKXc7IkSP58MMPad26NV9//TUtW7a0uqSgp0FxhX5KzeT+KT8UPR/X+Xp+2zvBwoqUUpdTvIlfhw4dSEhIYMKECYSF6S6wNPw6RyEiPUVkt4jsFZHnLvJ+fRH5TkQ2isgWEentz3qu1amcAlJmrQUgOsLJdxO7akgoFeQOHDhAjx49+Mtf/gLAuHHjePbZZzUkroDfgkJEnMC7QC8gERgqIokXLPZ7YJ4xphUwBJjir3qulcdjGP3BOs7mu4ivXIG1v+tOo+oxVpellLoEt9vN22+/TdOmTVm9enXRUYW6cv6M1LbAXmPMfgARmQv0A4rfgMEA52Z/KwGH/VjPVSt0e7hvyiq2HToDwP8MaUlspH4bUSpY7dy5kzFjxvDjjz/Sq1cvpk2bRv369a0uy7b8ubeLB9KKPU8H2l2wzB+AJSLyH0AM0P1iKxKRccA4IOD/s/cdz2LA1B/IzCkEYNqDt5LUsGpAa1BKXZm9e/eye/du5syZwwMPPKCnq18jq6+jGAp8YIypC/QG5ojIv9VkjJlhjEkyxiTVqFEjYMWlncwh+Y3lRSHx+z4J9GxaO2Cfr5QqvQ0bNjB79mzAez3EgQMHePDBBzUkyoA/g+IQUK/Y87q+14obA8wDMMb8CEQBQdFa1e2bkwCIiwpj0ROdGNvpeourUkpdKDc3l+eee4527drxxz/+kbw87z1f4uL0mqay4s+gWAc0FpFGIhKBd7J6wQXLpALJACKSgDcojvuxplJ7Zv4Wfj6WRYVwJ4ue7KT3r1YqCK1YsYIWLVrw6quvMnLkSDZu3KhN/PzAb3MUxhiXiIwHFgNOYLYxZruIvAisN8YsACYA74nIU3gntkeaIDg14c+Ld/HJT+kATHmgNXWrRFtckVLqQocOHSI5OZl69erxzTffkJycbHVJIcuvp+4YYxYBiy547b+KPd4BdPRnDVdq8fYjvPud92ZD/69vIt1uqWlxRUqp4rZu3UqzZs2Ij4/ns88+o1u3bsTE6Knq/mT1ZHZQ2XvsLP/x0UYA+javw5jbG1lckVLqnIyMDIYPH07z5s2Lmvj17dtXQyIA9GIAH4/He9OhAreH5nUr8frAFlaXpJTC237j448/Zvz48WRmZvL888/Trt2FZ9orf9Kg8Pnn1l/ZfzwbgJkjkogK1wZ/SgWDlJQU5syZQ1JSEt9++y3NmjWzuqRyR4MCyM53MenTrQAMvLUuNeP0rAmlrFS8iV+XLl1o3rw5//mf/6n9mSyicxTApE+3kpXvomJUGC/0a2J1OUqVa/v376d79+588MEHAIwZM4aJEydqSFio3AfFNzuOsmCzt8XU+yPbEB2hv4xKWcHtdvPWW2/RrFkz1q1bh8NR7ndPQaNc7xUzswv47WfeIac7E2tpDyelLLJjxw5Gjx7NmjVr6NOnD9OmTaNu3bpWl6V8ynVQPL9gO8fO5hMV7uC/72tqdTlKlVsHDhxg3759fPTRRwwZMkT7MwWZchsUaw+cLBpymjOmHTUr6gS2UoG0bt06Nm3axEMPPUSfPn3Yv38/FStWtLosdRHlchDQ4zE89rcNAPRpXoc2OuSkVMDk5OQwceJEbrvtNl555ZWiJn4aEsGrXAbFkh1HyMgqAOC5nrdYXI1S5ceyZcto3rw5b7zxBg899JA28bOJcjf0lFPg4s+LdwMwvtuN1KuqDf+UCoT09HTuvPNOGjRowNKlS+nWrZvVJalSKndHFK9+tYt9x7OJcDoY2bGh1eUoFfI2b94MQN26dfniiy/YsmWLhoTNlKug+CUjmzmrDwIweXALqsdGWlyRUqHr+PHjDBs2jJYtW7J8+XIAevfuTXS0HsXbTbkaenrxyx14DHRqXJ2+za+zuhylQpIxhrlz5/LEE09w+vRpXnjhBdq3b291WeoalJugWLrrKEt3HcMh8PJ92lRMKX8ZPnw4f/vb32jXrh2zZs2iSRNti2N35SIoPB7DhHnecdKBt9bTCWylypjH40FEEBG6devGrbfeyhNPPIHTqV2YQ0G5mKP4cuuvZOYUAjBab0akVJnau3cvycnJvP/++4C3id9TTz2lIRFCykVQzFuXBsCEO2/i5tp6UY9SZcHlcvH666/TrFkzNm7cSEREhNUlKT8J+aGnPUfPsnJvBuFO4YHbGlhdjlIhYdu2bYwaNYr169fTr18/pkyZwnXX6QkioSrkg+Lva1MB6Nv8OqrG6DcepcpCamoqBw8eZO7cuQwaNEib+IW4kA6KE1n5/G21Nyge1KMJpa7JmjVr2Lx5M+PGjaN3797s37+f2NhYq8tSARDScxSvL9lDgdtDi3qVubVBFavLUcqWsrOzefrpp2nfvj2vvfYa+fn5ABoS5UjIBoUxpmjY6f5W8RZXo5Q9LV26lObNm/Pmm2/yyCOP8NNPPxEZqR0NypuQHXrakn666PHgNvUsrEQpe0pPT+euu+6iUaNGLF++nM6dO1tdkrJIyB5R/LDvBAADb61LVLiez61UaW3cuBHwNvFbuHAhmzdv1pAo50I2KBZt/RWA266vZnElStnD0aNHGTx4MK1bty5q4tezZ08qVKhgcWXKaiEZFBsOZrL10GmiI5zc1bS21eUoFdSMMfz1r38lMTGRzz//nJdeeokOHTpYXZYKIiE5R/H1Nu/RRIcbqhEbGZL/iUqVmWHDhjF37lzat2/PrFmzSEhIsLokFWRCbi9qjGHx9qOAXjuh1KUUb+LXo0cP2rdvz+OPP679mdRFhdzQ0/bDZ0g9mUNcVBjtGun8hFIX2rNnD926dWP27NkAjBo1Sju9qhKFXFB8ve0IAJ0a16BChP7iK3WOy+Xitddeo0WLFmzZskUnqVWphdzQ06yVBwBoVb+yxZUoFTy2bNnC6NGj2bBhA/fddx/vvvsuderUsbosZRMhFRT7j2eRW+gGdH5CqeLS09NJS0vj448/pn///trET10Rvw49iUhPEdktIntF5LlLLDNIRHaIyHYR+ehaPm/9L5kAdLu5hl5kp8q9H374gWnTpgEUNfEbMGCAhoS6Yn4LChFxAu8CvYBEYKiIJF6wTGNgEtDRGNME+M9r+czVB7xXY3dqXONaVqOUrWVlZfHkk09y++2388YbbxQ18YuJibG4MmVX/jyiaAvsNcbsN8YUAHOBfhcs8xDwrjEmE8AYc+xaPvCLTYcBaBpf6VpWo5RtLVmyhKZNm/LOO+/w+OOPaxM/VSb8OUcRD6QVe54OtLtgmZsARGQV4AT+YIz5+sIVicg4YBxA/fr1L/phZ/IKcXsMAInXxV1j6UrZT1paGn369OGGG25gxYoV3H777VaXpEKE1afHhgGNga7AUOA9Efm305WMMTOMMUnGmKQaNS4+rPTlZu/V2DfUiNGrsVW5smHDBgDq1avHokWL2LRpk4aEKlP+DIpDQPH+3nV9rxWXDiwwxhQaYw4Ae/AGxxXbc/QsAJ1v0vkJVT4cOXKEgQMHkpSUVNTE78477yQqKsriylSo8WdQrAMai0gjEYkAhgALLljmc7xHE4hIdbxDUfuv5sN+PuYNihtq6F23VGgzxvDhhx+SmJjIwoULefnll7WJn/Irv43RGGNcIjIeWIx3/mG2MWa7iLwIrDfGLPC910NEdgBu4DfGmBNX+lnZ+S5W7fX+WPeEWmX236BUMBoyZAjz5s2jY8eOzJw5k1tuucXqklSI8+tgvjFmEbDogtf+q9hjAzzt+3PV1h/0Xj9xffUYalfSw24Veoo38evduzedOnXisccew+GweppRlQch8Vu2Oe0UANUr6mmAKvTs2rWLzp07M2vWLABSUlIYP368hoQKmJD4TVu223v5Rf/W8RZXolTZKSws5OWXX6ZFixbs2LGD2Fidf1PWCInzSE9mFwA6ka1Cx6ZNmxg1ahSbNm1iwIABvPPOO9SurXdrVNawfVD8kpHNLydyiIlw0qyuXpGtQsORI0c4cuQIn3zyCffff7/V5ahyzvZBsfXQaQDaNqpKZJg2AlT2tXLlSrZs2cJjjz1Gz5492bdvH9HR0VaXpZT95yi2+YLi5tratkPZ09mzZxk/fjydOnXirbfeKmripyGhgoXtg2LXEe+Fds112EnZ0OLFi2natClTpkzhySef1CZ+KijZeujJ7TFs8F1D0Uw7xiqbSUtLo2/fvtx4442sXLlSr65WQeuKjyhExCEiD/ijmCt1ICOLrHwXdSpFUa+qHqar4GeMYe3atYC3id9XX33Fxo0bNSRUULtkUIhInIhMEpH/FZEe4vUfeHsxDQpciZe2Jd07P3FL7YoWV6LU5f3666/079+fdu3aFTXx6969uzbxU0GvpKGnOUAm8CMwFvgtIMC9xphNAajtstYeOAnATbU0KFTwMsbwwQcf8PTTT5OXl8err75Kx44drS5LqVIrKSiuN8Y0AxCRmcCvQH1jTF5AKiuF1JM5ANSK029kKngNGjSI+fPn06lTJ2bOnMlNN91kdUlKXZGSgqLw3ANjjFtE0oMpJAB2+854ulmHnlSQcbvdiAgOh4O7776bO+64g4cfflj7MylbKum3toWInBGRsyJyFmhe7PmZQBVYkhO+1h2NqutN41Xw2LlzJ506dSpq4jdixAgeffRRDQllW5f8zTXGOI0xccaYir4/YcWeW351W06Bq+hxHW0troJAYWEhL730Ei1btmT37t1UqqSnbKvQcMmhJxGJAh4BbgS24L3xkOtSywfaiayCosciYmElSsHGjRsZOXIkW7ZsYfDgwbz99tvUrFnT6rKUKhMlzVF8iHee4nugN9AEeDIQRZXGuY6xToeGhLLe0aNHycjI4PPPP6dfv35Wl6NUmSopKBKLnfU0C1gbmJJKJyvfe3CjV2Qrq6xYsYKtW7fy+OOP07NnT/bu3UuFChWsLkupMlfS7Frxs56CZsjpnONnvY3Trqus8xMqsM6cOcNjjz1Gly5dePvtt4ua+GlIqFBVUlC09J3ldCYYz3o6dCoXgDqV9B+nCpxFixbRpEkTpk+fztNPP61N/FS5UNLQ02ZjTKuAVXKFfj7qvYaipt4nWwVIWloa/fr14+abb2b+/Pm0a9fO6pKUCoiSjihMwKq4Cj/uPwFAbT01VvmRMYbVq1cD3iZ+S5Ys4aefftKQUOVKSUcUNUXk6Uu9aYyZ7Id6Su3oGe+4cH3tGqv85PDhwzz66KMsWLCAZcuW0aVLF7p162Z1WUoFXElB4QRi8TYCDFo31Iy1ugQVYowxzJo1i4kTJ5Kfn8/rr7+uTfxUuVZSUPxqjHkxYJVcgdwCd9HjipG2vveSCkIDBgzg008/pUuXLsycOZMbb7zR6pKUslRJe9mgPZI4maNXZauyVbyJ37333kuPHj146KGHtD+TUpQ8mZ0csCqu0Elf+46EOpa3nFIhYNu2bXTs2LGoid/w4cO106tSxZTUFPBkIAu5EodPe6+hqFRBh53U1SsoKOCFF16gdevW7Nu3jypVqlhdklJByZZ72gKXBzjf70mpK7VhwwZGjhzJtm3bGDZsGG+99RY1atSwuiylgpItg+KUb46iZb3KFlei7OrEiROcOnWKhQsX0rdvX6vLUSqo2TIoDmR4b4Far4peQ6FK77vvvmPr1q088cQT9OjRg59//pmoKL1gU6nLseVs3cq9xwGoU1n7PKnLO336NA8//DB33HEHU6dOLWripyGhVOnYMigqRHgPhKLCbVm+CqCFCxeSmJjIzJkzmThxIhs2bNAmfkpdIVsOPW1NPwVo+w5VsrS0NPr3788tt9zC559/Tps2bawuSSlbsuVX8koVwgGooZ1j1QWMMfzwww/A+SZ+69ev15BQ6hr4NShEpKeI7BaRvSLyXAnL9RcRIyJJl1unx0BmTiERTge1KuoYszovPT2de+65h44dO7J8+XIAunbtSkREhMWVKWVvfgsKEXEC7wK9gERgqIgkXmS5injvxb2mNOt1ub3XUBS4PTj0ftkK8Hg8TJ8+ncTERL799lsmT57M7bffbnVZSoUMfx5RtAX2GmP2G2MKgLnAxe46/0fgVSCvNCt1eby3ydD2Heqc/v3788gjj9CmTRu2bdvGU089hdPptLospUKGP4MiHkgr9jzd91oREWkN1DPG/LOkFYnIOBFZLyLrMzK9E9m14nR+ojxzuVx4PN6jy/79+/Pee+/xzTffcP3111tcmVKhx7LJbBFxAJOBCZdb1hgzwxiTZIxJqlixIgBRIS4hAAARhUlEQVTHfDcuUuXPli1baN++Pe+99x4ADz74IGPHjtVOwkr5iT+D4hBQr9jzur7XzqkINAWWicgvwG3AgstNaBvfDVpbaPuOcic/P5/nn3+eW2+9lYMHD2pvJqUCxJ/XUawDGotII7wBMQQYdu5NY8xpoPq55yKyDJhojFlf0kpdHoMDqBCuY9Dlybp16xg5ciQ7duxg+PDhvPnmm1SrVs3qspQqF/wWFMYYl4iMBxbjva3qbGPMdhF5EVhvjFlwNev1+IKiSnR4GVargl1mZiZZWVksWrSIXr16WV2OUuWKX6/MNsYsAhZd8Np/XWLZrqVZp9tjCAMqaVCEvKVLl7J161aefPJJevTowZ49e7T9hlIWsN2V2W7fJEVclAZFqDp16hQPPfQQycnJTJ8+vaiJn4aEUtawXVCcm8yuEKFzFKHoiy++IDExkdmzZ/PMM89oEz+lgoDtmgK6fRfc6WR26ElNTWXgwIEkJCSwYMECkpIu29FFKRUAtjui8PgOKWIibZdx6iKMMXz//fcA1K9fn2+++YZ169ZpSCgVRGwXFL6RJyLDbFe6ukBqaip9+vShc+fORU38OnfurE38lAoyttvb5hW6Ab1pkZ15PB6mTJlCkyZNWLFiBW+//bY28VMqiNlu/CbM1zE23KlBYVf3338/X3zxBXfeeSczZsygYcOGVpeklCqB7YLiXPdYPT3WXlwuFw6HA4fDweDBg+nXrx8jR47U/kxK2YBtv5bHRtku48qtzZs3065dO2bMmAHA0KFDGTVqlIaEUjZh26DQoafgl5eXx+9//3uSkpJIT0+ndu3aVpeklLoKtvxaHq0X2wW9tWvXkpKSwq5du0hJSWHy5MlUrVrV6rKUUlfBlkERoafGBr0zZ86Qm5vL119/zV133WV1OUqpa2DLoChweawuQV3EkiVL2L59O0899RTdu3dn9+7d2n5DqRBgy6/mOQVuq0tQxWRmZjJq1CjuuusuZs2apU38lAoxtgyK66vHWF2C8vn0009JTExkzpw5TJo0ifXr12tAKBVibDn0pGc8BYfU1FSGDBlC06ZNWbRoEa1atbK6JKWUH9hyjxsepuffW8UYU9SXqX79+ixdupQ1a9ZoSCgVwmwZFCezCqwuoVw6ePAgvXr1omvXrkVhcfvttxMerlfJKxXKbBkU2mI8sDweD//7v/9LkyZNWLlyJe+88w6dOnWyuiylVIDYco9bu1KU1SWUK/feey8LFy7krrvuYvr06TRo0MDqkpRSAWTLoIjQyWy/KywsxOl04nA4GDp0KAMGDGD48OHan0mpcsiWe1w968m/fvrpJ9q2bcu0adMAbxO/ESNGaEgoVU7Zco/7y4lsq0sISbm5uUyaNIm2bdty5MgR6tWrZ3VJSqkgYMuhpybXVbK6hJCzevVqUlJS2LNnD6NHj+b111+nSpUqVpellAoCtgyKcKcOgZS17OxsCgsL+b//+z+6d+9udTlKqSBiy6AI06AoE19//TXbt29nwoQJJCcns2vXLiIiIqwuSykVZGw5RxHmsGXZQePEiROkpKTQq1cvPvzwQwoKvBcwakgopS7Glntch559c1WMMcyfP5/ExEQ++ugjfv/737Nu3ToNCKVUiWw59HTsbJ7VJdhSamoqw4YNo3nz5ixZsoQWLVpYXZJSygZseURxY81Yq0uwDWMMS5cuBaBBgwYsW7aM1atXa0gopUrNlkGhF9yVzoEDB+jRowfJyclFTfw6dOhAWJgtDySVUhax5R5XW3iUzO128z//8z80bdqUNWvWMHXqVG3ip5S6arb8aunyGKtLCGr9+vXjn//8J71792batGl6hbVS6prYMyjcHqtLCDrFm/gNHz6coUOHMmzYMO3PpJS6Zn4dwxGRniKyW0T2ishzF3n/aRHZISJbRORbESlV/+rqFfWezMWtX7+epKQkpk6dCsDgwYN54IEHNCSUUmXCb0EhIk7gXaAXkAgMFZHECxbbCCQZY5oD84HXSrNup+4AAW8Tv2effZZ27dpx/PhxvU+EUsov/HlE0RbYa4zZb4wpAOYC/YovYIz5zhiT43u6GqhbmhVrTsCPP/5IixYteO211xg9ejQ7duygb9++VpellApB/pyjiAfSij1PB9qVsPwY4KuLvSEi44BxABG1b8Tp0KTIzc3F4/HwzTffkJycbHU5SqkQFhST2SLyIJAEdLnY+8aYGcAMgMg6jU15beGxaNEitm/fzm9+8xvuuOMOdu7cSXh4uNVlKaVCnD+Hng4Bxc/LrOt77V+ISHfgd8A9xpj80qzYUc6OKDIyMnjwwQfp06cPf/vb34qa+GlIKKUCwZ9BsQ5oLCKNRCQCGAIsKL6AiLQCpuMNiWOlXXF5mcw2xjB37lwSEhKYN28ezz//PGvXrtUmfkqpgPLb0JMxxiUi44HFgBOYbYzZLiIvAuuNMQuAPwOxwMe+UzlTjTH3XG7d5eWAIjU1lZSUFFq0aMGsWbNo1qyZ1SUppcohv85RGGMWAYsueO2/ij2+qluphfLQkzGGb7/9lu7du9OgQQOWL19OmzZtcDqdVpemlCqnbNk0KTvfZXUJfrFv3z6Sk5O58847i5r43XbbbRoSSilL2TIoqsSE1hi92+1m8uTJNGvWjA0bNjB9+nRt4qeUChpBcXrslQq1yey7776br776ir59+zJ16lTq1i3VdYdKKRUQ9gyKEJijKCgoICwsDIfDwciRIxk+fDhDhgzR/kxKqaBjy6Enu19wt3btWm699VamTJkCwKBBgxg6dKiGhFIqKNkyKOx6RJGTk8OECRNo3749mZmZ3HDDDVaXpJRSl2XLoSc75sTKlStJSUlh//79PPzww7z66qtUqlTJ6rKUUuqy7BkUNkyKczcW+u677+jatavV5SilVKnZMijsctbTwoUL2blzJ8888wzdunVjx44dhIXZcpMrpcoxW85RBPtk9vHjxxk2bBj33HMPf//734ua+GlIKKXsyJZB4THG6hIuyhjDRx99REJCAvPnz+fFF19kzZo12sRPKWVrtvyKGxkenPmWmprKqFGjaNWqFbNmzaJJkyZWl6SUUtcsOPe4lyEEz9CTx+Nh8eLFADRo0IDvv/+eVatWaUgopUKGPYMiSHLi559/5o477qBnz56sWLECgLZt22oTP6VUSLFnUFj8+S6Xiz//+c80b96cTZs2MWvWLG3ip5QKWbaco7D6iKJv374sXryYfv36MWXKFK677jprC1JKKT+yZVBYcUyRn59PeHg4DoeDsWPHMnr0aAYOHKj9mZRSIc+eQ08B3jevXr2a1q1b8+677wIwYMAABg0apCGhlCoX7BkUAfqc7OxsnnrqKTp06MDZs2dp3LhxgD5ZKaWChy2HngLxTf77778nJSWFAwcO8Nhjj/HKK68QFxfn989VSqlgY8+gCMBnuFwuwsPDWb58OZ07dw7AJyqlVHCyZ1D4KSk+//xzdu7cyaRJk+jWrRvbt2/X/kxKqXLPpnMUZZsUR48eZdCgQdx3333Mnz9fm/gppVQx9gyKMsoJYwxz5swhMTGRL774gv/+7/9m9erV2sRPKaWKKddfmVNTUxk7dixJSUnMmjWLW265xeqSlFIq6JS7IwqPx8NXX30FeJv4rVq1ihUrVmhIKKXUJdg0KK4uKfbs2UPXrl3p3bs3y5cvByApKUmb+CmlVAnsGRRXuLzL5eLVV1+lefPmbN26lffff19PeVVKqVKy5RzFlR5Q9OnThyVLlnD//ffz7rvvUrt2bf8UppRSIUhMkN5W9FIi6zQ2B3dtpXalqBKXy8vLIzw8HKfTySeffAJA//79A1GiUkoFHRHZYIxJupqftefQ02WOKFatWkXLli2Lmvj1799fQ0Ippa6SPYPiEq9nZWXxxBNP0KlTJ/Ly8khISAhoXUopFYpsOUdxsaRYvnw5KSkppKamMn78eF5++WViY2MDX5tSSoUYWwbFpVp4REdH8/3339OxY8cAV6SUUqHLnkHhy4lPP/2UXbt28dvf/pYuXbqwdetWvSZCKaXKmF/nKESkp4jsFpG9IvLcRd6PFJF/+N5fIyINS7PeY0ePMGDAAPr3789nn31W1MRPQ0Ippcqe34JCRJzAu0AvIBEYKiKJFyw2Bsg0xtwIvAm8ern1unPO0CGpJV9++SWvvPIKP/zwgzbxU0opP/LnEUVbYK8xZr8xpgCYC/S7YJl+wIe+x/OBZLlMfw73meMkJDRh8+bNPPfcc4SHh5d54Uoppc7z5xxFPJBW7Hk60O5SyxhjXCJyGqgGZBRfSETGAeN8T/NX/7hqmzbxA6A6F2yrcky3xXm6Lc7TbXHezVf7g7aYzDbGzABmAIjI+qu9ujDU6LY4T7fFebotztNtcZ6IrL/an/Xn0NMhoF6x53V9r110GREJAyoBJ/xYk1JKqSvkz6BYBzQWkUYiEgEMARZcsMwCIMX3eACw1Nit+ZRSSoU4vw09+eYcxgOLAScw2xizXUReBNYbYxYAs4A5IrIXOIk3TC5nhr9qtiHdFufptjhPt8V5ui3Ou+ptYbvusUoppQLLlk0BlVJKBY4GhVJKqRIFbVD4q/2HHZViWzwtIjtEZIuIfCsiDayoMxAuty2KLddfRIyIhOypkaXZFiIyyPe7sV1EPgp0jYFSin8j9UXkOxHZ6Pt30tuKOv1NRGaLyDER2XaJ90VE3vZtpy0i0rpUKzbGBN0fvJPf+4DrgQhgM5B4wTKPAdN8j4cA/7C6bgu3RTcg2vf40fK8LXzLVQRWAKuBJKvrtvD3ojGwEajie17T6rot3BYzgEd9jxOBX6yu20/bojPQGth2ifd7A1/hvVnDbcCa0qw3WI8o/NL+w6Yuuy2MMd8ZY3J8T1fjvWYlFJXm9wLgj3j7huUFsrgAK822eAh41xiTCWCMORbgGgOlNNvCAHG+x5WAwwGsL2CMMSvwnkF6Kf2Avxiv1UBlEalzufUGa1BcrP1H/KWWMca4gHPtP0JNabZFcWPwfmMIRZfdFr5D6XrGmH8GsjALlOb34ibgJhFZJSKrRaRnwKoLrNJsiz8AD4pIOrAI+I/AlBZ0rnR/AtikhYcqHRF5EEgCulhdixVExAFMBkZaXEqwCMM7/NQV71HmChFpZow5ZWlV1hgKfGCMeUNE2uO9fqupMcZjdWF2EKxHFNr+47zSbAtEpDvwO+AeY0x+gGoLtMtti4pAU2CZiPyCdwx2QYhOaJfm9yIdWGCMKTTGHAD24A2OUFOabTEGmAdgjPkRiMLbMLC8KdX+5ELBGhTa/uO8y24LEWkFTMcbEqE6Dg2X2RbGmNPGmOrGmIbGmIZ452vuMcZcdTO0IFaafyOf4z2aQESq4x2K2h/IIgOkNNsiFUgGEJEEvEFxPKBVBocFwAjf2U+3AaeNMb9e7oeCcujJ+K/9h+2Uclv8GYgFPvbN56caY+6xrGg/KeW2KBdKuS0WAz1EZAfgBn5jjAm5o+5SbosJwHsi8hTeie2RofjFUkT+jvfLQXXffMzzQDiAMWYa3vmZ3sBeIAcYVar1huC2UkopVYaCdehJKaVUkNCgUEopVSINCqWUUiXSoFBKKVUiDQqllFIl0qBQqpRExC0im4r9aSgiXUXktO/5ThF53rds8dd3icjrVtev1NUKyusolApSucaYlsVf8LW3/94Y01dEYoBNIrLQ9/a51ysAG0XkM2PMqsCWrNS10yMKpcqIMSYb2ADceMHrucAmStF8TalgpEGhVOlVKDbs9NmFb4pINbz9pbZf8HoVvD2WVgSmTKXKlg49KVV6/zb05NNJRDYCHuBPvvYRXX2vb8YbEm8ZY44EsFalyowGhVLX7ntjTN9LvS4ijYDVIjLPGLMp0MUpda106EkpP/O1+P4T8KzVtSh1NTQolAqMaUBn31lSStmKdo9VSilVIj2iUEopVSINCqWUUiXSoFBKKVUiDQqllFIl0qBQSilVIg0KpZRSJdKgUEopVaL/D1ACoCCyevHbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel('FPR')\n",
    "    plt.ylabel('TPR')\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9301289849397728"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "roc_auc_score(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/ensemble/forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/ensemble/forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/ensemble/forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_clf = RandomForestClassifier(random_state=42)\n",
    "y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3, method='predict_proba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_scores_forest = y_probas_forest[:, 1]\n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5, y_scores_forest\n",
    "                                                     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8FNXawPHfSQiEEoog4qUrKCQQAoTeRSmCIkUpAqGEKooKvopXLupVBMWKKEVQLoqIigpKU4EAQgJEQiABpIeglCC9pJ73j7MpIKSRzexsnu/nE9izMzv7ZJLMs+ecmWeU1hohhBDiZjysDkAIIYRrk0QhhBAiU5IohBBCZEoShRBCiExJohBCCJEpSRRCCCEy5bREoZSap5Q6qZTadZPlSin1gVJqv1IqUinVwFmxCCGEyD1n9ig+AzplsrwzUNPxNRz42ImxCCGEyCWnJQqt9Xrg70xW6Qb8TxuhQGml1J3OikcIIUTuFLLwvSsCRzO0Yx3P/XX9ikqp4ZheB8WLF29Yq1atfAkwPySnaJJSNMkpKaReJP+Pa+X19c/p9H/19c9e19bXLtX/WOHa565frv8ZTZbve6PXZfW+pv2Pb/QG8dzgNVnE88/iA/r63XKD7/Kfr8v4Pd2ooMH127z+NWmLcrjvbrTtHO87UWAlnTtJSvwlSEmO01rfnpttWJkosk1rPRuYDRAYGKi3bdtmcUQ3l5yi+ftSAnEX4zl90fxvvlKfy/g4gYTkFKtDzjGVRVu4LqXMz8tDKcdjx/8ZHnsoZX6mKuN6oJTCw7HArJfh9Y7lGbfloRzPpb5vhsceSjniSV0vYyyO99fmvQp5mtjPnVMU9YZixSA5WfHXn1C2rKKkD8THw+FDisqVoGRJuHxZEREB9QMUPj5wOg527FA0a2qW7/tDER0NnTopShSHgwdg2zbFI49AsaKKPXsgLBT69wfvIorff4fwcMWI4eDhAb9tVOzeDSNHmO/j11/gyBHF8GHme/nhB8XZMzBkiPmevl4Mly8phg41+2rB54qkRAgeapZ/9ikUL67o08fsm0VfQoU7of19pr1kCVSvpggMNK9fsVJRswbce68iJRm2boUqlRVVKkNSEpw4AeXKQvHiHqxY/D/ijp9m6f/eOZLr3xtn1npSSlUDftRa17nBslnAOq31l472XqCt1vofPYqMrEgUVxOTOX0pgbgL8WkH+FMX469JBqn//3054YafNm+mRJFClCtRmDLFC+Pl4XHjP1rHkTjbf7TX/FEqx4HhuufStpPxPf75R5v+B51+AEnb/nUHiuvju3Y75rl/fh83OGhkfH3G97xBfNcffP65r9Lfn4z7NO37uNG+utH3ce1BLX1f3ejgl4P3v8HPLMv3RzkO4lm8P9cuT93/2aU1/P033HabeX1cHJw8CbVrm/bu3bB9O/TrZ9Zftw6OHYPHHzftRYvgzBkYNQquXIE33jAH6fHjzfKRI+GOO+CVV0y7YUNo3x7efNO0vbzgxRfN8sREKFzYLHvuOYiNhcqVYc4cCA6G6Gho0QLmz4eHH4YdO6BjR5g927S3bIGpU+Gtt+CuuyAsDF5+GRYsgHLlYPNm+OYb+M9/oFQp0162zGz7rrvM67dsgdGjTaLYsQOOHDHbBvN9X74MNWua9qVLZv+VKGHa8fHm/yJFzP/JyY6fkxMG/48dO8aoUaPo3bs3j6f+MAClVLjWOjA327QyUXQBxgAPAk2AD7TWjbPaZl4kCq01F+KTiLsQf00CiLv4z57A6YsJXIhPyva2lYIyxQpTrkRhyhYvQjmfIpQrUZhyJdL/L5vhsbeX5y19L0JkV3Iy7NwJFSqYrzNnzMH8/vvNAW7BApg71xx8q1eHl14yB9eTJ+H22+G//zUH0qQk8PSEZ5+F6dPh8GGoWBGGDIGff4ajjgHl7t3hwAGIjDSv6dQJUlJgzRqzPCgI7rwTpkwx7fHjwd8fBg407XfegUaNoFUrc9D9+WezvEIFs53Ll8HbGwrZYlzE+bTWfPLJJ4wfP57ExETeffddRowYkbbcJROFUupLoC1QDjgBTAK8ALTWM5X5ePMh5syoy8BgrXWWGSA3ieJSfBKLth5l2Y4/OXn+KnGXEkhIyv6Qj5enchz0HQf61Mc3eO62YoUp5CmXp4jc0dp8Gk39JBoebg7UnTub9htvmGVPPmnaDz4IDRrAa6+Ztq+vWfftt027QgXzif+dd9K3m/qpfN8+uOce+Pxz0wsID4fAQNi1C/z8TPuzz+DVV6FMGYiKMl+9eplPwkeOmCGOwEDTPnPGfPIvX968d0qKcz4xi386cOAAw4YNY+3atbRr1445c+Zw9913X7POrSQKp+VirXXfLJZr4AlnvT/A6YvxzN90mPmbj3DuSuI1y4oX9rzmk33ZEkW4vURhyvk4DvolCjueK0LJooVy3G0XBYPWcPasGVIoVswc1DdsgKZNzafso0dh4ULo3RuqVYOICHjqKfj2W/Mp/auvzCf19evNEM+4cfDuu2aoonBh+PBD+PVXiIkx77dxI+zdm54oKlRIXwbwyCNQt256e9QoqF/fPPb2hu++gzqO/n316vDXXyYJgBn6yfi5sWFD85XKz898papa1XylSt1OKkkS+Wfnzp2Eh4cze/ZsgoOD8/54pbW21VfDhg11VmJOX9ITv9+p731pua76/I+66vM/6h4f/aZX7PxTx5y+pC/HJ2W5DVGwxcenP/7uO61/+CG9PXy41m++mb4eaP3aa6a9ZYtpL1pk2hs3mvbq1aa9erXWd9yh9cqVpr1ypdY9emj999/p6z/9tNZJjl/RI0e03r07/b2T5FdXOOzcuVPPnz8/rR0XF5fp+sA2ncvjrlPnKJwhs6Gn6D/PMzPkAD/t/IvkFPN9ta9VnpFt76ZRtdvyM0zhgpKTzdj6hQtmMrNBA/MpPjzcDNV88onpFfz73zBzJpw+bV7XoYMZ2//LcZpFjx5QqxZMnmyGV6ZPh2bNoHFj0xMIDYWAADMpmpycPhErn7BFXkhISGDy5MlMnjyZO+64g3379uHt7Z3l61xy6Ck/bY85w3u/7CPkj1MAFPJQ9KhfkRFt7ubeCj4WRyfyQ3y8GQK64w7TnjDBDP9062bOkKlZE+bNg759ISQEHnoIli834/kxMfDll+Y1deuag37hwunbnjPn2vaSJemPPTxg7Nj0dpEi0KZNetvT03wJkRfCwsIYOnQoUVFR9O/fn3fffTdbSeJW2T5RxJ65TO/ZoSQkpVDUy5M+jSsT3OouKpYuanVoIg/Fx5uJ08qVzZlln35qzqZ5913zqb1UKWjdGlavNuvPnWsO2t26mWUjR5rTHAFatjRJI3W8vXNnuHo1/dTFrl3NV6qM4/BCWOXYsWO0atWKO+64gx9//JEuXbrk23vbvjP8v81HSEhKoc09t7PphfuY9JCfJAmbOHPGfIEZ2hk5Ev74w7Q/+8wc2PfvN+2JE80B+8IF0961C774Iv1UzRkz0s9pB/jzT3OePICPj0koTZqYdunSMHgwVKli2t7e6UlCCFfzh+OPomLFinz11VdERUXla5IAmyeKS/FJfLnFnPIxrsM9lCleOItXCCstWQI//mgenz5t5gc++MC0k5Nh1iw4dMi0/f3NWUGpQz4DBpizg7y8TPuNN8wZRqnn0A8dCmPGpL+XnFsv7O7s2bMMHz6cWrVqsX79egC6d+9OyZIl8z0WW/85Lfk9lgtXkwisWgb/SqWtDqfAu3QJ1q41Y/Q+PjBtmrm6de1aM5b/1lumR9CxI5Qtay7eat7cvLZePTMxnHpWX4MGZsI5Vd261572WVg+Ewg3tnTpUkaNGsXx48d57rnnaNSokaXx2LZHkZKi+fS3wwAMaVnd2mAKkIsX08/bf/VVqFHDlD4A01t46CFT2gHgX/8y66aWL/j8c5M0UnsFr7wCDzxgHqeWsRCioAsODqZbt26ULVuWsLAwpk6dStGi1g6n27ZHEfLHKQ7GXaJi6aJ08L3D6nDc1po15gKySZNM28fH9AhWrDAlHK5cMc+Bef7LL82wEZgrglPrAAFcd6GoEMIh9TIFpRSBgYFUrVqV559/nsIu0nW2bY9i3m9mMHtgs6pSMiMPXb1qPunv2WPaP/1kagAlOi5s//xz6NPHfPovUcLMFXTsaJaVLm2W5cPZekK4jaNHj9K1a1c+//xzAEaOHMnEiRNdJkmATRPFHycusGFfnDkdtlEVq8OxtaQkUyZi9mzTTkw0Zwt9/rmZM/jPf0xlztThoscfh0GDrIpWCPeRkpLCxx9/jJ+fH+vWrSM+dYzWBdkyUfy4408AujeoSKliXhZHYz8zZ5qL0c6fN0NHa9akF5Xz8TGnqv73v2YCulQpmTgWIq/t27ePdu3aMXr0aJo0acKuXbsIDg62OqybsmWiOBB3CYDAqmWyWLPgSkhIn3S+fBnatTM9BzAT0GFh5hRVHx9ziurBg+mvrVNHJpaFcKbo6GgiIyOZN28eq1evpnp11z4hx5aT2YcdiaJaueIWR+Jazp2DokVND6BZM3Ow37bNzBlcvmx6D2DuP5CxxFelStbEK0RBsmPHDiIiIggKCqJbt24cPHiQMteX3HVRtuxRpCaK6mULdqJIvQMZmFshli5t/gczIT18ePo9AcLCzD0IhBD5Kz4+nokTJxIYGMjEiRO5evUqgG2SBNiwR5GUormUkEypol4F8krspCRz4PfwMD0DpcwVz76+MGyYSRZwba0iIYQ1Nm/ezNChQ9m9ezcDBw7knXfeyZcifnnNdj2K+KRkAKqVLWZxJPnvp59MIti3z7R79DB3F/P2huLFzZlLGW8sI4SwzrFjx2jTpg0XL15k+fLlzJ8/n7Jly1odVq7YrkeRegvTgjI/ceGCuWn9Qw+ZchcPPpg+v/CEU+8PKITIjd27d1O7dm0qVqzI4sWLad++PT4+9r7dgQ17FI5EUUDmJ8aPN1VRT540t5pcvNjcNEcI4VrOnDnDkCFD8PX1ZcOGDQA88sgjtk8SYNMeRSGguhv3KObPN3MPAwea+yu3bGkqqQohXNN3333H6NGjOXXqFBMmTLC8iF9es2+Pwo0SRVwcjBplTmEFc+e1CRPMGUt+fqbEtlzXIIRrGjJkCD169KBChQps2bKFyZMn23LCOjO261EkOhJF5TLuc3Oir782d2x75BFTN2n6dChXTu6xLISryljEr2nTptSsWZPx48fj5eWelSJsdyhKcfyAfLzt/QP57jv4v/+DU6dMb+LkyfTieuXLS5IQwlUdOXKEzp07s2DBAgCGDx/OhAkT3DZJgA0ThQa8PBWFC9kudCD9jKXSpc3tO0+dMm0LblolhMiBlJQUZsyYQZ06ddi4cSOJqSWVCwBbHm2LFbbdiBlgehEPPGCSRYsWptaSr6/VUQkhsrJ3717atGnDmDFjaN68Obt27WLo0KFWh5VvbJkoihf2tDqEHHFcsU/58uaaiBMnTD0mN5vvEsJt7d27l6ioKD777DNWrlxJtWrVrA4pX9nyo3mxIvYJ+4knTNnukBBo0sQU5nPjoUwh3Mb27duJiIhg8ODBPPzwwxw8eJDSqTVyChjpUThBcrK5ohrMENNtt5my34UKSZIQwtVdvXqVF198kUaNGvHyyy+nFfErqEkCbJooirpworh0CerXN/eK1tr8//33UKSI1ZEJIbLy22+/ERAQwBtvvMHAgQOJiIhwu2sicsM+YzgZFHfRyeyUFChWDF58EQ4dsjoaIUROHDt2jHbt2lGxYkVWrVpFhw4drA7JZbjmETcLrjhHMWsWrFhhSn736WN1NEKI7IqOjsbX15eKFSvy7bff0q5dO0qUKGF1WC7FlkNPrjhHcfvt5mymixetjkQIkR1///03gwYNws/Pj/Xr1wPw0EMPSZK4AVsmCle5jmLjRpg3z0xed+8OmzbJhXNC2MG3336Lr68vX3zxBf/+979p3Lix1SG5NNc44uZQMRfpUfzwgxlqqlsX3KxYpBBua9CgQcyfP58GDRqwcuVKAgICrA7J5dmyR2Fl+Y74eHj9dfP4zTdhyxZJEkK4Oq11WiG/5s2bM2XKFMLCwiRJZJNTj7hKqU5Kqb1Kqf1KqRdusLyKUmqtUmq7UipSKfVgdrbr6WFdze2NG+Gll8xFdEqBTe9sKESBcejQITp06MD//vc/wBTxe/755ylUyJYDKpZwWqJQSnkCM4DOgC/QVyl1fWWjl4DFWuv6QB/go+xs28pE0b49fPutGW4SQriu5ORkPvjgA+rUqUNoaGhaj0LknDN7FI2B/Vrrg1rrBGAR0O26dTSQOv1bCvgzOxsuZEGi+Ppr+PlncxFdjx75/vZCiBzYvXs3rVq1YuzYsbRp04aoqCgGDRpkdVi25cxEURE4mqEd63guo5eB/kqpWGA58OSNNqSUGq6U2qaU2gbgkc+3e0tOhp9+MtdHnD+fr28thMiF/fv3s3fvXhYsWMBPP/1ElSpVrA7J1qyezO4LfKa1rgQ8CCxQSv0jJq31bK11oNY6EKCQZ/4kiqQkc1aTp6e569zu3VCqVL68tRAih8LDw5k3bx5groc4dOgQ/fv3R8l9hG+ZMxPFMaByhnYlx3MZDQUWA2itNwPeQLmsNpxfPYoFC2DgQPj7b/DxMWXChRCu5cqVK7zwwgs0adKE//73v2lF/ErKRU15xpmJYitQUylVXSlVGDNZvfS6dWKA9gBKqdqYRHEqqw3n1xxFv37mdqVlyuTL2wkhcmj9+vXUq1ePqVOnMmjQILZv3y5F/JzAaeeHaa2TlFJjgFWAJzBPax2llHoV2Ka1XgqMA+YopZ7BTGwP0tk4NcHDyYkiIsLcN6JZM/jPf5z6VkKIXDp27Bjt27encuXK/PLLL7Rv397qkNyWstspY0XurKm//GktPRpUcsr2tYYaNczXypXmWgkhhOvYuXMndR3np//444+0a9eO4sWLWxyV61NKhafO8+aU1ZPZueLM6yiUgs2b4a23JEkI4Uri4uIYMGAA/v7+aUX8unbtKkkiH0iiyGDDBlP9tXx58Pd3ylsIIXJIa83ixYvx9fVl0aJFTJo0iSZNmlgdVoFiy2vYPZ3wUT8xEYYNg8qVzYV1QgjXEBQUxIIFCwgMDOTXX39NG3YS+ceeicIJPQovL5g7V4abhHAFqXOnSinatGmDv78/Tz/9tNRnsogt93peX3B37BjceSe0aJGnmxVC5MLBgwcZNmwY/fv3Z/DgwQwdOtTqkAo8W85R5OWVlklJ0K0bDB6cZ5sUQuRCcnIy7733HnXr1mXr1q14eNjy8OSWbNmjyMsrs5WCAQNASsEIYZ3o6GiGDBlCWFgYXbp0YebMmVSq5JxT4EXO2TRR5M12UlJMHafRo80chRDCGocOHeLAgQMsXLiQPn36SH0mF2PLvp3i1n+JUlKgdm145hlJEkJYYevWrcyZMweALl26cPDgQfr27StJwgXZMlHkRY/CwwO6dgU5HVuI/HX58mXGjx9P06ZNeeONN9KK+Pn4+FgcmbgZWw495dUnjsmToUiRPNmUECIb1q1bR3BwMAcOHGDEiBFMnTpVivjZgE0Txa29fto0kyBGjMibeIQQWYuNjeWBBx6gatWqrFmzhnbt2lkdksgmmw495T5TXL4M4eEQGipzE0Lkhx07dgBQqVIlfvjhByIjIyVJ2IwtE8Wt9CiKFYOFC+Gzz+QqbCGc6dSpU/Tr14+AgABCQkIAePDBBylWrJjFkYmcsmWiyO1k9mefwdq1JkFIb0II59Ba8+WXX+Lr68s333zDK6+8QrNmzawOS9wCm85R5C5TfP65OS1Wer1COM+AAQP44osvaNKkCXPnzsXPz8/qkMQtsmeiyOXrVqyA2Ng8DUUIAaSkpKCUQilFu3btaNiwIU899RSenp5WhybygE2HnnKWKk6eTJ+8rl7dSUEJUUDt37+f9u3b8+mnnwIwdOhQnnnmGUkSbqRAJIpPP4X+/c1NiYQQeSMpKYlp06ZRt25dtm/fTuHCha0OSTiJPYeecjj29PjjkJwMJUo4Jx4hCppdu3YxePBgtm3bRrdu3fjoo4/417/+ZXVYwkkKRKKoVAlefNE5sQhREMXExHDkyBEWLVrEY489JvWZ3Jwth55yUhRw4kTYtMmJwQhRQISFhTF79mzAXA9x8OBBevfuLUmiALBlosju/Uzi4mDmTPj1V+fGI4Q7u3TpEs8++yzNmjXjzTffJD4+HoASMpZbYNhy6Cm7k9nlypkznhzFKYUQObRmzRqGDRvGwYMHGTVqFFOmTKGIVNIscGyZKLKTJlJSzFyGUlC0qNNDEsLtxMbG0rFjR6pXr05ISAitW7e2OiRhEVsOPWVnTPT33+H222HdOufHI4Q72b59O2CK+C1btowdO3ZIkijgbJoosl7Hxwcefhhq1HB+PEK4gxMnTtC7d28aNGiQVsSvU6dOFJUueYFny6Gn7MxR3HsvzJuXD8EIYXNaa7744gvGjh3LxYsXee2112jevLnVYQkXYstEkVWa+OMPSEwEX18pJS5EVvr168eiRYto1qwZc+fOpXbt2laHJFyMWw49TZ8O9etDQkL+xCOE3aSkpKC1BqBDhw68//77bNiwQZKEuCFbJoqsvP46fPON3A9biBv5448/aNeuHfMcY7ODBw+WSq8iU26ZKEqWNBPZQoh0SUlJvPnmm9SrV4/IyEiZpBbZZss5isy8/jrccQcEB1sdiRCuIzIykiFDhhAeHk737t2ZMWMGd955p9VhCZuwZaLIrNbTL7+YiWxJFEKki42N5ejRo3z99df07NlT6jOJHFGpE1pO2bhSnYD3AU/gE631lBus8xjwMqCBHVrrfplts8idNfW+XTuoUvbmN2jXWs52EmLTpk1ERkYycuRIwNRsKl68uMVRCasopcK11oG5ea3T5iiUUp7ADKAz4Av0VUr5XrdOTWAC0EJr7Qc8nTfvnRdbEcKeLl68yNixY2nZsiVvv/12WhE/SRIit5w5md0Y2K+1Pqi1TgAWAd2uW2cYMENrfQZAa30yOxu+WSIYNw4GDMh1vELY3urVq6lTpw7Tp0/niSee4Pfff5cifuKWOTNRVASOZmjHOp7L6B7gHqXUb0qpUMdQ1T8opYYrpbYppbZl9oYlS8LRo5mtIYT7Onr0KF26dMHb25v169czffp0fHx8rA5LuAGrT48tBNQE2gJ9gTlKqdLXr6S1nq21DsxqfG3SJCkCKAqe8PBwACpXrszy5cuJiIigZcuWFkcl3IkzE8UxoHKGdiXHcxnFAku11ola60PAH5jEkWNJSbmKUQjbOn78OI8++iiBgYFpRfweeOABvL29LY5MuBtnJoqtQE2lVHWlVGGgD7D0unW+x/QmUEqVwwxFHczNm731FvzrX3DxYu4DFsIOtNbMnz8fX19fli1bxuTJk6WIn3Aqp11HobVOUkqNAVZhTo+dp7WOUkq9CmzTWi91LOuglIoGkoHntNanc/N+gYHw2GMgd2cU7q5Pnz4sXryYFi1a8Mknn1CrVi2rQxJuzqnXUThDkTtr6v1RO6h8282voxDC3aSkpKCUQinF/PnzuXDhAqNHj8YjuzeQFwWeS15HkZ9OnoTYWKujEMI59uzZQ+vWrZk7dy4AQUFBjBkzRpKEyDe2/E27/jqKuXOhcmU5NVa4l8TERCZPnky9evWIjo6mhIyrCovYstbT9YYMgXPnoFIlqyMRIm9EREQwePBgIiIi6NWrF9OnT6dChQpWhyUKKLdIFHfcAVP+UUVKCPs6fvw4x48f59tvv6VHjx5WhyMKOJsOPaWPPV26BCtXwqlTFgYkRB7YuHEjH330EQCdOnXiwIEDkiSES7Blosho/Xro3BkiIqyORIjcuXDhAmPGjKFVq1a89957aUX8ihWTM/uEa7B9omjZEsLCoEULqyMRIudWrVpFnTp1+Oijjxg7dqwU8RMuyfZzFD4+0Lix1VEIkXNHjx6la9eu1KhRg40bN8rV1cJl5bhHoZTyUEo97oxgsh2D43+tYfp02L7dymiEyD6tNVu2bAFMEb8VK1awfft2SRLCpd00USilSiqlJiilPlRKdVDGk5haTI/lX4g3FxcHTz0Fv/5qdSRCZO2vv/6iZ8+eNGnSJK2I3/333y9F/ITLy2zoaQFwBtgMBAMvYj7MP6K1domp49tvN9dPCOHKtNZ89tlnPPvss1y9epWpU6fSQibVhI1kliju0lrXBVBKfQL8BVTRWl/Nl8iyqUQJkEoGwpU99thjfPPNN7Rq1YpPPvmEe+65x+qQhMiRzBJFYuoDrXWyUirWVZJE6mUUM2aY6ydeftnScIT4h+TkZJRSeHh48NBDD3HfffcxYsQIqc8kbCmz39p6SqnzSqkLSqkLgH+G9vn8CjAzkZFmMlsIV7J7925atWqVVsRv4MCBjBo1SpKEsK2b/uZqrT211iW11j6Or0IZ2iXzM8ibmTXLTGgL4QoSExN57bXXCAgIYO/evZQqVcrqkITIEzcdelJKeQMjgRpAJObGQy5xw1FFegmP6yvJCmGF7du3M2jQICIjI+nduzcffPAB5cuXtzosIfJEZn3h+UAgsBN4EHg7XyLKgcGD4euvrY5CCDhx4gRxcXF8//33LFq0SJKEcCuZTWb7ZjjraS6wJX9Cyh6tYfNmqFPH6khEQbV+/Xp27tzJE088QadOndi/fz9Fixa1Oiwh8lxmPYqMZz25xJBTRkrBnj0wbpzVkYiC5vz584wePZo2bdrwwQcfpBXxkyQh3FVmiSLAcZbTeVc760nmJYRVli9fjp+fH7NmzeLZZ5+VIn6iQMgsUexwnOVU0hXPetqwAbp2hcOHrY5EFBRHjx6lW7dulCpVik2bNvH2229TvHhxq8MSwukySxQ636LIhf37zdBTSctTlnBnWmtCQ0MBU8Rv9erV/P777zRp0sTiyITIP0rrG+cDpVQs8M7NXqi1vukyZypyZ00ds3cnd5T05u+/4bbbrIhCFAR//vkno0aNYunSpaxbt442bdpYHZIQuaaUCtdaB+bmtZmd9eQJlABcdkZAkoRwBq01c+fOZfz48cTHxzNt2jQp4icKtMwSxV9a61fzLZIpklwcAAAanklEQVQcuu8+aNQIpk61OhLhbnr16sWSJUto06YNn3zyCTVq1LA6JCEslVmicNmeBMA998CFC1ZHIdxFxiJ+jzzyCB06dGDYsGFSn0kIMp+juE1r/Xc+x5OljHMUQuSFXbt2ERwczNChQxk2bJjV4QjhFLcyR5FZUUCXSxIZ/fmn1REIu0tISOCVV16hQYMGHDhwgDJlylgdkhAuyZb96jN/Q8uW8PHHVkci7Co8PJyGDRvy8ssv8+ijjxIdHU2vXr2sDksIl5TZHIXLSk6GwEAzTyFEbpw+fZqzZ8+ybNkyunbtanU4Qri0m85RuKoid9bUR/fupLzMUYgcWrt2LTt37uSpp54C4OrVq3h7y++RKBicMkfhyuLi4KpL3JRV2MG5c+cYMWIE9913Hx9//HFaET9JEkJkjy0TxcIvoWhRSHK5mrbC1SxbtgxfX18++eQTxo8fT3h4uBTxEyKHbDlHUbs2PPkkFLJl9CK/HD16lJ49e1KrVi2+//57GjVqZHVIQtiSzFEIt6K1ZvPmzTRv3hyAdevW0bx5cwoXLmxxZEJYy2XnKJRSnZRSe5VS+5VSL2SyXk+llFZKZeubiI3NuxiF+4iNjeXhhx+mRYsWhISEANC2bVtJEkLcIqclCqWUJzAD6Az4An2VUr43WM8HGAuEZXfb3btDly55Famwu5SUFGbNmoWvry+//vor77zzDi1btrQ6LCHchjNH+RsD+7XWBwGUUouAbkD0dev9F5gKPJfdDQcHg79cQyEcevbsyffff899993HnDlzuOuuu6wOSQi34syhp4rA0QztWMdzaZRSDYDKWuufMtuQUmq4UmqbUmobwIiR0K1bXocr7CQpKYmUlBTAJIo5c+bwyy+/SJIQwgksOz1WKeWBuTHSuKzW1VrP1loHpk7EXLxgrs4WBVNkZCTNmjVjzpw5APTv35/g4GCU3ExdCKdwZqI4BlTO0K7keC6VD1AHWKeUOgw0BZZmZ0K75j3w9dd5GKmwhfj4eCZNmkTDhg05cuQIt99+u9UhCVEgOHOOYitQUylVHZMg+gD9Uhdqrc8B5VLbSql1wHit9basNvzyJKhfP8/jFS5s69atDBo0iOjoaAYMGMC7775L2bJlrQ5LiALBaYlCa52klBoDrMLcVnWe1jpKKfUqsE1rvTS32x4xEsr75FWkwg7OnDnDxYsXWb58OZ07d7Y6HCEKFFtecLc1ZCd+d3vj6Wl1NMKZ1qxZw86dOxk7dixghp6k/IYQueOyF9w5S/0GcPKk1VEIZzl79izDhg2jffv2zJo1K62InyQJIaxhy0Tx5lSQm5G5px9++AFfX1/mzZvH//3f/0kRPyFcgC3L6g0cqJAK0e4nJiaGRx99lNq1a7N06VICA3PVSxZC5DFb9ihiYqyOQOQVrTUbNmwAoEqVKvzyyy9s3bpVkoQQLsSWiaJ9e6sjEHkhJiaGLl260Lp167Qifq1bt5YifkK4GFsmimnTrI5A3IqUlBQ++ugj/Pz8WL9+PR988IEU8RPChdlyjkLqPNlbjx49+OGHH3jggQeYPXs21apVszokIUQmbJkoYmLgdj+roxA5kZSUhIeHBx4eHvTu3Ztu3boxaNAgqc8khA3Ycuipb1+rIxA5sWPHDpo0acLs2bMB6Nu3L4MHD5YkIYRN2DJRvPaa1RGI7Lh69SovvfQSgYGBxMbGUqFCBatDEkLkgi2Hntq1szoCkZUtW7YQFBTEnj17CAoK4p133uG2226zOiwhRC7YMlEcOQK317E6CpGZ8+fPc+XKFVauXEnHjh2tDkcIcQtsWRSwXeNdrPxByjq4mtWrVxMVFcUzzzwDSBE/IVxJgSsK+MQTVkcgMjpz5gyDBw+mY8eOzJ07V4r4CeFmbJkomjW3OgKRasmSJfj6+rJgwQImTJjAtm3bJEEI4WZsO0dRTq6jsFxMTAx9+vShTp06LF++nPpy20Eh3JItexTz5lkdQcGltU6ry1SlShXWrFlDWFiYJAkh3JgtE0VQkNURFExHjhyhc+fOtG3bNi1ZtGzZEi8vL4sjE0I4ky0TxV13WR1BwZKSksKHH36In58fGzduZPr06bRq1crqsIQQ+cSWcxRHj0K52lZHUXA88sgjLFu2jI4dOzJr1iyqVq1qdUhCiHxkyx7FTz9aHYH7S0xMJCUlBTC1mebPn8+KFSskSQhRANkyUXTvbnUE7u3333+ncePGzJw5EzCJYuDAgVLET4gCypaJQmrLOceVK1eYMGECjRs35vjx41SuXNnqkIQQLsCWieLYn1ZH4H5CQ0MJCAhgypQpBAUFER0dzUMPPWR1WEIIF2DLyezQzeB/j9VRuJdLly6RmJjIzz//zP333291OEIIF2LLooDRW3dxdyUpE3GrVq5cSVRUFOPGjQMgISGBwoULWxyVEMIZClxRwNKlrY7A3k6fPk1QUBCdO3dm/vz5JCQkAEiSEELckC0TxV/HrY7AnrTWfPPNN/j6+rJw4UJeeukltm7dKglCCJEpW85RRGyHOjWsjsJ+YmJi6NevH/7+/qxevZp69epZHZIQwgZs2aNo08bqCOxDa82aNWsAqFq1KuvWrSM0NFSShBAi22yZKIoVszoCezh06BAdOnSgffv2aUX8mjdvTqFCtuxICiEsYstEcfas1RG4tuTkZN5//33q1KlDWFgYH3/8sRTxE0Lkmi0/Wh6JgbsrWR2F6+rWrRs//fQTDz74IDNnzpQrrIUQt8SW11Ecid5FhTJyHUVGiYmJeHp64uHhwVdffUVSUhL9+vWT+kxCCMCFr6NQSnVSSu1VSu1XSr1wg+XPKqWilVKRSqlflVLZKk0q98m51rZt2wgMDOTjjz8GoHfv3jz++OOSJIQQecJpiUIp5QnMADoDvkBfpZTvdattBwK11v7AN8Cb2dn2hQt5Gal9Xblyheeff54mTZpw6tQpKQEuhHAKZ/YoGgP7tdYHtdYJwCKgW8YVtNZrtdaXHc1QIFszD+fO5WmctrR582bq1avHm2++yZAhQ4iOjqZr165WhyWEcEPOnMyuCBzN0I4FmmSy/lBgxY0WKKWGA8MBCleowZ3/yqsQ7evKlSukpKTwyy+/0L59e6vDEUK4MZc460kp1R8IBG54KZ3WejYwG8xktqctT+q9dcuXLycqKornnnuO++67j927d+MlEzZCCCdz5iH3GJDxvMxKjueuoZS6H/g38LDWOj47G75yJU/is424uDj69+9Ply5d+OKLL9KK+EmSEELkB2cmiq1ATaVUdaVUYaAPsDTjCkqp+sAsTJI4md0NX76c9TruQGvNokWLqF27NosXL2bSpEls2bJFivgJIfKV04aetNZJSqkxwCrAE5intY5SSr0KbNNaLwXeAkoAXztO5YzRWj+c1bZvu61gnPYZExNDUFAQ9erVY+7cudStW9fqkIQQBZAtL7j7a38UtxV3z0/VWmt+/fXXtLvMhYaG0qhRIzw9PS2OTAhhZ7dywZ1LTGbnVEICUNzqKPLegQMHGDZsGGvXrmXdunW0adOGpk2bWh2WEIC5+j82NparV69aHYrIhLe3N5UqVcrTOUxbJoqUFKsjyFupRfxeeuklvLy8mDVrlhTxEy4nNjYWHx8fqlWrJlf9uyitNadPnyY2Npbq1avn2XZtmSiKuFmZp4ceeogVK1bQtWtXPv74YypVkoqHwvVcvXpVkoSLU0pRtmxZTp06lafbtWWicIff04SEBAoVKoSHhweDBg1iwIAB9OnTR/4IhUuT30/X54yfUQG9dM1aW7ZsoWHDhnz00UcAPPbYY/Tt21f+CIUQLkkSRT66fPky48aNo1mzZpw5c4a7777b6pCEsJ3XX38dPz8//P39CQgIICwsjKSkJF588UVq1qxJQEAAAQEBvP7662mv8fT0JCAgAD8/P+rVq8fbb79NirtNdjqRLYee7Gjjxo0EBQVx8OBBRowYwdSpUylVqpTVYQlhK5s3b+bHH3/k999/p0iRIsTFxZGQkMBLL73E8ePH2blzJ97e3ly4cIG333477XVFixYlIiICgJMnT9KvXz/Onz/PK6+8YtW3YivSo8gnqTcWWrt2LTNnzpQkIWyvbVv47DPzODHRtD//3LQvXzbtr74y7XPnTHvJEtOOizPtZctM+/jx7L3nX3/9Rbly5SjiOKOlXLlylC5dmjlz5jB9+nS8vb0B8PHx4eWXX77hNsqXL8/s2bP58MMPsdt1ZFaRROFEy5Yt4803zS022rVrR3R0NG3btrU2KCFsrEOHDhw9epR77rmH0aNHExISwv79+6lSpQo+Pj7Z3s5dd91FcnIyJ09mu3JQgSZDT05w6tQpxo4dy5dffklAQABPP/00hQsXplAh2d3Cfaxbl/7Yy+vadrFi17ZLlbq2Xa7cte0KFbL3niVKlCA8PJwNGzawdu1aevfuzYsvvnjNOp9++invv/8+p0+fZtOmTXLP+DwgPYo8pLVm4cKF1K5dm2+++YZXX32VsLAwKeInRB7y9PSkbdu2vPLKK3z44YcsW7aMmJgYLjhufTl48GAiIiIoVaoUycnJN9zGwYMH8fT0pHz58vkZum1JoshDMTExDB48mBo1arB9+3YmTpwoSUKIPLR371727duX1o6IiODee+9l6NChjBkzJq28SHJyclo5/uudOnWKkSNHMmbMGDklPZtkLOQWpaSk8PPPP9OxY0eqVq3Khg0baNiwoRTxE8IJLl68yJNPPsnZs2cpVKgQNWrUYPbs2ZQqVYqJEydSp04dfHx8KFq0KEFBQfzrX+Z2mFeuXCEgIIDExEQKFSrEgAEDePbZZy3+buxDqsfegn379jFs2DBCQkIICQmhdevWVockhNPs3r2b2rVrWx2GyIYb/axupXqsDD3lQlJSEm+99Rb+/v5EREQwd+5cKeInhHBbMvSUC127dmXVqlV069aNjz76KK17K4QQ7kgSRTbFx8fj5eWFh4cHwcHBDBkyhEcffVQmw4QQbk+GnrIhNDSUBg0aMGPGDAB69erFY489JklCCFEg2DJR5Nfh+dKlSzzzzDM0b96cCxcuULNmzXx6ZyGEcB0y9HQTGzZsICgoiEOHDjF69GjeeOMNSpYsaXVYQgiR72zZo8gPSUlJeHl5ERISwowZMyRJCOGGqlWrRlxcXJ5vt2/fvvj7+/Puu+/m+bYBDh8+zMKFC52y7RuRHkUG33//Pbt372bChAm0a9eOqKgoqc8khAvSWqO1xsPD9T7rHj9+nK1bt7J///5svyYpKSlHx5rURNGvX7/chJhjchQETpw4wZNPPsnXX39NgwYNGDdunBTxEyIT1V74ySnbPTyly82XHT5Mx44dadKkCeHh4SxfvpwpU6awdetWrly5Qq9evdLuL1GtWjWCgoJYtmwZiYmJfP3119SqVYvTp0/Tt29fjh07RrNmza4pM/7OO+8wb948AIKDg3n66ac5fPgwnTp1omnTpmzatIlGjRoxePBgJk2axMmTJ/niiy9o3LjxNXF26NCBY8eOERAQwPTp0/Hx8WHkyJFcvnyZu+++m3nz5lGmTBnatm1LQEAAGzdupG/fvgwcOJCRI0cSExMDwHvvvUeLFi0ICQlh7NixgLnN6fr163nhhRfYvXs3AQEBBAUF8cwzz+Tpz+F6rpeO85HWmgULFuDr68sPP/zA66+/TmhoqNRnEsJF7du3j9GjRxMVFUXVqlV5/fXX2bZtG5GRkYSEhBAZGZm2brly5fj9998ZNWoU06ZNA+CVV16hZcuWREVF0b1797SDcnh4OJ9++ilhYWGEhoYyZ84ctm/fDsD+/fsZN24ce/bsYc+ePSxcuJCNGzcybdo0Jk+e/I8Yly5dyt13301ERAStWrVi4MCBTJ06lcjISOrWrXvNzZISEhLYtm0b48aNY+zYsTzzzDNs3bqVb7/9luDgYACmTZvGjBkziIiIYMOGDRQtWpQpU6bQqlUrIiIinJ4koID3KGJiYggODiYwMJC5c+dSq1Ytq0MSwhYy++TvTFWrVqVp06Zp7cWLFzN79mySkpL466+/iI6Oxt/fH4AePXoA0LBhQ5Y47pi0fv36tMddunShTJkygLkDZffu3SlevHjaazds2MDDDz9M9erVqVu3LgB+fn60b98epRR169bl8OHDmcZ77tw5zp49S5s2bQAICgri0UcfTVveu3fvtMe//PIL0dHRae3z589z8eJFWrRowbPPPsvjjz9Ojx49qFSpUs533C0qcIkiJSWFVatW0blzZ6pWrcpvv/1G/fr1pYifEDaQeiAHOHToENOmTWPr1q2UKVOGQYMGpVWPBdLugufp6UlSUlKu3zN1OwAeHh5pbQ8Pj1vaLlz7/aSkpBAaGpp2l75UL7zwAl26dGH58uW0aNGCVatW3dJ75kaBGnr6448/aNu2LQ8++CAhISEABAYGSpIQwobOnz9P8eLFKVWqFCdOnGDFihVZvqZ169ZpZwutWLGCM2fOANCqVSu+//57Ll++zKVLl/juu+/ypH5bqVKlKFOmDBs2bABgwYIFab2L63Xo0IHp06entVPv8X3gwAHq1q3L888/T6NGjdizZw8+Pj5p99/IDwUiUSQlJTF16lT8/f3ZuXMnn376qVR6FcLm6tWrR/369alVqxb9+vWjRYsWWb5m0qRJrF+/Hj8/P5YsWUKVKlUAaNCgAYMGDaJx48Y0adKE4OBg6tevnydxzp8/n+eeey6tiOh//vOfG673wQcfsG3bNvz9/fH19WXmzJmAmdSuU6cO/v7+eHl50blzZ/z9/fH09KRevXpOOwU3I1uWGT++P4oyOSgz3rFjR1avXk2PHj2YMWMGFbJ730UhRBopM24feV1m3G3nKK5evYqXlxeenp4MHz6c4cOH07NnT6vDEkII23HLoafffvuNgICAtCJ+PXv2lCQhhBC55FaJ4uLFizz11FO0atWKq1evSjdZiDxmt6HqgsgZPyO3SRQhISHUqVOHDz/8kDFjxrBr1y4eeOABq8MSwm14e3tz+vRpSRYuTGvN6dOn/3GK7a1yqzmKYsWKsWHDhmyd/SCEyJlKlSoRGxvLqVOnrA5FZMLb2zvPL8qz9VlPS5YsYc+ePbz44osAJCcnyzURQghxA7dy1pNTh56UUp2UUnuVUvuVUi/cYHkRpdRXjuVhSqlq2dnuiRPH6dWrFz179uS7774jISEBQJKEEEI4gdMShVLKE5gBdAZ8gb5KKd/rVhsKnNFa1wDeBaZmtd3ky+dp2rAeP/74I2+88QabNm2SIn5CCOFEzuxRNAb2a60Paq0TgEVAt+vW6QbMdzz+BmivsrgRdfL5U9Su7ceOHTt44YUX8PLyyvPAhRBCpHPmZHZF4GiGdizQ5GbraK2TlFLngLLANbecUkoNB4Y7mvGhm3/bJZVeASjHdfuqAJN9kU72RTrZF+nuze0LbXHWk9Z6NjAbQCm1LbcTMu5G9kU62RfpZF+kk32RTim1LbevdebQ0zGgcoZ2JcdzN1xHKVUIKAWcdmJMQgghcsiZiWIrUFMpVV0pVRjoAyy9bp2lQJDjcS9gjbbb+bpCCOHmnDb05JhzGAOsAjyBeVrrKKXUq8A2rfVSYC6wQCm1H/gbk0yyMttZMduQ7It0si/Syb5IJ/siXa73he0uuBNCCJG/3KbWkxBCCOeQRCGEECJTLpsonFX+w46ysS+eVUpFK6UilVK/KqWqWhFnfshqX2RYr6dSSiul3PbUyOzsC6XUY47fjSil1ML8jjG/ZONvpIpSaq1Sarvj7+RBK+J0NqXUPKXUSaXUrpssV0qpDxz7KVIp1SBbG9Zau9wXZvL7AHAXUBjYAfhet85oYKbjcR/gK6vjtnBftAOKOR6PKsj7wrGeD7AeCAUCrY7bwt+LmsB2oIyjXd7quC3cF7OBUY7HvsBhq+N20r5oDTQAdt1k+YPACkABTYGw7GzXVXsUTin/YVNZ7gut9Vqt9WVHMxRzzYo7ys7vBcB/MXXDruZncPksO/tiGDBDa30GQGt9Mp9jzC/Z2RcaKOl4XAr4Mx/jyzda6/WYM0hvphvwP22EAqWVUndmtV1XTRQ3Kv9R8WbraK2TgNTyH+4mO/sio6GYTwzuKMt94ehKV9Za/5SfgVkgO78X9wD3KKV+U0qFKqU65Vt0+Ss7++JloL9SKhZYDjyZP6G5nJweTwCblPAQ2aOU6g8EAm2sjsUKSikP4B1gkMWhuIpCmOGntphe5nqlVF2t9VlLo7JGX+AzrfXbSqlmmOu36mitU6wOzA5ctUch5T/SZWdfoJS6H/g38LDWOj6fYstvWe0LH6AOsE4pdRgzBrvUTSe0s/N7EQss1Vonaq0PAX9gEoe7yc6+GAosBtBabwa8MQUDC5psHU+u56qJQsp/pMtyXyil6gOzMEnCXcehIYt9obU+p7Uup7WuprWuhpmveVhrnetiaC4sO38j32N6EyilymGGog7mZ5D5JDv7IgZoD6CUqo1JFAXxnq5LgYGOs5+aAue01n9l9SKXHHrSziv/YTvZ3BdvASWArx3z+TFa64ctC9pJsrkvCoRs7otVQAelVDSQDDyntXa7Xnc298U4YI5S6hnMxPYgd/xgqZT6EvPhoJxjPmYS4AWgtZ6JmZ95ENgPXAYGZ2u7brivhBBC5CFXHXoSQgjhIiRRCCGEyJQkCiGEEJmSRCGEECJTkiiEEEJkShKFENmklEpWSkVk+KqmlGqrlDrnaO9WSk1yrJvx+T1KqWlWxy9EbrnkdRRCuKgrWuuAjE84yttv0Fp3VUoVByKUUssci1OfLwpsV0p9p7X+LX9DFuLWSY9CiDyitb4EhAM1rnv+ChBBNoqvCeGKJFEIkX1FMww7fXf9QqVUWUx9qajrni+DqbG0Pn/CFCJvydCTENn3j6Enh1ZKqe1ACjDFUT6ireP5HZgk8Z7W+ng+xipEnpFEIcSt26C17nqz55VS1YFQpdRirXVEfgcnxK2SoSchnMxR4nsK8LzVsQiRG5IohMgfM4HWjrOkhLAVqR4rhBAiU9KjEEIIkSlJFEIIITIliUIIIUSmJFEIIYTIlCQKIYQQmZJEIYQQIlOSKIQQQmTq/wFNM3Tp/GznKwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fpr, tpr, \"b:\", label=\"SGD\")\n",
    "plot_roc_curve(fpr_forest, tpr_forest, \"random forest\")\n",
    "plt.legend(loc='lower right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9929919688888965"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_train_5, y_scores_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### multiclass classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([9], dtype=int8)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.fit(X_train, y_train)\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-637160.37097315, -503086.76724873, -822528.33253539,\n",
       "        -219210.47715032, -197099.78962804, -334529.73985706,\n",
       "        -846656.25348322, -412427.98797736, -262339.64229933,\n",
       "        -178713.34385808]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_digit_scores = sgd_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "[0 1 2 3 4 5 6 7 8 9]\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "print(np.argmax(some_digit_scores))\n",
    "print(sgd_clf.classes_)\n",
    "print(sgd_clf.classes_[9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/ensemble/forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([9], dtype=int8)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_clf.fit(X_train, y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0. , 0. , 0. , 0.1, 0. , 0. , 0. , 0. , 0. , 0.9]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.8719756 , 0.83109155, 0.87953193])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring='accuracy', n_jobs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.90921816, 0.9100955 , 0.91158674])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 입력 스케일 조정\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))\n",
    "cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring='accuracy', n_jobs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 에러 분석"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n",
      "/csehome/insung151/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[5739,    3,   22,   11,   10,   49,   41,    9,   36,    3],\n",
       "       [   2, 6493,   40,   28,    6,   48,    6,    9,   99,   11],\n",
       "       [  60,   44, 5306,  109,   84,   23,   97,   50,  169,   16],\n",
       "       [  50,   42,  139, 5330,    3,  238,   36,   56,  139,   98],\n",
       "       [  19,   26,   36,    9, 5388,   10,   59,   25,   84,  186],\n",
       "       [  70,   41,   29,  184,   71, 4632,  109,   32,  164,   89],\n",
       "       [  33,   22,   48,    1,   44,   96, 5623,    6,   45,    0],\n",
       "       [  27,   25,   65,   30,   53,   12,    5, 5799,   17,  232],\n",
       "       [  52,  158,   67,  147,   13,  162,   62,   28, 5040,  122],\n",
       "       [  47,   33,   28,   84,  174,   31,    2,  197,   85, 5268]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = cross_val_predict(sgd_clf, X_train_scaled, y_train, cv=3)\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAACvZJREFUeJzt3c2L3fUVx/HPJzMZncRSDe3GTGyCFIuoJToUNZCFcdFWiZsurCjUTTatRhFEu/EfENFFEYZYNwZdxCxKLNaCuqiL0MkDjJmxImqTaMSUUBUhzNPp4l7BmnTub+o99zfX836BkIw3Xw6Xeec+5HfPOCIEoJZ1bQ8AYPAIHyiI8IGCCB8oiPCBgggfKKi18G3/3PY/bL9n+7G25mjK9hbbb9ietX3C9t62Z2rC9ojtY7YPtT1LE7Yvt33A9ju252zf0vZMvdh+uPs98bbtF21f2vZMvbQSvu0RSX+Q9AtJ10r6te1r25hlFRYlPRIR10q6WdJvh2BmSdoraa7tIVbhGUmvRsRPJP1Ua3x225slPShpMiKukzQi6e52p+qtrUf8n0l6LyLej4h5SS9JuqulWRqJiDMRcbT76y/U+Ybc3O5UK7M9IekOSfvanqUJ29+XtFPSc5IUEfMR8e92p2pkVNK47VFJGyR93PI8PbUV/mZJp772+9Na4xF9ne2tkrZLOtzuJD09LelRScttD9LQNklnJT3ffXmyz/bGtodaSUR8JOlJSSclnZH0WUS81u5UvfHm3irZvkzSy5IeiojP257nf7F9p6RPI+JI27OswqikGyU9GxHbJX0paU2//2P7CnWerW6TdKWkjbbvbXeq3toK/yNJW772+4nu19Y02+vViX5/RBxse54edkjabftDdV5K3Wb7hXZH6um0pNMR8dUzqQPq/EWwlt0u6YOIOBsRC5IOSrq15Zl6aiv8v0v6se1ttsfUeTPkTy3N0ohtq/Pacy4inmp7nl4i4vGImIiIrercv69HxJp+JIqITySdsn1N90u7JM22OFITJyXdbHtD93tkl9b4G5JS56nVwEXEou3fSfqLOu+C/jEiTrQxyyrskHSfpBnbx7tf+31E/LnFmb6LHpC0v/uA8L6k+1ueZ0URcdj2AUlH1fmXn2OSptqdqjfzsVygHt7cAwoifKAgwgcKInygIMIHCmo9fNt72p5hNYZtXomZB2HY5m09fElDdYdp+OaVmHkQhmretRA+gAFLuYBn06ZNMTEx0ei2586d06ZNmxrddmZm5tuMBZQQEe51m5RLdicmJvTKK6/0/dyrrrqq72fiQp1LzodL1hWomfdFm1fN8lQfKIjwgYIIHyiI8IGCCB8oqFH4w7YDH8DKeoY/pDvwAaygySP+0O3AB7CyJuEP9Q58ABfq25t7tvfYnrY9fe7cuX4dCyBBk/Ab7cCPiKmImIyIyabX3gNoR5Pwh24HPoCV9fyQzpDuwAewgkafzuv+0Ah+cATwHcGVe0BBhA8URPhAQYQPFET4QEEpyzZtpywTy9xRtm7d8P0dOGx75obxJzOPjIyknb20tJRybpNlm8P33Q7gWyN8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKCgRj808/+Rsa46cwX28ePHU8696aabUs6V8tZVZ619zlxVvby8nHLuMK7XboJHfKAgwgcKInygIMIHCiJ8oCDCBwoifKCgnuHb3mL7Dduztk/Y3juIwQDkaXIBz6KkRyLiqO3vSTpi+68RMZs8G4AkPR/xI+JMRBzt/voLSXOSNmcPBiDPql7j294qabukwxnDABiMxtfq275M0suSHoqIzy/y//dI2tPH2QAkaRS+7fXqRL8/Ig5e7DYRMSVpqnv7nE+PAOiLJu/qW9JzkuYi4qn8kQBka/Iaf4ek+yTdZvt4979fJs8FIFHPp/oR8TdJHsAsAAaEK/eAgggfKIjwgYIIHyiI8IGCnLGp1XZ0/vl/eIyO5iwcPnLkSMq5knTDDTeknDs+Pp5y7vnz51POlaSs77dh2wy8tLSkiOh5Z/CIDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQWnrtft+qPJWKGfKuH+/MjMzk3Lu9ddfn3LuunV5jzMZq6ql3PXaGSvd5+fntby8zHptABcifKAgwgcKInygIMIHCiJ8oCDCBwpqHL7tEdvHbB/KHAhAvtU84u+VNJc1CIDBaRS+7QlJd0jalzsOgEFo+oj/tKRHJeVcFwlgoHqGb/tOSZ9GxJEet9tje9r2dN+mA5CiySP+Dkm7bX8o6SVJt9l+4Zs3ioipiJiMiMk+zwigz3qGHxGPR8RERGyVdLek1yPi3vTJAKTh3/GBglb1geCIeFPSmymTABgYHvGBgggfKIjwgYIIHyiI8IGC0rbsZmxUzdxYm2VsbCzt7IWFhZRzDx3K+QDm7t27U86VpKWlpZRz169fn3KulDPz0tKSIoItuwAuRPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFJS2Zdfuuehz1TK37GbMKw3nzBkbkiXp3XffTTlXkq6++uqUc7PuYynve4MtuwAuivCBgggfKIjwgYIIHyiI8IGCCB8oqFH4ti+3fcD2O7bnbN+SPRiAPKMNb/eMpFcj4le2xyRtSJwJQLKe4dv+vqSdkn4jSRExL2k+dywAmZo81d8m6ayk520fs73P9sbkuQAkahL+qKQbJT0bEdslfSnpsW/eyPYe29O2p/s8I4A+axL+aUmnI+Jw9/cH1PmL4L9ExFRETEbEZD8HBNB/PcOPiE8knbJ9TfdLuyTNpk4FIFXTd/UfkLS/+47++5LuzxsJQLZG4UfEcUk8hQe+I7hyDyiI8IGCCB8oiPCBgggfKIjwgYJYr92VtVJ6GNdrLy8vp5yb6dSpUynnbtmyJeVcSRofH+/7mefPn9fy8jLrtQFciPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKChty27G1trR0aY/3Hf1FhcXU84dGRlJOVeSFhYWUs4dGxtLOTfrPpbythm/9dZbKedK0s6dO/t+5uLioiKCLbsALkT4QEGEDxRE+EBBhA8URPhAQYQPFNQofNsP2z5h+23bL9q+NHswAHl6hm97s6QHJU1GxHWSRiTdnT0YgDxNn+qPShq3PSppg6SP80YCkK1n+BHxkaQnJZ2UdEbSZxHxWvZgAPI0eap/haS7JG2TdKWkjbbvvcjt9tietj3d/zEB9FOTp/q3S/ogIs5GxIKkg5Ju/eaNImIqIiYjYrLfQwLorybhn5R0s+0Nti1pl6S53LEAZGryGv+wpAOSjkqa6f6ZqeS5ACRq9AH3iHhC0hPJswAYEK7cAwoifKAgwgcKInygIMIHCiJ8oKC09dqda32QsWb8K1mru7PWdmetwJakSy65JOXczJXg09P9v7r9nnvu0ezsLOu1AVyI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oKGvL7llJ/2x48x9I+lffh8gzbPNKzDwIa2XeH0XED3vdKCX81bA9HRGTrQ6xCsM2r8TMgzBs8/JUHyiI8IGC1kL4U20PsErDNq/EzIMwVPO2/hofwOCthUd8AANG+EBBhA8URPhAQYQPFPQfyGO3VXJ1dbgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(conf_mx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_sums = conf_mx.sum(axis=1, keepdims=True)\n",
    "norm_conf_mx = conf_mx / row_sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADD9JREFUeJzt3UuInfUZx/HfL3PJmDEkgdSFudQsSkoMiDJIrBAkdtGL6KaLiBZahYFgvRHQtBs34kqqLjQYtQUvWMIoWKT0gm0X3YROYsBOEpOSxDExkhS8TBriZGaeLuYErLU575H3f945Pt8PCJnxzcPDZL7znnPmnXccEQKQy6KmFwDQfYQPJET4QEKEDyRE+EBChA8k1Fj4tr9n+13b/7S9o6k9qrK9xvZfbB+wPWH7/qZ3qsJ2n+23bb/Z9C5V2F5ue8z2IdsHbd/Q9E7t2H6w9TnxD9uv2h5qeqd2Ggnfdp+kpyV9X9IGSbfb3tDELh2YkbQ9IjZI2iTpnh7YWZLul3Sw6SU68JSk30fEtyVdowW+u+1Vku6TNBIRGyX1Sdra7FbtNXXGv17SPyPiaERMS/qNpNsa2qWSiDgVEftaf57S/Cfkqma3ujTbqyX9UNLzTe9She1lkjZLekGSImI6Ij5udqtK+iVdZrtf0hJJHzS8T1tNhb9K0vufe/uEFnhEn2f7KknXStrT7CZtPSnpIUlzTS9S0TpJZyT9uvX05Hnbw00vdSkRcVLS45ImJZ2S9ElE/LHZrdrjxb0O2b5c0muSHoiIT5ve5/+xfYuk0xGxt+ldOtAv6TpJOyPiWkn/lrSgX/+xvULzj1bXSbpS0rDtO5vdqr2mwj8pac3n3l7det+CZntA89G/EhGvN71PGzdKutX2cc0/ldpi++VmV2rrhKQTEXHxkdSY5r8QLGTflXQsIs5ExAVJr0v6TsM7tdVU+H+X9C3b62wPav7FkN82tEsltq35554HI+KXTe/TTkT8PCJWR8RVmv/4/jkiFvSZKCI+lPS+7fWtd90s6UCDK1UxKWmT7SWtz5GbtcBfkJTmH1p1XUTM2P6ZpD9o/lXQX0XERBO7dOBGST+W9I7t/a33/SIiftfgTl9H90p6pXVCOCrppw3vc0kRscf2mKR9mv/Oz9uSdjW7VXvmx3KBfHhxD0iI8IGECB9IiPCBhAgfSKjx8G2PNr1DJ3ptX4mdu6HX9m08fEk99QFT7+0rsXM39NS+CyF8AF1W5AIe2z13VVB/f7WLGOfm5rRoUfWvl3Nzzf9gXERo/mrSaoaGytxHopOPxezsrPr6+iofPz09/VVWamtwcLDScTMzM5U/hy46f/78V1mprYho+4/dyCW7X1UnnwidWrlyZZG5Z8+eLTJXmg+6hKuvvrrI3HPnzhWZK0lHjx4tMnft2rVF5krSkSNHap85Oztb6Tge6gMJET6QEOEDCRE+kBDhAwlVCr/X7oEP4NLaht+j98AHcAlVzvg9dw98AJdWJfyevgc+gP9V25V7rZ9O6qkfVACyqhJ+pXvgR8Qute4u2ovX6gOZVHmo33P3wAdwaW3P+D16D3wAl1DpOX7rl0bwiyOArwmu3AMSInwgIcIHEiJ8ICHCBxIqds+9Tm7uWFXV+4l9FcuWLSsyt+R9Ak+fPl1kbqmbQL777rtF5krlPs6bNm0qMleSJicna59Z9d+OMz6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkVub320qVLNTIyUvvcjz76qPaZF+3fv7/I3B07dhSZK0knTpwoMvett94qMnd0dLTIXEkaHx8vMveOO+4oMleS3njjjdpnfvbZZ5WO44wPJET4QEKEDyRE+EBChA8kRPhAQoQPJNQ2fNtrbP/F9gHbE7bv78ZiAMqpcgHPjKTtEbHP9lJJe23/KSIOFN4NQCFtz/gRcSoi9rX+PCXpoKRVpRcDUE5Hz/FtXyXpWkl7SiwDoDsqX6tv+3JJr0l6ICI+/ZL/PyppVJIWL15c24IA6lfpjG97QPPRvxIRr3/ZMRGxKyJGImJkcHCwzh0B1KzKq/qW9IKkgxHxy/IrASityhn/Rkk/lrTF9v7Wfz8ovBeAgto+x4+Iv0lyF3YB0CVcuQckRPhAQoQPJET4QEKEDyRU5C67s7Ozmpqaqn3uzMxM7TMvevrpp4vMveeee4rMlaS+vr4ic0t9nNevX19kriRdccUVReYePny4yFxJ2rp1a+0zx8bGKh3HGR9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQcEbUPHRgYiBUrVtQ+d2hoqPaZFw0PDxeZOzk5WWSuJJ07d67I3FIfi+uvv77IXEk6cuRIkblbtmwpMleStm/fXvvM22+/XRMTE21/yS1nfCAhwgcSInwgIcIHEiJ8ICHCBxIifCChyuHb7rP9tu03Sy4EoLxOzvj3SzpYahEA3VMpfNurJf1Q0vNl1wHQDVXP+E9KekjSXMFdAHRJ2/Bt3yLpdETsbXPcqO1x2+Nzc3x9ABayKmf8GyXdavu4pN9I2mL75S8eFBG7ImIkIkYWLeKbBcBC1rbQiPh5RKyOiKskbZX054i4s/hmAIrh1Awk1N/JwRHxV0l/LbIJgK7hjA8kRPhAQoQPJET4QEKEDyTU0av6VS1dulQ33XRT7XNL3UlVkqamporMnZiYKDJXkh5++OEic3fv3l1kbsk71p48ebLI3G3bthWZK0nPPfdc7TPPnDlT6TjO+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQo6I2ocODQ3FmjVrap979uzZ2mdetHr16iJz9+7dW2SuJF1zzTVF5t5www1F5u7cubPIXEkaHBwsMnfjxo1F5krSvn37isyNCLc7hjM+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kFCl8G0vtz1m+5Dtg7bLfKMXQFdU/TXZT0n6fUT8yPagpCUFdwJQWNvwbS+TtFnSTyQpIqYlTZddC0BJVR7qr5N0RtKvbb9t+3nbw4X3AlBQlfD7JV0naWdEXCvp35J2fPEg26O2x22Pz87O1rwmgDpVCf+EpBMRsaf19pjmvxD8l4jYFREjETHS19dX544AatY2/Ij4UNL7tte33nWzpANFtwJQVNVX9e+V9ErrFf2jkn5abiUApVUKPyL2SxopvAuALuHKPSAhwgcSInwgIcIHEiJ8ICHCBxKq+n38jszNzen8+fO1z120qNzXqeXLlxeZu3bt2iJzJWnx4sVF5r700ktF5g4MDBSZK0nT02V+buzYsWNF5krS1NRU7TM3b95c6TjO+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQkXusjs4OFjk7rI7duyofeZFzz77bJG5jz32WJG5krRt27Yic3fv3l1k7hNPPFFkriQdPny4yNz33nuvyFxJeuaZZ2qfefr06UrHccYHEiJ8ICHCBxIifCAhwgcSInwgIcIHEqoUvu0HbU/Y/oftV20PlV4MQDltw7e9StJ9kkYiYqOkPklbSy8GoJyqD/X7JV1mu1/SEkkflFsJQGltw4+Ik5IelzQp6ZSkTyLij6UXA1BOlYf6KyTdJmmdpCslDdu+80uOG7U9bnv8woUL9W8KoDZVHup/V9KxiDgTERckvS7pO188KCJ2RcRIRIwMDAzUvSeAGlUJf1LSJttLbFvSzZIOll0LQElVnuPvkTQmaZ+kd1p/Z1fhvQAUVOnn8SPiEUmPFN4FQJdw5R6QEOEDCRE+kBDhAwkRPpAQ4QMJOSJqHzo8PBwbNmyofe7HH39c+8yLSl1mvHnz5iJzJenuu+8uMveuu+4qMvf48eNF5krSoUOHisx99NFHi8yVpBdffLHI3Ihwu2M44wMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRW5y67tM5Leq3j4Skn/qn2JcnptX4mdu2Gh7PvNiPhGu4OKhN8J2+MRMdLoEh3otX0ldu6GXtuXh/pAQoQPJLQQwt/V9AId6rV9JXbuhp7at/Hn+AC6byGc8QF0GeEDCRE+kBDhAwkRPpDQfwAqodgEaZY8/wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.0005065 , 0.00371433, 0.00185717, 0.00168833,\n",
       "        0.00827283, 0.00692217, 0.0015195 , 0.006078  , 0.0005065 ],\n",
       "       [0.00029665, 0.        , 0.00593296, 0.00415307, 0.00088994,\n",
       "        0.00711955, 0.00088994, 0.00133492, 0.01468407, 0.00163156],\n",
       "       [0.01007049, 0.00738503, 0.        , 0.01829473, 0.01409869,\n",
       "        0.00386036, 0.01628063, 0.00839208, 0.02836522, 0.00268546],\n",
       "       [0.00815528, 0.00685043, 0.02267167, 0.        , 0.00048932,\n",
       "        0.03881912, 0.0058718 , 0.00913391, 0.02267167, 0.01598434],\n",
       "       [0.00325231, 0.00445053, 0.00616227, 0.00154057, 0.        ,\n",
       "        0.00171174, 0.01009928, 0.00427936, 0.01437864, 0.03183841],\n",
       "       [0.01291275, 0.00756318, 0.00534957, 0.03394208, 0.01309721,\n",
       "        0.        , 0.02010699, 0.00590297, 0.03025272, 0.01641764],\n",
       "       [0.00557621, 0.00371747, 0.00811085, 0.00016898, 0.00743494,\n",
       "        0.0162217 , 0.        , 0.00101386, 0.00760392, 0.        ],\n",
       "       [0.00430966, 0.00399042, 0.0103751 , 0.00478851, 0.0084597 ,\n",
       "        0.0019154 , 0.00079808, 0.        , 0.00271349, 0.03703113],\n",
       "       [0.00888737, 0.02700393, 0.01145103, 0.02512391, 0.00222184,\n",
       "        0.02768757, 0.01059648, 0.00478551, 0.        , 0.02085114],\n",
       "       [0.00790049, 0.00554715, 0.00470667, 0.01412002, 0.02924861,\n",
       "        0.00521096, 0.00033619, 0.03311481, 0.01428812, 0.        ]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.fill_diagonal(norm_conf_mx, 0)\n",
    "plt.matshow(norm_conf_mx, cmap=plt.cm.gray)\n",
    "plt.show()\n",
    "norm_conf_mx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### multilabel classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=None, n_neighbors=5, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "y_train_large = (y_train >= 7)\n",
    "y_train_odd = (y_train % 2 == 1)\n",
    "y_multilabel = np.c_[y_train_large, y_train_odd]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train, y_multilabel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ True,  True]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### multioutput multiclass classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise = np.random.randint(0, 100, (len(X_train), 784))\n",
    "X_train_mod = X_train + noise\n",
    "noise = np.random.randint(0, 100, (len(X_test), 784))\n",
    "X_test_mod = X_test + noise\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAC4CAYAAAD61bdSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAEhpJREFUeJzt3Uls1tUXxvEDZaaWMlqgk0CZkUpFAUvBgGBciUQXGuPKxLgy0Y2KLkRFXWB0YaJCYiRxo8aEYMRgmlLQQmRQWsokIPM8lalAqf/tPznPkbfSvnrp97N8ct721+nwhnvPvV3++usvAwCkp+u//QAAgH+GBg4AiaKBA0CiaOAAkCgaOAAkigYOAImigQNAorpl85OtWbPGbTofNWqUrG1paXFZQUGBrP3tt99c1qdPH1mbm5vrsu7du8vamzdvZlx79uxZl/Xo0UPWdu3q/908fPiwy4YMGSJfr762bt30j1J9XPX5zcyam5tdFn3P1fzA0aNHZW1ZWZnLou/jnj17XDZz5kxZu2nTJpdNnz69iyzueAxUoKO5323egQNAomjgAJAoGjgAJCqr/wc+bdo0l23cuFHWzp0712UNDQ2yduzYsS7LycmRter/pRsbG2Xt0KFDXbZv3z5Ze//997tM/R+tmf4/YfV/8zt27JCv79evn8smTpwoa9XHHTRokKzdvn27y4YNGyZrlWjd4a677nLZzp07Za36P3e1vmBmdu3atYyfDbgT8Q4cABJFAweARNHAASBRNHAASBQNHAAS1SWbN/JUV1e7TzZw4EBZe/nyZZepyUgzs+nTp7tMTRWa6R0NFy5ckLVHjhxx2fDhw2WtonaLmJnt37/fZSUlJS47dOiQfP3gwYNddvz4cVlbXFzsMjXlaqZ3p7S2tspa9bPo0kUPQfbt21fmitoRNGHCBFm7e/dul5WXlzOJiTsVk5gAcKeggQNAomjgAJAoGjgAJCqri5itra3uk/3555+y9uTJky7Lz8+XtepoUrUIaqYXTdWioJnZtm3bXDZgwABZW1pa6rKamhpZO2nSJJep0fJo0VYdHVtUVCRre/fu7bLomNvTp0+7LC8vT9aqMf9oEbN///4uu3LliqxVR91OnjxZ1qoF6YKCAhYxcadiERMA7hQ0cABIFA0cABJFAweARNHAASBRWb3QQe04iXZPnD9/3mXHjh2Tteoyhai2LZcHq90tBw8elLW9evVyWXQZQn19vcvashtIjZZHu25OnDiR0evN9OUa0XNVVVW5rK6uTtaq3Snqkgcz/bzRRR6FhYUyBzoL3oEDQKJo4ACQKBo4ACSKBg4AicrqKP369evdJ1Nj4WZ6UfDUqVOyVi2IXbx4UdaWl5dn/HHVWHc0Wr53716XRbe/X79+3WXqazhz5ox8/Y0bN1w2fvx4WatG9NV4vZnZRx995LLoXPXKykqXTZs2Tdaq4wei88vVcQnRee2qdsyYMYzSw5k6darL+vTpI2uXLl3qsoqKinZ/pn+AUXoAuFPQwAEgUTRwAEgUDRwAEkUDB4BEZXWUfsaMGS5TOyrM9Ch8dLu5GruPLi3YtWuXy6JLGpRoB4fagVFbWytr1bOpSwuiYwbOnTuXUWamb5WPdsfs37/fZeqSBzOzUaNGuSy6gEIdXxAdM6B+lmo3UPQM6NyiHWXq9/jAgQOydtmyZS77j+xCcXgHDgCJooEDQKJo4ACQKBo4ACQqq4uYR44ccVljY6OsVQt40QKFGvdWZ3mbmY0bN85lLS0tslaNt69atUrWfvLJJy6LFiG/+uorl61evdpl0bnd6kb3aKGvrKzMZd9//72sVWdxX716VdaqxedokVkda7Bv3z5Zq852j36W6ucePQNu7e2335b5G2+84bJFixbJ2sWLF7frM7XV5s2bZa4WLLN5jEhH4R04ACSKBg4AiaKBA0CiaOAAkCgaOAAkKqu7UHbv3u0ydTGAmdnRo0ddpnYomOnLFJqammStWo0uKCiQtVu3bnXZBx98IGvV6H80Wq4uKBgyZIjLBg4cKF+vdvNERweoG+G/+eYbWbtjxw6XvfLKK7L22WefddmYMWNkrdrlU1paKmvV2P3QoUNl7aVLl1zGLpTMqB1dy5cvl7Xqd+jdd9+VtdnchaJ+X5977jlZq76GqqoqWfv888/f3oNlEe/AASBRNHAASBQNHAASRQMHgERldRHzoYcecll0O3n//v1dVldXJ2vnzJnjspMnT8raa9euuSy6nVp9vrNnz8patdDWq1cvWatG1idNmuSy+vp6+frc3NyMMjOzWbNmuSwnJ0fWqnPR1TnlZvrc7uioA3Vud/S9UT839bnMzO677z6Z49bUeHx0PrYaOY8WADtC9FyzZ892WfR3rxYxn376aVk7ZcqUzB/uX8Y7cABIFA0cABJFAweARNHAASBRNHAASFRWd6HU1NS4LLrtWV1aoC5jMNOXA/zxxx+ydsSIES6LdnB88cUXLlO7WMzMCgsLXbZgwQJZq762tWvXumz8+PHy9ep5ozFotTPkgQcekLV5eXkuGzt2rKzt16+fy9TFDWZmO3fudJnaZWRmlp+f7zJ1eYSZ2S+//OIytSMJ3nfffecytVPDzGzw4MEuW7p0abs/U+Tzzz+XubppPvoa1N/SE088cXsP9h/AO3AASBQNHAASRQMHgETRwAEgUVldxCwuLnaZuvndTJ953bt3b1mrxvFnzpwpa7ds2eKyd955R9aqW88HDRoka9UiSXl5uaxVC4tq5P3QoUPy9Rs2bHDZxx9/LGujUXjl/fffd1lJSYmsVWewR2d8q8VRtTBmpo8qUOfIm7FgmQl1bruZHjmPFgCfeeYZl3XUuHltba3Lor9P9bzRTfNffvmly6K/5ZTwDhwAEkUDB4BE0cABIFE0cABIFA0cABKV1V0o6uZ2ddi/mR6fViPoZvoigugyBHWRwObNm2WtulW+a1f9b96LL77osuhWefX59u/fL2uVVatWuaysrEzWrl+/3mXz5s2TtWrnT7RLSI3Sq9eb6d08Ue3169ddNmPGDFmrdqyo34XOTB1jYKZ3cERHN7z22mvt+kx/Z8mSJS6LdseoPDq+IjoSInW8AweARNHAASBRNHAASBQNHAASldVFzIkTJ7pMnelspkfLo3Oh1ah1dGP5ypUrXRbdel5UVOSyaNFV3UofjfXee++9Ljt37pzL1FixmVlDQ4PLonHzuXPnuuzll1+WtWq8XT2XmV5A6tOnj6xVI8vR9+bMmTMu27Ztm6xVZ7t3ZmpxPFqgV9//aAGwI0bOFy1aJPMff/zRZdHvivp9/fbbb2/vwRLDO3AASBQNHAASRQMHgETRwAEgUVldxFQLX2pBz8zs6tWrLosuFG5paXGZOvfbTF92rBbOzPT545cvX5a1jz/+uMvUIqiZ/j78+uuvLlNnbpuZtba2uiyaNOvWzf+Ily1bJmvVlOqOHTtkrVocjaZU1dRm9D1XHzc6e3rTpk0uq6qqkrWdgfpZRVOMirro2Mxs9erVGX8MdQ6/ei61WGnWtudVtVOnTs349RF1eXplZWXGr48WfTviEmXegQNAomjgAJAoGjgAJIoGDgCJooEDQKKyugtFjcRG52B3797dZT179pS16izs6EZ3NX7bt29fWatu7lZnW5vpHRiXLl2Stepc9IKCApdduHBBvn7IkCEui87B/umnn1wWHTPQ2Njosuim+ffee89l0TEDaudPbm6urFVnUpeXl8vajroZPVVq90Q0hq5y9fOPaqPdImpnUFtuj8/085vpv88TJ07I2rY8gzp+YMWKFbf9cT/88EOXvfTSS7I2U7wDB4BE0cABIFE0cABIFA0cABKV1UVMpaKiQubr1q1zWTTOeurUKZdFC2rqYuQ333xT1v7www8uq6urk7VqgfXw4cOy9sCBAy5TlwcXFhbK16tFzOrqalk7f/58l128eFHWqoXFnJycjGujr1ct6qjLi830MQHREQrquIRHHnlE1nYGahHz9ddfl7Xq3Oxdu3Zl/Llud+S9PWoXLlzosug8cPVxo+Mn/gvfh0zxDhwAEkUDB4BE0cABIFE0cABIFA0cABLVpS0jrbfrwoUL7pM1NTXJWjWGrm5+jz5Gjx49ZG1eXp7L1C4WM33pgNrFYqbHvaPV7NmzZ7tM/RyiyyOWL1/usjlz5shaNW5cXFwsa9Wt8vX19bJ22LBhGb3ezGzr1q0umz59uqxVK/UNDQ2yVh018Oijj7b/Un9msveH9B8Q/c2oHVJtGTdXt9UvXry4jU93x3LfSN6BA0CiaOAAkCgaOAAkigYOAInK6ij9kSNHXDZ69GhZq84Jz8/Pl7VqrDsaeY8W2hR13vTvv/8ua9WN7l9//bWsVV+zWtSZOHGifP1TTz3lsmhMt3fv3i6LxuPVoqk6P91MLzJ366Z/nYYPH+4ydZxA9AzqVnuz+Hx4dLwlS5bIXP0eqmzBggXy9a+++urtPVgnwztwAEgUDRwAEkUDB4BE0cABIFE0cABIVFZH6Zubm90ni27CVjenRxcG7N27N+NnUJcDRDtTrl696rIJEybI2g0bNrjsySeflLUPP/ywy9ROCzUyb6Z3vKhxdTN9eUN0g/0999zjshs3bshaddO8upTCTF/SEO0oam5udllZWZms3b59u8vKy8sZpW9nq1evdtljjz0ma1U/UeP10e3x+FuM0gPAnYIGDgCJooEDQKJo4ACQqKyO0qtFwZEjR8raTBdDzPTC14gRI2TtsWPHXBYtYqpx/mgEfN26dS5TZ2ab6cW3Tz/91GXqvOvoGaLxePV9jBYbFXWeuJkej49G6dW56NHPRy3GRjfYq/Pa0f7U2HxbblhfsWJFez4O/g/vwAEgUTRwAEgUDRwAEkUDB4BE0cABIFFZ3YWyceNGl6ndDGb65vRo7F+NZa9du1bWqksSzp8/L2tbWlpctm3bNlm7cuVKl0WXIbzwwgsumzp1qsuGDh0qX6/G4/v27Str1Y6TaLeI2qFTVFQka6MRe6WioiKjz2Wmv+ZoJ8yYMWMyfgbc2meffSbz2tpal0W7UN566y2XzZs37/YeDCHegQNAomjgAJAoGjgAJIoGDgCJyuoi5sCBA10WjZurM76jRauzZ8+6LFpAPHTokMvUIpuZHtWuqamRtWrsvrS0VNbOnz/fZeqGdfUxzfTCZDRWro4vUAumZmanT592WW5urqxdv369y6IjCdRi8JQpU2TtzZs3XRaN/qvnLSwslLW4tZ07d8o805vmzcwWLVrUrs+Ev8c7cABIFA0cABJFAweARNHAASBRNHAASFRWd6GonQdqB4mZviH93LlzsrZ79+4uUyPzZmabN292WXV1taxVO2SisW51U7y6ad5Mj72rSwuiyyNmzZrlsujyB3UJRrRj5dSpUy4bNGiQrJ02bZrLmpqaZK3a+aNutTczu/vuuzN6LrP4GAb8M+pSErP4CAv8+3gHDgCJooEDQKJo4ACQKBo4ACQqq4uYajFKjU6b6TOvo0WyK1euuCxa4FIj9tEN6Wqxr6qqStaqryOqVQupeXl5LispKZGvP378uMtaW1tl7YABA1wWncU9duxYl12+fFnWqs+3e/duWasWbdVipZm+wT5aSI3ONcc/M27cOJmr39eFCxd29OMgA7wDB4BE0cABIFE0cABIFA0cABJFAweARHXJ5phsTU2N+2SVlZWy9ueff3bZ5MmTZe3BgwddFl3+oGrVKL6ZHvNXI/5mZtu3b3dZtLtF7YRRuy/UeL2ZHm2ObokfOXKky9TYv5ne+bNv3z5Z++CDD7osuoBC7RaJLvJQP/cJEybIWrUbp7y8XN800PGYN0dHc7/bvAMHgETRwAEgUTRwAEgUDRwAEpXVWWR1O3l0xndOTo7LotvJ1dh8NFquFtTUedVmZs3NzS4rKyuTtcXFxS6LFgvVgqc6ZiD6XPn5+S5TC5Bm+tztaAS9oqLCZdFi45o1a1ymRubN9DEB0eJoz549XRadGa+OUAA6E96BA0CiaOAAkCgaOAAkigYOAImigQNAorK6CyU3N9dle/fulbWFhYUZf1y1g6MtRwSMHj1a5mqXw5YtW2StGvPv2lX/+6jG3ouKilwW7Y5ROzhmzpwpa9XOn2jkXV1gce3aNVmrDv/fs2ePrFU/yx49eshaJfo+RjnQWfAXAACJooEDQKJo4ACQKBo4ACQqq+eBAwDaD+/AASBRNHAASBQNHAASRQMHgETRwAEgUTRwAEgUDRwAEkUDB4BE0cABIFE0cABIFA0cABJFAweARNHAASBRNHAASBQNHAASRQMHgETRwAEgUTRwAEgUDRwAEkUDB4BE0cABIFE0cABIFA0cABL1P0zKiiGRoGVWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_digit(data):\n",
    "    image = data.reshape(28, 28)\n",
    "    plt.imshow(image, cmap = matplotlib.cm.binary,\n",
    "               interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")\n",
    "    \n",
    "some_index = 5500\n",
    "plt.subplot(121); plot_digit(X_test_mod[some_index])\n",
    "plt.subplot(122); plot_digit(y_test_mod[some_index])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABU9JREFUeJzt3bGOTVscwOGZayQ0IiEUolZQSQiFRu8FFEpRaiRaKhWleADvgUZjHkChUY4QiYJEmFvce6s7s3DOnH3OzO/7yvnn7L2T8ctKrLP2rG9vb68BB99fy34AYBpihwixQ4TYIULsELEx8f381z8s3vpOP7SyQ4TYIULsECF2iBA7RIgdIsQOEWKHCLFDhNghQuwQIXaIEDtEiB0ixA4RYocIsUOE2CFC7BAhdogQO0SIHSLEDhFihwixQ4TYIULsECF2iBA7RIgdIsQOEWKHCLFDhNghQuwQIXaIEDtEiB0iNpb9ALAom5ubu85+/Pgx/Ozly5f3+nGWzsoOEWKHCLFDhNghQuwQIXaIEDtErG9vb095v0lvxsH28+fP4fzQoUMzX3viLvba+k4/tLJDhNghQuwQIXaIEDtEiB0iHHFl35pna+39+/d7+CT7g5UdIsQOEWKHCLFDhNghQuwQIXaIsM++D3z79m04f/To0a6zBw8eDD+7ykc5nz59urBrnz17dmHXXlVWdogQO0SIHSLEDhFihwixQ4TYIcKrpFfAly9fhvNjx47NfO1V3kff2toazk+fPj2c3717dzh/8uTJHz/TAeFV0lAmdogQO0SIHSLEDhFihwixQ4R99gn86jz60aNHF3bvZe+zj75DMM/3B9bW1tY+fPgwnJ88eXKu6+9j9tmhTOwQIXaIEDtEiB0ixA4RYocI742fwCL30dfWlr+XPvLy5cuZP3vt2rXhPLyPPhMrO0SIHSLEDhFihwixQ4TYIcIR1z2wvr7jicLJLHPr7dmzZ8P5nTt3Zr725ubmcH7x4sWZr33AOeIKZWKHCLFDhNghQuwQIXaIEDtEOOK6Am7fvj2c37hxY6In+b93794N5/Pso7969Wo4t4++t6zsECF2iBA7RIgdIsQOEWKHCLFDhPPse+DEiRPD+adPn4bzU6dODedbW1vD+eh3+Kuz9r/6c9JHjhwZzufx9u3buT7/8ePHmT/79evX4fzcuXPD+ZkzZ2a+9wScZ4cysUOE2CFC7BAhdogQO0SIHSLss/+mZb8bntXy+vXr4fzKlSsTPcmO7LNDmdghQuwQIXaIEDtEiB0ixA4R3hv/m+b5PsKv9uiPHz8+nH/+/HnmezOb69evD+dXr14dzif+/spvsbJDhNghQuwQIXaIEDtEiB0ibL1NYBW3Yf4z79HdW7duDef37t3bdXbhwoW57s2fsbJDhNghQuwQIXaIEDtEiB0ixA4R9tkPuHn30e/fvz+cP3z4cDg/fPjwXPdn71jZIULsECF2iBA7RIgdIsQOEWKHCPvsB8Djx49n/uzz58+H85s3b858bVaLlR0ixA4RYocIsUOE2CFC7BAhdohYn/id5qv7AvUVNu+Z9JFVfqc9M9vxH4yVHSLEDhFihwixQ4TYIULsEOGI6wp48+bNwq79/fv3hV2b/cXKDhFihwixQ4TYIULsECF2iBA7RNhnXwGXLl2a6/MvXrzYdbax4VfMP6zsECF2iBA7RIgdIsQOEWKHCLFDhE3YA+D8+fPLfgT2ASs7RIgdIsQOEWKHCLFDhNghQuwQYZ99BfizyUzByg4RYocIsUOE2CFC7BAhdogQO0SIHSLEDhFihwixQ4TYIULsECF2iBA7RIgdIsQOEWKHCLFDhNghQuwQIXaIEDtEiB0ixA4RYocIsUOE2CFC7BAhdoiY+k82r098P+BfVnaIEDtEiB0ixA4RYocIsUOE2CFC7BAhdogQO0SIHSLEDhFihwixQ4TYIULsECF2iBA7RIgdIsQOEWKHCLFDhNgh4m+4YaqdYfQs9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "knn_clf.fit(X_train_mod, y_train_mod)\n",
    "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n",
    "plot_digit(clean_digit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 연습문제"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=None, n_neighbors=5, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1번\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5], dtype=int8)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.97025595, 0.96719836, 0.96974546])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "cross_val_score(knn_clf, X_train, y_train, cv=3, scoring='accuracy', n_jobs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2번 (training set expansion)\n",
    "\n",
    "from scipy.ndimage.interpolation import shift\n",
    "\n",
    "def shift_image(image, dx, dy):\n",
    "    image = image.reshape((28, 28))\n",
    "    shifted_image = shift(image, [dy, dx], cval=0, mode=\"constant\")\n",
    "    return shifted_image.reshape([-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqEAAADTCAYAAACx18q9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHStJREFUeJzt3X+0HHWZ5/HPJ8EEM8BBIQKGxCuYgWEXBDeCM8aZcBCC7pnFjByUAwwwIC4L50Q2gcEQRtaD/JhBRTAJhA2ELGp0NoCAMkxEkqxDxiGwYkBMgpqEQICwMRAY85Nn/6iKdu79dm737e663V3v1zn33O6nq6ueSu5zv8+tqm+XI0IAAABAkYYMdgIAAAAoH5pQAAAAFI4mFAAAAIWjCQUAAEDhaEIBAABQOJpQAAAAFI4mtI3YXm17ap3vCdunNzmPa2w/08x1AkWppY56L2P7YNv/bPst203/3Drbz9i+ps73TLW9utm5AJ2iE2vZ9nm236xznSNs/2/br+djek+DaXYMmtAmsz3K9mzb62xvs/2i7TtsH1rD2z8saWadmzxE0oP1Zwp0Htsjbc/MB56ttl+x/ajtk+tcVe9amyrpvZKOVVZTLfkDD0CGWt7N30j6c0njleX8wkAOSnWivQY7gW5i+/2SHpf0G0nnSlol6XBJX5H0hO0/jYjVifcNi4htEbGh3m1GxMuNZQ10lAWSRki6QNLzkt4j6S8kHVDPShK19gFJT0bEqmYkCaBf1PIffEDScxGxfFfA9iCmUxyOhDbXDElvS/p4RDwaEWsj4jFJH8/jMyTJ9iLbs2zfZHuDpH/J471PK/yx7cW2t9heYfuTtt+0fV7FMr//C892T/7807YX2v5327+o/MvS9lDbc2z/xvbvbK+yfYVtfhbQ1mzvL+ljkq7M62tNRDwRETdFxPxei+9t+3bbb+RnJS7vta7f11p+yvs0SX+d18/citPg/5jHVle89y9tP5nX5W9sf8X2sIrX32P7+3l9rbH9NzXu3xW2X85rfJ6kfXq9PsT21bZfyI8cLbd9WsXr823fVvH82jz3j1TEXrB9dv54ru2HbE/Oz9j81vZdtkfUki8wUN1ey4n9rbod24skTZb053l+i/LY+yT9Qx7r2ltb0ng0ie13SzpV0oyI+PfK1/LnMyV9wva78vDZkqysEP86sb4hku6TtEPSRySdJ+lLkobXkM5XJN0i6YOSnpA03/auAW2IpBclnSHpTyRdJWmapPNr3FVgsLyZf/0X23v3s+xlkpZL+pCkGyX9ve0/rbLshyX9SNL3lJ0Km5zHJOlzeezDkmR7oqRvSfqmpP+g7DTa6ZKuq1jfXGVHNj4u6VPK6rtnT8naPkPStcpq/EOSVkj6770Wmyzpckl/K+loZb8f7rV9bP76IkkTKpafIOm1XTHbH5B0aL7cLh+T9B/zXD8jaVK+HaCVuraWe6thO38l6S5JS/P8/ir/Wifpy3nskHq22VEigq8mfEk6QVJImlTl9Un568crGwR+nlhmtaSp+eOJyhrQURWv/1m+jvMqYiHp9PxxT/788xWvj8pj4/eQ+w2SflTx/BpJzwz2vylffPX+kvRpSRslbVH2S/smSSf0Wma1pO/0iq2SNL3XMlMrnj8kaW6v9/y+tipiSyRd3Sv2KWUDqiX9cf6+j1a8/j5JOyVds4f9elzSHb1iP5K0uuL5i5L+rtcyiyTdkz8+Mt/2IcpOc25V1rA+kr9+oaTnK947V9ILkoZWxO6o/F3AF1+t+uriWj5P0pu1bid//k1JixL7PrXadrrliyOhg+fJfl4/UtJLEfFiRewJZaf1+/Pziscv5d/fsytg+7/aXmZ7g7NZfJdJGlPDeoFBFRELlE06+EtJDyv7w+xfbU/rtejPez1/SRU10ID/JOmq/JT5m3n9fFvSH0k6WNnZhbcl/VtFzmv0hzqs5k+UDcSVfv/c9n7K9vtfei3zE0lH5dv5paSXlR35/DNJv5L0XUkftf2OPL6o1/t/ERE7K543698J2KMuruV6t1NqTExqnueV/dV0lLLTZL0dlb/+fP78rRbmsn3Xg4gIZxc4D5Ek25+RdLOyGYSPS3pD0iXKjtQCbS8itkhamH992fb/lHSN7ZsiYlu+2Pbeb1NzLj8aIul/SPrHxGuVEySKvIarcluLJZ0o6VVJj0XEatuvKTsF+ReSvtjrva36dwL6VZJarnU7pUQT2iQR8f9sPyLpv9n+elRcF5pf6H+JpIcjYqNrm/X2S0nvtf3eiNj1l9c4NV584yX9NCK+WZHf4Q2uExhMv1D2u2xvSdv6WbYe2yUN7RV7StKREfF8YnnZ/qWyGj1e2R95sj1G2RGfPXlO2bXfd1bEfj+hKCLesP2SpI9KerRimfHK9n+XRZKmSHpF0jcqYp9T3+tBgXbTDbXc2x63swfb1DfnrkMT2lyXKvth/ZHt6dr9I5qcv16rhcomJ9ydz/x7p6SvKbtOtJG/zFZKOs/2J5Qdlf2ssiMkv21gnUDL2T5A2dGEO5Wdotus7A+zKyQ9GhFvNHmTqyWdZHuxpK0R8VtlEwUesr1G2eSHHcom9hwfEVdExArb/yTpdtsXSfqdsrr9XT/b+oakebafUNYonq7sOvONFcv8g7KjRauUXc5ztrKJRR+qWGaRpFnKrl1bVBG7Q9KvImJdff8EQPN1eS33tsft9JPzx2zfk+f8Wp3b7QicdmmiiPiVskJ6VtL/kvRrZdd+PCfpwxHxmzrW9bayU+TDlV2TcreyZjaUXcg9ULcrK4RvK7vGtEfSVxtYH1CUNyX9q7IZr4uV1dl1yn6WP9OC7U1Rdmr7BUn/V5Ii4hFJ/zmP/1v+daWktRXvO0/ZZwX/WNmNJL6tbECpKiK+q2xC4FfybR2tbMCrdIuyRvTvJT2j7PfDpyPi6Yr17LoudGX84fMTFyk74LColp0GCtC1tdxbjdtJ+TtJo5Vd2921p+13zcxCB7D9QUk/kzQuIvqb2AQAANC2aELbmO1JyiYwrVJ2xPJryk7rHxf8xwEAgA7GNaHtbV9lH847Wtk1m4skXUYDCgAAOh1HQgEAAFA4JiYBAACgcA01obZPtb3C9vO2r2xWUgBag5oFOgf1im434NPxtocq+8zJkyWtU/ZxP2dGxC+qvefAAw+Mnp6eAW0PaKbVq1frtddeq+muAd2i3pqlXtEuqFfGWHSWWmu2kYlJx0t6PiJ+LUm250s6TbvfvWM3PT09WrZsWQObBJpj3Lhxg53CYKirZqlXtAvqlTEWnaXWmm3kdPwoZR/8usu6PAagPVGzQOegXtH1Wj4xyfZFtpfZXrZhQ9d+6D/QFahXoLNQs+hkjTShLyr7/MpdDs1ju4mI2RExLiLGjRw5soHNAWhQvzVLvQJtgzEWXa+RJvQJSWNtv9/2MEmflfRAc9IC0ALULNA5qFd0vQFPTIqIHbYvlfSIpKGS7oyIZ5uWGYCmomaBzkG9ogwaum1nRPxQ0g+blAuAFqNmgc5BvaLbccckAAAAFI4mFAAAAIWjCQUAAEDhaEIBAABQOJpQAAAAFI4mFAAAAIWjCQUAAEDhaEIBAABQOJpQAAAAFI4mFAAAAIWjCQUAAEDhaEIBAABQOJpQAAAAFI4mFAAAAIWjCQUAAEDhaEIBAABQOJpQAAAAFI4mFAAAAIWjCQUAAEDhaEIBAABQuL0aebPt1ZI2S9opaUdEjGtGUt1g+/btNcUkadasWXWt+/HHH0/GDzjggGR87dq1yfgjjzzSJzZx4sTksocddlgyPnPmzGQc7YmaBTpHJ9RrtXFtMMa7esY6ifGuHTTUhOZOjIjXmrAeAMWgZoHOQb2ia3E6HgAAAIVrtAkNSf9s+0nbFzUjIQAtRc0CnYN6RVdr9HT8+Ih40fZ7JC20/cuIWFK5QF44F0nSmDFjGtwcgAbtsWapV6CtMMaiqzV0JDQiXsy/vyrpPknHJ5aZHRHjImLcyJEjG9kcgAb1V7PUK9A+GGPR7QZ8JNT2H0kaEhGb88enSPpy0zLrEMuXL0/Gp0yZ0ie2ePHi5LI7duxoak61GjKk798gCxcuTC47evToZHzr1q3J+PDhwweeGFqCmgU6R7vVaz1jndRe411qrJMY79pBI6fjD5J0n+1d6/l2RPxTU7IC0ArULNA5qFd0vQE3oRHxa0kfbGIuAFqImgU6B/WKMuAjmgAAAFA4mlAAAAAUjiYUAAAAhWvGbTtLYcuWLcn4WWedlYw/++yzrUyncJs3b07GV6xYkYwfc8wxrUwHANAiqfGuLGOdxHhXJI6EAgAAoHA0oQAAACgcTSgAAAAKRxMKAACAwtGEAgAAoHDMjq/RypUrk/F6ZgYeccQRyXi1GXftZNOmTcn4pZdemowvWbKklekAAFokNd7VOwue8Q614EgoAAAACkcTCgAAgMLRhAIAAKBwNKEAAAAoHE0oAAAACsfs+BrdfvvtdS1/9NFH94nNnDkzuWxPT08y/vDDDyfj1157bTK+du3a2pJrouOOO67wbQIAWqee8S411kmMd6gNR0IBAABQOJpQAAAAFI4mFAAAAIWjCQUAAEDh+m1Cbd9p+1Xbz1TE3m17oe1V+fd3tTZNALWiZoHOQb2izGqZHT9X0jclzauIXSnp0Yi4wfaV+fO/bX56xZs+fXoyPnfu3LrWc/jhh/eJnXDCCcllhw4dmoxfcMEFyfiZZ56ZjF933XXJ+PXXX5+M12O//fZLxqdOndrwutF0c1Wimm2G7du31xSTpFmzZtW17scffzwZP+CAA5LxarN+H3nkkT6xiRMnJpc97LDDkvFqM5YxqOZqkOq1GeNdaqyTBme8a8ZYJzHeFanfI6ERsUTSxl7h0yTdnT++W9KnmpwXgAGiZoHOQb2izAZ6TehBEbE+f/yypIOalA+A1qBmgc5BvaIUGp6YFBEhKaq9bvsi28tsL9uwYUOjmwPQoD3VLPUKtBfGWHSzgTahr9g+RJLy769WWzAiZkfEuIgYN3LkyAFuDkCDaqpZ6hVoC4yxKIWB3rbzAUnnSroh//79pmU0yKr9Jblly5a61nP//ff3iVX7BXHDDTfUte5q7rvvvobXMWzYsGT8lltuScZHjx7d8DZRiK6t2XosX748GZ8yZUqf2OLFi5PL7tixo6k51WrIkL7HDBYuXJhctlpdbt26NRkfPnz4wBNDKxRSr80Y71JjncR4h9rU8hFN35G0VNIRttfZvkBZYZxse5Wkj+fPAbQBahboHNQryqzfI6ERkf58BOmkJucCoAmoWaBzUK8oM+6YBAAAgMLRhAIAAKBwNKEAAAAo3EBnx2MAXn/99WT84osvLjiT6rZt25aMP/jgg8n4unXrkvGzzz47GWd2IYpQbXbvWWedlYw/++yzrUyncJs3b07GV6xYkYwfc8wxrUwHJcR4x3hXC46EAgAAoHA0oQAAACgcTSgAAAAKRxMKAACAwtGEAgAAoHDMju+l2v1u991332R8+/btyXi995pvdwsWLKgrfttttyXjTz/9dDK+//77DywxIGHlypXJeD2z4I844ohkvNoM83ayadOmZPzSSy9NxpcsWdLKdNCm6hnvyjLWSa0d7xjrdseRUAAAABSOJhQAAACFowkFAABA4WhCAQAAUDiaUAAAABSO2fG9TJ8+PRm/8MILk/Fq95596623+sQee+yx5LKXX355jdl1jmr32K12P2FmDKKZbr/99rqWP/roo/vEZs6cmVy2p6cnGX/44YeT8WuvvTYZX7t2bW3JNdFxxx1X+DbRvuoZ7+oZ6yTGOyk93jHW7Y4joQAAACgcTSgAAAAKRxMKAACAwtGEAgAAoHD9NqG277T9qu1nKmLX2H7R9s/yr0+2Nk0AtaJmgc5BvaLMapkdP1fSNyXN6xX/ekTc1PSMBtnee++djFebEVuPxYsXN7wOSTr55JOT8RNPPDEZP/3005Pxm2++uU+s2kzHpUuXJuPV7tFdzT333JOMX3XVVXWtB3s0VyWp2Wqze+fOnVvXeg4//PA+sRNOOCG57NChQ5PxCy64IBk/88wzk/HrrrsuGb/++uuT8Xrst99+yfjUqVMbXjeabq4GqV67bbyrZ6yTBme8Y6zbXb9HQiNiiaSNBeQCoAmoWaBzUK8os0auCb3U9s/zUwnvalpGAFqFmgU6B/WKrjfQJnSWpMMlHStpvaSvVlvQ9kW2l9letmHDhgFuDkCDaqpZ6hVoC4yxKIUBNaER8UpE7IyItyXdIen4PSw7OyLGRcS4kSNHDjRPAA2otWapV2DwMcaiLAbUhNo+pOLpJEnPVFsWwOCjZoHOQb2iLPqdHW/7O5ImSDrQ9jpJX5I0wfaxkkLSakmfb2GOHSl1WmTatGnJZUeMGJGMX3311cn45MmTk/Hhw4fXmF3m1ltv7RPbuXNnctkZM2Yk45dddlkyfuihhybj55xzTo3ZYaDKVLPVTj9u2bKlrvXcf//9fWLVjirdcMMNda27mvvuu6/hdQwbNiwZv+WWW5Lx0aNHN7xNNFen12u1Gmyn8S411kmMd+2g3yY0IlKfLzKnBbkAaAJqFugc1CvKjDsmAQAAoHA0oQAAACgcTSgAAAAKRxMKAACAwtVy73gMwMSJE/vEtm7dmlx21KhRyfgVV1zR1JxqsWrVqmS82qzAaqrNaBwzZkzdOQGD4fXXX0/GL7744oIzqW7btm3J+IMPPpiMr1u3Lhk/++yzk3Fm06M/qbFOYryTGO9qwZFQAAAAFI4mFAAAAIWjCQUAAEDhaEIBAABQOCYmNeill15KxtesWVPzOm688cZmpZO0ffv2ZPySSy7pE7vrrruass1TTjmlKesB9qTarTX33XffZLxaLdR7m892t2DBgrrit912WzL+9NNPJ+P777//wBJDx2rGWCe1dryrZ6yTGO/aAUdCAQAAUDiaUAAAABSOJhQAAACFowkFAABA4WhCAQAAUDhmxzfoqaeeSsY3bdpU8zoeeuihZHz8+PHJ+Jw5c2petyTNmDEjGd+4cWNd60kZNmxYMj58+PCG1w30Z/r06cn4hRdemIxXu83lW2+91Sf22GOPJZe9/PLLa8yuc1S7nWe1W5cyO758mjHWSa0d71o51kmMd63AkVAAAAAUjiYUAAAAhaMJBQAAQOFoQgEAAFA4mlAAAAAUrt/Z8bZHS5on6SBJIWl2RHzD9rslfVdSj6TVks6IiN+2LtXBVe2+ueeff37D654/f35d8XYyadKkZHzs2LEFZwKpfPW69957J+M9PT0Nr3vx4sUNr0OSTj755GT8xBNPTMZPP/30ZPzmm2/uE0vN6pekpUuXJuMrV65Mxqu55557kvGrrrqqrvWgunas2dR414yxTmK8w+5qORK6Q9KUiDhK0kckXWL7KElXSno0IsZKejR/DmBwUa9AZ6FmUVr9NqERsT4insofb5b0nKRRkk6TdHe+2N2SPtWqJAHUhnoFOgs1izKr65pQ2z2SjpP0U0kHRcT6/KWXlZ1KSL3nItvLbC/bsGFDA6kCqAf1CnQWahZlU3MTansfSQskfSEi3qh8LSJC2bUsfUTE7IgYFxHjRo4c2VCyAGpDvQKdhZpFGdXUhNp+h7Li+FZE3JuHX7F9SP76IZJebU2KAOpBvQKdhZpFWdUyO96S5kh6LiK+VvHSA5LOlXRD/v37LcmwTVS7V/KECROS8XvvvTcZ71QHH3xwMj579uyCM8GeUK8DkzqNOW3atOSyI0aMSMavvvrqZHzy5MnJeL33m7711lv7xHbu3Jlctto9tC+77LJk/NBDD03GzznnnBqzw0C1Y82mxruyjHUS412R+m1CJX1U0jmSltv+WR6bpqwwvmf7AklrJJ3RmhQB1IF6BToLNYvS6rcJjYifSHKVl09qbjoAGkG9Ap2FmkWZccckAAAAFI4mFAAAAIWjCQUAAEDhapmYBFWfETtv3rxkPHXv5vXr1yeWbC977ZX+kai2n/vss08r0wEKMXHixD6xrVu3JpcdNWpUMn7FFVc0NadarFq1KhmvNgu+mmoz+MeMGVN3Tuh8qfGunrFOYrxDbTgSCgAAgMLRhAIAAKBwNKEAAAAoHE0oAAAACkcTCgAAgMIxO75B73znO5Pxeu8L3S5OPfXUZPykk7hxBzrfSy+9lIyvWbOm5nXceOONzUonafv27cn4JZdc0id21113NWWbp5xySlPWg+7VbWOdxHjXDjgSCgAAgMLRhAIAAKBwNKEAAAAoHE0oAAAACkcTCgAAgMIxO75FfvCDH/SJbd68ObnstGnTkvEf//jHTcml2j2tv/jFL/aJVZsBCXSDp556KhnftGlTzet46KGHkvHx48cn43PmzKl53ZI0Y8aMZHzjxo11rSdl2LBhyXgnz3DG4EqNddLgjHf1jHUS41074EgoAAAACkcTCgAAgMLRhAIAAKBwNKEAAAAoXL8Tk2yPljRP0kGSQtLsiPiG7WskfU7ShnzRaRHxw1Yl2mmOPPLImpdduHBhCzNBmVCvmWq35zz//PMbXvf8+fPrireTSZMmJeNjx44tOBPs0uk1W89YJzHeYXe1zI7fIWlKRDxle19JT9re9VP09Yi4qXXpAagT9Qp0FmoWpdVvExoR6yWtzx9vtv2cpFGtTgxA/ahXoLNQsyizuq4Jtd0j6ThJP81Dl9r+ue07bb+rynsusr3M9rINGzakFgHQAtQr0FmoWZRNzU2o7X0kLZD0hYh4Q9IsSYdLOlbZX3FfTb0vImZHxLiIGDdy5MgmpAygP9Qr0FmoWZRRTU2o7XcoK45vRcS9khQRr0TEzoh4W9Idko5vXZoAakW9Ap2FmkVZ1TI73pLmSHouIr5WET8kv5ZFkiZJeqY1KQKoFfWa2X///ZPxCRMmJOP33ntvC7Mp3sEHH5yMz549u+BM0B9qFmVWy+z4j0o6R9Jy2z/LY9MknWn7WGUfKbFa0udbkiGAelCvQGehZlFatcyO/4kkJ15qu88rA8qOegU6CzWLMuOOSQAAACgcTSgAAAAKRxMKAACAwtUyMQkAOsqIESOS8Xnz5iXjS5cu7RNbv359Ysn2stde6V/h1fZzn332aWU6AFAXjoQCAACgcDShAAAAKBxNKAAAAApHEwoAAIDC0YQCAACgcI6I4jZmb5C0Jn96oKTXCtv44GE/29P7ImLkYCfRzqjXrtdJ+0q91oCa7Wqdtp811WyhTehuG7aXRcS4Qdl4gdhPdIOy/P+WZT+lcu1rGZXl/5f97GycjgcAAEDhaEIBAABQuMFsQmcP4raLxH6iG5Tl/7cs+ymVa1/LqCz/v+xnBxu0a0IBAABQXpyOBwAAQOEKb0Jtn2p7he3nbV9Z9PZbyfadtl+1/UxF7N22F9pelX9/12Dm2Ay2R9t+zPYvbD9re3Ie77p9RffWLPXaffuK7q1XqRw1W7Z6LbQJtT1U0gxJn5B0lKQzbR9VZA4tNlfSqb1iV0p6NCLGSno0f97pdkiaEhFHSfqIpEvy/8du3NdS6/KanSvqtdv2tdS6vF6lctRsqeq16COhx0t6PiJ+HRHbJM2XdFrBObRMRCyRtLFX+DRJd+eP75b0qUKTaoGIWB8RT+WPN0t6TtIodeG+ontrlnrtvn1F99arVI6aLVu9Ft2EjpL0QsXzdXmsmx0UEevzxy9LOmgwk2k22z2SjpP0U3X5vpZU2Wq2q3+GqdeuV7Z6lbr457gM9crEpAJF9lEEXfNxBLb3kbRA0hci4o3K17ptX1E+3fYzTL2i23XTz3FZ6rXoJvRFSaMrnh+ax7rZK7YPkaT8+6uDnE9T2H6HsgL5VkTcm4e7cl9Lrmw125U/w9RraZStXqUu/DkuU70W3YQ+IWms7ffbHibps5IeKDiHoj0g6dz88bmSvj+IuTSFbUuaI+m5iPhaxUtdt68oXc123c8w9VoqZatXqct+jstWr4V/WL3tT0q6WdJQSXdGxFcKTaCFbH9H0gRJB0p6RdKXJN0v6XuSxkhaI+mMiOh9YXVHsT1e0v+RtFzS23l4mrLrVrpqX9G9NUu9Uq/dqFvrVSpHzZatXrljEgAAAArHxCQAAAAUjiYUAAAAhaMJBQAAQOFoQgEAAFA4mlAAAAAUjiYUAAAAhaMJBQAAQOFoQgEAAFC4/w9YR3D8wqAdkQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x216 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "image = X_train[1000]\n",
    "shifted_image_down = shift_image(image, 0, 5)\n",
    "shifted_image_left = shift_image(image, -5, 0)\n",
    "\n",
    "plt.figure(figsize=(12,3))\n",
    "plt.subplot(131)\n",
    "plt.title(\"Original\", fontsize=14)\n",
    "plt.imshow(image.reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\n",
    "plt.subplot(132)\n",
    "plt.title(\"Shifted down\", fontsize=14)\n",
    "plt.imshow(shifted_image_down.reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\n",
    "plt.subplot(133)\n",
    "plt.title(\"Shifted left\", fontsize=14)\n",
    "plt.imshow(shifted_image_left.reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# augmentation\n",
    "\n",
    "X_train_augmented = [image for image in X_train]\n",
    "y_train_augmented = [label for label in y_train]\n",
    "\n",
    "for dx, dy in ((1, 0), (-1, 0), (0, 1), (0, -1)):\n",
    "    for image, label in zip(X_train, y_train):\n",
    "        X_train_augmented.append(shift_image(image, dx, dy))\n",
    "        y_train_augmented.append(label)\n",
    "\n",
    "X_train_augmented = np.array(X_train_augmented)\n",
    "y_train_augmented = np.array(y_train_augmented)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffle_idx = np.random.permutation(len(X_train_augmented))\n",
    "X_train_augmented = X_train_augmented[shuffle_idx]\n",
    "y_train_augmented = y_train_augmented[shuffle_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=None, n_neighbors=4, p=2,\n",
       "           weights='distance')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf = KNeighborsClassifier(n_neighbors=4, weights='distance')\n",
    "knn_clf.fit(X_train_augmented, y_train_augmented)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = knn_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.ndimage.interpolation import shift\n",
    "def shift_digit(digit_array, dx, dy, new=0):\n",
    "    return shift(digit_array.reshape(28, 28), [dy, dx], cval=new).reshape(784)\n",
    "\n",
    "plot_digit(shift_digit(some_digit, 5, 1, new=100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3번\n",
    "import os\n",
    "\n",
    "TITANIC_PATH = os.path.join(\"datasets\", \"titanic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def load_titanic_data(filename, titanic_path=TITANIC_PATH):\n",
    "    csv_path = os.path.join(titanic_path, filename)\n",
    "    return pd.read_csv(csv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = load_titanic_data(\"train.csv\")\n",
    "test_data = load_titanic_data(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 12 columns):\n",
      "PassengerId    891 non-null int64\n",
      "Survived       891 non-null int64\n",
      "Pclass         891 non-null int64\n",
      "Name           891 non-null object\n",
      "Sex            891 non-null object\n",
      "Age            714 non-null float64\n",
      "SibSp          891 non-null int64\n",
      "Parch          891 non-null int64\n",
      "Ticket         891 non-null object\n",
      "Fare           891 non-null float64\n",
      "Cabin          204 non-null object\n",
      "Embarked       889 non-null object\n",
      "dtypes: float64(2), int64(5), object(5)\n",
      "memory usage: 83.6+ KB\n"
     ]
    }
   ],
   "source": [
    "train_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "위의 정보를 확인해보면 cabin column에 null 데이터가 상당히 많다는 것을 알 수 있다. 따라서 cabin 데이터는 무시한다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>714.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.383838</td>\n",
       "      <td>2.308642</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>0.523008</td>\n",
       "      <td>0.381594</td>\n",
       "      <td>32.204208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>257.353842</td>\n",
       "      <td>0.486592</td>\n",
       "      <td>0.836071</td>\n",
       "      <td>14.526497</td>\n",
       "      <td>1.102743</td>\n",
       "      <td>0.806057</td>\n",
       "      <td>49.693429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>223.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.125000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>668.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>512.329200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       PassengerId    Survived      Pclass         Age       SibSp  \\\n",
       "count   891.000000  891.000000  891.000000  714.000000  891.000000   \n",
       "mean    446.000000    0.383838    2.308642   29.699118    0.523008   \n",
       "std     257.353842    0.486592    0.836071   14.526497    1.102743   \n",
       "min       1.000000    0.000000    1.000000    0.420000    0.000000   \n",
       "25%     223.500000    0.000000    2.000000   20.125000    0.000000   \n",
       "50%     446.000000    0.000000    3.000000   28.000000    0.000000   \n",
       "75%     668.500000    1.000000    3.000000   38.000000    1.000000   \n",
       "max     891.000000    1.000000    3.000000   80.000000    8.000000   \n",
       "\n",
       "            Parch        Fare  \n",
       "count  891.000000  891.000000  \n",
       "mean     0.381594   32.204208  \n",
       "std      0.806057   49.693429  \n",
       "min      0.000000    0.000000  \n",
       "25%      0.000000    7.910400  \n",
       "50%      0.000000   14.454200  \n",
       "75%      0.000000   31.000000  \n",
       "max      6.000000  512.329200  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    549\n",
       "1    342\n",
       "Name: Survived, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"Survived\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PassengerId</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.005007</td>\n",
       "      <td>-0.035144</td>\n",
       "      <td>0.036847</td>\n",
       "      <td>-0.057527</td>\n",
       "      <td>-0.001652</td>\n",
       "      <td>0.012658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Survived</th>\n",
       "      <td>-0.005007</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.338481</td>\n",
       "      <td>-0.077221</td>\n",
       "      <td>-0.035322</td>\n",
       "      <td>0.081629</td>\n",
       "      <td>0.257307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pclass</th>\n",
       "      <td>-0.035144</td>\n",
       "      <td>-0.338481</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.369226</td>\n",
       "      <td>0.083081</td>\n",
       "      <td>0.018443</td>\n",
       "      <td>-0.549500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Age</th>\n",
       "      <td>0.036847</td>\n",
       "      <td>-0.077221</td>\n",
       "      <td>-0.369226</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.308247</td>\n",
       "      <td>-0.189119</td>\n",
       "      <td>0.096067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SibSp</th>\n",
       "      <td>-0.057527</td>\n",
       "      <td>-0.035322</td>\n",
       "      <td>0.083081</td>\n",
       "      <td>-0.308247</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.414838</td>\n",
       "      <td>0.159651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Parch</th>\n",
       "      <td>-0.001652</td>\n",
       "      <td>0.081629</td>\n",
       "      <td>0.018443</td>\n",
       "      <td>-0.189119</td>\n",
       "      <td>0.414838</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.216225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fare</th>\n",
       "      <td>0.012658</td>\n",
       "      <td>0.257307</td>\n",
       "      <td>-0.549500</td>\n",
       "      <td>0.096067</td>\n",
       "      <td>0.159651</td>\n",
       "      <td>0.216225</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             PassengerId  Survived    Pclass       Age     SibSp     Parch  \\\n",
       "PassengerId     1.000000 -0.005007 -0.035144  0.036847 -0.057527 -0.001652   \n",
       "Survived       -0.005007  1.000000 -0.338481 -0.077221 -0.035322  0.081629   \n",
       "Pclass         -0.035144 -0.338481  1.000000 -0.369226  0.083081  0.018443   \n",
       "Age             0.036847 -0.077221 -0.369226  1.000000 -0.308247 -0.189119   \n",
       "SibSp          -0.057527 -0.035322  0.083081 -0.308247  1.000000  0.414838   \n",
       "Parch          -0.001652  0.081629  0.018443 -0.189119  0.414838  1.000000   \n",
       "Fare            0.012658  0.257307 -0.549500  0.096067  0.159651  0.216225   \n",
       "\n",
       "                 Fare  \n",
       "PassengerId  0.012658  \n",
       "Survived     0.257307  \n",
       "Pclass      -0.549500  \n",
       "Age          0.096067  \n",
       "SibSp        0.159651  \n",
       "Parch        0.216225  \n",
       "Fare         1.000000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "        \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "num_pipeline = Pipeline([\n",
    "        (\"select_numeric\", DataFrameSelector([\"Age\", \"SibSp\", \"Parch\", \"Fare\"])),\n",
    "        (\"imputer\", SimpleImputer(strategy=\"median\")),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[22.    ,  1.    ,  0.    ,  7.25  ],\n",
       "       [38.    ,  1.    ,  0.    , 71.2833],\n",
       "       [26.    ,  0.    ,  0.    ,  7.925 ],\n",
       "       ...,\n",
       "       [28.    ,  1.    ,  2.    , 23.45  ],\n",
       "       [26.    ,  0.    ,  0.    , 30.    ],\n",
       "       [32.    ,  0.    ,  0.    ,  7.75  ]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_pipeline.fit_transform(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MostFrequentImputer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        self.most_frequent_ = pd.Series([X[c].value_counts().index[0] for c in X],\n",
    "                                        index=X.columns)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        return X.fillna(self.most_frequent_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "cat_pipeline = Pipeline([\n",
    "        (\"select_cat\", DataFrameSelector([\"Pclass\", \"Sex\", \"Embarked\"])),\n",
    "        (\"imputer\", MostFrequentImputer()),\n",
    "        (\"cat_encoder\", OneHotEncoder(sparse=False)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 1., ..., 0., 0., 1.],\n",
       "       [1., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 1., ..., 0., 0., 1.],\n",
       "       ...,\n",
       "       [0., 0., 1., ..., 0., 0., 1.],\n",
       "       [1., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 1., ..., 0., 1., 0.]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_pipeline.fit_transform(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import FeatureUnion\n",
    "\n",
    "preprocess_pipeline = FeatureUnion(transformer_list=[\n",
    "        (\"num_pipeline\", num_pipeline),\n",
    "        (\"cat_pipeline\", cat_pipeline),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[22.,  1.,  0., ...,  0.,  0.,  1.],\n",
       "       [38.,  1.,  0., ...,  1.,  0.,  0.],\n",
       "       [26.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       ...,\n",
       "       [28.,  1.,  2., ...,  0.,  0.,  1.],\n",
       "       [26.,  0.,  0., ...,  1.,  0.,  0.],\n",
       "       [32.,  0.,  0., ...,  0.,  1.,  0.]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = preprocess_pipeline.fit_transform(train_data)\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train_data[\"Survived\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SVM\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "svm_clf = SVC(gamma=\"auto\")\n",
    "svm_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "\"['AgeBucket', 'RelativesOnboard'] not in index\"",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-85-be78743d5324>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreprocess_pipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msvm_clf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    820\u001b[0m         Xs = Parallel(n_jobs=self.n_jobs)(\n\u001b[1;32m    821\u001b[0m             \u001b[0mdelayed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_transform_one\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrans\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 822\u001b[0;31m             for name, trans, weight in self._iter())\n\u001b[0m\u001b[1;32m    823\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mXs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    824\u001b[0m             \u001b[0;31m# All transformers are None\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    915\u001b[0m             \u001b[0;31m# remaining jobs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    916\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    918\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    757\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 759\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    760\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    761\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    714\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    715\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 716\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    717\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    718\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    180\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    547\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    548\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 549\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    550\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    223\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    224\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 225\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    227\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    223\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    224\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 225\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    227\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_transform_one\u001b[0;34m(transformer, X, y, weight, **fit_params)\u001b[0m\n\u001b[1;32m    603\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_transform_one\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 605\u001b[0;31m     \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    606\u001b[0m     \u001b[0;31m# if we have a weight for this transformer, multiply output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    607\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mweight\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_transform\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    449\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransform\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    450\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtransform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 451\u001b[0;31m                 \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    452\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mXt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    453\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-c4c8b5533962>\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattribute_names\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2922\u001b[0m                 \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2923\u001b[0m             indexer = self.loc._convert_to_indexer(key, axis=1,\n\u001b[0;32m-> 2924\u001b[0;31m                                                    raise_missing=True)\n\u001b[0m\u001b[1;32m   2925\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2926\u001b[0m         \u001b[0;31m# take() does not accept boolean indexers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_convert_to_indexer\u001b[0;34m(self, obj, axis, is_setter, raise_missing)\u001b[0m\n\u001b[1;32m   1352\u001b[0m                 kwargs = {'raise_missing': True if is_setter else\n\u001b[1;32m   1353\u001b[0m                           raise_missing}\n\u001b[0;32m-> 1354\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_listlike_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1355\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1356\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_get_listlike_indexer\u001b[0;34m(self, key, axis, raise_missing)\u001b[0m\n\u001b[1;32m   1159\u001b[0m         self._validate_read_indexer(keyarr, indexer,\n\u001b[1;32m   1160\u001b[0m                                     \u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_axis_number\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1161\u001b[0;31m                                     raise_missing=raise_missing)\n\u001b[0m\u001b[1;32m   1162\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1163\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/hands-on-machine-learning-KC8V5HHm/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_validate_read_indexer\u001b[0;34m(self, key, indexer, axis, raise_missing)\u001b[0m\n\u001b[1;32m   1250\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'loc'\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mraise_missing\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1251\u001b[0m                 \u001b[0mnot_found\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1252\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"{} not in index\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnot_found\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1254\u001b[0m             \u001b[0;31m# we skip the warning on Categorical/Interval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: \"['AgeBucket', 'RelativesOnboard'] not in index\""
     ]
    }
   ],
   "source": [
    "X_test = preprocess_pipeline.transform(test_data)\n",
    "y_pred = svm_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7365250822835092"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "svm_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, n_jobs=-1)\n",
    "svm_scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8149526160481217"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# RandomForest\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_clf = RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "forest_scores = cross_val_score(forest_clf, X_train, y_train, cv=10)\n",
    "forest_scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf8AAAD8CAYAAAB5Eq2kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGlhJREFUeJzt3X+U3XV95/HnO5MEdBVNTOrpQpIJGnEw2iBjQCkqpQpL16R2V01kNbQqh1bogrbH6GCh6FioR2HdE1fRRVKrww93K1MPK9oFJdsawqT8zhQIMZDQdpuQYauCCZl57x/f78hlnJCb5P6Y3O/zcc49934/31/vcEhe9/v5fu7nG5mJJEmqjmntLkCSJLWW4S9JUsUY/pIkVYzhL0lSxRj+kiRVjOEvSVLFGP6SJFWM4S9JUsUY/pIkVcz0dhfQLHPmzMnu7u52lyFJUsts3LhxZ2bO3d92HRv+3d3dDA0NtbsMSZJaJiIerWc7u/0lSaoYw1+SpIox/CVJqhjDX5KkijH8JUmqGMNfkqSKMfwlSYVtG2Dd54p3dbSO/Z2/JOkAbNsAa5fB6B7omgmrBmHe0nZXpSYx/CWpQiKivg0/edLzrs7MBlSjdrHbX5IqJDMnfz12B/mplxfbfOrlxfK+tjX4D3uGvySp6OJfNVh8tsu/4xn+kqTCeOAb/B3P8JckqWIMf0mSKsbwlySpYgx/SZIqxvCXJKliDH9JkirG8JckqWIMf0mSKsbwlySpYgx/SZIqxvCXJKliDH9JkirG8JckqWIMf0mSKsbwlySpYgx/SZIqxvCXJKliDH9JkirG8JckqWIMf0mSKsbwlySpYgx/SZIqxvCXJKliDH9JkirG8JckFbZteO67OlZLwz8izoyIByNic0SsnmT9/Ii4LSLuioh7I+Kssr07Ip6OiLvL15daWbckdbxtG2DtsuLz2mV+Aehw01t1oojoAtYAbwO2A3dGxGBmbqrZ7GLghsz8bxFxPHAz0F2ueyQzl7SqXkmqlK3rYHRP8Xl0T7E8b2l7a1LTtPLKfymwOTO3ZOYe4Dpg+YRtEjiq/PwS4B9bWJ8kVVf3qdA1s/jcNbNYVsdqZfgfDWyrWd5ettW6FPhPEbGd4qr/gpp1C8vbAT+MiEn/r4yIcyNiKCKGduzY0cDSJanDzVsKqwaLz6sGvervcFNtwN9K4NrMPAY4C/h6REwD/gmYn5knAB8BvhkRR03cOTOvzszezOydO3duSwuXpMPeeOAb/B2vleH/ODCvZvmYsq3WB4AbADLzR8CRwJzM3J2ZT5TtG4FHgFc1vWJJkjpQK8P/TmBRRCyMiJnACmBwwjaPAacDREQPRfjviIi55YBBIuJYYBGwpWWVS5LUQVo22j8z90bE+cAtQBdwTWY+EBGXAUOZOQh8FPhKRFxEMfjvnMzMiHgzcFlEPAOMAedl5q5W1S5JUieJzGx3DU3R29ubQ0ND7S5Dkg4f2zYQ808iH7vD+/6HqYjYmJm9+9tuqg34kyS1g5P8VIrhL0mafJIfdSzDX5LkJD8V07IBf5Kk5ps9ezYjIyOHdIz45P+FT5500PvPmjWLXbsckz2VGf6S1EFGRkZo90DuiGjr+bV/dvtLklQxhr8kSRVj+EuSVDGGvySpMHQtfP2dxbs6muGvKWVgYIDFixfT1dXF4sWLGRgYaHdJUjUMXQvf+c/wyK3Fu18AOpqj/TVlDAwM0NfXx8f6r2L3nEUcsfNh+vouBGDlypVtrk7qcMM3/fJy7zltKUXN55W/poz+/n4+1n8VV26awVW3buHKTTP4WP9V9Pf3t7s0qfP1LH/+ZXUUr/w1ZQwPD7N7ziL27N3CWMIze8fYPWcRw8PD7S5N6nzjV/nDNxXB71V/RzP8NWX09PRwxM6HmTl9Bs/sHWPG9GkcsfNhenp62l2aVA295xj6FWH4a8ro6+ujr+/C59zzv6LvQrv9JanBDH9NGeOD+vr7P8Hw8DA9PT309/c72E86AHnJUXDpS9pfg6a0aPcc0M3S29ubQ0ND7S5DkloqIqbE3P7trqGqImJjZvbubztH+0uSVDGGvySpsG0DrPtc8a6O5j1/SVIR+GuXwege6JoJqwZh3tJ2V6Um8cpfkgRb1xXBn6PF+9Z17a5ITWT4S5Kg+9Tiij+6ivfuU9tdkZrIbn9JUtHFv2qwuOLvPtUu/w5n+EuSCvOWGvoVYbe/JEkVY/hLklQxhr8kSRVj+EuSVDGGvyRJFWP4S5JUMYa/JEkVU1f4R8RVEbG42cVIkqTmq/fK/w3APRGxISLOjYgXN7MoSZLUPHWFf2aeAhwP3AZcAvxTRPxFRLzlQE4WEWdGxIMRsTkiVk+yfn5E3BYRd0XEvRFxVs26j5f7PRgRZxzIeSVJ0rPqvuefmQ9m5seAecAK4EXA9yLi4YhYHRGzn2//iOgC1gD/juKLxMqIOH7CZhcDN2TmCeU5vljue3y5/BrgTOCL5fEkSdIBOpgBfzOAo4CXAF3AY8D7gMci4r3Ps99SYHNmbsnMPcB1wPIJ22R5bMrj/2P5eTlwXWbuzswfA5vL40mSGmXbBlj3ueJdHa3uB/tERC/wexRX4E8Ba4EPlmFMRPw+cCXwzX0c4mhgW83yduCkCdtcStGbcAHwb4DfrNl3/YR9j663dknSfmzbAGuXweie4pG+qwZ9yE8Hq3e0/33A31F0+Z8DLMjMvvHgL90IzD3EelYC12bmMcBZwNcjou7eiXIw4lBEDO3YseMQS5GkCtm6rgj+HC3et65rd0VqonqD9QZgYWa+IzMHM3N04gaZuTMzn+94j1N8eRh3TNlW6wPlucjMHwFHAnPq3JfMvDozezOzd+7cQ/0eIkkV0n1qccUfXcV796ntrkhNVG/4XwE8MbExIo6MiJl1HuNOYFFELCz3WQEMTtjmMeD08tg9FOG/o9xuRUQcERELgUWAN6UkqVHmLS26+n+jzy7/Cqj3nv+NwA+Bz09oPw94K/Db+ztAZu6NiPOBWygGCl6TmQ9ExGXAUGYOAh8FvhIRF1EM/jsnMxN4ICJuADYBe4EPT9b7IEk6BPOWGvoVEUW27mejiJ3AWzPz/gntrwFuy8xfaVJ9B623tzeHhobaXYYktVREUM+/651eQ1VFxMbM7N3fdvV2+7+Q4op7ojHA2f4kSTqM1Bv+91KMxJ/ovcD9k7RLkqQpqt57/pcBN0XEK4Fby7bTgXcB72xGYZIkqTnqndv/ZuAdwALgC+VrPrAsM7/TvPIkSVKj1T3DX2Z+F/huE2uRJEktcDBz+0uSpMNYvdP7zoyIP42IhyLi5xExWvtqdpGSJKlx6r3y/xSwCvgcxc/7/pji8bxPAH/QnNIkSVIz1Bv+7wbOy8wvA6PATZn5h8AlwNuaVZwkSWq8esP/5RRT6wL8FHhp+fm7wNsbXZQkSWqeesP/MeDflp83A2eUn98IPN3ooiRJUvPU+1O/v6KY1Gc98F+AgYj4EHA08Nkm1SZJOggR0dbzz5o1q63n1/7VFf6Z+fGaz9+KiG3AKcBDTvIjSVPHoT5Qx4fyVMN+wz8iZgB/CXwiMx8ByMw7gDuaXJskSWqC/d7zz8xnKAb1+VVQkjrZtg3PfVfHqnfA3/8EfqeZhUiS2mjbBli7rPi8dplfADpcvQP+HgMujohTgSHgZ7UrM/PzjS5MktRCW9fB6J7i8+ieYnne0vbWpKapN/zPAUaA15WvWgkY/pJ0OOs+FbpmFp+7ZhbL6lj1jvZf2OxCJEltNG8prBqET55UvHvV39F8qp8kqTAe+AZ/x6vryj8ivvB868t5/iVJ0mGg3nv+r52wPAN4NdAF3NXQiiRJUlPVe8//tIltEXEk8N+BdY0uSpIkNc9B3/PPzJ8DnwH6GleOJElqtkMd8DcHeFEjCpHGbXx0hDW3bWbjoyPtLkWqFmf4q4x6B/x9ZGIT8KvA2cDNjS5K1bXx0RHO/up69uwdY+b0aXzjgydz4gKfECY13cQZ/vy5X0erd8DfBROWx4AdwNeAP2toRaqEeh852vvpfa/zyWNSAznDX6XU1e2fmQsnvF6RmSdn5icy8yfNLlKdJzMnfQ1t3cVxFxedScddfDNDW3ftc1tJDeQMf5US9fwjGhEzgWnlIL/a9iOBsczc06T6Dlpvb28ODQ21uwwdhI2PjtDbPZuhrbvs8pdaadsGYv5J5GN3eNV/mIqIjZnZu7/t6h3wdyPwB5O0nwfccCCFSfszHvgGv9RizvBXGfWG/ynA9yZp/z7wpsaVI0mSmq3e8H8hsHeS9jHgxY0rR5IkNVu94X8vsHKS9vcC9zeuHEmS1Gz1/tTvMuCmiHglcGvZdjrwLuCdzShM1XX5zcO/eF99Vk+bq5GkzlPvT/1uBt4BLAC+UL7mA8sy8zv1niwizoyIByNic0SsnmT9lRFxd/l6KCKerFk3WrNusN5z6vBy+c3DfOn2LQB86fYtv/giIElqnLp+6teQE0V0AQ8BbwO2A3cCKzNz0z62vwA4ITN/r1z+aWbWPZWwP/Vrn9mzZzMy0v6peWfNmsWuXbvaXYZ0WIkI59E4jNX7U796p/d9C0Bm/nCS9szM2+s4zFJgc2ZuKfe9DlgOTBr+FGMMLqmnPk0tIyMjB/2PR+2VP8B5bz72oLv+651FUJKqpt4Bf1cCk/3o+qhyXT2OBrbVLG8v235JRCwAFvLs+AKAIyNiKCLWR8Rv72O/c8tthnbs2FFnWZpKVp/Vw3lvPpbul73wkIJfkrRv9Q74Ow64Z5L2+8t1jbYC+FZmjta0LcjMxyPiWODWiLgvMx+p3SkzrwauhqLbvwl1qQVWn9Vj6EtSE9V75f80xVP8JjoaqHdq38eBeTXLx5Rtk1kBDNQ2ZObj5fsW4AfACXWeV5Ik1aj3yv8W4IqIWJaZIwARMZviiX631HmMO4FFEbGQIvRXUMwT8BwR8WqKWww/qmmbBTyVmbsjYg7FjIN/Xud51WJ5yVFw6UvaXUZRh6T6bdvw7LtT/Ha0esP/j4Dbga0RcW/Z9jqKx/q+p54DZObeiDif4stCF3BNZj4QEZcBQ5k5/vO9FcB1+dwRYz3AlyNijKK34vJ9/UpA7Rd/+q9TYrRwRJCXtrsK6TCxbQOsXVZ8XrsMVg36BaCD1f1Tv4h4IXA2sKRsugv4BnBKZv5Nc8o7eP7Ur32myk+Fpkod0lTSqF/B+HdramroT/0AMvMp4CvlwY8GfpdiwF83xZW81BAbHx1h/ZYnOPnYl/lkP6nB9hna41f+o3uga6ZX/h2u7vAvJ+lZDnwAeDvFfP9fpnjcr9QQGx8d4eyvrmfP3jFmTp/GNz54sl8ApFaYt7QI/K3roPtUg7/D7Tf8I+I44IPA+4GfAd8EzgDe5313Ndr6LU+wZ+8YYwnP7B1j/ZYnDH+pVeYtNfQr4nl/6hcR64D1FKPv352Zx2bmxYA3e9QUJx/7MmZOn0ZXwIzp0zj52Je1uyRJ6jj7u/J/I7AGuDozH2hBPaq4ExfM4hsfPNl7/pLURPsL/zdQdPn/n4jYCvwFEybfkRrtxAWzDH1JaqLn7fbPzLsy88MUs/t9HlhGMT//NOC3ysl3JEnSYaSu6X0z8+eZ+fXMPI1iwp3PAhcB/xwR/6uZBUqSpMaqd27/X8jMzZm5mmKe/ndT/9z+kiRpCjjg8B+XmaOZeVNmLm9kQdLGR0dYc9tmNj460u5SJKkj1T3Jj9QKTvIjSc130Ff+UjNMNsmPJKmxDH9NKU7yI0nNZ7e/phQn+ZGk5jP8NeU4yY8kNZfd/pIkVYzhL0lSxRj+kiRVjPf81RQR0e4SmDXLcQOSNBmv/NVwmXlIr6GtuwAY2rrrkI6za9euNv+XkKSpyfDXlDI+wx/A2V9d7xS/ktQEdvurLeq5LfDgp8+i99P7Xp+ZDaxIkqrD8Fdb7Cu4x6/8n9k7xgzn9pekpjD8NaU4w58kNZ/hrynHGf4kqbkc8CdJUsUY/pIkVYzhL0lSxRj+kiRVjOEvSVLFGP6SJFWM4S9JUsUY/pIkVUxLwz8izoyIByNic0SsnmT9lRFxd/l6KCKerFm3KiIeLl+rWlm3JEmdpGUz/EVEF7AGeBuwHbgzIgYzc9P4Npl5Uc32FwAnlJ9nA5cAvUACG8t9feSbJEkHqJVX/kuBzZm5JTP3ANcBy59n+5XAQPn5DOD7mbmrDPzvA2c2tVpJkjpUK8P/aGBbzfL2su2XRMQCYCFw64HuK0mSnt9UHfC3AvhWZo4eyE4RcW5EDEXE0I4dO5pUmiRJh7dWhv/jwLya5WPKtsms4Nku/7r3zcyrM7M3M3vnzp17iOVKktSZWhn+dwKLImJhRMykCPjBiRtFxKuBWcCPappvAd4eEbMiYhbw9rJNkiQdoJaN9s/MvRFxPkVodwHXZOYDEXEZMJSZ418EVgDXZWbW7LsrIj5F8QUC4LLM3NWq2iVJ6iRRk7Edpbe3N4eGhtpdhiRJLRMRGzOzd3/bTdUBf5IkqUkMf0mSKsbwlySpYgx/SZIqxvCXJKliDH9JkirG8JckqWIMf0mSKsbwlySpYgx/SZIqxvCXJKliDH9JkirG8JckqWIMf0mSKsbwlySpYgx/SZIqxvCXJKliDH9JkirG8JckqWIMf0mSKsbw15QyMDDA4sWL6erqYvHixQwMDLS7JEnqONPbXYA0bmBggL6+Pj7WfxW75yziiJ0P09d3IQArV65sc3WS1DkiM9tdQ1P09vbm0NBQu8vQAVi8eDEX9H2GKzfNYM/eMWZOn8ZFxz/Df+3/BPfff3+7y5OkKS8iNmZm7/62s9tfU8bw8DC75yxiz94xxhKe2TvG7jmLGB4ebndpktRRDH9NGT09PRyx82FmTp9GV8CM6dM4YufD9PT0tLs0Seoo3vPXlNHX10df34XPued/Rd+F9Pf3t7s0Seoohr+mjPFBff39n2B4eJienh76+/sd7CdJDeaAP0mSOoQD/iRJ0qQMf0mSKsbw15TiDH+S1HwO+NOU4Qx/ktQaDvjTlOEMf5J0aBzwp8OOM/xJUmu0NPwj4syIeDAiNkfE6n1s8+6I2BQRD0TEN2vaRyPi7vI12Lqq1SrO8CdJrdGye/4R0QWsAd4GbAfujIjBzNxUs80i4OPAKZk5EhG/UnOIpzNzSavqVes5w58ktUYrB/wtBTZn5haAiLgOWA5sqtnmQ8CazBwByMx/aWF9ajNn+JOk1mhl+B8NbKtZ3g6cNGGbVwFExN8CXcClmfndct2RETEE7AUuz8xvN7letcHKlSsNe0lqsqn2U7/pwCLgrcAxwO0R8drMfBJYkJmPR8SxwK0RcV9mPlK7c0ScC5wLMH/+/NZWLknSYaKVA/4eB+bVLB9TttXaDgxm5jOZ+WPgIYovA2Tm4+X7FuAHwAkTT5CZV2dmb2b2zp07t/F/AkmSOkArw/9OYFFELIyImcAKYOKo/W9TXPUTEXMobgNsiYhZEXFETfspPHesgCRJqlPLuv0zc29EnA/cQnE//5rMfCAiLgOGMnOwXPf2iNgEjAJ/nJlPRMSbgC9HxBjFF5bLa38lIEmS6ucMf5IkdYh6Z/jr2PCPiB3Ao+2uQwdtDrCz3UVIFeTfvcPbgszc76C3jg1/Hd4iYqieb6+SGsu/e9Xg3P6SJFWM4S9JUsUY/pqqrm53AVJF+XevArznL0lSxXjlL0lSxRj+armI6IuIByLi3oi4OyIuiYg/m7DNkogYLj9vjYh1E9bfHRH3t7JuqdEiYnT8/+WI+OuIeGmDjtvdjL8fEXFpRDxe1nx3RFze6HPUnGtJRJzVrONXneGvloqINwL/Hnh9Zr4O+E3gNuA9EzZdAQzULL84IuaVx+hpRa1SCzydmUsyczGwC/hwuwuqw5VlzUsyc3W9O0VE1wGeZwlg+DeJ4a9W+1VgZ2buBsjMnZl5OzASEbWPeH43zw3/G3j2C8LKCeukTvAjikefExEvioj/HRF/HxH3RcTysr07IoYj4itl79n3IuIF5boTI+KeiLiHmi8REXFkRHytPM5dEXFa2X5ORHw7Ir5f9q6dHxEfKbdZHxGz6y08Ik4v97svIq6peRbL1oi4IiL+HnhXRLwiIr4bERsjYl1EvLrc7l1l78c9EXF7+fyXy4D3lD0MEy8OdIgMf7Xa94B5EfFQRHwxIt5Stg9QXO0TEScDuzLz4Zr9/gfwO+XndwB/3aqCpWYrr4pP59mHnf0ceGdmvh44DfhcRES5bhGwJjNfAzwJ/Iey/WvABZn5axMO/2EgM/O1FF+c10bEkeW6xRR/r94A9ANPZeYJFF9E3r+Pci+q6fY/ozzWtcB7ynNMB36/ZvsnMvP1mXkdxS8JLsjME4E/Ar5YbvMnwBll7csyc0/Zdn3Zw3D9fv4T6gAZ/mqpzPwpcCJwLrADuD4izgGuB/5jREzjl7v8AZ6g6B1YAQwDT7WsaKl5XhARdwP/DLwc+H7ZHsBnIuJe4G8oegReXq77cWbeXX7eCHSXYwVeWvaiAXy95hy/DvwlQGb+A8W0568q192WmT/JzB3A/+PZL9X3Ad37qLm22/8W4LiypofK9WuBN9dsfz0UvRnAm4Abyz/zlyl6AgH+Frg2Ij5E8eA3NVnLnuonjcvMUeAHwA8i4j5gVWZeGxE/Bt5CcSXzxkl2vR5YA5zTolKlZns6M5dExAspnmr6YeALwNnAXODEzHwmIrYC41fru2v2HwVecAjnrz3WWM3yGI3Lh5+V79OAJzNzycQNMvO88rbfbwEbI+LEBp1b++CVv1oqIo6LiEU1TUt49gFMA8CVwJbM3D7J7n8F/DnFP5JSx8jMp4A/BD4aEdOBlwD/Ugb/acCC/ez/JPBkRPx62XR2zep148sR8SpgPvBgA8t/kKL34ZXl8vuAH05S478CP46Id5W1RET8Wvn5FZl5R2b+CUWP4DzgJ8CLG1inahj+arUXUdxz3FR2aR4PXFquuxF4DfsYzFd2T15R3g+UOkpm3gXcS3Ff/htAb9kz9n7gH+o4xO8Ca8ou9ahp/yIwrTzW9cA54wNuG1T3z8tz31ieYwz40j42Pxv4QDko8QFgedn+2XKw4P3A3wH3UPwK6HgH/DWHM/xJklQxXvlLklQxhr8kSRVj+EuSVDGGvyRJFWP4S5JUMYa/JEkVY/hLklQxhr8kSRXz/wFmpeey1qyUZwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "plt.figure(figsize=(8, 4))\n",
    "plt.plot([1]*10, svm_scores, \".\")\n",
    "plt.plot([2]*10, forest_scores, \".\")\n",
    "plt.boxplot([svm_scores, forest_scores], labels=(\"SVM\",\"Random Forest\"))\n",
    "plt.ylabel(\"Accuracy\", fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AgeBucket</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.576923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15.0</th>\n",
       "      <td>0.362745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30.0</th>\n",
       "      <td>0.423256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45.0</th>\n",
       "      <td>0.404494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60.0</th>\n",
       "      <td>0.240000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75.0</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Survived\n",
       "AgeBucket          \n",
       "0.0        0.576923\n",
       "15.0       0.362745\n",
       "30.0       0.423256\n",
       "45.0       0.404494\n",
       "60.0       0.240000\n",
       "75.0       1.000000"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"AgeBucket\"] = train_data[\"Age\"] // 15 * 15\n",
    "train_data[[\"AgeBucket\", \"Survived\"]].groupby(['AgeBucket']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RelativesOnboard</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.303538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.552795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.578431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.724138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.136364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Survived\n",
       "RelativesOnboard          \n",
       "0                 0.303538\n",
       "1                 0.552795\n",
       "2                 0.578431\n",
       "3                 0.724138\n",
       "4                 0.200000\n",
       "5                 0.136364\n",
       "6                 0.333333\n",
       "7                 0.000000\n",
       "10                0.000000"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"RelativesOnboard\"] = train_data[\"SibSp\"] + train_data[\"Parch\"]\n",
    "train_data[[\"RelativesOnboard\", \"Survived\"]].groupby(['RelativesOnboard']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_pipeline = Pipeline([\n",
    "        (\"select_numeric\", DataFrameSelector([\"AgeBucket\", \"RelativesOnboard\", \"Fare\"])),\n",
    "        (\"imputer\", SimpleImputer(strategy=\"median\")),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import FeatureUnion\n",
    "\n",
    "preprocess_pipeline = FeatureUnion(transformer_list=[\n",
    "        (\"num_pipeline\", num_pipeline),\n",
    "        (\"cat_pipeline\", cat_pipeline),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8182606968562025"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = preprocess_pipeline.fit_transform(train_data)\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_clf = RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "forest_scores = cross_val_score(forest_clf, X_train, y_train, cv=10)\n",
    "forest_scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=None,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'n_estimators': [10, 40, 70, 100, 130]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = {'n_estimators': [10, 40, 70, 100, 130]}\n",
    "grid_search = GridSearchCV(forest_clf, param_grid, cv=10, n_jobs=-1, scoring='accuracy')\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8260381593714927 {'n_estimators': 10}\n",
      "0.8125701459034792 {'n_estimators': 40}\n",
      "0.8103254769921436 {'n_estimators': 70}\n",
      "0.8181818181818182 {'n_estimators': 100}\n",
      "0.813692480359147 {'n_estimators': 130}\n"
     ]
    }
   ],
   "source": [
    "cvres = grid_search.cv_results_\n",
    "for mean_score, params in zip(cvres['mean_test_score'], cvres['params']):\n",
    "    print(mean_score, params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7532152990579956"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_scores = cross_val_score(knn_clf, X_train, y_train, cv=10)\n",
    "knn_scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=None,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'n_estimators': [10, 40, 70, 100, 130], 'max_features': [4, 5, 10], 'bootstrap': [True, False]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = {'n_estimators': [10, 40, 70, 100, 130], 'max_features': [4, 5, 10], 'bootstrap': [True, False]}\n",
    "grid_search = GridSearchCV(forest_clf, param_grid, cv=10, n_jobs=-1, scoring='accuracy')\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8237934904601572 {'bootstrap': True, 'max_features': 4, 'n_estimators': 10}\n",
      "0.813692480359147 {'bootstrap': True, 'max_features': 4, 'n_estimators': 40}\n",
      "0.8125701459034792 {'bootstrap': True, 'max_features': 4, 'n_estimators': 70}\n",
      "0.8170594837261503 {'bootstrap': True, 'max_features': 4, 'n_estimators': 100}\n",
      "0.813692480359147 {'bootstrap': True, 'max_features': 4, 'n_estimators': 130}\n",
      "0.819304152637486 {'bootstrap': True, 'max_features': 5, 'n_estimators': 10}\n",
      "0.8125701459034792 {'bootstrap': True, 'max_features': 5, 'n_estimators': 40}\n",
      "0.8148148148148148 {'bootstrap': True, 'max_features': 5, 'n_estimators': 70}\n",
      "0.8170594837261503 {'bootstrap': True, 'max_features': 5, 'n_estimators': 100}\n",
      "0.8148148148148148 {'bootstrap': True, 'max_features': 5, 'n_estimators': 130}\n",
      "0.8282828282828283 {'bootstrap': True, 'max_features': 10, 'n_estimators': 10}\n",
      "0.8226711560044894 {'bootstrap': True, 'max_features': 10, 'n_estimators': 40}\n",
      "0.8237934904601572 {'bootstrap': True, 'max_features': 10, 'n_estimators': 70}\n",
      "0.8271604938271605 {'bootstrap': True, 'max_features': 10, 'n_estimators': 100}\n",
      "0.8305274971941639 {'bootstrap': True, 'max_features': 10, 'n_estimators': 130}\n",
      "0.8058361391694725 {'bootstrap': False, 'max_features': 4, 'n_estimators': 10}\n",
      "0.8058361391694725 {'bootstrap': False, 'max_features': 4, 'n_estimators': 40}\n",
      "0.8013468013468014 {'bootstrap': False, 'max_features': 4, 'n_estimators': 70}\n",
      "0.8024691358024691 {'bootstrap': False, 'max_features': 4, 'n_estimators': 100}\n",
      "0.7991021324354658 {'bootstrap': False, 'max_features': 4, 'n_estimators': 130}\n",
      "0.8035914702581369 {'bootstrap': False, 'max_features': 5, 'n_estimators': 10}\n",
      "0.8080808080808081 {'bootstrap': False, 'max_features': 5, 'n_estimators': 40}\n",
      "0.8069584736251403 {'bootstrap': False, 'max_features': 5, 'n_estimators': 70}\n",
      "0.8069584736251403 {'bootstrap': False, 'max_features': 5, 'n_estimators': 100}\n",
      "0.8069584736251403 {'bootstrap': False, 'max_features': 5, 'n_estimators': 130}\n",
      "0.813692480359147 {'bootstrap': False, 'max_features': 10, 'n_estimators': 10}\n",
      "0.8148148148148148 {'bootstrap': False, 'max_features': 10, 'n_estimators': 40}\n",
      "0.8125701459034792 {'bootstrap': False, 'max_features': 10, 'n_estimators': 70}\n",
      "0.8148148148148148 {'bootstrap': False, 'max_features': 10, 'n_estimators': 100}\n",
      "0.813692480359147 {'bootstrap': False, 'max_features': 10, 'n_estimators': 130}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features=10, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=130, n_jobs=None,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvres = grid_search.cv_results_\n",
    "for mean_score, params in zip(cvres['mean_test_score'], cvres['params']):\n",
    "    print(mean_score, params)\n",
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_forest_clf = RandomForestClassifier(**grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RelativesOnboard</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1100.438735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1090.243243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1115.824561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1109.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1079.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>951.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1122.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1031.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1205.750000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  PassengerId\n",
       "RelativesOnboard             \n",
       "0                 1100.438735\n",
       "1                 1090.243243\n",
       "2                 1115.824561\n",
       "3                 1109.571429\n",
       "4                 1079.428571\n",
       "5                  951.000000\n",
       "6                 1122.250000\n",
       "7                 1031.500000\n",
       "10                1205.750000"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()\n",
    "test_data[\"AgeBucket\"] = test_data[\"Age\"] // 15 * 15\n",
    "test_data[[\"AgeBucket\", \"PassengerId\"]].groupby(['AgeBucket']).mean()\n",
    "test_data[\"RelativesOnboard\"] = test_data[\"SibSp\"] + test_data[\"Parch\"]\n",
    "test_data[[\"RelativesOnboard\", \"PassengerId\"]].groupby(['RelativesOnboard']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1,\n",
       "       1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1,\n",
       "       1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1,\n",
       "       1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0,\n",
       "       1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1,\n",
       "       0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1,\n",
       "       1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0,\n",
       "       0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0,\n",
       "       1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1,\n",
       "       0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1,\n",
       "       0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1,\n",
       "       1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0,\n",
       "       1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0,\n",
       "       0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0,\n",
       "       1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1,\n",
       "       0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0])"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = preprocess_pipeline.transform(test_data)\n",
    "best_forest_clf.fit(X_train, y_train)\n",
    "best_forest_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame()\n",
    "df['PassengerId'] = test_data['PassengerId']\n",
    "df['Survived'] = best_forest_clf.predict(X_test)\n",
    "df.to_csv('datasets/titanic/results.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4번\n",
    "import requests\n",
    "import os\n",
    "\n",
    "DOWNLOAD_ROOT = \"http://spamassassin.apache.org/old/publiccorpus/\"\n",
    "HAM_URL = DOWNLOAD_ROOT + \"20030228_easy_ham.tar.bz2\"\n",
    "SPAM_URL = DOWNLOAD_ROOT + \"20030228_spam.tar.bz2\"\n",
    "SPAM_PATH = os.path.join(\"datasets\", \"spam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tarfile\n",
    "\n",
    "def fetch_remote_data(url, path, filename):\n",
    "    if not os.path.isdir(path):\n",
    "        os.makedirs(path)\n",
    "    tgz_path = os.path.join(path, filename)\n",
    "    resp = requests.get(url, stream=True, verify=False)\n",
    "    with open(tgz_path, 'wb') as f:\n",
    "        f.write(resp.content)\n",
    "    tgz_file = tarfile.open(tgz_path)\n",
    "    tgz_file.extractall(path=path)\n",
    "    tgz_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'fetch_remote_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-7e3680a84efc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfetch_remote_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mHAM_URL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSPAM_PATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ham.tgz'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mfetch_remote_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSPAM_URL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSPAM_PATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'spam.tgz'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'fetch_remote_data' is not defined"
     ]
    }
   ],
   "source": [
    "fetch_remote_data(HAM_URL, SPAM_PATH, 'ham.tgz')\n",
    "fetch_remote_data(SPAM_URL, SPAM_PATH, 'spam.tgz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "HAM_DIR = os.path.join(SPAM_PATH, \"easy_ham\")\n",
    "SPAM_DIR = os.path.join(SPAM_PATH, \"spam\")\n",
    "ham_filenames = [name for name in sorted(os.listdir(HAM_DIR)) if len(name) > 20]\n",
    "spam_filenames = [name for name in sorted(os.listdir(SPAM_DIR)) if len(name) > 20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2500\n",
      "500\n"
     ]
    }
   ],
   "source": [
    "print(len(ham_filenames))\n",
    "print(len(spam_filenames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import email\n",
    "import email.policy\n",
    "\n",
    "def load_email(is_spam, filename, spam_path=SPAM_PATH):\n",
    "    directory = \"spam\" if is_spam else \"easy_ham\"\n",
    "    with open(os.path.join(spam_path, directory, filename), \"rb\") as f:\n",
    "        return email.parser.BytesParser(policy=email.policy.default).parse(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ham_emails = [load_email(is_spam=False, filename=name) for name in ham_filenames]\n",
    "spam_emails = [load_email(is_spam=True, filename=name) for name in spam_filenames]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Martin A posted:\n",
      "Tassos Papadopoulos, the Greek sculptor behind the plan, judged that the\n",
      " limestone of Mount Kerdylio, 70 miles east of Salonika and not far from the\n",
      " Mount Athos monastic community, was ideal for the patriotic sculpture. \n",
      " \n",
      " As well as Alexander's granite features, 240 ft high and 170 ft wide, a\n",
      " museum, a restored amphitheatre and car park for admiring crowds are\n",
      "planned\n",
      "---------------------\n",
      "So is this mountain limestone or granite?\n",
      "If it's limestone, it'll weather pretty fast.\n",
      "\n",
      "------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "4 DVDs Free +s&p Join Now\n",
      "http://us.click.yahoo.com/pt6YBB/NXiEAA/mG3HAA/7gSolB/TM\n",
      "---------------------------------------------------------------------~->\n",
      "\n",
      "To unsubscribe from this group, send an email to:\n",
      "forteana-unsubscribe@egroups.com\n",
      "\n",
      " \n",
      "\n",
      "Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n"
     ]
    }
   ],
   "source": [
    "print(ham_emails[1].get_content().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_email_structure(email):\n",
    "    if isinstance(email, str):\n",
    "        return email\n",
    "    \n",
    "    payload = email.get_payload()\n",
    "    if isinstance(payload, list):\n",
    "        return \"multipart({})\".format(\", \".join([\n",
    "            get_email_structure(sub_email)\n",
    "            for sub_email in payload\n",
    "        ]))\n",
    "    \n",
    "    else:\n",
    "        return email.get_content_type()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def structures_counter(emails):\n",
    "    structures = Counter()\n",
    "    for email in emails:\n",
    "        structure = get_email_structure(email)\n",
    "        structures[structure] += 1\n",
    "    return structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 2408),\n",
       " ('multipart(text/plain, application/pgp-signature)', 66),\n",
       " ('multipart(text/plain, text/html)', 8),\n",
       " ('multipart(text/plain, text/plain)', 4),\n",
       " ('multipart(text/plain)', 3),\n",
       " ('multipart(text/plain, application/octet-stream)', 2),\n",
       " ('multipart(text/plain, text/enriched)', 1),\n",
       " ('multipart(text/plain, application/ms-tnef, text/plain)', 1),\n",
       " ('multipart(multipart(text/plain, text/plain, text/plain), application/pgp-signature)',\n",
       "  1),\n",
       " ('multipart(text/plain, video/mng)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain))', 1),\n",
       " ('multipart(text/plain, application/x-pkcs7-signature)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), text/rfc822-headers)',\n",
       "  1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), multipart(multipart(text/plain, application/x-pkcs7-signature)))',\n",
       "  1),\n",
       " ('multipart(text/plain, application/x-java-applet)', 1)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(ham_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 218),\n",
       " ('text/html', 183),\n",
       " ('multipart(text/plain, text/html)', 45),\n",
       " ('multipart(text/html)', 20),\n",
       " ('multipart(text/plain)', 19),\n",
       " ('multipart(multipart(text/html))', 5),\n",
       " ('multipart(text/plain, image/jpeg)', 3),\n",
       " ('multipart(text/html, application/octet-stream)', 2),\n",
       " ('multipart(text/plain, application/octet-stream)', 1),\n",
       " ('multipart(text/html, text/plain)', 1),\n",
       " ('multipart(multipart(text/html), application/octet-stream, image/jpeg)', 1),\n",
       " ('multipart(multipart(text/plain, text/html), image/gif)', 1),\n",
       " ('multipart/alternative', 1)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(spam_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Return-Path : <12a1mailbot1@web.de>\n",
      "Delivered-To : zzzz@localhost.spamassassin.taint.org\n",
      "Received : from localhost (localhost [127.0.0.1])\tby phobos.labs.spamassassin.taint.org (Postfix) with ESMTP id 136B943C32\tfor <zzzz@localhost>; Thu, 22 Aug 2002 08:17:21 -0400 (EDT)\n",
      "Received : from mail.webnote.net [193.120.211.219]\tby localhost with POP3 (fetchmail-5.9.0)\tfor zzzz@localhost (single-drop); Thu, 22 Aug 2002 13:17:21 +0100 (IST)\n",
      "Received : from dd_it7 ([210.97.77.167])\tby webnote.net (8.9.3/8.9.3) with ESMTP id NAA04623\tfor <zzzz@spamassassin.taint.org>; Thu, 22 Aug 2002 13:09:41 +0100\n",
      "From : 12a1mailbot1@web.de\n",
      "Received : from r-smtp.korea.com - 203.122.2.197 by dd_it7  with Microsoft SMTPSVC(5.5.1775.675.6);\t Sat, 24 Aug 2002 09:42:10 +0900\n",
      "To : dcek1a1@netsgo.com\n",
      "Subject : Life Insurance - Why Pay More?\n",
      "Date : Wed, 21 Aug 2002 20:31:57 -1600\n",
      "MIME-Version : 1.0\n",
      "Message-ID : <0103c1042001882DD_IT7@dd_it7>\n",
      "Content-Type : text/html; charset=\"iso-8859-1\"\n",
      "Content-Transfer-Encoding : quoted-printable\n"
     ]
    }
   ],
   "source": [
    "for header, value in spam_emails[0].items():\n",
    "    print(header,\":\",value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = np.array(ham_emails + spam_emails)\n",
    "y = np.array([0] * len(ham_emails) + [1] * len(spam_emails))\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from html import unescape\n",
    "\n",
    "def html_to_plain_text(html):\n",
    "    text = re.sub('<head.*?>.*?</head>', '', html, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<a\\s.*?>', ' HYPERLINK ', text, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<.*?>', '', text, flags=re.M | re.S)\n",
    "    text = re.sub(r'(\\s*\\n)+', '\\n', text, flags=re.M | re.S)\n",
    "    return unescape(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HTML><HEAD><TITLE></TITLE><META http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1252\"><STYLE>A:link {TEX-DECORATION: none}A:active {TEXT-DECORATION: none}A:visited {TEXT-DECORATION: none}A:hover {COLOR: #0033ff; TEXT-DECORATION: underline}</STYLE><META content=\"MSHTML 6.00.2713.1100\" name=\"GENERATOR\"></HEAD>\n",
      "<BODY text=\"#000000\" vLink=\"#0033ff\" link=\"#0033ff\" bgColor=\"#CCCC99\"><TABLE borderColor=\"#660000\" cellSpacing=\"0\" cellPadding=\"0\" border=\"0\" width=\"100%\"><TR><TD bgColor=\"#CCCC99\" valign=\"top\" colspan=\"2\" height=\"27\">\n",
      "<font size=\"6\" face=\"Arial, Helvetica, sans-serif\" color=\"#660000\">\n",
      "<b>OTC</b></font></TD></TR><TR><TD height=\"2\" bgcolor=\"#6a694f\">\n",
      "<font size=\"5\" face=\"Times New Roman, Times, serif\" color=\"#FFFFFF\">\n",
      "<b>&nbsp;Newsletter</b></font></TD><TD height=\"2\" bgcolor=\"#6a694f\"><div align=\"right\"><font color=\"#FFFFFF\">\n",
      "<b>Discover Tomorrow's Winners&nbsp;</b></font></div></TD></TR><TR><TD height=\"25\" colspan=\"2\" bgcolor=\"#CCCC99\"><table width=\"100%\" border=\"0\"  ...\n"
     ]
    }
   ],
   "source": [
    "html_spam_emails = [email for email in X_train[y_train==1]\n",
    "                    if get_email_structure(email) == \"text/html\"]\n",
    "sample_html_spam = html_spam_emails[7]\n",
    "print(sample_html_spam.get_content().strip()[:1000], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "OTC\n",
      " Newsletter\n",
      "Discover Tomorrow's Winners \n",
      "For Immediate Release\n",
      "Cal-Bay (Stock Symbol: CBYI)\n",
      "Watch for analyst \"Strong Buy Recommendations\" and several advisory newsletters picking CBYI.  CBYI has filed to be traded on the OTCBB, share prices historically INCREASE when companies get listed on this larger trading exchange. CBYI is trading around 25 cents and should skyrocket to $2.66 - $3.25 a share in the near future.\n",
      "Put CBYI on your watch list, acquire a position TODAY.\n",
      "REASONS TO INVEST IN CBYI\n",
      "A profitable company and is on track to beat ALL earnings estimates!\n",
      "One of the FASTEST growing distributors in environmental & safety equipment instruments.\n",
      "Excellent management team, several EXCLUSIVE contracts.  IMPRESSIVE client list including the U.S. Air Force, Anheuser-Busch, Chevron Refining and Mitsubishi Heavy Industries, GE-Energy & Environmental Research.\n",
      "RAPIDLY GROWING INDUSTRY\n",
      "Industry revenues exceed $900 million, estimates indicate that there could be as much as $25 billi ...\n"
     ]
    }
   ],
   "source": [
    "print(html_to_plain_text(sample_html_spam.get_content())[:1000], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def email_to_text(email):\n",
    "    html = None\n",
    "    for part in email.walk():\n",
    "        ctype = part.get_content_type()\n",
    "        if not ctype in (\"text/plain\", \"text/html\"):\n",
    "            continue\n",
    "        try:\n",
    "            content = part.get_content()\n",
    "        except: # in case of encoding issues\n",
    "            content = str(part.get_payload())\n",
    "        if ctype == \"text/plain\":\n",
    "            return content\n",
    "        else:\n",
    "            html = content\n",
    "    if html:\n",
    "        return html_to_plain_text(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "OTC\n",
      " Newsletter\n",
      "Discover Tomorrow's Winners \n",
      "For Immediate Release\n",
      "Cal-Bay (Stock Symbol: CBYI)\n",
      "Wat ...\n"
     ]
    }
   ],
   "source": [
    "print(email_to_text(sample_html_spam)[:100], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computations => comput\n",
      "Computation => comput\n",
      "Computing => comput\n",
      "Computed => comput\n",
      "Compute => comput\n",
      "Compulsive => compuls\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "stemmer = nltk.PorterStemmer()\n",
    "for word in (\"Computations\", \"Computation\", \"Computing\", \"Computed\", \"Compute\", \"Compulsive\"):\n",
    "    print(word, \"=>\", stemmer.stem(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['github.com', 'https://youtu.be/7Pq-S557XQU?t=3m32s']\n"
     ]
    }
   ],
   "source": [
    "import urlextract\n",
    "\n",
    "url_extractor = urlextract.URLExtract()\n",
    "print(url_extractor.find_urls(\"Will it detect github.com and https://youtu.be/7Pq-S557XQU?t=3m32s\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class EmailToWordCounterTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, strip_headers=True, lower_case=True, remove_punctuation=True,\n",
    "                 replace_urls=True, replace_numbers=True, stemming=True):\n",
    "        self.strip_headers = strip_headers\n",
    "        self.lower_case = lower_case\n",
    "        self.remove_punctuation = remove_punctuation\n",
    "        self.replace_urls = replace_urls\n",
    "        self.replace_numbers = replace_numbers\n",
    "        self.stemming = stemming\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        X_transformed = []\n",
    "        for email in X:\n",
    "            text = email_to_text(email) or \"\"\n",
    "            \n",
    "            if self.lower_case:\n",
    "                text = text.lower()\n",
    "            \n",
    "            if self.replace_urls and url_extractor is not None:\n",
    "                urls = list(set(url_extractor.find_urls(text)))\n",
    "                urls.sort(key=lambda url: len(url), reverse=True)\n",
    "                for url in urls:\n",
    "                    text = text.replace(url, \" URL \")\n",
    "            \n",
    "            if self.replace_numbers:\n",
    "                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?', 'NUMBER', text)\n",
    "            \n",
    "            if self.remove_punctuation:\n",
    "                text = re.sub(r'\\W+', ' ', text, flags=re.M)\n",
    "            word_counts = Counter(text.split())\n",
    "            \n",
    "            if self.stemming and stemmer is not None:\n",
    "                stemmed_word_counts = Counter()\n",
    "                for word, count in word_counts.items():\n",
    "                    stemmed_word = stemmer.stem(word)\n",
    "                    stemmed_word_counts[stemmed_word] += count\n",
    "                word_counts = stemmed_word_counts\n",
    "            X_transformed.append(word_counts)\n",
    "        return np.array(X_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([Counter({'chuck': 1, 'murcko': 1, 'wrote': 1, 'stuff': 1, 'yawn': 1, 'r': 1}),\n",
       "       Counter({'the': 11, 'of': 9, 'and': 8, 'all': 3, 'christian': 3, 'to': 3, 'by': 3, 'jefferson': 2, 'i': 2, 'have': 2, 'superstit': 2, 'one': 2, 'on': 2, 'been': 2, 'ha': 2, 'half': 2, 'rogueri': 2, 'teach': 2, 'jesu': 2, 'some': 1, 'interest': 1, 'quot': 1, 'url': 1, 'thoma': 1, 'examin': 1, 'known': 1, 'word': 1, 'do': 1, 'not': 1, 'find': 1, 'in': 1, 'our': 1, 'particular': 1, 'redeem': 1, 'featur': 1, 'they': 1, 'are': 1, 'alik': 1, 'found': 1, 'fabl': 1, 'mytholog': 1, 'million': 1, 'innoc': 1, 'men': 1, 'women': 1, 'children': 1, 'sinc': 1, 'introduct': 1, 'burnt': 1, 'tortur': 1, 'fine': 1, 'imprison': 1, 'what': 1, 'effect': 1, 'thi': 1, 'coercion': 1, 'make': 1, 'world': 1, 'fool': 1, 'other': 1, 'hypocrit': 1, 'support': 1, 'error': 1, 'over': 1, 'earth': 1, 'six': 1, 'histor': 1, 'american': 1, 'john': 1, 'e': 1, 'remsburg': 1, 'letter': 1, 'william': 1, 'short': 1, 'again': 1, 'becom': 1, 'most': 1, 'pervert': 1, 'system': 1, 'that': 1, 'ever': 1, 'shone': 1, 'man': 1, 'absurd': 1, 'untruth': 1, 'were': 1, 'perpetr': 1, 'upon': 1, 'a': 1, 'larg': 1, 'band': 1, 'dupe': 1, 'import': 1, 'led': 1, 'paul': 1, 'first': 1, 'great': 1, 'corrupt': 1}),\n",
       "       Counter({'url': 5, 's': 3, 'group': 3, 'to': 3, 'in': 2, 'forteana': 2, 'martin': 2, 'an': 2, 'and': 2, 'we': 2, 'is': 2, 'yahoo': 2, 'unsubscrib': 2, 'y': 1, 'adamson': 1, 'wrote': 1, 'for': 1, 'altern': 1, 'rather': 1, 'more': 1, 'factual': 1, 'base': 1, 'rundown': 1, 'on': 1, 'hamza': 1, 'career': 1, 'includ': 1, 'hi': 1, 'belief': 1, 'that': 1, 'all': 1, 'non': 1, 'muslim': 1, 'yemen': 1, 'should': 1, 'be': 1, 'murder': 1, 'outright': 1, 'know': 1, 'how': 1, 'unbias': 1, 'memri': 1, 'don': 1, 't': 1, 'html': 1, 'rob': 1, 'sponsor': 1, 'number': 1, 'dvd': 1, 'free': 1, 'p': 1, 'join': 1, 'now': 1, 'from': 1, 'thi': 1, 'send': 1, 'email': 1, 'your': 1, 'use': 1, 'of': 1, 'subject': 1})],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few = X_train[:3]\n",
    "X_few_wordcounts = EmailToWordCounterTransformer().fit_transform(X_few)\n",
    "X_few_wordcounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, vocabulary_size=1000):\n",
    "        self.vocabulary_size = vocabulary_size\n",
    "        \n",
    "    def fit(self, X, y=None):\n",
    "        total_count = Counter()\n",
    "        for word_count in X:\n",
    "            for word, count in word_count.items():\n",
    "                total_count[word] += min(count, 10)\n",
    "        most_common = total_count.most_common()[:self.vocabulary_size]\n",
    "        self.most_common_ = most_common\n",
    "        self.vocabulary_ = {word: index + 1 for index, (word, count) in enumerate(most_common)}\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        rows = []\n",
    "        cols = []\n",
    "        data = []\n",
    "        for row, word_count in enumerate(X):\n",
    "            for word, count in word_count.items():\n",
    "                rows.append(row)\n",
    "                cols.append(self.vocabulary_.get(word, 0))\n",
    "                data.append(count)\n",
    "        return csr_matrix((data, (rows, cols)), shape=(len(X), self.vocabulary_size + 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x11 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 20 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer = WordCounterToVectorTransformer(vocabulary_size=10)\n",
    "X_few_vectors = vocab_transformer.fit_transform(X_few_wordcounts)\n",
    "X_few_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 6,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [99, 11,  9,  8,  1,  3,  3,  1,  3,  2,  3],\n",
       "       [65,  0,  1,  2,  5,  3,  1,  2,  0,  1,  0]], dtype=int64)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few_vectors.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'the': 1,\n",
       " 'of': 2,\n",
       " 'and': 3,\n",
       " 'url': 4,\n",
       " 'to': 5,\n",
       " 'all': 6,\n",
       " 'in': 7,\n",
       " 'christian': 8,\n",
       " 'on': 9,\n",
       " 'by': 10}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "preprocess_pipeline = Pipeline([\n",
    "    (\"email_to_wordcount\", EmailToWordCounterTransformer()),\n",
    "    (\"wordcount_to_vector\", WordCounterToVectorTransformer()),\n",
    "])\n",
    "\n",
    "X_train_transformed = preprocess_pipeline.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.1s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  ................................................................\n",
      "[CV] .................................. , score=0.98375, total=   0.1s\n",
      "[CV]  ................................................................\n",
      "[CV] .................................... , score=0.985, total=   0.1s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................... , score=0.9925, total=   0.1s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9870833333333334"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
    "score = cross_val_score(log_clf, X_train_transformed, y_train, cv=3, verbose=3)\n",
    "score.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 94.90%\n",
      "Recall: 97.89%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "X_test_transformed = preprocess_pipeline.transform(X_test)\n",
    "\n",
    "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
    "log_clf.fit(X_train_transformed, y_train)\n",
    "\n",
    "y_pred = log_clf.predict(X_test_transformed)\n",
    "\n",
    "print(\"Precision: {:.2f}%\".format(100 * precision_score(y_test, y_pred)))\n",
    "print(\"Recall: {:.2f}%\".format(100 * recall_score(y_test, y_pred)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}