02-knn_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-Nearest Neighbours from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing required libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading and preprocessing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SepalLength</th>\n",
       "      <th>SepalWidth</th>\n",
       "      <th>PetalLength</th>\n",
       "      <th>PetalWidth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SepalLength  SepalWidth  PetalLength  PetalWidth\n",
       "0          5.1         3.5          1.4         0.2\n",
       "1          4.9         3.0          1.4         0.2\n",
       "2          4.7         3.2          1.3         0.2\n",
       "3          4.6         3.1          1.5         0.2\n",
       "4          5.0         3.6          1.4         0.2"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('iris.data.txt', header=None, names = ['SepalLength', 'SepalWidth', 'PetalLength', 'PetalWidth', 'Species'])\n",
    "y = data.Species\n",
    "X = data.drop(['Species'], axis = 1)\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert target to categorical and then numerical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'Iris-setosa', 1: 'Iris-versicolor', 2: 'Iris-virginica'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = y.astype('category')\n",
    "species_mapping = {k:category for (k, category) in enumerate(list(y.cat.categories.values))}\n",
    "y = y.cat.codes\n",
    "                   \n",
    "species_mapping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Split train-test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((105, 4), (45, 4))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normalise the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = StandardScaler()\n",
    "X_train = s.fit_transform(X_train)\n",
    "X_test = s.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def knn(X_train, y_train, X_test, k, t):\n",
    "    '''\n",
    "    Computes target y for the test set.\n",
    "    Inputs:\n",
    "    X_train - Normalised train data\n",
    "    y_train - Target of the training data\n",
    "    k - No: of nearest neighbours\n",
    "    X_test - Normalised test data\n",
    "    t - Type of problem, either 'classification' or 'regression'\n",
    "    \n",
    "    Output:\n",
    "    y_pred - Test predictions\n",
    "    \n",
    "    '''\n",
    "    y_pred = [0]*len(X_test)\n",
    "    cls = y_train.unique() \n",
    "    for row in range(len(X_test)):\n",
    "        idx = np.argsort(np.linalg.norm(X_test[row] - X_train, axis=1))[0:k]\n",
    "        class_count = {c:0 for c in cls}\n",
    "        if t == 'classification':\n",
    "            for c in cls:\n",
    "                class_count[c] = np.sum(y_train.values[idx] == c)\n",
    "                y_pred[row] = sorted(class_count, key = lambda x: class_count[x], reverse=True)[0]\n",
    "        if t == 'regression':\n",
    "            y_pred[row] = np.mean(y_train.values[idx])\n",
    "    return(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = 'classification'\n",
    "y_pred = knn(X_train, y_train, X_test, 10, t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compute prediction accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97777777777777775"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_pred, y_test.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Choosing an optimal k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = []\n",
    "for i in range(100):\n",
    "    result.append(accuracy_score(knn(X_train, y_train, X_test, i, t), y_test.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x116257a90>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XucVXW9//HXZ24MIzADw4DIDAwD\ng0ZqooPgLUC0g+VPKisvdcxOZV5Iq9Pp2K9Ov5O/LufX6dTJFD3mJa28oGVReY6ZgpkJMmBqgMow\nKDPcBRmQ+9778/tj79luhoHZDLNm7T3r/Xw8eLjX2muv9VluWJ/9Wd/LMndHREQEoCDsAEREJHco\nKYiISJqSgoiIpCkpiIhImpKCiIikKSmIiEiakoKIiKQpKYiISJqSgoiIpBWFHcCRGjp0qNfW1oYd\nhohIXlmyZMmb7l7V1XZ5lxRqa2tpbGwMOwwRkbxiZm9ks51uH4mISJqSgoiIpCkpiIhImpKCiIik\nKSmIiEhaYEnBzO42s01m9rdDvG9mdrOZNZnZS2Z2alCxiIhIdoKsFH4KzDzM+xcA9ak/VwG3BRiL\niIhkIbCk4O5/ArYeZpNZwH2etBCoMLMRQcWTqW33fn61tJWOjyLdH0/wi0VvsGd/POt9/ffL61m1\n+e2D1v9l1ZssXfPWQeuXrWtj/iubjjxoEZFeEGabwkigJWO5NbXuIGZ2lZk1mlnj5s2bj/rAf1i2\ngS/NfZGX17YdsP7JFRv52qN/485nmrPaz8qNO7j2/qV8+eEXD0gwbbv287n7lnDNz5cckGDiCefz\n97/A5362hA1te476PEREelqYScE6WeedrMPd73D3BndvqKrqcpR2l/bGEgAsaj6wkFmYWr7zz6vZ\nsWd/l/u5+akm3OGFNdt4ZuWb6fV3PbuaHXtjbNy+l4cWv5P3fvviOprf3Mm+eILbn1511OchItLT\nwkwKrUBNxnI1sK43DhxPJHPPwuYtB6xf2LyF6sH92bZrP/c9d/gR4Ss37uB3L63jM2eP4bjyUv7z\nj6/h7rTt3s89z65m5ruP5fQxQ5izoIk9++PEE87NT63khGMHcklDDfc/v4aN21UtiEhuCTMpzAOu\nSPVCmgK0ufv63jhwLJUUnl+9NZ0gtu7cxysbdnDppBrOPWEYP3mmmbf3xg65jx8/1UT/4kKunT6O\na6ePY+mabfy56U3u/vNqduyJcf2Mer4wo56N2/cyt7GF3720jubNO7l+Rj2zzx1HIuHctkDVgojk\nliC7pD4APAccb2atZvZpM7vazK5ObfIY0Aw0AT8Brg0qlo7iieTtox17Y6xYvx1IJgiAKXWV3DCj\nnm279nPvX17v9PNNm3bw25fWccUZtQw5poSPNlQzoryUf3/8Ve5+djV/9+7hTDhuEGeMreT02iHM\nmb+Km59cyfHDBzLz3cdSM6SMi0+tVrUgIjknsFlS3f2yLt534Lqgjn848cQ7rxc2b+HEkeUsbN5C\naXEBJ1dXUFJUwPTjq7jzmWZqK4/BOrR+PNzYQmlRIZ89ZwwA/YoKuXbaWP7lN8sAuH5GPQBmxg3n\n1fPxOxcBcOvlp1JQkNzZddPH8cjSVr71+xVccOKxAZ9xsCrKijlz7NCwwxCRHpB3U2f3hPZKoXpw\nfxY2b+Ez59SxsHkLDaOHUFKULJ6+cN54PjjnWa67f2mn+7hm2lgqB/RLL39sUg3/9admTq4u593H\nlafXnzm2kjPqKmnbvf+Ai/+oyjI+1lDNA8+38NsXe6UpJVD3f3ayEoNIHxDJpNDepnDW2KE89rf1\nbHl7L69s2MGX3/fOMIn31FTwzFems3PvwWMWCgzqqgYcsK5fUSH/fcM56aTSzsy4+8pJJNzTVUK7\nb150Ip86awzeaZ+r/BBPOFfe8zw/+uNKJQWRPiCSSSGecAoMzhxXyUONLdyb6mk0ua7ygO2qB5cd\n0X4HlhZ3ur5/SWGn60uKChg/fOARHSMXXT11LDf9bjnPrdrCGWMru/6AiOSsSE6IF0s4RQUFTB6T\nvIDd8+fVqfaE8i4+KZ25fPIoqgb240dPvhZ2KCJylCKZFOIJp7DAOLa8lNrKMnbsjXHa6MH0K+r8\nF70cXmlxIVdPHcvC5q0Hjf0QkfwSydtHsbhTlLq/P6Wukte37GLKGN32OBofnzyK259exQ+feI3v\nfPikbu3DgJohZRQXRvK3ikhOiGRSiCcSFBYmk8KZ44by4OIWzhynRtKjUVpcyOfeW8e3fr+CGf/x\ndLf3c/nkUXznQ91LKiJy9CKZFJJtCsmkcOFJIxhRXsppoweHHFX+u/LMWqoH90/PLXWk/rB8I3MX\nt3DN1LHUDDmyRn4R6RmRTArtbQoABQXGpNohIUfUNxQVFjDzxO7Pfj6lrpInlm9kzoImvvvhk3sw\nMhHJViRv3rb3PpLcMnxQKZdNquHhxlZatu4KOxyRSIrklTGzUpDccs20cRSYMUeTBYqEIpJJIbNN\nQXLLseWlXHp6DQ83ttD6lqoFkd4W0TaFhCqFHHbNtLE8+HwLU/99AYUdZyPsYOKoCh747JSDphDJ\nZT9b+Ab3L1rDo9eeSWnxkY2NWbttN7NueZbtu7t+CFSmK84YzdcvnHBEn5FoimRSiMV1+yiXjSjv\nz82XTeTF1m2H3W5D2x4efWEtf1i+4agauHvTzr0xfvCHV3lr137uX7SGfzh7zBF9/tb5TWzfvZ9P\nnV1LQRcJs92LLdv46V9e55Nn1qpXl3QpkkkhnnCKCpUUctnME49lZhdTisfiCf7aso0fPdnE+yYc\nmxfVwn3PvcFbu/ZTW1nG7U+v4vLJo7KuFtZu283DjS1cMqmGr17wrqyPub5tN1O/t0C9uiQrkW1T\nKFTvo7xXVFjA588dx4r123lixcaww+nSzr0xfvJMM1PHV/HdD5/Mph17eeD5NVl//rYFTQBcO23c\nER13RHn/VDuNenVJ1yJ5ZYyrobnPuOg9x1FbWcaP/rgSz/E5yH+28A227tzHDefVJ5/KN2YItz+9\nij37D56evaN123bz0OIWPtZQw3EV/Y/42NdMG6teXZKVSCaFmBqa+4xktVDP8vXbeWJ57lYLu/bF\nuONPzbx3fBWnjkqOnv/CeclneD+0uKXLz7c/z/va6UdWJbQbUd6fSybV8MgS9eqSwwu0TcHMZgI/\nAgqBO9393zq8Pxq4G6gCtgKfcPfWIGOCZKWgSdf6jlmnHMePn1rJP//yJX78VFPY4XTq7b2xZJWQ\nelQrwBl1qWd4L2jikkk1B7QtrH5zJ//08IvpKUNe2bCdjzbUMLIbVUK7a6aN5cHFa5izYNVh55f6\n75fXc9vTq9IPfxo1pIwfXzYxL9ps5OgFlhTMrBC4FTgfaAUWm9k8d1+esdn3gfvc/V4zOxf4LvD3\nQcXULpZwSov1F7yvKCos4FsfPIl7nl1Nrt5AqhrYjwtPHnHAHFtmxhfOq+fyOxcxt7GFK86oTb/3\nH394lWXrtqcfWlQ9ePgBCaU7jqtIVgsPLW7huunjOk0we2Nxbvrdcgw4YcQgNm7fw+9fXs+NF5yg\nnksREWSlcDrQ5O7NAGb2IDALyEwKE4Avpl7PB34dYDxpalPoe86uH8rZ9fk30+0ZYyuZVDuYOfNX\nccmkGvoVFbJy4w5+//J6rpk6lq/MPKFHj3fNtHE8tLiFOfOb+HYn1cLcxlbWt+3hZ58+nXPqq2h8\nfSsfuf05Vm7aoaQQEUHeQxkJZN4sbU2ty/QicHHq9YeAgWYW+IMNkuMUdPtIwmdm3DBjPBu272Fu\nqm3h5qea6F9cyGfOqevx442s6M9HG2qY29jCum27D3hvbyzObfObOHVUBWenppKvH5Z8XOzKjW/3\neCySm4K8Mnb2U7xjdf9lYKqZvQBMBdYCsYN2ZHaVmTWaWePmzZuPOjBVCpJLzhpXScPowcxZsIpl\n69r43UvruOKMWoYcUxLI8a6dNhZ4p/G63SNLWlnXtocbzhuPpQbGlZcVM2xgP15TUoiMIJNCK1CT\nsVwNrMvcwN3XufuH3X0i8LXUuraOO3L3O9y9wd0bqqqqjjqwWMZDdkTCZmbccF4969v2cOU9i+lf\nXMhnzzmykc5HonpwGR85Ldm2sL4tWS3siyWYM38VE0dV8N4Ot+Hqhw+gadOOwOKR3BJkUlgM1JvZ\nGDMrAS4F5mVuYGZDzaw9hq+S7IkUOFUKkmvOHjeUU0dVsHnHXv7+jNFUDugX6PGunTaWhDtX/3wp\nX374RT57XyNrt+3mhhn16SqhXf2wgazc9DaJRK4240tPCiwpuHsMmA08DqwA5rr7MjO7ycwuSm02\nDXjVzF4DhgPfDiqeTDFNnS05xsz42gfexWmjB3NVAG0JHdUMKePa6eN4c8denlu1haZNb/OBk0Yw\ndfzBlXj98AHs2hdnXdvuTvYkfU2g4xTc/THgsQ7rvpHx+hHgkSBj6IwqBclFp40ewi+vObPXjvel\n88fzpfPHd7nd+OHvNDZXD1YPpL4ukl1wNPeRSPbqhw0AYKXaFSIhkldGVQoi2asoK6FKPZAiI5JJ\nIRbX3EciR6J+2ABWblJSiIJIJgVVCiJHZvzwgTRt3JHzM9HK0YtkUoglXOMURI7AuGED2Lkvzrq2\nPWGHIgGLZFJQpSByZNp7IL22UY3NfV3kkoK7q/eRyBFq74HUpMbmPi9yV8b2QZmqFESyN/iYEoYO\nKFGlEAGRSwqxRPKhJep9JHJk2qe7kL4t0BHNuSieKhVUKYgcmfrhA/jlklb+vPLNw25XXGhMqh2i\nJ7XlqcglhVgqKahSEDkyJ44s577n3uATdy3qctubZr37gCfJSf6IXFKIx1UpiHTHxadWUz9sQPqH\n1aF897EVBzxJTvJL5JJCulIojFxzishRKSwwJo4a3OV2Xzr/eD5x1yLmLm7h71Ut5J3IXRnVpiAS\nrMwnye2NxcMOR45Q5JKCeh+JBCvzSXJzG1vDDkeOUORuH6lSEAne2eOGctrowcyZ38T57xp+0I8w\nM6g8puSgp7xJ+CKXFNT7SCR4ZsYNM+q54u7nmfLdJzvd5uqpY7nxghN6OTLpSuSSwjuVQuTunIn0\nqnPqhzLn46eyZee+g957YvlGfvqX1fzD2bUMG1gaQnRyKJFLCrG4KgWR3mBmvP+kEZ2+d/a4ocz4\njwXc8XQzX79wQi9HJocT6M9lM5tpZq+aWZOZ3djJ+6PMbL6ZvWBmL5nZ+4OMB9SmIJILxgw9hg+e\nMpKfL3qDzTv2hh2OZAgsKZhZIXArcAEwAbjMzDr+JPg6MNfdJwKXAnOCiqdduveRnqcgEqrZ545j\nXyzBT55pDjsUyRBkpXA60OTuze6+D3gQmNVhGwcGpV6XA+sCjAdQpSCSK+qqBjDrlJHc99zrvPm2\nqoVcEWRSGAm0ZCy3ptZl+lfgE2bWCjwGfD7AeAD1PhLJJe3Vwj3Prg47FEkJMil0dtXtOGnKZcBP\n3b0aeD/wMzM7KCYzu8rMGs2scfPmzUcVlHofieSOsVUDmDymkgWvHt2/a+k5QV4ZW4GajOVqDr49\n9GlgLoC7PweUAkM77sjd73D3BndvqKqqOqqg3qkUjmo3ItJDptRVsnz9dtp27Q87FCHYpLAYqDez\nMWZWQrIheV6HbdYAMwDM7F0kk0KgPxkS6aSgrCCSC6bUDcEdnn99a9ihCAEmBXePAbOBx4EVJHsZ\nLTOzm8zsotRm/wh81sxeBB4ArnT3w8/Le5RiamgWySnvqamgX1EBC5u3hB2KEPDgNXd/jGQDcua6\nb2S8Xg6cFWQMHcU1IZ5ITiktLuTUUYOVFHJE5O6hqFIQyT2T64Yk2xV2q10hbJFLCnF1SRXJOVPq\nKnGHxavVrhC2yCWFWFxdUkVyzSk1FZSoXSEnRO7KmK4UNM2FSM4oLS5kYk0FC1crKYQtcklBbQoi\nuWlKXSXL16ldIWyRSwrqfSSSm6bUVZJwaNR4hVBFLimoUhDJTRNHqV0hF0QuKaj3kUhuKi0u5IRj\nB/LKhh1hhxJpkUsKMU2IJ5KzBpeVsF1tCqGK3JVRlYJI7qooK+YtTYwXqsglhXfGKSgpiOSaiv7F\nbNu1L+wwIi1ySSGeSGAGBUoKIjmnoqyE7Xti6Ypeel/kkkIs4aoSRHJURVkxgNoVQhS5pBBPuNoT\nRHJUe1LYpqQQmsglhWSlELnTFskLFf1LANSuEKLIXR1VKYjkrvL2SkE9kEITuaQQSyTUpiCSowaX\npSqF3aoUwtJlUjCz2WY2uDeC6Q2qFERyV0V/VQphy6ZSOBZYbGZzzWymmeX1FTUWV+8jkVw1SEkh\ndF0mBXf/OlAP3AVcCaw0s++Y2diuPptKIq+aWZOZ3djJ+z80s7+m/rxmZtu6cQ5HJJ5wPUtBJEcV\nFhiDSos0fXaIirLZyN3dzDYAG4AYMBh4xMyecPevdPYZMysEbgXOB1pJVhvz3H15xn6/mLH954GJ\n3T6TLKn3kUhuqygr4S31PgpNNm0K15vZEuB7wLPASe5+DXAacPFhPno60OTuze6+D3gQmHWY7S8D\nHsg68m5Sm4JIbhtcVqzbRyHKplIYCnzY3d/IXOnuCTO78DCfGwm0ZCy3ApM729DMRgNjgKeyiOeo\nqPeRSG4rLyvR4LUQZXMf5TEg/SgkMxtoZpMB3H3FYT7X2ZX3UBOaXAo84u7xTndkdpWZNZpZ4+bN\nm7MI+dBUKYjktor+xbTp9lFoskkKtwFvZyzvTK3rSitQk7FcDaw7xLaXcphbR+5+h7s3uHtDVVVV\nFoc+NM19JJLbKsqKVSmEKJukYO6e/oXv7gmyu+20GKg3szFmVkLywj/voJ2bHU+y4fq57EI+OqoU\nRHJbRf9i2nbv10ypIckmKTSnGpuLU39uAJq7+pC7x4DZwOPACmCuuy8zs5vM7KKMTS8DHsxMPEFK\njlNQ7yORXFVRVoI77NijaiEM2fzivxq4Gfg6yTaBJ4Grstm5uz9Gsk0ic903Oiz/azb76imqFERy\nW0XG/EcVqWkvpPd0mRTcfRPJWz99QiyRoF9xVsMzRCQEmj47XF1eHc2sFPg08G6gtH29u/9DgHEF\nRpWCSG4r1/TZocrm5vrPSM5/9HfA0yR7Ee0IMqggqfeRSG4brOmzQ5VNUhjn7v8C7HT3e4EPACcF\nG1ZwVCmI5Lb2dgRVCuHIJim0p+ttZnYiUA7UBhZRwDT3kUhuG1SavKutNoVwZNPiekfqeQpfJznO\nYADwL4FGFSBVCiK5raiwgIGlRbp9FJLDJgUzKwC2u/tbwJ+Aul6JKkCa+0gk91WUFWv67JAc9j5K\navTy7F6KpVfE46oURHLdYE2fHZpsbq4/YWZfNrMaMxvS/ifwyAISSzhFesiOSE4r76/ps8OSTZtC\n+3iE6zLWOXl6K0ltCiK5r6KshNa3docdRiRlM6J5TG8E0lvU+0gk91X0L1aX1JBkM6L5is7Wu/t9\nPR9O8FQpiOS+9obmRMIp0L/XXpXN7aNJGa9LgRnAUiAvk4J6H4nkvoqyEhIOO/bEKE+NcJbekc3t\no89nLptZOcmpL/KSKgWR3FfRv31SvH1KCr2sOzfXdwH1PR1Ib9HcRyK5r0LzH4UmmzaF3/LOs5UL\ngAnA3CCDCkoi4bhDoRqaRXKaps8OTzZtCt/PeB0D3nD31oDiCVQs9Xg/jVMQyW2aPjs82SSFNcB6\nd98DYGb9zazW3V8PNLIAtD/zVW0KIrmtffpsTXXR+7K5j/IwkMhYjqfW5Z1YInkaalMQyW3lqYbm\nt3YqKfS2bJJCkbuna7jU66wenGpmM83sVTNrMrMbD7HNx8xsuZktM7P7swu7e1QpiOSHosICBvYr\nYttu3T7qbdncPtpsZhe5+zwAM5sFvNnVh8ysELgVOB9oBRab2Tx3X56xTT3wVeAsd3/LzIZ15ySy\nFVNSEMkb5WXFtKn3Ua/LJilcDfzCzG5JLbcCnY5y7uB0oMndmwHM7EFgFrA8Y5vPArempubG3Tdl\nG3h3qFIQyR8VZcXqfRSCbAavrQKmmNkAwNw92+czjwRaMpZbgckdthkPYGbPAoXAv7r7/3TckZld\nBVwFMGrUqCwPf7B07yMlBZGcd+ygUpav287+eILiQnUj7y1d/p82s++YWYW7v+3uO8xssJl9K4t9\nd3bl9Q7LRSQHwk0DLgPuNLOKgz7kfoe7N7h7Q1VVVRaH7lwiXSnoL5hIrrt88ijWte3hV0vzsgd8\n3srm6niBu29rX0jd6nl/Fp9rBWoylquBdZ1s8xt33+/uq4FXCXC0tCoFkfwx/fhhnFxdzi3zm9gf\nT3T9AekR2SSFQjPr175gZv2BfofZvt1ioN7MxphZCXApyWc8Z/o1MD2136Ekbyc1ZxN4d8RTXVLV\npiCS+8yMG2bU07J1N48uXRt2OJGRTVL4OfCkmX3azD4NPAHc29WH3D1G8lGejwMrgLnuvszMbjKz\ni1KbPQ5sMbPlwHzgn9x9S3dOJBuqFETyy7knDOOkkaoWelM2Dc3fM7OXgPNIthP8DzA6m527+2PA\nYx3WfSPjtQNfSv0JXCyu3kci+aS9WvjMfY08+sJaPtZQ0/WH5Khk2+K6geSo5otJPk9hRWARBSiu\nuY9E8s6Mdw3jxJGDuOuZ1WGHEgmHTApmNt7MvmFmK4BbSHYvNXef7u63HOpzuSym3kciecfMuPDk\n43h14w427dgTdjh93uGujq+QrAr+l7uf7e4/JjnvUd6Kq01BJC9NqasE4PnVW0OOpO87XFK4mORt\no/lm9hMzm0HnYw/yRky9j0Ty0onHDeKYkkIWNgfWD0VSDpkU3P1Rd78EOAFYAHwRGG5mt5nZ+3op\nvh6lSkEkPxUVFtBQO4SFzaoUgtblzXV33+nuv3D3C0kOQPsr0OmMp7lOE+KJ5K8pdZU0bXqbN9/e\nG3YofdoRtbi6+1Z3/y93PzeogIIUj7dXCmpoFsk3U+qGALBI1UKgInV1VKUgkr9OHFmudoVeEKmk\noHEKIvmruLCA02qHsGi1kkKQIpUU1PtIJL9NqRvCaxvVrhCkSCUF9T4SyW8arxC8SCUFtSmI5LeT\nRpZTVlLIIrUrBCabx3H2Ge9UCpHKhSJ9RnFhAaeNHsyvlq7lr61tgR3HgBtm1DP9hEAfG5+TIpUU\nVCmI5L/PnFPHPc+uxjs+x7EHvbJhO9/87TLOqR9KUcQeBRqppBBPzceuNgWR/DV1fBVTx3f/sbzZ\neHzZBj73syX85q/ruPi06kCPlWsilQLTlYK6pIrIYbxvwnAmjBjELfObiEXs4T6RSgrqfSQi2TAz\nrp9Rz+o3d/Lblzo+Wr5vi1RSUJuCiGTrfROGc8KxA/nxk03pH5RREK02BfU+EpEsFRQYXzivnqt/\nvpSrf76EymNKDrv95LohfGhi/rc/BJoUzGwm8COgELjT3f+tw/tXAv8OrE2tusXd7wwqnvZKQYWC\niGTjfROO5dwThvFS67bDbrdrX5xfLV3LlLpKRpT376XoghFYUjCzQuBW4HygFVhsZvPcfXmHTR9y\n99lBxZEpnkhQVGCYKSuISNcKCoy7r5zU5XYtW3cx/fsLuH3BKr4568ReiCw4Qd5HOR1ocvdmd98H\nPAjMCvB4XYolXO0JItLjaoaU8ZHTqnng+RY2tOX3c6SDTAojgZaM5dbUuo4uNrOXzOwRM6vpbEdm\ndpWZNZpZ4+bNm7sdUDzu6nkkIoG4bvo4Eu7c/vSqsEM5KkEmhc6uvh2b8H8L1Lr7ycAfgXs725G7\n3+HuDe7eUFXV/UErqhREJCg1Q8q4+NRq7n9+DRu352+1EGRSaAUyf/lXAwd0+HX3Le7ePgfuT4DT\nAoyHeMIjN2RdRHrPddPHEU84X3v0Ze55djX3PLua+a9uCjusIxJk76PFQL2ZjSHZu+hS4PLMDcxs\nhLuvTy1eBKwIMB5VCiISqFGVZVw6qYZfLFrDH1ckk0GBwR+/NJW6qgEhR5edwJKCu8fMbDbwOMku\nqXe7+zIzuwlodPd5wPVmdhEQA7YCVwYVD7zT+0hEJCjf+uCJ/NPfHQ9A2+79zPzPZ7jlqSZ+cMkp\nIUeWnUDHKbj7Y8BjHdZ9I+P1V4GvBhlDJlUKIhI0M6OiLDnQraKshE9MGcVdf17N7HPH5UW1EKkb\n7PGEeh+JSO+66r1jKSkq4Jb5TWGHkpVIJQVVCiLS26oG9uMTk0fz6xfWsvrNnWGH06VIJYXkOIVI\nnbKI5ICrptYlq4Wncr9aiNQVUpWCiIRh2MBSPj55NL/+61pez/FqIVJJIZ5IUKQH7IhICD43tY6i\nAsv5toVIJQVVCiISlvZq4dEX1vLGltytFiKVFNT7SETCdHV7tZDDbQuRSgqqFEQkTMMGlXL55FH8\n6oW1rNmyK+xwOhWppJCsFCJ1yiKSY66eOpbCAuOW+SvDDqVTkbpCqlIQkbANH1TK5aeP4pdLc7Na\niFRS0NxHIpILrpmWrBZuzcGeSJFKCrG4KgURCd/wQaVcNqmGXy5tpWVrblULkUoKyecpKCmISPiu\nmTaOAjPmLMitaiFySaFQDc0ikgOOLS/l0tNreLgxt6qFSF0hYxqnICI55JppY1PVQu481zlSSSGu\n3kcikkNGlPfnkkk1PLKkhda3cqNaiFRSiCUSFJqSgojkjmumjQXImWohUkkhnnAK1dAsIjnkuIr+\nfKyhhocbW1i7bXfY4QSbFMxsppm9amZNZnbjYbb7iJm5mTUEGY/aFEQkF107fRwAc3Jg3EJgScHM\nCoFbgQuACcBlZjahk+0GAtcDi4KKpV1c4xREJAeNrOjPRxtqmNvYwrqQq4UgK4XTgSZ3b3b3fcCD\nwKxOtvu/wPeAPQHGAqhSEJHcdW2qbeG2kNsWgkwKI4GWjOXW1Lo0M5sI1Lj77wKMIy3uGqcgIrmp\nenAZHzmthocWh1stBHmF7OwnuaffNCsAfgj8Y5c7MrvKzBrNrHHz5s3dDkjPUxCRXHbd9LE4zu1P\nh1ctBJkUWoGajOVqYF3G8kDgRGCBmb0OTAHmddbY7O53uHuDuzdUVVV1Kxh31zgFEclpyWqhmgef\nb2F9WzjVQpBJYTFQb2ZjzKwEuBSY1/6mu7e5+1B3r3X3WmAhcJG7NwYRTDyRLFJUKYhILrt22jgS\n7tweUttCYEnB3WPAbOBxYAUQzuZXAAAHvElEQVQw192XmdlNZnZRUMc9lFgqKWicgojkspohyWrh\ngcUtbGgLvP/NQQJtdXX3x9x9vLuPdfdvp9Z9w93ndbLttKCqBFClICL547rp40gkwmlbiExXnHSl\noN5HIpLjaoaU8eFTR3L/82vYuL13q4XIXCFVKYhIPpk9vZ54CNVCZJJCLJEAUO8jEckLoyrL+PDE\nkdy/aA2berFaiExSUKUgIvlm9rnjiCWc259u7rVjRiYpxOLtbQpKCiKSH0ZXHsOHJo7kF4ve6LVq\nITJJIV0pqEuqiOSR2dOT1cJ//al3qoXIJAX1PhKRfFQ79Bg+eMpIfr7wDTbtCL5aiMwVUm0KIpKv\nPp9qW3h06drAj1UU+BFyhHofiUi+qh16DL+57izefdygwI8VmaSgSkFE8tmJI8t75TiRuX30TpuC\nkoKIyKFEJim8UylE5pRFRI5YZK6QGqcgItK1yCQFjVMQEelaZJKCeh+JiHQtMklBvY9ERLoWmaSg\n3kciIl2LTFJQ7yMRka5F5gqpSkFEpGuBJgUzm2lmr5pZk5nd2Mn7V5vZy2b2VzP7s5lNCCqWeKqh\nWW0KIiKHFlhSMLNC4FbgAmACcFknF/373f0kdz8F+B7wg6Di0TgFEZGuBVkpnA40uXuzu+8DHgRm\nZW7g7tszFo8BPKhgNE5BRKRrQU6INxJoyVhuBSZ33MjMrgO+BJQA5wYVjNoURES6FmSl0NnV96BK\nwN1vdfexwD8DX+90R2ZXmVmjmTVu3ry5W8Go95GISNeCvEK2AjUZy9XAusNs/yDwwc7ecPc73L3B\n3Ruqqqq6FYwqBRGRrgWZFBYD9WY2xsxKgEuBeZkbmFl9xuIHgJVBBaPeRyIiXQusTcHdY2Y2G3gc\nKATudvdlZnYT0Oju84DZZnYesB94C/hkUPHUVh7D+086Vg3NIiKHYe6BdfgJRENDgzc2NoYdhohI\nXjGzJe7e0NV2anUVEZE0JQUREUlTUhARkTQlBRERSVNSEBGRNCUFERFJU1IQEZE0JQUREUnLu8Fr\nZrYZeKObHx8KvNmD4eSLKJ53FM8ZonneUTxnOPLzHu3uXU4el3dJ4WiYWWM2I/r6miiedxTPGaJ5\n3lE8ZwjuvHX7SERE0pQUREQkLWpJ4Y6wAwhJFM87iucM0TzvKJ4zBHTekWpTEBGRw4tapSAiIocR\nmaRgZjPN7FUzazKzG8OOJwhmVmNm881shZktM7MbUuuHmNkTZrYy9d/BYcfa08ys0MxeMLPfpZbH\nmNmi1Dk/lHr6X59iZhVm9oiZvZL6zs+IyHf9xdTf77+Z2QNmVtrXvm8zu9vMNpnZ3zLWdfrdWtLN\nqWvbS2Z26tEcOxJJwcwKgVuBC4AJwGVmNiHcqAIRA/7R3d8FTAGuS53njcCT7l4PPJla7mtuAFZk\nLP8/4Iepc34L+HQoUQXrR8D/uPsJwHtInn+f/q7NbCRwPdDg7ieSfKrjpfS97/unwMwO6w713V4A\n1Kf+XAXcdjQHjkRSAE4Hmty92d33AQ8Cs0KOqce5+3p3X5p6vYPkRWIkyXO9N7XZvcAHw4kwGGZW\nTfIZ33emlg04F3gktUlfPOdBwHuBuwDcfZ+7b6OPf9cpRUB/MysCyoD19LHv293/BGztsPpQ3+0s\n4D5PWghUmNmI7h47KklhJNCSsdyaWtdnmVktMBFYBAx39/WQTBzAsPAiC8R/Al8BEqnlSmCbu8dS\ny33x+64DNgP3pG6b3Wlmx9DHv2t3Xwt8H1hDMhm0AUvo+983HPq77dHrW1SSgnWyrs92uzKzAcAv\ngS+4+/aw4wmSmV0IbHL3JZmrO9m0r33fRcCpwG3uPhHYSR+7VdSZ1H30WcAY4DjgGJK3Tzrqa9/3\n4fTo3/eoJIVWoCZjuRpYF1IsgTKzYpIJ4Rfu/qvU6o3t5WTqv5vCii8AZwEXmdnrJG8LnkuycqhI\n3V6Avvl9twKt7r4otfwIySTRl79rgPOA1e6+2d33A78CzqTvf99w6O+2R69vUUkKi4H6VA+FEpIN\nU/NCjqnHpe6l3wWscPcfZLw1D/hk6vUngd/0dmxBcfevunu1u9eS/F6fcvePA/OBj6Q261PnDODu\nG4AWMzs+tWoGsJw+/F2nrAGmmFlZ6u97+3n36e875VDf7TzgilQvpClAW/ttpu6IzOA1M3s/yV+Q\nhcDd7v7tkEPqcWZ2NvAM8DLv3F//3yTbFeYCo0j+o/qou3dsxMp7ZjYN+LK7X2hmdSQrhyHAC8An\n3H1vmPH1NDM7hWTjegnQDHyK5A+9Pv1dm9k3gUtI9rZ7AfgMyXvofeb7NrMHgGkkZ0LdCPwf4Nd0\n8t2mkuMtJHsr7QI+5e6N3T52VJKCiIh0LSq3j0REJAtKCiIikqakICIiaUoKIiKSpqQgIiJpSgoi\nR8nMajNnsxTJZ0oKIiKSpqQg0oPMrC41Qd2ksGMR6Q4lBZEekppy4pckR5QuDjseke4o6noTEclC\nFcm5aC5292VhByPSXaoURHpGG8k57c8KOxCRo6FKQaRn7CP5JKzHzextd78/7IBEukNJQaSHuPvO\n1EN/njCzne7eF6dvlj5Os6SKiEia2hRERCRNSUFERNKUFEREJE1JQURE0pQUREQkTUlBRETSlBRE\nRCRNSUFERNL+P0NC0jNT1ttFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11620f2b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('Accuracy')\n",
    "ax.plot(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.0, 1)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(result), np.argmax(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.28888888888888886, 1.0, 0.9555555555555556, 1.0, 0.97777777777777775]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}