k-nearest-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import geohash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>time_cat</th>\n",
       "      <th>time_num</th>\n",
       "      <th>time_cos</th>\n",
       "      <th>time_sin</th>\n",
       "      <th>day_cat</th>\n",
       "      <th>day_num</th>\n",
       "      <th>day_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>weekend</th>\n",
       "      <th>x_start</th>\n",
       "      <th>y_start</th>\n",
       "      <th>z_start</th>\n",
       "      <th>location_start</th>\n",
       "      <th>location_end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>16.833333333333332:49.999999999999886</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.173648</td>\n",
       "      <td>-0.984808</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.396825</td>\n",
       "      <td>-0.797133</td>\n",
       "      <td>0.603804</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.460713</td>\n",
       "      <td>-0.120608</td>\n",
       "      <td>0.879316</td>\n",
       "      <td>eyckx6e80</td>\n",
       "      <td>eycs8byzz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>12.333333333333334:20.0</td>\n",
       "      <td>0.569444</td>\n",
       "      <td>-0.906308</td>\n",
       "      <td>-0.422618</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.367063</td>\n",
       "      <td>-0.670981</td>\n",
       "      <td>0.741474</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>u0qjdqxen</td>\n",
       "      <td>u0qjf9jxq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>12.35:21.0</td>\n",
       "      <td>0.570833</td>\n",
       "      <td>-0.902585</td>\n",
       "      <td>-0.430511</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.367262</td>\n",
       "      <td>-0.671905</td>\n",
       "      <td>0.740637</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>u0qjdqxen</td>\n",
       "      <td>u0qjf9jxq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>12.35:21.0</td>\n",
       "      <td>0.570833</td>\n",
       "      <td>-0.902585</td>\n",
       "      <td>-0.430511</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.367262</td>\n",
       "      <td>-0.671905</td>\n",
       "      <td>0.740637</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>u0qjdqxen</td>\n",
       "      <td>u0qjf9jxq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>12.35:21.0</td>\n",
       "      <td>0.570833</td>\n",
       "      <td>-0.902585</td>\n",
       "      <td>-0.430511</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.367262</td>\n",
       "      <td>-0.671905</td>\n",
       "      <td>0.740637</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>u0qjdqxen</td>\n",
       "      <td>u0qjf9jxq</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year  month  day                               time_cat  time_num  \\\n",
       "0  2017      3   22  16.833333333333332:49.999999999999886  0.777778   \n",
       "1  2017      5   24                12.333333333333334:20.0  0.569444   \n",
       "2  2017      5   24                             12.35:21.0  0.570833   \n",
       "3  2017      5   24                             12.35:21.0  0.570833   \n",
       "4  2017      5   24                             12.35:21.0  0.570833   \n",
       "\n",
       "   time_cos  time_sin    day_cat   day_num   day_cos   day_sin  weekend  \\\n",
       "0  0.173648 -0.984808  Wednesday  0.396825 -0.797133  0.603804        0   \n",
       "1 -0.906308 -0.422618  Wednesday  0.367063 -0.670981  0.741474        0   \n",
       "2 -0.902585 -0.430511  Wednesday  0.367262 -0.671905  0.740637        0   \n",
       "3 -0.902585 -0.430511  Wednesday  0.367262 -0.671905  0.740637        0   \n",
       "4 -0.902585 -0.430511  Wednesday  0.367262 -0.671905  0.740637        0   \n",
       "\n",
       "    x_start   y_start   z_start location_start location_end  \n",
       "0 -0.460713 -0.120608  0.879316      eyckx6e80    eycs8byzz  \n",
       "1  0.612977 -0.738237 -0.281542      u0qjdqxen    u0qjf9jxq  \n",
       "2  0.612977 -0.738237 -0.281542      u0qjdqxen    u0qjf9jxq  \n",
       "3  0.612977 -0.738237 -0.281542      u0qjdqxen    u0qjf9jxq  \n",
       "4  0.612977 -0.738237 -0.281542      u0qjdqxen    u0qjf9jxq  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featuredDataset = pd.read_csv('featured-dataset.csv')\n",
    "featuredDataset = featuredDataset.drop(featuredDataset.columns[0], axis=1)\n",
    "featuredDataset.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1537, 17)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featuredDataset.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# Get the longitude and latitude from the geohash\n",
    "def decodegeo(geo, which):\n",
    "    if len(geo) >= 6:\n",
    "        geodecoded = geohash.decode(geo)\n",
    "        return geodecoded[which]\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def further_data_prep(df):  \n",
    "\n",
    "    df['start_lat'] = df['location_start'].apply(lambda geo: decodegeo(geo, 0))\n",
    "    df['start_lon'] = df['location_start'].apply(lambda geo: decodegeo(geo, 1))\n",
    "    df['end_lat'] = df['location_end'].apply(lambda geo: decodegeo(geo, 0))\n",
    "    df['end_lon'] = df['location_end'].apply(lambda geo: decodegeo(geo, 1))\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>time_cat</th>\n",
       "      <th>time_num</th>\n",
       "      <th>time_cos</th>\n",
       "      <th>time_sin</th>\n",
       "      <th>day_cat</th>\n",
       "      <th>day_num</th>\n",
       "      <th>day_cos</th>\n",
       "      <th>...</th>\n",
       "      <th>weekend</th>\n",
       "      <th>x_start</th>\n",
       "      <th>y_start</th>\n",
       "      <th>z_start</th>\n",
       "      <th>location_start</th>\n",
       "      <th>location_end</th>\n",
       "      <th>start_lat</th>\n",
       "      <th>start_lon</th>\n",
       "      <th>end_lat</th>\n",
       "      <th>end_lon</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>16.833333333333332:49.999999999999886</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.173648</td>\n",
       "      <td>-0.984808</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.396825</td>\n",
       "      <td>-0.797133</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.460713</td>\n",
       "      <td>-0.120608</td>\n",
       "      <td>0.879316</td>\n",
       "      <td>eyckx6e80</td>\n",
       "      <td>eycs8byzz</td>\n",
       "      <td>38.773520</td>\n",
       "      <td>-9.168756</td>\n",
       "      <td>38.765237</td>\n",
       "      <td>-9.098074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>12.333333333333334:20.0</td>\n",
       "      <td>0.569444</td>\n",
       "      <td>-0.906308</td>\n",
       "      <td>-0.422618</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.367063</td>\n",
       "      <td>-0.670981</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>u0qjdqxen</td>\n",
       "      <td>u0qjf9jxq</td>\n",
       "      <td>47.409289</td>\n",
       "      <td>8.546956</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>12.35:21.0</td>\n",
       "      <td>0.570833</td>\n",
       "      <td>-0.902585</td>\n",
       "      <td>-0.430511</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.367262</td>\n",
       "      <td>-0.671905</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>u0qjdqxen</td>\n",
       "      <td>u0qjf9jxq</td>\n",
       "      <td>47.409289</td>\n",
       "      <td>8.546956</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>12.35:21.0</td>\n",
       "      <td>0.570833</td>\n",
       "      <td>-0.902585</td>\n",
       "      <td>-0.430511</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.367262</td>\n",
       "      <td>-0.671905</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>u0qjdqxen</td>\n",
       "      <td>u0qjf9jxq</td>\n",
       "      <td>47.409289</td>\n",
       "      <td>8.546956</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>12.35:21.0</td>\n",
       "      <td>0.570833</td>\n",
       "      <td>-0.902585</td>\n",
       "      <td>-0.430511</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.367262</td>\n",
       "      <td>-0.671905</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>u0qjdqxen</td>\n",
       "      <td>u0qjf9jxq</td>\n",
       "      <td>47.409289</td>\n",
       "      <td>8.546956</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   year  month  day                               time_cat  time_num  \\\n",
       "0  2017      3   22  16.833333333333332:49.999999999999886  0.777778   \n",
       "1  2017      5   24                12.333333333333334:20.0  0.569444   \n",
       "2  2017      5   24                             12.35:21.0  0.570833   \n",
       "3  2017      5   24                             12.35:21.0  0.570833   \n",
       "4  2017      5   24                             12.35:21.0  0.570833   \n",
       "\n",
       "   time_cos  time_sin    day_cat   day_num   day_cos    ...     weekend  \\\n",
       "0  0.173648 -0.984808  Wednesday  0.396825 -0.797133    ...           0   \n",
       "1 -0.906308 -0.422618  Wednesday  0.367063 -0.670981    ...           0   \n",
       "2 -0.902585 -0.430511  Wednesday  0.367262 -0.671905    ...           0   \n",
       "3 -0.902585 -0.430511  Wednesday  0.367262 -0.671905    ...           0   \n",
       "4 -0.902585 -0.430511  Wednesday  0.367262 -0.671905    ...           0   \n",
       "\n",
       "    x_start   y_start   z_start  location_start location_end  start_lat  \\\n",
       "0 -0.460713 -0.120608  0.879316       eyckx6e80    eycs8byzz  38.773520   \n",
       "1  0.612977 -0.738237 -0.281542       u0qjdqxen    u0qjf9jxq  47.409289   \n",
       "2  0.612977 -0.738237 -0.281542       u0qjdqxen    u0qjf9jxq  47.409289   \n",
       "3  0.612977 -0.738237 -0.281542       u0qjdqxen    u0qjf9jxq  47.409289   \n",
       "4  0.612977 -0.738237 -0.281542       u0qjdqxen    u0qjf9jxq  47.409289   \n",
       "\n",
       "   start_lon    end_lat   end_lon  \n",
       "0  -9.168756  38.765237 -9.098074  \n",
       "1   8.546956  47.423751  8.555195  \n",
       "2   8.546956  47.423751  8.555195  \n",
       "3   8.546956  47.423751  8.555195  \n",
       "4   8.546956  47.423751  8.555195  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featuredDataset = further_data_prep(featuredDataset)\n",
    "featuredDataset.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train-test split\n",
    "For Cross Validation, we split the data into 80% train set and 20% test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns_all_features = featuredDataset.columns\n",
    "columns_X = ['day_num', 'x_start', 'y_start', 'z_start']\n",
    "columns_y = ['end_lat', 'end_lon']\n",
    "X = featuredDataset[columns_X]\n",
    "y = featuredDataset[columns_y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X: (1537, 4)\n",
      "y: (1537, 2)\n",
      "X_train: (1229, 4)\n",
      "y_train: (1229, 2)\n",
      "X_test: (308, 4)\n",
      "y_test: (308, 2)\n"
     ]
    }
   ],
   "source": [
    "print ('X: ({}, {})'.format(*X.shape))\n",
    "print ('y: ({}, {})'.format(*y.shape))\n",
    "print ('X_train: ({}, {})'.format(*X_train.shape))\n",
    "print ('y_train: ({}, {})'.format(*y_train.shape))\n",
    "print ('X_test: ({}, {})'.format(*X_test.shape))\n",
    "print ('y_test: ({}, {})'.format(*y_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Funtion for cross-validation over a grid of parameters\n",
    "\n",
    "def cv_optimize(clf, parameters, X, y, n_jobs=1, n_folds=5, score_func=None):\n",
    "    if score_func:\n",
    "        gs = GridSearchCV(clf, param_grid=parameters, cv=n_folds, n_jobs=n_jobs, scoring=score_func)\n",
    "    else:\n",
    "        gs = GridSearchCV(clf, param_grid=parameters, n_jobs=n_jobs, cv=n_folds)\n",
    "    gs.fit(X, y)\n",
    "    print (\"BEST\", gs.best_params_, gs.best_score_, gs.cv_results_)\n",
    "    best = gs.best_estimator_\n",
    "    return best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Grid Search Cross Validation\n",
    "Grid Search CV implements an exhaustive search over specified parameter values for an estimator.\n",
    "Important members are fit, predict.\n",
    "\n",
    "It iterates through a dictionary of hyper parameters and choose the combination that better fits the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BEST {'n_neighbors': 5} -3.15864936119 {'split0_test_score': array([-0.00151617, -0.00060575, -0.00048735]), 'split1_test_score': array([ -4.34287932e-05,  -7.69864045e-05,  -2.25453448e-05]), 'split2_test_score': array([-15.80417493, -15.80246456, -15.77927105]), 'split3_test_score': array([-0.00066441, -0.00045609, -0.0004521 ]), 'split4_test_score': array([-0.00070371, -0.00037731, -0.00017443]), 'mean_test_score': array([-3.16399231, -3.16336768, -3.15864936]), 'std_test_score': array([ 6.32330521,  6.32276204,  6.31351975]), 'rank_test_score': array([3, 2, 1], dtype=int32), 'split0_train_score': array([-0.        , -0.98888218, -2.53179367]), 'split1_train_score': array([-0.        , -0.98889181, -2.53703436]), 'split2_train_score': array([-0.        , -0.00015217, -0.0002098 ]), 'split3_train_score': array([-0.        , -0.99280409, -2.53649995]), 'split4_train_score': array([-0.        , -0.98783378, -2.52992351]), 'mean_train_score': array([ 0.        , -0.7917128 , -2.02709226]), 'std_train_score': array([ 0.        ,  0.39578396,  1.01344486]), 'mean_fit_time': array([ 0.00926137,  0.00324183,  0.00331125]), 'std_fit_time': array([  8.34417344e-03,   2.08696759e-05,   7.74885471e-05]), 'mean_score_time': array([ 0.0024837 ,  0.00250416,  0.00263515]), 'std_score_time': array([  1.13831786e-04,   3.86677598e-05,   1.60522114e-05]), 'param_n_neighbors': masked_array(data = [1 2 5],\n",
      "             mask = [False False False],\n",
      "       fill_value = ?)\n",
      ", 'params': ({'n_neighbors': 1}, {'n_neighbors': 2}, {'n_neighbors': 5})}\n"
     ]
    }
   ],
   "source": [
    "# Create a k-Nearest Neighbors Regression estimator\n",
    "knn_estimator = KNeighborsRegressor()\n",
    "#knn_parameters = {\"n_neighbors\": [1,2,5,10,20,50,100]}\n",
    "knn_parameters = {\"n_neighbors\": [1,2,5]}\n",
    "knn_best = cv_optimize(knn_estimator, knn_parameters, X_train, y_train, score_func='neg_mean_squared_error')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model accuracy: R-Squared and Root-mean-squared deviation\n",
    "R-squared is a statistical measure of how close the data are to the fitted regression line. It ranges from 0 to 1, being 1 the best coefficient.\n",
    "RMSE is the square root of the mean square error. In other worids the distance, on average, of a data point from the fitted line, measured along a vertical line."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############# based on standard predict ################\n",
      "R^2 on training data: 0.97231458\n",
      "R^2 on test data:     0.99999609\n"
     ]
    }
   ],
   "source": [
    "# Fit the best Random Forest and calculate R^2 values for training and test sets\n",
    "knn_reg=knn_best.fit(X_train, y_train)\n",
    "knn_training_accuracy = knn_reg.score(X_train, y_train)\n",
    "knn_test_accuracy = knn_reg.score(X_test, y_test)\n",
    "print (\"############# based on standard predict ################\")\n",
    "print (\"R^2 on training data: %0.8f\" % (knn_training_accuracy))\n",
    "print (\"R^2 on test data:     %0.8f\" % (knn_test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.020617576356285399"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the Root Mean Squared Error\n",
    "np.sqrt(mean_squared_error(knn_reg.predict(X_test),y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day_num</th>\n",
       "      <th>x_start</th>\n",
       "      <th>y_start</th>\n",
       "      <th>z_start</th>\n",
       "      <th>end_lat</th>\n",
       "      <th>end_lon</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>645</th>\n",
       "      <td>0.384325</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1100</th>\n",
       "      <td>0.387500</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>0.372421</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1510</th>\n",
       "      <td>0.401984</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858</th>\n",
       "      <td>0.388294</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1219</th>\n",
       "      <td>0.390079</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>0.374802</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567</th>\n",
       "      <td>0.382738</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1391</th>\n",
       "      <td>0.394444</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>0.386508</td>\n",
       "      <td>0.612977</td>\n",
       "      <td>-0.738237</td>\n",
       "      <td>-0.281542</td>\n",
       "      <td>47.423751</td>\n",
       "      <td>8.555195</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       day_num   x_start   y_start   z_start    end_lat   end_lon\n",
       "645   0.384325  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "1100  0.387500  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "270   0.372421  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "1510  0.401984  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "858   0.388294  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "1219  0.390079  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "123   0.374802  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "567   0.382738  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "1391  0.394444  0.612977 -0.738237 -0.281542  47.423751  8.555195\n",
       "763   0.386508  0.612977 -0.738237 -0.281542  47.423751  8.555195"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampleds = pd.DataFrame(featuredDataset, columns=(columns_X + columns_y))\n",
    "sampleds = sampleds.sample(10)\n",
    "sampleds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533],\n",
       "       [ 47.42375135,   8.55519533]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = knn_reg.predict(sampleds.iloc[:,:-2])\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the model\n",
    "We dump the trained model into a file, so that we can later load and use it without having to fit it again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['k_nearest_model.pkl']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.externals import joblib\n",
    "joblib.dump(knn_reg, 'k_nearest_model.pkl') "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}