Project2_Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1: Regression Task:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Data exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraris and dataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from sklearn import preprocessing\n",
    "\n",
    "df = pd.read_csv('bikesharing.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.isnull().sum()\n",
    "df = df.fillna(df.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(['casual','instant','dteday','registered'],axis = 1)\n",
    "\n",
    "seed = 1498920\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 17379 entries, 0 to 17378\n",
      "Data columns (total 13 columns):\n",
      "season        17379 non-null int64\n",
      "yr            17379 non-null int64\n",
      "mnth          17379 non-null int64\n",
      "hr            17379 non-null int64\n",
      "holiday       17379 non-null int64\n",
      "weekday       17379 non-null int64\n",
      "workingday    17379 non-null int64\n",
      "weathersit    17379 non-null int64\n",
      "temp          17379 non-null float64\n",
      "atemp         17379 non-null float64\n",
      "hum           17379 non-null float64\n",
      "windspeed     17379 non-null float64\n",
      "cnt           17379 non-null int64\n",
      "dtypes: float64(4), int64(9)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Splitting data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split train test 70%\n",
    "bool_index = np.random.rand(df.shape[0]) < 0.7\n",
    "train = df[bool_index].reset_index(drop = True)\n",
    "test = df[~bool_index].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Scaling data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X features and y\n",
    "# Standardize X features\n",
    "scaler = preprocessing.StandardScaler()\n",
    "\n",
    "\n",
    "train_x = train.iloc[:,0:12]\n",
    "train_y = train.iloc[:,12]\n",
    "scaler.fit(train_x)\n",
    "train_x_scaled = scaler.transform(train_x)\n",
    "\n",
    "test_x = test.iloc[:,0:12]\n",
    "test_y = test.iloc[:,12] \n",
    "test_x_scaled = scaler.transform(test_x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "dt_reg = DecisionTreeRegressor(random_state=0)\n",
    "bag_reg = BaggingRegressor(dt_reg, n_estimators=500, max_samples=100, bootstrap=True, random_state=0, oob_score=True)\n",
    "\n",
    "bag_reg.fit(train_x_scaled, train_y)\n",
    "y_pred = bag_reg.predict(test_x_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103.83903402376203"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, y_pred))\n",
    "rms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "113.82936972085706"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "rnd_reg = RandomForestRegressor(random_state=0)\n",
    "bag_rnd = BaggingRegressor(rnd_reg, n_estimators=500, max_samples=100, bootstrap=True, random_state=0, oob_score=True)\n",
    "\n",
    "bag_rnd.fit(train_x_scaled, train_y)\n",
    "\n",
    "y_pred_rf = bag_rnd.predict(test_x_scaled)\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, y_pred_rf))\n",
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Pasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "dt_reg = DecisionTreeRegressor(random_state=0)\n",
    "bag_reg = BaggingRegressor(dt_reg, n_estimators=500, max_samples=100, bootstrap=False, random_state=0)\n",
    "\n",
    "bag_reg.fit(train_x_scaled, train_y)\n",
    "y_pred = bag_reg.predict(test_x_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103.74226374247431"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, y_pred))\n",
    "rms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "113.75255512203731"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "rnd_reg = RandomForestRegressor(random_state=0)\n",
    "bag_rnd = BaggingRegressor(rnd_reg, n_estimators=500, max_samples=100, bootstrap=False, random_state=0)\n",
    "\n",
    "bag_rnd.fit(train_x_scaled, train_y)\n",
    "\n",
    "y_pred_rf = bag_rnd.predict(test_x_scaled)\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, y_pred_rf))\n",
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103.74226374247431"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "\n",
    "dt_reg = DecisionTreeRegressor(random_state=0)\n",
    "ada_reg = AdaBoostRegressor(dt_reg)\n",
    "\n",
    "ada_reg.fit(train_x_scaled, train_y)\n",
    "y_pred_ada = bag_reg.predict(test_x_scaled)\n",
    "rms = sqrt(mean_squared_error(test_y, y_pred_ada))\n",
    "rms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103.74226374247431"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_reg = RandomForestRegressor(random_state=0, max_depth = 1, max_leaf_nodes=2, min_samples_split=2)\n",
    "ada_reg = AdaBoostRegressor(dt_reg)\n",
    "\n",
    "ada_reg.fit(train_x_scaled, train_y)\n",
    "y_pred = bag_reg.predict(test_x_scaled)\n",
    "rms = sqrt(mean_squared_error(test_y, y_pred))\n",
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters: {'learning_rate': 0.8, 'max_depth': 2, 'n_estimators': 200}\n",
      "Best cross-validation score: 0.7766\n",
      "Average cross-validation score: 0.7463\n"
     ]
    }
   ],
   "source": [
    "# GridSearch\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from  sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.model_selection import cross_val_score\n",
    "gb = GradientBoostingRegressor(random_state=42)\n",
    "\n",
    "param = {'max_depth':[1,2,3], 'n_estimators':[100,200], 'learning_rate':[0.8, 1.0]}\n",
    "gb_grid_search = GridSearchCV(gb, param, cv=5)\n",
    "gb_grid_search.fit(train_x_scaled, train_y)\n",
    "print(\"Best parameters: {}\".format(gb_grid_search.best_params_))\n",
    "print(\"Best cross-validation score: {:.4f}\".format(gb_grid_search.best_score_))\n",
    "\n",
    "# Cross-validation\n",
    "gb_scores = cross_val_score(gb, train_x_scaled, train_y, cv=5)\n",
    "print(\"Average cross-validation score: {:.4f}\".format(gb_scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66.5177573922792"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.8, random_state=42)\n",
    "gb.fit(train_x_scaled, train_y)\n",
    "y_pred_gb = gb.predict(test_x_scaled)\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, y_pred_gb))\n",
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 9. Applying PCA and running all algorithms from Project 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "# 0.90 = 90% variance retained\n",
    "pca = PCA(0.90)\n",
    "# fit on train set only\n",
    "pca.fit(train_x_scaled)\n",
    "\n",
    "# transform for both train and test\n",
    "pca_train_x = pca.transform(train_x_scaled)\n",
    "pca_test_x = pca.transform(test_x_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12126, 12)\n",
      "(12126, 9)\n"
     ]
    }
   ],
   "source": [
    "print(train_x_scaled.shape)\n",
    "print(pca_train_x.shape)\n",
    "\n",
    "# 90% variance is retained by using 9 components (variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression models using GridSearchCV and cross-validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. SVM kernel (linear, poly, rbf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   9 | elapsed:   18.5s remaining:   37.2s\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   9 | elapsed:   21.0s remaining:   26.3s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   9 | elapsed:   32.7s remaining:   26.1s\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   9 | elapsed:   32.9s remaining:   16.4s\n",
      "[Parallel(n_jobs=-1)]: Done   7 out of   9 | elapsed:   37.3s remaining:   10.6s\n",
      "[Parallel(n_jobs=-1)]: Done   9 out of   9 | elapsed:   46.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   9 out of   9 | elapsed:   46.0s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "             estimator=SVR(C=1.0, cache_size=200, coef0=0.0, degree=3,\n",
       "                           epsilon=0.1, gamma='auto_deprecated', kernel='rbf',\n",
       "                           max_iter=-1, shrinking=True, tol=0.001,\n",
       "                           verbose=False),\n",
       "             iid='warn', n_jobs=-1,\n",
       "             param_grid={'kernel': ['linear', 'poly', 'rbf']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring=None, verbose=10)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.svm import SVR\n",
    "\n",
    "svclassifier = SVR()\n",
    "param_grid = {'kernel' : ['linear','poly', 'rbf']}\n",
    "\n",
    "grid_search_SVR = GridSearchCV(estimator = svclassifier, param_grid = param_grid, \n",
    "                          cv = 3, n_jobs = -1,verbose = 10,return_train_score = True)\n",
    "\n",
    "\n",
    "grid_search_SVR.fit(pca_train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "results = grid_search_SVR.cv_results_\n",
    "num_param = len(list(param_grid.values())[0])\n",
    "train_scores = np.hstack((results['split0_train_score'].reshape((num_param,1)),results['split1_train_score'].reshape((num_param,1)),results['split2_train_score'].reshape((num_param,1))))\n",
    "val_scores = np.hstack((results['split0_test_score'].reshape((num_param,1)),results['split1_test_score'].reshape((num_param,1)),results['split2_test_score'].reshape((num_param,1))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_mean = np.mean(train_scores,axis = 1)\n",
    "val_mean = np.mean(val_scores,axis = 1)  \n",
    "width = 0.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEjCAYAAAAsbUY2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZgdZZn38e+PQDJi2DQYgQYSMESDCpoW5FWhoxGDCzAOI+AGM/Jm8CWjDmZGUGTJqOPSbkgcjMqIC0YWZaITxQVaQUSTQAQDEwxhaxNlSVgCGAjc7x/Pc0LlcE71Sfep7k7n97muvrqWp566z6k6ddf6lCICMzOzZrYZ6gDMzGx4c6IwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK7VVJApJoyStk7RXO8va4JA0XdIdg1G3pOWSXlNFHJK+JunD/Z3erL8kfVvS2f2dflgmiryhrv09JemxQv87Nre+iHgyIsZGxF3tLNtfkk6SFJLeWtU8thaS/ijp3Q2Gf1DSdZtbX0RMjoir2xDXSZJ66uo+KSI+MdC6+5in16thQtI1kk4s9L9O0lpJxwxhWP0yLBNF3lCPjYixwF3AWwrDvlNfXtK2gx/lgJwArMn/B5WkUYM9z4p9E3hGogDeBVw4yLEMNa9XLRrseCUdAVwGnBARl/Zj+qH9fiNiWP8BdwDT64Z9DPge8F3gYeBE4BDgOuABYDVwLrBdLr8tEMCE3P/tPP7HefrfABM3t2wefwRwK/Ag8CXg18CJJZ9nH+Ap4BjgcWDXuvFvBZYCDwErgMPz8OcC38ifbS1wWR5+EtBTmL5R/HOBnwCPAF3AkXkeD5MS8UfrYjg0f5cPAneTNrqHAKuAbQrljgUWN/mcTecBvCDH+G6gF7gXOK0wfnvgW/lzLgM+BNzRZD57AxuAjsKwlwB/BZ5T+I5uybHcBpxUKDu9WHeOp6uVOIAzgJW53mXAkXXzfxJYB9xXWBZnF6Y/OS/j+4HLgd3qluE/5fFrgXP7+J1s9etVs3mUxLtzHn4vaTtzOqBcfj/gV7me+4CL8vBtSNuDe/K4G4EpTWK9hrRtOoq0XXpD3fgpwM9Jyf1/gb8rjGsUb1/bor7qOzt3Pw9YmGNaA/yqz+1wOzbmVf7RPFE8DrwlL7hnAa8ADs4r9D6kjfeskpX8PqAT2I6UdL7dj7LPywvsqDzuVOAJyhPFOcC1ufsW4H2Fcf8nL7zX5c+1JzA5j7sCuAjYBRgNHLoZP+i1pB/kNsAY4LXAi3P/AfnzvTmXn5g/09tyXeOAA/O45cDrC/P6IfD+Jp+zbB61RHE+8DfAy4H1wKQ8vhvoyZ91b+BmmiSKXP4qNk00nwEuLfS/Ja8TynE9Brw0jytLFKVx5O9ot/wZ305KCuMbLZcGP9bDSRubA/N38GXgyrpl+N/ATsAE0g96esl34PWqfB6N4r0I+D6wQ14/VpD2+AEuIe0YbJOXz6vy8DcBv8vLZRvSxvn5TWK9hrQDsBaYVjduB+BPpJ2lbYGppB2GySXxlm2LWqmvtu59Bjgv1zEaOKzP7XDVG/qB/tE8UVzZx3SzgUtKVvLzC2WPBP7Qj7L/CFxdGCfSnlnDRJHHr+TpBPZRYElh/NeBzzSYbk/SXvNODca18oO+oI/v6rzafHNMlzQp9xHgwtw9DngUeF6Ly7E4j1qieH5h/PXAMbn7ruIyB/4f5YniRODm3L1N/sG8paT8j4BTcndZotjcOP4AvKnRciksi9qP9ULgE4VxO5KOQDoKy/CVhfHfB2Z7ver3PDaJl7SR3ADsVxh2CvDz3H0R8J/AHnX1HE7aWz+YwlFQk3leQzqC+w3wN3Xj3gFcVTfs68BHmn2/lG+LWqmvtu59Iq9P+7byHUfE8LxG0aK7iz2SXijpfyT9WdJDwBzSStfMnwvdjwJj+1F292IckZZCb0k9h5J+nBfn/ouAl0t6ce7fk3RqpN6epNMXD5bUXab+uzpEUo+keyU9SNoo1L6rZjFAOg1ztKTtgeNIK+Y9jQr2MQ8AIqLZ97pbXcx39vH5LgX2ktRJ2vBvRzo8r8XyZkm/lbRG0gOkH3vZulFTGoekEyX9XtIDud4XtlgvpHVnY30R8RBpD3KPQplW11GvV33Poz7e5wGj2HSZ3snT3/8HSevRYkk3SToBICJ+SjoS/k/gL5LOl7RDyTw/TDol+H1JowvD9wZeVVt38vpzLGmdaxRvTbN1opX6aj6ZP+svJN0m6V9L4geG6cXsFkVd/1dIe3QviIgdgTNJe1pVWk3aAwRAktj0h17vBNJ3fqOkP5OuZwRPX4y9G9i3wXR3A+Mk7dhg3COkc+k1z29Qpv67mk+6sLZnROwEfI2nv6tmMRDpTrDFpFNt7yL9wJspm0df/kz60deU3qocEetIe0jvznFdFBEbACQ9i5RI/oN0Wmhn4KctxtI0Dkn7kDYW7wWem+v930K99d95vVWkH3etvh1Ip3/+1EJc9bxe9TGPBvHeQzqC27swbC/y9x8RqyPdpbYb6UhjnqSJedwXIuLlpFNgU0innJtZR7qOuSvwvcKNN3cDv4iInQt/YyNiVpN4+9JKfeT4H4qIf4mICcDRwIckHVZW+ZacKOrtQLq49IikF5EuBFbtR6Q9t7fkFeD9pBXiGfLe0jHAe0jnpWt//wK8M9/V8HXgJEnTJG0jqUPS5Ii4m3SRaq6knSVtJ+nQXPXvgZdKekneKJ7VQtw7AGsi4q+SXknai6v5NjBD0t9J2lbSOEkHFMZ/k3TR74Wkc+j9mUdfLgY+nD/rXsAzVvYGLgSOB/6WTe92GkM6D3sv8KSkN5PO1Q80jrGkH/K9pH2Ek0jfSc1fgA5J2zWp+7vAeyS9VNIYUiK7OiLKjkifwevVZs1jo4h4grQD8QlJY3MS+JdcB5LeJqm20/cAaVk/Kemg/LctKZk+Tko4TeWjxcNJ15q+nZfJAmB/SW/P3/t2ud7JZXWVaLm+vL3aN+/YPpjjL/0MIylRfJC0Z/Uw6ejie1XPMCL+Qjq8+xzpwtG+wA2kC7P13ppj+3ZE/Ln2B3yVdDH+9RFxLfB/SXc2PEi6SFvbo31n/n8raSP0zzmGm0nnHHtIFwV/1ULo7wX+Q9LDpEPj2ikLIuJ20sXfD5EuoF5Puoun5jLShb9LI+Kx/syjBWeRjtbuIJ1C+mYL01xFOhS/PSJuqA2MiAdIG4AfkD7PMaQEP6A4IuJG0nL6XS7zQuC3hWl/BvyRdHqieLqgNv1PSKdHf5Cn34t0nnlzeb1qfR71/h9pQ3878EvSDkZtGR8MLJL0COlo9ZR85LMzKfE+QFovVgOfL5lHLba1wOtJRyEXkJbZG0jf/2rS0et/kHZsNls+fdhqfZOBK0lHO78GvhgR15TVX7sVzNog7ymsIl2UHfBDW8NR3gu5nXTBvmeIw7ERwuvV8DaSjiiGhKQZknbKpw8+SrqT4ndDHFaV3kY6YvrlUAdiI4rXq2FsS3uieTh6NfAd0nnwZcDREdHo1NMWT9I1wCTgHeFDUWsTr1fDn089mZlZKZ96si1GvqNjsVLjkKsl/VjSq4c6roHIzwScNIBp/yrpYUkPSVoi6bR8GrTVOkLSC/ozf9t6OFHYFkHSqcAXSHfijCfdJfRl0r33W7NZEbED6cGqD5JuF12YLw6btYUThQ17knYi3Up6SkR8PyIeiYgnIuKHEfGvucwYSV+QtCr/faG2Zy2pS1KvpH+TdE8+Gjla0hsl3ar0xPaHC/M7W9IlSm34P6z0ZO5+kk7P098t6fBifJK+nuv9k6SP5Tvgak9vXyOpW6mJ6duVWhJF0seB1wDn5aOk85R8Ps/nQUk36uknrJvK30kPqVmHQ0htEpHvpf+N0tO6q/M8RudxtVtef5/nf6ykXST9SOnJ57W5u6PhTG2r4URhW4JDSA2z/aCkzEeAV5IeNjsAOIjUumvN83Mde5Ce2v8q6Z7zqaSN9ZlKT1vXvIX0hPAupGdjriD9XvYgJa2vFMpeSLrb7QXAy0gPVxVPJx1MehZhHPBp4OuSFBEfAa4mHRXUnqI9nNQkx36ke/aPJT2j05LCU861ly89SXqOZBzpe3wd6fkBIqL2cN0Bef7fy5/xv0hPLO9FakDxvFbnbyOTE4VtCZ5LapNoQ0mZdwBzIuKeiLiX1JrquwrjnwA+np/InU/acH4xIh6OiGWkO9ZeWih/dURcked5CemJ+08Wpp+g9DTzeFITDR/Ie/X3kB7AKj4xfGdEfDUiniQlld1Ip88aeYL09PELSTeb3BIRq/v6guqsAp4DEBFLIuK6iNgQEXeQElzT5hoi4v6IuCwiHo2Ih4GPl5W3rYNvj7Utwf2kNom2LUkWmzSyl7t3L9aRN9SQ9pIhPYlMYVix0b36cfc1mH5snsd2wOrCZYFt2LRBt41PZkfEo7lcwwb+IuJKSeeR3kWwl6QfkFqNfahR+Sb2AK4FkLQfqeWATlLbTdsCS5pNqNQkyOeBGaSjKYAdJI0qfH7byviIwrYEvyG9COjokjKbNLJHOm2yqsqgsrtJD4qNKzTGtmNE7N/i9M+4Pz0izo2IqcD+pFNQfbbuWSNpT9LptFrLAP9JaqxwUm4s88OUN4j4QVITDwfn8rXTU744vhVzorBhL7djcyap8bqjJW2fGz07QtKnc7HvAmdI2lXSuFz+24MQ22pSa7SflbSjUqN7+6qP1jgL/kJq4wgASa+QdLBSY4KP8PSb8krl7+QwUoN6vyO9wQzSaayHgHWSXkhqK6np/HP5x4AHJD2H1hoDtBHOicK2CBHxOVJzzmeQWmy9m9Sa6+W5yMdIF3FvBG4iNQj3sUEK792kJ/NvJr1T4lIavwegkS8Cx+Q7jM4lvcDoq7meO0mn3bpLpj9PqYG8v5BuH74MmBERT+Xxs0lv33s411vfWObZwIX5rqi35TqeRXqT2nWkV3HaVs5PZpuZWSkfUZiZWSknCjMzK+VEYWZmpZwozMys1Ih54G7cuHExYcKEoQ6jLR555BGe/exnD3UY1kZepiPPSFmmS5YsuS8idi0rU2mikDSDdPvfKOBrEfHJuvEnA6eQ7hNfB8yMiJslTQBuIbWPA3BdRJxcNq8JEyawePHi9n6AIdLT00NXV9dQh2Ft5GU68oyUZSrpzr7KVJYocuuZc0kvFO8lvah8QX5pe81FEXF+Ln8kqamBGXncbRFxYFXxmZlZa6q8RnEQsCIiVkbE46SG1DZ5d0Bd+zXPpkFzBmZmNrSqPPW0B5s2jNZLam55E5JOIT1xOxp4bWHUREk3kJofOCMirm4w7UxgJsD48ePp6elpW/BDad26dSPms1jiZTrybE3LtMpE0agRsUYNoM0lteHzdlLzDCcAq4G9IuJ+SVOByyXtX9+CZkTMA+YBdHZ2xkg4Xwgj59ynPc3LdOTZmpZplaeeeoE9C/0dlLfmOZ/cOmhErI+I+3P3EuA2UiuaZmY2yKpMFIuASZIm5lcvHgcsKBaQNKnQ+ybgj3n4roVXSe4DTAJWVhirmZk1Udmpp4jYIGkW6RWSo4ALImKZpDnA4ohYAMySNJ30Vq+1pNNOkNrAnyNpA+nW2ZMjYk1VsZqZWXOVPkcREQt5ul382rAzC93vbzLdZaTmks3MbIi5CQ8zMyvlRGFmWwxp+PwtWTL0MdT+quZEYWZmpZwozMyslBOFmZmVcqIwM7NSThTZUF+M2lovkpnZ8OdEYWZmpZwozMyslBOFmZmVcqIwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK+VEYWZmpZwozMyslBOFmZmVcqIwM7NSThRmZlbKicLMzEpVmigkzZC0XNIKSac1GH+ypJskLZV0jaQphXGn5+mWS3pDlXGamVlzlSUKSaOAucARwBTg+GIiyC6KiJdExIHAp4HP5WmnAMcB+wMzgC/n+szMbJBVeURxELAiIlZGxOPAfOCoYoGIeKjQ+2wgcvdRwPyIWB8RtwMrcn1mZjbItq2w7j2Auwv9vcDB9YUknQKcCowGXluY9rq6afeoJkwzMytTZaJo9CLNeMaAiLnAXElvB84ATmh1WkkzgZkA48ePp6enp9/Bdnf3e9K26+hYR3d3z1CHAcAAvtIht2TJUEfwtI6OdXz2sz1DHQYAU6cOdQT9599pY5X/TiOikj/gEOCKQv/pwOkl5bcBHmxUFrgCOKRsflOnTo2BgOHz19191ZDHUPvbkg31d+dl2n5D/d2NxGUKLI4o355XeY1iETBJ0kRJo0kXpxcUC0iaVOh9E/DH3L0AOE7SGEkTgUnA7yqM1czMmqjs1FNEbJA0i3Q0MAq4ICKWSZpDymALgFmSpgNPAGtJp53I5S4GbgY2AKdExJNVxWpmZs1VeY2CiFgILKwbdmah+/0l034c+Hh10ZmZWSv8ZLaZmZVyojAzs1JOFGZmVsqJwszMSjlRmJlZKScKMzMr5URhZmalnCjMzKyUE4WZmZVyojAzs1JOFGZmVsqJwszMSlXaKODW5izObks9uzN5wHWd06ZYzMx8RGFmZqWcKMzMrJQThZmZlXKiMDOzUk4UZmZWyonCzMxKOVGYmVkpJwozMyvlRGFmZqUqTRSSZkhaLmmFpNMajD9V0s2SbpT0C0l7F8Y9KWlp/ltQZZxmZtZcZU14SBoFzAVeD/QCiyQtiIibC8VuADoj4lFJ7wU+DRybxz0WEQdWFZ+ZmbWmyiOKg4AVEbEyIh4H5gNHFQtExFUR8WjuvQ7oqDAeMzPrB0VENRVLxwAzIuKk3P8u4OCImNWk/HnAnyPiY7l/A7AU2AB8MiIubzDNTGAmwPjx46fOnz+/3/EuWdLvSTfandUDrwQY0zGG9b3rB1THKnZrSyxTp7almiHRjmXaLh0d6+jtHTvUYQBepu0yUpbptGnTlkREZ1mZKluPVYNhDbOSpHcCncBhhcF7RcQqSfsAV0q6KSJu26SyiHnAPIDOzs7o6urqd7DTpvV70o3a1Xrs5O7JLJ+9fEB1nMPxbYmlov2IQdGOZdou3d09zJ7dNdRhAF6m7bI1LdMqTz31AnsW+juAVfWFJE0HPgIcGREbd6MjYlX+vxLoAV5WYaxmZtZElYliETBJ0kRJo4HjgE3uXpL0MuArpCRxT2H4LpLG5O5xwKuA4kVwMzMbJJWdeoqIDZJmAVcAo4ALImKZpDnA4ohYAHwGGAtcIgngrog4EngR8BVJT5GS2Sfr7pYyM7NBUukb7iJiIbCwbtiZhe7pTaa7FnhJlbGZmVlr/GS2mZmVcqIwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK+VEYWZmpZwozMyslBOFmZmVcqIwM7NSThRmZlbKicLMzEpV2iigmdlw064XjO3O5AHXdU6bYqmajyjMzKyUE4WZmZVyojAzs1JOFGZmVsqJwszMSjlRmJlZKScKMzMr5URhZmalKk0UkmZIWi5phaTTGow/VdLNkm6U9AtJexfGnSDpj/nvhCrjNDOz5ipLFJJGAXOBI4ApwPGSptQVuwHojIiXApcCn87TPgc4CzgYOAg4S9IuVcVqZmbNVXlEcRCwIiJWRsTjwHzgqGKBiLgqIh7NvdcBHbn7DcDPImJNRKwFfgbMqDBWMzNrospEsQdwd6G/Nw9r5j3Aj/s5rZmZVaTKRgHVYFg0LCi9E+gEDtucaSXNBGYCjB8/np6enn4FCtDd3e9JN9qdyQOvBBjTMYbJ3QOrq5uetsQygK90yLVjmbZLR8c6urt7hjoMwMvUv9PNV2Wi6AX2LPR3AKvqC0maDnwEOCwi1hem7aqbtqd+2oiYB8wD6OzsjK6urvoiLZs2rd+TbtSuViknd09m+ezlA6rjHI5vSyzRMLVvGdqxTNulu7uH2bO7hjoMwMvUv9PNV3rqSdJrC90T68a9tY+6FwGTJE2UNBo4DlhQV8fLgK8AR0bEPYVRVwCHS9olX8Q+PA8zM7NB1tc1iuKB3mV1484omzAiNgCzSBv4W4CLI2KZpDmSjszFPgOMBS6RtFTSgjztGuDfSclmETAnDzMzs0HW16knNelu1P8MEbEQWFg37MxC9/SSaS8ALuhrHmZmVq2+jiiiSXejfjMzG4H6OqLYJ58OUqGb3D+x+WRmZjZS9JUoig/I1d+YNoxuPjQzs6qUJoqI+GWxX9J2wIuBP9XdpWRmZiNUX7fHni9p/9y9E/B74JvADZLacwOwmZkNa31dzH5NRCzL3f8A3BoRLwGmAv9WaWRmZjYs9JUoHi90vx64HCAi/lxZRGZmNqz0lSgekPTm/AT1q4CfAEjaFnhW1cGZmdnQ6+uup38CzgWeD3ygcCTxOuB/qgzMzMyGh77uerqVBu+BiIgrcNtLZmZbhdJEIencsvER8b72hmNmZsNNX6eeTgb+AFxMaiK8z/adzMxsZOkrUewG/D1wLLAB+B5wWX49qZmZbQVK73qKiPsj4vyImAacCOwMLJP0rsEIzszMhl5Lb7iT9HLgeNKzFD8GllQZlJmZDR99Xcw+B3gz6cVD84HT8wuJzMxsK9HXEcVHgZXAAfnvE5IgXdSOiHhpteGZmdlQ6ytR+J0TZmZbub4euLuz0XBJo4DjgIbjzcxs5OirmfEdJZ0u6TxJhyv5Z9LpqLcNTohmZjaU+jr19C1gLfAb4CTgX4HRwFERsbTi2MzMbBjo853Z+f0TSPoacB+wV0Q8XHlkZmY2LPTVzPgTtY6IeBK4fXOShKQZkpZLWiHptAbjD5V0vaQNko6pG/ekpKX5b0Gr8zQzs/bq64jiAEkP5W4Bz8r9tdtjd2w2Yb7gPZf0kF4vsEjSgoi4uVDsLtIT37MbVPFYRBzY2scwM7Oq9HXX06gB1H0QsCIiVgJImg8cBWxMFBFxRx731ADmY2ZmFVJEVFNxOpU0IyJOyv3vAg6OiFkNyn4D+FFEXFoYtgFYSmqM8JMRcXmD6WYCMwHGjx8/df78+f2Od0kbGiXZndUDrwQY0zGG9b3rB1THKnZrSyxTp7almiHRjmXaLh0d6+jtHTvUYQBepv6dbmratGlLIqKzrExLbT31U6MmyTcnK+0VEask7QNcKemmiLhtk8oi5gHzADo7O6Orq6vfwU6b1u9JNzqLswdeCTC5ezLLZy8fUB3ncHxbYqloP2JQtGOZtkt3dw+zZ3cNdRiAl6l/p5uvr4vZA9EL7Fno7yC906IlEbEq/18J9AAva2dwZmbWmioTxSJgkqSJkkaTnuRu6e4lSbtIGpO7xwGvonBtw8zMBk9liSK3MjuL9G7tW4CLI2KZpDmSjgSQ9ApJvaSXI31F0rI8+YuAxZJ+D1xFukbhRGFmNgSqvEZBRCwEFtYNO7PQvYh0Sqp+umuBl1QZm5mZtabKU09mZjYCOFGYmVkpJwozMyvlRGFmZqWcKMzMrJQThZmZlXKiMDOzUk4UZmZWyonCzMxKOVGYmVkpJwozMyvlRGFmZqWcKMzMrJQThZmZlXKiMDOzUpW+j8JsS9eu9yvvzuQB13VOm2Ix21w+ojAzs1JOFGZmVsqJwszMSjlRmJlZKScKMzMrVWmikDRD0nJJKySd1mD8oZKul7RB0jF1406Q9Mf8d0KVcZqZWXOVJQpJo4C5wBHAFOB4SVPqit0FnAhcVDftc4CzgIOBg4CzJO1SVaxmZtZclUcUBwErImJlRDwOzAeOKhaIiDsi4kbgqbpp3wD8LCLWRMRa4GfAjApjNTOzJqpMFHsAdxf6e/Owqqc1M7M2qvLJbDUYFu2cVtJMYCbA+PHj6enpaTm4et3d/Z50o92ZPPBKgDEdY5jcPbC6uulpSywD+EqHnJdpY16mXqabq8pE0QvsWejvAFZtxrRdddP21BeKiHnAPIDOzs7o6uqqL9KyadP6PelG7WruYXL3ZJbPXj6gOs7h+LbEEq2m9mHIy7QxL9OzB14JW9cyrfLU0yJgkqSJkkYDxwELWpz2CuBwSbvki9iH52FmZjbIKksUEbEBmEXawN8CXBwRyyTNkXQkgKRXSOoF/h74iqRledo1wL+Tks0iYE4eZmZmg6zS1mMjYiGwsG7YmYXuRaTTSo2mvQC4oMr4zMysb34y28zMSjlRmJlZKScKMzMr5URhZmalnCjMzKyUE4WZmZVyojAzs1JOFGZmVsqJwszMSjlRmJlZKScKMzMr5URhZmalnCjMzKyUE4WZmZVyojAzs1JOFGZmVsqJwszMSjlRmJlZKScKMzMr5URhZmalnCjMzKyUE4WZmZWqNFFImiFpuaQVkk5rMH6MpO/l8b+VNCEPnyDpMUlL89/5VcZpZmbNbVtVxZJGAXOB1wO9wCJJCyLi5kKx9wBrI+IFko4DPgUcm8fdFhEHVhWfmZm1psojioOAFRGxMiIeB+YDR9WVOQq4MHdfCrxOkiqMyczMNpMiopqKpWOAGRFxUu5/F3BwRMwqlPlDLtOb+28DDgbGAsuAW4GHgDMi4uoG85gJzAQYP3781Pnz5/c73iVL+j3pRruzeuCVAGM6xrC+d/2A6ljFbm2JZerUtlQzJLxMG/My9TItmjZt2pKI6CwrU9mpJ6DRkUF9VmpWZjWwV0TcL2kqcLmk/SPioU0KRswD5gF0dnZGV1dXv4OdNq3fk250FmcPvBJgcvdkls9ePqA6zuH4tsRS0X7EoPAybczL9OyBV8LWtUyrPPXUC+xZ6O8AVjUrI2lbYCdgTUSsj4j7ASJiCXAbsF+FsZqZWRNVJopFwCRJEyWNBo4DFtSVWQCckLuPAa6MiJC0a74YjqR9gEnAygpjNTOzJio79RQRGyTNAq4ARgEXRMQySXOAxRGxAPg68C1JK4A1pGQCcCgwR9IG4Eng5IhYU1WsZmbWXJXXKIiIhcDCumFnFrr/Cvx9g+kuAy6rMjYzM2uNn8w2M7NSThRmZlbKicLMzEo5UZiZWSknCjMzK+VEYWZmpZwozMyslBOFmZmVcqIwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK+VEYWZmpZwozMyslBOFmZmVcqIwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK1VpopA0Q9JySSskndZg/BhJ38vjfytpQmHc6Xn4cklvqDJOMzNrrrJEIWkUMHCeLFEAAAdcSURBVBc4ApgCHC9pSl2x9wBrI+IFwOeBT+VppwDHAfsDM4Av5/rMzGyQVXlEcRCwIiJWRsTjwHzgqLoyRwEX5u5LgddJUh4+PyLWR8TtwIpcn5mZDbIqE8UewN2F/t48rGGZiNgAPAg8t8VpzcxsEGxbYd1qMCxaLNPKtEiaCczMveskLd+sCNvsnHZVNJtxwH0Dq6Q90ajRktiKeJmOPF6mz7B3XwWqTBS9wJ6F/g5gVZMyvZK2BXYC1rQ4LRExD5jXxpiHBUmLI6JzqOOw9vEyHXm2pmVa5amnRcAkSRMljSZdnF5QV2YBcELuPga4MiIiDz8u3xU1EZgE/K7CWM3MrInKjigiYoOkWcAVwCjggohYJmkOsDgiFgBfB74laQXpSOK4PO0ySRcDNwMbgFMi4smqYjUzs+aUduBtOJE0M59WsxHCy3Tk2ZqWqROFmZmVchMeZmZWyomiIpLWNRh2sqR3D0U81pikJyUtlfQHSZdI2r6P8s9YroNBUqekc/so0yXpR4MV00hWt178UNLOefgESY/lcb+XdK2kyXlcl6QH87ilkn4+tJ+ifZwoBlFEnB8R36yqfiVeppvnsYg4MCJeDDwOnDzUATUSEYsj4n1DHcdWpLherAFOKYy7LY87gNSyxIcL467O4w6MiOmDGXCVvFEZRJLOljQ7d/dI+pSk30m6VdJr8vBRkj4jaZGkGyX9Ux4+VtIvJF0v6SZJR+XhEyTdIunLwPVs+vyJbZ6rgRcASDo1703+QdIH6gtK+lZtGeT+70g6UtKJkr4v6SeS/ijp04Uyx+dl9wdJnyoMX5fXhSWSfi7poLx+rJR0ZC6z8Wghj79W0g3FPVqrzG9o3jLEjsDaQYxlaESE/yr4A9Y1GHY2MDt39wCfzd1vBH6eu2cCZ+TuMcBiYCLpVuYd8/BxpPavBEwAngJeOdSfeUv8qy2n/P3+N/BeYCpwE/BsYCywDHhZXfnDgMtz907A7bmOE4GVedjfAHeSkvfuwF3ArrnclcDRefoAjsjdPwB+CmwHHAAszcO7gB/l7h2BbXP3dOCy+jL+a9t6MQq4BJiR+ycAjwFLgduA1cBehe//wTxuKfCRof4c7fqr8sls69v38/8lpBUQ4HDgpZKOyf07kR447AU+IelQUmLYAxify9wZEdcNSsQjz7MkLc3dV5Oe7Xkv8IOIeARA0veB1wA31CaKiF9KmivpecBbSRvrDalNS34REQ/maW8mNZHwXKAnIu7Nw78DHApcTjrl9ZNc9U3A+oh4QtJNPL1eFO0EXChpEinJbNeWb8KKauvFBNLv82eFcbdFxIEAko4ltQ4xI4+7OiLePJiBDgYniqG1Pv9/kqeXhYB/jogrigUlnUjaG52aNyJ3kPZYAR6pPtQR67Haj74mt2Dcim8B7yA9KPqPheHrC921ZVtW5xORd0lJOwHrASLiqdy0Tb1/B66KiL9VeodLT4vxWusei4gDJe0E/Ih0jaLRzQQLgP8a1MiGgK9RDD9XAO+VtB2ApP0kPZu0F3lPThLTaKEhL+u3XwFHS9o+f/d/SzraqPcN4AOQWhPoo87fAodJGqf0bpXjgV/2M76dgD/l7hP7WYe1IB8Zvg+YXftN1nk16RTUiOYjiupsL6m30P+5Fqf7Gulw9/q8Z3svcDTwHeCHkhaTzn/+bxtjtYKIuF7SN3i6fbGvRcQNDcr9RdItpNNHfdW5WtLpwFWko4uFEfHf/Qzx06RTT6eSrnVYhSLiBkm/Jx05Xg3sm09LiXTa8KShjG8w+Mlss37Kz1zcBLy8dk3CbCTyqSezfpA0nXRU9yUnCRvpfERhZmalfERhZmalnCjMzKyUE4WZmZVyojArUWwtVtIbc/tNe1U8zx5JW8W7mG3L4ERh1gJJrwO+RGrz564Wp/FzSjYieEU260Nu2ferwBsj4rY8bFfgfKB2dPGBiPi1pLNJDQBOAO6T9FPgSGB7YF9SG1L/lus4HDiH1PjjbcA/RETxCGYUqe2pTlKbThdExOer/bRmz+REYVZuDKlV2a6IKD4N/0Xg8xFxTT4VdQXwojxuKvDqiHgst9F1IPAyUhtOyyV9idQC6RnA9Ih4RNKHgFOBOYV5HAjsEemdCCi/PMdssDlRmJV7ArgWeA/w/sLw6cCUQvuBO0raIXcviIjHCmUbtSa7MzAF+HWuYzTpvQdFK4F9cmL5H1Lz42aDzonCrNxTwNuAn0v6cER8Ig/fBjikLiGQN/r1rfk2a032ZxFxfLMZR8RaSQcAbyC1Xvo2Nm2l1mxQ+GK2WR8i4lHgzcA7JL0nD/4pMKtWRtKBjaYtcR3wKkm1N+ptL2m/YgFJ44BtIuIy4KPAy/v5EcwGxEcUZi2IiDWSZgC/knQfqenpuZJuJP2OfsVmvG87Iu7N1y++K2lMHnwGcGuh2B7Af+np96CfPsCPYdYvbuvJzMxK+dSTmZmVcqIwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK+VEYWZmpZwozMys1P8HNbour/aCDn8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.grid(True)\n",
    "plt.bar(['Linear','Polynomial','RBF'], train_mean, width=width, \n",
    "        color='b', label='Training Accuracy')\n",
    "plt.bar(['Linear','Polynomial','RBF'], val_mean, \n",
    "        width=0.5*width, color='r', alpha=0.5, label='Validation Accuracy')\n",
    "plt.title('Comments Data')\n",
    "plt.suptitle('Training Accuracy and Validation Accuracy across Kernels')\n",
    "plt.ylabel('RMSE')\n",
    "plt.xlabel('Kernels')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM_final = grid_search_SVR.best_estimator_\n",
    "pred_test_y = SVM_final.predict(pca_test_x) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from math import sqrt\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, pred_test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151.93758160270653"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   5 tasks      | elapsed:    2.4s\n",
      "[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:    4.6s\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:    9.7s\n",
      "[Parallel(n_jobs=-1)]: Done  27 out of  30 | elapsed:   16.6s remaining:    1.8s\n",
      "[Parallel(n_jobs=-1)]: Done  30 out of  30 | elapsed:   18.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "             estimator=KNeighborsRegressor(algorithm='auto', leaf_size=30,\n",
       "                                           metric='minkowski',\n",
       "                                           metric_params=None, n_jobs=None,\n",
       "                                           n_neighbors=5, p=2,\n",
       "                                           weights='uniform'),\n",
       "             iid='warn', n_jobs=-1,\n",
       "             param_grid={'n_neighbors': [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring=None, verbose=10)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "knn_model = KNeighborsRegressor()\n",
    "param_grid = {'n_neighbors' : [i for i in range(1,21,2)]}\n",
    "\n",
    "grid_search_KNN = GridSearchCV(estimator = knn_model, param_grid = param_grid, \n",
    "                          cv = 3, n_jobs = -1,verbose = 10,return_train_score = True)\n",
    "\n",
    "grid_search_KNN.fit(pca_train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_k = grid_search_KNN.best_params_['n_neighbors']\n",
    "\n",
    "results = grid_search_KNN.cv_results_\n",
    "num_param = len(list(param_grid.values())[0])\n",
    "train_scores = np.hstack((results['split0_train_score'].reshape((num_param,1)),results['split1_train_score'].reshape((num_param,1)),results['split2_train_score'].reshape((num_param,1))))\n",
    "val_scores = np.hstack((results['split0_test_score'].reshape((num_param,1)),results['split1_test_score'].reshape((num_param,1)),results['split2_test_score'].reshape((num_param,1))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_mean = np.mean(train_scores,axis = 1)\n",
    "val_mean = np.mean(val_scores,axis = 1)  \n",
    "width = 0.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Number of Neighbors')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEjCAYAAADHWv01AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3debwcVZn/8c83CYsaVgMRCBjAAIMjIDeAuJIBMaAQxgGEQRYFGRUYRsEB1EHk5y4uo6Aoi4IIYVMnw0RRIQFRWRIEZBGJLBKJLAIhQQUCz++PczpUOl19q29u3e6b+32/Xv26XVWnqp5bVV1P1amqU4oIzMzMWhnV7QDMzKx3OUmYmVkpJwkzMyvlJGFmZqWcJMzMrJSThJmZlVqpkoSk0ZIWS9pkMMva0JC0m6T7h2Laku6W9KY64pB0tqSPDnR8s4GSdIGkUwZzml1NEnkn3fi8IOlvhe6DOp1eRDwfEWMj4o+DWXagJB0hKSS9s655jBSS7pF0SIv+x0m6vtPpRcSWEfGLQYjrCEmzm6Z9RER8ZkWn3c88vV31CEnXSTqs0L2rpCck7dvFsAZNV5NE3kmPjYixwB+BvQr9vt9cXtKYoY9yhRwKPJ7/DilJo4d6njU7H1guSQAHA+cNcSzd5u2qoqGOV9IewOXAoRFx2QDG773lGxE98QHuB3Zr6vcp4GLgImARcBiwM3A98CSwAPgasEouPwYIYGLuviAP/3Ee/9fApp2WzcP3AH4PLAS+DvwSOKzN/7MZ8AKwL/AssF7T8HcCtwBPAfOA3XP/lwPfzf/bE8Dluf8RwOzC+K3iPwP4CfA0sAuwd57HIlIS/q+mGN6cl+VC4EHSDndn4CFgVKHcu4A5Jf9n6TyAV+UYDwHmA48CJxaGvxT4Xv4/7wBOAO4vmc8rgSXAhEK/1wB/B9YtLKO7cix/AI4olN2tOO0czy5V4gA+Dtybp3sHsHfT/J8HFgOPFdbFKYXx35/X8V+AHwEbNK3Df8vDnwC+1s/vZMRvV2XzaBPv2rn/o6T9zEmAcvktgGvzdB4DLsz9R5H2B4/kYbcBW5fEeh1p3zSNtF96W9PwrYGfkxL774B/KQxrFW9/+6L+pndK/r4+MDPH9Dhw7YD2zQMZqY4P5UniWWCvvNJeAuwA7JQ35s1IO+6j22zgjwGTgVVICeeCAZRdP6+saXnYh4HnaJ8kPgn8Kn+/C/j3wrDX5xW3a/6/Nga2zMOuBC4E1gFWBd7cwY/5CdKPcRSwGvBPwD/m7m3z//eOXH7T/D/tn6c1DtguD7sbeGthXv8LHFvyf7abRyNJnAmsDmwPPANMysNPA2bn//WVwJ2UJIlcfhbLJpkvApcVuvfK24RyXH8DtsnD2iWJtnHkZbRB/h//lZQQxrdaLy1+qLuTdjTb5WXwDeDqpnX4P8BawETSj3m3NsvA21X7ebSK90LgB8AaefuYRzrSB7iUdFAwKq+fN+T+bwduzOtlFGnH/IqSWK8jJf8ngClNw9YA/kQ6UBoD9JEOFrZsE2+7fVGV6TW2vS8Cp+dprAq8ZUD75sHc0a/Ih/IkcXU/4x0PXNpmAz+zUHZv4PYBlH0v8IvCMJGOyFomiTz8Xl5MXv8FzC0MPwf4YovxNiYdLa/VYliVH/O5/Syr0xvzzTFdWlLuY8B5+fs44K/A+hXXY3EejSTxisLwm4F98/c/Ftc58EHaJ4nDgDvz91H5x7JXm/JXAEfl7+2SRKdx3A68vdV6KayLxg/1POAzhWFrks48JhTW4esKw38AHO/tasDzWCZe0g5yCbBFod9RwM/z9wuBbwIbNU1nd9JR+k4Uzn5K5nkd6czt18DqTcMOAmY19TsH+FjZ8qX9vqjK9Brb3mfy9rR5lWVc9hkOdzc9WOyQtJWk/5P0Z0lPAaeSNrgyfy58/yswdgBlNyzGEWkNzG8znTeTfpiX5O4Lge0l/WPu3phUHdJsY1KVxcI2026neVntLGm2pEclLSTtEBrLqiwGSFUv+0h6KXAAaaN8pFXBfuYBQESULdcNmmJ+oJ//7zJgE0mTSTv9VUin5I1Y3iHpBkmPS3qS9ENvt200tI1D0mGSbpX0ZJ7uVhWnC2nbWTq9iHiKdOS4UaFM1W3U21X/82iOd31gNMuu0wd4cfkfR9qO5kj6raRDASLip6Qz4G8CD0s6U9Iabeb5UVI14A8krVro/0rgDY1tJ28/7yJtc63ibSjbJqpMr+Fz+X+9StIfJH2kTfylhkOSiKbub5GO5F4VEWsCJ5OOsOq0gHTkB4AkseyPvNmhpGV7m6Q/k65fBC9eeH0Q2LzFeA8C4ySt2WLY06S684ZXtCjTvKymky6ibRwRawFn8+KyKouBSHd8zSFVrx1M+nGXaTeP/vyZ9INvaHs7ckQsJh0ZHZLjujAilgBIegkpiXyWVBW0NvDTirGUxiFpM9KO4gPAy/N0f1eYbvMyb/YQ6YfdmN4apCqfP1WIq5m3q37m0SLeR0hnbq8s9NuEvPwjYkGku9E2IJ1hfFvSpnnYVyNie1K119akauYyi0nXLdcDLi7cZPMgcFVErF34jI2Io0vi7U+V6ZHjfyoiPhQRE4F9gBMkvaWDeQHDI0k0W4N0IelpSf9AuuhXtytIR2x75ZV/LGljWE4+StoXOJxUD934fAh4d7574RzgCElTJI2SNEHSlhHxIOmC1BmS1pa0iqQ350nfCmwj6TV5h/iJCnGvATweEX+X9DrS0VvDBcBUSf8iaYykcZK2LQw/n3SBbytSnflA5tGfS4CP5v91E2C5Db2F84ADgX9m2buaViPVuz4KPC/pHaS6+RWNYyzpR/wo6fjgCNIyaXgYmCBplZJpXwQcLmkbSauRktgvIqLdmehyvF11NI+lIuI50sHDZySNzQngQ3kaSNpfUuOA70nSun5e0o75M4aUSJ8lJZtS+Sxxd9K1pQvyOpkBvFrSv+blvkqe7pbtptVG5enl/dXm+aB2YY6/7f/QynBMEseRjqgWkc4qLq57hhHxMOmU7suki0SbA78hXYRt9s4c2wUR8efGBziLdOH9rRHxK+B9pDsYFpIuyDaOZN+d//6etAM6JsdwJ6mOcTbpAuC1FUL/APBZSYtIp8ONagoi4j7Shd4TSBdLbybdrdNwOeki32UR8beBzKOCT5DO0u4nVRudX2GcWaTT7/si4jeNnhHxJOnH/0PS/7MvKbmvUBwRcRtpPd2Yy2wF3FAY92fAPaQqiWIVQWP8n5CqRH+Yx9+EVK/cKW9X1efR7IOknfx9wDWkg4vGOt4JuEnS06Sz1KPyGc/apKT7JGm7WAB8pc08GrE9AbyVdPZxLmmdvY20/BeQzlo/Szqo6ViuMqw6vS2Bq0lnOb8E/jsirut0no3bwKwD+QjhIdIF2BV+IKsX5aOP+0gX52d3ORxbSXi7Gn6G45lEV0iaKmmtXGXwX6Q7Jm7sclh12p90pnRNtwOxlYq3q2FmuD3B3E1vBL5Pqve+A9gnIlpVNw17kq4DJgEHhU81bZB4uxqeXN1kZmalXN1kPSXftTFHqZHHBZJ+LOmN3Y5rReT7/Y9YgXH/LmmRpKckzZV0Yq72rDqNkPSqgczfzEnCeoakDwNfJd1tM550J9A3SPfVj2RHR8QapAemjiPdDjozXwQ2q5WThPUESWuRbhU9KiJ+EBFPR8RzEfG/EfGRXGY1SV+V9FD+fLVxRC1pF0nzJf2npEfyWcg+kvaU9Hulp7A/WpjfKZIuVWp/f5HS07ZbSDopj/+gpN2L8Uk6J0/3T5I+le9yazyRfZ2k05SaiL5PqTVQJH0aeBNwej47Ol3JV/J8Fkq6TS8+NV0qL5PZpGYadia1L0S+T/7XSk/gLsjzWDUPa9zSemue/7skrSPpCqWnmZ/I3ye0nKmNeE4S1it2JjWw9sM2ZT4GvI70ENm2wI6kFlobXpGnsRHpSfyzSPeT95F21CcrPUHdsBfpqd91SM+9XEn6TWxESljfKpQ9j3RH26uA15IemipWIe1Ees5gHPAF4BxJioiPAb8gnQ00nozdndTExhak+/HfRXr+ppLCk8uNlyY9T3pGZBxpOe5KejaAiGg8NLdtnv/F+X/8Dukp5E1IDSGeXnX+NrI4SViveDmpfaElbcocBJwaEY9ExKOkFlEPLgx/Dvh0fsp2Ommn+d8RsSgi7iDdlbZNofwvIuLKPM9LSU/Rf64w/kSlJ5THk5pc+I98NP8I6cGq4lPAD0TEWRHxPCmhbECqMmvlOdITxVuRbh65KyIW9LeAmjwErAsQEXMj4vqIWBIR95OSW2nzCxHxl4i4PCL+GhGLgE+3K28jm2+BtV7xF1L7QmPaJIplGsvL3zcsTiPvpCEdHUN6uphCv2Ljec3DHmsx/tg8j1WABYXLAKNYtmG2pU9bR8Rfc7mWDfVFxNWSTie9R2ATST8ktfz6VKvyJTYCfgUgaQtSawCTSe0wjQHmlo2o1MTHV4CppLMogDUkjS78/2aAzySsd/ya9AKffdqUWaaxPFJVyUN1BpU9SHoAbFyhUbU1I+LVFcdf7j7ziPhaRPQBryZVO1VuoVPSxqQqtMbT/t8kNTo4KTd6+VHaN2x4HKnJhp1y+UaVlC+E23KcJKwn5DZpTiY1QrePpJfmxsv2kPSFXOwi4OOS1pM0Lpe/YAhiW0BqUfZLktZUajxvc1VvUfNhUntFAEjaQdJOSo0CPs2Lb7drKy+Tt5AaxruR9NYxSFVXTwGLJW1FaveodP65/N+AJyWtS7VG/WyEcpKwnhERXyY1x/xxUqurD5JaZP1RLvIp0gXb24Dfkhp2+9QQhXcI6Wn7O0nvg7iM1m34t/LfwL75TqKvkV48dFaezgOkqrbT2ox/ulJDdw+TbhG+HJgaES/k4ceT3pi3KE+3udHLU4Dz8t1P++dpvIT09rPrSa/ONGvJT1ybmVkpn0mYmVkpJwkzMyvlJGFmZqWcJMzMrNSwe5hu3LhxMXHixKXdTz/9NC972cu6F1AFjnFw9HqMvR4fOMbBMhxjnDt37mMRsV7HE4qIYfXp6+uLolmzZkWvc4yDo9dj7PX4IhzjYBmOMQJzYgD7XFc3mZlZKScJMzMr5SRhZmalnCTMzKyUk4SZmZWqLUlIOje/nvH2kuGS9DVJ8/LrG7evKxYzMxuYOs8kvkt6qUmZPYBJ+XMkqU18MzPrIbUliYi4Fni8TZFpwPn5Ft7rgbUlVW162czMhkCtTYVLmghcERH/2GLYFaT3CV+Xu68CToiIOS3KHkk622D8+PF906dPXzps8eLFjB3b8i2Ry5lb+kLHZfX1VStXVScxdotjXHG9Hh84xsEyHGOcMmXK3IiY3PGEBvIEXtUPMBG4vWTY/wFvLHRfBfT1N80VeeIaqn0G23B8OrMX9XqMvR5fhGMcLMMxRobhE9fzgY0L3RMYmvcVm5lZRd1MEjOAQ/JdTq8DFkZ6l7CZmfWI2lqBlXQRsAswTtJ80svWVwGIiDNJL3HfE5gH/BV4T12xmJnZwNSWJCLiwH6GB3BUXfM3M7MV5yeuzcyslJOEmZmVcpIwM7NSThJmZlbKScLMzEo5SZiZWSknCTMzK+UkYWZmpZwkzMyslJOEmZmVcpIwM7NSThJmZlbKScLMzEo5SZiZWSknCTMzK+UkYWZmpZwkzMyslJOEmZmVcpIwM7NSThJmZlbKScLMzEo5SZiZWSknCTMzK+UkYWZmpZwkzMyslJOEmZmVcpIwM7NSThJmZlbKScLMzEo5SZiZWalak4SkqZLuljRP0okthm8iaZak30i6TdKedcbTC6RqHzOzXlBbkpA0GjgD2APYGjhQ0tZNxT4OXBIRrwUOAL5RVzxmZta5Os8kdgTmRcS9EfEsMB2Y1lQmgDXz97WAh2qMx8zMOqSIqGfC0r7A1Ig4IncfDOwUEUcXymwA/BRYB3gZsFtEzG0xrSOBIwHGjx/fN3369KXDFi9ezNixYyvFNHe5KbfW11etXFXFGLsVQ386WY7d0usx9np84BgHy3CMccqUKXMjYnLHE4qIWj7AfsDZhe6Dga83lfkwcFz+vjNwJzCq3XT7+vqiaNasWVEVVPsMtmKM3Yqhkxh7Va/H2OvxRTjGwTIcYwTmxAD25XVWN80HNi50T2D56qTDgUsAIuLXwOrAuBpjMjOzDtSZJG4CJknaVNKqpAvTM5rK/BHYFUDSP5CSxKM1xmRmZh2oLUlExBLgaOBK4C7SXUx3SDpV0t652HHA+yTdClwEHJZPi8zMrAeMqXPiETETmNnU7+TC9zuBN9QZg5mZDZyfuDYzs1JOEmZmVspJwszMSjlJmJlZKSeJEahVg4Jz57qRQTNbnpOEmZmVcpIwM7NSThJmZlbKScLMzEo5SZiZWSknCTMzK1Vr201mrVS9vdZNPZp1n88kzMyslJOEmZmVcpIwM7NSviZhI5Kvi5hV4zMJMzMr5SRhZmalnCTMzKyUr0mYdYmvi9hw4DMJMzMr5SRhZmalXN1kNoJVqfJyddfI5jMJMzMr5SRhZl3j9633PicJMzMr5SRhZmalfOHazEY0P6/Sns8kzMyslM8kzMy6rJfPZmo9k5A0VdLdkuZJOrGkzP6S7pR0h6QL64zHzMw60/ZMQtI72w2PiB+0GXc0cAbwVmA+cJOkGRFxZ6HMJOAk4A0R8YSk9TsJ3szM6tVfddNe+e/6wOuBq3P3FGA2UJokgB2BeRFxL4Ck6cA04M5CmfcBZ0TEEwAR8UgnwZuZWb0UFSq5JF0BvC8iFuTuDUg799IzDUn7AlMj4ojcfTCwU0QcXSjzI+D3wBuA0cApEfGTFtM6EjgSYPz48X3Tp09fOmzx4sWMHTu2wr+aHtKpoq+vWrmqijF2K4aiVjFMmLCY+fOXXY51xTDQZdDJuq4rhnY6ja8b20JzjFViWJm3xbIYWinGMJjb4kBj6E9zjFOmTJkbEZM7DA0iot8PcHtT96jmfi3G2Q84u9B9MPD1pjJXAD8EVgE2JVVLrd1uun19fVE0a9asqCpd9un/M9iKMXYrhqJW8zvttFlDFsNAl0En67quGNrpNL5ubAvNMY70bbEshv6Ww2BuiwONoT/Lr2vmRD/7+lafqnc3zZZ0JXAREMABwKx+xpkPbFzongA81KLM9RHxHHCfpLuBScBNFeMyM7MaVbq7KVIV0ZnAtsB2wLcj4ph+RrsJmCRpU0mrkhLLjKYyPyJd30DSOGAL4N7q4ZuZWZ06eU7iZmBRRPxc0kslrRERi8oKR8QSSUcDV5KuN5wbEXdIOpV02jMjD9td0p3A88BHIuIvA/93zMxsMFVKEpLeR7pwvC6wObAR6cxi13bjRcRMYGZTv5ML3wP4cP6YmVmPqfow3VGkO5CeAoiIe0i3xZqZ2UqsapJ4JiKebXRIGkO6gG1mZiuxqkniGkkfBV4i6a3ApcD/1heWmZn1gqpJ4kTgUeC3wL+RrjN8vK6gzMysN/R74Tq3wXReRLwbOKv+kMzMrFf0eyYREc8D6+VnHczMbASp+pzE/cAvJc0Anm70jIgv1xGUmZn1hqpJ4qH8GQWsUV84ZmbWSyoliYj4ZN2BmJlZ76n6xPV6wH8CrwZWb/SPiH+qKS4zM+sBVW+B/T7wO1Jz3p8kXaNwS61mZiu5qkni5RFxDvBcRFwTEe8FXldjXGZm1gOqXrh+Lv9dIOntpIvYE+oJyczMekXVJPEpSWsBxwFfB9YEPlRbVGZm1hOq3t10Rf66kPySIDMzW/lVvbvpO7Ro9TVfmzAzs5VU1eqmKwrfVwf+meXfV21mZiuZqtVNlxe7JV0E/LyWiMzMrGdUvQW22SRgk8EMxMzMek/VaxKLSNcklP/+GTihxrjMzKwHVK1ucqN+ZmYjUNUzie3bDY+ImwcnHDMz6yVV7276BrA9cBupymkb4AbSk9gBuKE/M7OVUNUL1/cDfRExOSL6gNcC8yJiiluCNTNbeVVNEltFxG8bHRFxO7BdPSGZmVmvqFrddJeks4ELSNVL7wbuqi0qMzPrCVWTxHuADwDH5u5rgW/WEpGZmfWMqrfA/h34CvAVSesCE3I/MzNbiVW6JiFptqQ1c4K4BfiOpC/XG5qZmXVb1QvXa0XEU8A7ge/kO5x2628kSVMl3S1pnqQT25TbV1JImlwxHjMzGwJVk8QYSRsA+7Nsi7ClJI0GzgD2ALYGDpS0dYtyawD/TnruwszMekjVJHEqcCXp2YibJG0G3NPPODvm8vdGxLPAdGBai3L/D/gC4GscZmY9plKSiIhLI2KbiPhg7r43Iv6ln9E2Ah4sdM/P/ZaS9Fpg48Kb78zMrIcoYrkXzrUfQbo5Itq25ZTL7Qe8LSKOyN0HAztGxDG5exRwNXBYRNwvaTZwfETMaTGtI4EjAcaPH983ffr0pcMWL17M2LFjK8U+d26lYvT1VStXVTHGbsVQ1CqGCRMWM3/+ssuxrhgGugw6Wdd1xdBOp/F1Y1tojrFKDCvztlgWQyvFGAZzWxxoDP1pjnHKlClzI6Lz674R0dEH+E3FcjsDVxa6TwJOKnSvBTxGavLjflJ100PA5HbT7evri6JZs2ZFVVDtM9iKMXYrhqJW8zvttFlDFsNAl0En67quGNrpNL5ubAvNMY70bbEshv6Ww2BuiwONoT/Lr2vmRMX9fPEzkJcO/V/FcjcBkyRtKmlV4ABgRiE5LYyIcRExMSImAtcDe0eLMwkzM+uOjpNERHxc0mhJB/VTbglwNOmC913AJRFxh6RTJe09sHDNzGwotX3iWtKawFGkC84zgJ/l7o+QHqr7frvxI2ImMLOp38klZXepGrSZmQ2N/prl+B7wBPBr4AhSclgVmBYRt9Qcm5mZdVl/SWKziHgNQG4F9jFgk4hYVHtkZmbWdf1dk3iu8SUingfuc4IwMxs5+juT2FbSU/m7gJfkbpHux1qz1ujMzKyr2iaJiBg9VIGYmVnvGchzEmZmNkJUfTOdmVmtPsEpAGzIlku/fzL/HeoYGro9/27E0MxJwsx6cudkvcFJwqzLvIO2XuZrEmZmVspnEjbi+UjerJzPJMzMrJTPJKyrfBRv1tt8JmFmZqWcJMzMrJSThJmZlXKSMDOzUr5wPYIVLxo3mkLwRWMzK/KZhJmZlXKSMDOzUk4SZmZWyknCzMxKOUmYmVkpJwkzMyvlJGFmZqWcJMzMrJSThJmZlXKSMDOzUk4SZmZWym03dYlftmNmw0GtZxKSpkq6W9I8SSe2GP5hSXdKuk3SVZJeWWc8ZmbWmdqShKTRwBnAHsDWwIGStm4q9htgckRsA1wGfKGueMzMrHN1nknsCMyLiHsj4llgOjCtWCAiZkXEX3Pn9cCEGuMxM7MO1ZkkNgIeLHTPz/3KHA78uMZ4zMysQ4qIeiYs7Qe8LSKOyN0HAztGxDEtyr4bOBp4S0Q802L4kcCRAOPHj++bPn360mGLFy9m7NixlWKaO7da7H191cpVVYyxEcOGLFiu3ENsUFsMRa1iWG3Cajwz/5khiaG4Htoth+b5d7Ku64qhnU7j68a20BxjlRiGYlssxtDYFrsZQ0PZehjMbbEYw2BuC80xTpkyZW5ETO40tjqTxM7AKRHxttx9EkBEfLap3G7A10kJ4pH+pjt58uSYM2fO0u7Zs2ezyy67VIypWuyDvUiKMTZi6O/upppWS2kMW562JXcff/eQxFBcD+2WQ/P8O1nXdcXQTqfxdWNbaI6xSgxDsS0WY2hsi92MoaFsPQzmtliMYTC3heXXtQaUJOqsbroJmCRpU0mrAgcAM4oFJL0W+Bawd5UEYWZmQ6u2JBERS0hVSFcCdwGXRMQdkk6VtHcu9kVgLHCppFskzSiZnJmZdUGtD9NFxExgZlO/kwvfd6tz/mZmtmLcLIeZmZVykjAzs1JOEmZmVspJwszMSjlJmJlZKScJMzMr5SRhZmalnCTMzKyUk4SZmZVykjAzs1JOEmZmVspJwszMSjlJmJlZKScJMzMr5SRhZmalnCTMzKyUk4SZmZVykjAzs1JOEmZmVspJwszMSjlJmJlZKScJMzMr5SRhZmalnCTMzKyUk4SZmZVykjAzs1JOEmZmVspJwszMSjlJmJlZKScJMzMrVWuSkDRV0t2S5kk6scXw1SRdnIffIGlinfGYmVlnaksSkkYDZwB7AFsDB0rauqnY4cATEfEq4CvA5+uKx8zMOlfnmcSOwLyIuDcingWmA9OaykwDzsvfLwN2laQaYzIzsw4oIuqZsLQvMDUijsjdBwM7RcTRhTK35zLzc/cfcpnHmqZ1JHAkwPjx4/umT5++dNjixYsZO3Zs5wEuWLB8vw026Hw6FQw4xiHkGFdcr8cHjnGwDMcYp0yZMjciJnc6nTGDGtWyWp0RNGekKmWIiG8D3waYPHly7LLLLkuHzZ49m2J3Zaecsny/Aw/sfDoVDDjGIeQYV1yvxweOcbCMpBjrrG6aD2xc6J4APFRWRtIYYC3g8RpjMjOzDtSZJG4CJknaVNKqwAHAjKYyM4BD8/d9gaujrvovMzPrWG3VTRGxRNLRwJXAaODciLhD0qnAnIiYAZwDfE/SPNIZxAF1xbOcVtVNZma2jDqvSRARM4GZTf1OLnz/O7BfnTGYmdnA+YlrMzMr5SRhZmalnCTMzKyUk4SZmZVykjAzs1JOEmZmVspJwszMSjlJmJlZqdpaga2LpEeBBwq9xgGPlRTvFY5xcPR6jL0eHzjGwTIcY3xlRKzX6USGXZJoJmnOQJq/HUqOcXD0eoy9Hh84xsEykmJ0dZOZmZVykjAzs1IrQ5L4drcDqMAxDo5ej7HX4wPHOFhGTIzD/pqEmZnVZ2U4kzAzs5oMmyQhaaqkuyXNk3Rii+GrSbo4D79B0sQhjG1jSbMk3SXpDknHtiizi6SFkm7Jn5NbTWsIYr1f0m9zDHNaDJekr+XleJuk7Ycwti0Ly+cWSU9J+o+mMl1ZjpLOlfSIpNsL/daV9DNJ9+S/65SMe2guc4+kQ1uVqSm+L0r6XV6PP5S0dsm4bbeJmmM8RdKfCutzz5Jx2/7+a47x4kJ890u6pWTcoVqOLfc3tW2PEdHzH9Kb7f4AbAasCtwKbN1U5oPAmfn7AcDFQxjfBsBZxw4AAAgeSURBVMD2+fsawO9bxLcLcEUPLMv7gXFthu8J/BgQ8Drghi6u8z+T7u3u+nIE3gxsD9xe6PcF4MT8/UTg8y3GWxe4N/9dJ39fZ4ji2x0Yk79/vlV8VbaJmmM8BTi+wrbQ9vdfZ4xNw78EnNzl5dhyf1PX9jhcziR2BOZFxL0R8SwwHZjWVGYacF7+fhmwqyQNRXARsSAibs7fFwF3ARsNxbxrMA04P5LrgbUlbdCFOHYF/hARD/RbcghExLWkV+wWFbe584B9Woz6NuBnEfF4RDwB/AyYOhTxRcRPI2JJ7rwemDDY8+1EyTKsosrvf1C0izHvT/YHLqpj3lW12d/Usj0OlySxEfBgoXs+y++El5bJP4yFwMuHJLqCXM31WuCGFoN3lnSrpB9LevWQBvaiAH4qaa6kI1sMr7Ksh8IBlP8Ye2E5AoyPiAWQfrjA+i3K9MryfC/pDLGV/raJuh2dq8TOLaki6ZVl+Cbg4Yi4p2T4kC/Hpv1NLdvjcEkSrc4Imm/LqlKmVpLGApcD/xERTzUNvplUdbIt8HXgR0MZW8EbImJ7YA/gKElvbhreC8txVWBv4NIWg3tlOVbVC8vzY8AS4PslRfrbJur0TWBzYDtgAak6p1nXl2F2IO3PIoZ0OfazvykdrUW/tstyuCSJ+cDGhe4JwENlZSSNAdZiYKe2AyJpFdIK+35E/KB5eEQ8FRGL8/eZwCqSxg1VfIU4Hsp/HwF+SDqVL6qyrOu2B3BzRDzcPKBXlmP2cKMqLv99pEWZri7PfGHyHcBBkSulm1XYJmoTEQ9HxPMR8QJwVsm8u75N5n3KO4GLy8oM5XIs2d/Usj0OlyRxEzBJ0qb5KPMAYEZTmRlA40r9vsDVZT+KwZbrKs8B7oqIL5eUeUXjGomkHUnL/i9DEV8hhpdJWqPxnXRh8/amYjOAQ5S8DljYOIUdQqVHbL2wHAuK29yhwP+0KHMlsLukdXJVyu65X+0kTQVOAPaOiL+WlKmyTdQZY/F61z+XzLvK779uuwG/i4j5rQYO5XJss7+pZ3us+0r8IF7R35N0Ff8PwMdyv1NJPwCA1UnVE/OAG4HNhjC2N5JO2W4DbsmfPYH3A+/PZY4G7iDdmXE98PouLMPN8vxvzbE0lmMxTgFn5OX8W2DyEMf4UtJOf61Cv64vR1LSWgA8RzoaO5x0zesq4J78d91cdjJwdmHc9+btch7wniGMbx6p/rmxTTbu/tsQmNlumxjCGL+Xt7PbSDu5DZpjzN3L/f6HKsbc/7uNbbBQtlvLsWx/U8v26Ceuzcys1HCpbjIzsy5wkjAzs1JOEmZmVspJwszMSjlJmJlZKScJ6xpJIelLhe7jJZ0ySNP+rqR9B2Na/cxnv9wa56ym/hPz/3dMod/pkg7rZ3rvl3RIP2UOk3R6ybDFHYRv1i8nCeumZ4B3dvGJ6ZYkje6g+OHAByNiSothjwDH5gfAKomIMyPi/A7mP2jyU8Vmy3CSsG5aQnrF4oeaBzSfCTSOkJXeJ3GNpEsk/V7S5yQdJOlGpbb8Ny9MZjdJv8jl3pHHH630noWbcqNy/1aY7ixJF5Ie7mqO58A8/dslfT73O5n0YNOZkr7Y4v97lPRQ03Jt9kvaXNJPcmNwv5C0Ve5/iqTj8/cdcoy/zjEXn+DdMI9/j6QvNE37S5JulnSVpPVyv+0kXa8X3y2xTu4/W9JnJF1DSmj75f/xVknXtvifbIRxkrBuOwM4SNJaHYyzLXAs8BrgYGCLiNgROBs4plBuIvAW4O2kHfnqpCP/hRGxA7AD8D5Jm+byO5KelN26ODNJG5Lex/BPpIbodpC0T0ScCswhtYv0kZJYPwcc1+Ls5NvAMRHRBxwPfKPFuN8hPeW7M/B807DtgHflZfAuSY32eF5Gavdqe+Aa4BO5//nACRGxDSkJfqIwrbUj4i0R8SXgZOBtkRpQ3Lvkf7IRxEnCuipS65XnA//ewWg3RWpT/xlSMw0/zf1/S0oMDZdExAuRmna+F9iK1FbNIUpvF7uB1JTBpFz+xoi4r8X8dgBmR8SjkZqh/z7p5TRV/r/7SM3E/Gujn1Lrna8HLs1xfIv0IhkKZdYG1oiIX+VeFzZN+qqIWBgRfwfuBF6Z+7/Ai43QXQC8MSfgtSPimtz/vKb4i43W/RL4rqT3kV72YyOc6yCtF3yV1AT4dwr9lpAPYnKDZsV6/WcK318odL/Astt0c5szQWqb6piIWKZRM0m7AE+XxLeiL6/6DOlFWI3qm1HAkxGxXZtx+ptncRk8T/lvuUq7O0v/74h4v6SdSGdft0jaLiK61YCi9QCfSVjXRcTjwCWkqqCG+4G+/H0asMoAJr2fpFH5OsVmwN2kFi8/oNTUMpK2yK12tnMD8BZJ43K10YGkqpxKIuJ3pKP9d+Tup4D7JO2XY5CkbZvGeQJYpNQSL6SWT6sYRWoFGdLZy3URsRB4QtKbcv+Dy+KXtHlE3BARJwOPsWyz0jYC+UzCesWXSC28NpwF/I+kG0kXf8uO8tu5m7QzHE+q2/+7pLNJVVI35zOUR2n9mselImKBpJOAWaQj/JkR0aoZ5nY+Dfym0H0Q8E1JHyclwOmkFkSLDgfOkvQ0MJv0tsX+PA28WtLcXP5duf+hpOsyLyVVvb2nZPwvSppE+j+vahGTjTBuBdasR0kaG/kFS5JOJDWjfWyXw7IRxmcSZr3r7fkMZgzwAHBYd8OxkchnEmZmVsoXrs3MrJSThJmZlXKSMDOzUk4SZmZWyknCzMxKOUmYmVmp/w84dhoCq6rc5QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.grid(True)\n",
    "plt.bar( [i for i in range(1,21,2)], train_mean, width=width, \n",
    "        color='b', label='Training Accuracy')\n",
    "plt.bar([i for i in range(1,21,2)], val_mean, \n",
    "        width=0.5*width, color='r', alpha=0.5, label='Validation Accuracy')\n",
    "plt.title('Comments Data')\n",
    "plt.suptitle('Training Accuracy and Validation Accuracy across Kernels')\n",
    "plt.ylabel('R-squared')\n",
    "plt.xlabel('Number of Neighbors')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "KNN_final = grid_search_KNN.best_estimator_\n",
    "pred_test_y = KNN_final.predict(pca_test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "118.1769602519825"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, pred_test_y))\n",
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "linreg_model = LinearRegression()\n",
    "parameters = {'fit_intercept':[True,False], 'normalize':[True,False], 'copy_X':[True, False]}\n",
    "linreg_model.fit(pca_train_x,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.formula.api as sm\n",
    "import statsmodels.api as sm\n",
    "\n",
    "linreg_model = sm.OLS(train_y,pca_train_x).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results = linreg_model.fit()\n",
    "#results.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg_model.summary()\n",
    "\n",
    "pred_test_y = linreg_model.predict(pca_test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "239.51366865837832"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rms = sqrt(mean_squared_error(test_y, pred_test_y))\n",
    "rms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "\n",
    "class PolynomialRegression(BaseEstimator):\n",
    "    def __init__(self, deg=None):\n",
    "        self.deg = deg\n",
    "    \n",
    "    def fit(self, X, y, deg=None):\n",
    "        self.model = LinearRegression(fit_intercept=False)\n",
    "        self.model.fit(np.vander(X, N=self.deg + 1), y)\n",
    "    \n",
    "    def predict(self, x):\n",
    "        return self.model.predict(np.vander(x, N=self.deg + 1))\n",
    "    \n",
    "    @property\n",
    "    def coef_(self):\n",
    "        return self.model.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PolynomialRegression(deg=5)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poly_reg = PolynomialRegression()\n",
    "degrees = np.arange(1, 8)\n",
    "grid_search_poly = GridSearchCV(poly_reg,\n",
    "                        param_grid={'deg': degrees},\n",
    "                        scoring='neg_mean_squared_error')\n",
    "grid_search_poly.fit(train_y, pca_train_x)\n",
    "grid_search_poly.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_degree = grid_search_poly.best_params_['deg']\n",
    "poly_final = grid_search_poly.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "\n",
    "poly = PolynomialFeatures(degree=best_degree)\n",
    "train_x_poly = poly.fit_transform(pca_train_x)\n",
    "poly_lr = LinearRegression()\n",
    "poly_lr.fit(train_x_poly, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1606.3532361836228"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_x_poly = poly.fit_transform(pca_test_x)\n",
    "\n",
    "pred_test_y = poly_lr.predict(test_x_poly)\n",
    "rms = sqrt(mean_squared_error(test_y, pred_test_y))\n",
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4. Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.0624s.) Setting batch_size=6.\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:    0.0s remaining:    0.4s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:    0.0s remaining:    0.2s\n",
      "[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "             estimator=Ridge(alpha=1.0, copy_X=True, fit_intercept=True,\n",
       "                             max_iter=None, normalize=False, random_state=None,\n",
       "                             solver='auto', tol=0.001),\n",
       "             iid='warn', n_jobs=-1,\n",
       "             param_grid={'alpha': array([0.5, 0.7, 0.9, 1.1, 1.3, 1.5, 1.7, 1.9])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring=None, verbose=10)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge, Lasso\n",
    "\n",
    "ridge_model = Ridge()\n",
    "\n",
    "param_grid = {'alpha' : np.arange(0.5,2.0,0.2)}\n",
    "\n",
    "grid_search_ridge = GridSearchCV(estimator = ridge_model, param_grid = param_grid, \n",
    "                          cv = 3, n_jobs = -1,verbose = 10,return_train_score = True)\n",
    "\n",
    "grid_search_ridge.fit(pca_train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144.82485410842116"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_alpha = grid_search_ridge.best_params_['alpha']\n",
    "\n",
    "ridge_final = grid_search_ridge.best_estimator_\n",
    "pred_test_y = ridge_final.predict(pca_test_x)\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, pred_test_y))\n",
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5. Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.0585s.) Setting batch_size=6.\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:    0.0s remaining:    0.3s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:    0.0s remaining:    0.2s\n",
      "[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "             estimator=Lasso(alpha=1.0, copy_X=True, fit_intercept=True,\n",
       "                             max_iter=1000, normalize=False, positive=False,\n",
       "                             precompute=False, random_state=None,\n",
       "                             selection='cyclic', tol=0.0001, warm_start=False),\n",
       "             iid='warn', n_jobs=-1,\n",
       "             param_grid={'alpha': array([0.5, 0.7, 0.9, 1.1, 1.3, 1.5, 1.7, 1.9])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring=None, verbose=10)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso_model = Lasso()\n",
    "\n",
    "param_grid = {'alpha' : np.arange(0.5,2.0,0.2)}\n",
    "\n",
    "grid_search_lasso = GridSearchCV(estimator = lasso_model, param_grid = param_grid, \n",
    "                          cv = 3, n_jobs = -1,verbose = 10,return_train_score = True)\n",
    "\n",
    "grid_search_lasso.fit(pca_train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144.8266162413143"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_alpha = grid_search_lasso.best_params_['alpha']\n",
    "\n",
    "lasso_final = grid_search_lasso.best_estimator_\n",
    "pred_test_y = lasso_final.predict(pca_test_x)\n",
    "\n",
    "rms = sqrt(mean_squared_error(test_y, pred_test_y))\n",
    "rms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Result Table before and after applying PCA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before PCA:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model Selection</th>\n",
       "      <th>Best cross-validation RMSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVR Kernel RBF</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Lasso Regression</td>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ridge Regression</td>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Polynomial Regression</td>\n",
       "      <td>324</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model Selection  Best cross-validation RMSE\n",
       "0                    KNN                         116\n",
       "2         SVR Kernel RBF                         154\n",
       "5       Lasso Regression                         172\n",
       "4       Ridge Regression                         186\n",
       "1      Linear Regression                         224\n",
       "3  Polynomial Regression                         324"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Before PCA:')\n",
    "\n",
    "data1 = {'Model Selection':[\"KNN\", \"Linear Regression\", \"SVR Kernel RBF\", \"Polynomial Regression\", \"Ridge Regression\", \"Lasso Regression\"],\n",
    "       'Best cross-validation RMSE':[116, 224, 154, 324, 186, 172]}\n",
    "data2 = {'Model Selection':[\"KNN\", \"Linear Regression\", \"SVR Kernel RBF\", \"Polynomial Regression\", \"Ridge Regression\", \"Lasso Regression\"],\n",
    "       'RMSE after PCA':[118, 239, 151, 1606, 144, 144]}\n",
    "\n",
    "df1 = pd.DataFrame(data1)\n",
    "df1.sort_values(['Best cross-validation RMSE'], ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After PCA:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model Selection</th>\n",
       "      <th>RMSE after PCA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ridge Regression</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Lasso Regression</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVR Kernel RBF</td>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Polynomial Regression</td>\n",
       "      <td>1606</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model Selection  RMSE after PCA\n",
       "0                    KNN             118\n",
       "4       Ridge Regression             144\n",
       "5       Lasso Regression             144\n",
       "2         SVR Kernel RBF             151\n",
       "1      Linear Regression             239\n",
       "3  Polynomial Regression            1606"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('After PCA:')\n",
    "\n",
    "df2 = pd.DataFrame(data2)\n",
    "df2.sort_values(['RMSE after PCA'], ascending=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Result Comparison Table (Before PCA vs After PCA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model Selection</th>\n",
       "      <th>Best cross-validation RMSE</th>\n",
       "      <th>Best cross-validation RMSE after PCA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN</td>\n",
       "      <td>116</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVR Kernel RBF</td>\n",
       "      <td>154</td>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Lasso Regression</td>\n",
       "      <td>172</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ridge Regression</td>\n",
       "      <td>186</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>224</td>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Polynomial Regression</td>\n",
       "      <td>324</td>\n",
       "      <td>1606</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model Selection  Best cross-validation RMSE  \\\n",
       "0                    KNN                         116   \n",
       "2         SVR Kernel RBF                         154   \n",
       "5       Lasso Regression                         172   \n",
       "4       Ridge Regression                         186   \n",
       "1      Linear Regression                         224   \n",
       "3  Polynomial Regression                         324   \n",
       "\n",
       "   Best cross-validation RMSE after PCA  \n",
       "0                                   118  \n",
       "2                                   151  \n",
       "5                                   144  \n",
       "4                                   144  \n",
       "1                                   239  \n",
       "3                                  1606  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = {'Model Selection':[\"KNN\", \"Linear Regression\", \"SVR Kernel RBF\", \"Polynomial Regression\", \"Ridge Regression\", \"Lasso Regression\"],\n",
    "       'Best cross-validation RMSE':[116, 224, 154, 324, 186, 172],\n",
    "       'Best cross-validation RMSE after PCA':[118, 239, 151, 1606, 144, 144]}\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "df.sort_values(['Best cross-validation RMSE'], ascending=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KNN is the best model as it provides the best model results with the lowest cross-validation RMSE (both before and after PCA). PCA doesn't help in getting better results in this dataset as the cross-validation RMSE after PCA seem to be higher in general compared to those before PCA. Since we aim to have a model that runs fast and is easy to implement, KNN is the best model to be selected. It also requires no assumption and only uses one hyperparameter."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 10. Applying Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12126, 12)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the keras model\n",
    "model = Sequential()\n",
    "\n",
    "# Hidden layers\n",
    "model.add(Dense(12, input_dim=12, kernel_initializer='normal', activation='relu'))\n",
    "\n",
    "# Output\n",
    "model.add(Dense(1, kernel_initializer='normal',))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile model\n",
    "model.compile(loss='mean_squared_error', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "12126/12126 [==============================] - 5s 423us/step - loss: 46899.1513\n",
      "Epoch 2/10\n",
      "12126/12126 [==============================] - 6s 526us/step - loss: 23051.1178\n",
      "Epoch 3/10\n",
      "12126/12126 [==============================] - 7s 542us/step - loss: 20678.7490\n",
      "Epoch 4/10\n",
      "12126/12126 [==============================] - 7s 544us/step - loss: 19705.5033\n",
      "Epoch 5/10\n",
      "12126/12126 [==============================] - 5s 453us/step - loss: 19404.6318\n",
      "Epoch 6/10\n",
      "12126/12126 [==============================] - 6s 482us/step - loss: 19323.7343\n",
      "Epoch 7/10\n",
      "12126/12126 [==============================] - 6s 480us/step - loss: 19292.0175\n",
      "Epoch 8/10\n",
      "12126/12126 [==============================] - 5s 454us/step - loss: 19277.6221\n",
      "Epoch 9/10\n",
      "12126/12126 [==============================] - 6s 468us/step - loss: 19262.5476\n",
      "Epoch 10/10\n",
      "12126/12126 [==============================] - 6s 476us/step - loss: 19241.8433\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x26cdab5f1d0>"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model: fit epochs and batch_size\n",
    "model.fit(train_x_scaled, train_y, epochs = 10, batch_size = 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have improved our loss from 46899.1513 to 19241.8433."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12126/12126 [==============================] - 0s 37us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "19208.477147744903"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluation\n",
    "model.evaluate(train_x_scaled, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5253/5253 [==============================] - 0s 26us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "20341.76542106148"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(test_x_scaled, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 36.491898],\n",
       "       [ 80.924904],\n",
       "       [ 86.71492 ],\n",
       "       ...,\n",
       "       [ 95.54914 ],\n",
       "       [186.18881 ],\n",
       "       [220.76636 ]], dtype=float32)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Making predictions\n",
    "test_y_predictions = model.predict(test_x_scaled)\n",
    "test_y_predictions"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}