P12-CrossValidation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Cross Validation using Scikit-Learn"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Python Imports"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split, KFold, cross_val_score\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.metrics import r2_score"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Load and display the Diamond Prices data set\n",
    "\n",
    "Source: [R Data Sets - Diamond Pricing](https://vincentarelbundock.github.io/Rdatasets/csv/Ecdat/Diamond.csv)\n",
    "\n",
    "Data Dictionary:\n",
    "\n",
    "    - carat : weight of diamond stones in carat unit\n",
    "    - colour: a factor with levels (D,E,F,G,H,I). D is the highest quality\n",
    "    - clarity: a factor with levels (IF,VVS1,VVS2,VS1,VS2). IF is the highest quality\n",
    "    - certification: certification body, a factor with levels ( GIA, IGI, HRD)\n",
    "    - price: price in Singapore dollars\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "     carat colour clarity certification  price\n0     0.30      D     VS2           GIA   1302\n1     0.30      E     VS1           GIA   1510\n2     0.30      G    VVS1           GIA   1510\n3     0.30      G     VS1           GIA   1260\n4     0.31      D     VS1           GIA   1641\n..     ...    ...     ...           ...    ...\n303   1.01      I     VS1           HRD   8175\n304   1.02      F    VVS2           HRD  10796\n305   1.06      H    VVS2           HRD   9890\n306   1.02      H     VS2           HRD   8959\n307   1.09      I    VVS2           HRD   9107\n\n[308 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>carat</th>\n      <th>colour</th>\n      <th>clarity</th>\n      <th>certification</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.30</td>\n      <td>D</td>\n      <td>VS2</td>\n      <td>GIA</td>\n      <td>1302</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.30</td>\n      <td>E</td>\n      <td>VS1</td>\n      <td>GIA</td>\n      <td>1510</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.30</td>\n      <td>G</td>\n      <td>VVS1</td>\n      <td>GIA</td>\n      <td>1510</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.30</td>\n      <td>G</td>\n      <td>VS1</td>\n      <td>GIA</td>\n      <td>1260</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.31</td>\n      <td>D</td>\n      <td>VS1</td>\n      <td>GIA</td>\n      <td>1641</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>303</th>\n      <td>1.01</td>\n      <td>I</td>\n      <td>VS1</td>\n      <td>HRD</td>\n      <td>8175</td>\n    </tr>\n    <tr>\n      <th>304</th>\n      <td>1.02</td>\n      <td>F</td>\n      <td>VVS2</td>\n      <td>HRD</td>\n      <td>10796</td>\n    </tr>\n    <tr>\n      <th>305</th>\n      <td>1.06</td>\n      <td>H</td>\n      <td>VVS2</td>\n      <td>HRD</td>\n      <td>9890</td>\n    </tr>\n    <tr>\n      <th>306</th>\n      <td>1.02</td>\n      <td>H</td>\n      <td>VS2</td>\n      <td>HRD</td>\n      <td>8959</td>\n    </tr>\n    <tr>\n      <th>307</th>\n      <td>1.09</td>\n      <td>I</td>\n      <td>VVS2</td>\n      <td>HRD</td>\n      <td>9107</td>\n    </tr>\n  </tbody>\n</table>\n<p>308 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = './data/diamond-prices.csv'\n",
    "diamond_prices_df = pd.read_csv(url, usecols=['carat', 'colour', 'clarity', 'certification', 'price'])\n",
    "diamond_prices_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Ensure there are no more missing values"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "carat            0\ncolour           0\nclarity          0\ncertification    0\nprice            0\ndtype: int64"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diamond_prices_df.isnull().sum()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Using the data dictionary, create a Python dict of the ordinal features with a dictionary that maps the labels to their respective numerical values"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "ord_features_dict = {\n",
    "    'colour': {'D': 6, 'E': 5, 'F': 4, 'G': 3, 'H': 2 , 'I': 1},\n",
    "    'clarity': {'IF': 5, 'VVS1': 4, 'VVS2': 3, 'VS1': 2, 'VS2': 1}\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Replace the ordinal feature labels with their numerical equivalent in the data set"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "     carat  colour  clarity certification  price\n0     0.30       6        1           GIA   1302\n1     0.30       5        2           GIA   1510\n2     0.30       3        4           GIA   1510\n3     0.30       3        2           GIA   1260\n4     0.31       6        2           GIA   1641\n..     ...     ...      ...           ...    ...\n303   1.01       1        2           HRD   8175\n304   1.02       4        3           HRD  10796\n305   1.06       2        3           HRD   9890\n306   1.02       2        1           HRD   8959\n307   1.09       1        3           HRD   9107\n\n[308 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>carat</th>\n      <th>colour</th>\n      <th>clarity</th>\n      <th>certification</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.30</td>\n      <td>6</td>\n      <td>1</td>\n      <td>GIA</td>\n      <td>1302</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.30</td>\n      <td>5</td>\n      <td>2</td>\n      <td>GIA</td>\n      <td>1510</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.30</td>\n      <td>3</td>\n      <td>4</td>\n      <td>GIA</td>\n      <td>1510</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.30</td>\n      <td>3</td>\n      <td>2</td>\n      <td>GIA</td>\n      <td>1260</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.31</td>\n      <td>6</td>\n      <td>2</td>\n      <td>GIA</td>\n      <td>1641</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>303</th>\n      <td>1.01</td>\n      <td>1</td>\n      <td>2</td>\n      <td>HRD</td>\n      <td>8175</td>\n    </tr>\n    <tr>\n      <th>304</th>\n      <td>1.02</td>\n      <td>4</td>\n      <td>3</td>\n      <td>HRD</td>\n      <td>10796</td>\n    </tr>\n    <tr>\n      <th>305</th>\n      <td>1.06</td>\n      <td>2</td>\n      <td>3</td>\n      <td>HRD</td>\n      <td>9890</td>\n    </tr>\n    <tr>\n      <th>306</th>\n      <td>1.02</td>\n      <td>2</td>\n      <td>1</td>\n      <td>HRD</td>\n      <td>8959</td>\n    </tr>\n    <tr>\n      <th>307</th>\n      <td>1.09</td>\n      <td>1</td>\n      <td>3</td>\n      <td>HRD</td>\n      <td>9107</td>\n    </tr>\n  </tbody>\n</table>\n<p>308 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for key, val in ord_features_dict.items():\n",
    "    diamond_prices_df[key] = diamond_prices_df[key].map(val)\n",
    "diamond_prices_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Create a dummy binary variable for the nominal feature `certification`"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "     certification.GIA  certification.HRD  certification.IGI\n0                    1                  0                  0\n1                    1                  0                  0\n2                    1                  0                  0\n3                    1                  0                  0\n4                    1                  0                  0\n..                 ...                ...                ...\n303                  0                  1                  0\n304                  0                  1                  0\n305                  0                  1                  0\n306                  0                  1                  0\n307                  0                  1                  0\n\n[308 rows x 3 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>certification.GIA</th>\n      <th>certification.HRD</th>\n      <th>certification.IGI</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>303</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>304</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>305</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>306</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>307</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>308 rows × 3 columns</p>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cert_encoded_df = pd.get_dummies(diamond_prices_df[['certification']], prefix_sep='.', sparse=False)\n",
    "cert_encoded_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Drop the nominal feature and replace with dummy variables"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "     carat  colour  clarity  price  certification.GIA  certification.HRD  \\\n0     0.30       6        1   1302                  1                  0   \n1     0.30       5        2   1510                  1                  0   \n2     0.30       3        4   1510                  1                  0   \n3     0.30       3        2   1260                  1                  0   \n4     0.31       6        2   1641                  1                  0   \n..     ...     ...      ...    ...                ...                ...   \n303   1.01       1        2   8175                  0                  1   \n304   1.02       4        3  10796                  0                  1   \n305   1.06       2        3   9890                  0                  1   \n306   1.02       2        1   8959                  0                  1   \n307   1.09       1        3   9107                  0                  1   \n\n     certification.IGI  \n0                    0  \n1                    0  \n2                    0  \n3                    0  \n4                    0  \n..                 ...  \n303                  0  \n304                  0  \n305                  0  \n306                  0  \n307                  0  \n\n[308 rows x 7 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>carat</th>\n      <th>colour</th>\n      <th>clarity</th>\n      <th>price</th>\n      <th>certification.GIA</th>\n      <th>certification.HRD</th>\n      <th>certification.IGI</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.30</td>\n      <td>6</td>\n      <td>1</td>\n      <td>1302</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.30</td>\n      <td>5</td>\n      <td>2</td>\n      <td>1510</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.30</td>\n      <td>3</td>\n      <td>4</td>\n      <td>1510</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.30</td>\n      <td>3</td>\n      <td>2</td>\n      <td>1260</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.31</td>\n      <td>6</td>\n      <td>2</td>\n      <td>1641</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>303</th>\n      <td>1.01</td>\n      <td>1</td>\n      <td>2</td>\n      <td>8175</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>304</th>\n      <td>1.02</td>\n      <td>4</td>\n      <td>3</td>\n      <td>10796</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>305</th>\n      <td>1.06</td>\n      <td>2</td>\n      <td>3</td>\n      <td>9890</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>306</th>\n      <td>1.02</td>\n      <td>2</td>\n      <td>1</td>\n      <td>8959</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>307</th>\n      <td>1.09</td>\n      <td>1</td>\n      <td>3</td>\n      <td>9107</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>308 rows × 7 columns</p>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diamond_prices_df = diamond_prices_df.drop('certification', axis=1)\n",
    "diamond_prices_df = pd.concat([diamond_prices_df, cert_encoded_df], axis=1)\n",
    "diamond_prices_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Display the feature information about the data set"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 308 entries, 0 to 307\n",
      "Data columns (total 7 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   carat              308 non-null    float64\n",
      " 1   colour             308 non-null    int64  \n",
      " 2   clarity            308 non-null    int64  \n",
      " 3   price              308 non-null    int64  \n",
      " 4   certification.GIA  308 non-null    uint8  \n",
      " 5   certification.HRD  308 non-null    uint8  \n",
      " 6   certification.IGI  308 non-null    uint8  \n",
      "dtypes: float64(1), int64(3), uint8(3)\n",
      "memory usage: 10.7 KB\n"
     ]
    }
   ],
   "source": [
    "diamond_prices_df.info()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Create the training, validation, and test data sets"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "X_other, X_test, y_other, y_test = train_test_split(diamond_prices_df[['carat', 'colour', 'clarity', 'certification.GIA', 'certification.HRD', 'certification.IGI']], diamond_prices_df['price'], test_size=0.15, random_state=101)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_other, y_other, test_size=0.15, random_state=101)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Display the shape (rows, columns) of the training data set"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "(221, 6)"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Display the shape (rows, columns) of the validation data set"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "(40, 6)"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val.shape"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Scale the training, validation, and test data sets"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "        carat    colour   clarity  certification.GIA  certification.HRD  \\\n99   0.320281  1.242712 -1.440025           0.995485          -0.575609   \n209 -0.586717  0.529363 -0.658095          -1.004535          -0.575609   \n233 -0.441597 -0.897335  0.905765          -1.004535           1.737292   \n199 -1.167196 -0.897335  1.687695          -1.004535          -0.575609   \n168 -1.602555 -0.183986 -1.440025          -1.004535          -0.575609   \n..        ...       ...       ...                ...                ...   \n73   0.574241 -1.610683  0.123835           0.995485          -0.575609   \n108  0.392841 -1.610683 -0.658095           0.995485          -0.575609   \n22  -1.058356 -0.183986 -0.658095           0.995485          -0.575609   \n304  1.372399  0.529363  0.123835          -1.004535           1.737292   \n206 -0.840676 -1.610683  0.905765          -1.004535          -0.575609   \n\n     certification.IGI  \n99           -0.575609  \n209           1.737292  \n233          -0.575609  \n199           1.737292  \n168           1.737292  \n..                 ...  \n73           -0.575609  \n108          -0.575609  \n22           -0.575609  \n304          -0.575609  \n206           1.737292  \n\n[221 rows x 6 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>carat</th>\n      <th>colour</th>\n      <th>clarity</th>\n      <th>certification.GIA</th>\n      <th>certification.HRD</th>\n      <th>certification.IGI</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>99</th>\n      <td>0.320281</td>\n      <td>1.242712</td>\n      <td>-1.440025</td>\n      <td>0.995485</td>\n      <td>-0.575609</td>\n      <td>-0.575609</td>\n    </tr>\n    <tr>\n      <th>209</th>\n      <td>-0.586717</td>\n      <td>0.529363</td>\n      <td>-0.658095</td>\n      <td>-1.004535</td>\n      <td>-0.575609</td>\n      <td>1.737292</td>\n    </tr>\n    <tr>\n      <th>233</th>\n      <td>-0.441597</td>\n      <td>-0.897335</td>\n      <td>0.905765</td>\n      <td>-1.004535</td>\n      <td>1.737292</td>\n      <td>-0.575609</td>\n    </tr>\n    <tr>\n      <th>199</th>\n      <td>-1.167196</td>\n      <td>-0.897335</td>\n      <td>1.687695</td>\n      <td>-1.004535</td>\n      <td>-0.575609</td>\n      <td>1.737292</td>\n    </tr>\n    <tr>\n      <th>168</th>\n      <td>-1.602555</td>\n      <td>-0.183986</td>\n      <td>-1.440025</td>\n      <td>-1.004535</td>\n      <td>-0.575609</td>\n      <td>1.737292</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>73</th>\n      <td>0.574241</td>\n      <td>-1.610683</td>\n      <td>0.123835</td>\n      <td>0.995485</td>\n      <td>-0.575609</td>\n      <td>-0.575609</td>\n    </tr>\n    <tr>\n      <th>108</th>\n      <td>0.392841</td>\n      <td>-1.610683</td>\n      <td>-0.658095</td>\n      <td>0.995485</td>\n      <td>-0.575609</td>\n      <td>-0.575609</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>-1.058356</td>\n      <td>-0.183986</td>\n      <td>-0.658095</td>\n      <td>0.995485</td>\n      <td>-0.575609</td>\n      <td>-0.575609</td>\n    </tr>\n    <tr>\n      <th>304</th>\n      <td>1.372399</td>\n      <td>0.529363</td>\n      <td>0.123835</td>\n      <td>-1.004535</td>\n      <td>1.737292</td>\n      <td>-0.575609</td>\n    </tr>\n    <tr>\n      <th>206</th>\n      <td>-0.840676</td>\n      <td>-1.610683</td>\n      <td>0.905765</td>\n      <td>-1.004535</td>\n      <td>-0.575609</td>\n      <td>1.737292</td>\n    </tr>\n  </tbody>\n</table>\n<p>221 rows × 6 columns</p>\n</div>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = StandardScaler()\n",
    "X_other_s = pd.DataFrame(scaler.fit_transform(X_other), columns=X_other.columns, index=X_other.index)\n",
    "X_train_s = pd.DataFrame(scaler.fit_transform(X_train), columns=X_train.columns, index=X_train.index)\n",
    "X_val_s = pd.DataFrame(scaler.transform(X_val), columns=X_val.columns, index=X_val.index)\n",
    "X_test_s = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns, index=X_test.index)\n",
    "X_train_s"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Hold Out Train-Validate-Test Cross Validation"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Iteration 1 with hyperparameter `alpha=100`"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "0.8393041847346832"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_one = Ridge(alpha=100)\n",
    "model_one.fit(X_train_s, y_train)\n",
    "y_val_pred1 = model_one.predict(X_val_s)\n",
    "r2_score(y_val, y_val_pred1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Iteration 2 with hyperparameter `alpha=5`"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "0.9528101881304383"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_two = Ridge(alpha=5)\n",
    "model_two.fit(X_train_s, y_train)\n",
    "y_val_pred2 = model_two.predict(X_val_s)\n",
    "r2_score(y_val, y_val_pred2)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Final model evaluation using the test set"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "0.9302466842403143"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model_two.predict(X_test_s)\n",
    "r2_score(y_test, y_pred)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## k-Fold Cross Validation"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Create a 5-fold split"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "folds_5 = KFold(n_splits=5, random_state=101, shuffle=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Iteration 1 with hyperparameter `alpha=100`"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "0.8223482129601845"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_one = Ridge(alpha=100)\n",
    "scores = cross_val_score(model_one, X_other_s, y_other, scoring='r2', cv=folds_5)\n",
    "scores.mean()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Iteration 2 with hyperparameter `alpha=5`"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": "0.9541076637596422"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_two = Ridge(alpha=5)\n",
    "scores2 = cross_val_score(model_two, X_other_s, y_other, scoring='r2', cv=folds_5)\n",
    "scores2.mean()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Final model evaluation using the test set"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "data": {
      "text/plain": "0.9274938024707198"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_two.fit(X_other_s, y_other)\n",
    "y_pred = model_two.predict(X_test_s)\n",
    "r2_score(y_test, y_pred)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}