1. Getting the data ready.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6980e0fc-b4e4-4d53-b1c7-01526776f759",
   "metadata": {},
   "source": [
    "## Standard Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "63fd9253-ed3e-4a20-ad89-b51287c0a62e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56368ca5-c1f9-47f5-9a3d-3b8dfde2d2aa",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Preprocessing data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1471f8ab-bc76-4fd8-b9cc-98792fb5ed76",
   "metadata": {},
   "source": [
    "### Three main things we have to do:\n",
    "1. Split the data into lables (`X` & `y`)\n",
    "2. Filling (also called imputing) or disregarding missing values\n",
    "3. Converting non-numerical to numerical values (feature encoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e6e475fd-755c-4403-826f-69d8c054d4d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/heart-disease.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e2e3f8b2-7408-4e25-85f2-bbc06fa14d58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((303, 13), (303,))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_column = \"target\"\n",
    "X = df.drop(target_column, axis=1)\n",
    "y = df[target_column]\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb31b069-18e9-4507-b33f-3c28133f8d09",
   "metadata": {
    "tags": []
   },
   "source": [
    " ## 1. Split data into testing and training sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1758703a-842c-4492-942e-c2b35d008b24",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e3124b7-607e-4689-8b6c-59eb405c4500",
   "metadata": {},
   "source": [
    "### 1.1 Make sure its all numerical"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc1c97e4-2079-48f0-a107-dea81c5cc4f7",
   "metadata": {},
   "source": [
    "Well the heart-deases data set is already numerical. Let's load a dataset with some text values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a6fde060-616e-4baa-8e75-a8b8077aa534",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431</td>\n",
       "      <td>4</td>\n",
       "      <td>15323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714</td>\n",
       "      <td>5</td>\n",
       "      <td>19943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714</td>\n",
       "      <td>4</td>\n",
       "      <td>28343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365</td>\n",
       "      <td>4</td>\n",
       "      <td>13434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577</td>\n",
       "      <td>3</td>\n",
       "      <td>14043</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors  Price\n",
       "0   Honda  White          35431      4  15323\n",
       "1     BMW   Blue         192714      5  19943\n",
       "2   Honda  White          84714      4  28343\n",
       "3  Toyota  White         154365      4  13434\n",
       "4  Nissan   Blue         181577      3  14043"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/car-sales-extended.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15a516da-fee5-4f54-8346-2c6bd253b221",
   "metadata": {},
   "source": [
    "##### Observation: Let's try to predict `Price`. \n",
    "As it is a numeric column, it will be a <b>regression problem</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "405b75bc-d6ff-483d-901e-0e57ec728240",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 5)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "72b34b20-eb91-4326-b3e8-0094baa43bde",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 5 columns):\n",
      " #   Column         Non-Null Count  Dtype \n",
      "---  ------         --------------  ----- \n",
      " 0   Make           1000 non-null   object\n",
      " 1   Colour         1000 non-null   object\n",
      " 2   Odometer (KM)  1000 non-null   int64 \n",
      " 3   Doors          1000 non-null   int64 \n",
      " 4   Price          1000 non-null   int64 \n",
      "dtypes: int64(3), object(2)\n",
      "memory usage: 39.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da466002-5e29-4ef5-ab23-2380a3890681",
   "metadata": {},
   "source": [
    "Let's produce an error by trying to train a model without turning texts into numbers aka encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fc21ebbe-c3b4-4795-8c77-e60f0400a4d4",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "could not convert string to float: 'Toyota'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[20], line 15\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msklearn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mensemble\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m RandomForestRegressor\n\u001b[0;32m     14\u001b[0m model \u001b[38;5;241m=\u001b[39m RandomForestRegressor()\n\u001b[1;32m---> 15\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     17\u001b[0m \u001b[38;5;66;03m# Evaluate the model\u001b[39;00m\n\u001b[0;32m     18\u001b[0m model\u001b[38;5;241m.\u001b[39mscore(X_test, y_test)\n",
      "File \u001b[1;32mC:\\ProgramData\\anaconda3\\lib\\site-packages\\sklearn\\ensemble\\_forest.py:345\u001b[0m, in \u001b[0;36mBaseForest.fit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m issparse(y):\n\u001b[0;32m    344\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msparse multilabel-indicator for y is not supported.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 345\u001b[0m X, y \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_data\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    346\u001b[0m \u001b[43m    \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmulti_output\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccept_sparse\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcsc\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mDTYPE\u001b[49m\n\u001b[0;32m    347\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    348\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m sample_weight \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    349\u001b[0m     sample_weight \u001b[38;5;241m=\u001b[39m _check_sample_weight(sample_weight, X)\n",
      "File \u001b[1;32mC:\\ProgramData\\anaconda3\\lib\\site-packages\\sklearn\\base.py:565\u001b[0m, in \u001b[0;36mBaseEstimator._validate_data\u001b[1;34m(self, X, y, reset, validate_separately, **check_params)\u001b[0m\n\u001b[0;32m    563\u001b[0m         y \u001b[38;5;241m=\u001b[39m check_array(y, input_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124my\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mcheck_y_params)\n\u001b[0;32m    564\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 565\u001b[0m         X, y \u001b[38;5;241m=\u001b[39m check_X_y(X, y, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mcheck_params)\n\u001b[0;32m    566\u001b[0m     out \u001b[38;5;241m=\u001b[39m X, y\n\u001b[0;32m    568\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m no_val_X \u001b[38;5;129;01mand\u001b[39;00m check_params\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mensure_2d\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mTrue\u001b[39;00m):\n",
      "File \u001b[1;32mC:\\ProgramData\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:1106\u001b[0m, in \u001b[0;36mcheck_X_y\u001b[1;34m(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)\u001b[0m\n\u001b[0;32m   1101\u001b[0m         estimator_name \u001b[38;5;241m=\u001b[39m _check_estimator_name(estimator)\n\u001b[0;32m   1102\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m   1103\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mestimator_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m requires y to be passed, but the target y is None\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1104\u001b[0m     )\n\u001b[1;32m-> 1106\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[43mcheck_array\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1107\u001b[0m \u001b[43m    \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1108\u001b[0m \u001b[43m    \u001b[49m\u001b[43maccept_sparse\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maccept_sparse\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1109\u001b[0m \u001b[43m    \u001b[49m\u001b[43maccept_large_sparse\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maccept_large_sparse\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1110\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[43m    \u001b[49m\u001b[43morder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1112\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcopy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcopy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1113\u001b[0m \u001b[43m    \u001b[49m\u001b[43mforce_all_finite\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_all_finite\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1114\u001b[0m \u001b[43m    \u001b[49m\u001b[43mensure_2d\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mensure_2d\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1115\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_nd\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mallow_nd\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1116\u001b[0m \u001b[43m    \u001b[49m\u001b[43mensure_min_samples\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mensure_min_samples\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1117\u001b[0m \u001b[43m    \u001b[49m\u001b[43mensure_min_features\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mensure_min_features\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1118\u001b[0m \u001b[43m    \u001b[49m\u001b[43mestimator\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mestimator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1119\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mX\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1120\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1122\u001b[0m y \u001b[38;5;241m=\u001b[39m _check_y(y, multi_output\u001b[38;5;241m=\u001b[39mmulti_output, y_numeric\u001b[38;5;241m=\u001b[39my_numeric, estimator\u001b[38;5;241m=\u001b[39mestimator)\n\u001b[0;32m   1124\u001b[0m check_consistent_length(X, y)\n",
      "File \u001b[1;32mC:\\ProgramData\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:879\u001b[0m, in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator, input_name)\u001b[0m\n\u001b[0;32m    877\u001b[0m         array \u001b[38;5;241m=\u001b[39m xp\u001b[38;5;241m.\u001b[39mastype(array, dtype, copy\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m    878\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 879\u001b[0m         array \u001b[38;5;241m=\u001b[39m \u001b[43m_asarray_with_order\u001b[49m\u001b[43m(\u001b[49m\u001b[43marray\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mxp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mxp\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    880\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ComplexWarning \u001b[38;5;28;01mas\u001b[39;00m complex_warning:\n\u001b[0;32m    881\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    882\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mComplex data not supported\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(array)\n\u001b[0;32m    883\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcomplex_warning\u001b[39;00m\n",
      "File \u001b[1;32mC:\\ProgramData\\anaconda3\\lib\\site-packages\\sklearn\\utils\\_array_api.py:185\u001b[0m, in \u001b[0;36m_asarray_with_order\u001b[1;34m(array, dtype, order, copy, xp)\u001b[0m\n\u001b[0;32m    182\u001b[0m     xp, _ \u001b[38;5;241m=\u001b[39m get_namespace(array)\n\u001b[0;32m    183\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m xp\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnumpy\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnumpy.array_api\u001b[39m\u001b[38;5;124m\"\u001b[39m}:\n\u001b[0;32m    184\u001b[0m     \u001b[38;5;66;03m# Use NumPy API to support order\u001b[39;00m\n\u001b[1;32m--> 185\u001b[0m     array \u001b[38;5;241m=\u001b[39m \u001b[43mnumpy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43masarray\u001b[49m\u001b[43m(\u001b[49m\u001b[43marray\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    186\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m xp\u001b[38;5;241m.\u001b[39masarray(array, copy\u001b[38;5;241m=\u001b[39mcopy)\n\u001b[0;32m    187\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[1;32mC:\\ProgramData\\anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:2070\u001b[0m, in \u001b[0;36mNDFrame.__array__\u001b[1;34m(self, dtype)\u001b[0m\n\u001b[0;32m   2069\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__array__\u001b[39m(\u001b[38;5;28mself\u001b[39m, dtype: npt\u001b[38;5;241m.\u001b[39mDTypeLike \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m np\u001b[38;5;241m.\u001b[39mndarray:\n\u001b[1;32m-> 2070\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43masarray\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_values\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mValueError\u001b[0m: could not convert string to float: 'Toyota'"
     ]
    }
   ],
   "source": [
    "# setting features and labels\n",
    "target_var = \"Price\"\n",
    "X = df.drop(target_var, axis=1)\n",
    "y = df[target_var]\n",
    "\n",
    "# split the sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Fit the model. Let's do a RandomForestRegression\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the model\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d95e7744-d348-4b8f-9e1a-ad4f7d9984fe",
   "metadata": {},
   "source": [
    "As we can see, we need to convert strings to numbers.\n",
    "\n",
    "This method is called feature encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8b29513a-0058-4cdb-9d5d-47696c0e6850",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature encoding\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35780c07-5ee3-4ab1-9ef6-0e28959f2962",
   "metadata": {},
   "source": [
    " yes, we are using Doors too! Because, The number of doors can be at most two to five types "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "119eebc2-67d3-486b-8340-2fc8c231712a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    856\n",
       "5     79\n",
       "3     65\n",
       "Name: Doors, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"Doors\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a9205df-0090-44d4-9803-92e8f61bc4d3",
   "metadata": {},
   "source": [
    "So, there will be either 4, 5 or 3 doors in each car"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8de6133e-06f2-4fd4-b3b8-319aabf4d321",
   "metadata": {},
   "source": [
    "<b>One Hot Encoding</b> can be defined as a process of transforming categorical variables into numerical format before fitting and training a Machine Learning algorithm.\n",
    "\n",
    "![One Hot Encoding](https://www.statology.org/wp-content/uploads/2021/09/oneHot1.png)\n",
    "\n",
    "<b>Column Transformer</b> is a scikit-learn class that applise transformers to pandas DataFrame (or columns of an array). This is useful for heterogeneous or columnar data, to combine several feature extraction mechanisms or transformations into a single transformer.\n",
    "\n",
    "![Column Transformer](https://ubc-cs.github.io/cpsc330/_images/column-transformer.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "333690e2-e22c-4ce7-98ea-ecf5ca19282e",
   "metadata": {},
   "source": [
    "<b>Call signature:</b>\n",
    "```python\n",
    "class sklearn.compose.ColumnTransformer(transformers, *, remainder='drop', sparse_threshold=0.3, n_jobs=None, transformer_weights=None, verbose=False, verbose_feature_names_out=True)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ddd87d5-1344-4af5-aa08-cdbc3595702a",
   "metadata": {},
   "source": [
    "Here, `transformers` is a list of tuples.\n",
    "List of <b><i>`(name, transformer, columns)`</i></b> tuples specifying the transformer objects to be applied to subsets of the data. where,\n",
    "\n",
    "<b>name: str</b> a name for future reference<br/>\n",
    "<b>transformer:</b> {`drop`, `passthrough`} or estimator class<br/>\n",
    "<b>columns: str, array-like of str, int, slice etc.</b> Index of dataframe or similar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e0805ac8-3391-4c7a-82f8-baf0bb60f896",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 3.54310e+04],\n",
       "       [1.00000e+00, 0.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        1.00000e+00, 1.92714e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 8.47140e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 1.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 6.66040e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 2.15883e+05],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 2.48360e+05]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer(\n",
    "                transformers=[(\"one_hot\", one_hot, categorical_features)],\n",
    "                remainder=\"passthrough\")\n",
    "\n",
    "# Apply the transformer on features X\n",
    "transformed_X = transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d291be25-cad4-48d0-ba39-6d42a8509a34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35431.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>192714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>84714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>154365.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>181577.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0    1    2    3    4    5    6    7    8    9    10   11        12\n",
       "0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0   35431.0\n",
       "1  1.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  192714.0\n",
       "2  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0   84714.0\n",
       "3  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0  154365.0\n",
       "4  0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  0.0  1.0  0.0  0.0  181577.0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# a bit clearer visualization\n",
    "pd.DataFrame(transformed_X).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2055d8cc-0d0e-405b-8d6d-e5661b25a3cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Make_BMW</th>\n",
       "      <th>Make_Honda</th>\n",
       "      <th>Make_Nissan</th>\n",
       "      <th>Make_Toyota</th>\n",
       "      <th>Colour_Black</th>\n",
       "      <th>Colour_Blue</th>\n",
       "      <th>Colour_Green</th>\n",
       "      <th>Colour_Red</th>\n",
       "      <th>Colour_White</th>\n",
       "      <th>Doors_3</th>\n",
       "      <th>Doors_4</th>\n",
       "      <th>Doors_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35431</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>192714</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>84714</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>154365</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>181577</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Odometer (KM)  Make_BMW  Make_Honda  Make_Nissan  Make_Toyota  \\\n",
       "0          35431         0           1            0            0   \n",
       "1         192714         1           0            0            0   \n",
       "2          84714         0           1            0            0   \n",
       "3         154365         0           0            0            1   \n",
       "4         181577         0           0            1            0   \n",
       "\n",
       "   Colour_Black  Colour_Blue  Colour_Green  Colour_Red  Colour_White  Doors_3  \\\n",
       "0             0            0             0           0             1        0   \n",
       "1             0            1             0           0             0        0   \n",
       "2             0            0             0           0             1        0   \n",
       "3             0            0             0           0             1        0   \n",
       "4             0            1             0           0             0        1   \n",
       "\n",
       "   Doors_4  Doors_5  \n",
       "0        1        0  \n",
       "1        0        1  \n",
       "2        1        0  \n",
       "3        1        0  \n",
       "4        0        0  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Similar implementation with pandas get_dummies method\n",
    "pd.get_dummies(X, columns=categorical_features).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "9f4f12ac-0b32-4287-af7c-ce71240657b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors\n",
       "0   Honda  White          35431      4\n",
       "1     BMW   Blue         192714      5\n",
       "2   Honda  White          84714      4\n",
       "3  Toyota  White         154365      4\n",
       "4  Nissan   Blue         181577      3"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compare with actual X\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7382d6b1-3ec6-478f-972a-257afa2598ba",
   "metadata": {},
   "source": [
    "So, now all are data are in numbers. Let's retry and refit the model!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "1a49a3f7-5d26-4e1f-8afb-94183c68bbba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.417628391779279"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test_train split\n",
    "X_train, X_test, y_train, y_test = train_test_split(transformed_X, y, test_size=0.2)\n",
    "\n",
    "# Let's choose the Random Forest Regressor again\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the model\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2d85c40-a012-4dc5-8434-3eb3d09d95db",
   "metadata": {},
   "source": [
    "## 2. Handling missing data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8c9ff7b-bec8-4bf6-b16f-5d2b17944c87",
   "metadata": {},
   "source": [
    "1. Fill them with some value (also know as imputation)\n",
    "2. Remove the sample with missing data altogether."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "ae2fa326-d224-4174-b787-0cda9cb05604",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors    Price\n",
       "0   Honda  White        35431.0    4.0  15323.0\n",
       "1     BMW   Blue       192714.0    5.0  19943.0\n",
       "2   Honda  White        84714.0    4.0  28343.0\n",
       "3  Toyota  White       154365.0    4.0  13434.0\n",
       "4  Nissan   Blue       181577.0    3.0  14043.0"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's load data!\n",
    "df = pd.read_csv('data/car-sales-extended-missing-data.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "5a18fe40-6d8c-418d-b0e0-016b9dd6dd51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 5 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   Make           951 non-null    object \n",
      " 1   Colour         950 non-null    object \n",
      " 2   Odometer (KM)  950 non-null    float64\n",
      " 3   Doors          950 non-null    float64\n",
      " 4   Price          950 non-null    float64\n",
      "dtypes: float64(3), object(2)\n",
      "memory usage: 39.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b74edf7-e26d-447e-b017-f63d2ce02916",
   "metadata": {},
   "source": [
    "So, looks like there are some missing data.\n",
    "Let's explore a little more."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "92eef8c2-cd98-4552-aed0-122b7052a2ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5ff773c-f1e7-45fd-995f-9896adf5011c",
   "metadata": {},
   "source": [
    " <b>Note:</b> `isna()` and `isnull()` are actually same.\n",
    " `isnull()` is a method of Pandas Series objects, \n",
    " while `isna()` is a method of both Series and DataFrame objects\n",
    " so, `isna()` would be a preferred choice."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8de61372-3363-4506-a1c1-ee3f0d6785e9",
   "metadata": {},
   "source": [
    "Anyways, as there are some missing values in the set,\n",
    "how do we proceed?\n",
    "\n",
    "#### Option-1: Fill missing data with Pandas\n",
    "For the <i>categorical</i> columns, we will fill the values with string: `missing`.<br>\n",
    "For the <i>numerical values</i> we may use `mean` or similar statistics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "43496b8a-76b1-4370-80db-e06210a1199d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill the \"Make\" column\n",
    "df[\"Make\"].fillna(\"missing\", inplace=True)\n",
    "\n",
    "# Fill the \"Colour\" column\n",
    "df[\"Colour\"].fillna(\"missing\", inplace=True)\n",
    "\n",
    "# Fill the \"Odometer (KM)\" column\n",
    "avg_odometer_distance = df[\"Odometer (KM)\"].mean()\n",
    "df[\"Odometer (KM)\"].fillna(avg_odometer_distance, inplace=True)\n",
    "\n",
    "# Fill the \"Doors\" column\n",
    "# Well, Doors column is a categorical feature. So we will fill it with the \n",
    "# `mode` in this case. the statistical term `mode` means the most occuring value.\n",
    "most_frequent_number_of_doors = df[\"Doors\"].mode()[0]\n",
    "df[\"Doors\"].fillna(most_frequent_number_of_doors, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "81a15194-caf5-4641-b240-a13b366d1d8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make              0\n",
       "Colour            0\n",
       "Odometer (KM)     0\n",
       "Doors             0\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's check the missing values again\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06263abe-acc9-41e0-8d51-0b92f4fa0c82",
   "metadata": {},
   "source": [
    "Now, there are 50 prices that we don't know. As these price is the variable we are\n",
    "trying to predict, we have no other option but to dorp it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "bbe2314f-be17-4cd6-abe5-372554a0f836",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "acf6a026-53eb-4fa3-b9b8-70329163da8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 950 entries, 0 to 999\n",
      "Data columns (total 5 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   Make           950 non-null    object \n",
      " 1   Colour         950 non-null    object \n",
      " 2   Odometer (KM)  950 non-null    float64\n",
      " 3   Doors          950 non-null    float64\n",
      " 4   Price          950 non-null    float64\n",
      "dtypes: float64(3), object(2)\n",
      "memory usage: 44.5+ KB\n"
     ]
    }
   ],
   "source": [
    "# Let's check our current status of the data\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f794b469-1447-499a-98ed-57682ad23b17",
   "metadata": {},
   "source": [
    "Ok, so there are no missing values in the features. \n",
    "We can also see that, the size of our data set is reduced to 950 from 1000 rows.\n",
    "Which means all the null values in Price column (target variable) is dropped.\n",
    "\n",
    "So now we can proceed to next steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "bccc7c78-a223-4f4f-958a-8d83f484cfc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature and label declaration\n",
    "target_variable = \"Price\"\n",
    "X = df.drop(target_variable, axis=1)\n",
    "y = df[target_variable]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53036c38-fbe1-464b-b4ca-a86300cce371",
   "metadata": {},
   "source": [
    "But wait, there are strings in our dataframe. So, we need to do some encoding first.\n",
    "We'll do the same as before:\n",
    "\n",
    "    1. Use one hot encoder to take care of Make, Colur and Doors\n",
    "    2. Then, transform everything to our dataframe at once, with the help of scikit-learn `ColumnTransformer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "0cba3f5b-e40e-4942-9563-23ddc97c9ebf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "# Make the transformer\n",
    "categorical_columns = [\"Make\", \"Colour\", \"Doors\"]\n",
    "\n",
    "onehot_enc = OneHotEncoder()\n",
    "column_transformer = ColumnTransformer(\n",
    "                        transformers=[(\"onehot_enc\", onehot_enc, categorical_columns)],\n",
    "                        remainder=\"passthrough\"\n",
    "                    )\n",
    "\n",
    "# Apply the transformer on X\n",
    "transformed_X = column_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f9cfddf-fc10-4c3a-b66c-47d7064cfc8d",
   "metadata": {},
   "source": [
    "#### Option-2: Fill missing data with Scikit-Learn\n",
    "So, what we did with pandas before, was actually called imputing, meaning we filled our missing values following some sort of strategies.<br>\n",
    "However, Scikit-Learn also provides a simialr functionality, which is called Imputer.\n",
    "The plus point is, we can directly combine the Imputer object to the ColumnTransformer class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "19703f81-41a4-4966-b801-003e81ac4353",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Honda', 'White', 4.0, 35431.0],\n",
       "       ['BMW', 'Blue', 5.0, 192714.0],\n",
       "       ['Honda', 'White', 4.0, 84714.0],\n",
       "       ...,\n",
       "       ['Nissan', 'Blue', 4.0, 66604.0],\n",
       "       ['Honda', 'White', 4.0, 215883.0],\n",
       "       ['Toyota', 'Blue', 4.0, 248360.0]], dtype=object)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "df = pd.read_csv('data/car-sales-extended-missing-data.csv')\n",
    "target_variable = \"Price\"\n",
    "\n",
    "# We can't take the rows, where there is no Price. So, drop it!\n",
    "df = df.dropna(subset=target_variable)\n",
    "\n",
    "X = df.drop(target_variable, axis=1)\n",
    "y = df[target_variable]\n",
    "\n",
    "# Fill the categorical values with 'missing' & numerical values with mean or mode\n",
    "categorical_imputer = SimpleImputer(strategy=\"constant\", fill_value=\"missing\")\n",
    "\n",
    "most_frequent_number_of_doors = df[\"Doors\"].mode()[0]\n",
    "door_imputer = SimpleImputer(strategy=\"constant\", fill_value=4)\n",
    "\n",
    "odometer_imputer = SimpleImputer(strategy=\"mean\")\n",
    "\n",
    "# Define the columns\n",
    "categorical_features = [\"Make\", \"Colour\"]\n",
    "door_features = [\"Doors\"]\n",
    "odometer_features = [\"Odometer (KM)\"]\n",
    "\n",
    "# Create a transformer to fill the X\n",
    "impute_transformer = ColumnTransformer(\n",
    "    transformers=[\n",
    "        (\"categorical_imputer\", categorical_imputer, categorical_features),\n",
    "        (\"door_imputer\", door_imputer, door_features),\n",
    "        (\"odometer_imputer\", odometer_imputer, odometer_features),\n",
    "    ],\n",
    "    remainder=\"passthrough\"\n",
    ")\n",
    "\n",
    "filled_X = impute_transformer.fit_transform(X)\n",
    "filled_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "e4c2accd-66f4-48b5-8f51-637c63d63e8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>4.0</td>\n",
       "      <td>35431.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>5.0</td>\n",
       "      <td>192714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>4.0</td>\n",
       "      <td>84714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>4.0</td>\n",
       "      <td>154365.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>3.0</td>\n",
       "      <td>181577.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour Doors Odometer (KM)\n",
       "0   Honda  White   4.0       35431.0\n",
       "1     BMW   Blue   5.0      192714.0\n",
       "2   Honda  White   4.0       84714.0\n",
       "3  Toyota  White   4.0      154365.0\n",
       "4  Nissan   Blue   3.0      181577.0"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's just make it a data frame.\n",
    "X_filled = pd.DataFrame(filled_X, columns=[\"Make\", \"Colour\", \"Doors\", \"Odometer (KM)\"])\n",
    "X_filled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "2afb86c7-7a80-4df4-9f6c-a4b68212342e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 950 entries, 0 to 949\n",
      "Data columns (total 4 columns):\n",
      " #   Column         Non-Null Count  Dtype \n",
      "---  ------         --------------  ----- \n",
      " 0   Make           950 non-null    object\n",
      " 1   Colour         950 non-null    object\n",
      " 2   Doors          950 non-null    object\n",
      " 3   Odometer (KM)  950 non-null    object\n",
      "dtypes: object(4)\n",
      "memory usage: 29.8+ KB\n"
     ]
    }
   ],
   "source": [
    "X_filled.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b37f218f-d458-493f-9d31-f77696aad661",
   "metadata": {},
   "source": [
    "Now, Let the encoding begin!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "80db7b18-77f7-46ec-a592-5743b6c74c10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<950x15 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 3800 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "onehot_enc = OneHotEncoder()\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "\n",
    "onehot_transformer = ColumnTransformer(transformers=[(\"onehot_enc\", onehot_enc, categorical_features)], remainder=\"passthrough\")\n",
    "\n",
    "X_transformed = onehot_transformer.fit_transform(X_filled)\n",
    "X_transformed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de4105b6-4a05-47a6-b2ea-f702ae49db7e",
   "metadata": {},
   "source": [
    "Let the training begin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "5ecb0aaa-38a8-48b6-8467-d3da99637f74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.23243492420386014"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Train-test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_transformed, y, test_size=0.2)\n",
    "\n",
    "# Fit to a model\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Evaluation\n",
    "model.score(X_test, y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}