5_c_Housing_Model_Selection_LazyPredict (1).ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o9YjQpALTzD8"
      },
      "source": [
        "Change scoring to balanced accuracy scoring, run all the models, and figure out voting classifier\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V2iLqTjIb4cw"
      },
      "source": [
        "## 1 Data Reading"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Hil4XLeVbWTN"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.preprocessing import OrdinalEncoder\n",
        "from sklearn import set_config\n",
        "\n",
        "# reading\n",
        "url = \"https://drive.google.com/file/d/1JsZV_7u1TGiQA863-YtYxlia1VNxAJZj/view?usp=sharing\"\n",
        "path = 'https://drive.google.com/uc?export=download&id='+url.split('/')[-2]\n",
        "data = df = pd.read_csv(path)\n",
        "\n",
        "# X and y creation\n",
        "## Removing ID as it is unique to each property, and removing MSSubClass as it is a categorization\n",
        "## of property types, but there is no way to tell which one is better / worse\n",
        "X = data\n",
        "y = X.pop(\"Expensive\")\n",
        "Id = X.pop(\"Id\")\n",
        "\n",
        "# # Feature Engineering\n",
        "# X.loc[:, \"Cabin\"] = X.Cabin.str[0]\n",
        "\n",
        "# data splitting\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GBOv230zFzO4"
      },
      "source": [
        "## 2 Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        },
        "id": "PAJbbw0oF1n0",
        "outputId": "cd9f0a04-a2fe-4dd4-8d17-0b6efda40c74"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ColumnTransformer(transformers=[('num_pipe',\n",
              "                                 Pipeline(steps=[('simpleimputer',\n",
              "                                                  SimpleImputer())]),\n",
              "                                 Index(['LotArea', 'LotFrontage', 'TotalBsmtSF', 'BedroomAbvGr', 'Fireplaces',\n",
              "       'PoolArea', 'GarageCars', 'WoodDeckSF', 'ScreenPorch', 'MSSubClass',\n",
              "       'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea',\n",
              "       'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', '1stFlrSF',...\n",
              "       'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope',\n",
              "       'Neighborhood', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle',\n",
              "       'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'BsmtFinType2',\n",
              "       'HeatingQC', 'Electrical', 'Functional', 'GarageType', 'GarageFinish',\n",
              "       'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence',\n",
              "       'MiscFeature', 'SaleType', 'SaleCondition'],\n",
              "      dtype='object'))])"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>ColumnTransformer(transformers=[(&#x27;num_pipe&#x27;,\n",
              "                                 Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
              "                                                  SimpleImputer())]),\n",
              "                                 Index([&#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;Fireplaces&#x27;,\n",
              "       &#x27;PoolArea&#x27;, &#x27;GarageCars&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;ScreenPorch&#x27;, &#x27;MSSubClass&#x27;,\n",
              "       &#x27;OverallQual&#x27;, &#x27;OverallCond&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;MasVnrArea&#x27;,\n",
              "       &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;, &#x27;BsmtUnfSF&#x27;, &#x27;1stFlrSF&#x27;,...\n",
              "       &#x27;LotShape&#x27;, &#x27;LandContour&#x27;, &#x27;Utilities&#x27;, &#x27;LotConfig&#x27;, &#x27;LandSlope&#x27;,\n",
              "       &#x27;Neighborhood&#x27;, &#x27;Condition2&#x27;, &#x27;BldgType&#x27;, &#x27;HouseStyle&#x27;, &#x27;RoofStyle&#x27;,\n",
              "       &#x27;RoofMatl&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;MasVnrType&#x27;, &#x27;BsmtFinType2&#x27;,\n",
              "       &#x27;HeatingQC&#x27;, &#x27;Electrical&#x27;, &#x27;Functional&#x27;, &#x27;GarageType&#x27;, &#x27;GarageFinish&#x27;,\n",
              "       &#x27;GarageQual&#x27;, &#x27;GarageCond&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;, &#x27;Fence&#x27;,\n",
              "       &#x27;MiscFeature&#x27;, &#x27;SaleType&#x27;, &#x27;SaleCondition&#x27;],\n",
              "      dtype=&#x27;object&#x27;))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[(&#x27;num_pipe&#x27;,\n",
              "                                 Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
              "                                                  SimpleImputer())]),\n",
              "                                 Index([&#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;Fireplaces&#x27;,\n",
              "       &#x27;PoolArea&#x27;, &#x27;GarageCars&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;ScreenPorch&#x27;, &#x27;MSSubClass&#x27;,\n",
              "       &#x27;OverallQual&#x27;, &#x27;OverallCond&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;MasVnrArea&#x27;,\n",
              "       &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;, &#x27;BsmtUnfSF&#x27;, &#x27;1stFlrSF&#x27;,...\n",
              "       &#x27;LotShape&#x27;, &#x27;LandContour&#x27;, &#x27;Utilities&#x27;, &#x27;LotConfig&#x27;, &#x27;LandSlope&#x27;,\n",
              "       &#x27;Neighborhood&#x27;, &#x27;Condition2&#x27;, &#x27;BldgType&#x27;, &#x27;HouseStyle&#x27;, &#x27;RoofStyle&#x27;,\n",
              "       &#x27;RoofMatl&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;MasVnrType&#x27;, &#x27;BsmtFinType2&#x27;,\n",
              "       &#x27;HeatingQC&#x27;, &#x27;Electrical&#x27;, &#x27;Functional&#x27;, &#x27;GarageType&#x27;, &#x27;GarageFinish&#x27;,\n",
              "       &#x27;GarageQual&#x27;, &#x27;GarageCond&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;, &#x27;Fence&#x27;,\n",
              "       &#x27;MiscFeature&#x27;, &#x27;SaleType&#x27;, &#x27;SaleCondition&#x27;],\n",
              "      dtype=&#x27;object&#x27;))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">num_pipe</label><div class=\"sk-toggleable__content\"><pre>Index([&#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;Fireplaces&#x27;,\n",
              "       &#x27;PoolArea&#x27;, &#x27;GarageCars&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;ScreenPorch&#x27;, &#x27;MSSubClass&#x27;,\n",
              "       &#x27;OverallQual&#x27;, &#x27;OverallCond&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;MasVnrArea&#x27;,\n",
              "       &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;, &#x27;BsmtUnfSF&#x27;, &#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;,\n",
              "       &#x27;LowQualFinSF&#x27;, &#x27;GrLivArea&#x27;, &#x27;BsmtFullBath&#x27;, &#x27;BsmtHalfBath&#x27;, &#x27;FullBath&#x27;,\n",
              "       &#x27;HalfBath&#x27;, &#x27;KitchenAbvGr&#x27;, &#x27;TotRmsAbvGrd&#x27;, &#x27;GarageYrBlt&#x27;, &#x27;GarageArea&#x27;,\n",
              "       &#x27;OpenPorchSF&#x27;, &#x27;EnclosedPorch&#x27;, &#x27;3SsnPorch&#x27;, &#x27;MiscVal&#x27;, &#x27;MoSold&#x27;,\n",
              "       &#x27;YrSold&#x27;],\n",
              "      dtype=&#x27;object&#x27;)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">cat_pipe</label><div class=\"sk-toggleable__content\"><pre>Index([&#x27;MSZoning&#x27;, &#x27;Condition1&#x27;, &#x27;Heating&#x27;, &#x27;Street&#x27;, &#x27;CentralAir&#x27;,\n",
              "       &#x27;Foundation&#x27;, &#x27;ExterQual&#x27;, &#x27;ExterCond&#x27;, &#x27;BsmtQual&#x27;, &#x27;BsmtCond&#x27;,\n",
              "       &#x27;BsmtExposure&#x27;, &#x27;BsmtFinType1&#x27;, &#x27;KitchenQual&#x27;, &#x27;FireplaceQu&#x27;, &#x27;Alley&#x27;,\n",
              "       &#x27;LotShape&#x27;, &#x27;LandContour&#x27;, &#x27;Utilities&#x27;, &#x27;LotConfig&#x27;, &#x27;LandSlope&#x27;,\n",
              "       &#x27;Neighborhood&#x27;, &#x27;Condition2&#x27;, &#x27;BldgType&#x27;, &#x27;HouseStyle&#x27;, &#x27;RoofStyle&#x27;,\n",
              "       &#x27;RoofMatl&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;MasVnrType&#x27;, &#x27;BsmtFinType2&#x27;,\n",
              "       &#x27;HeatingQC&#x27;, &#x27;Electrical&#x27;, &#x27;Functional&#x27;, &#x27;GarageType&#x27;, &#x27;GarageFinish&#x27;,\n",
              "       &#x27;GarageQual&#x27;, &#x27;GarageCond&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;, &#x27;Fence&#x27;,\n",
              "       &#x27;MiscFeature&#x27;, &#x27;SaleType&#x27;, &#x27;SaleCondition&#x27;],\n",
              "      dtype=&#x27;object&#x27;)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(fill_value=&#x27;NA&#x27;, strategy=&#x27;constant&#x27;)</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">columntransformer: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[(&#x27;cat_ordinal&#x27;,\n",
              "                                 OrdinalEncoder(categories=[[&#x27;NA&#x27;, &#x27;Pave&#x27;,\n",
              "                                                             &#x27;Grvl&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;N&#x27;, &#x27;Y&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;No&#x27;, &#x27;Mn&#x27;,\n",
              "                                                             &#x27;Av&#x27;, &#x27;Gd&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;,\n",
              "                                                             &#x27;Rec&#x27;, &#x27;BLQ&#x27;,\n",
              "                                                             &#x27;ALQ&#x27;, &#x27;GLQ&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;...\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;N&#x27;, &#x27;P&#x27;,\n",
              "                                                             &#x27;Y&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;,\n",
              "                                                             &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;MnWw&#x27;,\n",
              "                                                             &#x27;GdWo&#x27;, &#x27;MnPrv&#x27;,\n",
              "                                                             &#x27;GdPrv&#x27;]]),\n",
              "                                 array([ 3,  4,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 17, 19, 25, 29, 32,\n",
              "       33, 34, 35, 36, 37, 38, 39])),\n",
              "                                (&#x27;cat_onehot&#x27;,\n",
              "                                 OneHotEncoder(handle_unknown=&#x27;ignore&#x27;,\n",
              "                                               sparse_output=False),\n",
              "                                 array([ 0,  1,  2,  5, 15, -1, 20, 21, 22, 23, 16, 18, 20, 21, 22, 23, 24,\n",
              "       26, 27, 28, -1, 31, 40, 41, 42]))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">cat_ordinal</label><div class=\"sk-toggleable__content\"><pre>[ 3  4  6  7  8  9 10 11 12 13 14 15 17 19 25 29 32 33 34 35 36 37 38 39]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OrdinalEncoder</label><div class=\"sk-toggleable__content\"><pre>OrdinalEncoder(categories=[[&#x27;NA&#x27;, &#x27;Pave&#x27;, &#x27;Grvl&#x27;], [&#x27;NA&#x27;, &#x27;N&#x27;, &#x27;Y&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;No&#x27;, &#x27;Mn&#x27;, &#x27;Av&#x27;, &#x27;Gd&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;, &#x27;GLQ&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Pave&#x27;, &#x27;Grvl&#x27;],\n",
              "                           [&#x27;NA...\n",
              "                            &#x27;Metal&#x27;, &#x27;Membran&#x27;, &#x27;CompShg&#x27;, &#x27;ClyTile&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;, &#x27;GLQ&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Sal&#x27;, &#x27;Sev&#x27;, &#x27;Maj2&#x27;, &#x27;Maj1&#x27;, &#x27;Mod&#x27;, &#x27;Min2&#x27;,\n",
              "                            &#x27;Min1&#x27;, &#x27;Typ&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Detchd&#x27;, &#x27;CarPort&#x27;, &#x27;BuiltIn&#x27;, &#x27;Basment&#x27;,\n",
              "                            &#x27;Attchd&#x27;, &#x27;2Types&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Unf&#x27;, &#x27;RFn&#x27;, &#x27;Fin&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;N&#x27;, &#x27;P&#x27;, &#x27;Y&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;MnWw&#x27;, &#x27;GdWo&#x27;, &#x27;MnPrv&#x27;, &#x27;GdPrv&#x27;]])</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">cat_onehot</label><div class=\"sk-toggleable__content\"><pre>[ 0  1  2  5 15 -1 20 21 22 23 16 18 20 21 22 23 24 26 27 28 -1 31 40 41\n",
              " 42]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OneHotEncoder</label><div class=\"sk-toggleable__content\"><pre>OneHotEncoder(handle_unknown=&#x27;ignore&#x27;, sparse_output=False)</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "from sklearn.impute import KNNImputer\n",
        "from sklearn.pipeline import make_pipeline\n",
        "\n",
        "# 0. Set the config so that we can view our preprocessor\n",
        "set_config(display=\"diagram\")\n",
        "\n",
        "# 1. defining categorical & ordinal columns\n",
        "X_cat = X.select_dtypes(exclude=\"number\").copy()\n",
        "X_num = X.select_dtypes(include=\"number\").copy()\n",
        "\n",
        "# 2. numerical pipeline , default strategy we search for the best parameters later\n",
        "numeric_pipe = make_pipeline(\n",
        "    SimpleImputer(\"mean\"))\n",
        "\n",
        "# 3. categorical pipeline\n",
        "\n",
        "# # 3.1 defining ordinal & onehot columns\n",
        "# .get_indexer() get's the index to solve the problem described above about losing column names\n",
        "ordinal_cols = X_cat.columns.get_indexer([\"Street\",\n",
        "                                          \"CentralAir\",\n",
        "                                          \"ExterQual\",\n",
        "                                          \"ExterCond\",\n",
        "                                          \"BsmtQual\",\n",
        "                                          \"BsmtCond\",\n",
        "                                          \"BsmtExposure\",\n",
        "                                          \"BsmtFinType1\",\n",
        "                                          \"KitchenQual\",\n",
        "                                          \"FireplaceQu\",\n",
        "                                          \"Alley\",\n",
        "                                          \"LotShape\",\n",
        "                                          \"Utilities\",\n",
        "                                          \"LandSlope\",\n",
        "                                          \"RoofMatl\",\n",
        "                                          \"BsmtFinType2\",\n",
        "                                          \"Functional\",\n",
        "                                          \"GarageType\",\n",
        "                                          \"GarageFinish\",\n",
        "                                          \"GarageQual\",\n",
        "                                          \"GarageCond\",\n",
        "                                          \"PavedDrive\",\n",
        "                                          \"PoolQC\",\n",
        "                                          \"Fence\"])\n",
        "\n",
        "\n",
        "onehot_cols = X_cat.columns.get_indexer([\"MSZoning\",\n",
        "                                         \"Condition1\",\n",
        "                                         \"Heating\",\n",
        "                                         \"Foundation\",\n",
        "                                         \"LotShape\",\n",
        "                                         \"LotCongig\",\n",
        "                                         \"Neighborhood\",\n",
        "                                         \"Condition2\",\n",
        "                                         \"BldgType\",\n",
        "                                         \"HouseStyle\",\n",
        "                                         \"LandContour\",\n",
        "                                         \"LotConfig\",\n",
        "                                         \"Neighborhood\",\n",
        "                                         \"Condition2\",\n",
        "                                         \"BldgType\",\n",
        "                                         \"HouseStyle\",\n",
        "                                         \"RoofStyle\",\n",
        "                                         \"Exterior1st\",\n",
        "                                         \"Exterior2nd\",\n",
        "                                         \"MasVnrType\",\n",
        "                                         \"HeatingQc\",\n",
        "                                         \"Electrical\",\n",
        "                                         \"MiscFeature\",\n",
        "                                         \"SaleType\",\n",
        "                                         \"SaleCondition\"])\n",
        "\n",
        "\n",
        "# # 3.2. defining the categorical encoder\n",
        "\n",
        "Street_cats = [ \"NA\", \"Pave\", \"Grvl\"]\n",
        "CentralAir_cats = [\"NA\",\"N\", \"Y\" ]\n",
        "ExterQual_cats = [\"NA\",\"Po\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "ExterCond_cats = [\"NA\",\"Po\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "BsmtQual_cats = [\"NA\",\"Po\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "BsmtCond_cats = [\"NA\",\"Po\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "BsmtExposure_cats = [\"NA\", \"No\",\"Mn\",\"Av\",\"Gd\"]\n",
        "BsmtFinType1_cats = [\"NA\", \"Unf\",\"LwQ\",\"Rec\",\"BLQ\",\"ALQ\",\"GLQ\"]\n",
        "KitchenQual_cats = [\"NA\",\"Po\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "FireplaceQu_cats = [\"NA\",\"Po\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "Alley_cats = [ \"NA\", \"Pave\", \"Grvl\"]\n",
        "LotShape_cats = [\"NA\", \"IR3\",\"IR2\",\"IR1\",\"Reg\"]\n",
        "Utilities_cats = [\"NA\", \"ELO\",\"NoSeWa\",\"NoSewr\",\"AllPub\"]\n",
        "LandSlope_cats = [\"NA\", \"Sev\",\"Mod\",\"Gtl\"]\n",
        "RoofMatl_cats = [\"NA\",\"WdShngl\",\"WdShake\",\"Tar&Grv\",\"Roll\",\"Metal\",\"Membran\",\"CompShg\",\"ClyTile\"]\n",
        "BsmtFinType2_cats = [\"NA\", \"Unf\",\"LwQ\",\"Rec\",\"BLQ\",\"ALQ\",\"GLQ\"]\n",
        "Functional_cats = [\"NA\",\"Sal\",\"Sev\",\"Maj2\",\"Maj1\",\"Mod\",\"Min2\",\"Min1\",\"Typ\"]\n",
        "GarageType_cats = [\"NA\", \"Detchd\",\"CarPort\",\"BuiltIn\",\"Basment\",\"Attchd\",\"2Types\"]\n",
        "GarageFinish_cats = [\"NA\", \"Unf\",\"RFn\",\"Fin\"]\n",
        "GarageQual_cats = [\"NA\",\"Po\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "GarageCond_cats = [\"NA\",\"Po\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "PavedDrive_cats = [\"NA\",\"N\",\"P\",\"Y\"]\n",
        "PoolQC_cats = [\"NA\",\"Fa\",\"TA\",\"Gd\",\"Ex\"]\n",
        "Fence_cats = [\"NA\", \"MnWw\",\"GdWo\",\"MnPrv\",\"GdPrv\"]\n",
        "\n",
        "\n",
        "\n",
        "# # # 3.2.1. we manually establish the order of the categories for our ordinal feature (Cabin) from worst to best, including \"N_A\"\n",
        "\n",
        "ordinal_cats = [Street_cats,\n",
        "                CentralAir_cats,\n",
        "                ExterQual_cats,\n",
        "                ExterCond_cats,\n",
        "                BsmtQual_cats,\n",
        "                BsmtCond_cats,\n",
        "                BsmtExposure_cats,\n",
        "                BsmtFinType1_cats,\n",
        "                KitchenQual_cats,\n",
        "                FireplaceQu_cats,\n",
        "                Alley_cats,\n",
        "                LotShape_cats,\n",
        "                Utilities_cats,\n",
        "                LandSlope_cats,\n",
        "                RoofMatl_cats,\n",
        "                BsmtFinType2_cats,\n",
        "                Functional_cats,\n",
        "                GarageType_cats,\n",
        "                GarageFinish_cats,\n",
        "                GarageQual_cats,\n",
        "                GarageCond_cats,\n",
        "                PavedDrive_cats,\n",
        "                PoolQC_cats,\n",
        "                Fence_cats]\n",
        "\n",
        "ordinal_encoder = OrdinalEncoder(categories=ordinal_cats)\n",
        "\n",
        "# # # 3.2.2. defining the categorical encoder: a ColumnTransformer with 2 branches: ordinal & onehot\n",
        "categorical_encoder = ColumnTransformer(\n",
        "    transformers=[\n",
        "        (\"cat_ordinal\", ordinal_encoder, ordinal_cols),\n",
        "        (\"cat_onehot\", OneHotEncoder(sparse_output=False, handle_unknown=\"ignore\"), onehot_cols),\n",
        "    ]\n",
        ")\n",
        "\n",
        "# # 3.3. categorical pipeline = \"N_A\" imputer + categorical encoder\n",
        "categorical_pipe = make_pipeline(SimpleImputer(strategy=\"constant\", fill_value=\"NA\"),\n",
        "                                 categorical_encoder\n",
        "                                )\n",
        "\n",
        "# 4. full preprocessing: a ColumnTransformer with 2 branches: numeric & categorical\n",
        "full_preprocessing = ColumnTransformer(\n",
        "    transformers=[\n",
        "        (\"num_pipe\", numeric_pipe, X_num.columns),\n",
        "        (\"cat_pipe\", categorical_pipe, X_cat.columns),\n",
        "    ]\n",
        ")\n",
        "\n",
        "full_preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4aii_JBkDfMK"
      },
      "source": [
        "## 3 LazyPredict\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "quGutrsxDeew",
        "outputId": "80ccde54-2da7-4163-cf50-9143b8092c02"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting lazypredict\n",
            "  Downloading lazypredict-0.2.12-py2.py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from lazypredict) (8.1.7)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from lazypredict) (1.2.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from lazypredict) (1.5.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from lazypredict) (4.66.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from lazypredict) (1.3.2)\n",
            "Requirement already satisfied: lightgbm in /usr/local/lib/python3.10/dist-packages (from lazypredict) (4.1.0)\n",
            "Requirement already satisfied: xgboost in /usr/local/lib/python3.10/dist-packages (from lazypredict) (2.0.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from lightgbm->lazypredict) (1.23.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from lightgbm->lazypredict) (1.11.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->lazypredict) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->lazypredict) (2023.3.post1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->lazypredict) (3.2.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->lazypredict) (1.16.0)\n",
            "Installing collected packages: lazypredict\n",
            "Successfully installed lazypredict-0.2.12\n"
          ]
        }
      ],
      "source": [
        "# pip install lazypredict"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "opRgnLNhEFnq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "8492b1f3-ff59-4f0f-8213-c7a66f3b1c32"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 97%|█████████▋| 28/29 [00:06<00:00,  6.45it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[LightGBM] [Info] Number of positive: 169, number of negative: 999\n",
            "[LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001158 seconds.\n",
            "You can set `force_row_wise=true` to remove the overhead.\n",
            "And if memory is not enough, you can set `force_col_wise=true`.\n",
            "[LightGBM] [Info] Total Bins 3215\n",
            "[LightGBM] [Info] Number of data points in the train set: 1168, number of used features: 162\n",
            "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.144692 -> initscore=-1.776856\n",
            "[LightGBM] [Info] Start training from score -1.776856\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 29/29 [00:06<00:00,  4.45it/s]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                               Accuracy  Balanced Accuracy  ROC AUC  F1 Score  \\\n",
              "Model                                                                           \n",
              "Perceptron                         0.95               0.92     0.92      0.95   \n",
              "XGBClassifier                      0.97               0.91     0.91      0.96   \n",
              "BernoulliNB                        0.86               0.90     0.90      0.87   \n",
              "LGBMClassifier                     0.96               0.89     0.89      0.96   \n",
              "LinearSVC                          0.95               0.88     0.88      0.94   \n",
              "ExtraTreesClassifier               0.96               0.87     0.87      0.95   \n",
              "AdaBoostClassifier                 0.95               0.87     0.87      0.95   \n",
              "LogisticRegression                 0.95               0.87     0.87      0.94   \n",
              "BaggingClassifier                  0.95               0.86     0.86      0.95   \n",
              "LinearDiscriminantAnalysis         0.93               0.86     0.86      0.93   \n",
              "SVC                                0.95               0.85     0.85      0.94   \n",
              "NearestCentroid                    0.83               0.85     0.85      0.84   \n",
              "DecisionTreeClassifier             0.93               0.84     0.84      0.93   \n",
              "RandomForestClassifier             0.95               0.83     0.83      0.94   \n",
              "SGDClassifier                      0.93               0.83     0.83      0.93   \n",
              "ExtraTreeClassifier                0.91               0.83     0.83      0.91   \n",
              "CalibratedClassifierCV             0.94               0.82     0.82      0.93   \n",
              "KNeighborsClassifier               0.93               0.81     0.81      0.92   \n",
              "PassiveAggressiveClassifier        0.93               0.79     0.79      0.92   \n",
              "RidgeClassifierCV                  0.92               0.79     0.79      0.92   \n",
              "RidgeClassifier                    0.92               0.79     0.79      0.91   \n",
              "LabelSpreading                     0.91               0.76     0.76      0.90   \n",
              "LabelPropagation                   0.91               0.76     0.76      0.90   \n",
              "GaussianNB                         0.50               0.68     0.68      0.55   \n",
              "QuadraticDiscriminantAnalysis      0.84               0.53     0.53      0.78   \n",
              "DummyClassifier                    0.84               0.50     0.50      0.76   \n",
              "\n",
              "                               Time Taken  \n",
              "Model                                      \n",
              "Perceptron                           0.10  \n",
              "XGBClassifier                        0.25  \n",
              "BernoulliNB                          0.16  \n",
              "LGBMClassifier                       0.26  \n",
              "LinearSVC                            0.18  \n",
              "ExtraTreesClassifier                 0.30  \n",
              "AdaBoostClassifier                   1.14  \n",
              "LogisticRegression                   0.17  \n",
              "BaggingClassifier                    0.38  \n",
              "LinearDiscriminantAnalysis           0.20  \n",
              "SVC                                  0.16  \n",
              "NearestCentroid                      0.12  \n",
              "DecisionTreeClassifier               0.10  \n",
              "RandomForestClassifier               0.38  \n",
              "SGDClassifier                        0.15  \n",
              "ExtraTreeClassifier                  0.06  \n",
              "CalibratedClassifierCV               0.94  \n",
              "KNeighborsClassifier                 0.09  \n",
              "PassiveAggressiveClassifier          0.09  \n",
              "RidgeClassifierCV                    0.23  \n",
              "RidgeClassifier                      0.12  \n",
              "LabelSpreading                       0.26  \n",
              "LabelPropagation                     0.17  \n",
              "GaussianNB                           0.06  \n",
              "QuadraticDiscriminantAnalysis        0.18  \n",
              "DummyClassifier                      0.06  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-44ebbaed-e6d1-4b3d-80ea-b31b5d8694ad\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Balanced Accuracy</th>\n",
              "      <th>ROC AUC</th>\n",
              "      <th>F1 Score</th>\n",
              "      <th>Time Taken</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Model</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Perceptron</th>\n",
              "      <td>0.95</td>\n",
              "      <td>0.92</td>\n",
              "      <td>0.92</td>\n",
              "      <td>0.95</td>\n",
              "      <td>0.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>XGBClassifier</th>\n",
              "      <td>0.97</td>\n",
              "      <td>0.91</td>\n",
              "      <td>0.91</td>\n",
              "      <td>0.96</td>\n",
              "      <td>0.25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>BernoulliNB</th>\n",
              "      <td>0.86</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LGBMClassifier</th>\n",
              "      <td>0.96</td>\n",
              "      <td>0.89</td>\n",
              "      <td>0.89</td>\n",
              "      <td>0.96</td>\n",
              "      <td>0.26</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LinearSVC</th>\n",
              "      <td>0.95</td>\n",
              "      <td>0.88</td>\n",
              "      <td>0.88</td>\n",
              "      <td>0.94</td>\n",
              "      <td>0.18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ExtraTreesClassifier</th>\n",
              "      <td>0.96</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.95</td>\n",
              "      <td>0.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AdaBoostClassifier</th>\n",
              "      <td>0.95</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.95</td>\n",
              "      <td>1.14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LogisticRegression</th>\n",
              "      <td>0.95</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.94</td>\n",
              "      <td>0.17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>BaggingClassifier</th>\n",
              "      <td>0.95</td>\n",
              "      <td>0.86</td>\n",
              "      <td>0.86</td>\n",
              "      <td>0.95</td>\n",
              "      <td>0.38</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LinearDiscriminantAnalysis</th>\n",
              "      <td>0.93</td>\n",
              "      <td>0.86</td>\n",
              "      <td>0.86</td>\n",
              "      <td>0.93</td>\n",
              "      <td>0.20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SVC</th>\n",
              "      <td>0.95</td>\n",
              "      <td>0.85</td>\n",
              "      <td>0.85</td>\n",
              "      <td>0.94</td>\n",
              "      <td>0.16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>NearestCentroid</th>\n",
              "      <td>0.83</td>\n",
              "      <td>0.85</td>\n",
              "      <td>0.85</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>DecisionTreeClassifier</th>\n",
              "      <td>0.93</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.93</td>\n",
              "      <td>0.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>RandomForestClassifier</th>\n",
              "      <td>0.95</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0.94</td>\n",
              "      <td>0.38</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SGDClassifier</th>\n",
              "      <td>0.93</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0.93</td>\n",
              "      <td>0.15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ExtraTreeClassifier</th>\n",
              "      <td>0.91</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0.91</td>\n",
              "      <td>0.06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CalibratedClassifierCV</th>\n",
              "      <td>0.94</td>\n",
              "      <td>0.82</td>\n",
              "      <td>0.82</td>\n",
              "      <td>0.93</td>\n",
              "      <td>0.94</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>KNeighborsClassifier</th>\n",
              "      <td>0.93</td>\n",
              "      <td>0.81</td>\n",
              "      <td>0.81</td>\n",
              "      <td>0.92</td>\n",
              "      <td>0.09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PassiveAggressiveClassifier</th>\n",
              "      <td>0.93</td>\n",
              "      <td>0.79</td>\n",
              "      <td>0.79</td>\n",
              "      <td>0.92</td>\n",
              "      <td>0.09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>RidgeClassifierCV</th>\n",
              "      <td>0.92</td>\n",
              "      <td>0.79</td>\n",
              "      <td>0.79</td>\n",
              "      <td>0.92</td>\n",
              "      <td>0.23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>RidgeClassifier</th>\n",
              "      <td>0.92</td>\n",
              "      <td>0.79</td>\n",
              "      <td>0.79</td>\n",
              "      <td>0.91</td>\n",
              "      <td>0.12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LabelSpreading</th>\n",
              "      <td>0.91</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.26</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LabelPropagation</th>\n",
              "      <td>0.91</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>GaussianNB</th>\n",
              "      <td>0.50</td>\n",
              "      <td>0.68</td>\n",
              "      <td>0.68</td>\n",
              "      <td>0.55</td>\n",
              "      <td>0.06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>QuadraticDiscriminantAnalysis</th>\n",
              "      <td>0.84</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.78</td>\n",
              "      <td>0.18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>DummyClassifier</th>\n",
              "      <td>0.84</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-44ebbaed-e6d1-4b3d-80ea-b31b5d8694ad')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-44ebbaed-e6d1-4b3d-80ea-b31b5d8694ad button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-44ebbaed-e6d1-4b3d-80ea-b31b5d8694ad');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-181410d4-587a-4fa3-b2c3-f6224db400b1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-181410d4-587a-4fa3-b2c3-f6224db400b1')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-181410d4-587a-4fa3-b2c3-f6224db400b1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "import lazypredict\n",
        "from lazypredict.Supervised import LazyClassifier\n",
        "\n",
        "\n",
        "# Initialize and fit LazyClassifier\n",
        "clf = LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None)\n",
        "models, predictions = clf.fit(X_train, X_test, y_train, y_test)\n",
        "\n",
        "# Print the performance of each model\n",
        "models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m-pQuzdyFdzp"
      },
      "source": [
        "The top 5 models based on Accuracy are\n",
        "1. XGBClassifier\n",
        "2. LGBMClassifier\n",
        "3. ExtraTreesClassifier\n",
        "4. Perceptron\n",
        "5. LogisticRegression."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "FwVHGas4vWkh"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4rIzgwsQHVPQ"
      },
      "source": [
        "## 4 Modelling\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nIMMWW_nPfWm"
      },
      "source": [
        "### 4.1 XGB Classifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "fBJAmazoPhZZ"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from xgboost import XGBClassifier\n",
        "\n",
        "\n",
        "# XGBoost pipeline\n",
        "xgb_full_pipeline = make_pipeline(full_preprocessing,\n",
        "                                  StandardScaler(),\n",
        "                                  XGBClassifier())\n",
        "\n",
        "# Parameter grid for XGBoost\n",
        "xgb_param_grid = {\n",
        "    \"columntransformer__num_pipe__simpleimputer__strategy\": [\"mean\", \"median\"],\n",
        "    \"xgbclassifier__n_estimators\": [100, 150, 200],\n",
        "    \"xgbclassifier__learning_rate\": [0.01, 0.1, 0.2],\n",
        "    \"xgbclassifier__max_depth\": [3, 5, 7],\n",
        "    \"xgbclassifier__min_child_weight\": [1, 3, 5]\n",
        "}\n",
        "\n",
        "# GridSearchCV for XGBoost\n",
        "xgb_search = GridSearchCV(xgb_full_pipeline,\n",
        "                          xgb_param_grid,\n",
        "                          cv=5,\n",
        "                          verbose=1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jqhBzfI_SZde",
        "outputId": "9deb0289-df3d-4eef-f8b4-e79872171165"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 162 candidates, totalling 810 fits\n",
            "The best average score in cross validation was 0.9503539855471186\n",
            "The best parameters are {'columntransformer__num_pipe__simpleimputer__strategy': 'median', 'xgbclassifier__learning_rate': 0.1, 'xgbclassifier__max_depth': 5, 'xgbclassifier__min_child_weight': 1, 'xgbclassifier__n_estimators': 100}\n"
          ]
        }
      ],
      "source": [
        "xgb_search.fit(X_train, y_train)\n",
        "\n",
        "print(f\"The best average score in cross validation was {xgb_search.best_score_}\")\n",
        "print(f\"The best parameters are {xgb_search.best_params_}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qFOu2ckgUE-b",
        "outputId": "0da92814-8898-44b5-ac44-e8d102c69581"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9623287671232876"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "accuracy_score(xgb_search.predict(X_test), y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9MNpo3hKOrxo"
      },
      "source": [
        "### 4.2 LGBMClassifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "OddFo1rESpt7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6c1ceeba-0d4e-4578-f53f-492b83fed2eb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: lightgbm in /usr/local/lib/python3.10/dist-packages (4.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from lightgbm) (1.23.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from lightgbm) (1.11.3)\n"
          ]
        }
      ],
      "source": [
        "# pip install lightgbm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 72,
      "metadata": {
        "id": "sYcOh1n6O1AO"
      },
      "outputs": [],
      "source": [
        "from lightgbm import LGBMClassifier\n",
        "from scipy.stats import randint, uniform\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "\n",
        "\n",
        "#  pipeline\n",
        "lgb_full_pipeline = make_pipeline(full_preprocessing,\n",
        "                                  StandardScaler(),\n",
        "                                  LGBMClassifier())\n",
        "\n",
        "# Parameter grid\n",
        "lgb_param_grid = {\n",
        "    'columntransformer__num_pipe__simpleimputer__strategy': [\"mean\", \"median\"],\n",
        "    'lgbmclassifier__learning_rate': [0.01, 0.05, 0.1],\n",
        "    'lgbmclassifier__n_estimators': [50, 100, 200],\n",
        "    'lgbmclassifier__max_depth': [3, 5, 7],\n",
        "    'lgbmclassifier__min_child_samples': [10, 20, 30],\n",
        "    'lgbmclassifier__subsample': [0.8, 0.9, 1.0],\n",
        "    'lgbmclassifier__colsample_bytree': [0.8, 0.9, 1.0],\n",
        "    'lgbmclassifier__reg_alpha': [0.0, 0.1, 0.5],\n",
        "    'lgbmclassifier__reg_lambda': [0.0, 0.1, 0.5],\n",
        "    'lgbmclassifier__importance_type': ['split', 'gain'],\n",
        "    'lgbmclassifier__bagging_fraction': [0.8, 0.9, 1.0],\n",
        "    'lgbmclassifier__bagging_freq': [0, 5, 10],\n",
        "    'lgbmclassifier__boosting_type': ['gbdt', 'dart']\n",
        "}\n",
        "\n",
        "# RandomizedSearchCV - GridSearchCV would be over 2 million fits\n",
        "lgb_search = RandomizedSearchCV(lgb_full_pipeline,\n",
        "                          lgb_param_grid,\n",
        "                          cv=5,\n",
        "                          verbose=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RQCoaOwpO8_K"
      },
      "outputs": [],
      "source": [
        "lgb_search.fit(X_train, y_train)\n",
        "\n",
        "print(f\"The best average score in cross validation was {lgb_search.best_score_}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "metadata": {
        "id": "lKHfxNolPCfA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2af7407a-75dc-4493-d094-907167fd313a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[LightGBM] [Warning] bagging_fraction is set=0.8, subsample=0.8 will be ignored. Current value: bagging_fraction=0.8\n",
            "[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10\n",
            "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9657534246575342"
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ],
      "source": [
        "accuracy_score(lgb_search.predict(X_test), y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 4.3 ExtraTreesClassifier\n"
      ],
      "metadata": {
        "id": "V4gUYaZ9wsGW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.impute import SimpleImputer\n",
        "from scipy.stats import randint\n",
        "\n",
        "\n",
        "# Create the ExtraTreesClassifier pipeline\n",
        "ext_full_pipeline = make_pipeline(\n",
        "    full_preprocessing,\n",
        "    StandardScaler(),\n",
        "    ExtraTreesClassifier(random_state=42)  # Set any other parameters you need\n",
        ")\n",
        "\n",
        "# Parameter grid\n",
        "ext_param_grid = {\n",
        "    \"columntransformer__num_pipe__simpleimputer__strategy\": [\"mean\", \"median\"],\n",
        "    \"extratreesclassifier__n_estimators\": randint(50, 100).rvs(10),\n",
        "    \"extratreesclassifier__max_depth\": [None] + list(randint(5, 30).rvs(10))\n",
        "}\n",
        "\n",
        "# GridSearchCV\n",
        "ext_search = GridSearchCV(\n",
        "    ext_full_pipeline,\n",
        "    ext_param_grid,\n",
        "    cv=5,\n",
        "    verbose=1\n",
        ")\n",
        "\n",
        "# Fit the grid search\n",
        "ext_search.fit(X_train, y_train)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "id": "RL5GZLgXwu_X",
        "outputId": "6e016240-509d-4b67-f813-ab0d09b48c9c"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 220 candidates, totalling 1100 fits\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GridSearchCV(cv=5,\n",
              "             estimator=Pipeline(steps=[('columntransformer',\n",
              "                                        ColumnTransformer(transformers=[('num_pipe',\n",
              "                                                                         Pipeline(steps=[('simpleimputer',\n",
              "                                                                                          SimpleImputer())]),\n",
              "                                                                         Index(['LotArea', 'LotFrontage', 'TotalBsmtSF', 'BedroomAbvGr', 'Fireplaces',\n",
              "       'PoolArea', 'GarageCars', 'WoodDeckSF', 'ScreenPorch', 'MSSubClass',\n",
              "       'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAd...\n",
              "      dtype='object'))])),\n",
              "                                       ('standardscaler', StandardScaler()),\n",
              "                                       ('extratreesclassifier',\n",
              "                                        ExtraTreesClassifier(random_state=42))]),\n",
              "             param_grid={'columntransformer__num_pipe__simpleimputer__strategy': ['mean',\n",
              "                                                                                  'median'],\n",
              "                         'extratreesclassifier__max_depth': [None, 24, 15, 12,\n",
              "                                                             13, 12, 13, 5, 16,\n",
              "                                                             26, 7],\n",
              "                         'extratreesclassifier__n_estimators': array([66, 86, 89, 89, 65, 76, 80, 54, 62, 71])},\n",
              "             verbose=1)"
            ],
            "text/html": [
              "<style>#sk-container-id-4 {color: black;background-color: white;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5,\n",
              "             estimator=Pipeline(steps=[(&#x27;columntransformer&#x27;,\n",
              "                                        ColumnTransformer(transformers=[(&#x27;num_pipe&#x27;,\n",
              "                                                                         Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
              "                                                                                          SimpleImputer())]),\n",
              "                                                                         Index([&#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;Fireplaces&#x27;,\n",
              "       &#x27;PoolArea&#x27;, &#x27;GarageCars&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;ScreenPorch&#x27;, &#x27;MSSubClass&#x27;,\n",
              "       &#x27;OverallQual&#x27;, &#x27;OverallCond&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAd...\n",
              "      dtype=&#x27;object&#x27;))])),\n",
              "                                       (&#x27;standardscaler&#x27;, StandardScaler()),\n",
              "                                       (&#x27;extratreesclassifier&#x27;,\n",
              "                                        ExtraTreesClassifier(random_state=42))]),\n",
              "             param_grid={&#x27;columntransformer__num_pipe__simpleimputer__strategy&#x27;: [&#x27;mean&#x27;,\n",
              "                                                                                  &#x27;median&#x27;],\n",
              "                         &#x27;extratreesclassifier__max_depth&#x27;: [None, 24, 15, 12,\n",
              "                                                             13, 12, 13, 5, 16,\n",
              "                                                             26, 7],\n",
              "                         &#x27;extratreesclassifier__n_estimators&#x27;: array([66, 86, 89, 89, 65, 76, 80, 54, 62, 71])},\n",
              "             verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-39\" type=\"checkbox\" ><label for=\"sk-estimator-id-39\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5,\n",
              "             estimator=Pipeline(steps=[(&#x27;columntransformer&#x27;,\n",
              "                                        ColumnTransformer(transformers=[(&#x27;num_pipe&#x27;,\n",
              "                                                                         Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
              "                                                                                          SimpleImputer())]),\n",
              "                                                                         Index([&#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;Fireplaces&#x27;,\n",
              "       &#x27;PoolArea&#x27;, &#x27;GarageCars&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;ScreenPorch&#x27;, &#x27;MSSubClass&#x27;,\n",
              "       &#x27;OverallQual&#x27;, &#x27;OverallCond&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAd...\n",
              "      dtype=&#x27;object&#x27;))])),\n",
              "                                       (&#x27;standardscaler&#x27;, StandardScaler()),\n",
              "                                       (&#x27;extratreesclassifier&#x27;,\n",
              "                                        ExtraTreesClassifier(random_state=42))]),\n",
              "             param_grid={&#x27;columntransformer__num_pipe__simpleimputer__strategy&#x27;: [&#x27;mean&#x27;,\n",
              "                                                                                  &#x27;median&#x27;],\n",
              "                         &#x27;extratreesclassifier__max_depth&#x27;: [None, 24, 15, 12,\n",
              "                                                             13, 12, 13, 5, 16,\n",
              "                                                             26, 7],\n",
              "                         &#x27;extratreesclassifier__n_estimators&#x27;: array([66, 86, 89, 89, 65, 76, 80, 54, 62, 71])},\n",
              "             verbose=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-40\" type=\"checkbox\" ><label for=\"sk-estimator-id-40\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;columntransformer&#x27;,\n",
              "                 ColumnTransformer(transformers=[(&#x27;num_pipe&#x27;,\n",
              "                                                  Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
              "                                                                   SimpleImputer())]),\n",
              "                                                  Index([&#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;Fireplaces&#x27;,\n",
              "       &#x27;PoolArea&#x27;, &#x27;GarageCars&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;ScreenPorch&#x27;, &#x27;MSSubClass&#x27;,\n",
              "       &#x27;OverallQual&#x27;, &#x27;OverallCond&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;MasVnrArea&#x27;,\n",
              "       &#x27;BsmtFinSF1&#x27;...\n",
              "       &#x27;RoofMatl&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;MasVnrType&#x27;, &#x27;BsmtFinType2&#x27;,\n",
              "       &#x27;HeatingQC&#x27;, &#x27;Electrical&#x27;, &#x27;Functional&#x27;, &#x27;GarageType&#x27;, &#x27;GarageFinish&#x27;,\n",
              "       &#x27;GarageQual&#x27;, &#x27;GarageCond&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;, &#x27;Fence&#x27;,\n",
              "       &#x27;MiscFeature&#x27;, &#x27;SaleType&#x27;, &#x27;SaleCondition&#x27;],\n",
              "      dtype=&#x27;object&#x27;))])),\n",
              "                (&#x27;standardscaler&#x27;, StandardScaler()),\n",
              "                (&#x27;extratreesclassifier&#x27;,\n",
              "                 ExtraTreesClassifier(random_state=42))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-41\" type=\"checkbox\" ><label for=\"sk-estimator-id-41\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">columntransformer: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[(&#x27;num_pipe&#x27;,\n",
              "                                 Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
              "                                                  SimpleImputer())]),\n",
              "                                 Index([&#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;Fireplaces&#x27;,\n",
              "       &#x27;PoolArea&#x27;, &#x27;GarageCars&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;ScreenPorch&#x27;, &#x27;MSSubClass&#x27;,\n",
              "       &#x27;OverallQual&#x27;, &#x27;OverallCond&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;MasVnrArea&#x27;,\n",
              "       &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;, &#x27;BsmtUnfSF&#x27;, &#x27;1stFlrSF&#x27;,...\n",
              "       &#x27;LotShape&#x27;, &#x27;LandContour&#x27;, &#x27;Utilities&#x27;, &#x27;LotConfig&#x27;, &#x27;LandSlope&#x27;,\n",
              "       &#x27;Neighborhood&#x27;, &#x27;Condition2&#x27;, &#x27;BldgType&#x27;, &#x27;HouseStyle&#x27;, &#x27;RoofStyle&#x27;,\n",
              "       &#x27;RoofMatl&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;MasVnrType&#x27;, &#x27;BsmtFinType2&#x27;,\n",
              "       &#x27;HeatingQC&#x27;, &#x27;Electrical&#x27;, &#x27;Functional&#x27;, &#x27;GarageType&#x27;, &#x27;GarageFinish&#x27;,\n",
              "       &#x27;GarageQual&#x27;, &#x27;GarageCond&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;, &#x27;Fence&#x27;,\n",
              "       &#x27;MiscFeature&#x27;, &#x27;SaleType&#x27;, &#x27;SaleCondition&#x27;],\n",
              "      dtype=&#x27;object&#x27;))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-42\" type=\"checkbox\" ><label for=\"sk-estimator-id-42\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">num_pipe</label><div class=\"sk-toggleable__content\"><pre>Index([&#x27;LotArea&#x27;, &#x27;LotFrontage&#x27;, &#x27;TotalBsmtSF&#x27;, &#x27;BedroomAbvGr&#x27;, &#x27;Fireplaces&#x27;,\n",
              "       &#x27;PoolArea&#x27;, &#x27;GarageCars&#x27;, &#x27;WoodDeckSF&#x27;, &#x27;ScreenPorch&#x27;, &#x27;MSSubClass&#x27;,\n",
              "       &#x27;OverallQual&#x27;, &#x27;OverallCond&#x27;, &#x27;YearBuilt&#x27;, &#x27;YearRemodAdd&#x27;, &#x27;MasVnrArea&#x27;,\n",
              "       &#x27;BsmtFinSF1&#x27;, &#x27;BsmtFinSF2&#x27;, &#x27;BsmtUnfSF&#x27;, &#x27;1stFlrSF&#x27;, &#x27;2ndFlrSF&#x27;,\n",
              "       &#x27;LowQualFinSF&#x27;, &#x27;GrLivArea&#x27;, &#x27;BsmtFullBath&#x27;, &#x27;BsmtHalfBath&#x27;, &#x27;FullBath&#x27;,\n",
              "       &#x27;HalfBath&#x27;, &#x27;KitchenAbvGr&#x27;, &#x27;TotRmsAbvGrd&#x27;, &#x27;GarageYrBlt&#x27;, &#x27;GarageArea&#x27;,\n",
              "       &#x27;OpenPorchSF&#x27;, &#x27;EnclosedPorch&#x27;, &#x27;3SsnPorch&#x27;, &#x27;MiscVal&#x27;, &#x27;MoSold&#x27;,\n",
              "       &#x27;YrSold&#x27;],\n",
              "      dtype=&#x27;object&#x27;)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-43\" type=\"checkbox\" ><label for=\"sk-estimator-id-43\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-44\" type=\"checkbox\" ><label for=\"sk-estimator-id-44\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">cat_pipe</label><div class=\"sk-toggleable__content\"><pre>Index([&#x27;MSZoning&#x27;, &#x27;Condition1&#x27;, &#x27;Heating&#x27;, &#x27;Street&#x27;, &#x27;CentralAir&#x27;,\n",
              "       &#x27;Foundation&#x27;, &#x27;ExterQual&#x27;, &#x27;ExterCond&#x27;, &#x27;BsmtQual&#x27;, &#x27;BsmtCond&#x27;,\n",
              "       &#x27;BsmtExposure&#x27;, &#x27;BsmtFinType1&#x27;, &#x27;KitchenQual&#x27;, &#x27;FireplaceQu&#x27;, &#x27;Alley&#x27;,\n",
              "       &#x27;LotShape&#x27;, &#x27;LandContour&#x27;, &#x27;Utilities&#x27;, &#x27;LotConfig&#x27;, &#x27;LandSlope&#x27;,\n",
              "       &#x27;Neighborhood&#x27;, &#x27;Condition2&#x27;, &#x27;BldgType&#x27;, &#x27;HouseStyle&#x27;, &#x27;RoofStyle&#x27;,\n",
              "       &#x27;RoofMatl&#x27;, &#x27;Exterior1st&#x27;, &#x27;Exterior2nd&#x27;, &#x27;MasVnrType&#x27;, &#x27;BsmtFinType2&#x27;,\n",
              "       &#x27;HeatingQC&#x27;, &#x27;Electrical&#x27;, &#x27;Functional&#x27;, &#x27;GarageType&#x27;, &#x27;GarageFinish&#x27;,\n",
              "       &#x27;GarageQual&#x27;, &#x27;GarageCond&#x27;, &#x27;PavedDrive&#x27;, &#x27;PoolQC&#x27;, &#x27;Fence&#x27;,\n",
              "       &#x27;MiscFeature&#x27;, &#x27;SaleType&#x27;, &#x27;SaleCondition&#x27;],\n",
              "      dtype=&#x27;object&#x27;)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-45\" type=\"checkbox\" ><label for=\"sk-estimator-id-45\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(fill_value=&#x27;NA&#x27;, strategy=&#x27;constant&#x27;)</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-46\" type=\"checkbox\" ><label for=\"sk-estimator-id-46\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">columntransformer: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[(&#x27;cat_ordinal&#x27;,\n",
              "                                 OrdinalEncoder(categories=[[&#x27;NA&#x27;, &#x27;Pave&#x27;,\n",
              "                                                             &#x27;Grvl&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;N&#x27;, &#x27;Y&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;No&#x27;, &#x27;Mn&#x27;,\n",
              "                                                             &#x27;Av&#x27;, &#x27;Gd&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;,\n",
              "                                                             &#x27;Rec&#x27;, &#x27;BLQ&#x27;,\n",
              "                                                             &#x27;ALQ&#x27;, &#x27;GLQ&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;...\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;,\n",
              "                                                             &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;N&#x27;, &#x27;P&#x27;,\n",
              "                                                             &#x27;Y&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;,\n",
              "                                                             &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                                                            [&#x27;NA&#x27;, &#x27;MnWw&#x27;,\n",
              "                                                             &#x27;GdWo&#x27;, &#x27;MnPrv&#x27;,\n",
              "                                                             &#x27;GdPrv&#x27;]]),\n",
              "                                 array([ 3,  4,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 17, 19, 25, 29, 32,\n",
              "       33, 34, 35, 36, 37, 38, 39])),\n",
              "                                (&#x27;cat_onehot&#x27;,\n",
              "                                 OneHotEncoder(handle_unknown=&#x27;ignore&#x27;,\n",
              "                                               sparse_output=False),\n",
              "                                 array([ 0,  1,  2,  5, 15, -1, 20, 21, 22, 23, 16, 18, 20, 21, 22, 23, 24,\n",
              "       26, 27, 28, -1, 31, 40, 41, 42]))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-47\" type=\"checkbox\" ><label for=\"sk-estimator-id-47\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">cat_ordinal</label><div class=\"sk-toggleable__content\"><pre>[ 3  4  6  7  8  9 10 11 12 13 14 15 17 19 25 29 32 33 34 35 36 37 38 39]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-48\" type=\"checkbox\" ><label for=\"sk-estimator-id-48\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OrdinalEncoder</label><div class=\"sk-toggleable__content\"><pre>OrdinalEncoder(categories=[[&#x27;NA&#x27;, &#x27;Pave&#x27;, &#x27;Grvl&#x27;], [&#x27;NA&#x27;, &#x27;N&#x27;, &#x27;Y&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;No&#x27;, &#x27;Mn&#x27;, &#x27;Av&#x27;, &#x27;Gd&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;, &#x27;GLQ&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Pave&#x27;, &#x27;Grvl&#x27;],\n",
              "                           [&#x27;NA...\n",
              "                            &#x27;Metal&#x27;, &#x27;Membran&#x27;, &#x27;CompShg&#x27;, &#x27;ClyTile&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Unf&#x27;, &#x27;LwQ&#x27;, &#x27;Rec&#x27;, &#x27;BLQ&#x27;, &#x27;ALQ&#x27;, &#x27;GLQ&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Sal&#x27;, &#x27;Sev&#x27;, &#x27;Maj2&#x27;, &#x27;Maj1&#x27;, &#x27;Mod&#x27;, &#x27;Min2&#x27;,\n",
              "                            &#x27;Min1&#x27;, &#x27;Typ&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Detchd&#x27;, &#x27;CarPort&#x27;, &#x27;BuiltIn&#x27;, &#x27;Basment&#x27;,\n",
              "                            &#x27;Attchd&#x27;, &#x27;2Types&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Unf&#x27;, &#x27;RFn&#x27;, &#x27;Fin&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Po&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;N&#x27;, &#x27;P&#x27;, &#x27;Y&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;Fa&#x27;, &#x27;TA&#x27;, &#x27;Gd&#x27;, &#x27;Ex&#x27;],\n",
              "                           [&#x27;NA&#x27;, &#x27;MnWw&#x27;, &#x27;GdWo&#x27;, &#x27;MnPrv&#x27;, &#x27;GdPrv&#x27;]])</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-49\" type=\"checkbox\" ><label for=\"sk-estimator-id-49\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">cat_onehot</label><div class=\"sk-toggleable__content\"><pre>[ 0  1  2  5 15 -1 20 21 22 23 16 18 20 21 22 23 24 26 27 28 -1 31 40 41\n",
              " 42]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-50\" type=\"checkbox\" ><label for=\"sk-estimator-id-50\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OneHotEncoder</label><div class=\"sk-toggleable__content\"><pre>OneHotEncoder(handle_unknown=&#x27;ignore&#x27;, sparse_output=False)</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-51\" type=\"checkbox\" ><label for=\"sk-estimator-id-51\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-52\" type=\"checkbox\" ><label for=\"sk-estimator-id-52\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">ExtraTreesClassifier</label><div class=\"sk-toggleable__content\"><pre>ExtraTreesClassifier(random_state=42)</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"The best average score in cross validation was {ext_search.best_score_}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mWbxH-p-x0ZE",
        "outputId": "2eab6823-932c-49a3-80b4-fe2108a39fca"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The best average score in cross validation was 0.9357873885770882\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(ext_search.predict(X_test), y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YZ8SSjrTx3zF",
        "outputId": "ff161bbb-8de4-485d-ca8c-46fd1201554b"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9452054794520548"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hsDgPEIPHb2f"
      },
      "source": [
        "### 4.4 Perceptron"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "BBX3Jk_2HegT"
      },
      "outputs": [],
      "source": [
        "from sklearn.linear_model import Perceptron\n",
        "from scipy.stats import uniform\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "\n",
        "# Pipeline\n",
        "per_full_pipeline = make_pipeline(full_preprocessing, Perceptron())\n",
        "\n",
        "# define parameter grid\n",
        "per_param_grid = {\n",
        "    \"perceptron__alpha\": uniform(0.0001, 0.1).rvs(10),\n",
        "    \"perceptron__max_iter\": [10, 50, 100],\n",
        "    \"perceptron__tol\": uniform(0.001, 0.01).rvs(10),\n",
        "    \"perceptron__eta0\": uniform(0.01, 1.0).rvs(10),\n",
        "}\n",
        "\n",
        "# define GridSearchCV\n",
        "per_search = GridSearchCV(per_full_pipeline,\n",
        "                          per_param_grid,\n",
        "                          cv=5,\n",
        "                          verbose=1)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yYdvpQEjIHv3",
        "outputId": "75892aca-7c9c-47bb-e778-9b58192d4493"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 3000 candidates, totalling 15000 fits\n",
            "The best average score in cross-validation was 0.8878617805656432\n",
            "The best parameters are {'perceptron__alpha': 0.048253501600573095, 'perceptron__eta0': 0.7403442286745863, 'perceptron__max_iter': 10, 'perceptron__tol': 0.010883503986712461}\n"
          ]
        }
      ],
      "source": [
        "per_search.fit(X_train, y_train)\n",
        "\n",
        "print(f\"The best average score in cross-validation was {per_search.best_score_}\")\n",
        "print(f\"The best parameters are {per_search.best_params_}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aSJAF4roKRND",
        "outputId": "aba9685e-aefc-47ff-ec4b-a8ce9e46cdd7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9075342465753424"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ],
      "source": [
        "accuracy_score(per_search.predict(X_test), y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dg2IU5AIudnl"
      },
      "source": [
        "### 4.5 LogisticRegression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "YZnESrRJvfLt"
      },
      "outputs": [],
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "#  pipeline\n",
        "log_full_pipeline = make_pipeline(full_preprocessing,\n",
        "                                  StandardScaler(),\n",
        "                                  LogisticRegression())\n",
        "\n",
        "# Parameter grid\n",
        "log_param_grid = {\n",
        "    \"columntransformer__num_pipe__simpleimputer__strategy\": [\"mean\", \"median\"],\n",
        "    \"logisticregression__C\": uniform(0.1, 1.0).rvs(10),\n",
        "    \"logisticregression__penalty\": ['l1', 'l2'],\n",
        "    \"logisticregression__solver\": ['liblinear', 'saga'],\n",
        "    \"logisticregression__max_iter\": [100, 200, 500, 1000],\n",
        "}\n",
        "\n",
        "# GridSearchCV\n",
        "log_search = GridSearchCV(log_full_pipeline,\n",
        "                          log_param_grid,\n",
        "                          cv=5,\n",
        "                          verbose=1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tAAJvfPlzb_3",
        "outputId": "dafedd15-bddf-4f88-d213-5dcfe4c8e070"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 320 candidates, totalling 1600 fits\n",
            "The best average score in cross validation was 0.9494956164484061\n"
          ]
        }
      ],
      "source": [
        "log_search.fit(X_train, y_train)\n",
        "\n",
        "print(f\"The best average score in cross validation was {log_search.best_score_}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KzX7fNjdznUl",
        "outputId": "2bd813e0-d479-46e9-ac18-78b46506001a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.934931506849315"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ],
      "source": [
        "accuracy_score(log_search.predict(X_test), y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 4.6 Best Parameters"
      ],
      "metadata": {
        "id": "2uIJkLOs8tLb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "parameters = {\"xgb\" : xgb_search.best_params_,\n",
        "              \"lgbm\": lgb_search.best_params_,\n",
        "              \"ext\": ext_search.best_params_,\n",
        "              \"per\": per_search.best_params_,\n",
        "              \"log\": log_search.best_params_}\n"
      ],
      "metadata": {
        "id": "MYu2eJux8ve2"
      },
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "parameters"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eKyDwP_YvO_4",
        "outputId": "12f440c7-d03e-4874-ba1a-6b118fee375a"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'xgb': {'columntransformer__num_pipe__simpleimputer__strategy': 'median',\n",
              "  'xgbclassifier__learning_rate': 0.1,\n",
              "  'xgbclassifier__max_depth': 5,\n",
              "  'xgbclassifier__min_child_weight': 1,\n",
              "  'xgbclassifier__n_estimators': 100},\n",
              " 'lgbm': {'lgbmclassifier__subsample': 0.8,\n",
              "  'lgbmclassifier__reg_lambda': 0.1,\n",
              "  'lgbmclassifier__reg_alpha': 0.1,\n",
              "  'lgbmclassifier__n_estimators': 50,\n",
              "  'lgbmclassifier__min_child_samples': 20,\n",
              "  'lgbmclassifier__max_depth': 5,\n",
              "  'lgbmclassifier__learning_rate': 0.1,\n",
              "  'lgbmclassifier__importance_type': 'split',\n",
              "  'lgbmclassifier__colsample_bytree': 0.8,\n",
              "  'lgbmclassifier__boosting_type': 'gbdt',\n",
              "  'lgbmclassifier__bagging_freq': 10,\n",
              "  'lgbmclassifier__bagging_fraction': 0.8,\n",
              "  'columntransformer__num_pipe__simpleimputer__strategy': 'median'},\n",
              " 'ext': {'columntransformer__num_pipe__simpleimputer__strategy': 'mean',\n",
              "  'extratreesclassifier__max_depth': 12,\n",
              "  'extratreesclassifier__n_estimators': 54},\n",
              " 'per': {'perceptron__alpha': 0.048253501600573095,\n",
              "  'perceptron__eta0': 0.7403442286745863,\n",
              "  'perceptron__max_iter': 10,\n",
              "  'perceptron__tol': 0.010883503986712461},\n",
              " 'log': {'columntransformer__num_pipe__simpleimputer__strategy': 'mean',\n",
              "  'logisticregression__C': 0.7009413280452148,\n",
              "  'logisticregression__max_iter': 100,\n",
              "  'logisticregression__penalty': 'l1',\n",
              "  'logisticregression__solver': 'liblinear'}}"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import VotingClassifier\n",
        "from xgboost import XGBClassifier\n",
        "from lightgbm import LGBMClassifier\n",
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "from sklearn.linear_model import Perceptron\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "# Initialize your models with the best parameters\n",
        "xgb_model = XGBClassifier(learning_rate=0.1, max_depth=5, min_child_weight=1, n_estimators=100)\n",
        "lgbm_model = LGBMClassifier(subsample = 0.8,reg_lambda = 0.1,reg_alpha= 0.1,n_estimators= 50,min_child_samples= 20, max_depth= 5, learning_rate= 0.1, importance_type = 'split', colsample_bytree= 0.8,boosting_type= 'gbdt',bagging_freq= 10, bagging_fraction= 0.8,)\n",
        "extra_model = ExtraTreesClassifier(n_estimators=54, max_depth=12, random_state=42)\n",
        "per_model = Perceptron(alpha=0.0482535, eta0 = 0.7403442286745863, max_iter = 10, tol = 0.010883503986712461 )\n",
        "log_model = LogisticRegression(C = 0.7009413280452148, max_iter = 100, penalty = \"l1\", solver =\"liblinear\" )\n",
        "\n",
        "# Create a dictionary of your models\n",
        "models = {\n",
        "    'xgb': xgb_model,\n",
        "    'lgbm': lgbm_model,\n",
        "    'ext': extra_model,\n",
        "    'per': per_model,\n",
        "    'log': log_model\n",
        "}\n",
        "\n",
        "voting_classifier = make_pipeline(\n",
        "    full_preprocessing,\n",
        "    VotingClassifier(\n",
        "        estimators=[(name, model) for name, model in models.items()],\n",
        "        voting='hard'\n",
        "    )\n",
        ")\n",
        "\n",
        "# Fit the ensemble model on your training data\n",
        "voting_classifier.fit(X_train, y_train)\n",
        "\n",
        "# Predict using the ensemble model\n",
        "voting_predictions = voting_classifier.predict(X_test)\n",
        "\n",
        "# Calculate accuracy\n",
        "final_accuracy = accuracy_score(y_test, voting_predictions)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "przv-mKZBdra",
        "outputId": "6aeb567d-338b-4f34-bdb8-2026e0d6bb52"
      },
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[LightGBM] [Warning] bagging_fraction is set=0.8, subsample=0.8 will be ignored. Current value: bagging_fraction=0.8\n",
            "[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10\n",
            "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
            "[LightGBM] [Warning] bagging_fraction is set=0.8, subsample=0.8 will be ignored. Current value: bagging_fraction=0.8\n",
            "[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10\n",
            "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
            "[LightGBM] [Info] Number of positive: 169, number of negative: 999\n",
            "[LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001536 seconds.\n",
            "You can set `force_row_wise=true` to remove the overhead.\n",
            "And if memory is not enough, you can set `force_col_wise=true`.\n",
            "[LightGBM] [Info] Total Bins 3240\n",
            "[LightGBM] [Info] Number of data points in the train set: 1168, number of used features: 171\n",
            "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.144692 -> initscore=-1.776856\n",
            "[LightGBM] [Info] Start training from score -1.776856\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[LightGBM] [Warning] bagging_fraction is set=0.8, subsample=0.8 will be ignored. Current value: bagging_fraction=0.8\n",
            "[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10\n",
            "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "final_accuracy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cr7PHMqJ0W50",
        "outputId": "4bd2a140-ed91-41c1-8ae4-bc2005f3dadc"
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.952054794520548"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}