cv_load_autogluon.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype = {\n",
    "    'Usage': 'category',\n",
    "    'Description': 'category',\n",
    "    'status': 'category',\n",
    "}\n",
    "df = pd.read_csv('data/reservations.csv.gz', dtype=dtype, parse_dates=['created', 'arrival', 'departure'])\n",
    "df.drop(columns='Usage', inplace=True)\n",
    "\n",
    "df.loc[df['cancel_date'] == '0001-01-01T00:00:00', ['cancel_date']] = None\n",
    "df['cancel_date'] = pd.to_datetime(df['cancel_date'])\n",
    "\n",
    "df['arrival_year'] = df['arrival'].dt.year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>owner_id</th>\n",
       "      <th>contract_id</th>\n",
       "      <th>resort_id</th>\n",
       "      <th>Description</th>\n",
       "      <th>cancel_date</th>\n",
       "      <th>created</th>\n",
       "      <th>arrival</th>\n",
       "      <th>departure</th>\n",
       "      <th>status</th>\n",
       "      <th>arrival_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>28</td>\n",
       "      <td>Fixed-Annual</td>\n",
       "      <td>2017-12-19</td>\n",
       "      <td>2013-07-09</td>\n",
       "      <td>2015-10-17</td>\n",
       "      <td>2015-10-24</td>\n",
       "      <td>active</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>28</td>\n",
       "      <td>Fixed-Annual</td>\n",
       "      <td>2017-12-19</td>\n",
       "      <td>2014-12-20</td>\n",
       "      <td>2016-10-15</td>\n",
       "      <td>2016-10-22</td>\n",
       "      <td>active</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>28</td>\n",
       "      <td>Fixed-Annual</td>\n",
       "      <td>2017-12-19</td>\n",
       "      <td>2015-11-17</td>\n",
       "      <td>2017-10-21</td>\n",
       "      <td>2017-10-28</td>\n",
       "      <td>active</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>28</td>\n",
       "      <td>Fixed-Annual</td>\n",
       "      <td>NaT</td>\n",
       "      <td>2013-07-09</td>\n",
       "      <td>2015-10-24</td>\n",
       "      <td>2015-10-31</td>\n",
       "      <td>active</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>28</td>\n",
       "      <td>Fixed-Annual</td>\n",
       "      <td>NaT</td>\n",
       "      <td>2014-12-20</td>\n",
       "      <td>2016-10-22</td>\n",
       "      <td>2016-10-29</td>\n",
       "      <td>active</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   owner_id  contract_id  resort_id   Description cancel_date    created  \\\n",
       "0        10           10         28  Fixed-Annual  2017-12-19 2013-07-09   \n",
       "1        10           10         28  Fixed-Annual  2017-12-19 2014-12-20   \n",
       "2        10           10         28  Fixed-Annual  2017-12-19 2015-11-17   \n",
       "3       100          100         28  Fixed-Annual         NaT 2013-07-09   \n",
       "4       100          100         28  Fixed-Annual         NaT 2014-12-20   \n",
       "\n",
       "     arrival  departure  status  arrival_year  \n",
       "0 2015-10-17 2015-10-24  active          2015  \n",
       "1 2016-10-15 2016-10-22  active          2016  \n",
       "2 2017-10-21 2017-10-28  active          2017  \n",
       "3 2015-10-24 2015-10-31  active          2015  \n",
       "4 2016-10-22 2016-10-29  active          2016  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "active       215049\n",
       "cancelled    118973\n",
       "no-show       53850\n",
       "Name: status, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.status.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "appearances = {}\n",
    "for contract_id in df['contract_id'].unique():\n",
    "    subset_df = df.loc[df['contract_id'] == contract_id].sort_values(by=['arrival', 'created'])\n",
    "    # save the last known state\n",
    "    # fill in blanks for bad years\n",
    "    # handle case where cancel year might come after a series of misses\n",
    "    yearly_state = {arrival_year: status for (created, arrival_year, status) in subset_df[['created', 'arrival_year', 'status']].itertuples(index=False, name=None)}\n",
    "    earliest = subset_df['arrival_year'].min()\n",
    "    latest = min(subset_df['arrival_year'].max(), 2019)\n",
    "    activity = [(year, yearly_state.get(year, 'no-show')) for year in range(earliest, latest + 1)]    \n",
    "    if activity:\n",
    "        resort_id = subset_df['resort_id'].values[0]\n",
    "        appearances[str(contract_id)] = [resort_id] + activity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each unique contract's status in every year."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = []\n",
    "for r in list(appearances.values()):\n",
    "    resort_id, activity = r[0], r[1:]\n",
    "    row = [None] * 5\n",
    "    row[-len(activity):] = [s for year, s in activity]\n",
    "    rows.append([resort_id] + row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[28, None, None, 'active', 'active', 'active'],\n",
       " [28, 'active', 'active', 'active', 'active', 'active'],\n",
       " [30, 'active', 'active', 'active', 'cancelled', 'cancelled'],\n",
       " [39, 'no-show', 'active', 'active', 'active', 'active'],\n",
       " [38, 'cancelled', 'active', 'cancelled', 'active', 'active'],\n",
       " [38, 'active', 'cancelled', 'active', 'active', 'active'],\n",
       " [39, 'cancelled', 'cancelled', 'cancelled', 'cancelled', 'cancelled'],\n",
       " [39, 'active', 'cancelled', 'cancelled', 'cancelled', 'no-show'],\n",
       " [39, 'cancelled', 'active', 'active', 'active', 'active'],\n",
       " [39, 'active', 'active', 'active', 'active', 'active']]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each resort's status in each year, within each unique contract."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(rows, columns=['resort_id', 'year_2015', 'year_2016', 'year_2017', 'year_2018', 'year_2019'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "resort_id    0.000000\n",
       "year_2015    0.515846\n",
       "year_2016    0.205276\n",
       "year_2017    0.098678\n",
       "year_2018    0.048311\n",
       "year_2019    0.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()/df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "active       38655\n",
       "cancelled    18509\n",
       "no-show      11909\n",
       "Name: year_2019, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['year_2019'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>resort_id</th>\n",
       "      <th>year_2015</th>\n",
       "      <th>year_2016</th>\n",
       "      <th>year_2017</th>\n",
       "      <th>year_2018</th>\n",
       "      <th>year_2019</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "      <td>cancelled</td>\n",
       "      <td>cancelled</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>39</td>\n",
       "      <td>no-show</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>38</td>\n",
       "      <td>cancelled</td>\n",
       "      <td>active</td>\n",
       "      <td>cancelled</td>\n",
       "      <td>active</td>\n",
       "      <td>active</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   resort_id  year_2015 year_2016  year_2017  year_2018  year_2019\n",
       "0         28       None      None     active     active     active\n",
       "1         28     active    active     active     active     active\n",
       "2         30     active    active     active  cancelled  cancelled\n",
       "3         39    no-show    active     active     active     active\n",
       "4         38  cancelled    active  cancelled     active     active"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('resort_id', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_data, test_data = train_test_split(df, test_size=0.30, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.to_csv('data/capital_train.csv', encoding='utf-8', index=False)\n",
    "test_data.to_csv('data/capital_test.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "active       27073\n",
       "cancelled    12947\n",
       "no-show       8331\n",
       "Name: year_2019, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.year_2019.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "active       11582\n",
       "cancelled     5562\n",
       "no-show       3578\n",
       "Name: year_2019, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.year_2019.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary of class variable: \n",
      " count      48351\n",
      "unique         3\n",
      "top       active\n",
      "freq       27073\n",
      "Name: year_2019, dtype: object\n"
     ]
    }
   ],
   "source": [
    "label_column = 'year_2019'\n",
    "print(\"Summary of class variable: \\n\", train_data[label_column].describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No output_directory specified. Models will be saved in: AutogluonModels/ag-20210330_072731/\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to AutogluonModels/ag-20210330_072731/\n",
      "AutoGluon Version:  0.0.15\n",
      "Train Data Rows:    48351\n",
      "Train Data Columns: 4\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'multiclass' (because dtype of label-column == object).\n",
      "\t3 unique label values:  ['cancelled', 'active', 'no-show']\n",
      "\tIf 'multiclass' is not the correct problem_type, please manually specify the problem_type argument in fit() (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Train Data Class Count: 3\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    6212.4 MB\n",
      "\tTrain Data (Original)  Memory Usage: 11.06 MB (0.2% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting CategoryFeatureGenerator...\n",
      "\t\t\tFitting CategoryMemoryMinimizeFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('object', []) : 4 | ['year_2015', 'year_2016', 'year_2017', 'year_2018']\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('category', []) : 4 | ['year_2015', 'year_2016', 'year_2017', 'year_2018']\n",
      "\t0.1s = Fit runtime\n",
      "\t4 features in original data used to generate 4 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.58 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.15s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric argument of fit()\n",
      "AutoGluon will early stop models using evaluation metric: 'accuracy'\n",
      "Fitting model: NeuralNetClassifier ...\n",
      "\t0.6736\t = Validation accuracy score\n",
      "\t60.23s\t = Training runtime\n",
      "\t0.11s\t = Validation runtime\n",
      "Fitting model: KNeighborsClassifierUnif ...\n",
      "\tNo valid features to train KNeighborsClassifierUnif... Skipping this model.\n",
      "Fitting model: KNeighborsClassifierDist ...\n",
      "\tNo valid features to train KNeighborsClassifierDist... Skipping this model.\n",
      "Fitting model: RandomForestClassifierGini ...\n",
      "\t0.6732\t = Validation accuracy score\n",
      "\t1.24s\t = Training runtime\n",
      "\t0.21s\t = Validation runtime\n",
      "Fitting model: RandomForestClassifierEntr ...\n",
      "\t0.6732\t = Validation accuracy score\n",
      "\t1.23s\t = Training runtime\n",
      "\t0.21s\t = Validation runtime\n",
      "Fitting model: ExtraTreesClassifierGini ...\n",
      "\t0.6744\t = Validation accuracy score\n",
      "\t1.14s\t = Training runtime\n",
      "\t0.21s\t = Validation runtime\n",
      "Fitting model: ExtraTreesClassifierEntr ...\n",
      "\t0.6744\t = Validation accuracy score\n",
      "\t1.03s\t = Training runtime\n",
      "\t0.21s\t = Validation runtime\n",
      "Fitting model: LightGBMClassifier ...\n",
      "\t0.6744\t = Validation accuracy score\n",
      "\t0.81s\t = Training runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: LightGBMClassifierXT ...\n",
      "\t0.6728\t = Validation accuracy score\n",
      "\t0.59s\t = Training runtime\n",
      "\t0.01s\t = Validation runtime\n",
      "Fitting model: CatboostClassifier ...\n",
      "\t0.6768\t = Validation accuracy score\n",
      "\t3.65s\t = Training runtime\n",
      "\t0.01s\t = Validation runtime\n",
      "Fitting model: LightGBMClassifierCustom ...\n",
      "\t0.674\t = Validation accuracy score\n",
      "\t1.44s\t = Training runtime\n",
      "\t0.04s\t = Validation runtime\n",
      "Fitting model: weighted_ensemble_k0_l1 ...\n",
      "\t0.6768\t = Validation accuracy score\n",
      "\t0.52s\t = Training runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 74.38s ...\n"
     ]
    }
   ],
   "source": [
    "import autogluon as ag\n",
    "from autogluon import TabularPrediction as task\n",
    "\n",
    "predictor = task.fit(train_data=train_data, label=label_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = test_data[label_column]\n",
    "test_data_nolab = test_data.drop(labels=[label_column],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.6706881575137535\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.6706881575137535,\n",
      "    \"accuracy_score\": 0.6706881575137535,\n",
      "    \"balanced_accuracy_score\": 0.5651247356842359,\n",
      "    \"matthews_corrcoef\": 0.4102990853305558\n",
      "}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions:   ['active' 'cancelled' 'active' ... 'active' 'active' 'active']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Detailed (per-class) classification report:\n",
      "{\n",
      "    \"active\": {\n",
      "        \"precision\": 0.7244427934621099,\n",
      "        \"recall\": 0.8419098601277845,\n",
      "        \"f1-score\": 0.7787716636051434,\n",
      "        \"support\": 11582\n",
      "    },\n",
      "    \"cancelled\": {\n",
      "        \"precision\": 0.5970003895597974,\n",
      "        \"recall\": 0.5510607695073715,\n",
      "        \"f1-score\": 0.5731114435302916,\n",
      "        \"support\": 5562\n",
      "    },\n",
      "    \"no-show\": {\n",
      "        \"precision\": 0.5084586466165414,\n",
      "        \"recall\": 0.3024035774175517,\n",
      "        \"f1-score\": 0.37924991237294076,\n",
      "        \"support\": 3578\n",
      "    },\n",
      "    \"accuracy\": 0.6706881575137535,\n",
      "    \"macro avg\": {\n",
      "        \"precision\": 0.6099672765461496,\n",
      "        \"recall\": 0.5651247356842359,\n",
      "        \"f1-score\": 0.5770443398361252,\n",
      "        \"support\": 20722\n",
      "    },\n",
      "    \"weighted avg\": {\n",
      "        \"precision\": 0.6529426521669596,\n",
      "        \"recall\": 0.6706881575137535,\n",
      "        \"f1-score\": 0.6545862099826578,\n",
      "        \"support\": 20722\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "y_pred = predictor.predict(test_data_nolab)\n",
    "print(\"Predictions:  \", y_pred)\n",
    "perf = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted probabilities of class 'cancelled': [[0.7598184  0.17107713 0.06910452]\n",
      " [0.15061373 0.75407445 0.09531184]\n",
      " [0.87844694 0.07554352 0.04600956]\n",
      " ...\n",
      " [0.82452995 0.08081903 0.094651  ]\n",
      " [0.82452995 0.08081903 0.094651  ]\n",
      " [0.52298206 0.36374778 0.11327014]]\n"
     ]
    }
   ],
   "source": [
    "pred_probs = predictor.predict_proba(test_data_nolab)\n",
    "positive_class = [label for label in predictor.class_labels if predictor.class_labels_internal_map[label]==1][0]\n",
    "print(f\"Predicted probabilities of class '{positive_class}':\", pred_probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Summary of fit() ***\n",
      "Estimated performance of each model:\n",
      "                        model  score_val  pred_time_val   fit_time  pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  fit_order\n",
      "0          CatboostClassifier     0.6768       0.010369   3.654064                0.010369           3.654064            0       True          8\n",
      "1     weighted_ensemble_k0_l1     0.6768       0.011087   4.173240                0.000718           0.519176            1       True         10\n",
      "2          LightGBMClassifier     0.6744       0.018447   0.809488                0.018447           0.809488            0       True          6\n",
      "3    ExtraTreesClassifierGini     0.6744       0.206662   1.135898                0.206662           1.135898            0       True          4\n",
      "4    ExtraTreesClassifierEntr     0.6744       0.207086   1.032413                0.207086           1.032413            0       True          5\n",
      "5    LightGBMClassifierCustom     0.6740       0.040228   1.442169                0.040228           1.442169            0       True          9\n",
      "6         NeuralNetClassifier     0.6736       0.114016  60.229692                0.114016          60.229692            0       True          1\n",
      "7  RandomForestClassifierEntr     0.6732       0.207045   1.234125                0.207045           1.234125            0       True          3\n",
      "8  RandomForestClassifierGini     0.6732       0.207224   1.236910                0.207224           1.236910            0       True          2\n",
      "9        LightGBMClassifierXT     0.6728       0.011456   0.591041                0.011456           0.591041            0       True          7\n",
      "Number of models trained: 10\n",
      "Types of models trained:\n",
      "{'XTModel', 'LGBModel', 'RFModel', 'CatboostModel', 'WeightedEnsembleModel', 'TabularNeuralNetModel'}\n",
      "Bagging used: False \n",
      "Stack-ensembling used: False \n",
      "Hyperparameter-tuning used: False \n",
      "User-specified hyperparameters:\n",
      "{'default': {'NN': [{}], 'GBM': [{}, {'extra_trees': True, 'AG_args': {'name_suffix': 'XT'}}], 'CAT': [{}], 'RF': [{'criterion': 'gini', 'AG_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'AG_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}], 'XT': [{'criterion': 'gini', 'AG_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'AG_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}], 'KNN': [{'weights': 'uniform', 'AG_args': {'name_suffix': 'Unif'}}, {'weights': 'distance', 'AG_args': {'name_suffix': 'Dist'}}], 'custom': [{'num_boost_round': 10000, 'num_threads': -1, 'objective': 'multiclass', 'num_classes': 3, 'verbose': -1, 'boosting_type': 'gbdt', 'learning_rate': 0.03, 'num_leaves': 128, 'feature_fraction': 0.9, 'min_data_in_leaf': 3, 'two_round': True, 'seed_value': 0, 'AG_args': {'model_type': 'GBM', 'name_suffix': 'Custom', 'disable_in_hpo': True}}]}}\n",
      "Feature Metadata (Processed):\n",
      "(raw dtype, special dtypes):\n",
      "('category', []) : 4 | ['year_2015', 'year_2016', 'year_2017', 'year_2018']\n",
      "Plot summary of models saved to file: AutogluonModels/ag-20210330_072731/SummaryOfModels.html\n",
      "*** End of fit() summary ***\n"
     ]
    }
   ],
   "source": [
    "results = predictor.fit_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoGluon infers problem type is:  multiclass\n",
      "AutoGluon identified the following types of features:\n",
      "('category', []) : 4 | ['year_2015', 'year_2016', 'year_2017', 'year_2018']\n"
     ]
    }
   ],
   "source": [
    "print(\"AutoGluon infers problem type is: \", predictor.problem_type)\n",
    "print(\"AutoGluon identified the following types of features:\")\n",
    "print(predictor.feature_metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CatboostClassifier</td>\n",
       "      <td>0.670688</td>\n",
       "      <td>0.6768</td>\n",
       "      <td>0.032980</td>\n",
       "      <td>0.010369</td>\n",
       "      <td>3.654064</td>\n",
       "      <td>0.032980</td>\n",
       "      <td>0.010369</td>\n",
       "      <td>3.654064</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>weighted_ensemble_k0_l1</td>\n",
       "      <td>0.670688</td>\n",
       "      <td>0.6768</td>\n",
       "      <td>0.035819</td>\n",
       "      <td>0.011087</td>\n",
       "      <td>4.173240</td>\n",
       "      <td>0.002840</td>\n",
       "      <td>0.000718</td>\n",
       "      <td>0.519176</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LightGBMClassifier</td>\n",
       "      <td>0.670061</td>\n",
       "      <td>0.6744</td>\n",
       "      <td>0.107309</td>\n",
       "      <td>0.018447</td>\n",
       "      <td>0.809488</td>\n",
       "      <td>0.107309</td>\n",
       "      <td>0.018447</td>\n",
       "      <td>0.809488</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesClassifierGini</td>\n",
       "      <td>0.669578</td>\n",
       "      <td>0.6744</td>\n",
       "      <td>0.221292</td>\n",
       "      <td>0.206662</td>\n",
       "      <td>1.135898</td>\n",
       "      <td>0.221292</td>\n",
       "      <td>0.206662</td>\n",
       "      <td>1.135898</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesClassifierEntr</td>\n",
       "      <td>0.669578</td>\n",
       "      <td>0.6744</td>\n",
       "      <td>0.224360</td>\n",
       "      <td>0.207086</td>\n",
       "      <td>1.032413</td>\n",
       "      <td>0.224360</td>\n",
       "      <td>0.207086</td>\n",
       "      <td>1.032413</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>LightGBMClassifierCustom</td>\n",
       "      <td>0.669385</td>\n",
       "      <td>0.6740</td>\n",
       "      <td>0.308849</td>\n",
       "      <td>0.040228</td>\n",
       "      <td>1.442169</td>\n",
       "      <td>0.308849</td>\n",
       "      <td>0.040228</td>\n",
       "      <td>1.442169</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>RandomForestClassifierEntr</td>\n",
       "      <td>0.669289</td>\n",
       "      <td>0.6732</td>\n",
       "      <td>0.218579</td>\n",
       "      <td>0.207045</td>\n",
       "      <td>1.234125</td>\n",
       "      <td>0.218579</td>\n",
       "      <td>0.207045</td>\n",
       "      <td>1.234125</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>RandomForestClassifierGini</td>\n",
       "      <td>0.669289</td>\n",
       "      <td>0.6732</td>\n",
       "      <td>0.224014</td>\n",
       "      <td>0.207224</td>\n",
       "      <td>1.236910</td>\n",
       "      <td>0.224014</td>\n",
       "      <td>0.207224</td>\n",
       "      <td>1.236910</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NeuralNetClassifier</td>\n",
       "      <td>0.669240</td>\n",
       "      <td>0.6736</td>\n",
       "      <td>1.106056</td>\n",
       "      <td>0.114016</td>\n",
       "      <td>60.229692</td>\n",
       "      <td>1.106056</td>\n",
       "      <td>0.114016</td>\n",
       "      <td>60.229692</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LightGBMClassifierXT</td>\n",
       "      <td>0.667551</td>\n",
       "      <td>0.6728</td>\n",
       "      <td>0.039465</td>\n",
       "      <td>0.011456</td>\n",
       "      <td>0.591041</td>\n",
       "      <td>0.039465</td>\n",
       "      <td>0.011456</td>\n",
       "      <td>0.591041</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        model  score_test  score_val  pred_time_test  \\\n",
       "0          CatboostClassifier    0.670688     0.6768        0.032980   \n",
       "1     weighted_ensemble_k0_l1    0.670688     0.6768        0.035819   \n",
       "2          LightGBMClassifier    0.670061     0.6744        0.107309   \n",
       "3    ExtraTreesClassifierGini    0.669578     0.6744        0.221292   \n",
       "4    ExtraTreesClassifierEntr    0.669578     0.6744        0.224360   \n",
       "5    LightGBMClassifierCustom    0.669385     0.6740        0.308849   \n",
       "6  RandomForestClassifierEntr    0.669289     0.6732        0.218579   \n",
       "7  RandomForestClassifierGini    0.669289     0.6732        0.224014   \n",
       "8         NeuralNetClassifier    0.669240     0.6736        1.106056   \n",
       "9        LightGBMClassifierXT    0.667551     0.6728        0.039465   \n",
       "\n",
       "   pred_time_val   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0       0.010369   3.654064                 0.032980                0.010369   \n",
       "1       0.011087   4.173240                 0.002840                0.000718   \n",
       "2       0.018447   0.809488                 0.107309                0.018447   \n",
       "3       0.206662   1.135898                 0.221292                0.206662   \n",
       "4       0.207086   1.032413                 0.224360                0.207086   \n",
       "5       0.040228   1.442169                 0.308849                0.040228   \n",
       "6       0.207045   1.234125                 0.218579                0.207045   \n",
       "7       0.207224   1.236910                 0.224014                0.207224   \n",
       "8       0.114016  60.229692                 1.106056                0.114016   \n",
       "9       0.011456   0.591041                 0.039465                0.011456   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           3.654064            0       True          8  \n",
       "1           0.519176            1       True         10  \n",
       "2           0.809488            0       True          6  \n",
       "3           1.135898            0       True          4  \n",
       "4           1.032413            0       True          5  \n",
       "5           1.442169            0       True          9  \n",
       "6           1.234125            0       True          3  \n",
       "7           1.236910            0       True          2  \n",
       "8          60.229692            0       True          1  \n",
       "9           0.591041            0       True          7  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor.leaderboard(test_data, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Computing raw permutation importance for 4 features on weighted_ensemble_k0_l1 ...\n",
      "\t0.12s\t= Expected runtime\n",
      "\t0.12s\t= Actual runtime\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "year_2018    0.125\n",
       "year_2017    0.040\n",
       "year_2016    0.018\n",
       "year_2015    0.014\n",
       "dtype: float64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor.feature_importance(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}