otto-model-orders.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "41dc6691",
   "metadata": {
    "papermill": {
     "duration": 0.005805,
     "end_time": "2023-03-23T11:15:52.268428",
     "exception": false,
     "start_time": "2023-03-23T11:15:52.262623",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Orders Model\n",
    "A model predicting which aids user is going to order is trained in this notebook. This same notebook was also used for cross-validating the orders model. Unlike the clicks model, for orders, prediction is made in a separate notebook. On kaggle platform, notebooks with GPU have less memory available, and it was hard to fit all the required data into 13 GB of available RAM, so I had to move prediction to a different notebook without GPU support, but with 30Gb RAM available.\n",
    "For orders, the prediction is made using two different models, LGBM model and  catboost model, trained on two cross-validation datasets.\n",
    "This notebook uses input from two \"parallel\" notebooks that produce w2vec features for orders, one for one of cross-validation sets and half of the test set and the other one for another cross-validation set the other half of the test set.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cdeb933f",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-23T11:15:52.279132Z",
     "iopub.status.busy": "2023-03-23T11:15:52.278659Z",
     "iopub.status.idle": "2023-03-23T11:15:55.368077Z",
     "shell.execute_reply": "2023-03-23T11:15:55.367140Z"
    },
    "papermill": {
     "duration": 3.097639,
     "end_time": "2023-03-23T11:15:55.370562",
     "exception": false,
     "start_time": "2023-03-23T11:15:52.272923",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "from lightgbm.sklearn import LGBMRanker\n",
    "from catboost import CatBoostRanker, Pool\n",
    "from sklearn.model_selection import GroupKFold\n",
    "import joblib\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d9c337ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:15:55.382709Z",
     "iopub.status.busy": "2023-03-23T11:15:55.381297Z",
     "iopub.status.idle": "2023-03-23T11:15:55.386705Z",
     "shell.execute_reply": "2023-03-23T11:15:55.385929Z"
    },
    "papermill": {
     "duration": 0.012731,
     "end_time": "2023-03-23T11:15:55.388507",
     "exception": false,
     "start_time": "2023-03-23T11:15:55.375776",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function was used to test new features before adding them to the pipeline.\n",
    "# Now it only deletes the day_of_week column, which is used to construct some features.\n",
    "def prepare_df(df):\n",
    "    del df['day_of_week']\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3514605f",
   "metadata": {
    "papermill": {
     "duration": 0.004216,
     "end_time": "2023-03-23T11:15:55.397348",
     "exception": false,
     "start_time": "2023-03-23T11:15:55.393132",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Load and prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8a59bcd8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:15:55.407709Z",
     "iopub.status.busy": "2023-03-23T11:15:55.406960Z",
     "iopub.status.idle": "2023-03-23T11:16:04.398858Z",
     "shell.execute_reply": "2023-03-23T11:16:04.397914Z"
    },
    "papermill": {
     "duration": 8.999653,
     "end_time": "2023-03-23T11:16:04.401409",
     "exception": false,
     "start_time": "2023-03-23T11:15:55.401756",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load the train/cross-validation data.\n",
    "\n",
    "df_train = pd.read_parquet('/kaggle/input/otto-orders-w2vec/train_features_with_w2v_cv1.parquet')\n",
    "#df_train = pd.read_parquet('/kaggle/input/otto-orders-w2vec-part1/train_features_with_w2v_cv2.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ff06d9a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:04.412064Z",
     "iopub.status.busy": "2023-03-23T11:16:04.411760Z",
     "iopub.status.idle": "2023-03-23T11:16:07.230122Z",
     "shell.execute_reply": "2023-03-23T11:16:07.228629Z"
    },
    "papermill": {
     "duration": 2.826079,
     "end_time": "2023-03-23T11:16:07.232318",
     "exception": false,
     "start_time": "2023-03-23T11:16:04.406239",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0 GB\n"
     ]
    }
   ],
   "source": [
    "# A few checks and preparations.\n",
    "df_train = prepare_df(df_train)\n",
    "\n",
    "assert len(df_train[df_train.duplicated(subset=['session','order_predictions'], keep=False)]) == 0\n",
    "\n",
    "size = df_train.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b7e9c1b9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.243192Z",
     "iopub.status.busy": "2023-03-23T11:16:07.242513Z",
     "iopub.status.idle": "2023-03-23T11:16:07.248597Z",
     "shell.execute_reply": "2023-03-23T11:16:07.247596Z"
    },
    "papermill": {
     "duration": 0.014985,
     "end_time": "2023-03-23T11:16:07.251964",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.236979",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_defined\n"
     ]
    }
   ],
   "source": [
    "# Set the LGBM model's parameters.\n",
    "parameters = {\n",
    "    \"objective\" : \"lambdarank\",\n",
    "    \"metric\" : \"ndcg\",\n",
    "    \"boosting_type\" : \"gbdt\",\n",
    "    'min_child_samples' : 100,\n",
    "    \"n_estimators\" : 299,\n",
    "    \"num_leaves\" : 128,\n",
    "    \"importance_type\" : 'gain',\n",
    "    'max_depth' : 8,\n",
    "    'learning_rate' : 0.07,\n",
    "    'device': 'gpu',\n",
    "    'gpu_platform_id': 0,\n",
    "    'gpu_device_id': 0,    \n",
    "}\n",
    "model = LGBMRanker(**parameters)\n",
    "\n",
    "print('model_defined')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8f86cf93",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.262089Z",
     "iopub.status.busy": "2023-03-23T11:16:07.261813Z",
     "iopub.status.idle": "2023-03-23T11:16:07.270240Z",
     "shell.execute_reply": "2023-03-23T11:16:07.268867Z"
    },
    "papermill": {
     "duration": 0.016602,
     "end_time": "2023-03-23T11:16:07.273054",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.256452",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_defined\n"
     ]
    }
   ],
   "source": [
    "# Set the catboost model's parameters.\n",
    "catboost_parameters = {\n",
    "    'iterations': 800,\n",
    "    'loss_function': 'QuerySoftMax',    \n",
    "    'learning_rate': 0.12,\n",
    "    'depth' : 8,\n",
    "    'verbose': 50,\n",
    "    'random_seed': 0, \n",
    "    'task_type' : 'GPU'\n",
    "}\n",
    "model_catboost = CatBoostRanker(**catboost_parameters)\n",
    "print('model_defined')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a51de62a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.285692Z",
     "iopub.status.busy": "2023-03-23T11:16:07.285432Z",
     "iopub.status.idle": "2023-03-23T11:16:07.289840Z",
     "shell.execute_reply": "2023-03-23T11:16:07.288945Z"
    },
    "papermill": {
     "duration": 0.011883,
     "end_time": "2023-03-23T11:16:07.291841",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.279958",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# A few global parameters, used both for creating submission and cross-validation.\n",
    "CROSS_VALIDATE = False # Should be changed to False to produce submission.\n",
    "frac = 0.4 #fracture of records with target==False to be dropped from train to reduce memory usage\n",
    "x_cols = list(df_train.columns[3:])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3ded0bd",
   "metadata": {
    "papermill": {
     "duration": 0.004449,
     "end_time": "2023-03-23T11:16:07.300864",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.296415",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d1fbec2c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.312322Z",
     "iopub.status.busy": "2023-03-23T11:16:07.310905Z",
     "iopub.status.idle": "2023-03-23T11:16:07.327355Z",
     "shell.execute_reply": "2023-03-23T11:16:07.326363Z"
    },
    "papermill": {
     "duration": 0.023908,
     "end_time": "2023-03-23T11:16:07.329419",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.305511",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3 µs, sys: 1 µs, total: 4 µs\n",
      "Wall time: 7.15 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Cell to cross-validate the LGBM model.\n",
    "\n",
    "if CROSS_VALIDATE:\n",
    "    # Define the splits and prepare a column to save results.\n",
    "    n_splits = 4\n",
    "    groups_by_session = df_train['session'].copy().tolist()\n",
    "    group_kfold = GroupKFold(n_splits=n_splits)    \n",
    "    df_importances = pd.DataFrame({'columns':x_cols})\n",
    "    df_train['cv_prediction'] = -1\n",
    "    df_train['cv_prediction'] = df_train['cv_prediction'].astype(np.float32)\n",
    "    \n",
    "    # Fit the model and save the results.\n",
    "    for i, (train_index, test_index) in enumerate(group_kfold.split(df_train[x_cols], df_train['target'], groups_by_session)):\n",
    "        train_index = otto_common.remove_frac(train_index, df_train, frac)\n",
    "        gc.collect()\n",
    "        print('start_fitting')\n",
    "        model.fit(\n",
    "            df_train[x_cols].iloc[train_index],\n",
    "            df_train.iloc[train_index, 2].astype(np.int8),\n",
    "            group=df_train.iloc[train_index].groupby('session').size(),\n",
    "        )\n",
    "        column_name = 'imp_' + str(i)\n",
    "        df_importances[column_name] = model.feature_importances_\n",
    "        df_train['cv_prediction'].iloc[test_index] = model.predict(df_train[x_cols].iloc[test_index])\n",
    "        gc.collect()\n",
    "    del groups_by_session, group_kfold, train_index, test_index\n",
    "    gc.collect()\n",
    "    df_importances['imp_avg'] = df_importances.mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "03d53a96",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.340046Z",
     "iopub.status.busy": "2023-03-23T11:16:07.339772Z",
     "iopub.status.idle": "2023-03-23T11:16:07.342989Z",
     "shell.execute_reply": "2023-03-23T11:16:07.342262Z"
    },
    "papermill": {
     "duration": 0.010559,
     "end_time": "2023-03-23T11:16:07.344862",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.334303",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# View feature_importances. Two cells were used to print feature importances so that it would be possible to compare the values between two runs.\n",
    "#df_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ea1d0d25",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.355261Z",
     "iopub.status.busy": "2023-03-23T11:16:07.354957Z",
     "iopub.status.idle": "2023-03-23T11:16:07.358398Z",
     "shell.execute_reply": "2023-03-23T11:16:07.357650Z"
    },
    "papermill": {
     "duration": 0.011054,
     "end_time": "2023-03-23T11:16:07.360541",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.349487",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "30701b95",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.377208Z",
     "iopub.status.busy": "2023-03-23T11:16:07.376956Z",
     "iopub.status.idle": "2023-03-23T11:16:07.380798Z",
     "shell.execute_reply": "2023-03-23T11:16:07.379945Z"
    },
    "papermill": {
     "duration": 0.012094,
     "end_time": "2023-03-23T11:16:07.382814",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.370720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Print the LGBM cross-validation results.\n",
    "if CROSS_VALIDATE:\n",
    "    otto_common.calculate_recall(df_train, 'cv_prediction', 311027)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0cf450ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.393035Z",
     "iopub.status.busy": "2023-03-23T11:16:07.392768Z",
     "iopub.status.idle": "2023-03-23T11:16:07.401106Z",
     "shell.execute_reply": "2023-03-23T11:16:07.399891Z"
    },
    "papermill": {
     "duration": 0.01554,
     "end_time": "2023-03-23T11:16:07.402927",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.387387",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2 µs, sys: 1 µs, total: 3 µs\n",
      "Wall time: 6.68 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Cell to cross-validate the catboost model.\n",
    "\n",
    "if CROSS_VALIDATE:\n",
    "    n_splits = 4\n",
    "    groups_by_session = df_train['session'].copy().tolist()\n",
    "    group_kfold = GroupKFold(n_splits=n_splits)    \n",
    "    df_importances = pd.DataFrame({'columns':x_cols})\n",
    "    df_train['catboost_prediction'] = -1\n",
    "    df_train['catboost_prediction'] = df_train['catboost_prediction'].astype(np.float32)\n",
    "    \n",
    "    # Fitting the model and saving the results.\n",
    "    for i, (train_index, test_index) in enumerate(group_kfold.split(df_train[x_cols], df_train['target'], groups_by_session)):\n",
    "        train_index = otto_common.remove_frac(train_index, df_train, frac)\n",
    "        train_pool = Pool(\n",
    "            data=df_train[x_cols].iloc[train_index],\n",
    "            label=df_train.iloc[train_index, 2].astype(np.int8),\n",
    "            group_id=df_train.iloc[train_index, 0]\n",
    "        )\n",
    "        cv_pool = Pool(\n",
    "            data=df_train[x_cols].iloc[test_index],\n",
    "            label=df_train.iloc[test_index, 2].astype(np.int8),\n",
    "            group_id=df_train.iloc[test_index, 0]\n",
    "        )\n",
    "        gc.collect()\n",
    "        print('start_fitting')\n",
    "        model_catboost.fit(train_pool, eval_set=cv_pool)\n",
    "\n",
    "        df_train['catboost_prediction'].iloc[test_index] = model_catboost.predict(cv_pool)\n",
    "        del train_pool, cv_pool\n",
    "        gc.collect()\n",
    "    del groups_by_session, group_kfold, train_index, test_index\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "60fb8b7f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.413807Z",
     "iopub.status.busy": "2023-03-23T11:16:07.413083Z",
     "iopub.status.idle": "2023-03-23T11:16:07.417484Z",
     "shell.execute_reply": "2023-03-23T11:16:07.416665Z"
    },
    "papermill": {
     "duration": 0.01185,
     "end_time": "2023-03-23T11:16:07.419411",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.407561",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Print the catboost cross-validation results.\n",
    "if CROSS_VALIDATE:\n",
    "    otto_common.calculate_recall(df_train, 'catboost_prediction', 311027)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "449599e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.435113Z",
     "iopub.status.busy": "2023-03-23T11:16:07.434280Z",
     "iopub.status.idle": "2023-03-23T11:16:07.438809Z",
     "shell.execute_reply": "2023-03-23T11:16:07.438012Z"
    },
    "papermill": {
     "duration": 0.016644,
     "end_time": "2023-03-23T11:16:07.440758",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.424114",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Save both LGBM and catboost cross-validation results to file.\n",
    "if CROSS_VALIDATE:\n",
    "    df_train.to_parquet('cv_predictions_orders.parquet')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ced6f8cd",
   "metadata": {
    "papermill": {
     "duration": 0.004532,
     "end_time": "2023-03-23T11:16:07.450092",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.445560",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Fit the test models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a7a5407f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:16:07.461008Z",
     "iopub.status.busy": "2023-03-23T11:16:07.460252Z",
     "iopub.status.idle": "2023-03-23T11:20:48.189777Z",
     "shell.execute_reply": "2023-03-23T11:20:48.188772Z"
    },
    "papermill": {
     "duration": 280.737915,
     "end_time": "2023-03-23T11:20:48.192683",
     "exception": false,
     "start_time": "2023-03-23T11:16:07.454768",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Fit the LGBM model.\n",
    "if not CROSS_VALIDATE:\n",
    "    if frac > 0:\n",
    "        remove_index = df_train.loc[df_train['target'] == False].sample(frac=frac, random_state=25).index\n",
    "        df_train = df_train.drop(remove_index)\n",
    "        del remove_index\n",
    "        gc.collect()\n",
    "    model.fit(df_train[x_cols],\n",
    "            df_train.iloc[:,2].astype(np.int8),\n",
    "            group=df_train.groupby('session').size())\n",
    "    \n",
    "    del df_train\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "eeaf6aeb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:20:48.205149Z",
     "iopub.status.busy": "2023-03-23T11:20:48.203908Z",
     "iopub.status.idle": "2023-03-23T11:20:48.359348Z",
     "shell.execute_reply": "2023-03-23T11:20:48.358574Z"
    },
    "papermill": {
     "duration": 0.163671,
     "end_time": "2023-03-23T11:20:48.361665",
     "exception": false,
     "start_time": "2023-03-23T11:20:48.197994",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Save the LGBM model to file.\n",
    "if not CROSS_VALIDATE:\n",
    "    joblib.dump(model, 'lgb.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "52d8de47",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:20:48.373706Z",
     "iopub.status.busy": "2023-03-23T11:20:48.372700Z",
     "iopub.status.idle": "2023-03-23T11:21:00.409329Z",
     "shell.execute_reply": "2023-03-23T11:21:00.407164Z"
    },
    "papermill": {
     "duration": 12.044895,
     "end_time": "2023-03-23T11:21:00.411647",
     "exception": false,
     "start_time": "2023-03-23T11:20:48.366752",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0 GB\n"
     ]
    }
   ],
   "source": [
    "# Load and prepare second cross-validation dataset.\n",
    "if not CROSS_VALIDATE:\n",
    "    df_train = pd.read_parquet('/kaggle/input/otto-orders-w2vec-part1/train_features_with_w2v_cv2.parquet')\n",
    "    df_train = prepare_df(df_train)\n",
    "    \n",
    "    assert len(df_train[df_train.duplicated(subset=['session','order_predictions'], keep=False)]) == 0\n",
    "\n",
    "    size = df_train.memory_usage(deep='True').sum()\n",
    "    print(naturalsize(size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c522bb2e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:21:00.423152Z",
     "iopub.status.busy": "2023-03-23T11:21:00.422844Z",
     "iopub.status.idle": "2023-03-23T11:25:33.571722Z",
     "shell.execute_reply": "2023-03-23T11:25:33.570840Z"
    },
    "papermill": {
     "duration": 273.157229,
     "end_time": "2023-03-23T11:25:33.574167",
     "exception": false,
     "start_time": "2023-03-23T11:21:00.416938",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 3.3912960\ttotal: 222ms\tremaining: 2m 57s\n",
      "50:\tlearn: 1.6223405\ttotal: 11.2s\tremaining: 2m 43s\n",
      "100:\tlearn: 1.5991825\ttotal: 22.1s\tremaining: 2m 33s\n",
      "150:\tlearn: 1.5890697\ttotal: 33.8s\tremaining: 2m 25s\n",
      "200:\tlearn: 1.5819851\ttotal: 44.6s\tremaining: 2m 12s\n",
      "250:\tlearn: 1.5762063\ttotal: 55.7s\tremaining: 2m 1s\n",
      "300:\tlearn: 1.5713802\ttotal: 1m 7s\tremaining: 1m 52s\n",
      "350:\tlearn: 1.5668827\ttotal: 1m 18s\tremaining: 1m 40s\n",
      "400:\tlearn: 1.5626345\ttotal: 1m 29s\tremaining: 1m 29s\n",
      "450:\tlearn: 1.5589706\ttotal: 1m 41s\tremaining: 1m 18s\n",
      "500:\tlearn: 1.5554598\ttotal: 1m 52s\tremaining: 1m 7s\n",
      "550:\tlearn: 1.5521821\ttotal: 2m 4s\tremaining: 56.3s\n",
      "600:\tlearn: 1.5489818\ttotal: 2m 15s\tremaining: 44.9s\n",
      "650:\tlearn: 1.5457157\ttotal: 2m 26s\tremaining: 33.6s\n",
      "700:\tlearn: 1.5429188\ttotal: 2m 38s\tremaining: 22.4s\n",
      "750:\tlearn: 1.5401895\ttotal: 2m 49s\tremaining: 11s\n",
      "799:\tlearn: 1.5375254\ttotal: 2m 59s\tremaining: 0us\n"
     ]
    }
   ],
   "source": [
    "# Remove fraction of negative samples and fit the catboost model.\n",
    "if not CROSS_VALIDATE:\n",
    "    if frac > 0:\n",
    "        remove_index = df_train.loc[df_train['target'] == False].sample(frac=frac, random_state=25).index\n",
    "        df_train = df_train.drop(remove_index)\n",
    "        del remove_index\n",
    "        gc.collect()\n",
    "    train_pool = Pool(\n",
    "        data=df_train[x_cols],\n",
    "        label=df_train.iloc[:,2].astype(np.int8),\n",
    "        group_id=df_train.iloc[:,0]\n",
    "    )\n",
    "    model_catboost.fit(train_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "193fa7b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:25:33.591283Z",
     "iopub.status.busy": "2023-03-23T11:25:33.590970Z",
     "iopub.status.idle": "2023-03-23T11:25:33.617743Z",
     "shell.execute_reply": "2023-03-23T11:25:33.617018Z"
    },
    "papermill": {
     "duration": 0.037835,
     "end_time": "2023-03-23T11:25:33.620198",
     "exception": false,
     "start_time": "2023-03-23T11:25:33.582363",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Export the catboost model to file.\n",
    "if not CROSS_VALIDATE:\n",
    "    model_catboost.save_model(\"model\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 590.085724,
   "end_time": "2023-03-23T11:25:34.762322",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-23T11:15:44.676598",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}