otto-model-carts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f9d1283d",
   "metadata": {
    "papermill": {
     "duration": 0.004415,
     "end_time": "2023-03-23T11:00:33.836550",
     "exception": false,
     "start_time": "2023-03-23T11:00:33.832135",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Carts Model\n",
    "A model predicting which aids user is going to add to cart is trained in this notebook. This same notebook was also used for cross-validating the carts model. Unlike the clicks model, for carts, prediction is made in a separate notebook. On kaggle platform, notebooks with GPU have less memory available, and it was hard to fit all the required data into 13 GB of available RAM, so I had to move prediction to a different notebook without GPU support, but with 30Gb RAM available. This notebook uses input from two \"parallel\" notebooks that produce w2vec features for carts, one for cross-validation set and half of the test set and the other one for the other half of the test set. I've tried both catboost and LGBM models to predict carts, and LGBM showed better results. So, I've used the LGBM model to produce final results. But unlike clicks, for carts model I've removed the catboost code, to make the notebook shorter and clearer.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1d46aec9",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:33.845231Z",
     "iopub.status.busy": "2023-03-23T11:00:33.844775Z",
     "iopub.status.idle": "2023-03-23T11:00:36.653228Z",
     "shell.execute_reply": "2023-03-23T11:00:36.652286Z"
    },
    "papermill": {
     "duration": 2.815685,
     "end_time": "2023-03-23T11:00:36.655739",
     "exception": false,
     "start_time": "2023-03-23T11:00:33.840054",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from lightgbm.sklearn import LGBMRanker\n",
    "import joblib\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eab430de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:36.664601Z",
     "iopub.status.busy": "2023-03-23T11:00:36.664281Z",
     "iopub.status.idle": "2023-03-23T11:00:36.669256Z",
     "shell.execute_reply": "2023-03-23T11:00:36.668307Z"
    },
    "papermill": {
     "duration": 0.011742,
     "end_time": "2023-03-23T11:00:36.671470",
     "exception": false,
     "start_time": "2023-03-23T11:00:36.659728",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function was used to test new features before adding them to the pipeline.\n",
    "# Now it only deletes the day_of_week column, which is used to construct some features.\n",
    "def prepare_df(df):\n",
    "    del df['day_of_week']\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4e562c5",
   "metadata": {
    "papermill": {
     "duration": 0.003287,
     "end_time": "2023-03-23T11:00:36.678285",
     "exception": false,
     "start_time": "2023-03-23T11:00:36.674998",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Load and prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "94e5e111",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:36.686567Z",
     "iopub.status.busy": "2023-03-23T11:00:36.686267Z",
     "iopub.status.idle": "2023-03-23T11:00:49.990255Z",
     "shell.execute_reply": "2023-03-23T11:00:49.989159Z"
    },
    "papermill": {
     "duration": 13.310933,
     "end_time": "2023-03-23T11:00:49.992832",
     "exception": false,
     "start_time": "2023-03-23T11:00:36.681899",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load the train/cross-validation data.\n",
    "df_train = pd.read_parquet('/kaggle/input/otto-carts-w2vec/train_features_with_w2v_cv1.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1385beee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:50.002080Z",
     "iopub.status.busy": "2023-03-23T11:00:50.001155Z",
     "iopub.status.idle": "2023-03-23T11:00:55.183540Z",
     "shell.execute_reply": "2023-03-23T11:00:55.182510Z"
    },
    "papermill": {
     "duration": 5.189673,
     "end_time": "2023-03-23T11:00:55.186304",
     "exception": false,
     "start_time": "2023-03-23T11:00:49.996631",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7 GB\n"
     ]
    }
   ],
   "source": [
    "# A few checks and preparations.\n",
    "df_train = prepare_df(df_train)\n",
    "gc.collect()\n",
    "\n",
    "assert len(df_train[df_train.duplicated(subset=['session','cart_predictions'], keep=False)]) == 0\n",
    "\n",
    "size = df_train.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8824032e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:55.200078Z",
     "iopub.status.busy": "2023-03-23T11:00:55.199739Z",
     "iopub.status.idle": "2023-03-23T11:00:55.208953Z",
     "shell.execute_reply": "2023-03-23T11:00:55.208051Z"
    },
    "papermill": {
     "duration": 0.019217,
     "end_time": "2023-03-23T11:00:55.211601",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.192384",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_defined\n"
     ]
    }
   ],
   "source": [
    "# Set the LGBM model's parameters.\n",
    "parameters = {\n",
    "    \"objective\" : \"lambdarank\",\n",
    "    \"metric\" : \"ndcg\",\n",
    "    \"boosting_type\" : \"gbdt\",\n",
    "    'min_child_samples' : 100,\n",
    "    \"n_estimators\" : 250,\n",
    "    \"num_leaves\" : 128,\n",
    "    \"importance_type\" : 'gain',\n",
    "    'max_depth' : 8,\n",
    "    'learning_rate' : 0.07,\n",
    "    'random_state' : 22,\n",
    "    'device': 'gpu',\n",
    "    'gpu_platform_id': 0,\n",
    "    'gpu_device_id': 0,    \n",
    "}\n",
    "model = LGBMRanker(**parameters)\n",
    "\n",
    "print('model_defined')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "166edbe9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:55.227871Z",
     "iopub.status.busy": "2023-03-23T11:00:55.227458Z",
     "iopub.status.idle": "2023-03-23T11:00:55.238142Z",
     "shell.execute_reply": "2023-03-23T11:00:55.237193Z"
    },
    "papermill": {
     "duration": 0.023761,
     "end_time": "2023-03-23T11:00:55.245394",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.221633",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# A few global parameters, used both for creating submission and cross-validation.\n",
    "CROSS_VALIDATE = False # Should be changed to False to produce submission.\n",
    "frac = 0.65 # fracture of records with target==False to be dropped from train to reduce memory usage\n",
    "x_cols = list(df_train.columns[3:])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b01fa4bd",
   "metadata": {
    "papermill": {
     "duration": 0.005015,
     "end_time": "2023-03-23T11:00:55.255719",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.250704",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0c41c2ba",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:55.267321Z",
     "iopub.status.busy": "2023-03-23T11:00:55.266879Z",
     "iopub.status.idle": "2023-03-23T11:00:55.279679Z",
     "shell.execute_reply": "2023-03-23T11:00:55.277860Z"
    },
    "papermill": {
     "duration": 0.021193,
     "end_time": "2023-03-23T11:00:55.281991",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.260798",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6 µs, sys: 0 ns, total: 6 µs\n",
      "Wall time: 12.6 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Cell for cross-validation.\n",
    "\n",
    "if CROSS_VALIDATE:\n",
    "    # Define the splits and prepare a column to save results.\n",
    "    n_splits = 4\n",
    "    groups_by_session = df_train['session'].copy().tolist()\n",
    "    group_kfold = GroupKFold(n_splits=n_splits)\n",
    "    df_importances = pd.DataFrame({'columns':x_cols})\n",
    "    df_train['cv_prediction'] = -1\n",
    "    df_train['cv_prediction'] = df_train['cv_prediction'].astype(np.float32)\n",
    "    # Fit the model and save the results.\n",
    "    for i, (train_index, test_index) in enumerate(group_kfold.split(df_train[x_cols], df_train['target'], groups_by_session)):\n",
    "        train_index = otto_common.remove_frac(train_index, df_train, frac)\n",
    "        gc.collect()\n",
    "        print('start_fitting')\n",
    "\n",
    "        model.fit(\n",
    "            df_train[x_cols].iloc[train_index],\n",
    "            df_train.iloc[train_index, 2].astype(np.int8),\n",
    "            group=df_train.iloc[train_index].groupby('session').size(),\n",
    "        )\n",
    "        column_name = 'imp_' + str(i)\n",
    "        df_importances[column_name] = model.feature_importances_\n",
    "        df_train['cv_prediction'].iloc[test_index] = model.predict(df_train[x_cols].iloc[test_index])\n",
    "        gc.collect()\n",
    "    del groups_by_session\n",
    "    gc.collect()\n",
    "    df_importances['imp_avg'] = df_importances.mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "382b0ffb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:55.313799Z",
     "iopub.status.busy": "2023-03-23T11:00:55.313520Z",
     "iopub.status.idle": "2023-03-23T11:00:55.320782Z",
     "shell.execute_reply": "2023-03-23T11:00:55.319800Z"
    },
    "papermill": {
     "duration": 0.022613,
     "end_time": "2023-03-23T11:00:55.323135",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.300522",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# View feature_importances. Two cells were used to print feature importances so that it would be possible to compare values between two runs.\n",
    "#df_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1f546058",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:55.336569Z",
     "iopub.status.busy": "2023-03-23T11:00:55.336231Z",
     "iopub.status.idle": "2023-03-23T11:00:55.340240Z",
     "shell.execute_reply": "2023-03-23T11:00:55.339284Z"
    },
    "papermill": {
     "duration": 0.014799,
     "end_time": "2023-03-23T11:00:55.344667",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.329868",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ed79af73",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:55.357716Z",
     "iopub.status.busy": "2023-03-23T11:00:55.357419Z",
     "iopub.status.idle": "2023-03-23T11:00:55.361710Z",
     "shell.execute_reply": "2023-03-23T11:00:55.360894Z"
    },
    "papermill": {
     "duration": 0.014959,
     "end_time": "2023-03-23T11:00:55.365444",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.350485",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Check the cross-validation results.\n",
    "if CROSS_VALIDATE:\n",
    "    otto_common.calculate_recall(df_train, 'cv_prediction', 567353)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e561529",
   "metadata": {
    "papermill": {
     "duration": 0.005134,
     "end_time": "2023-03-23T11:00:55.375929",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.370795",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Fit the test model and save it to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0ec43e4b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:00:55.389540Z",
     "iopub.status.busy": "2023-03-23T11:00:55.389218Z",
     "iopub.status.idle": "2023-03-23T11:04:41.021126Z",
     "shell.execute_reply": "2023-03-23T11:04:41.020182Z"
    },
    "papermill": {
     "duration": 225.646054,
     "end_time": "2023-03-23T11:04:41.027401",
     "exception": false,
     "start_time": "2023-03-23T11:00:55.381347",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove a fraction of negative samples.\n",
    "if frac > 0:\n",
    "    remove_index = df_train.loc[df_train['target'] == False].sample(frac=frac, random_state=25).index\n",
    "    df_train = df_train.drop(remove_index)\n",
    "    del remove_index\n",
    "    gc.collect()\n",
    "\n",
    "# Fit the model.\n",
    "model.fit(df_train[x_cols],\n",
    "          df_train.iloc[:,2].astype(np.int8),\n",
    "          group=df_train.groupby('session').size())\n",
    "\n",
    "del df_train\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4a5df5c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T11:04:41.036632Z",
     "iopub.status.busy": "2023-03-23T11:04:41.035809Z",
     "iopub.status.idle": "2023-03-23T11:04:41.171923Z",
     "shell.execute_reply": "2023-03-23T11:04:41.171086Z"
    },
    "papermill": {
     "duration": 0.143012,
     "end_time": "2023-03-23T11:04:41.174201",
     "exception": false,
     "start_time": "2023-03-23T11:04:41.031189",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lgb.pkl']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the model to file.\n",
    "joblib.dump(model, 'lgb.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 256.162524,
   "end_time": "2023-03-23T11:04:42.099527",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-23T11:00:25.937003",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}