otto-carts-w2vec-part1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f2581e18",
   "metadata": {
    "papermill": {
     "duration": 0.004515,
     "end_time": "2023-03-21T12:24:20.392569",
     "exception": false,
     "start_time": "2023-03-21T12:24:20.388054",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# W2vec features for carts (cross-validation and part_0 of test)\n",
    "\n",
    "In this notebook, the w2vec features for cart model are built using w2vec model from \"W2vec model for carts and orders\" notebook. There are total 4 w2vec features for carts model, two of them are calculated for the last 5 session aids and candidate, another two - for last 20 session aids and candidate. Each group of features is calculated in a single cycle and also using pandarallel library, but anyway this calculation takes time. To decrease calculation time even further, the task is performed in two notebooks in parallel - this one, and another one with very similar code that calculates the same features for another chunk of test dataset and also for cross-validation dataset.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "990b2c55",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-21T12:24:20.402306Z",
     "iopub.status.busy": "2023-03-21T12:24:20.401439Z",
     "iopub.status.idle": "2023-03-21T12:24:22.286808Z",
     "shell.execute_reply": "2023-03-21T12:24:22.285040Z"
    },
    "papermill": {
     "duration": 1.89431,
     "end_time": "2023-03-21T12:24:22.290163",
     "exception": false,
     "start_time": "2023-03-21T12:24:20.395853",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "from gensim.models import Word2Vec\n",
    "from pandarallel import pandarallel\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4e53d0a8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:24:22.302048Z",
     "iopub.status.busy": "2023-03-21T12:24:22.300524Z",
     "iopub.status.idle": "2023-03-21T12:24:22.309191Z",
     "shell.execute_reply": "2023-03-21T12:24:22.307666Z"
    },
    "papermill": {
     "duration": 0.016917,
     "end_time": "2023-03-21T12:24:22.312320",
     "exception": false,
     "start_time": "2023-03-21T12:24:22.295403",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Returns min and max w2vec similarity for the candidate aid and last session aids.\n",
    "def agg_min_max(x, w2v_model):\n",
    "    similarities = []\n",
    "    for item in x.aid:\n",
    "        similarities.append(w2v_model.wv.similarity(item, x.cart_predictions))\n",
    "    return [np.min(similarities), np.max(similarities)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2619631c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:24:22.320145Z",
     "iopub.status.busy": "2023-03-21T12:24:22.319689Z",
     "iopub.status.idle": "2023-03-21T12:24:22.327372Z",
     "shell.execute_reply": "2023-03-21T12:24:22.325962Z"
    },
    "papermill": {
     "duration": 0.014481,
     "end_time": "2023-03-21T12:24:22.329875",
     "exception": false,
     "start_time": "2023-03-21T12:24:22.315394",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Returns mean and min w2vec similarity for the candidate aid and last session aids.\n",
    "def agg_mean_min(x, w2v_model):\n",
    "    similarities = []\n",
    "    for item in x.aid:\n",
    "        similarities.append(w2v_model.wv.similarity(item, x.cart_predictions))\n",
    "    return [np.mean(similarities), np.min(similarities)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bf397efb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:24:22.341806Z",
     "iopub.status.busy": "2023-03-21T12:24:22.340384Z",
     "iopub.status.idle": "2023-03-21T12:24:22.356820Z",
     "shell.execute_reply": "2023-03-21T12:24:22.355477Z"
    },
    "papermill": {
     "duration": 0.026454,
     "end_time": "2023-03-21T12:24:22.360288",
     "exception": false,
     "start_time": "2023-03-21T12:24:22.333834",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Import candidates and last aids for each session, then calculate the w2vec features for them in chunks.\n",
    "def calculate_w2vec_features(train_path, sessions_path, w2v_model,\n",
    "                            time_delta, n_max, feature_type, feature_name1, feature_name2):\n",
    "    \n",
    "    # Import all the required data and merge it together.\n",
    "    print(feature_type)\n",
    "    df_sessions = pd.read_parquet(sessions_path)\n",
    "    df_sessions = otto_common.filter_by_time_and_n_max(df_sessions, time_delta, n_max)\n",
    "    df_sessions = (df_sessions.groupby('session').agg({'aid': lambda x: x.tolist()}))\n",
    "    df = pd.read_parquet(train_path)\n",
    "    df = pd.merge(df, df_sessions, how='left', on='session')\n",
    "    \n",
    "    # Remove all the columns not relevant to this calculation.\n",
    "    df = df[['cart_predictions', 'aid']]\n",
    "    del df_sessions\n",
    "    gc.collect()\n",
    "    \n",
    "    # Prepare data chunks.\n",
    "    i = 0\n",
    "    chunk_size = 10000000\n",
    "    while i < 1000000000:\n",
    "        df_chunk = df.iloc[i:i+chunk_size,:].copy()\n",
    "        \n",
    "        # Calculate the features.\n",
    "        #pandarallel.initialize(nb_workers=4, progress_bar=True)\n",
    "        pandarallel.initialize(nb_workers=4)\n",
    "        if feature_type == 'mean_min':\n",
    "            df_chunk['features']  = df_chunk.parallel_apply(\n",
    "                lambda x: agg_mean_min(x, w2v_model), axis=1)\n",
    "        elif feature_type == 'min_max':\n",
    "            df_chunk['features']  = df_chunk.parallel_apply(\n",
    "                lambda x: agg_min_max(x, w2v_model), axis=1)\n",
    "        else:\n",
    "            print('feature_type_unknown')\n",
    "            \n",
    "        # Format the calculated features.\n",
    "        df_chunk[[feature_name1,feature_name2]] = pd.DataFrame(df_chunk.features.tolist(), index=df_chunk.index)\n",
    "        df_chunk = df_chunk[[feature_name1, feature_name2]]\n",
    "        df_chunk[feature_name1] = df_chunk[feature_name1].astype(np.float32)\n",
    "        df_chunk[feature_name2] = df_chunk[feature_name2].astype(np.float32)\n",
    "        gc.collect()\n",
    "        \n",
    "        # Merge the chunks together.\n",
    "        if i == 0:\n",
    "            df_all = df_chunk.copy()\n",
    "        else:\n",
    "            df_all = pd.concat([df_all, df_chunk])\n",
    "        print(i)\n",
    "        i += chunk_size\n",
    "        gc.collect()\n",
    "        if df_chunk.shape[0] < chunk_size:\n",
    "            return df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9e710280",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:24:22.371220Z",
     "iopub.status.busy": "2023-03-21T12:24:22.370430Z",
     "iopub.status.idle": "2023-03-21T12:24:22.380004Z",
     "shell.execute_reply": "2023-03-21T12:24:22.378175Z"
    },
    "papermill": {
     "duration": 0.018287,
     "end_time": "2023-03-21T12:24:22.382906",
     "exception": false,
     "start_time": "2023-03-21T12:24:22.364619",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define all the parameters for the w2vec features, except for paths, that are different for cross-validation and test datasets.\n",
    "def add_w2vec_data(train_path, sessions_path, w2vec_path):\n",
    "    w2v_model = Word2Vec.load(w2vec_path)\n",
    "    df_w2v_20 = calculate_w2vec_features(train_path, sessions_path, w2v_model, 3 * 60 * 60, 20, 'mean_min', 'w2v_20_mean', 'w2v_20_min')  \n",
    "    gc.collect()\n",
    "    df_w2v_5 = calculate_w2vec_features(train_path, sessions_path, w2v_model, 5 * 24 * 60 * 60, 5, 'min_max', 'w2v_5_min', 'w2v_5_max')\n",
    "    gc.collect()\n",
    "    df_all = pd.read_parquet(train_path)\n",
    "    df_all = pd.concat([df_all, df_w2v_20, df_w2v_5], axis=1)\n",
    "    return df_all"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4928c513",
   "metadata": {
    "papermill": {
     "duration": 0.00326,
     "end_time": "2023-03-21T12:24:22.389695",
     "exception": false,
     "start_time": "2023-03-21T12:24:22.386435",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## W2vec features for Part_1 of test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "afb7643d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:24:22.399104Z",
     "iopub.status.busy": "2023-03-21T12:24:22.398657Z",
     "iopub.status.idle": "2023-03-21T14:58:47.628593Z",
     "shell.execute_reply": "2023-03-21T14:58:47.627008Z"
    },
    "papermill": {
     "duration": 9265.241288,
     "end_time": "2023-03-21T14:58:47.634254",
     "exception": false,
     "start_time": "2023-03-21T12:24:22.392966",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_min\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "10000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "20000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "30000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "40000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "50000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "60000000\n",
      "min_max\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "10000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "20000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "30000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "40000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "50000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "60000000\n"
     ]
    }
   ],
   "source": [
    "# All the paths and w2vec feature calculation.\n",
    "test_path_part1 = '/kaggle/input/otto-feature-engineering-carts/test_features_cart_part_1.parquet'\n",
    "sessions_path_test = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "w2vec_path_test = '/kaggle/input/otto-word2vec-exp/word2vec_test_exp.wordvectors'\n",
    "\n",
    "df_test1 = add_w2vec_data(test_path_part1, sessions_path_test, w2vec_path_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5a1c798c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T14:58:47.649380Z",
     "iopub.status.busy": "2023-03-21T14:58:47.647923Z",
     "iopub.status.idle": "2023-03-21T14:59:28.550495Z",
     "shell.execute_reply": "2023-03-21T14:59:28.549196Z"
    },
    "papermill": {
     "duration": 40.914741,
     "end_time": "2023-03-21T14:59:28.554244",
     "exception": false,
     "start_time": "2023-03-21T14:58:47.639503",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.5 GB\n"
     ]
    }
   ],
   "source": [
    "# Check size and export to file.\n",
    "size = df_test1.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_test1.to_parquet('test_features_with_w2v_cart_part_1.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 9323.140657,
   "end_time": "2023-03-21T14:59:29.823916",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-21T12:24:06.683259",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}