otto-carts-w2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "19875a2f",
   "metadata": {
    "papermill": {
     "duration": 0.005224,
     "end_time": "2023-03-21T12:27:01.000184",
     "exception": false,
     "start_time": "2023-03-21T12:27:00.994960",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# W2vec features for carts (cross-validation and part_0 of test)\n",
    "\n",
    "In this notebook, the w2vec features for carts model are built using w2vec model from \"W2vec model for carts and orders\" notebook. There are total 4 w2vec features for carts model, two of them are calculated for the last 5 session aids and candidate, another two - for last 20 session aids and candidate. Each group of features is calculated in a single cycle and also using pandarallel library, but anyway this calculation takes time. To decrease calculation time even further, the task is performed by two notebooks in parallel - this one, and another one with very similar code that calculates the same features for another chunk of test dataset.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4456b944",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-21T12:27:01.011076Z",
     "iopub.status.busy": "2023-03-21T12:27:01.010615Z",
     "iopub.status.idle": "2023-03-21T12:27:02.378809Z",
     "shell.execute_reply": "2023-03-21T12:27:02.377465Z"
    },
    "papermill": {
     "duration": 1.37696,
     "end_time": "2023-03-21T12:27:02.381764",
     "exception": false,
     "start_time": "2023-03-21T12:27:01.004804",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "from gensim.models import Word2Vec\n",
    "from pandarallel import pandarallel\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "69e71bfe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:27:02.392042Z",
     "iopub.status.busy": "2023-03-21T12:27:02.391466Z",
     "iopub.status.idle": "2023-03-21T12:27:02.397778Z",
     "shell.execute_reply": "2023-03-21T12:27:02.396530Z"
    },
    "papermill": {
     "duration": 0.014136,
     "end_time": "2023-03-21T12:27:02.400159",
     "exception": false,
     "start_time": "2023-03-21T12:27:02.386023",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Returns min and max w2vec similarity for the candidate aid and last session aids.\n",
    "def agg_min_max(x, w2v_model):\n",
    "    similarities = []\n",
    "    for item in x.aid:\n",
    "        similarities.append(w2v_model.wv.similarity(item, x.cart_predictions))\n",
    "    return [np.min(similarities), np.max(similarities)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d2a3c7e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:27:02.410279Z",
     "iopub.status.busy": "2023-03-21T12:27:02.409861Z",
     "iopub.status.idle": "2023-03-21T12:27:02.416403Z",
     "shell.execute_reply": "2023-03-21T12:27:02.414960Z"
    },
    "papermill": {
     "duration": 0.01437,
     "end_time": "2023-03-21T12:27:02.418822",
     "exception": false,
     "start_time": "2023-03-21T12:27:02.404452",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Returns mean and min w2vec similarity for the candidate aid and last session aids.\n",
    "def agg_mean_min(x, w2v_model):\n",
    "    similarities = []\n",
    "    for item in x.aid:\n",
    "        similarities.append(w2v_model.wv.similarity(item, x.cart_predictions))\n",
    "    return [np.mean(similarities), np.min(similarities)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cdaf24d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:27:02.430304Z",
     "iopub.status.busy": "2023-03-21T12:27:02.429879Z",
     "iopub.status.idle": "2023-03-21T12:27:02.444081Z",
     "shell.execute_reply": "2023-03-21T12:27:02.442550Z"
    },
    "papermill": {
     "duration": 0.023623,
     "end_time": "2023-03-21T12:27:02.446986",
     "exception": false,
     "start_time": "2023-03-21T12:27:02.423363",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Import candidates and last aids for each session, then calculate the w2vec features for them in chunks.\n",
    "def calculate_w2vec_features(train_path, sessions_path, w2v_model,\n",
    "                            time_delta, n_max, feature_type, feature_name1, feature_name2):\n",
    "    \n",
    "    # Import all the required data and merge it together.\n",
    "    print(feature_type)\n",
    "    df_sessions = pd.read_parquet(sessions_path)\n",
    "    df_sessions = otto_common.filter_by_time_and_n_max(df_sessions, time_delta, n_max)\n",
    "    df_sessions = (df_sessions.groupby('session').agg({'aid': lambda x: x.tolist()}))\n",
    "    df = pd.read_parquet(train_path)\n",
    "    df = pd.merge(df, df_sessions, how='left', on='session')\n",
    "    \n",
    "    # Remove all the columns not relevant to this calculation.\n",
    "    df = df[['cart_predictions', 'aid']]\n",
    "    del df_sessions\n",
    "    gc.collect()\n",
    "    \n",
    "    # Prepare data chunks\n",
    "    i = 0\n",
    "    chunk_size = 10000000\n",
    "    while i < 1000000000:\n",
    "        df_chunk = df.iloc[i:i+chunk_size,:].copy()\n",
    "        \n",
    "        # Calculate the features.\n",
    "        #pandarallel.initialize(nb_workers=4, progress_bar=True)\n",
    "        pandarallel.initialize(nb_workers=4)\n",
    "        if feature_type == 'mean_min':\n",
    "            df_chunk['features']  = df_chunk.parallel_apply(\n",
    "                lambda x: agg_mean_min(x, w2v_model), axis=1)\n",
    "        elif feature_type == 'min_max':\n",
    "            df_chunk['features']  = df_chunk.parallel_apply(\n",
    "                lambda x: agg_min_max(x, w2v_model), axis=1)\n",
    "        else:\n",
    "            print('feature_type_unknown')\n",
    "            \n",
    "        # Format the calculated features.\n",
    "        df_chunk[[feature_name1,feature_name2]] = pd.DataFrame(df_chunk.features.tolist(), index=df_chunk.index)\n",
    "        df_chunk = df_chunk[[feature_name1, feature_name2]]\n",
    "        df_chunk[feature_name1] = df_chunk[feature_name1].astype(np.float32)\n",
    "        df_chunk[feature_name2] = df_chunk[feature_name2].astype(np.float32)\n",
    "        gc.collect()\n",
    "        \n",
    "        # Merge the chunks together.\n",
    "        if i == 0:\n",
    "            df_all = df_chunk.copy()\n",
    "        else:\n",
    "            df_all = pd.concat([df_all, df_chunk])\n",
    "        print(i)\n",
    "        i += chunk_size\n",
    "        gc.collect()\n",
    "        if df_chunk.shape[0] < chunk_size:\n",
    "            return df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fa946bea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:27:02.457933Z",
     "iopub.status.busy": "2023-03-21T12:27:02.457527Z",
     "iopub.status.idle": "2023-03-21T12:27:02.465869Z",
     "shell.execute_reply": "2023-03-21T12:27:02.464557Z"
    },
    "papermill": {
     "duration": 0.016923,
     "end_time": "2023-03-21T12:27:02.468400",
     "exception": false,
     "start_time": "2023-03-21T12:27:02.451477",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define all the parameters for the w2vec features, except for paths, that are different for cross-validation and test datasets.\n",
    "def add_w2vec_data(train_path, sessions_path, w2vec_path):\n",
    "    w2v_model = Word2Vec.load(w2vec_path)\n",
    "    df_w2v_20 = calculate_w2vec_features(train_path, sessions_path, w2v_model, 3 * 60 * 60, 20, 'mean_min', 'w2v_20_mean', 'w2v_20_min')  \n",
    "    gc.collect()\n",
    "    df_w2v_5 = calculate_w2vec_features(train_path, sessions_path, w2v_model, 5 * 24 * 60 * 60, 5, 'min_max', 'w2v_5_min', 'w2v_5_max')\n",
    "    gc.collect()\n",
    "    df_all = pd.read_parquet(train_path)\n",
    "    df_all = pd.concat([df_all, df_w2v_20, df_w2v_5], axis=1)\n",
    "    return df_all"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e35bf5bb",
   "metadata": {
    "papermill": {
     "duration": 0.003858,
     "end_time": "2023-03-21T12:27:02.476486",
     "exception": false,
     "start_time": "2023-03-21T12:27:02.472628",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## W2vec features for cross-validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c2115a8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:27:02.486738Z",
     "iopub.status.busy": "2023-03-21T12:27:02.486358Z",
     "iopub.status.idle": "2023-03-21T13:15:58.268682Z",
     "shell.execute_reply": "2023-03-21T13:15:58.267318Z"
    },
    "papermill": {
     "duration": 2935.792311,
     "end_time": "2023-03-21T13:15:58.272972",
     "exception": false,
     "start_time": "2023-03-21T12:27:02.480661",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_min\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "10000000\n",
      "min_max\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "10000000\n"
     ]
    }
   ],
   "source": [
    "# All the paths and w2vec feature calculation.\n",
    "train_path_cv1 = '/kaggle/input/otto-feature-engineering-carts/cv_features_cart.parquet'\n",
    "sessions_path_cv1 = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "w2vec_path_cv = '/kaggle/input/otto-word2vec-exp/word2vec_cv_exp.wordvectors'\n",
    "\n",
    "df_cv1 = add_w2vec_data(train_path_cv1, sessions_path_cv1, w2vec_path_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4af88f9d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T13:15:58.288118Z",
     "iopub.status.busy": "2023-03-21T13:15:58.287042Z",
     "iopub.status.idle": "2023-03-21T13:15:58.300180Z",
     "shell.execute_reply": "2023-03-21T13:15:58.299006Z"
    },
    "papermill": {
     "duration": 0.025002,
     "end_time": "2023-03-21T13:15:58.303305",
     "exception": false,
     "start_time": "2023-03-21T13:15:58.278303",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n%%time\\n# This cell was used to test while developing the solution.\\n\\nw2v_model = Word2Vec.load(w2vec_path_cv)\\ndf_w2v_20 = calculate_w2vec_feature(train_path_cv1, sessions_path_cv1, w2v_model, 3 * 60 * 60, 20, 'mean_min', 'w2v_20_mean', 'w2v_20_min')  \\n\""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "%%time\n",
    "# This cell was used to test while developing the solution.\n",
    "\n",
    "w2v_model = Word2Vec.load(w2vec_path_cv)\n",
    "df_w2v_20 = calculate_w2vec_feature(train_path_cv1, sessions_path_cv1, w2v_model, 3 * 60 * 60, 20, 'mean_min', 'w2v_20_mean', 'w2v_20_min')  \n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8225009a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T13:15:58.316631Z",
     "iopub.status.busy": "2023-03-21T13:15:58.315378Z",
     "iopub.status.idle": "2023-03-21T13:16:09.301065Z",
     "shell.execute_reply": "2023-03-21T13:16:09.300210Z"
    },
    "papermill": {
     "duration": 10.994757,
     "end_time": "2023-03-21T13:16:09.303547",
     "exception": false,
     "start_time": "2023-03-21T13:15:58.308790",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7 GB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check size and export to file.\n",
    "size = df_cv1.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_cv1.to_parquet('train_features_with_w2v_cv1.parquet')\n",
    "\n",
    "del df_cv1, train_path_cv1, sessions_path_cv1, w2vec_path_cv\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "754c5ef6",
   "metadata": {
    "papermill": {
     "duration": 0.004779,
     "end_time": "2023-03-21T13:16:09.313520",
     "exception": false,
     "start_time": "2023-03-21T13:16:09.308741",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## W2vec features for Part_0 of test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3176f5db",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T13:16:09.326767Z",
     "iopub.status.busy": "2023-03-21T13:16:09.325653Z",
     "iopub.status.idle": "2023-03-21T15:47:21.389050Z",
     "shell.execute_reply": "2023-03-21T15:47:21.387631Z"
    },
    "papermill": {
     "duration": 9072.073624,
     "end_time": "2023-03-21T15:47:21.392484",
     "exception": false,
     "start_time": "2023-03-21T13:16:09.318860",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_min\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "10000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "20000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "30000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "40000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "50000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "60000000\n",
      "min_max\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "10000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "20000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "30000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "40000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "50000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "60000000\n"
     ]
    }
   ],
   "source": [
    "# All the paths and w2vec feature calculation.\n",
    "test_path_part0 = '/kaggle/input/otto-feature-engineering-carts/test_features_cart_part_0.parquet'\n",
    "sessions_path_test = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "w2vec_path_test = '/kaggle/input/otto-word2vec-exp/word2vec_test_exp.wordvectors'\n",
    "\n",
    "df_test0 = add_w2vec_data(test_path_part0, sessions_path_test, w2vec_path_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d768cabf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T15:47:21.408353Z",
     "iopub.status.busy": "2023-03-21T15:47:21.407851Z",
     "iopub.status.idle": "2023-03-21T15:48:01.583181Z",
     "shell.execute_reply": "2023-03-21T15:48:01.582025Z"
    },
    "papermill": {
     "duration": 40.186916,
     "end_time": "2023-03-21T15:48:01.586386",
     "exception": false,
     "start_time": "2023-03-21T15:47:21.399470",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.5 GB\n"
     ]
    }
   ],
   "source": [
    "# Check size and export to file.\n",
    "size = df_test0.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_test0.to_parquet('test_features_with_w2v_cart_part_0.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "34c2d967",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T15:48:01.602476Z",
     "iopub.status.busy": "2023-03-21T15:48:01.602030Z",
     "iopub.status.idle": "2023-03-21T15:48:01.606538Z",
     "shell.execute_reply": "2023-03-21T15:48:01.605301Z"
    },
    "papermill": {
     "duration": 0.015172,
     "end_time": "2023-03-21T15:48:01.608854",
     "exception": false,
     "start_time": "2023-03-21T15:48:01.593682",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#del df_test0, test_path_part0\n",
    "#gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "90132658",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T15:48:01.624561Z",
     "iopub.status.busy": "2023-03-21T15:48:01.624099Z",
     "iopub.status.idle": "2023-03-21T15:48:01.628950Z",
     "shell.execute_reply": "2023-03-21T15:48:01.627711Z"
    },
    "papermill": {
     "duration": 0.015665,
     "end_time": "2023-03-21T15:48:01.631363",
     "exception": false,
     "start_time": "2023-03-21T15:48:01.615698",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#test_path_part1 = '/kaggle/input/otto-feature-engineering-carts/test_features_cart_part_1.parquet'\n",
    "\n",
    "#df_test1 = add_w2vec_data(test_path_part1, sessions_path_test, w2vec_path_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5cfc560a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T15:48:01.647495Z",
     "iopub.status.busy": "2023-03-21T15:48:01.646342Z",
     "iopub.status.idle": "2023-03-21T15:48:01.651654Z",
     "shell.execute_reply": "2023-03-21T15:48:01.650718Z"
    },
    "papermill": {
     "duration": 0.015893,
     "end_time": "2023-03-21T15:48:01.654009",
     "exception": false,
     "start_time": "2023-03-21T15:48:01.638116",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df_test1.to_parquet('test_features_with_w2v_cart_part_1.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 12072.387705,
   "end_time": "2023-03-21T15:48:02.909216",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-21T12:26:50.521511",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}