data-preparation-uxml-time.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "print(f\"SciPy version: {scipy.__version__}\")\n",
    "from collections import OrderedDict\n",
    "import scipy.sparse as sp\n",
    "import time\n",
    "import random\n",
    "from constants import (DATA_OCT, DATA_NOV, EXPORT_DIR, UX_CONSTANTS, SEED, NEW_USER_ID, NEW_PRODUCT_ID, T, USECOLS, \n",
    "                       EVENT_THRESHOLD, ALL_DATA_PATH, TRAIN_DATA_PATH, VAL_DATA_PATH, TEST_DATA_PATH, VAL_THRESHOLD, TEST_THRESHOLD)\n",
    "random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ux_constants = pd.Series(pd.read_csv(UX_CONSTANTS, index_col=0, squeeze=True, header=None), dtype='float32')\n",
    "VIEW     = ux_constants['view_to_purchase']\n",
    "CART     = ux_constants['cart_to_purchase']\n",
    "REMOVE   = ux_constants['remove_to_purchase']\n",
    "PURCHASE = ux_constants['purchase_to_purchase']\n",
    "\n",
    "def event_to_ux(event):\n",
    "    event_weights = {\n",
    "        'view': VIEW,\n",
    "        'cart': CART,\n",
    "        'remove_from_cart': REMOVE,\n",
    "        'purchase': PURCHASE,   \n",
    "    }\n",
    "    return event_weights.get(event, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4.65 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "COL = [\"event_type\",\"product_id\",\"user_id\"]\n",
    "DT = {'product_id': np.int64, 'user_id': np.int64, 'event_type': 'category'}\n",
    "df = pd.concat([pd.read_csv(DATA_OCT, engine='c', sep=',', usecols=COL, dtype=DT)\n",
    "                ,pd.read_csv(DATA_NOV, engine='c', sep=',', usecols=COL, dtype=DT)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cart</td>\n",
       "      <td>5773203</td>\n",
       "      <td>463240011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cart</td>\n",
       "      <td>5773353</td>\n",
       "      <td>463240011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cart</td>\n",
       "      <td>5881589</td>\n",
       "      <td>429681830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cart</td>\n",
       "      <td>5723490</td>\n",
       "      <td>463240011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cart</td>\n",
       "      <td>5881449</td>\n",
       "      <td>429681830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4635832</th>\n",
       "      <td>view</td>\n",
       "      <td>5830317</td>\n",
       "      <td>457678989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4635833</th>\n",
       "      <td>view</td>\n",
       "      <td>5733064</td>\n",
       "      <td>422196217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4635834</th>\n",
       "      <td>view</td>\n",
       "      <td>5867785</td>\n",
       "      <td>572579084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4635835</th>\n",
       "      <td>view</td>\n",
       "      <td>5779406</td>\n",
       "      <td>540006764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4635836</th>\n",
       "      <td>view</td>\n",
       "      <td>5880201</td>\n",
       "      <td>579969854</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8738120 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        event_type  product_id    user_id\n",
       "0             cart     5773203  463240011\n",
       "1             cart     5773353  463240011\n",
       "2             cart     5881589  429681830\n",
       "3             cart     5723490  463240011\n",
       "4             cart     5881449  429681830\n",
       "...            ...         ...        ...\n",
       "4635832       view     5830317  457678989\n",
       "4635833       view     5733064  422196217\n",
       "4635834       view     5867785  572579084\n",
       "4635835       view     5779406  540006764\n",
       "4635836       view     5880201  579969854\n",
       "\n",
       "[8738120 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 8738120 entries, 0 to 4635836\n",
      "Data columns (total 3 columns):\n",
      " #   Column      Dtype   \n",
      "---  ------      -----   \n",
      " 0   event_type  category\n",
      " 1   product_id  int64   \n",
      " 2   user_id     int64   \n",
      "dtypes: category(1), int64(2)\n",
      "memory usage: 208.3 MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8738120, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_dim = df.shape\n",
    "start_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We start with 713,100 unique users.\n"
     ]
    }
   ],
   "source": [
    "print(f\"We start with {len(df.user_id.unique()):,} unique users.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We will \u001b[91mdrop 535,508 (75.10%) users,\u001b[0m for not meeting the minimum \u001b[91m5\u001b[0m event requirement.\n"
     ]
    }
   ],
   "source": [
    "drop_visitors = set(df.user_id.value_counts()[df.user_id.value_counts()<EVENT_THRESHOLD].index)\n",
    "print(f\"We will {T.R}drop {len(drop_visitors):,} ({len(drop_visitors)*100/len(df.user_id.unique()):.2f}%) users,{T.E} \"+\n",
    "      f\"for not meeting the minimum {T.R}{EVENT_THRESHOLD}{T.E} event requirement.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df.user_id.isin(drop_visitors)]\n",
    "df.reset_index(inplace=True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This way we have reduced the number of total events by \u001b[92m10.95%\u001b[0m.\n"
     ]
    }
   ],
   "source": [
    "print(f\"This way we have reduced the number of total events by {T.G}{100-len(df)*100/start_dim[0]:.2f}%{T.E}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We will have \u001b[94m177,592 unique users.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "new_user_id = pd.DataFrame()\n",
    "new_user_id['user_id']=df.user_id.unique()\n",
    "print(f\"We will have {T.B}{len(new_user_id):,} unique users.{T.E}\")\n",
    "new_user_id.to_csv(NEW_USER_ID, index = True, header=True)\n",
    "uid_lookup = pd.Series(index=new_user_id.user_id,data=new_user_id.index)\n",
    "uid_lookup = uid_lookup.to_dict(OrderedDict)\n",
    "del new_user_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We will have \u001b[94m44,780 unique features\u001b[0m (products for e-commerce).\n"
     ]
    }
   ],
   "source": [
    "new_product_id = pd.DataFrame()\n",
    "new_product_id['product_id']=df.product_id.unique()\n",
    "print(f\"We will have {T.B}{len(new_product_id):,} unique features{T.E} (products for e-commerce).\")\n",
    "new_product_id.to_csv(NEW_PRODUCT_ID, index = True, header=True)\n",
    "pid_lookup = pd.Series(index=new_product_id.product_id,data=new_product_id.index)\n",
    "pid_lookup = pid_lookup.to_dict(OrderedDict)\n",
    "del new_product_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = time.time() # we start the timer after loading the dataframe\n",
    "number_of_users = df['user_id'].unique().shape[0]\n",
    "number_of_features = df['product_id'].unique().shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def user_experience_matrix(df):\n",
    "    last_index = df.shape[0]-1      \n",
    "    # Use np.float32 for torch.cuda.FloatTensor.or np.float16 for torch.cuda.HalfTensor (float64 not recommended)\n",
    "    uxm = sp.dok_matrix((number_of_users, number_of_features), dtype=np.float32)   \n",
    "    print(f\"   Event |   User | Product | Event | Previous |   {T.b}New UX{T.E}\")\n",
    "    \n",
    "    for row in df.itertuples():        \n",
    "        uid = uid_lookup[row.user_id]\n",
    "        pid = pid_lookup[row.product_id]        \n",
    "        prev_ux = uxm[uid,pid]\n",
    "        ux = np.tanh(prev_ux+event_to_ux(row.event_type))   \n",
    "#       ux = prev_ux + 1 # test case calculating the number of events between the user-product pair\n",
    "        uxm[uid,pid] = ux        \n",
    "        if (row.Index % 500000 == 0) or (row.Index == last_index):\n",
    "            print(f\"{row.Index:8} | \"+\n",
    "                  f\"{uid:6} | \"+\n",
    "                  f\"{pid:7} |  \"+\n",
    "                  f\"{row.event_type[:4]} | \"+\n",
    "                  f\"{prev_ux:8.5f} | \"+\n",
    "                  f\"{T.b}{ux:8.5f}{T.E}\")       \n",
    "    return uxm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Event |   User | Product | Event | Previous |   \u001b[1mNew UX\u001b[0m\n",
      "       0 |      0 |       0 |  cart |  0.00000 | \u001b[1m 0.19295\u001b[0m\n",
      "  500000 |    946 |   22949 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      " 1000000 |   1287 |   33175 |  remo |  0.23090 | \u001b[1m 0.26649\u001b[0m\n",
      " 1500000 |  55881 |    4161 |  cart |  0.00000 | \u001b[1m 0.19295\u001b[0m\n",
      " 2000000 |  56372 |    4570 |  cart |  0.72553 | \u001b[1m 0.72634\u001b[0m\n",
      " 2500000 |  80486 |   18263 |  view |  0.47220 | \u001b[1m 0.48172\u001b[0m\n",
      " 3000000 |  92749 |   11986 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      " 3500000 |  18163 |   12778 |  purc |  0.36994 | \u001b[1m 0.87868\u001b[0m\n",
      " 4000000 | 114521 |   28527 |  view |  0.05298 | \u001b[1m 0.10561\u001b[0m\n",
      " 4500000 | 122760 |   17009 |  remo |  0.00000 | \u001b[1m 0.04216\u001b[0m\n",
      " 5000000 |   2867 |   26409 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      " 5500000 |  91468 |    3903 |  cart |  0.00000 | \u001b[1m 0.19295\u001b[0m\n",
      " 6000000 |  84718 |     564 |  remo |  0.24114 | \u001b[1m 0.27598\u001b[0m\n",
      " 6500000 | 156464 |   16120 |  remo |  0.51193 | \u001b[1m 0.50360\u001b[0m\n",
      " 7000000 |  40902 |   37660 |  cart |  0.00000 | \u001b[1m 0.19295\u001b[0m\n",
      " 7500000 | 142922 |   18118 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      " 7780863 |   6770 |   10891 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      "Elapsed time: 165.61 seconds\n",
      "Wall time: 2min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "uxm = user_experience_matrix(df)\n",
    "print(f\"Elapsed time: {time.time()-start_time:.2f} seconds\") # we stop the timer before the train-test-validaiton split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train - test - validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_to_npz(X,path):\n",
    "    X = X.tocoo()\n",
    "    sp.save_npz(path,X)\n",
    "    print(f\"{T.G}Sparse matrix saved to: {path}{T.E}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 70.00% \n",
      "Validation: 15.00% \n",
      "Test: 15.00%\n"
     ]
    }
   ],
   "source": [
    "print(f\"Train: {VAL_THRESHOLD*100:.2f}% \\nValidation: {(1-TEST_THRESHOLD)*100:.2f}% \\nTest: {(1-TEST_THRESHOLD)*100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of stored values: 3,687,560\n"
     ]
    }
   ],
   "source": [
    "NNZ = uxm.nnz\n",
    "print(f\"Number of stored values: {NNZ:,}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 30.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "uxm_train = sp.dok_matrix.copy(uxm)\n",
    "uxm_val = sp.dok_matrix((number_of_users, number_of_features), dtype=np.float32) \n",
    "uxm_test = sp.dok_matrix((number_of_users, number_of_features), dtype=np.float32) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 29.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rows,cols = uxm_train.nonzero()\n",
    "for row,col in zip(rows,cols):\n",
    "    rnd = random.random()\n",
    "    if rnd > TEST_THRESHOLD:\n",
    "        uxm_test[row,col] = uxm_train[row,col]\n",
    "        uxm_train[row,col] = 0\n",
    "    elif rnd > VAL_THRESHOLD:\n",
    "        uxm_val[row,col] = uxm_train[row,col]\n",
    "        uxm_train[row,col] = 0   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train data values: 2,581,863 (70.02%)\n",
      "Number of validation data values: 553,442 (15.01%)\n",
      "Number of test data values: 552,255 (14.98%)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of train data values: {uxm_train.nnz:,} ({uxm_train.nnz*100/NNZ:.2f}%)\")\n",
    "print(f\"Number of validation data values: {uxm_val.nnz:,} ({uxm_val.nnz*100/NNZ:.2f}%)\")\n",
    "print(f\"Number of test data values: {uxm_test.nnz:,} ({uxm_test.nnz*100/NNZ:.2f}%)\")\n",
    "errormessage = '''All datapoints should be in either the train, the test of the validation datasets. \n",
    "The reason might be a change in how .nnz of a DOK matrix (scipy.sparse.dok_matrix) is calculated. \n",
    "In version 1.4.1 SciPy setting the value to zero explicitly (X[i,j]=0) is not counted by .nnz'''\n",
    "assert NNZ - uxm_train.nnz - uxm_val.nnz - uxm_test.nnz == 0, errormessage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save_to_npz(uxm,ALL_DATA_PATH)\n",
    "# save_to_npz(uxm_train,TRAIN_DATA_PATH)\n",
    "# save_to_npz(uxm_val,VAL_DATA_PATH)\n",
    "# save_to_npz(uxm_test,TEST_DATA_PATH)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}