data-preparation-uxml.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SciPy version: 1.4.1\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "print(f\"SciPy version: {scipy.__version__}\")\n",
    "from collections import OrderedDict\n",
    "import scipy.sparse as sp\n",
    "import time\n",
    "import random\n",
    "from constants import (DATA_OCT, DATA_NOV, EXPORT_DIR, UX_CONSTANTS, SEED, NEW_USER_ID, NEW_PRODUCT_ID, T, USECOLS, \n",
    "                       EVENT_THRESHOLD, ALL_DATA_PATH, TRAIN_DATA_PATH, VAL_DATA_PATH, TEST_DATA_PATH, VAL_THRESHOLD, TEST_THRESHOLD)\n",
    "random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ux_constants = pd.Series(pd.read_csv(UX_CONSTANTS, index_col=0, squeeze=True, header=None), dtype='float32')\n",
    "VIEW     = ux_constants['view_to_purchase']\n",
    "CART     = ux_constants['cart_to_purchase']\n",
    "REMOVE   = ux_constants['remove_to_purchase']\n",
    "PURCHASE = ux_constants['purchase_to_purchase']\n",
    "\n",
    "def event_to_ux(event):\n",
    "    event_weights = {\n",
    "        'view': VIEW,\n",
    "        'cart': CART,\n",
    "        'remove_from_cart': REMOVE,\n",
    "        'purchase': PURCHASE,   \n",
    "    }\n",
    "    return event_weights.get(event, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([pd.read_csv(DATA_OCT, engine='c', sep=',',usecols=USECOLS)\n",
    "                ,pd.read_csv(DATA_NOV, engine='c', sep=',',usecols=USECOLS)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"event_type\"] = df[\"event_type\"].astype(\"category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 8738120 entries, 0 to 4635836\n",
      "Data columns (total 3 columns):\n",
      " #   Column      Dtype   \n",
      "---  ------      -----   \n",
      " 0   event_type  category\n",
      " 1   product_id  int64   \n",
      " 2   user_id     int64   \n",
      "dtypes: category(1), int64(2)\n",
      "memory usage: 208.3 MB\n"
     ]
    }
   ],
   "source": [
    "df.info()\n",
    "start_time = time.time() # we start the timer after loading the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8738120, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_dim = df.shape\n",
    "start_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We start with 713,100 unique users.\n"
     ]
    }
   ],
   "source": [
    "print(f\"We start with {len(df.user_id.unique()):,} unique users.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We will \u001b[91mdrop 535,508 (75.10%) users,\u001b[0m for not meeting the minimum \u001b[91m5\u001b[0m event requirement.\n"
     ]
    }
   ],
   "source": [
    "drop_visitors = set(df.user_id.value_counts()[df.user_id.value_counts()<EVENT_THRESHOLD].index)\n",
    "print(f\"We will {T.R}drop {len(drop_visitors):,} ({len(drop_visitors)*100/len(df.user_id.unique()):.2f}%) users,{T.E} \"+\n",
    "      f\"for not meeting the minimum {T.R}{EVENT_THRESHOLD}{T.E} event requirement.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df.user_id.isin(drop_visitors)]\n",
    "df.reset_index(inplace=True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This way we have reduced the number of total events by \u001b[92m10.95%\u001b[0m.\n"
     ]
    }
   ],
   "source": [
    "print(f\"This way we have reduced the number of total events by {T.G}{100-len(df)*100/start_dim[0]:.2f}%{T.E}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We will have \u001b[94m177,592 unique users.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "new_user_id = pd.DataFrame()\n",
    "new_user_id['user_id']=df.user_id.unique()\n",
    "print(f\"We will have {T.B}{len(new_user_id):,} unique users.{T.E}\")\n",
    "new_user_id.to_csv(NEW_USER_ID, index = True, header=True)\n",
    "uid_lookup = pd.Series(index=new_user_id.user_id,data=new_user_id.index)\n",
    "uid_lookup = uid_lookup.to_dict(OrderedDict)\n",
    "del new_user_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We will have \u001b[94m44,780 unique features\u001b[0m (products for e-commerce).\n"
     ]
    }
   ],
   "source": [
    "new_product_id = pd.DataFrame()\n",
    "new_product_id['product_id']=df.product_id.unique()\n",
    "print(f\"We will have {T.B}{len(new_product_id):,} unique features{T.E} (products for e-commerce).\")\n",
    "new_product_id.to_csv(NEW_PRODUCT_ID, index = True, header=True)\n",
    "pid_lookup = pd.Series(index=new_product_id.product_id,data=new_product_id.index)\n",
    "pid_lookup = pid_lookup.to_dict(OrderedDict)\n",
    "del new_product_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_users = df['user_id'].unique().shape[0]\n",
    "number_of_features = df['product_id'].unique().shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def user_experience_matrix(df):\n",
    "    last_index = df.shape[0]-1      \n",
    "    # Use np.float32 for torch.cuda.FloatTensor.or np.float16 for torch.cuda.HalfTensor (float64 not recommended)\n",
    "    uxm = sp.dok_matrix((number_of_users, number_of_features), dtype=np.float32)   \n",
    "    print(f\"   Event |   User | Product | Event | Previous |   {T.b}New UX{T.E}\")\n",
    "    \n",
    "    for row in df.itertuples():        \n",
    "        uid = uid_lookup[row.user_id]\n",
    "        pid = pid_lookup[row.product_id]        \n",
    "        prev_ux = uxm[uid,pid]\n",
    "        ux = np.tanh(prev_ux+event_to_ux(row.event_type))   \n",
    "#       ux = prev_ux + 1 # test case calculating the number of events between the user-product pair\n",
    "        uxm[uid,pid] = ux        \n",
    "        if (row.Index % 500000 == 0) or (row.Index == last_index):\n",
    "            print(f\"{row.Index:8} | \"+\n",
    "                  f\"{uid:6} | \"+\n",
    "                  f\"{pid:7} |  \"+\n",
    "                  f\"{row.event_type[:4]} | \"+\n",
    "                  f\"{prev_ux:8.5f} | \"+\n",
    "                  f\"{T.b}{ux:8.5f}{T.E}\")       \n",
    "    return uxm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Event |   User | Product | Event | Previous |   \u001b[1mNew UX\u001b[0m\n",
      "       0 |      0 |       0 |  cart |  0.00000 | \u001b[1m 0.19295\u001b[0m\n",
      "  500000 |    946 |   22949 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      " 1000000 |   1287 |   33175 |  remo |  0.23090 | \u001b[1m 0.26649\u001b[0m\n",
      " 1500000 |  55881 |    4161 |  cart |  0.00000 | \u001b[1m 0.19295\u001b[0m\n",
      " 2000000 |  56372 |    4570 |  cart |  0.72553 | \u001b[1m 0.72634\u001b[0m\n",
      " 2500000 |  80486 |   18263 |  view |  0.47220 | \u001b[1m 0.48172\u001b[0m\n",
      " 3000000 |  92749 |   11986 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      " 3500000 |  18163 |   12778 |  purc |  0.36994 | \u001b[1m 0.87868\u001b[0m\n",
      " 4000000 | 114521 |   28527 |  view |  0.05298 | \u001b[1m 0.10561\u001b[0m\n",
      " 4500000 | 122760 |   17009 |  remo |  0.00000 | \u001b[1m 0.04216\u001b[0m\n",
      " 5000000 |   2867 |   26409 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      " 5500000 |  91468 |    3903 |  cart |  0.00000 | \u001b[1m 0.19295\u001b[0m\n",
      " 6000000 |  84718 |     564 |  remo |  0.24114 | \u001b[1m 0.27598\u001b[0m\n",
      " 6500000 | 156464 |   16120 |  remo |  0.51193 | \u001b[1m 0.50360\u001b[0m\n",
      " 7000000 |  40902 |   37660 |  cart |  0.00000 | \u001b[1m 0.19295\u001b[0m\n",
      " 7500000 | 142922 |   18118 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      " 7780863 |   6770 |   10891 |  view |  0.00000 | \u001b[1m 0.05298\u001b[0m\n",
      "Elapsed time: 142.24 seconds\n"
     ]
    }
   ],
   "source": [
    "uxm = user_experience_matrix(df)\n",
    "print(f\"Elapsed time: {time.time()-start_time:.2f} seconds\") # we stop the timer before the train-test-validaiton split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train - test - validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_to_npz(X,path):\n",
    "    X = X.tocoo()\n",
    "    sp.save_npz(path,X)\n",
    "    print(f\"{T.G}Sparse matrix saved to: {path}{T.E}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 70.00% \n",
      "Validation: 15.00% \n",
      "Test: 15.00%\n"
     ]
    }
   ],
   "source": [
    "print(f\"Train: {VAL_THRESHOLD*100:.2f}% \\nValidation: {(1-TEST_THRESHOLD)*100:.2f}% \\nTest: {(1-TEST_THRESHOLD)*100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of stored values: 3,687,560\n"
     ]
    }
   ],
   "source": [
    "NNZ = uxm.nnz\n",
    "print(f\"Number of stored values: {NNZ:,}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "uxm_train = sp.dok_matrix.copy(uxm)\n",
    "uxm_val = sp.dok_matrix((number_of_users, number_of_features), dtype=np.float32) \n",
    "uxm_test = sp.dok_matrix((number_of_users, number_of_features), dtype=np.float32) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows,cols = uxm_train.nonzero()\n",
    "for row,col in zip(rows,cols):\n",
    "    rnd = random.random()\n",
    "    if rnd > TEST_THRESHOLD:\n",
    "        uxm_test[row,col] = uxm_train[row,col]\n",
    "        uxm_train[row,col] = 0\n",
    "    elif rnd > VAL_THRESHOLD:\n",
    "        uxm_val[row,col] = uxm_train[row,col]\n",
    "        uxm_train[row,col] = 0   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train data values: 2,581,863 (70.02%)\n",
      "Number of validation data values: 553,442 (15.01%)\n",
      "Number of test data values: 552,255 (14.98%)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of train data values: {uxm_train.nnz:,} ({uxm_train.nnz*100/NNZ:.2f}%)\")\n",
    "print(f\"Number of validation data values: {uxm_val.nnz:,} ({uxm_val.nnz*100/NNZ:.2f}%)\")\n",
    "print(f\"Number of test data values: {uxm_test.nnz:,} ({uxm_test.nnz*100/NNZ:.2f}%)\")\n",
    "errormessage = '''All datapoints should be in either the train, the test of the validation datasets. \n",
    "The reason might be a change in how .nnz of a DOK matrix (scipy.sparse.dok_matrix) is calculated. \n",
    "In version 1.4.1 SciPy setting the value to zero explicitly (X[i,j]=0) is not counted by .nnz'''\n",
    "assert NNZ - uxm_train.nnz - uxm_val.nnz - uxm_test.nnz == 0, errormessage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92mSparse matrix saved to: ./data/uxm.npz\u001b[0m\n",
      "\u001b[92mSparse matrix saved to: ./data/uxm_train.npz\u001b[0m\n",
      "\u001b[92mSparse matrix saved to: ./data/uxm_val.npz\u001b[0m\n",
      "\u001b[92mSparse matrix saved to: ./data/uxm_test.npz\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "save_to_npz(uxm,ALL_DATA_PATH)\n",
    "save_to_npz(uxm_train,TRAIN_DATA_PATH)\n",
    "save_to_npz(uxm_val,VAL_DATA_PATH)\n",
    "save_to_npz(uxm_test,TEST_DATA_PATH)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}