sample_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:00 - Initialized environment\n"
     ]
    }
   ],
   "source": [
    "# IMPORTS\n",
    "from itertools import chain, zip_longest\n",
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import random\n",
    "import os\n",
    "import json\n",
    "\n",
    "gc.collect()\n",
    "start = datetime.now()\n",
    "print('{} - Initialized environment'.format(\n",
    "    datetime.now() - start\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quick Guide\n",
    "Here we will train a Bayesian Personalized Ranking (BPR-MF) model on the MovieLens 1M dataset.\n",
    "\n",
    "To start, we assume that the data (ratings.dat, etc.) files have been extracted to the `datasets/ml_1m` folder.\n",
    "\n",
    "The following block will process the raw data file to make it into a pandas DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:00.092786 - Retrieved interactions df.\n"
     ]
    }
   ],
   "source": [
    "def get_interactions(filename):\n",
    "    # Separators are ::, but the c engine doesn't handle multiple-char separators. Thus we're just going to have to manually interleave and break.\n",
    "    columns = ['user_id', 'item_id', 'rating', 'timestamp']\n",
    "    interleave = list(map(str, np.arange(len(columns)-1)))\n",
    "    read_names = [x for x in chain(*zip_longest(columns, interleave)) if x is not None]\n",
    "    # Read the actual file in\n",
    "    interactions_df = pd.read_csv(filename, sep=':', header=None, names=read_names).drop(columns=interleave)\n",
    "    return interactions_df\n",
    "\n",
    "movie_dir = os.path.join('datasets', 'ml-1m')\n",
    "ratings_file = os.path.join(movie_dir, 'ratings.dat')\n",
    "interactions_loc = os.path.join(movie_dir, 'interactions.msgpack')\n",
    "\n",
    "start = datetime.now()\n",
    "try:\n",
    "    df = pd.read_msgpack(interactions_loc)\n",
    "    print('{} - Retrieved interactions df.'.format(datetime.now() - start))\n",
    "except Exception as e:\n",
    "    print('Error unpickling {}, reconstructing from ratings.dat: {}'.format(interactions_loc, e))\n",
    "    df = get_interactions(ratings_file)\n",
    "    print('{} - Processed interactions from ratings.dat'.format(datetime.now() - start))\n",
    "    df.to_msgpack(interactions_loc)\n",
    "    print('{} - Serialized interactions to {}'.format(datetime.now() - start, interactions_loc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>914</td>\n",
       "      <td>3</td>\n",
       "      <td>978301968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3408</td>\n",
       "      <td>4</td>\n",
       "      <td>978300275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2355</td>\n",
       "      <td>5</td>\n",
       "      <td>978824291</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp\n",
       "0        1     1193       5  978300760\n",
       "1        1      661       3  978302109\n",
       "2        1      914       3  978301968\n",
       "3        1     3408       4  978300275\n",
       "4        1     2355       5  978824291"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train/Test/Validation Splits\n",
    "Perform the following operations to prepare our MovieLens interactions data for the recommender system model:\n",
    "- K-core (preserve only users and items with more than 5 interactions)\n",
    "- Create temporal columns\n",
    "- Map user/item IDs to contiguous integer series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:00.434865 - Added proper temporal columns to df\n",
      "Removing 0/6040 users (0.00 %) and 290/3706 items (7.83 %) from 1000209 total interactions (95.53164% Sparsity)\n",
      "Removing 0/6040 users (0.00 %) and 0/3416 items (0.00 %) from 999611 total interactions (95.15520% Sparsity)\n",
      "0:00:00.363996 - Done: 5-core decomposition after 2 iterations\n",
      "0:00:00.450793 - Mapped u-i indices\n",
      "0:00:00.845737 - Created \"prior\" column\n",
      "6040 Users interacted with 3416 items 993571 times (95.1845% sparsity, 164.499 actions/user, 290.858 actions/item)\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 993571 entries, 289422 to 856074\n",
      "Data columns (total 12 columns):\n",
      "user_id        993571 non-null int64\n",
      "item_id        993571 non-null int64\n",
      "rating         993571 non-null int64\n",
      "timestamp      993571 non-null int64\n",
      "date           993571 non-null datetime64[ns]\n",
      "year           993571 non-null int32\n",
      "month          993571 non-null int32\n",
      "day_of_week    993571 non-null int32\n",
      "day_of_year    993571 non-null int32\n",
      "u              993571 non-null int32\n",
      "i              993571 non-null int32\n",
      "prior          993571 non-null int32\n",
      "dtypes: datetime64[ns](1), int32(7), int64(4)\n",
      "memory usage: 72.0 MB\n",
      "None\n",
      "\n",
      "0:00:00.337128 - Created full user : item, interaction mappings df and user : items dict\n",
      "0:00:00.514655 - Created holdout sets of most recent interactions/user\n",
      "0:00:04.002325 - Created bootstrap validation DF of size 2000000\n",
      "0:00:07.623639 - Created bootstrap testing DF of size 2000000\n",
      "0:00:07.888900 - Generated train/validation/test splits and user : items dictionary mappings\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "INITIALIZE data and train/validation/test splits\n",
    "'''\n",
    "from recsys_models.data import process_temporal_columns, kcore, map_user_items, print_basic_stats\n",
    "from recsys_models.data.sampling import train_test_validation_split, get_user_interactions_df\n",
    "\n",
    "user_col = 'user_id'\n",
    "item_col = 'item_id'\n",
    "\n",
    "# Get temporal columns\n",
    "df['date'] = pd.to_datetime(df['timestamp'].apply(datetime.utcfromtimestamp))\n",
    "df = process_temporal_columns(df)\n",
    "\n",
    "# K-core\n",
    "cores = 5\n",
    "df = kcore(df, user_col, item_col, cores)\n",
    "\n",
    "# User/item ix -> id mappings\n",
    "df = map_user_items(df, user_col, item_col)\n",
    "\n",
    "# Get stats\n",
    "gc.collect()\n",
    "print_basic_stats(df, user_col, item_col)\n",
    "n_users = df[user_col].nunique()\n",
    "n_items = df[item_col].nunique()\n",
    "\n",
    "# Create train, validation, and test DFs by holding out the latest interaction per user for test and second-to-last for validation\n",
    "start = datetime.now()\n",
    "eval_size = 2000000\n",
    "train_df, validation_df, test_df, all_int_by_user_df = train_test_validation_split(df, eval_size)\n",
    "train_items_by_user = train_df.groupby(['u'])['i'].agg(lambda x: set(x)).to_dict()\n",
    "print('{} - Generated train/validation/test splits and user : items dictionary mappings'.format(\n",
    "    datetime.now() - start\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_year</th>\n",
       "      <th>u</th>\n",
       "      <th>i</th>\n",
       "      <th>prior</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>289422</th>\n",
       "      <td>6040</td>\n",
       "      <td>2384</td>\n",
       "      <td>4</td>\n",
       "      <td>956703954</td>\n",
       "      <td>2000-04-25 23:05:54</td>\n",
       "      <td>2000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>115</td>\n",
       "      <td>6039</td>\n",
       "      <td>323</td>\n",
       "      <td>669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138498</th>\n",
       "      <td>6040</td>\n",
       "      <td>593</td>\n",
       "      <td>5</td>\n",
       "      <td>956703954</td>\n",
       "      <td>2000-04-25 23:05:54</td>\n",
       "      <td>2000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>115</td>\n",
       "      <td>6039</td>\n",
       "      <td>128</td>\n",
       "      <td>323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45032</th>\n",
       "      <td>6040</td>\n",
       "      <td>1961</td>\n",
       "      <td>4</td>\n",
       "      <td>956703977</td>\n",
       "      <td>2000-04-25 23:06:17</td>\n",
       "      <td>2000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>115</td>\n",
       "      <td>6039</td>\n",
       "      <td>41</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  item_id  rating  timestamp                date  year  month  \\\n",
       "289422     6040     2384       4  956703954 2000-04-25 23:05:54  2000      4   \n",
       "138498     6040      593       5  956703954 2000-04-25 23:05:54  2000      4   \n",
       "45032      6040     1961       4  956703977 2000-04-25 23:06:17  2000      4   \n",
       "\n",
       "        day_of_week  day_of_year     u    i  prior  \n",
       "289422            1          115  6039  323    669  \n",
       "138498            1          115  6039  128    323  \n",
       "45032             1          115  6039   41    128  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_year</th>\n",
       "      <th>u</th>\n",
       "      <th>i</th>\n",
       "      <th>prior</th>\n",
       "      <th>j</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2469</td>\n",
       "      <td>3702</td>\n",
       "      <td>4</td>\n",
       "      <td>974160232</td>\n",
       "      <td>2000-11-14 00:03:52</td>\n",
       "      <td>2000</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>318</td>\n",
       "      <td>2468</td>\n",
       "      <td>211</td>\n",
       "      <td>217</td>\n",
       "      <td>1586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4614</td>\n",
       "      <td>492</td>\n",
       "      <td>4</td>\n",
       "      <td>964143639</td>\n",
       "      <td>2000-07-21 01:40:39</td>\n",
       "      <td>2000</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>202</td>\n",
       "      <td>4613</td>\n",
       "      <td>1801</td>\n",
       "      <td>305</td>\n",
       "      <td>2045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2260</td>\n",
       "      <td>1923</td>\n",
       "      <td>3</td>\n",
       "      <td>974585842</td>\n",
       "      <td>2000-11-18 22:17:22</td>\n",
       "      <td>2000</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>322</td>\n",
       "      <td>2259</td>\n",
       "      <td>290</td>\n",
       "      <td>733</td>\n",
       "      <td>2681</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp                date  year  month  \\\n",
       "0     2469     3702       4  974160232 2000-11-14 00:03:52  2000     11   \n",
       "1     4614      492       4  964143639 2000-07-21 01:40:39  2000      7   \n",
       "2     2260     1923       3  974585842 2000-11-18 22:17:22  2000     11   \n",
       "\n",
       "   day_of_week  day_of_year     u     i  prior     j  \n",
       "0            1          318  2468   211    217  1586  \n",
       "1            4          202  4613  1801    305  2045  \n",
       "2            5          322  2259   290    733  2681  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_year</th>\n",
       "      <th>u</th>\n",
       "      <th>i</th>\n",
       "      <th>prior</th>\n",
       "      <th>j</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5632</td>\n",
       "      <td>2336</td>\n",
       "      <td>5</td>\n",
       "      <td>959015166</td>\n",
       "      <td>2000-05-22 17:06:06</td>\n",
       "      <td>2000</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>142</td>\n",
       "      <td>5631</td>\n",
       "      <td>435</td>\n",
       "      <td>42</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5633</td>\n",
       "      <td>1287</td>\n",
       "      <td>5</td>\n",
       "      <td>959014384</td>\n",
       "      <td>2000-05-22 16:53:04</td>\n",
       "      <td>2000</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>142</td>\n",
       "      <td>5632</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>1119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5903</td>\n",
       "      <td>1617</td>\n",
       "      <td>5</td>\n",
       "      <td>957465792</td>\n",
       "      <td>2000-05-04 18:43:12</td>\n",
       "      <td>2000</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>124</td>\n",
       "      <td>5902</td>\n",
       "      <td>246</td>\n",
       "      <td>905</td>\n",
       "      <td>1965</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp                date  year  month  \\\n",
       "0     5632     2336       5  959015166 2000-05-22 17:06:06  2000      5   \n",
       "1     5633     1287       5  959014384 2000-05-22 16:53:04  2000      5   \n",
       "2     5903     1617       5  957465792 2000-05-04 18:43:12  2000      5   \n",
       "\n",
       "   day_of_week  day_of_year     u    i  prior     j  \n",
       "0            0          142  5631  435     42    27  \n",
       "1            0          142  5632    6      9  1119  \n",
       "2            3          124  5902  246    905  1965  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(train_df.head(3))\n",
    "display(validation_df.head(3))\n",
    "display(test_df.head(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Models\n",
    "Now we can initialize our model with a few parameters, train a BPR-MF model, and compare it to the PopRec baseline (pick the most popular item as seen in the training set):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:06.907555 - PopRec:\n",
      "Training AUC:\t\t0.84690\n",
      "Validation AUC:\t\t0.80072\n",
      "Testing AUC:\t\t0.79488\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "PopRec Baseline - Pick the more popular item based on training interactions\n",
    "'''\n",
    "from recsys_models.models import pop_rec\n",
    "from recsys_models.data.sampling import sample_unobserved\n",
    "\n",
    "start = datetime.now()\n",
    "pop_auc_tr = pop_rec(train_df, sample_unobserved(train_df, train_items_by_user, n_items, len(test_df)))\n",
    "pop_auc_v = pop_rec(train_df, validation_df)\n",
    "pop_auc_t = pop_rec(train_df, test_df)\n",
    "print('{} - PopRec:\\nTraining AUC:\\t\\t{:.5f}\\nValidation AUC:\\t\\t{:.5f}\\nTesting AUC:\\t\\t{:.5f}'.format(\n",
    "    datetime.now() - start,\n",
    "    pop_auc_tr,\n",
    "    pop_auc_v,\n",
    "    pop_auc_t\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:00.075769 - Generated u-i-j matrices for validation and testing\n",
      "\n",
      "=== BEGIN Optimization for bpr-mf_5k_0.0001l2_0.0001l2bias ===\n",
      "    200 Max epochs, with early stoppage at 1e-05 Validation AUC change\n",
      "    1000 Iterations per epoch with 512-sized batches\n",
      "0:00:00.267288 - Prior: 0.50051 Validation AUC, 0.50301 Testing AUC\n",
      "[0:00:03.625273 - Epoch 1] 0.54022 Loss, 0.85301 Training AUC, 0.79636 Validation AUC (0.29585 Change)\n",
      "[0:00:06.588379 - Epoch 2] 0.44859 Loss, 0.85319 Training AUC, 0.79956 Validation AUC (0.00320 Change)\n",
      "[0:00:09.493608 - Epoch 3] 0.43666 Loss, 0.86849 Training AUC, 0.81308 Validation AUC (0.01352 Change)\n",
      "[0:00:12.403825 - Epoch 4] 0.42238 Loss, 0.88661 Training AUC, 0.82824 Validation AUC (0.01516 Change)\n",
      "[0:00:15.316034 - Epoch 5] 0.41697 Loss, 0.89285 Training AUC, 0.83717 Validation AUC (0.00893 Change)\n",
      "[0:00:18.308031 - Epoch 6] 0.41347 Loss, 0.89770 Training AUC, 0.84078 Validation AUC (0.00361 Change)\n",
      "[0:00:21.266120 - Epoch 7] 0.41205 Loss, 0.89975 Training AUC, 0.84310 Validation AUC (0.00232 Change)\n",
      "[0:00:24.194288 - Epoch 8] 0.40971 Loss, 0.90223 Training AUC, 0.84572 Validation AUC (0.00262 Change)\n",
      "[0:00:27.143370 - Epoch 9] 0.40962 Loss, 0.90138 Training AUC, 0.84689 Validation AUC (0.00117 Change)\n",
      "[0:00:30.182272 - Epoch 10] 0.40814 Loss, 0.90566 Training AUC, 0.84853 Validation AUC (0.00164 Change)\n",
      "[0:00:33.464463 - Epoch 11] 0.40859 Loss, 0.90471 Training AUC, 0.84958 Validation AUC (0.00105 Change)\n",
      "[0:00:36.586114 - Epoch 12] 0.40836 Loss, 0.90554 Training AUC, 0.85102 Validation AUC (0.00144 Change)\n",
      "[0:00:39.721725 - Epoch 13] 0.40745 Loss, 0.90614 Training AUC, 0.85128 Validation AUC (0.00026 Change)\n",
      "[0:00:42.667878 - Epoch 14] 0.40746 Loss, 0.90627 Training AUC, 0.85215 Validation AUC (0.00088 Change)\n",
      "[0:00:45.623938 - Epoch 15] 0.40742 Loss, 0.90760 Training AUC, 0.85243 Validation AUC (0.00028 Change)\n",
      "[0:00:48.707722 - Epoch 16] 0.40721 Loss, 0.90854 Training AUC, 0.85286 Validation AUC (0.00043 Change)\n",
      "[0:00:51.748591 - Epoch 17] 0.40714 Loss, 0.90701 Training AUC, 0.85298 Validation AUC (0.00012 Change)\n",
      "[0:00:54.752522 - Epoch 18] 0.40656 Loss, 0.90801 Training AUC, 0.85359 Validation AUC (0.00061 Change)\n",
      "[0:00:57.740533 - Epoch 19] 0.40596 Loss, 0.90852 Training AUC, 0.85442 Validation AUC (0.00083 Change)\n",
      "[0:01:00.749514 - Epoch 20] 0.40555 Loss, 0.90919 Training AUC, 0.85420 Validation AUC (-0.00022 Change)\n",
      "[0:01:00.867198 - Epoch 20] - STOPPED. Final test AUC: 0.84519\n",
      "0:00:00.002993 - Saved parameters to tf_models\\bpr-mf_5k_0.0001l2_0.0001l2bias_ml-1m\\params.pkl\n",
      "0:00:00.026897 - Saved weights to tf_models\\bpr-mf_5k_0.0001l2_0.0001l2bias_ml-1m\\weights.pkl\n",
      "0:01:02.821943 - Saved model to tf_models\\bpr-mf_5k_0.0001l2_0.0001l2bias_ml-1m\n",
      "\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Run BPR\n",
    "'''\n",
    "import tensorflow as tf\n",
    "from recsys_models.models.bpr import BPR_MF\n",
    "from recsys_models.pipeline import train_model\n",
    "\n",
    "# Set training parameters\n",
    "max_epochs = 200\n",
    "n_iterations = 1000\n",
    "batch_size = 512\n",
    "stopping_threshold = 1e-5\n",
    "\n",
    "# Get the validation and testing matrices\n",
    "start = datetime.now()\n",
    "validation_data = validation_df[['u', 'i', 'j']].values\n",
    "test_data = test_df[['u', 'i', 'j']].values\n",
    "print('{} - Generated u-i-j matrices for validation and testing'.format(\n",
    "    datetime.now() - start\n",
    "))\n",
    "\n",
    "# Initialize the graph\n",
    "tf.reset_default_graph()\n",
    "model = BPR_MF(n_users, n_items, k=5, lambda_emb=1e-4, lambda_bias=1e-4,\n",
    "               opt_type=tf.contrib.opt.LazyAdamOptimizer, opt_args={'learning_rate': 0.007})\n",
    "print('\\n=== BEGIN Optimization for {} ==='.format(model.model_id))\n",
    "print('    {} Max epochs, with early stoppage at {} Validation AUC change'.format(max_epochs, stopping_threshold))\n",
    "print('    {} Iterations per epoch with {}-sized batches'.format(n_iterations, batch_size))\n",
    "\n",
    "# Open session and initialize graph weights\n",
    "session = tf.Session()\n",
    "session.run(tf.global_variables_initializer())\n",
    "\n",
    "# Train the model!\n",
    "model, train_auc, validation_auc, test_auc = train_model(\n",
    "    session, model, train_df, validation_data, test_data,\n",
    "    n_iterations=n_iterations, batch_size=batch_size,\n",
    "    min_epochs=10, max_epochs=max_epochs,\n",
    "    stopping_threshold=stopping_threshold,\n",
    "    sample_columns=['u', 'i'], column_order=['u', 'i', 'j'],\n",
    "    n_items=n_items, items_by_user=train_items_by_user\n",
    ")\n",
    "\n",
    "# Save model\n",
    "suffix = '_ml-1m'\n",
    "full_model_id = '{}{}'.format(model.model_id, suffix)\n",
    "model_folder = os.path.join('tf_models', full_model_id)\n",
    "if not os.path.exists(model_folder):\n",
    "    os.makedirs(model_folder)\n",
    "model.save(session, 'tf_models', suffix=suffix)\n",
    "print('{} - Saved model to {}'.format(\n",
    "    datetime.now() - start, model_folder\n",
    "))\n",
    "\n",
    "# Cleanup\n",
    "session.close()\n",
    "gc.collect()\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Weights from Pretrained Model\n",
    "We can load weights from another model and initialize the weight matrices.\n",
    "\n",
    "We can thusly evaluate existing models using the RecSysModels framework:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:00.392998 - Pretrained model from tf_models\\bpr-mf_5k_0.0001l2_0.0001l2bias_ml-1m evaluated on test data, with AUC: 0.84519\n",
      "\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Retrieve pretrained weights and evaluate with model\n",
    "'''\n",
    "start = datetime.now()\n",
    "\n",
    "# Initialize TF session\n",
    "tf.reset_default_graph()\n",
    "session = tf.Session()\n",
    "\n",
    "# Retrieve the model we just trained\n",
    "model2 = BPR_MF.load(model_folder)\n",
    "session.run(tf.global_variables_initializer())\n",
    "\n",
    "# Evaluate on the test data:\n",
    "test_auc_2 = model2.evaluate_auc(session, test_data)\n",
    "print('{} - Pretrained model from {} evaluated on test data, with AUC: {:.5f}'.format(\n",
    "    datetime.now() - start,\n",
    "    model_folder,\n",
    "    test_auc_2\n",
    "))\n",
    "\n",
    "# Cleanup\n",
    "session.close()\n",
    "gc.collect()\n",
    "print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}