4_performance_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd -q data/actr_reco"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.stats as sp\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "plt.rcParams['figure.dpi'] = 200\n",
    "\n",
    "import glob\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileglob = \"preds/LFM-2b_2019_*_topn_preds_*.csv\"\n",
    "all_files = glob.glob(fileglob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 2335)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "limit = 150\n",
    "with open(\"sampled.txt\", \"r\") as f:\n",
    "    users = f.readlines()\n",
    "    \n",
    "users = [int(user.strip()) for user in users]\n",
    "# limit users\n",
    "users = users[:limit]\n",
    "all_files = [file for file in all_files if any(f\"preds_{user}\" in file for user in users)]\n",
    "len(users), len(all_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def next_item_hr(row):\n",
    "    assert len(row[\"exp\"])\n",
    "    if not len(row[\"pred\"]):\n",
    "        return 0\n",
    "    return 1 if row[\"exp\"][0] == row[\"pred\"][0] else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rprecision(row):\n",
    "    \"\"\"Only works for remaining session predictions correctly.\n",
    "    Not the original implementation due to duplicates.\"\"\"\n",
    "    R = len(set(row[\"exp\"]))\n",
    "    r = len(set(row[\"pred\"]).intersection(set(row[\"exp\"])))\n",
    "    return r/R"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate(pred_df):\n",
    "    pred_df[\"r-precision\"] = pred_df.apply(rprecision, axis=1)\n",
    "    pred_df[\"next_item_hr\"] = pred_df.apply(next_item_hr, axis=1)\n",
    "    \n",
    "    agg_df = pred_df.groupby(\"algo\").agg({\"r-precision\": \"sum\", \"next_item_hr\": \"sum\", \"pos\": \"count\", \"user\": \"nunique\", \"algo\": \"nunique\"})\n",
    "    return agg_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>r-precision</th>\n",
       "      <th>next_item_hr</th>\n",
       "      <th>pos</th>\n",
       "      <th>user</th>\n",
       "      <th>algo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>algo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MostRecent</th>\n",
       "      <td>130.733717</td>\n",
       "      <td>25</td>\n",
       "      <td>6657</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            r-precision  next_item_hr   pos  user  algo\n",
       "algo                                                   \n",
       "MostRecent   130.733717            25  6657     1     1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read first file\n",
    "def read_file_and_aggregate(filename):\n",
    "    pred_df = pd.read_csv(filename, names=[\n",
    "        \"algo\",\n",
    "        \"split_f\",\n",
    "        \"user\",\n",
    "        \"pos\",\n",
    "        \"pred\",\n",
    "        \"pred_len\",\n",
    "        \"exp\",\n",
    "        \"unique_exp\",\n",
    "        \"mu_s\"],\n",
    "        index_col=0)\n",
    "    pred_df[\"pred\"] = pred_df[\"pred\"].map(eval)\n",
    "    pred_df[\"exp\"] = pred_df[\"exp\"].map(eval)\n",
    "    return aggregate(pred_df)\n",
    "\n",
    "agg_df = read_file_and_aggregate(all_files[0])\n",
    "agg_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read reamining files\n",
    "for filename in tqdm.tqdm(all_files[1:], bar_format=\"{l_bar}{bar}{r_bar} %i\"):\n",
    "    try:\n",
    "        calc_df = read_file_and_aggregate(filename)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(filename)\n",
    "        continue\n",
    "        \n",
    "    assert len(calc_df.index) == 1\n",
    "    # Element-wise aggregation\n",
    "    if calc_df.index[0] not in agg_df.index:\n",
    "        agg_df = pd.concat([agg_df, calc_df])\n",
    "    else:\n",
    "        agg_df = calc_df.combine(agg_df, lambda s1, s2: s1 + s2, fill_value=0, overwrite=False)\n",
    "agg_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_df.to_csv(\"aggregate_results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_df[\"r-precision\"] = agg_df[\"r-precision\"] / agg_df[\"pos\"]\n",
    "agg_df[\"next_item_hr\"] = agg_df[\"next_item_hr\"] / agg_df[\"pos\"]\n",
    "agg_df = agg_df.sort_values(\"r-precision\")\n",
    "agg_df.style.background_gradient(cmap=sns.light_palette(\"green\", as_cmap=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_df.index = agg_df.index.map({\n",
    "    \"Word2Vec100\": \"Partial Matching\",\n",
    "    \"NoiseComponent\": \"Noise\",\n",
    "    \"AssociativeComponent\": \"Spreading\",\n",
    "    \n",
    "    \"AllPos\": \"Valuation(MP)\",\n",
    "    \"PosNeuNeg\": \"Valuation(Discrete)\",\n",
    "    \"ValueRatio\": \"Valuation(Ratio)\",\n",
    "    \n",
    "    \"BaseLevelComponent1.737\": \"Base-level(full)\",\n",
    "    \"BaseLevelComponent\": \"Base-level(default)\",\n",
    "    \"BaseLevelComponent0.86\": \"Base-level(week)\",\n",
    "    \n",
    "    \"MostRecent\": \"MostRecent\",\n",
    "    \"UserBasedTransitionProbability\": \"TransProb\",\n",
    "    \n",
    "    \"ActrRecommender(AssociativeComponent,AllPos)\": \"ACT-R(S,V)\",\n",
    "    \"ActrRecommender(BaseLevelComponent,AllPos)\": \"ACT-R(B,V)\",\n",
    "    \"ActrRecommender(BaseLevelComponent,AssociativeComponent)\": \"ACT-R(B,S)\",\n",
    "    \"ActrRecommender(BaseLevelComponent,AssociativeComponent,AllPos)\": \"ACT-R(B,S,V)\",\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrr}\n",
      "\\toprule\n",
      "{} &  r-precision &  next\\_item\\_hr \\\\\n",
      "algo                &              &               \\\\\n",
      "\\midrule\n",
      "NaN                 &      0.00000 &       0.00000 \\\\\n",
      "NaN                 &      0.00083 &       0.00007 \\\\\n",
      "NaN                 &      0.00121 &       0.00003 \\\\\n",
      "NaN                 &      0.01563 &       0.00093 \\\\\n",
      "TransProb           &      0.03839 &       0.15907 \\\\\n",
      "Partial Matching    &      0.03895 &       0.01320 \\\\\n",
      "Noise               &      0.03996 &       0.00289 \\\\\n",
      "Valuation(Discrete) &      0.04751 &       0.00533 \\\\\n",
      "Valuation(Ratio)    &      0.05987 &       0.01042 \\\\\n",
      "Valuation(MP)       &      0.08436 &       0.01477 \\\\\n",
      "NaN                 &      0.08951 &       0.02983 \\\\\n",
      "NaN                 &      0.08959 &       0.03046 \\\\\n",
      "NaN                 &      0.09128 &       0.03041 \\\\\n",
      "Spreading           &      0.09235 &       0.02117 \\\\\n",
      "Base-level(full)    &      0.09903 &       0.03200 \\\\\n",
      "ACT-R(B,V)          &      0.10069 &       0.02416 \\\\\n",
      "MostRecent          &      0.10167 &       0.05189 \\\\\n",
      "Base-level(default) &      0.10380 &       0.02451 \\\\\n",
      "Base-level(week)    &      0.10489 &       0.02883 \\\\\n",
      "ACT-R(S,V)          &      0.11009 &       0.02998 \\\\\n",
      "ACT-R(B,S)          &      0.11042 &       0.02972 \\\\\n",
      "ACT-R(B,S,V)        &      0.11119 &       0.02961 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(agg_df[[\"r-precision\", \"next_item_hr\"]].to_latex(float_format=\"{:.5f}\".format))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "polarice",
   "language": "python",
   "name": "polarice"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}