experiment_coat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import numpy as np\n",
    "import torch\n",
    "import pdb\n",
    "from sklearn.metrics import roc_auc_score\n",
    "np.random.seed(2020)\n",
    "torch.manual_seed(2020)\n",
    "import pdb\n",
    "\n",
    "from dataset import load_data\n",
    "from matrix_factorization import MF, MF_CVIB, MF_IPS, MF_SNIPS, MF_DR\n",
    "from utils import gini_index, ndcg_func, get_user_wise_ctr, rating_mat_to_sample, binarize, shuffle, minU\n",
    "mse_func = lambda x,y: np.mean((x-y)**2)\n",
    "acc_func = lambda x,y: np.sum(x == y) / len(x)\n",
    "\n",
    "dataset_name = \"coat\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===>Load from coat data set<===\n",
      "[train] rating ratio: 0.080000\n",
      "[test]  rating ratio: 0.053333\n",
      "# user: 290, # item: 300\n"
     ]
    }
   ],
   "source": [
    "if dataset_name == \"coat\":\n",
    "    train_mat, test_mat = load_data(\"coat\")        \n",
    "    x_train, y_train = rating_mat_to_sample(train_mat)\n",
    "    x_test, y_test = rating_mat_to_sample(test_mat)\n",
    "    num_user = train_mat.shape[0]\n",
    "    num_item = train_mat.shape[1]\n",
    "\n",
    "elif dataset_name == \"yahoo\":\n",
    "    x_train, y_train, x_test, y_test = load_data(\"yahoo\")\n",
    "    x_train, y_train = shuffle(x_train, y_train)\n",
    "    num_user = x_train[:,0].max() + 1\n",
    "    num_item = x_train[:,1].max() + 1\n",
    "\n",
    "print(\"# user: {}, # item: {}\".format(num_user, num_item))\n",
    "# binarize\n",
    "y_train = binarize(y_train)\n",
    "y_test = binarize(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MF-CVIB] epoch:30, xent:108.83231997489929\n",
      "***************[MF-CVIB]***************\n",
      "[MF-CVIB] test mse: 0.5794288551431656\n",
      "[MF-CVIB] test auc: 0.7469847095368782\n",
      "[MF] ndcg@5:0.655306, ndcg@10:0.712245\n",
      "Num User: 290\n",
      "Gini index: 0.31512991899429077\n",
      "Global utility: 0.5372413793103449\n",
      "***************[MF-CVIB]***************\n"
     ]
    }
   ],
   "source": [
    "\"MF CVIB\"\n",
    "mf_cvib = MF_CVIB(num_user, num_item)\n",
    "mf_cvib.fit(x_train, y_train, \n",
    "    lr=0.01,\n",
    "    batch_size=128,\n",
    "    lamb=1e-4,\n",
    "    alpha=10.0,\n",
    "    gamma=1e-5,\n",
    "    tol=1e-5,\n",
    "    verbose=False)\n",
    "\n",
    "test_pred = mf_cvib.predict(x_test)\n",
    "mse_mf = mse_func(y_test, test_pred)\n",
    "auc_mf = roc_auc_score(y_test, test_pred)\n",
    "ndcg_res = ndcg_func(mf_cvib, x_test, y_test)\n",
    "\n",
    "print(\"***\"*5 + \"[MF-CVIB]\" + \"***\"*5)\n",
    "print(\"[MF-CVIB] test mse:\", mse_func(y_test, test_pred))\n",
    "print(\"[MF-CVIB] test auc:\", auc_mf)\n",
    "print(\"[MF] ndcg@5:{:.6f}, ndcg@10:{:.6f}\".format(\n",
    "        np.mean(ndcg_res[\"ndcg_5\"]), np.mean(ndcg_res[\"ndcg_10\"])))\n",
    "user_wise_ctr = get_user_wise_ctr(x_test,y_test,test_pred)\n",
    "gi,gu = gini_index(user_wise_ctr)\n",
    "print(\"***\"*5 + \"[MF-CVIB]\" + \"***\"*5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MF] epoch:92, xent:17.17500016093254\n",
      "***************[MF]***************\n",
      "[MF] test mse: 0.24587457540363486\n",
      "[MF] test auc: 0.7032263627287906\n",
      "[MF] ndcg@5:0.588338, ndcg@10:0.664943\n",
      "Num User: 290\n",
      "Gini index: 0.3487279372865877\n",
      "Global utility: 0.4944827586206896\n",
      "***************[MF]***************\n"
     ]
    }
   ],
   "source": [
    "\"MF naive\"\n",
    "mf = MF(num_user, num_item)\n",
    "mf.fit(x_train, y_train, \n",
    "    lr=0.01,\n",
    "    batch_size=128,\n",
    "    lamb=1e-4,\n",
    "    tol=1e-5,\n",
    "    verbose=False)\n",
    "test_pred = mf.predict(x_test)\n",
    "mse_mf = mse_func(y_test, test_pred)\n",
    "auc_mf = roc_auc_score(y_test, test_pred)\n",
    "ndcg_res = ndcg_func(mf, x_test, y_test)\n",
    "\n",
    "print(\"***\"*5 + \"[MF]\" + \"***\"*5)\n",
    "print(\"[MF] test mse:\", mse_func(y_test, test_pred))\n",
    "print(\"[MF] test auc:\", auc_mf)\n",
    "print(\"[MF] ndcg@5:{:.6f}, ndcg@10:{:.6f}\".format(\n",
    "        np.mean(ndcg_res[\"ndcg_5\"]), np.mean(ndcg_res[\"ndcg_10\"])))\n",
    "user_wise_ctr = get_user_wise_ctr(x_test,y_test,test_pred)\n",
    "gi,gu = gini_index(user_wise_ctr)\n",
    "print(\"***\"*5 + \"[MF]\" + \"***\"*5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MF-IPS] epoch:121, xent:144.3805215358734\n",
      "***************[MF-IPS]***************\n",
      "[MF-IPS] test mse: 0.2891346891790511\n",
      "[MF-IPS] test auc: 0.6849764800770826\n",
      "[MF] ndcg@5:0.586394, ndcg@10:0.668824\n",
      "Num User: 290\n",
      "Gini index: 0.3433973354231987\n",
      "Global utility: 0.4855172413793104\n",
      "***************[MF-IPS]***************\n"
     ]
    }
   ],
   "source": [
    "\"MF IPS\"\n",
    "mf_ips = MF_IPS(num_user, num_item)\n",
    "\n",
    "ips_idxs = np.arange(len(y_test))\n",
    "np.random.shuffle(ips_idxs)\n",
    "y_ips = y_test[ips_idxs[:int(0.05 * len(ips_idxs))]]\n",
    "\n",
    "mf_ips.fit(x_train, y_train,  y_ips=y_ips,\n",
    "    lr=0.01,\n",
    "    batch_size=128,\n",
    "    lamb=1e-4,\n",
    "    tol=1e-5,\n",
    "    verbose=False)\n",
    "test_pred = mf_ips.predict(x_test)\n",
    "mse_mfips = mse_func(y_test, test_pred)\n",
    "auc_mfips = roc_auc_score(y_test, test_pred)\n",
    "ndcg_res = ndcg_func(mf_ips, x_test, y_test)\n",
    "\n",
    "print(\"***\"*5 + \"[MF-IPS]\" + \"***\"*5)\n",
    "print(\"[MF-IPS] test mse:\", mse_func(y_test, test_pred))\n",
    "print(\"[MF-IPS] test auc:\", auc_mfips)\n",
    "print(\"[MF] ndcg@5:{:.6f}, ndcg@10:{:.6f}\".format(\n",
    "        np.mean(ndcg_res[\"ndcg_5\"]), np.mean(ndcg_res[\"ndcg_10\"])))\n",
    "user_wise_ctr = get_user_wise_ctr(x_test,y_test,test_pred)\n",
    "gi,gu = gini_index(user_wise_ctr)\n",
    "print(\"***\"*5 + \"[MF-IPS]\" + \"***\"*5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MF-SNIPS] epoch:108, xent:16.620826572179794\n",
      "***************[MF-SNIPS]***************\n",
      "[MF-SNIPS] test mse: 0.24541254818914784\n",
      "[MF-SNIPS] test auc: 0.6906621691532132\n",
      "[MF] ndcg@5:0.603736, ndcg@10:0.680151\n",
      "Num User: 290\n",
      "Gini index: 0.32868059814142303\n",
      "Global utility: 0.5041379310344828\n",
      "***************[MF-SNIPS]***************\n"
     ]
    }
   ],
   "source": [
    "\"MF-SNIPS\"\n",
    "mf_snips = MF_SNIPS(num_user, num_item)\n",
    "\n",
    "ips_idxs = np.arange(len(y_test))\n",
    "np.random.shuffle(ips_idxs)\n",
    "y_ips = y_test[ips_idxs[:int(0.05 * len(ips_idxs))]]\n",
    "\n",
    "mf_snips.fit(x_train, y_train,  y_ips=y_ips,\n",
    "    lr=0.01,\n",
    "    batch_size=128,\n",
    "    lamb=1e-4,\n",
    "    tol=1e-5,\n",
    "    verbose=False)\n",
    "test_pred = mf_snips.predict(x_test)\n",
    "mse_mfsnips = mse_func(y_test, test_pred)\n",
    "auc_mfsnips = roc_auc_score(y_test, test_pred)\n",
    "ndcg_res = ndcg_func(mf_snips, x_test, y_test)\n",
    "\n",
    "print(\"***\"*5 + \"[MF-SNIPS]\" + \"***\"*5)\n",
    "print(\"[MF-SNIPS] test mse:\", mse_mfsnips)\n",
    "print(\"[MF-SNIPS] test auc:\", auc_mfsnips)\n",
    "print(\"[MF] ndcg@5:{:.6f}, ndcg@10:{:.6f}\".format(\n",
    "        np.mean(ndcg_res[\"ndcg_5\"]), np.mean(ndcg_res[\"ndcg_10\"])))\n",
    "user_wise_ctr = get_user_wise_ctr(x_test,y_test,test_pred)\n",
    "gi,gu = gini_index(user_wise_ctr)\n",
    "print(\"***\"*5 + \"[MF-SNIPS]\" + \"***\"*5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MF-DR] epoch:45, xent:24672.74530029297\n",
      "***************[MF-DR]***************\n",
      "[MF-DR] test mse: 0.2747846467958318\n",
      "[MF-DR] test auc: 0.6872889374005826\n",
      "[MF] ndcg@5:0.609119, ndcg@10:0.676230\n",
      "Num User: 290\n",
      "Gini index: 0.33700959437636646\n",
      "Global utility: 0.5006896551724137\n",
      "***************[MF-DR]***************\n"
     ]
    }
   ],
   "source": [
    "\"MF DR\"\n",
    "mf_dr = MF_DR(num_user, num_item)\n",
    "\n",
    "ips_idxs = np.arange(len(y_test))\n",
    "np.random.shuffle(ips_idxs)\n",
    "y_ips = y_test[ips_idxs[:int(0.05 * len(ips_idxs))]]\n",
    "\n",
    "mf_dr.fit(x_train, y_train,  y_ips=y_ips,\n",
    "    lr=0.05,\n",
    "    batch_size=128,\n",
    "    lamb=1e-4,\n",
    "    tol=1e-5,\n",
    "    verbose=False)\n",
    "test_pred = mf_dr.predict(x_test)\n",
    "mse_mfdr = mse_func(y_test, test_pred)\n",
    "auc_mfdr = roc_auc_score(y_test, test_pred)\n",
    "ndcg_res = ndcg_func(mf_dr, x_test, y_test)\n",
    "\n",
    "print(\"***\"*5 + \"[MF-DR]\" + \"***\"*5)\n",
    "print(\"[MF-DR] test mse:\", mse_mfdr)\n",
    "print(\"[MF-DR] test auc:\", auc_mfdr)\n",
    "print(\"[MF] ndcg@5:{:.6f}, ndcg@10:{:.6f}\".format(\n",
    "        np.mean(ndcg_res[\"ndcg_5\"]), np.mean(ndcg_res[\"ndcg_10\"])))\n",
    "user_wise_ctr = get_user_wise_ctr(x_test,y_test,test_pred)\n",
    "gi,gu = gini_index(user_wise_ctr)\n",
    "print(\"***\"*5 + \"[MF-DR]\" + \"***\"*5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}