notebooks/06-statistics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functools\n",
    "from functools import partial\n",
    "import itertools\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "import pickle\n",
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pingouin as pg\n",
    "import seaborn as sns\n",
    "import yaml\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "sns.set_context(\"poster\")\n",
    "sns.set(rc={\"figure.figsize\": (16, 12.0)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import kendalltau, rankdata, spearmanr, pearsonr, ttest_rel\n",
    "import torch.nn.functional as F\n",
    "\n",
    "pd.set_option(\"display.max_rows\", 120)\n",
    "pd.set_option(\"display.max_columns\", 120)\n",
    "\n",
    "logging.basicConfig(level=logging.INFO, stream=sys.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lda4rec.datasets import Interactions, DataLoader, random_train_test_split, MOVIELENS_1M,MetaData, get_dataset, items_per_user_train_test_split\n",
    "import lda4rec.evaluations as lda_eval\n",
    "from lda4rec.estimators import MFEst, PopEst, LDA4RecEst, SNMFEst\n",
    "from lda4rec.utils import process_ids, cmp_ranks, Config, split_along_dim_apply, plot_cat\n",
    "from lda4rec import lda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyro\n",
    "import pyro.distributions as dist\n",
    "import pyro.optim as optim\n",
    "import torch\n",
    "from pyro.distributions import constraints\n",
    "from pyro.infer import SVI, Predictive, Trace_ELBO, TraceEnum_ELBO, config_enumerate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:neptune.internal.hardware.gpu.gpu_monitor:Info (NVML): NVML Shared Library Not Found. GPU usage metrics may not be reported. For more information, see https://docs-legacy.neptune.ai/logging-and-managing-experiment-results/logging-experiment-data.html#hardware-consumption \n",
      "offline/817d0465-9ab1-4c54-9245-7b263d3083e3\n",
      "Remember to stop your run once you’ve finished logging your metadata (https://docs.neptune.ai/api-reference/run#stop). It will be stopped automatically only when the notebook kernel/interactive console is terminated.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<neptune.new.run.Run at 0x7f9dcc3a4790>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import neptune.new as neptune\n",
    "# init dummy neptune to avoid problems with logging\n",
    "neptune.init(mode='offline') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "from icecream import ic, install\n",
    "\n",
    "install()\n",
    "# configure icecream\n",
    "def ic_str(obj):\n",
    "    if hasattr(obj, \"shape\"):\n",
    "        return f\"{obj} \"  #\n",
    "    else:\n",
    "        return str(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "ic.configureOutput(argToStringFunction=ic_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfg = Config(Path('../configs/exp_2.yaml')) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'goodbooks'"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cfg['experiment']['dataset']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "for cfg in lda_eval.get_cfgs_from_path(Path('../configs')):\n",
    "    data, train, test, data_rng = lda_eval.get_train_test_data(cfg)\n",
    "    est = lda_eval.load_model(cfg, train)\n",
    "    v, t, h, b = est.get_lda_params()\n",
    "    med = np.median((1/t).flatten())\n",
    "    results.append(dict(dataset=cfg['experiment']['dataset'], rnd=cfg['experiment']['model_seed'], median=med))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rnd</th>\n",
       "      <th>median</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>amazon</th>\n",
       "      <td>18517721.4</td>\n",
       "      <td>0.010686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>goodbooks</th>\n",
       "      <td>18517721.4</td>\n",
       "      <td>0.003384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movielens-1m</th>\n",
       "      <td>18517721.4</td>\n",
       "      <td>0.016350</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     rnd    median\n",
       "dataset                           \n",
       "amazon        18517721.4  0.010686\n",
       "goodbooks     18517721.4  0.003384\n",
       "movielens-1m  18517721.4  0.016350"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(results)\n",
    "df.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'goodbooks', 'rnd': 1729, 'median': 0.003220169},\n",
       " {'dataset': 'goodbooks', 'rnd': 234829, 'median': 0.0037773054},\n",
       " {'dataset': 'movielens-1m', 'rnd': 234829, 'median': 0.016483523},\n",
       " {'dataset': 'amazon', 'rnd': 1981, 'median': 0.011081425},\n",
       " {'dataset': 'goodbooks', 'rnd': 666, 'median': 0.0033622752},\n",
       " {'dataset': 'movielens-1m', 'rnd': 1981, 'median': 0.0152998455},\n",
       " {'dataset': 'goodbooks', 'rnd': 1981, 'median': 0.0034285225},\n",
       " {'dataset': 'amazon', 'rnd': 234829, 'median': 0.01030039},\n",
       " {'dataset': 'amazon', 'rnd': 666, 'median': 0.010790214},\n",
       " {'dataset': 'movielens-1m', 'rnd': 666, 'median': 0.017283749},\n",
       " {'dataset': 'goodbooks', 'rnd': 92349402, 'median': 0.0031326886},\n",
       " {'dataset': 'amazon', 'rnd': 1729, 'median': 0.010667719},\n",
       " {'dataset': 'movielens-1m', 'rnd': 92349402, 'median': 0.016401894},\n",
       " {'dataset': 'amazon', 'rnd': 92349402, 'median': 0.010588221},\n",
       " {'dataset': 'movielens-1m', 'rnd': 1729, 'median': 0.01627946}]"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Assembling all the statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results(path):\n",
    "    for res_file in path.glob(\"result_exp_*.pickle\"):\n",
    "        with open(res_file, 'rb') as fh:\n",
    "            res = pickle.load(fh)\n",
    "        yield res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = list(get_results(Path('../evaluations/')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp1_df = pd.DataFrame([dict(dataset=res['experiment']['dataset'],\n",
    "                             dim=res['experiment']['est_params']['embedding_dim'],\n",
    "                             train_p=res['result']['ttest_cohort_user_interaction_train']['p-val'].loc['T-test'],\n",
    "                             train_d=res['result']['ttest_cohort_user_interaction_train']['cohen-d'].loc['T-test'],\n",
    "                             test_p=res['result']['ttest_cohort_user_interaction_test']['p-val'].loc['T-test'],\n",
    "                             test_d=res['result']['ttest_cohort_user_interaction_test']['cohen-d'].loc['T-test'])\n",
    "                        for res in results])\n",
    "exp1_df = exp1_df.groupby(['dataset', 'dim']).agg([np.mean, np.std])\n",
    "#exp1_df.loc['goodbooks'] = exp1_df.loc['goodbooks'] / (2*calc_pairs(GB_DIM))\n",
    "#exp1_df.loc['movielens-1m'] = exp1_df.loc['movielens-1m'] / (2*calc_pairs(ML_DIM))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">train_p</th>\n",
       "      <th colspan=\"2\" halign=\"left\">train_d</th>\n",
       "      <th colspan=\"2\" halign=\"left\">test_p</th>\n",
       "      <th colspan=\"2\" halign=\"left\">test_d</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>dim</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>amazon</th>\n",
       "      <th>256</th>\n",
       "      <td>7.739793e-41</td>\n",
       "      <td>8.916809e-41</td>\n",
       "      <td>0.124953</td>\n",
       "      <td>0.004170</td>\n",
       "      <td>5.608193e-10</td>\n",
       "      <td>1.130118e-09</td>\n",
       "      <td>0.059283</td>\n",
       "      <td>0.003940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>goodbooks</th>\n",
       "      <th>256</th>\n",
       "      <td>3.321596e-161</td>\n",
       "      <td>7.425478e-161</td>\n",
       "      <td>0.178114</td>\n",
       "      <td>0.010057</td>\n",
       "      <td>1.010080e-11</td>\n",
       "      <td>2.245296e-11</td>\n",
       "      <td>0.046596</td>\n",
       "      <td>0.004882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movielens-1m</th>\n",
       "      <th>64</th>\n",
       "      <td>6.663359e-213</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.608852</td>\n",
       "      <td>0.022047</td>\n",
       "      <td>2.440486e-15</td>\n",
       "      <td>5.454175e-15</td>\n",
       "      <td>0.162705</td>\n",
       "      <td>0.017907</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        train_p                  train_d            \\\n",
       "                           mean            std      mean       std   \n",
       "dataset      dim                                                     \n",
       "amazon       256   7.739793e-41   8.916809e-41  0.124953  0.004170   \n",
       "goodbooks    256  3.321596e-161  7.425478e-161  0.178114  0.010057   \n",
       "movielens-1m 64   6.663359e-213   0.000000e+00  0.608852  0.022047   \n",
       "\n",
       "                        test_p                  test_d            \n",
       "                          mean           std      mean       std  \n",
       "dataset      dim                                                  \n",
       "amazon       256  5.608193e-10  1.130118e-09  0.059283  0.003940  \n",
       "goodbooks    256  1.010080e-11  2.245296e-11  0.046596  0.004882  \n",
       "movielens-1m 64   2.440486e-15  5.454175e-15  0.162705  0.017907  "
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp1_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp2_df = pd.DataFrame([dict(dataset=res['experiment']['dataset'], \n",
    "                             dim=res['experiment']['est_params']['embedding_dim'],\n",
    "                             corr_pop_train=res['result']['corr_popularity_train'][0],\n",
    "                             p_val_train=res['result']['corr_popularity_train'][1],\n",
    "                             corr_pop_data=res['result']['corr_popularity_data'][0],\n",
    "                             p_val_data=res['result']['corr_popularity_data'][1])\n",
    "                        for res in results])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp2_df = exp2_df.groupby(['dataset', 'dim']).agg([np.mean, np.std])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">corr_pop_train</th>\n",
       "      <th colspan=\"2\" halign=\"left\">p_val_train</th>\n",
       "      <th colspan=\"2\" halign=\"left\">corr_pop_data</th>\n",
       "      <th colspan=\"2\" halign=\"left\">p_val_data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>dim</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>amazon</th>\n",
       "      <th>256</th>\n",
       "      <td>0.377344</td>\n",
       "      <td>0.003264</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.372699</td>\n",
       "      <td>0.003339</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>goodbooks</th>\n",
       "      <th>256</th>\n",
       "      <td>0.264629</td>\n",
       "      <td>0.004193</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.267341</td>\n",
       "      <td>0.004364</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movielens-1m</th>\n",
       "      <th>64</th>\n",
       "      <td>0.519759</td>\n",
       "      <td>0.005958</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.520511</td>\n",
       "      <td>0.005910</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 corr_pop_train           p_val_train      corr_pop_data  \\\n",
       "                           mean       std        mean  std          mean   \n",
       "dataset      dim                                                           \n",
       "amazon       256       0.377344  0.003264         0.0  0.0      0.372699   \n",
       "goodbooks    256       0.264629  0.004193         0.0  0.0      0.267341   \n",
       "movielens-1m 64        0.519759  0.005958         0.0  0.0      0.520511   \n",
       "\n",
       "                           p_val_data       \n",
       "                       std       mean  std  \n",
       "dataset      dim                            \n",
       "amazon       256  0.003339        0.0  0.0  \n",
       "goodbooks    256  0.004364        0.0  0.0  \n",
       "movielens-1m 64   0.005910        0.0  0.0  "
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp2_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp3_df = pd.DataFrame([dict(dataset=res['experiment']['dataset'], \n",
    "                             dim=res['experiment']['est_params']['embedding_dim'],\n",
    "                             corr_conf_pop_train=res['result']['corr_conformity_pop_train'][0],\n",
    "                             p_val_conf_pop_train=res['result']['corr_conformity_pop_train'][1],\n",
    "                            corr_conf_pop_data=res['result']['corr_conformity_pop_data'][0],\n",
    "                             p_val_conf_pop_data=res['result']['corr_conformity_pop_data'][1],\n",
    "                            corr_conf_pop_data_all=res['result']['corr_conformity_pop_data_all'][0],\n",
    "                             p_val_conf_pop_data_all=res['result']['corr_conformity_pop_data_all'][1],\n",
    "                             corr_conf_b=res['result']['corr_conformity_b'][0],\n",
    "                             p_val_conf_b=res['result']['corr_conformity_b'][1],)\n",
    "                        for res in results])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp3_df = exp3_df.groupby(['dataset', 'dim']).agg([np.mean, np.std])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">corr_conf_pop_train</th>\n",
       "      <th colspan=\"2\" halign=\"left\">p_val_conf_pop_train</th>\n",
       "      <th colspan=\"2\" halign=\"left\">corr_conf_pop_data</th>\n",
       "      <th colspan=\"2\" halign=\"left\">p_val_conf_pop_data</th>\n",
       "      <th colspan=\"2\" halign=\"left\">corr_conf_pop_data_all</th>\n",
       "      <th colspan=\"2\" halign=\"left\">p_val_conf_pop_data_all</th>\n",
       "      <th colspan=\"2\" halign=\"left\">corr_conf_b</th>\n",
       "      <th colspan=\"2\" halign=\"left\">p_val_conf_b</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>dim</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>amazon</th>\n",
       "      <th>256</th>\n",
       "      <td>0.081838</td>\n",
       "      <td>0.002615</td>\n",
       "      <td>2.916868e-73</td>\n",
       "      <td>6.522311e-73</td>\n",
       "      <td>0.091750</td>\n",
       "      <td>0.002602</td>\n",
       "      <td>1.663145e-92</td>\n",
       "      <td>3.718905e-92</td>\n",
       "      <td>0.091463</td>\n",
       "      <td>0.002512</td>\n",
       "      <td>1.850760e-92</td>\n",
       "      <td>4.138412e-92</td>\n",
       "      <td>-0.004453</td>\n",
       "      <td>0.005992</td>\n",
       "      <td>1.554960e-01</td>\n",
       "      <td>1.111974e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>goodbooks</th>\n",
       "      <th>256</th>\n",
       "      <td>-0.001633</td>\n",
       "      <td>0.001303</td>\n",
       "      <td>6.055143e-01</td>\n",
       "      <td>2.563741e-01</td>\n",
       "      <td>-0.000374</td>\n",
       "      <td>0.001304</td>\n",
       "      <td>7.789111e-01</td>\n",
       "      <td>2.305083e-01</td>\n",
       "      <td>-0.001573</td>\n",
       "      <td>0.001312</td>\n",
       "      <td>6.198387e-01</td>\n",
       "      <td>2.608295e-01</td>\n",
       "      <td>-0.044962</td>\n",
       "      <td>0.002004</td>\n",
       "      <td>1.608840e-48</td>\n",
       "      <td>3.597471e-48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movielens-1m</th>\n",
       "      <th>64</th>\n",
       "      <td>0.374866</td>\n",
       "      <td>0.008969</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.383584</td>\n",
       "      <td>0.009038</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.394357</td>\n",
       "      <td>0.009267</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.523747</td>\n",
       "      <td>0.009208</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 corr_conf_pop_train           p_val_conf_pop_train  \\\n",
       "                                mean       std                 mean   \n",
       "dataset      dim                                                      \n",
       "amazon       256            0.081838  0.002615         2.916868e-73   \n",
       "goodbooks    256           -0.001633  0.001303         6.055143e-01   \n",
       "movielens-1m 64             0.374866  0.008969         0.000000e+00   \n",
       "\n",
       "                               corr_conf_pop_data            \\\n",
       "                           std               mean       std   \n",
       "dataset      dim                                              \n",
       "amazon       256  6.522311e-73           0.091750  0.002602   \n",
       "goodbooks    256  2.563741e-01          -0.000374  0.001304   \n",
       "movielens-1m 64   0.000000e+00           0.383584  0.009038   \n",
       "\n",
       "                 p_val_conf_pop_data               corr_conf_pop_data_all  \\\n",
       "                                mean           std                   mean   \n",
       "dataset      dim                                                            \n",
       "amazon       256        1.663145e-92  3.718905e-92               0.091463   \n",
       "goodbooks    256        7.789111e-01  2.305083e-01              -0.001573   \n",
       "movielens-1m 64         0.000000e+00  0.000000e+00               0.394357   \n",
       "\n",
       "                           p_val_conf_pop_data_all               corr_conf_b  \\\n",
       "                       std                    mean           std        mean   \n",
       "dataset      dim                                                               \n",
       "amazon       256  0.002512            1.850760e-92  4.138412e-92   -0.004453   \n",
       "goodbooks    256  0.001312            6.198387e-01  2.608295e-01   -0.044962   \n",
       "movielens-1m 64   0.009267            0.000000e+00  0.000000e+00    0.523747   \n",
       "\n",
       "                            p_val_conf_b                \n",
       "                       std          mean           std  \n",
       "dataset      dim                                        \n",
       "amazon       256  0.005992  1.554960e-01  1.111974e-01  \n",
       "goodbooks    256  0.002004  1.608840e-48  3.597471e-48  \n",
       "movielens-1m 64   0.009208  0.000000e+00  0.000000e+00  "
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp3_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp4_df = pd.DataFrame([dict(dataset=res['experiment']['dataset'], \n",
    "                             dim=res['experiment']['est_params']['embedding_dim'],\n",
    "                             good_bad_train_p=res['result']['ttest_user_interaction_good_bad_train']['p-val'].loc['T-test'],\n",
    "                             good_bad_train_d=res['result']['ttest_user_interaction_good_bad_train']['cohen-d'].loc['T-test'],\n",
    "                             good_bad_test_p=res['result']['ttest_user_interaction_good_bad_test']['p-val'].loc['T-test'],\n",
    "                             good_bad_test_d=res['result']['ttest_user_interaction_good_bad_test']['cohen-d'].loc['T-test'],\n",
    "                             good_rnd_train_p=res['result']['ttest_user_interaction_good_rnd_train']['p-val'].loc['T-test'],\n",
    "                             good_rnd_train_d=res['result']['ttest_user_interaction_good_rnd_train']['cohen-d'].loc['T-test'],\n",
    "                             good_rnd_test_p=res['result']['ttest_user_interaction_good_rnd_test']['p-val'].loc['T-test'],\n",
    "                             good_rnd_test_d=res['result']['ttest_user_interaction_good_rnd_test']['cohen-d'].loc['T-test'])\n",
    "                        for res in results])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp4_df = exp4_df.groupby(['dataset', 'dim']).agg([np.mean, np.std])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">good_bad_train_p</th>\n",
       "      <th colspan=\"2\" halign=\"left\">good_bad_train_d</th>\n",
       "      <th colspan=\"2\" halign=\"left\">good_bad_test_p</th>\n",
       "      <th colspan=\"2\" halign=\"left\">good_bad_test_d</th>\n",
       "      <th colspan=\"2\" halign=\"left\">good_rnd_train_p</th>\n",
       "      <th colspan=\"2\" halign=\"left\">good_rnd_train_d</th>\n",
       "      <th colspan=\"2\" halign=\"left\">good_rnd_test_p</th>\n",
       "      <th colspan=\"2\" halign=\"left\">good_rnd_test_d</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>dim</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>amazon</th>\n",
       "      <th>256</th>\n",
       "      <td>3.332443e-312</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.478724</td>\n",
       "      <td>0.027948</td>\n",
       "      <td>7.705007e-130</td>\n",
       "      <td>1.722892e-129</td>\n",
       "      <td>0.862463</td>\n",
       "      <td>0.029791</td>\n",
       "      <td>1.306924e-299</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.435556</td>\n",
       "      <td>0.027294</td>\n",
       "      <td>1.353733e-124</td>\n",
       "      <td>3.027038e-124</td>\n",
       "      <td>0.842361</td>\n",
       "      <td>0.028882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>goodbooks</th>\n",
       "      <th>256</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.884263</td>\n",
       "      <td>0.058301</td>\n",
       "      <td>2.748834e-68</td>\n",
       "      <td>6.146576e-68</td>\n",
       "      <td>0.618658</td>\n",
       "      <td>0.036337</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.118543</td>\n",
       "      <td>0.041310</td>\n",
       "      <td>6.762792e-38</td>\n",
       "      <td>1.372516e-37</td>\n",
       "      <td>0.434077</td>\n",
       "      <td>0.022393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movielens-1m</th>\n",
       "      <th>64</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.881296</td>\n",
       "      <td>0.030967</td>\n",
       "      <td>6.777133e-71</td>\n",
       "      <td>1.370053e-70</td>\n",
       "      <td>0.607651</td>\n",
       "      <td>0.021897</td>\n",
       "      <td>2.630566e-255</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.172604</td>\n",
       "      <td>0.012945</td>\n",
       "      <td>4.412468e-43</td>\n",
       "      <td>7.926921e-43</td>\n",
       "      <td>0.449772</td>\n",
       "      <td>0.016349</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 good_bad_train_p      good_bad_train_d            \\\n",
       "                             mean  std             mean       std   \n",
       "dataset      dim                                                    \n",
       "amazon       256    3.332443e-312  0.0         1.478724  0.027948   \n",
       "goodbooks    256     0.000000e+00  0.0         2.884263  0.058301   \n",
       "movielens-1m 64      0.000000e+00  0.0         1.881296  0.030967   \n",
       "\n",
       "                 good_bad_test_p                good_bad_test_d            \\\n",
       "                            mean            std            mean       std   \n",
       "dataset      dim                                                            \n",
       "amazon       256   7.705007e-130  1.722892e-129        0.862463  0.029791   \n",
       "goodbooks    256    2.748834e-68   6.146576e-68        0.618658  0.036337   \n",
       "movielens-1m 64     6.777133e-71   1.370053e-70        0.607651  0.021897   \n",
       "\n",
       "                 good_rnd_train_p      good_rnd_train_d            \\\n",
       "                             mean  std             mean       std   \n",
       "dataset      dim                                                    \n",
       "amazon       256    1.306924e-299  0.0         1.435556  0.027294   \n",
       "goodbooks    256     0.000000e+00  0.0         2.118543  0.041310   \n",
       "movielens-1m 64     2.630566e-255  0.0         1.172604  0.012945   \n",
       "\n",
       "                 good_rnd_test_p                good_rnd_test_d            \n",
       "                            mean            std            mean       std  \n",
       "dataset      dim                                                           \n",
       "amazon       256   1.353733e-124  3.027038e-124        0.842361  0.028882  \n",
       "goodbooks    256    6.762792e-38   1.372516e-37        0.434077  0.022393  \n",
       "movielens-1m 64     4.412468e-43   7.926921e-43        0.449772  0.016349  "
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp4_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}