RecommendationSystem/Autoencoder/EASE/EASE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install bottleneck"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import bottleneck as bn"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset\n",
    "\n",
    "This dataset is formed by a scraper using solved.ac API.  \n",
    "If necessary, you can use [this scraper](https://github.com/Glanceyes/RECJOON/blob/main/airflow/dags/scrapers/problem_solved_scraper.py) in RECJOON project repository."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "DATA_DIR = '/content' # Write your own directory\n",
    "RANDOM_SEED = 98765\n",
    "FOLD_SIZE = 3000\n",
    "NUM_TOP_PROBLEMS = 10\n",
    "np.random.seed(98765)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>handle</th>\n",
       "      <th>problems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>koosaga</td>\n",
       "      <td>1000,1001,1002,1003,1004,1005,1006,1007,1008,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>cki86201</td>\n",
       "      <td>1000,1001,1002,1003,1004,1005,1006,1007,1008,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>mitnegativeinfinity</td>\n",
       "      <td>1000,1001,1019,1056,1067,1144,1150,1311,1372,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>ainta</td>\n",
       "      <td>1000,1001,1002,1003,1004,1005,1007,1008,1009,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>yclock</td>\n",
       "      <td>1000,1001,1002,1003,1004,1005,1007,1008,1009,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63871</th>\n",
       "      <td>63872</td>\n",
       "      <td>ohjcms1</td>\n",
       "      <td>2557,10718,15596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63872</th>\n",
       "      <td>63873</td>\n",
       "      <td>steven1010</td>\n",
       "      <td>2557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63873</th>\n",
       "      <td>63874</td>\n",
       "      <td>wwowwww</td>\n",
       "      <td>1000,1001,1008,1330,2438,2439,2557,2588,2739,2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63874</th>\n",
       "      <td>63875</td>\n",
       "      <td>yijh061016</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63875</th>\n",
       "      <td>63876</td>\n",
       "      <td>yoonkoo2001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>63876 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id               handle  \\\n",
       "0          1              koosaga   \n",
       "1          2             cki86201   \n",
       "2          3  mitnegativeinfinity   \n",
       "3          4                ainta   \n",
       "4          5               yclock   \n",
       "...      ...                  ...   \n",
       "63871  63872              ohjcms1   \n",
       "63872  63873           steven1010   \n",
       "63873  63874              wwowwww   \n",
       "63874  63875           yijh061016   \n",
       "63875  63876          yoonkoo2001   \n",
       "\n",
       "                                                problems  \n",
       "0      1000,1001,1002,1003,1004,1005,1006,1007,1008,1...  \n",
       "1      1000,1001,1002,1003,1004,1005,1006,1007,1008,1...  \n",
       "2      1000,1001,1019,1056,1067,1144,1150,1311,1372,1...  \n",
       "3      1000,1001,1002,1003,1004,1005,1007,1008,1009,1...  \n",
       "4      1000,1001,1002,1003,1004,1005,1007,1008,1009,1...  \n",
       "...                                                  ...  \n",
       "63871                                   2557,10718,15596  \n",
       "63872                                               2557  \n",
       "63873  1000,1001,1008,1330,2438,2439,2557,2588,2739,2...  \n",
       "63874                                                NaN  \n",
       "63875                                                NaN  \n",
       "\n",
       "[63876 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data = pd.read_csv(os.path.join(DATA_DIR, 'problems_solved.csv'), header=0)\n",
    "raw_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>problems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1000,1001,1002,1003,1004,1005,1006,1007,1008,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cki86201</td>\n",
       "      <td>1000,1001,1002,1003,1004,1005,1006,1007,1008,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mitnegativeinfinity</td>\n",
       "      <td>1000,1001,1019,1056,1067,1144,1150,1311,1372,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ainta</td>\n",
       "      <td>1000,1001,1002,1003,1004,1005,1007,1008,1009,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>yclock</td>\n",
       "      <td>1000,1001,1002,1003,1004,1005,1007,1008,1009,1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63871</th>\n",
       "      <td>ohjcms1</td>\n",
       "      <td>2557,10718,15596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63872</th>\n",
       "      <td>steven1010</td>\n",
       "      <td>2557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63873</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>1000,1001,1008,1330,2438,2439,2557,2588,2739,2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63874</th>\n",
       "      <td>yijh061016</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63875</th>\n",
       "      <td>yoonkoo2001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>63876 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    handle                                           problems\n",
       "0                  koosaga  1000,1001,1002,1003,1004,1005,1006,1007,1008,1...\n",
       "1                 cki86201  1000,1001,1002,1003,1004,1005,1006,1007,1008,1...\n",
       "2      mitnegativeinfinity  1000,1001,1019,1056,1067,1144,1150,1311,1372,1...\n",
       "3                    ainta  1000,1001,1002,1003,1004,1005,1007,1008,1009,1...\n",
       "4                   yclock  1000,1001,1002,1003,1004,1005,1007,1008,1009,1...\n",
       "...                    ...                                                ...\n",
       "63871              ohjcms1                                   2557,10718,15596\n",
       "63872           steven1010                                               2557\n",
       "63873              wwowwww  1000,1001,1008,1330,2438,2439,2557,2588,2739,2...\n",
       "63874           yijh061016                                                NaN\n",
       "63875          yoonkoo2001                                                NaN\n",
       "\n",
       "[63876 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user_problems = raw_data[['handle', 'problems']]\n",
    "df_user_problems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_48090/2504318994.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_user_problems.problems = df_user_problems.problems.str.split(',')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>problems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>[1000, 1001, 1002, 1003, 1004, 1005, 1006, 100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cki86201</td>\n",
       "      <td>[1000, 1001, 1002, 1003, 1004, 1005, 1006, 100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mitnegativeinfinity</td>\n",
       "      <td>[1000, 1001, 1019, 1056, 1067, 1144, 1150, 131...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ainta</td>\n",
       "      <td>[1000, 1001, 1002, 1003, 1004, 1005, 1007, 100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>yclock</td>\n",
       "      <td>[1000, 1001, 1002, 1003, 1004, 1005, 1007, 100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63871</th>\n",
       "      <td>ohjcms1</td>\n",
       "      <td>[2557, 10718, 15596]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63872</th>\n",
       "      <td>steven1010</td>\n",
       "      <td>[2557]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63873</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>[1000, 1001, 1008, 1330, 2438, 2439, 2557, 258...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63874</th>\n",
       "      <td>yijh061016</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63875</th>\n",
       "      <td>yoonkoo2001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>63876 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    handle                                           problems\n",
       "0                  koosaga  [1000, 1001, 1002, 1003, 1004, 1005, 1006, 100...\n",
       "1                 cki86201  [1000, 1001, 1002, 1003, 1004, 1005, 1006, 100...\n",
       "2      mitnegativeinfinity  [1000, 1001, 1019, 1056, 1067, 1144, 1150, 131...\n",
       "3                    ainta  [1000, 1001, 1002, 1003, 1004, 1005, 1007, 100...\n",
       "4                   yclock  [1000, 1001, 1002, 1003, 1004, 1005, 1007, 100...\n",
       "...                    ...                                                ...\n",
       "63871              ohjcms1                               [2557, 10718, 15596]\n",
       "63872           steven1010                                             [2557]\n",
       "63873              wwowwww  [1000, 1001, 1008, 1330, 2438, 2439, 2557, 258...\n",
       "63874           yijh061016                                                NaN\n",
       "63875          yoonkoo2001                                                NaN\n",
       "\n",
       "[63876 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user_problems.problems = df_user_problems.problems.str.split(',')\n",
    "df_user_problems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>problems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509347</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>10998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509348</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>11021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509349</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>11022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509350</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>14681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509351</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>15552</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8509032 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          handle problems\n",
       "0        koosaga     1000\n",
       "1        koosaga     1001\n",
       "2        koosaga     1002\n",
       "3        koosaga     1003\n",
       "4        koosaga     1004\n",
       "...          ...      ...\n",
       "8509347  wwowwww    10998\n",
       "8509348  wwowwww    11021\n",
       "8509349  wwowwww    11022\n",
       "8509350  wwowwww    14681\n",
       "8509351  wwowwww    15552\n",
       "\n",
       "[8509032 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user_problems = df_user_problems.explode('problems').reset_index(drop=True)\n",
    "df_user_problems = df_user_problems.dropna(axis=0)\n",
    "df_user_problems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>problems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509347</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>10998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509348</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>11021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509349</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>11022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509350</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>14681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509351</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>15552</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8509032 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          handle  problems\n",
       "0        koosaga      1000\n",
       "1        koosaga      1001\n",
       "2        koosaga      1002\n",
       "3        koosaga      1003\n",
       "4        koosaga      1004\n",
       "...          ...       ...\n",
       "8509347  wwowwww     10998\n",
       "8509348  wwowwww     11021\n",
       "8509349  wwowwww     11022\n",
       "8509350  wwowwww     14681\n",
       "8509351  wwowwww     15552\n",
       "\n",
       "[8509032 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user_problems = df_user_problems.astype({'handle':'str', 'problems': np.uint16})\n",
    "df_user_problems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_interaction_count(df, id):\n",
    "    interaction_count_groupby_id = df[[id]].groupby(id, as_index=False)\n",
    "    grouped_count = interaction_count_groupby_id.size()\n",
    "    return grouped_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def filter_data(df, min_user_interaction, min_problem_interaction):\n",
    "    user_interaction_count = get_interaction_count(df, 'handle')\n",
    "    problem_interaction_count = get_interaction_count(df, 'problems')\n",
    "\n",
    "    print(f\"Size of Dataframe Before Filtering: {df.size}\")\n",
    "\n",
    "    if min_user_interaction > 0:\n",
    "        df = df[df['handle'].isin(user_interaction_count[user_interaction_count['size'] >= min_user_interaction]['handle'])]\n",
    "\n",
    "    print(f\"Size of Dataframe After User Filtering: {df.size}\")\n",
    "\n",
    "    if min_problem_interaction > 0:\n",
    "        df = df[df['problems'].isin(problem_interaction_count[problem_interaction_count['size'] >= min_problem_interaction]['problems'])]\n",
    "\n",
    "    print(f\"Size of Dataframe After Problem Filtering: {df.size}\")\n",
    "\n",
    "    return df, user_interaction_count, problem_interaction_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of Dataframe Before Filtering: 17018064\n",
      "Size of Dataframe After User Filtering: 17005344\n",
      "Size of Dataframe After Problem Filtering: 16947992\n"
     ]
    }
   ],
   "source": [
    "df_user_problems_filtered, user_activity, problem_popularity = filter_data(df_user_problems, 5, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>problems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>koosaga</td>\n",
       "      <td>1004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509347</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>10998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509348</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>11021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509349</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>11022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509350</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>14681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509351</th>\n",
       "      <td>wwowwww</td>\n",
       "      <td>15552</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8473996 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          handle  problems\n",
       "0        koosaga      1000\n",
       "1        koosaga      1001\n",
       "2        koosaga      1002\n",
       "3        koosaga      1003\n",
       "4        koosaga      1004\n",
       "...          ...       ...\n",
       "8509347  wwowwww     10998\n",
       "8509348  wwowwww     11021\n",
       "8509349  wwowwww     11022\n",
       "8509350  wwowwww     14681\n",
       "8509351  wwowwww     15552\n",
       "\n",
       "[8473996 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user_problems_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000000000</td>\n",
       "      <td>734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000064</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000sb</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0004poppop</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000ian</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63549</th>\n",
       "      <td>zzzzlll</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63550</th>\n",
       "      <td>zzzzong123</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63551</th>\n",
       "      <td>zzzzz9887</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63552</th>\n",
       "      <td>zzzzzzzz</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63553</th>\n",
       "      <td>zzzzzzzzu2</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>63554 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           handle  size\n",
       "0      0000000000   734\n",
       "1         0000064   136\n",
       "2          0000sb    10\n",
       "3      0004poppop   145\n",
       "4          000ian     1\n",
       "...           ...   ...\n",
       "63549     zzzzlll     6\n",
       "63550  zzzzong123     6\n",
       "63551   zzzzz9887    27\n",
       "63552    zzzzzzzz    67\n",
       "63553  zzzzzzzzu2    45\n",
       "\n",
       "[63554 rows x 2 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_activity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>problems</th>\n",
       "      <th>size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000</td>\n",
       "      <td>52278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001</td>\n",
       "      <td>47009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1002</td>\n",
       "      <td>16183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1003</td>\n",
       "      <td>22466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1004</td>\n",
       "      <td>4686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19716</th>\n",
       "      <td>25070</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19717</th>\n",
       "      <td>25071</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19718</th>\n",
       "      <td>25074</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19719</th>\n",
       "      <td>25075</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19720</th>\n",
       "      <td>25083</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19721 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       problems   size\n",
       "0          1000  52278\n",
       "1          1001  47009\n",
       "2          1002  16183\n",
       "3          1003  22466\n",
       "4          1004   4686\n",
       "...         ...    ...\n",
       "19716     25070      1\n",
       "19717     25071      1\n",
       "19718     25074      1\n",
       "19719     25075      1\n",
       "19720     25083     92\n",
       "\n",
       "[19721 rows x 2 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "problem_popularity"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# EASE(Embarrassingly Shallow Autoencoders for Sparse Data\n",
    "\n",
    "The EASE model does not require validation, so it does not divide the dataset into training and validation datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before shuffling\n",
      " ['koosaga' 'cki86201' 'mitnegativeinfinity' ... 'hihimotga' 'number'\n",
      " 'wwowwww']\n",
      "\n",
      "After shuffling\n",
      " ['goonbamm' 'junqwoopark' 'skwlghddlek' ... 'ogisoo' 'doublej20'\n",
      " 'afternoon']\n"
     ]
    }
   ],
   "source": [
    "unique_user_id = pd.unique(df_user_problems_filtered['handle'])\n",
    "index_permutation = np.random.permutation(unique_user_id.size)\n",
    "unique_user_id_shuffled = unique_user_id[index_permutation]\n",
    "print(f\"Before shuffling\\n {unique_user_id}\\n\")\n",
    "print(f\"After shuffling\\n {unique_user_id_shuffled}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1000 1001 1002 ... 6677 2071 1845]\n"
     ]
    }
   ],
   "source": [
    "unique_problem_id = pd.unique(df_user_problems_filtered['problems'])\n",
    "print(unique_problem_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "user_to_id = dict((str(user_id), int(id)) for (id, user_id) in enumerate(unique_user_id))\n",
    "problem_to_id = dict((int(problem_id), int(id)) for (id, problem_id) in enumerate(unique_problem_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "id_to_user = dict((int(id), str(user_id)) for user_id, id in user_to_id.items())\n",
    "id_to_problem = dict((int(id), int(problem_id)) for problem_id, id in problem_to_id.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if not os.path.exists(DATA_DIR):\n",
    "    os.makedirs(DATA_DIR)\n",
    "\n",
    "with open(os.path.join(DATA_DIR, 'user_to_id.json'), 'w', encoding=\"utf-8\") as f:\n",
    "    json.dump(user_to_id, f, ensure_ascii=False, indent=\"\\t\")\n",
    "\n",
    "with open(os.path.join(DATA_DIR, 'problem_to_id.json'), 'w', encoding=\"utf-8\") as f:\n",
    "    json.dump(problem_to_id, f, ensure_ascii=False, indent=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Users: 61124\n",
      "Number of Fold Size: 3000\n"
     ]
    }
   ],
   "source": [
    "num_users = unique_user_id_shuffled.size\n",
    "num_heldout_users = FOLD_SIZE\n",
    "print(f\"Number of Users: {num_users}\")\n",
    "print(f\"Number of Fold Size: {num_heldout_users}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def numerize(df, user_to_id, problem_to_id):\n",
    "    user_id = df['handle'].apply(lambda x: user_to_id[str(x)])\n",
    "    problem_id = df['problems'].apply(lambda x: problem_to_id[str(x)])\n",
    "\n",
    "    return pd.DataFrame(data={'handle': user_id, 'problem': problem_id}, columns=['handle', 'problem'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def denumerize(df, id_to_user, id_to_problem):\n",
    "    user_id = df['user_id'].apply(lambda x: id_to_user[x])\n",
    "    problem_id = df['problem_id'].apply(lambda x: id_to_problem[x])\n",
    "\n",
    "    return pd.DataFrame(data={'handle': user_id, 'problem': problem_id}, columns=['handle', 'problem'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DIR, 'user_to_id.json'), 'r', encoding=\"utf-8\") as f:\n",
    "    user_to_id = json.load(f)\n",
    "\n",
    "with open(os.path.join(DATA_DIR, 'problem_to_id.json'), 'r', encoding=\"utf-8\") as f:\n",
    "    problem_to_id = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>problem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509347</th>\n",
       "      <td>61123</td>\n",
       "      <td>3646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509348</th>\n",
       "      <td>61123</td>\n",
       "      <td>3661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509349</th>\n",
       "      <td>61123</td>\n",
       "      <td>3662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509350</th>\n",
       "      <td>61123</td>\n",
       "      <td>5307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509351</th>\n",
       "      <td>61123</td>\n",
       "      <td>5729</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8473996 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         handle  problem\n",
       "0             0        0\n",
       "1             0        1\n",
       "2             0        2\n",
       "3             0        3\n",
       "4             0        4\n",
       "...         ...      ...\n",
       "8509347   61123     3646\n",
       "8509348   61123     3661\n",
       "8509349   61123     3662\n",
       "8509350   61123     5307\n",
       "8509351   61123     5729\n",
       "\n",
       "[8473996 rows x 2 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_for_ease = numerize(df_user_problems_filtered, user_to_id, problem_to_id)\n",
    "df_for_ease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>problem</th>\n",
       "      <th>solve</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509347</th>\n",
       "      <td>61123</td>\n",
       "      <td>3646</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509348</th>\n",
       "      <td>61123</td>\n",
       "      <td>3661</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509349</th>\n",
       "      <td>61123</td>\n",
       "      <td>3662</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509350</th>\n",
       "      <td>61123</td>\n",
       "      <td>5307</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509351</th>\n",
       "      <td>61123</td>\n",
       "      <td>5729</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8473996 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         handle  problem  solve\n",
       "0             0        0      1\n",
       "1             0        1      1\n",
       "2             0        2      1\n",
       "3             0        3      1\n",
       "4             0        4      1\n",
       "...         ...      ...    ...\n",
       "8509347   61123     3646      1\n",
       "8509348   61123     3661      1\n",
       "8509349   61123     3662      1\n",
       "8509350   61123     5307      1\n",
       "8509351   61123     5729      1\n",
       "\n",
       "[8473996 rows x 3 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_for_ease['solve'] = [1] * len(df_for_ease)\n",
    "df_for_ease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pivot_table = df_for_ease.pivot_table(index=[\"handle\"], columns=[\"problem\"], values=\"solve\")\n",
    "X = pivot_table.to_numpy()\n",
    "X = np.nan_to_num(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 1. 1. ... 0. 0. 0.]\n",
      " [1. 1. 1. ... 0. 0. 0.]\n",
      " [1. 1. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [1. 1. 0. ... 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "print(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class EASE:\n",
    "    \"\"\"\n",
    "    Embarrassingly Shallow Autoencoders model class\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, lambda_):\n",
    "        self.B = None\n",
    "        self.lambda_ = lambda_\n",
    "\n",
    "    def train(self, interaction_matrix):\n",
    "        \"\"\"\n",
    "        train pass\n",
    "        :param interaction_matrix: interaction_matrix\n",
    "        \"\"\"\n",
    "        G = interaction_matrix.T @ interaction_matrix\n",
    "        diag = list(range(G.shape[0]))\n",
    "        G[diag, diag] += self.lambda_\n",
    "        P = np.linalg.inv(G)\n",
    "\n",
    "        # B = P * (X^T * X − diagMat(γ))\n",
    "        self.B = P / -np.diag(P)\n",
    "        min_dim = min(*self.B.shape)\n",
    "        self.B[range(min_dim), range(min_dim)] = 0\n",
    "\n",
    "    def forward(self, user_row):\n",
    "        \"\"\"\n",
    "        forward pass\n",
    "        \"\"\"\n",
    "        return user_row @ self.B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "ease = EASE(300)\n",
    "ease.train(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.00727725e+00  1.02440200e+00  1.01068748e+00 ...  1.11855588e-03\n",
      "   3.12755723e-03 -5.54864549e-03]\n",
      " [ 1.00036939e+00  1.01667548e+00  9.88474423e-01 ...  1.31113134e-03\n",
      "   6.84148040e-03  2.71095626e-03]\n",
      " [ 1.08367825e+00  8.31202138e-01  1.67785703e-02 ... -5.30959702e-03\n",
      "   4.92269876e-03  5.50936932e-03]\n",
      " ...\n",
      " [-1.71914208e-02  7.86388777e-03  5.43994123e-02 ...  6.76805079e-04\n",
      "   8.47921342e-04 -3.42467570e-03]\n",
      " [-3.50431692e-03 -2.41779668e-02 -6.70369826e-02 ... -1.65399198e-03\n",
      "  -4.73558723e-03  8.10726865e-03]\n",
      " [ 9.85755405e-01  9.85760876e-01  8.15752727e-03 ... -1.13759033e-04\n",
      "  -3.53057683e-04  7.46113530e-04]]\n"
     ]
    }
   ],
   "source": [
    "result = ease.forward(X[:, :])\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([    0,     0,     0, ..., 61123, 61123, 61123]), array([   0,    1,    2, ..., 3662, 5307, 5729]))\n"
     ]
    }
   ],
   "source": [
    "print(X.nonzero())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[       -inf        -inf        -inf ...  0.00111856  0.00312756\n",
      "  -0.00554865]\n",
      " [       -inf        -inf        -inf ...  0.00131113  0.00684148\n",
      "   0.00271096]\n",
      " [       -inf        -inf  0.01677857 ... -0.0053096   0.0049227\n",
      "   0.00550937]\n",
      " ...\n",
      " [-0.01719142  0.00786389  0.05439941 ...  0.00067681  0.00084792\n",
      "  -0.00342468]\n",
      " [-0.00350432 -0.02417797 -0.06703698 ... -0.00165399 -0.00473559\n",
      "   0.00810727]\n",
      " [       -inf        -inf  0.00815753 ... -0.00011376 -0.00035306\n",
      "   0.00074611]]\n"
     ]
    }
   ],
   "source": [
    "result[X.nonzero()] = -np.inf\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[8844 8783 8807 ... 8725 8912 9010]\n",
      " [3158 5573 5979 ... 2593 3220 3442]\n",
      " [6323 6268 1835 ... 7783 7030 4395]\n",
      " ...\n",
      " [8476 8467 8464 ... 3602 5920 8457]\n",
      " [7250 3679 3524 ... 8494 3027  755]\n",
      " [  87 1012 1008 ... 3490 5768 1352]]\n"
     ]
    }
   ],
   "source": [
    "top_problems_by_user = bn.argpartition(-result, NUM_TOP_PROBLEMS, axis=1)[:, :NUM_TOP_PROBLEMS]\n",
    "print(top_problems_by_user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "user_result = []\n",
    "problem_result =[]\n",
    "\n",
    "for id, top_k in enumerate(top_problems_by_user):\n",
    "    user_result.extend([id] * NUM_TOP_PROBLEMS)\n",
    "    problem_result.extend(top_k)\n",
    "\n",
    "df_user_result = pd.DataFrame(user_result, columns=['user_id'])\n",
    "df_problem_result = pd.DataFrame(problem_result, columns=['problem_id'])\n",
    "df_result = pd.concat([df_user_result, df_problem_result], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>problem_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>8844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>8783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>8807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>8745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>8841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611235</th>\n",
       "      <td>61123</td>\n",
       "      <td>3944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611236</th>\n",
       "      <td>61123</td>\n",
       "      <td>1026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611237</th>\n",
       "      <td>61123</td>\n",
       "      <td>3490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611238</th>\n",
       "      <td>61123</td>\n",
       "      <td>5768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611239</th>\n",
       "      <td>61123</td>\n",
       "      <td>1352</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>611240 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  problem_id\n",
       "0             0        8844\n",
       "1             0        8783\n",
       "2             0        8807\n",
       "3             0        8745\n",
       "4             0        8841\n",
       "...         ...         ...\n",
       "611235    61123        3944\n",
       "611236    61123        1026\n",
       "611237    61123        3490\n",
       "611238    61123        5768\n",
       "611239    61123        1352\n",
       "\n",
       "[611240 rows x 2 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle</th>\n",
       "      <th>problem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7749</th>\n",
       "      <td>0000000000</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7748</th>\n",
       "      <td>0000000000</td>\n",
       "      <td>3004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7741</th>\n",
       "      <td>0000000000</td>\n",
       "      <td>2231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7742</th>\n",
       "      <td>0000000000</td>\n",
       "      <td>2743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7743</th>\n",
       "      <td>0000000000</td>\n",
       "      <td>13866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432753</th>\n",
       "      <td>zzzzzzzzu2</td>\n",
       "      <td>2839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432752</th>\n",
       "      <td>zzzzzzzzu2</td>\n",
       "      <td>10809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432751</th>\n",
       "      <td>zzzzzzzzu2</td>\n",
       "      <td>11654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432750</th>\n",
       "      <td>zzzzzzzzu2</td>\n",
       "      <td>1157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432755</th>\n",
       "      <td>zzzzzzzzu2</td>\n",
       "      <td>4673</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>611240 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            handle  problem\n",
       "7749    0000000000     2010\n",
       "7748    0000000000     3004\n",
       "7741    0000000000     2231\n",
       "7742    0000000000     2743\n",
       "7743    0000000000    13866\n",
       "...            ...      ...\n",
       "432753  zzzzzzzzu2     2839\n",
       "432752  zzzzzzzzu2    10809\n",
       "432751  zzzzzzzzu2    11654\n",
       "432750  zzzzzzzzu2     1157\n",
       "432755  zzzzzzzzu2     4673\n",
       "\n",
       "[611240 rows x 2 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_infer = denumerize(df_result, id_to_user, id_to_problem)\n",
    "df_infer.columns = ['handle', 'problem']\n",
    "df_infer = df_infer.sort_values('handle')\n",
    "df_infer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3891     6236\n",
       "3890    10266\n",
       "3893    10953\n",
       "3894     8111\n",
       "3896     4781\n",
       "3897     1620\n",
       "3898    14918\n",
       "3899     1550\n",
       "3892    10211\n",
       "3895     9202\n",
       "Name: problem, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_infer[df_infer['handle'] == 'glanceyes']['problem']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df_infer.to_csv(os.path.join(DATA_DIR, 'inference.csv'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DIR, \"model.txt\"), 'wb') as f:\n",
    "    pickle.dump(ease, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}