contentbased.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook I implement a content based recommender system, based on the available description of groups (their degree, year, semester)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import math\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = pd.read_pickle(\"groups.pickle\").rename({\"title\": \"group_title\"}, axis=1)\n",
    "users = pd.read_pickle(\"users.pkl\")\n",
    "groups_membership = pd.read_pickle(\"df_membership.pkl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "degrees_courses = pd.read_csv(\"database/university_coursedegree.csv\").drop(\n",
    "    columns=[\"id\"]\n",
    ")\n",
    "degrees = (\n",
    "    pd.read_csv(\"database/university_degree.csv\")\n",
    "    .rename(\n",
    "        {\n",
    "            \"id\": \"degree_id\",\n",
    "            \"name\": \"degree_name\",\n",
    "            \"group_id\": \"degree_group_id\",\n",
    "            \"type\": \"degree_type\",\n",
    "        },\n",
    "        axis=1,\n",
    "    )\n",
    "    .drop(columns=[\"department_id\", \"slug\", \"icon\"])\n",
    ")\n",
    "courses = (\n",
    "    pd.read_csv(\"database/university_course.csv\")\n",
    "    .drop(columns=[\"cfu\", \"wiki_link\", \"slug_unimi\"])\n",
    "    .rename(\n",
    "        {\"id\": \"course_id\", \"group_id\": \"course_group_id\", \"name\": \"course_name\"},\n",
    "        axis=1,\n",
    "    )\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>course_id</th>\n",
       "      <th>course_name</th>\n",
       "      <th>course_group_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Linguaggi di programmazione</td>\n",
       "      <td>-1.001436e+12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30</td>\n",
       "      <td>Linguaggi e traduttori</td>\n",
       "      <td>-1.001190e+12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Architettura degli elaboratori 1</td>\n",
       "      <td>-1.001188e+12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Matematica del continuo</td>\n",
       "      <td>-1.001389e+12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Linguaggi formali e automi</td>\n",
       "      <td>-1.001217e+12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   course_id                       course_name  course_group_id\n",
       "0         23       Linguaggi di programmazione    -1.001436e+12\n",
       "1         30            Linguaggi e traduttori    -1.001190e+12\n",
       "2          3  Architettura degli elaboratori 1    -1.001188e+12\n",
       "3          1           Matematica del continuo    -1.001389e+12\n",
       "4          4        Linguaggi formali e automi    -1.001217e+12"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "courses.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>degree_id</th>\n",
       "      <th>degree_name</th>\n",
       "      <th>degree_type</th>\n",
       "      <th>degree_group_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>71</td>\n",
       "      <td>Lingue e culture per la comunicazione e la coo...</td>\n",
       "      <td>M</td>\n",
       "      <td>-1001520507045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>97</td>\n",
       "      <td>Scienza, tecnica e didattica dello sport</td>\n",
       "      <td>M</td>\n",
       "      <td>-1001508132234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>98</td>\n",
       "      <td>Scienze agrarie</td>\n",
       "      <td>M</td>\n",
       "      <td>-1001549161533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>Sicurezza informatica</td>\n",
       "      <td>M</td>\n",
       "      <td>-1001215714502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>155</td>\n",
       "      <td>Economics and political science (EPS)</td>\n",
       "      <td>M</td>\n",
       "      <td>-1001452282561</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   degree_id                                        degree_name degree_type  \\\n",
       "0         71  Lingue e culture per la comunicazione e la coo...           M   \n",
       "1         97           Scienza, tecnica e didattica dello sport           M   \n",
       "2         98                                    Scienze agrarie           M   \n",
       "3          7                              Sicurezza informatica           M   \n",
       "4        155              Economics and political science (EPS)           M   \n",
       "\n",
       "   degree_group_id  \n",
       "0   -1001520507045  \n",
       "1   -1001508132234  \n",
       "2   -1001549161533  \n",
       "3   -1001215714502  \n",
       "4   -1001452282561  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.to_pickle(degrees, \"degrees.pkl\")\n",
    "degrees.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_id</th>\n",
       "      <th>group_title</th>\n",
       "      <th>course_id</th>\n",
       "      <th>course_name</th>\n",
       "      <th>year</th>\n",
       "      <th>semester</th>\n",
       "      <th>course_degree_id</th>\n",
       "      <th>course_degree_name</th>\n",
       "      <th>course_degree_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1001563734995</td>\n",
       "      <td>Quantitative biology | StudentiUniMi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1001557200491</td>\n",
       "      <td>Educazione professionale | StudentiUniMi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1001774201871</td>\n",
       "      <td>Algebra 1 | Matematica</td>\n",
       "      <td>232.0</td>\n",
       "      <td>Algebra 1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Matematica</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1001724030284</td>\n",
       "      <td>Lettatura italiana Baragetti | StudentiUniMi</td>\n",
       "      <td>239.0</td>\n",
       "      <td>Letteratura italiana Baragetti</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>Storia</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1001531478970</td>\n",
       "      <td>Archeologia | StudentiUniMi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        group_id                                   group_title  course_id  \\\n",
       "0 -1001563734995          Quantitative biology | StudentiUniMi        NaN   \n",
       "1 -1001557200491      Educazione professionale | StudentiUniMi        NaN   \n",
       "2 -1001774201871                        Algebra 1 | Matematica      232.0   \n",
       "3 -1001724030284  Lettatura italiana Baragetti | StudentiUniMi      239.0   \n",
       "4 -1001531478970                   Archeologia | StudentiUniMi        NaN   \n",
       "\n",
       "                      course_name  year  semester  course_degree_id  \\\n",
       "0                             NaN   NaN       NaN               NaN   \n",
       "1                             NaN   NaN       NaN               NaN   \n",
       "2                       Algebra 1   1.0       1.0               9.0   \n",
       "3  Letteratura italiana Baragetti   1.0       1.0              23.0   \n",
       "4                             NaN   NaN       NaN               NaN   \n",
       "\n",
       "  course_degree_name course_degree_type  \n",
       "0                NaN                NaN  \n",
       "1                NaN                NaN  \n",
       "2         Matematica                  B  \n",
       "3             Storia                  B  \n",
       "4                NaN                NaN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joined_course_degrees = (\n",
    "    groups.merge(courses.rename({\"course_group_id\": \"group_id\"}, axis=1), how=\"left\")\n",
    "    .merge(degrees_courses, how=\"left\")\n",
    "    .merge(degrees, how=\"left\")\n",
    "    .drop(columns=[\"degree_group_id\"])\n",
    "    .rename(\n",
    "        {\n",
    "            \"course_group_id\": \"group_id\",\n",
    "            \"degree_id\": \"course_degree_id\",\n",
    "            \"degree_name\": \"course_degree_name\",\n",
    "            \"degree_type\": \"course_degree_type\",\n",
    "        },\n",
    "        axis=1,\n",
    "    )\n",
    ")\n",
    "joined_course_degrees.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_id</th>\n",
       "      <th>group_title</th>\n",
       "      <th>group_degree_id</th>\n",
       "      <th>group_degree_name</th>\n",
       "      <th>group_degree_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1001563734995</td>\n",
       "      <td>Quantitative biology | StudentiUniMi</td>\n",
       "      <td>88.0</td>\n",
       "      <td>Quantitative biology</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1001557200491</td>\n",
       "      <td>Educazione professionale | StudentiUniMi</td>\n",
       "      <td>61.0</td>\n",
       "      <td>Educazione professionale</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1001774201871</td>\n",
       "      <td>Algebra 1 | Matematica</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1001724030284</td>\n",
       "      <td>Lettatura italiana Baragetti | StudentiUniMi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1001531478970</td>\n",
       "      <td>Archeologia | StudentiUniMi</td>\n",
       "      <td>40.0</td>\n",
       "      <td>Archeologia</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        group_id                                   group_title  \\\n",
       "0 -1001563734995          Quantitative biology | StudentiUniMi   \n",
       "1 -1001557200491      Educazione professionale | StudentiUniMi   \n",
       "2 -1001774201871                        Algebra 1 | Matematica   \n",
       "3 -1001724030284  Lettatura italiana Baragetti | StudentiUniMi   \n",
       "4 -1001531478970                   Archeologia | StudentiUniMi   \n",
       "\n",
       "   group_degree_id         group_degree_name group_degree_type  \n",
       "0             88.0      Quantitative biology                 M  \n",
       "1             61.0  Educazione professionale                 B  \n",
       "2              NaN                       NaN               NaN  \n",
       "3              NaN                       NaN               NaN  \n",
       "4             40.0               Archeologia                 M  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joined_groups_degrees = (\n",
    "    groups.rename({\"group_id\": \"degree_group_id\"}, axis=1)\n",
    "    .merge(degrees, how=\"left\")\n",
    "    .rename(\n",
    "        {\n",
    "            \"degree_group_id\": \"group_id\",\n",
    "            \"degree_id\": \"group_degree_id\",\n",
    "            \"degree_name\": \"group_degree_name\",\n",
    "            \"degree_type\": \"group_degree_type\",\n",
    "        },\n",
    "        axis=1,\n",
    "    )\n",
    ")\n",
    "joined_groups_degrees.head()  # information on main groups for each degree\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After performing some joins, I manage to have the information I need for each group."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_id</th>\n",
       "      <th>group_title</th>\n",
       "      <th>course_id</th>\n",
       "      <th>course_name</th>\n",
       "      <th>year</th>\n",
       "      <th>semester</th>\n",
       "      <th>degree_id</th>\n",
       "      <th>degree_name</th>\n",
       "      <th>degree_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1001563734995</td>\n",
       "      <td>Quantitative biology | StudentiUniMi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>88.0</td>\n",
       "      <td>Quantitative biology</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1001557200491</td>\n",
       "      <td>Educazione professionale | StudentiUniMi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>61.0</td>\n",
       "      <td>Educazione professionale</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1001774201871</td>\n",
       "      <td>Algebra 1 | Matematica</td>\n",
       "      <td>232.0</td>\n",
       "      <td>Algebra 1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Matematica</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1001724030284</td>\n",
       "      <td>Lettatura italiana Baragetti | StudentiUniMi</td>\n",
       "      <td>239.0</td>\n",
       "      <td>Letteratura italiana Baragetti</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>Storia</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1001531478970</td>\n",
       "      <td>Archeologia | StudentiUniMi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>Archeologia</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        group_id                                   group_title  course_id  \\\n",
       "0 -1001563734995          Quantitative biology | StudentiUniMi        NaN   \n",
       "1 -1001557200491      Educazione professionale | StudentiUniMi        NaN   \n",
       "2 -1001774201871                        Algebra 1 | Matematica      232.0   \n",
       "3 -1001724030284  Lettatura italiana Baragetti | StudentiUniMi      239.0   \n",
       "4 -1001531478970                   Archeologia | StudentiUniMi        NaN   \n",
       "\n",
       "                      course_name  year  semester  degree_id  \\\n",
       "0                             NaN   NaN       NaN       88.0   \n",
       "1                             NaN   NaN       NaN       61.0   \n",
       "2                       Algebra 1   1.0       1.0        9.0   \n",
       "3  Letteratura italiana Baragetti   1.0       1.0       23.0   \n",
       "4                             NaN   NaN       NaN       40.0   \n",
       "\n",
       "                degree_name  degree_type  \n",
       "0      Quantitative biology          2.0  \n",
       "1  Educazione professionale          1.0  \n",
       "2                Matematica          1.0  \n",
       "3                    Storia          1.0  \n",
       "4               Archeologia          2.0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joined_groups_courses_degrees = joined_groups_degrees.merge(\n",
    "    joined_course_degrees, on=[\"group_id\", \"group_title\"]\n",
    ")\n",
    "\n",
    "copied = joined_groups_courses_degrees.copy()\n",
    "for column in [\"degree_id\", \"degree_name\", \"degree_type\"]:\n",
    "    copied[column] = joined_groups_courses_degrees[f\"group_{column}\"].combine_first(\n",
    "        joined_groups_courses_degrees[f\"course_{column}\"]\n",
    "    )\n",
    "    copied = copied.drop(columns=[f\"group_{column}\", f\"course_{column}\"])\n",
    "\n",
    "combined_groups_courses_degrees = copied\n",
    "combined_groups_courses_degrees.replace(-2, 6, inplace=True)\n",
    "combined_groups_courses_degrees.replace(\"B\", 1, inplace=True)\n",
    "combined_groups_courses_degrees.replace(\"M\", 2, inplace=True)\n",
    "combined_groups_courses_degrees.replace(\"C\", 3, inplace=True)\n",
    "\n",
    "extra_groups = combined_groups_courses_degrees.loc[\n",
    "    combined_groups_courses_degrees[\"degree_id\"].isna()\n",
    "]\n",
    "\n",
    "combined_groups_courses_degrees.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I add a new course degree for a specific category of groups (For24 groups, not officially in a degree), but of common interest for users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adding FOR24 groups\n",
    "for_24 = extra_groups[\n",
    "    (extra_groups.group_title.str.contains(\"FOR\"))\n",
    "    | (extra_groups.group_id == -1001672191242)\n",
    "]\n",
    "combined_groups_courses_degrees.loc[\n",
    "    combined_groups_courses_degrees.group_id.isin(for_24.group_id), \"degree_id\"\n",
    "] = (degrees.degree_id.max() + 1)\n",
    "combined_groups_courses_degrees[\n",
    "    combined_groups_courses_degrees.group_id.isin(for_24.group_id)\n",
    "]\n",
    "extra_groups = combined_groups_courses_degrees.loc[\n",
    "    combined_groups_courses_degrees[\"degree_id\"].isna()\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adding degrees to some non-classified groups\n",
    "combined_groups_courses_degrees.loc[\n",
    "    combined_groups_courses_degrees.group_id == -1001448722502, \"degree_id\"\n",
    "] = 3.0\n",
    "combined_groups_courses_degrees.loc[\n",
    "    combined_groups_courses_degrees.group_id == -1001525285619, \"degree_id\"\n",
    "] = 5.0\n",
    "extra_groups = combined_groups_courses_degrees.loc[\n",
    "    combined_groups_courses_degrees[\"degree_id\"].isna()\n",
    "]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then I devide the groups that don't have a course degree into groups for international matters and general extra groups (like notes exchange, job adverts..)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "international_groups = extra_groups[\n",
    "    (extra_groups.group_id == -1001353996639)\n",
    "    | (extra_groups.group_id == -1001699979466)\n",
    "]\n",
    "utility_groups = extra_groups[\n",
    "    ~extra_groups.group_id.isin(international_groups.group_id)\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_groups_courses_degrees.to_pickle(\"groups_degrees.pkl\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here I generate a vector representation of each group. Each group is described by a vector of features, in which the first 158 correspond to the course degrees. <br>\n",
    "If the vector_rep[i]+1 = 1 (with i <= 158), then i is the associated course degree for that group (from the degree id in the degree table). <br>\n",
    "After these features I add  the year representation to the vector (from first to fifth year, with elective courses encoded as in the sixth position). <br>\n",
    "I then add two other two features that, if set to 1, indicate if the group is about international matters or a general utility group respectivly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1001563734995    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n",
       "-1001557200491    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n",
       "-1001774201871    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.6, ...\n",
       "-1001724030284    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n",
       "-1001531478970    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n",
       "dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def generate_group_representation(group):\n",
    "    # [1° 2° 3° 4° 5° year and elective(6)]\n",
    "    # we are adding a new degree for FOR24 which wasn't in the degree table\n",
    "    YEARS_OFFSET = degrees.degree_id.max()\n",
    "    YEARS_VALUES = 6\n",
    "\n",
    "    # [1 2]\n",
    "    SEMESTER_OFFSET = YEARS_OFFSET + YEARS_VALUES\n",
    "    SEMESTER_VALUES = 2\n",
    "\n",
    "    \"\"\"\n",
    "    # [1 bachelor 2 master 3 single cycle]\n",
    "    TYPE_OFFSET = SEMESTER_OFFSET + SEMESTER_VALUES\n",
    "    TYPE_VALUES = 3\n",
    "    \"\"\"\n",
    "\n",
    "    EXTRA_OFFSET = SEMESTER_OFFSET + SEMESTER_VALUES\n",
    "    EXTRA_VALUES = 2\n",
    "\n",
    "    qb_representation = np.zeros(\n",
    "        degrees.degree_id.max()\n",
    "        + 1\n",
    "        + YEARS_VALUES\n",
    "        + SEMESTER_VALUES\n",
    "        #+ TYPE_VALUES\n",
    "        + EXTRA_VALUES\n",
    "    )\n",
    "\n",
    "    degree_ids = combined_groups_courses_degrees[\n",
    "        combined_groups_courses_degrees.group_id == group\n",
    "    ].degree_id.values\n",
    "    years = combined_groups_courses_degrees[\n",
    "        combined_groups_courses_degrees.group_id == group\n",
    "    ].year\n",
    "\n",
    "    for degree_id in degree_ids:\n",
    "        for year in years:\n",
    "            if not math.isnan(degree_id) and not math.isnan(year):\n",
    "                qb_representation[int(degree_id) - 1] = 0.6\n",
    "                qb_representation[int(year) + YEARS_OFFSET] = 0.2\n",
    "            elif math.isnan(year) and not math.isnan(degree_id):\n",
    "                qb_representation[int(degree_id) - 1] = 1\n",
    "\n",
    "    semesters = combined_groups_courses_degrees[\n",
    "        combined_groups_courses_degrees.group_id == group\n",
    "    ].semester\n",
    "    for semester in semesters:\n",
    "        if not math.isnan(semester):\n",
    "            qb_representation[int(semester) + SEMESTER_OFFSET] = 0.1\n",
    "\n",
    "    if group in international_groups.group_id.values:\n",
    "        qb_representation[EXTRA_OFFSET + 1] = 1\n",
    "\n",
    "    if group in utility_groups.group_id.values:\n",
    "        qb_representation[EXTRA_OFFSET + 2] = 1\n",
    "\n",
    "    return qb_representation\n",
    "\n",
    "\n",
    "group_description = pd.Series(dtype=object)\n",
    "for group_id in combined_groups_courses_degrees.group_id.unique():\n",
    "    group_description.loc[group_id] = generate_group_representation(group_id)\n",
    "\n",
    "group_description.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.to_pickle(group_description, \"group_description.pkl\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then I represent each user as a vector of the same dimension, with the values beeing an average between the vectors of all groups he is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a6c671e57cc49aea2d160654e14e4a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/6174 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "1292286374    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n",
       "1775427491    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n",
       "167020566     [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n",
       "1131687203    [0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.0, 0.0, 0.0, ...\n",
       "1922202100    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n",
       "dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def generate_user_profile(user_id):\n",
    "    in_groups = groups_membership[groups_membership.user_id == user_id].group_id\n",
    "    user_profile = np.mean(group_description[in_groups])\n",
    "    return user_profile\n",
    "\n",
    "\n",
    "user_profile = pd.Series(dtype=object)\n",
    "for user in tqdm(\n",
    "    groups_membership.user_id.unique(), total=len(groups_membership.user_id.unique())\n",
    "):\n",
    "    user_profile.loc[user] = generate_user_profile(user)\n",
    "\n",
    "user_profile.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I compute the probability of a particular user being interested in a particular group (in which he isn't in) as the cosine similarity between the user vector representation and the representation of that group. The most probable k are the ones I'm gonna recommend to that user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d801c3ba0e0949cca182a26e81afa844",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4183 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>77632</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001563734995</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77633</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001557200491</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77634</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001774201871</td>\n",
       "      <td>0.010962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77635</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001724030284</td>\n",
       "      <td>0.010962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77636</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001531478970</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_id       group_id  prediction\n",
       "77632  350313104 -1001563734995    0.000000\n",
       "77633  350313104 -1001557200491    0.000000\n",
       "77634  350313104 -1001774201871    0.010962\n",
       "77635  350313104 -1001724030284    0.010962\n",
       "77636  350313104 -1001531478970    0.000000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainset = pd.read_pickle(\"trainset.pkl\")\n",
    "\n",
    "\n",
    "def get_not_in_groups(user_id):\n",
    "    in_groups = trainset[trainset.user_id == user_id].group_id\n",
    "    not_in_groups = groups[~(groups.group_id.isin(in_groups))].group_id\n",
    "    return not_in_groups\n",
    "\n",
    "\n",
    "def get_prediction(user, group):\n",
    "    up = user_profile[user]\n",
    "    return np.dot(up, group_description[group])\n",
    "\n",
    "\n",
    "predictions = list()\n",
    "for user in tqdm(trainset.user_id.unique(), total=len(trainset.user_id.unique())):\n",
    "    not_in = get_not_in_groups(user)\n",
    "    user_predictions = list()\n",
    "    for group in not_in:\n",
    "        pred = get_prediction(user, group)\n",
    "        predictions.append((int(user), int(group), pred))\n",
    "\n",
    "predictions = pd.DataFrame(predictions, columns=(\"user_id\", \"group_id\", \"prediction\"))\n",
    "predictions[predictions.user_id==350313104][:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>prediction</th>\n",
       "      <th>group_title</th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>username</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>77765</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001467027267</td>\n",
       "      <td>0.324038</td>\n",
       "      <td>Sicurezza delle architetture orientate ai serv...</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77876</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001348261542</td>\n",
       "      <td>0.324038</td>\n",
       "      <td>Modellazione e analisi di sistemi - Informatic...</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77898</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001434472267</td>\n",
       "      <td>0.319423</td>\n",
       "      <td>Information management - Sicurezza Informatica</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77849</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001452073794</td>\n",
       "      <td>0.317115</td>\n",
       "      <td>Statistical methods for machine learning - Inf...</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77666</th>\n",
       "      <td>350313104</td>\n",
       "      <td>-1001556502415</td>\n",
       "      <td>0.314038</td>\n",
       "      <td>Privacy and data protection - Informatica magi...</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_id       group_id  prediction  \\\n",
       "77765  350313104 -1001467027267    0.324038   \n",
       "77876  350313104 -1001348261542    0.324038   \n",
       "77898  350313104 -1001434472267    0.319423   \n",
       "77849  350313104 -1001452073794    0.317115   \n",
       "77666  350313104 -1001556502415    0.314038   \n",
       "\n",
       "                                             group_title first_name last_name  \\\n",
       "77765  Sicurezza delle architetture orientate ai serv...    Alessia       NaN   \n",
       "77876  Modellazione e analisi di sistemi - Informatic...    Alessia       NaN   \n",
       "77898     Information management - Sicurezza Informatica    Alessia       NaN   \n",
       "77849  Statistical methods for machine learning - Inf...    Alessia       NaN   \n",
       "77666  Privacy and data protection - Informatica magi...    Alessia       NaN   \n",
       "\n",
       "       username  \n",
       "77765  aleceres  \n",
       "77876  aleceres  \n",
       "77898  aleceres  \n",
       "77849  aleceres  \n",
       "77666  aleceres  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_predictions_groups_users = predictions.merge(groups).merge(users)\n",
    "merged_predictions_groups_users.loc[\n",
    "    merged_predictions_groups_users.username == \"aleceres\"\n",
    "].sort_values(by=[\"prediction\"], ascending=False)[:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "testset = pd.read_pickle(\"testset.pkl\")\n",
    "\n",
    "\n",
    "def get_top_k(user, k):\n",
    "    predictions_user = predictions[(predictions.user_id == user)].loc[\n",
    "        :, [\"group_id\", \"prediction\"]\n",
    "    ]\n",
    "    return predictions_user.sort_values(by=\"prediction\", ascending=False)[:k]\n",
    "\n",
    "\n",
    "def is_relevant(user, item):\n",
    "    return len(testset[(testset.user_id == user) & (testset.group_id == item)]) != 0\n",
    "\n",
    "\n",
    "def HR(user, k):\n",
    "    recommended_items = get_top_k(user, k)\n",
    "    return sum(is_relevant(user, item) for item in recommended_items.group_id)\n",
    "\n",
    "\n",
    "def average_precision(user, k):\n",
    "    recommended_items = get_top_k(user, k)\n",
    "    return sum(\n",
    "        is_relevant(user, row[1].group_id) * (1 / (rank + 1) * 1)\n",
    "        for rank, row in enumerate(recommended_items.iterrows())\n",
    "    )\n",
    "\n",
    "\n",
    "def RR(user, k):\n",
    "    recommended_items = get_top_k(user, k)\n",
    "    return sum(\n",
    "        is_relevant(user, row[1].group_id) * (1 / (rank + 1))\n",
    "        for rank, row in enumerate(recommended_items.iterrows())\n",
    "    )\n",
    "\n",
    "\n",
    "def precision(user, k):\n",
    "    recommended_items = get_top_k(user, k)\n",
    "    return sum(is_relevant(user, item) for item in recommended_items.group_id) * 1 / k\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I compute precision and recall metrics in order to evaluate the recommender system we just implemented."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average P@8: 0.07171697114111729\n"
     ]
    }
   ],
   "source": [
    "k = 8\n",
    "print(f\"Average P@{k}: {np.mean([precision(user,k) for user in testset.user_id])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HR@8: 0.5737357691289383\n"
     ]
    }
   ],
   "source": [
    "print(f\"HR@{k}: {np.mean([HR(user,k) for user in testset.user_id])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@8: 0.2809873671470176\n"
     ]
    }
   ],
   "source": [
    "print(f\"MAP@{k}: {np.mean([average_precision(user,k) for user in testset.user_id])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@8: 0.2809873671470176\n"
     ]
    }
   ],
   "source": [
    "print(f\"MRR@{k}: {np.mean([RR(user,k) for user in testset.user_id])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8169db0d00414f1abebf27d708f0a772",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/6174 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def get_not_in_groups(user_id):\n",
    "    in_groups = groups_membership[groups_membership.user_id == user_id].group_id\n",
    "    not_in_groups = groups[~(groups.group_id.isin(in_groups))].group_id\n",
    "    return [group for group in not_in_groups]\n",
    "\n",
    "\n",
    "all_predictions = list()\n",
    "for user in tqdm(\n",
    "    groups_membership.user_id.unique(), total=len(groups_membership.user_id.unique())\n",
    "):\n",
    "    not_in = get_not_in_groups(user)\n",
    "    for group in not_in:\n",
    "        pred = get_prediction(user, group)\n",
    "        all_predictions.append((int(user), int(group), pred))\n",
    "\n",
    "all_predictions = pd.DataFrame(\n",
    "    all_predictions, columns=(\"user_id\", \"group_id\", \"prediction\")\n",
    ")\n",
    "all_merged_predictions = all_predictions.merge(users).merge(groups)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>prediction</th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>username</th>\n",
       "      <th>group_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>908300</th>\n",
       "      <td>26170256</td>\n",
       "      <td>-1001466214340</td>\n",
       "      <td>0.272500</td>\n",
       "      <td>Marco</td>\n",
       "      <td>Aceti</td>\n",
       "      <td>acetimarco</td>\n",
       "      <td>Elaborazione dei segnali - Informatica Musical...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619244</th>\n",
       "      <td>26170256</td>\n",
       "      <td>-1001189502801</td>\n",
       "      <td>0.266071</td>\n",
       "      <td>Marco</td>\n",
       "      <td>Aceti</td>\n",
       "      <td>acetimarco</td>\n",
       "      <td>Linguaggi e traduttori - Informatica | Informa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2155021</th>\n",
       "      <td>26170256</td>\n",
       "      <td>-1001283074624</td>\n",
       "      <td>0.266071</td>\n",
       "      <td>Marco</td>\n",
       "      <td>Aceti</td>\n",
       "      <td>acetimarco</td>\n",
       "      <td>Programmazione Dichiarativa - Informatica | In...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1605265</th>\n",
       "      <td>26170256</td>\n",
       "      <td>-1001300632521</td>\n",
       "      <td>0.265357</td>\n",
       "      <td>Marco</td>\n",
       "      <td>Aceti</td>\n",
       "      <td>acetimarco</td>\n",
       "      <td>Crittografia 1 - Informatica | Informatica mus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1525878</th>\n",
       "      <td>26170256</td>\n",
       "      <td>-1001338485997</td>\n",
       "      <td>0.265357</td>\n",
       "      <td>Marco</td>\n",
       "      <td>Aceti</td>\n",
       "      <td>acetimarco</td>\n",
       "      <td>Sistemi informativi - Informatica | Informatic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1707795</th>\n",
       "      <td>26170256</td>\n",
       "      <td>-1001435818491</td>\n",
       "      <td>0.265357</td>\n",
       "      <td>Marco</td>\n",
       "      <td>Aceti</td>\n",
       "      <td>acetimarco</td>\n",
       "      <td>Linguaggi di Programmazione - Informatica | In...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1804225</th>\n",
       "      <td>26170256</td>\n",
       "      <td>-1001404399387</td>\n",
       "      <td>0.265357</td>\n",
       "      <td>Marco</td>\n",
       "      <td>Aceti</td>\n",
       "      <td>acetimarco</td>\n",
       "      <td>Elaborazione Delle Immagini - Informatica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1851668</th>\n",
       "      <td>26170256</td>\n",
       "      <td>-1001469541498</td>\n",
       "      <td>0.265357</td>\n",
       "      <td>Marco</td>\n",
       "      <td>Aceti</td>\n",
       "      <td>acetimarco</td>\n",
       "      <td>Sicurezza e Privatezza - Informatica | Informa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          user_id       group_id  prediction first_name last_name    username  \\\n",
       "908300   26170256 -1001466214340    0.272500      Marco     Aceti  acetimarco   \n",
       "619244   26170256 -1001189502801    0.266071      Marco     Aceti  acetimarco   \n",
       "2155021  26170256 -1001283074624    0.266071      Marco     Aceti  acetimarco   \n",
       "1605265  26170256 -1001300632521    0.265357      Marco     Aceti  acetimarco   \n",
       "1525878  26170256 -1001338485997    0.265357      Marco     Aceti  acetimarco   \n",
       "1707795  26170256 -1001435818491    0.265357      Marco     Aceti  acetimarco   \n",
       "1804225  26170256 -1001404399387    0.265357      Marco     Aceti  acetimarco   \n",
       "1851668  26170256 -1001469541498    0.265357      Marco     Aceti  acetimarco   \n",
       "\n",
       "                                               group_title  \n",
       "908300   Elaborazione dei segnali - Informatica Musical...  \n",
       "619244   Linguaggi e traduttori - Informatica | Informa...  \n",
       "2155021  Programmazione Dichiarativa - Informatica | In...  \n",
       "1605265  Crittografia 1 - Informatica | Informatica mus...  \n",
       "1525878  Sistemi informativi - Informatica | Informatic...  \n",
       "1707795  Linguaggi di Programmazione - Informatica | In...  \n",
       "1804225          Elaborazione Delle Immagini - Informatica  \n",
       "1851668  Sicurezza e Privatezza - Informatica | Informa...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_merged_predictions[all_merged_predictions.username == \"acetimarco\"].sort_values(\n",
    "    by=[\"prediction\"], ascending=False\n",
    ")[:8]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_merged_predictions.to_pickle(\"all_merged_predictions.pkl\")\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}