dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook I create the tables I'm gonna use for the recommender systems and compute some statistics in order to better understand the dataset I'm working with."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I start importing the users table, removing the columns I don't need and banned users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = pd.read_csv(\"database/telegrambot_user.csv\")\n",
    "users.drop(\n",
    "    columns=[\"reputation\", \"warn_count\", \"permissions_level\", \"last_seen\"], inplace=True\n",
    ")\n",
    "users = users[users.banned == \"f\"]\n",
    "users.drop(columns=\"banned\", inplace=True)\n",
    "users.rename({\"id\": \"user_id\"}, inplace=True, axis=1)\n",
    "users.to_pickle(\"users.pkl\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then I do the same for the groups table, removing the groups I don't want to recommend."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = pd.read_csv(\"database/telegrambot_group.csv\")\n",
    "groups.drop(\n",
    "    columns=[\n",
    "        \"profile_picture\",\n",
    "        \"invite_link\",\n",
    "        \"owner_id\",\n",
    "        \"bot_id\",\n",
    "        \"welcome_model\",\n",
    "        \"ignore_admin_tagging\",\n",
    "        \"description\",\n",
    "    ],\n",
    "    inplace=True,\n",
    ")\n",
    "groups.rename({\"id\": \"group_id\"}, inplace=True, axis=1)\n",
    "to_delete = [\n",
    "    -1001394272076,\n",
    "    -1001586960044,\n",
    "    -1001581680126,\n",
    "    -1001317570364,\n",
    "    -1001597995872,\n",
    "    -1001399541270,\n",
    "    -1001374431808,\n",
    "    -1001429342185,\n",
    "    -1001314188573,\n",
    "    -1001161201627,\n",
    "]\n",
    "groups = groups[~(groups.group_id.isin(to_delete))]\n",
    "groups.to_pickle(\"groups.pickle\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Group membership is the table that links each user with each group in which he has been (including the groups he has left). I remove userbots from it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>username</th>\n",
       "      <th>id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>last_seen</th>\n",
       "      <th>messages_count</th>\n",
       "      <th>status</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>350313104</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "      <td>3112</td>\n",
       "      <td>-1001568744489</td>\n",
       "      <td>2021-09-13 21:34:00.243643+00</td>\n",
       "      <td>3</td>\n",
       "      <td>member</td>\n",
       "      <td>Bioinformatics for computational genomics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1802</th>\n",
       "      <td>350313104</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "      <td>14052</td>\n",
       "      <td>-1001461601993</td>\n",
       "      <td>2021-10-13 18:17:29.48649+00</td>\n",
       "      <td>2</td>\n",
       "      <td>member</td>\n",
       "      <td>Tirocini e tesi - StudentiUniMi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018</th>\n",
       "      <td>350313104</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "      <td>16395</td>\n",
       "      <td>-1001405226631</td>\n",
       "      <td>2021-11-15 13:24:07.170593+00</td>\n",
       "      <td>3</td>\n",
       "      <td>member</td>\n",
       "      <td>Videogame Design and Programming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2203</th>\n",
       "      <td>350313104</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "      <td>17326</td>\n",
       "      <td>-1001342690802</td>\n",
       "      <td>2022-04-02 17:21:46.070376+00</td>\n",
       "      <td>7</td>\n",
       "      <td>administrator</td>\n",
       "      <td>Bioinformatics - Informatica magistrale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3196</th>\n",
       "      <td>350313104</td>\n",
       "      <td>Alessia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aleceres</td>\n",
       "      <td>16341</td>\n",
       "      <td>-1001357462160</td>\n",
       "      <td>2022-04-02 17:21:45.456724+00</td>\n",
       "      <td>9</td>\n",
       "      <td>administrator</td>\n",
       "      <td>Fisica - Informatica</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id first_name last_name  username     id       group_id  \\\n",
       "190   350313104    Alessia       NaN  aleceres   3112 -1001568744489   \n",
       "1802  350313104    Alessia       NaN  aleceres  14052 -1001461601993   \n",
       "2018  350313104    Alessia       NaN  aleceres  16395 -1001405226631   \n",
       "2203  350313104    Alessia       NaN  aleceres  17326 -1001342690802   \n",
       "3196  350313104    Alessia       NaN  aleceres  16341 -1001357462160   \n",
       "\n",
       "                          last_seen  messages_count         status  \\\n",
       "190   2021-09-13 21:34:00.243643+00               3         member   \n",
       "1802   2021-10-13 18:17:29.48649+00               2         member   \n",
       "2018  2021-11-15 13:24:07.170593+00               3         member   \n",
       "2203  2022-04-02 17:21:46.070376+00               7  administrator   \n",
       "3196  2022-04-02 17:21:45.456724+00               9  administrator   \n",
       "\n",
       "                                          title  \n",
       "190   Bioinformatics for computational genomics  \n",
       "1802            Tirocini e tesi - StudentiUniMi  \n",
       "2018           Videogame Design and Programming  \n",
       "2203    Bioinformatics - Informatica magistrale  \n",
       "3196                       Fisica - Informatica  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_membership = pd.read_csv(\"database/telegrambot_groupmembership.csv\")\n",
    "df_membership = users.merge(group_membership, on=[\"user_id\"]).merge(\n",
    "    groups, on=[\"group_id\"]\n",
    ")\n",
    "df_membership = df_membership[df_membership.status != \"kicked\"]\n",
    "df_membership = df_membership[df_membership.username != \"studentiunimibot\"]\n",
    "df_membership = df_membership[df_membership.username != \"areyouseeingthistoo\"]\n",
    "pd.to_pickle(df_membership, \"df_membership.pkl\")\n",
    "df_membership[df_membership.username == \"aleceres\"].head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I compute some statistics in order to better understand the dataset. Here is the description of the number of groups in which each user is: on average, a user is in around 3 groups with a standard deviation of 5, but with the presence of outliers (users in around 100 groups)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    2948.000000\n",
       "mean        3.912822\n",
       "std         4.774117\n",
       "min         1.000000\n",
       "25%         1.000000\n",
       "50%         2.000000\n",
       "75%         5.000000\n",
       "max       119.000000\n",
       "Name: group_id, dtype: float64"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groups_per_user = (\n",
    "    df_membership[df_membership.status != \"left\"]\n",
    "    .groupby(by=[\"user_id\", \"username\"])\n",
    "    .count()[\"group_id\"]\n",
    "    .sort_values(ascending=False)\n",
    ")\n",
    "groups_per_user.describe()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On the x axis, I have the number of groups; on the y axis, the percentage of users that are in that number of groups. Also here we can see that most of the users tend to be in a range of 1 to 15 groups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'percentage of users')"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXbklEQVR4nO3df7QfdX3n8edLkJ+KqIlUAzSpm+JGqqjhRysu/uoW0AVXUaHtWl3W7B5Epah7cLFW6dlzdO1qWcUfICzq6YLIqhslShWxP6j8CEiRBKMRqQS1pBYBQfn53j9mrny9uTeZ3GS+N/fO83HOPfc7853vzPuTuef7ysxn5jOpKiRJw/Wo2S5AkjS7DAJJGjiDQJIGziCQpIEzCCRp4Hae7QK21oIFC2rx4sWzXYYkzSnXXnvtP1fVwqne6y0IkpwHvBS4vaoOnOL9AGcCRwP3Aq+tquu2tN7FixezevXq7V2uJM1rSf5xuvf6PDV0PnDkZt4/Clja/qwAPtJjLZKkafQWBFX1N8C/bGaRY4FPVuNKYO8kT+6rHknS1Gazs3gRcOvI9IZ23iaSrEiyOsnqjRs3jqU4SRqKOXHVUFWdXVXLq2r5woVT9nVIkmZoNoPgNmC/kel923mSpDGazSBYCbwmjcOAO6vqR7NYjyQNUp+Xj14APB9YkGQD8KfAowGq6qPAKppLR9fTXD76ur5qkSRNr7cgqKoTtvB+AW/oa/uSpG7mRGexJKk/c26IiW2x+LRLZm3bt7znJbO2bUnaHI8IJGngDAJJGjiDQJIGziCQpIEzCCRp4AwCSRo4g0CSBs4gkKSBMwgkaeAMAkkaOINAkgbOIJCkgTMIJGngDAJJGjiDQJIGziCQpIEzCCRp4AwCSRo4g0CSBs4gkKSBMwgkaeAMAkkaOINAkgbOIJCkgTMIJGngDAJJGjiDQJIGziCQpIEzCCRp4AwCSRo4g0CSBs4gkKSB6zUIkhyZZF2S9UlOm+L9/ZNcnuSbSW5IcnSf9UiSNtVbECTZCTgLOApYBpyQZNmkxd4BXFRVzwKOBz7cVz2SpKn1eURwCLC+qm6uqvuBC4FjJy1TwF7t68cBP+yxHknSFPoMgkXArSPTG9p5o94F/GGSDcAq4I1TrSjJiiSrk6zeuHFjH7VK0mDNdmfxCcD5VbUvcDTwqSSb1FRVZ1fV8qpavnDhwrEXKUnzWZ9BcBuw38j0vu28UScCFwFU1TeA3YAFPdYkSZqkzyC4BliaZEmSXWg6g1dOWuYHwIsAkvxrmiDw3I8kjVFvQVBVDwInA5cCN9FcHbQmyRlJjmkXewvw+iT/AFwAvLaqqq+aJEmb2rnPlVfVKppO4NF57xx5vRZ4bp81SJI2b7Y7iyVJs8wgkKSBMwgkaeAMAkkaOINAkgbOIJCkgTMIJGngDAJJGritCoIkj0/yjL6KkSSN3xaDIMnXk+yV5AnAdcA5Sd7ff2mSpHHockTwuKq6C3g58MmqOhR4cb9lSZLGpUsQ7JzkycCrgC/2XI8kacy6BMG7aUYQXV9V1yT5DeC7/ZYlSRqXzY4+2j6Afr+q+mUHcVXdDLyi78IkSeOx2SOCqnqI5nGSkqR5qsvzCK5I8iHg08A9EzOr6rreqpIkjU2XIDio/X3GyLwCXrjdq5Ekjd0Wg6CqXjCOQiRJs6PLDWX7JDk3yZfa6WVJTuy/NEnSOHS5fPR8mstHn9JOfwc4pad6JElj1iUIFlTVRcDDAFX1IPBQr1VJksamSxDck+SJNB3EJDkMuLPXqiRJY9PlqqFTgZXAU5NcASwEjuu1KknS2HS5aui6JEcABwAB1lXVA71XJkkaiy5XDb0S2L2q1gAvAz6d5Nl9FyZJGo8ufQR/UlV3JzkceBFwLvCRfsuSJI1LlyCYuELoJcA5VXUJsEt/JUmSxqlLENyW5GPAq4FVSXbt+DlJ0hzQ5Qv9VTQ3lP1eVf0UeALwtj6LkiSNT5fLRxcAqwGS7N/O+3ZvFUmSxqpLEFxCczNZgN2AJcA64Ok91iVJGpMu9xH81uh0e+noSb1VJEkaq63u9G0fSHNoD7VIkmbBFo8Ikpw6Mvko4NnAD3urSJI0Vl2OCB478rMrTZ/BsV1WnuTIJOuSrE9y2jTLvCrJ2iRrkvyfroVLkraPLn0E757JipPsBJwF/C6wAbgmycqqWjuyzFLg7cBzq+qOJE+aybYkSTPX541hhwDrq+rmqrofuJBNjyReD5xVVXcAVNXtPdYjSZpCn0GwCLh1ZHpDO2/UbwK/meSKJFcmOXKqFSVZkWR1ktUbN27sqVxJGqZpgyDJe9vfr+xx+zsDS4HnAycA5yTZe/JCVXV2VS2vquULFy7ssRxJGp7NHREcnSQ05/Bn4jZgv5Hpfdt5ozYAK6vqgar6Ps3zkJfOcHuSpBnYXBB8GbgDeEaSu5LcPfq7w7qvAZYmWZJkF+B4miedjfo8zdEASRbQnCq6eSvbIEnaBtMGQVW9rar2Bi6pqr2q6rGjv7e04vYh9yfTDFh3E3BRVa1JckaSY9rFLgV+kmQtcDnwtqr6ybY2SpLUXZfLR49Nsg9wcDvrqqrq1GNbVauAVZPmvXPkddE8E/lUJEmzouujKq8GXkkzJPXVSXx4vSTNE11GH30HcPDENf5JFgJfBS7uszBJ0nh0uY/gUZNu9PpJx89JkuaALkcEX05yKXBBO/1qJp33lyTNXV06i9+W5OXA4e2ss6vqc/2WJUkaly5HBFTVZ4HP9lyLJGkWeK5fkgbOIJCkgesUBEl2T3JA38VIksavyw1l/w64nmbsIZIclGTymEGSpDmqyxHBu2geMvNTgKq6HljSW0WSpLHqEgQPVNWdk+ZVH8VIksavy+Wja5L8PrBT+4zhNwF/329ZkqRx6XJE8Ebg6cB9NHcX3wWc0mNNkqQx6nJn8b3A6e2PJGme2WIQJPkCm/YJ3AmsBj5WVb/oozBJ0nh0OTV0M/Az4Jz25y7gbprHSp7TX2mSpHHo0ln8O1V18Mj0F5JcU1UHJ1nTV2GSpPHockTwmCT7T0y0rx/TTt7fS1WSpLHpckTwFuDvknwPCM3NZCcl2RP4RJ/FSZL61+WqoVXt/QNPa2etG+kg/ou+CpMkjUen5xEAS4EDgN2AZyahqj7ZX1mSpHHpcvnonwLPB5bRPKLyKODvAINAkuaBLp3FxwEvAn5cVa8Dngk8rteqJElj0yUIfl5VDwMPJtkLuB3Yr9+yJEnj0qWPYHWSvWluHruW5uayb/RZlCRpfLpcNXRS+/KjSb4M7FVVN/RbliRpXLo8oeyyiddVdUtV3TA6T5I0t017RJBkN2APYEGSx9PcTAawF7BoDLVJksZgc6eG/jPNcweeQtM3MBEEdwEf6rcsSdK4TBsEVXUmcGaSN1bVB8dYkyRpjLp0Fn8wye8Ai0eX985iSZofutxZ/CngqcD1wEPt7MI7iyVpXuhyH8FyYFlVTX5K2RYlORI4E9gJ+HhVvWea5V4BXAwcXFWrt3Y7kqSZ63Jn8Y3Ar23tipPsBJxFMzbRMuCEJMumWO6xwJuBq7Z2G5KkbdfliGABsDbJ1cB9EzOr6pgtfO4QYH1V3QyQ5ELgWGDtpOX+DHgv8LauRUuStp8uQfCuGa57EXDryPQG4NDRBZI8G9ivqi5JMm0QJFkBrADYf//9p1tMkjQDWzw1VFV/DdwCPLp9fQ1w3bZuOMmjgPfTPAFtSzWcXVXLq2r5woULt3XTkqQRXYaYeD1NR+7H2lmLgM93WPdt/Ooopfu28yY8FjgQ+HqSW4DDgJVJlndYtyRpO+nSWfwG4Lk0dxRTVd8FntThc9cAS5MsSbILcDywcuLNqrqzqhZU1eKqWgxcCRzjVUOSNF5dguC+qrp/YiLJzjT3EWxWVT0InAxcCtwEXFRVa5KckWRLHc2SpDHp0ln810n+G7B7kt8FTgK+0GXlVbWK5vGWo/PeOc2yz++yTknS9tXliOA0YCPwLZqB6FYB7+izKEnS+HQ5ItgdOK+qzoFf3ii2O3Bvn4VJksajyxHBZTRf/BN2B77aTzmSpHHrEgS7VdXPJiba13v0V5IkaZy6BME97R3AACR5DvDz/kqSJI1Tlz6CNwOfSfJDmqeU/Rrw6l6rkiSNzWaDoO0Yfh7wNOCAdva6qnqg78IkSeOx2VNDVfUQcEJVPVBVN7Y/hoAkzSNdTg1dkeRDwKeBeyZmVtU2DzwnSZp9XYLgoPb3GSPzCnjhdq9GkjR2XR5e/4JxFCJJmh1dhqHeJ8m5Sb7UTi9LcmL/pUmSxqHLfQTn04wg+pR2+jvAKT3VI0kasy5BsKCqLgIehl8OL/1Qr1VJksam653FT6R9BkGSw4A7e61KkjQ2Xa4aOpXmyWJPTXIFsBA4rteqJElj0+WqoeuSHEFzZ3HwzmJJmle2GARJdqN5KtnhNKeH/jbJR6vqF30XJ0nqX5dTQ58E7gY+2E7/PvAp4JV9FSVJGp8uQXBgVS0bmb48ydq+CpIkjVeXq4aua68UAiDJocDq/kqSJI1TlyOC5wB/n+QH7fT+wLok3wKqqp7RW3WSpN51CYIje69CkjRrulw++o/jKESSNDu69BFIkuYxg0CSBs4gkKSBMwgkaeAMAkkaOINAkgbOIJCkgTMIJGngDAJJGjiDQJIGrtcgSHJkknVJ1ic5bYr3T02yNskNSS5L8ut91iNJ2lRvQZBkJ+As4ChgGXBCkmWTFvsmsLwdwfRi4H/0VY8kaWp9HhEcAqyvqpur6n7gQuDY0QWq6vKquredvBLYt8d6JElT6DMIFgG3jkxvaOdN50TgS1O9kWRFktVJVm/cuHE7lihJ2iE6i5P8IbAceN9U71fV2VW1vKqWL1y4cLzFSdI81+XBNDN1G7DfyPS+7bxfkeTFwOnAEVV1X4/1SJKm0OcRwTXA0iRLkuwCHA+sHF0gybOAjwHHVNXtPdYiSZpGb0FQVQ8CJwOXAjcBF1XVmiRnJDmmXex9wGOAzyS5PsnKaVYnSepJn6eGqKpVwKpJ89458vrFfW5fkrRlO0RnsSRp9hgEkjRwBoEkDZxBIEkDZxBI0sAZBJI0cAaBJA2cQSBJA2cQSNLAGQSSNHAGgSQNnEEgSQNnEEjSwBkEkjRwBoEkDZxBIEkDZxBI0sAZBJI0cAaBJA2cQSBJA2cQSNLAGQSSNHAGgSQNnEEgSQNnEEjSwBkEkjRwBoEkDZxBIEkDZxBI0sAZBJI0cAaBJA2cQSBJA7fzbBcwFItPu2RWtnvLe14yK9uVNHd4RCBJA9drECQ5Msm6JOuTnDbF+7sm+XT7/lVJFvdZjyRpU70FQZKdgLOAo4BlwAlJlk1a7ETgjqr6V8AHgPf2VY8kaWp99hEcAqyvqpsBklwIHAusHVnmWOBd7euLgQ8lSVVVj3UNymz1TYD9E9Jc0WcQLAJuHZneABw63TJV9WCSO4EnAv88ulCSFcCKdvJnSdZtZS0LJq9zDpszbUm347s5054O5lNbwPbsyGbSll+f7o05cdVQVZ0NnD3TzydZXVXLt2NJs2Y+tQXmV3vmU1vA9uzItndb+uwsvg3Yb2R633belMsk2Rl4HPCTHmuSJE3SZxBcAyxNsiTJLsDxwMpJy6wE/qh9fRzwNfsHJGm8ejs11J7zPxm4FNgJOK+q1iQ5A1hdVSuBc4FPJVkP/AtNWPRhxqeVdkDzqS0wv9ozn9oCtmdHtl3bEv8DLknD5p3FkjRwBoEkDdy8DoItDXGxo0uyX5LLk6xNsibJm9v5T0jylSTfbX8/frZr7SrJTkm+meSL7fSSdniR9e1wI7vMdo1dJdk7ycVJvp3kpiS/PVf3TZI/bv/GbkxyQZLd5tK+SXJektuT3Dgyb8p9kcb/att1Q5Jnz17lU5umPe9r/9ZuSPK5JHuPvPf2tj3rkvze1m5v3gZBxyEudnQPAm+pqmXAYcAb2jacBlxWVUuBy9rpueLNwE0j0+8FPtAOM3IHzbAjc8WZwJer6mnAM2naNef2TZJFwJuA5VV1IM3FHcczt/bN+cCRk+ZNty+OApa2PyuAj4ypxq1xPpu25yvAgVX1DOA7wNsB2u+E44Gnt5/5cPv919m8DQJGhrioqvuBiSEu5oyq+lFVXde+vpvmi2YRTTs+0S72CeBls1LgVkqyL/AS4OPtdIAX0gwvAnOrLY8D/g3NlW9U1f1V9VPm6L6huYJw9/Z+nj2AHzGH9k1V/Q3NlYejptsXxwKfrMaVwN5JnjyWQjuaqj1V9VdV9WA7eSXNvVnQtOfCqrqvqr4PrKf5/utsPgfBVENcLJqlWrZZOzLrs4CrgH2q6kftWz8G9pmturbSXwD/FXi4nX4i8NORP+65tI+WABuB/92e6vp4kj2Zg/umqm4D/hz4AU0A3Alcy9zdNxOm2xfz4bvhPwJfal9vc3vmcxDMG0keA/xf4JSqumv0vfYGvB3+GuAkLwVur6prZ7uW7WRn4NnAR6rqWcA9TDoNNIf2zeNp/le5BHgKsCebnpaY0+bKvugiyek0p43/cnutcz4HQZchLnZ4SR5NEwJ/WVWfbWf/08ShbPv79tmqbys8FzgmyS00p+leSHOOfe/2dATMrX20AdhQVVe10xfTBMNc3DcvBr5fVRur6gHgszT7a67umwnT7Ys5+92Q5LXAS4E/GBmFYZvbM5+DoMsQFzu09hz6ucBNVfX+kbdGh+b4I+D/jbu2rVVVb6+qfatqMc2++FpV/QFwOc3wIjBH2gJQVT8Gbk1yQDvrRTRDrM+5fUNzSuiwJHu0f3MTbZmT+2bEdPtiJfCa9uqhw4A7R04h7bCSHElzavWYqrp35K2VwPFpHvS1hKYT/OqtWnlVzdsf4Gia3vXvAafPdj0zqP9wmsPZG4Dr25+jac6tXwZ8F/gq8ITZrnUr2/V84Ivt699o/2jXA58Bdp3t+raiHQcBq9v983ng8XN13wDvBr4N3Ah8Cth1Lu0b4AKa/o0HaI7WTpxuXwChuaLwe8C3aK6WmvU2dGjPepq+gInvgo+OLH962551wFFbuz2HmJCkgZvPp4YkSR0YBJI0cAaBJA2cQSBJA2cQSNLAGQQapCRfT9L7g8yTvKkdmXS73QUqbW+9PapSmq+S7FyPjMGzJScBL66qDTPc1k5V9dBMPit15RGBdlhJFrf/mz6nHSv/r5Ls3r73y//RJ1nQDl1Bktcm+Xw7/vwtSU5Ocmo7MNyVSZ4wson/kOT6dgz+Q9rP79mOBX91+5ljR9a7MsnXaG5Smlzrqe16bkxySjvvozQ3ZX0pyR9PWn6PJBeledbE59px/yfa87Mk/zPJPwC/Pc26F08aq/6tSd418m9z5hRtO6Kdd33btsdu4y7SPGEQaEe3FDirqp4O/BR4RYfPHAi8HDgY+O/AvdUMDPcN4DUjy+1RVQfR/K/9vHbe6TTDXxwCvAB4XzuqKDRjCR1XVUeMbizJc4DXAYfSPDfi9UmeVVX/Bfgh8IKq+sCkGk8C7qjmWRN/Ajxn5L09gauq6pnAz6dad4d/g6na9lbgDe3857XrlgwC7fC+X1XXt6+vBRZ3+MzlVXV3VW2kGVL5C+38b036/AXwy7Hf92qf+PRvgdOSXA98HdgN2L9d/itVNXnMe2iGAvlcVd1TVT+jGbTteVuo8XCawfeoqhtphqmY8BDNQIMzXfd0bbsCeH+SNwF7b8XpLc1zBoF2dPeNvH6IR/q1HuSRv9/dNvOZh0emH+ZX+8Umj69SNOPQvKKqDmp/9q+qiSeq3TOD+mfiFx36BUbbD5v+G2zStqp6D/CfgN2BK5I8bdvK1HxhEGiuuoVHTqcct5nlNufVAEkOpxmB8k7gUuCN7SicdDwN87fAy9rz/nsC/76dtzlXAK9qt7EM+K2tXPc/AU9K8sQku9IMTbzZtiV5alV9q6reSzM6r0EgwKuGNHf9OXBRkhXAJTNcxy+SfBN4NM0TnwD+jOZJajckeRTwfTb9kv0VVXVdkvN5ZOjfj1fVN7ew7Q8Dn0iylmbUzzU0p7E6rzvJGe3829p1bKltpyR5Ac2R0RoeecKVBs7RR6VZkObh4o+uql8keSrNMMkHVPN87W1d99eBt1bV6m1dl4bBIwJpduwBXJ7mCXQBTtoeISDNhEcEkjRwdhZL0sAZBJI0cAaBJA2cQSBJA2cQSNLA/X9cQoivX97n8AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(groups_per_user, weights=np.ones(len(groups_per_user)) / len(groups_per_user))\n",
    "plt.xlabel(\"number of groups\")\n",
    "plt.ylabel(\"percentage of users\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I compute similar statistics for the number of users per group. Here we can see the 5 most popular groups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "group_id        title                                                                            \n",
       "-1001456212600  Chat | Studenti UniMi                                                                1002\n",
       "-1001260565113  Compravendita libri e appunti - StudentiUniMi                                         713\n",
       "-1001157628331  Programmazione 1 - Informatica, Informatica Musicale, Informatica per la Com Dig.     435\n",
       "-1001461601993  Tirocini e tesi - StudentiUniMi                                                       430\n",
       "-1001348138483  Annunci di lavoro - StudentiUniMi                                                     338\n",
       "Name: user_id, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_per_group = (\n",
    "    df_membership[df_membership.status != \"left\"]\n",
    "    .groupby(by=[\"group_id\", \"title\"])\n",
    "    .count()[\"user_id\"]\n",
    "    .sort_values(ascending=False)[:20]\n",
    ")\n",
    "users_per_group[:5]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we can visualize the statistics related to the number of users per group: the distribution is less uniform than the previous one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'percentage of groups')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAY6klEQVR4nO3debgldX3n8feHJsgmsnRHEWgbDaNhHERoQA3uS1AzkFFQ0BnFJcRBVMLE2I6Ok5B/1PExMUqCqIxLoggETQMtKCqOkhFoEIEGkRZbaUYFN3CJQMN3/qi6cLje7lvd3Drndtf79TznuVW/qlP17XMKPqe2X6WqkCQN11aTLkCSNFkGgSQNnEEgSQNnEEjSwBkEkjRwW0+6gI21cOHCWrJkyaTLkKTNyhVXXPHjqlo007TNLgiWLFnCypUrJ12GJG1WknxvfdM8NCRJA2cQSNLAGQSSNHAGgSQNnEEgSQNnEEjSwBkEkjRwBoEkDZxBIEkDt9ndWfxgLFl2/sTWveadL5zYuiVpQ9wjkKSBMwgkaeAMAkkaOINAkgbOIJCkgTMIJGngDAJJGjiDQJIGziCQpIEzCCRp4AwCSRo4g0CSBs4gkKSBMwgkaeB6DYIkhyW5IcnqJMtmmH5sktuSXNW+XttnPZKk39bb8wiSLABOAZ4LrAUuT7K8qq6bNuunq+qEvuqQJG1Yn3sEBwOrq+qmqroLOAM4osf1SZI2QZ9BsAdw88j42rZtuhcnuTrJ2Un26rEeSdIMJn2y+FxgSVXtB3wB+NhMMyU5LsnKJCtvu+22sRYoSVu6PoPgFmD0F/6ebdt9quonVXVnO/ph4MCZFlRVp1XV0qpaumjRol6KlaSh6jMILgf2SbJ3km2Ao4HlozMk2X1k9HDg+h7rkSTNoLerhqpqXZITgAuBBcDpVbUqycnAyqpaDrwxyeHAOuCnwLF91SNJmllvQQBQVSuAFdPa3jEy/FbgrX3WIEnasEmfLJYkTZhBIEkDZxBI0sAZBJI0cAaBJA2cQSBJA2cQSNLAGQSSNHAGgSQNnEEgSQNnEEjSwBkEkjRwBoEkDZxBIEkDZxBI0sAZBJI0cAaBJA2cQSBJA2cQSNLAGQSSNHAGgSQNnEEgSQO3UUGQZJck+/VVjCRp/GYNgiQXJ9kpya7AlcCHkry3/9IkSePQZY/gYVV1B/Ai4ONVdQjwnH7LkiSNS5cg2DrJ7sBLgPN6rkeSNGZdguBk4ELgO1V1eZJHAzf2W5YkaVy2nm2GqjoLOGtk/CbgxX0WJUkany4nix+d5NwktyW5Ncm/tHsFkqQtQJdDQ58EzgR2Bx5Js3fwqS4LT3JYkhuSrE6ybAPzvThJJVnaZbmSpLnTJQi2r6pPVNW69vWPwLazvSnJAuAU4PnAvsAxSfadYb6HAm8CLt240iVJc6FLEHwuybIkS5I8KslfACuS7NreW7A+BwOrq+qmqroLOAM4Yob5/hp4F/Cbja5ekvSgzXqymOayUYA/ndZ+NFDA+s4X7AHcPDK+FjhkdIYkBwB7VdX5Sd68vgKSHAccB7B48eIOJUuSuupy1dDefaw4yVbAe4FjO9RwGnAawNKlS6uPeiRpqGYNgiSvmKm9qj4+y1tvAfYaGd+zbZvyUODxwMVJAB4BLE9yeFWtnK0uSdLc6HJo6KCR4W2BZ9P0OTRbEFwO7JNkb5oAOBp42dTEqrodWDg1nuRi4M8NAUkary6Hht4wOp5kZ5oTv7O9b12SE2juSl4AnF5Vq5KcDKysquWbVrIkaS512SOY7ldAp/MGVbUCWDGt7R3rmfcZm1CLJOlB6nKO4Fyaq4Og+WX/+zQ3mEmStgBd9gjeMzK8DvheVa3tqR5J0pjNekNZVX0F+BbNVT67AHf1XZQkaXy6dDr3EuAy4Ciam8suTXJk34VJksajy6GhtwEHVdWtAEkWARcBZ/dZmCRpPLr0NbTVVAi0ftLxfZKkzUCXPYILklzI/V1Pv5Rpl4RKkjZfGwyCNH0//B3N3cWHts2nVdVn+i5MkjQeGwyCqqokK6rqPwDnjKkmSdIYdTnWf2WSg2afTZK0OepyjuAQ4OVJvkfTvURodhb267UySdJYdAmCP+y9CknSxHQJgl90bJMkbYY6nSMAbgO+DdzYDq9JcmWSA/ssTpLUvy5B8AXgBVW1sKp2A54PnAccD/x9n8VJkvrXJQieVFUXTo1U1eeBJ1fV14GH9FaZJGksupwj+EGSt3D/U8leCvwoyQLg3t4qkySNRZc9gpfRPHj+s8BnaB5I/zKah9S8pLfKJElj0eWZxT8G3rCeyavnthxJ0rjZi6gkDZxBIEkDt94gSPKu9u9R4ytHkjRuG9ojeEHbDfVbx1WMJGn8NnSy+ALgZ8COSe6g7WyO+zud22kM9UmSerbePYKqenNV7QycX1U7VdVDR/+Or0RJUp+6XD56RJKH0zylDODSqrqt37IkSeMy61VD7cniy4CjaG4guyzJkX0XJkkajy5dTLwdOKiqbgVIsgi4CDi7z8IkSePR5T6CraZCoPWTju+TJG0GuvwP/YIkFyY5NsmxwPnAii4LT3JYkhuSrE6ybIbpr0tyTZKrknwtyb4bV74k6cHqcrL4zUleBBzaNp1WVZ+Z7X1t76SnAM8F1gKXJ1leVdeNzPbJqjq1nf9w4L3AYRv5b5AkPQhdzhFQVecA52zksg8GVlfVTQBJzgCOAO4Lgqq6Y2T+HWjuU5AkjVGnINhEewA3j4yvBQ6ZPlOS1wMnAdsAz5ppQUmOA44DWLx48ZwXKklDNvGTvlV1SlU9BngLzRVKM81zWlUtraqlixYtGm+BkrSF6xQESbZL8tiNXPYtNA+xmbJn27Y+ZwB/vJHrkCQ9SF1uKPuPwFU0fQ+RZP8kyzss+3JgnyR7J9kGOBp4wPuS7DMy+kLgxo51S5LmSJdzBH9Jc+L3YoCquirJ3rO9qarWJTkBuJDmsZanV9WqJCcDK6tqOXBCkucAd9N0cPfKTfpXSJI2WZcguLuqbm96pL5Pp6t7qmoF0+45qKp3jAy/qctyJEn96RIEq5K8DFjQHsp5I/Cv/ZYlSRqXLieL3wD8e+BO4FPAHcCJPdYkSRqjLncW/xp4W/uSJG1hZg2CJOfy2+cEbgdWAh+sqt/0UZgkaTy6HBq6Cfgl8KH2dQfwC+DfteOSpM1Yl5PFT6mqg0bGz01yeVUdlGRVX4VJksajyx7Bjknu6+CnHd6xHb2rl6okSWPTZY/gvwFfS/IdIMDewPFJdgA+1mdxkqT+dblqaEV7/8Dj2qYbRk4Q/21fhUmSxqNrN9T7AI8FtgWekISq+nh/ZUmSxqXL5aP/E3gGsC9NdxHPB74GGASStAXocrL4SODZwA+r6lXAE4CH9VqVJGlsugTBv1XVvcC6JDsBt/LA5wxIkjZjXc4RrEyyM83NY1fQ3Fz2f/ssaku0ZNn5E1nvmne+cCLrlbT56HLV0PHt4KlJLgB2qqqr+y1LkjQuXZ5Q9sWp4apaU1VXj7ZJkjZv690jSLItsD2wMMkuNDeTAewE7DGG2iRJY7ChQ0N/SvPcgUfSnBuYCoI7gA/0W5YkaVzWGwRV9T7gfUneUFXvH2NNkqQx6nKy+P1JngIsGZ3fO4slacvQ5c7iTwCPAa4C7mmbC+8slqQtQpf7CJYC+1bV9KeUSZK2AF3uLL4WeETfhUiSJqPLHsFC4LoklwF3TjVW1eG9VSVJGpsuQfCXfRchSZqcLlcNfSXJo4B9quqiJNsDC/ovTZI0Dl26mPgT4Gzgg23THsBne6xJkjRGXU4Wvx74A5o7iqmqG4Hf7bMoSdL4dAmCO6vqrqmRJFvT3EcwqySHJbkhyeoky2aYflKS65JcneSL7SEoSdIYdQmCryT578B2SZ4LnAWcO9ubkiwATqF5tOW+wDFJ9p022zeApVW1H83hp3dvTPGSpAevSxAsA24DrqHpiG4F8PYO7zsYWF1VN7V7FGcAR4zOUFVfrqpft6NfB/bsWrgkaW50uXx0O+D0qvoQ3PdLfzvg1xt8V3NS+eaR8bXAIRuY/zXA52aakOQ44DiAxYsXdyhZktRVlz2CL9L8j3/KdsBFc1lEkv9M05XF/5ppelWdVlVLq2rpokWL5nLVkjR4XfYItq2qX06NVNUv23sJZnMLD3zI/Z5t2wMkeQ7wNuDpVXXn9OmSpH512SP4VZIDpkaSHAj8W4f3XQ7sk2TvJNsARwPLR2dI8kSa+xMOr6pbu5ctSZorXfYI3gScleT/0Tyl7BHAS2d7U1WtS3ICcCHNncinV9WqJCcDK6tqOc2hoB3b5QN83z6MJGm8NhgE7YnhpwKPAx7bNt9QVXd3WXhVraC5ymi07R0jw8/ZqGolSXNug4eGquoe4Jiquruqrm1fnUJAkrR56HJo6JIkHwA+DfxqqrGqruytKknS2HQJgv3bvyePtBXwrDmvRpI0dl26oX7mOAqRJE1Gl26oH57kI0k+147vm+Q1/ZcmSRqHLvcRfJTmEtBHtuPfBk7sqR5J0ph1CYKFVXUmcC809wcA9/RalSRpbLreWbwb7TMIkjwJuL3XqiRJY9PlqqGTaLqGeEySS4BFwJG9ViVJGpsuVw1dmeTpNHcWh424s1iSNP/NGgRJtgWOBw6lOTz01SSnVtVv+i5OktS/LoeGPg78Anh/O/4y4BPAUX0VJUkany5B8PiqGn3W8JeTXNdXQZKk8epy1dCV7ZVCACQ5BFjZX0mSpHHqskdwIPCvSb7fji8GbkhyDVBVtV9v1UmSetclCA7rvQpJ0sR0uXz0e+MoRJI0GV3OEUiStmAGgSQNnEEgSQNnEEjSwBkEkjRwBoEkDZxBIEkDZxBI0sAZBJI0cAaBJA2cQSBJA2cQSNLA9RoESQ5LckOS1UmWzTD9aUmuTLIuyZF91iJJmllvQZBkAXAK8HxgX+CYJPtOm+37wLHAJ/uqQ5K0YV2eR7CpDgZWV9VNAEnOAI4A7nvMZVWtaafd22MdkqQN6DMI9gBuHhlfCxyyKQtKchxwHMDixYsffGUaiyXLzp/Iete884UTWa+0udosThZX1WlVtbSqli5atGjS5UjSFqXPILgF2GtkfM+2TZI0j/QZBJcD+yTZO8k2wNHA8h7XJ0naBL0FQVWtA04ALgSuB86sqlVJTk5yOECSg5KsBY4CPphkVV/1SJJm1ufJYqpqBbBiWts7RoYvpzlkJEmakM3iZLEkqT8GgSQNnEEgSQNnEEjSwBkEkjRwBoEkDZxBIEkDZxBI0sAZBJI0cAaBJA2cQSBJA2cQSNLAGQSSNHC99j4qaTx8LOj4TOqzhv4+b/cIJGngDAJJGjiDQJIGziCQpIEzCCRp4AwCSRo4g0CSBs4gkKSBMwgkaeAMAkkaOINAkgbOIJCkgTMIJGngDAJJGjiDQJIGrtcgSHJYkhuSrE6ybIbpD0ny6Xb6pUmW9FmPJOm39RYESRYApwDPB/YFjkmy77TZXgP8rKp+D/gb4F191SNJmlmfewQHA6ur6qaqugs4Azhi2jxHAB9rh88Gnp0kPdYkSZqmz0dV7gHcPDK+FjhkffNU1boktwO7AT8enSnJccBx7egvk9wwx7UunL7OeWBOasrc72PN+8+qh3/zppj3n9NcmIPPej5+TjA/61qYdz2omh61vgmbxTOLq+o04LS+lp9kZVUt7Wv5m2I+1gTzsy5r6saaupuPdfVZU5+Hhm4B9hoZ37Ntm3GeJFsDDwN+0mNNkqRp+gyCy4F9kuydZBvgaGD5tHmWA69sh48EvlRV1WNNkqRpejs01B7zPwG4EFgAnF5Vq5KcDKysquXAR4BPJFkN/JQmLCaht8NOD8J8rAnmZ13W1I01dTcf6+rv8Lg/wCVp2LyzWJIGziCQpIEbRBAk2SvJl5Ncl2RVkje17bsm+UKSG9u/u7TtSfJ3bdcXVyc5oIeatk1yWZJvtjX9Vdu+d9vdxuq2+41t2vaxdceRZEGSbyQ5bz7UlGRNkmuSXJVkZds2se+uXc/OSc5O8q0k1yd58jyo6bHtZzT1uiPJifOgrj9rt/Frk3yq3fYnvU29qa1nVZIT27axfk5JTk9ya5JrR9o2uoYkr2znvzHJK2da16yqaot/AbsDB7TDDwW+TdPtxbuBZW37MuBd7fALgM8BAZ4EXNpDTQF2bId/B7i0XdeZwNFt+6nAf22HjwdObYePBj7d4+d1EvBJ4Lx2fKI1AWuAhdPaJvbdtev5GPDadngbYOdJ1zStvgXAD2luIprkdr4H8F1gu5Ft6dhJblPA44Frge1pLpi5CPi9cX9OwNOAA4BrN3W7BnYFbmr/7tIO77LRtfS9Qc7HF/AvwHOBG4Dd27bdgRva4Q8Cx4zMf998PdWzPXAlzZ3XPwa2btufDFzYDl8IPLkd3rqdLz3UsifwReBZwHnthjfpmtbw20Ewse+O5n6X707/t86X7aldx/OASyZdF/f3HrBru42cB/zhJLcp4CjgIyPj/wP4i0l8TsASHhgEG1UDcAzwwZH2B8zX9TWIQ0Oj2l3NJ9L8An94Vf2gnfRD4OHt8EzdY+zRQy0LklwF3Ap8AfgO8POqWjfDeh/QHQcw1R3HXPtbmv8o7m3Hd5sHNRXw+SRXpOluBCb73e0N3Ab87/YQ2oeT7DDhmqY7GvhUOzyxuqrqFuA9wPeBH9BsI1cw2W3qWuCpSXZLsj3Nr+29mB/f38bWMCe1DSoIkuwI/DNwYlXdMTqtmjgd67W0VXVPVe1P8yv8YOBx41z/dEn+CLi1qq6YZB0zOLSqDqDpyfb1SZ42OnEC393WNLv0/1BVTwR+RbMbP8ma7tMebz8cOGv6tHHX1R7jPoImPB8J7AAcNq71z6Sqrqfp6fjzwAXAVcA90+aZ2Pc3iRoGEwRJfocmBP6pqs5pm3+UZPd2+u40v8yhW/cYc6aqfg58mWYXeec03W1MX+84uuP4A+DwJGtoeot9FvC+Cdc09auSqroV+AxNaE7yu1sLrK2qS9vxs2mCYV5sTzSBeWVV/agdn2RdzwG+W1W3VdXdwDk029mkt6mPVNWBVfU04Gc05w3nw/e3sTXMSW2DCIIkobmL+fqqeu/IpNEuLl5Jc+5gqv0V7Zn6JwG3j+yuzVVNi5Ls3A5vR3PO4nqaQDhyPTX12h1HVb21qvasqiU0hxa+VFUvn2RNSXZI8tCpYZpj39cywe+uqn4I3JzksW3Ts4HrJlnTNMdw/2GhqfVPqq7vA09Ksn373+HUZzWxbQogye+2fxcDL6K5OGI+fH8bW8OFwPOS7NLufT2vbds4c3kSZr6+gENpdrGuptkNvIrmuOBuNCdGb6S5cmDXdv7QPFTnO8A1wNIeatoP+EZb07XAO9r2RwOXAatpdu0f0rZv246vbqc/uufP7Bncf9XQxGpq1/3N9rUKeFvbPrHvrl3P/sDK9vv7LM0VGxOtqV3XDjS/oB820jbpz+qvgG+12/kngIdMejsHvkoTSN8Enj2Jz4kmrH8A3E2zl/maTakBeHX7ea0GXrUptdjFhCQN3CAODUmS1s8gkKSBMwgkaeAMAkkaOINAkgbOIJCAJBcn6f1h5UnemKa30n/qe11SV709qlIaiiRb1/395szmeOA5VbW2z5pgo+vSgLlHoM1GkiXtr+kPtf3If769K/sBv+iTLGy7ySDJsUk+2/btvibJCUlOajuL+3qSXUdW8V/S9ON/bZKD2/fvkKbf+Mva9xwxstzlSb5EcwPQ9FpPapdzbe7v7/5UmhupPpfkz6bNf2ySD4yMn5fkGW3HhB9tl3PN1PuSPCbJBWk64vtqkse17R9NcmqSS4F3J3l67n8+wTem7tKWRrlHoM3NPjTd7P5JkjOBFwP/OMt7Hk/T4+y2NHdfvqWqnpjkb4BX0PS4CrB9Ve2fplO709v3vY2mm4NXt12CXJbkonb+A4D9quqnoytLciDwKppuxQNcmuQrVfW6JIcBz6yqH3f89+4P7FFVj2+XvXPbfhrwuqq6MckhwN/T9A0FTX8zT6mqe5KcC7y+qi5J0+nibzquVwNiEGhz892quqodvoKmP/fZfLmqfgH8IsntwLlt+zU0XX1M+RRAVf2fJDu1/9N9Hk1HfH/ezrMtsLgd/sL0EGgdCnymqn4FkOQc4Kk0XYpsrJuARyd5P3A+TXfcOwJPAc5quu8Bmm4bppxVVVO9aV4CvLc9J3HOOA5JafPjoSFtbu4cGb6H+3/MrOP+7XnbDbzn3pHxe3ngj6Hp/a0UzS/6F1fV/u1rcTXdGEPT/fRcGa0f2n9DVf0MeAJwMfA64MPtfD8fqWn/qvr9kffeV1dVvRN4LbAdcMnUISRplEGgLcUa4MB2+MgNzLchLwVIcihN74630/Tk+Ia250ySPLHDcr4K/HHb4+YOwH9q2zZkDbB/kq2S7EXT1TZJFgJbVdU/A2+neeTqHcB3kxzVzpMkT5hpoUkeU1XXVNW7gMuZ8DMvND95aEhbivcAZ6Z5gtn5m7iM3yT5Bs0zpF/dtv01zTmEq5NsRfOIyj/a0EKq6sokH6XpPRPgw1U122GhS9plX0fTHfmVbfseNE9Cm/rR9tb278uBf0jy9rbeM2h60pzuxCTPpNn7WUXz3FvpAex9VJIGzkNDkjRwBoEkDZxBIEkDZxBI0sAZBJI0cAaBJA2cQSBJA/f/Ab5z/UfODN0gAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(users_per_group, weights=np.ones(len(users_per_group)) / len(users_per_group))\n",
    "plt.xlabel(\"number of users\")\n",
    "plt.ylabel(\"percentage of groups\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I also create a trainset and a testset, which I will use to evaluate the recommender systems later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_membership.sort_values(by = ['last_seen'], inplace = True)\n",
    "testset = df_membership[(df_membership.messages_count >= 3) & (df_membership.messages_count <800)]\n",
    "testset = testset.drop_duplicates(subset = ['user_id'], keep = 'last')\n",
    "\n",
    "train_set = df_membership.drop(list(testset.index))\n",
    "testset = testset[testset.user_id.isin(train_set.user_id)]\n",
    "\n",
    "train_set.to_pickle(\"trainset.pkl\")\n",
    "testset.to_pickle(\"testset.pkl\")"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
  },
  "kernelspec": {
   "display_name": "Python 3.9.12 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}