datasets/datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# filtered_twitter_users.csv\n",
    "#### https://github.com/AmmarRashed/EventOrient/blob/master/datasets/filtered_twitter_users.csv\n",
    "### 2849 Twitter users crawled from a list of sehir accounts using snowball sampling\n",
    "#### Id = Twitter ID\n",
    "#### followers_count = followers count on Twitter\n",
    "#### friends_count = friends count on Twitter\n",
    "#### is_org = is a sehir institutional account or just a user\n",
    "#### lang = the most used language by that twitter account\n",
    "#### match_name = most similar name, to twitter name, in the gmail contacts of sehir.edu and std.sehir.edu domains\n",
    "#### match_ratio = the similarity score between twitter name and most similar name from Sehir domain gmail contacts. Calculated using fuzzywuzzy WRatio  (match_ratio > 90, 291122559 id added manually)\n",
    "#### name = Twitter name\n",
    "#### screen_name = twitter screen name\n",
    "#### truncated_id = first 9 digits of the Twitter Id, to fit in libraries like SnapPy\n",
    "#### community = the community that account belongs to as calculated by girvan newman algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>is_org</th>\n",
       "      <th>lang</th>\n",
       "      <th>match_name</th>\n",
       "      <th>match_ratio</th>\n",
       "      <th>name</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>truncated_id</th>\n",
       "      <th>community</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3.072510e+07</th>\n",
       "      <td>170.0</td>\n",
       "      <td>319.0</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>kaveh nematipour</td>\n",
       "      <td>100.0</td>\n",
       "      <td>Kaveh Nematipour</td>\n",
       "      <td>knematipour</td>\n",
       "      <td>30725104</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.784500e+08</th>\n",
       "      <td>174.0</td>\n",
       "      <td>362.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>seyma inal</td>\n",
       "      <td>95.0</td>\n",
       "      <td>Seyma İnal</td>\n",
       "      <td>seyma_i</td>\n",
       "      <td>178450017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8.418170e+17</th>\n",
       "      <td>6.0</td>\n",
       "      <td>123.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>salih salih</td>\n",
       "      <td>95.0</td>\n",
       "      <td>Salih Çiriş</td>\n",
       "      <td>salih_ciris</td>\n",
       "      <td>841817036</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9.909540e+07</th>\n",
       "      <td>202921.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>murat demir</td>\n",
       "      <td>96.0</td>\n",
       "      <td>Murat Özdemir</td>\n",
       "      <td>Murat_Ozdemir_</td>\n",
       "      <td>99095397</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9.678976e+08</th>\n",
       "      <td>436.0</td>\n",
       "      <td>420.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>ahmet kaya</td>\n",
       "      <td>100.0</td>\n",
       "      <td>Ahmet Kaya</td>\n",
       "      <td>HocazadeAhmed</td>\n",
       "      <td>967897650</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              followers_count  friends_count  is_org lang        match_name  \\\n",
       "id                                                                            \n",
       "3.072510e+07            170.0          319.0   False   en  kaveh nematipour   \n",
       "1.784500e+08            174.0          362.0   False   tr        seyma inal   \n",
       "8.418170e+17              6.0          123.0   False   tr       salih salih   \n",
       "9.909540e+07         202921.0           45.0   False   tr       murat demir   \n",
       "9.678976e+08            436.0          420.0   False   tr        ahmet kaya   \n",
       "\n",
       "              match_ratio              name     screen_name  truncated_id  \\\n",
       "id                                                                          \n",
       "3.072510e+07        100.0  Kaveh Nematipour     knematipour      30725104   \n",
       "1.784500e+08         95.0        Seyma İnal         seyma_i     178450017   \n",
       "8.418170e+17         95.0       Salih Çiriş     salih_ciris     841817036   \n",
       "9.909540e+07         96.0     Murat Özdemir  Murat_Ozdemir_      99095397   \n",
       "9.678976e+08        100.0        Ahmet Kaya   HocazadeAhmed     967897650   \n",
       "\n",
       "             community  \n",
       "id                      \n",
       "3.072510e+07         1  \n",
       "1.784500e+08         1  \n",
       "8.418170e+17         1  \n",
       "9.909540e+07         1  \n",
       "9.678976e+08         3  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_users = pd.read_csv(\"../datasets/filtered_twitter_users.csv\", index_col=\"id\")\n",
    "twitter_users.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2849"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(twitter_users)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# filtered_twitter_connections.csv\n",
    "#### https://github.com/AmmarRashed/EventOrient/blob/master/datasets/filtered_twitter_connections.csv\n",
    "### 9031 Twitter following connections crawled at two different dates\n",
    "### from_user_id, to_user_id: the twitter id of the follower and the followed twitter account respectively in this following connection\n",
    "### formation: the date of the changes in the connection state (e.g. {'2018.05.08':True, '2018.05.18':False} means this connection was _created_ on  2018.05.08, and _removed_ on 2018.05.18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9026</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>737056442</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9027</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9028</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>1666891914</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9029</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>3064906390</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9030</th>\n",
       "      <td>997958654354755584</td>\n",
       "      <td>1536995378</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            from_user_id  to_user_id             formation\n",
       "id                                                        \n",
       "9026  995255575398240256   737056442  {'2018.05.24': True}\n",
       "9027  995255575398240256  1222491402  {'2018.05.24': True}\n",
       "9028  995255575398240256  1666891914  {'2018.05.24': True}\n",
       "9029  995255575398240256  3064906390  {'2018.05.24': True}\n",
       "9030  997958654354755584  1536995378  {'2018.05.24': True}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped_cons = pd.read_csv(\"../datasets/filtered_twitter_connections.csv\", index_col=\"id\")\n",
    "grouped_cons.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# contacts.csv\n",
    "#### https://github.com/AmmarRashed/EventOrient/blob/master/datasets/contacts.csv\n",
    "## 10557 gmail contacts  from std.sehir.edu.tr and sehir.edu.tr domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>First Name</th>\n",
       "      <th>Last Name</th>\n",
       "      <th>Primary Email</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10552</th>\n",
       "      <td>10552</td>\n",
       "      <td>Sukru</td>\n",
       "      <td>Olkun</td>\n",
       "      <td>sukruolkun@std.sehir.edu.tr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10553</th>\n",
       "      <td>10553</td>\n",
       "      <td>Zeynep Begum</td>\n",
       "      <td>Tanis</td>\n",
       "      <td>zeyneptanis@std.sehir.edu.tr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10554</th>\n",
       "      <td>10554</td>\n",
       "      <td>Anastasiya</td>\n",
       "      <td>Osipova</td>\n",
       "      <td>anastasiyaosipova@std.sehir.edu.tr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10555</th>\n",
       "      <td>10555</td>\n",
       "      <td>Omer</td>\n",
       "      <td>Koca</td>\n",
       "      <td>omerkoca@std.sehir.edu.tr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10556</th>\n",
       "      <td>10556</td>\n",
       "      <td>Muhammed Murat</td>\n",
       "      <td>Yilmaz</td>\n",
       "      <td>muratyilmaz@std.sehir.edu.tr</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id      First Name Last Name                       Primary Email\n",
       "10552  10552           Sukru     Olkun         sukruolkun@std.sehir.edu.tr\n",
       "10553  10553    Zeynep Begum     Tanis        zeyneptanis@std.sehir.edu.tr\n",
       "10554  10554      Anastasiya   Osipova  anastasiyaosipova@std.sehir.edu.tr\n",
       "10555  10555            Omer      Koca           omerkoca@std.sehir.edu.tr\n",
       "10556  10556  Muhammed Murat    Yilmaz        muratyilmaz@std.sehir.edu.tr"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sehir_directory = pd.read_csv('contacts.csv')\n",
    "sehir_directory.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10557"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sehir_directory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# fb_users.csv\n",
    "#### https://github.com/AmmarRashed/EventOrient/blob/master/datasets/fb_users.csv\n",
    "### 2754 facebook accounts from Şehir-related groups\n",
    "#### Sehir Dersler&Hocalar\t2346\n",
    "#### Sehir Duyuru\t309\n",
    "#### Sehir Lessons&Teachers 39\n",
    "#### Sehir Mezunlari\t35\n",
    "#### Sehir Muhendislik 25\n",
    "## .\n",
    "#### id = Facebook account id\n",
    "#### membership = group\n",
    "#### full_name is the concatenation  first_name, midle_name, last_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>membership</th>\n",
       "      <th>first_name</th>\n",
       "      <th>middle_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2749</th>\n",
       "      <td>1734068939990594</td>\n",
       "      <td>Sehir Muhendislik</td>\n",
       "      <td>zeynep</td>\n",
       "      <td>kezban</td>\n",
       "      <td>turgut</td>\n",
       "      <td>zeynep kezban turgut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2750</th>\n",
       "      <td>910830722417581</td>\n",
       "      <td>Sehir Muhendislik</td>\n",
       "      <td>reha</td>\n",
       "      <td>NaN</td>\n",
       "      <td>karltekin</td>\n",
       "      <td>reha karltekin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2751</th>\n",
       "      <td>10155959451848548</td>\n",
       "      <td>Sehir Muhendislik</td>\n",
       "      <td>aysenur</td>\n",
       "      <td>NaN</td>\n",
       "      <td>eser</td>\n",
       "      <td>aysenur eser</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2752</th>\n",
       "      <td>769153509959215</td>\n",
       "      <td>Sehir Muhendislik</td>\n",
       "      <td>hakan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>gurler</td>\n",
       "      <td>hakan gurler</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2753</th>\n",
       "      <td>1797189933648857</td>\n",
       "      <td>Sehir Muhendislik</td>\n",
       "      <td>esra</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uslupat</td>\n",
       "      <td>esra uslupat</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     id         membership first_name middle_name  last_name  \\\n",
       "2749   1734068939990594  Sehir Muhendislik     zeynep      kezban     turgut   \n",
       "2750    910830722417581  Sehir Muhendislik       reha         NaN  karltekin   \n",
       "2751  10155959451848548  Sehir Muhendislik    aysenur         NaN       eser   \n",
       "2752    769153509959215  Sehir Muhendislik      hakan         NaN     gurler   \n",
       "2753   1797189933648857  Sehir Muhendislik       esra         NaN    uslupat   \n",
       "\n",
       "                 full_name  \n",
       "2749  zeynep kezban turgut  \n",
       "2750        reha karltekin  \n",
       "2751          aysenur eser  \n",
       "2752          hakan gurler  \n",
       "2753          esra uslupat  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fb_users = pd.read_csv('fb_users2.csv', usecols=[\"id\",\n",
    "                                                \"membership\",\n",
    "                                                \"first_name\",\n",
    "                                                \"middle_name\",\n",
    "                                                \"last_name\",\n",
    "                                                \"full_name\"])\n",
    "fb_users.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>first_name</th>\n",
       "      <th>middle_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>membership</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Sehir Dersler&amp;Hocalar</th>\n",
       "      <td>2346</td>\n",
       "      <td>2341</td>\n",
       "      <td>506</td>\n",
       "      <td>2341</td>\n",
       "      <td>2341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sehir Duyuru</th>\n",
       "      <td>309</td>\n",
       "      <td>305</td>\n",
       "      <td>61</td>\n",
       "      <td>306</td>\n",
       "      <td>306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sehir Lessons&amp;Teachers</th>\n",
       "      <td>39</td>\n",
       "      <td>39</td>\n",
       "      <td>10</td>\n",
       "      <td>39</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sehir Mezunlari</th>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>7</td>\n",
       "      <td>34</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sehir Muhendislik</th>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          id  first_name  middle_name  last_name  full_name\n",
       "membership                                                                 \n",
       "Sehir Dersler&Hocalar   2346        2341          506       2341       2341\n",
       "Sehir Duyuru             309         305           61        306        306\n",
       "Sehir Lessons&Teachers    39          39           10         39         39\n",
       "Sehir Mezunlari           35          35            7         34         35\n",
       "Sehir Muhendislik         25          25            3         25         25"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fb_users.groupby('membership').count()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}