notebooks/calculating_communities.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import snap\n",
    "import time, pandas as pd, pickle, json, networkx as nx, numpy as np\n",
    "from networkx.readwrite import json_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1636</th>\n",
       "      <td>319629696</td>\n",
       "      <td>2612870692</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5403</th>\n",
       "      <td>2207630876</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7823</th>\n",
       "      <td>4704017456</td>\n",
       "      <td>1400588754</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4159</th>\n",
       "      <td>1030991929</td>\n",
       "      <td>139032492</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3929</th>\n",
       "      <td>891024134</td>\n",
       "      <td>995806488</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      from_user_id  to_user_id             formation\n",
       "id                                                  \n",
       "1636     319629696  2612870692  {'2018.05.24': True}\n",
       "5403    2207630876   106086098  {'2018.05.08': True}\n",
       "7823    4704017456  1400588754  {'2018.05.24': True}\n",
       "4159    1030991929   139032492  {'2018.05.24': True}\n",
       "3929     891024134   995806488  {'2018.05.24': True}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cons = pd.read_csv(\"../REST/static/filtered_twitter_connections.csv\", index_col=\"id\")\n",
    "cons.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>is_org</th>\n",
       "      <th>lang</th>\n",
       "      <th>match_name</th>\n",
       "      <th>match_ratio</th>\n",
       "      <th>name</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>truncated_id</th>\n",
       "      <th>community</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4.158962e+09</th>\n",
       "      <td>17.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>mustafa karan</td>\n",
       "      <td>92.0</td>\n",
       "      <td>mustafa kar</td>\n",
       "      <td>mustafa07301267</td>\n",
       "      <td>415896221</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9.410563e+17</th>\n",
       "      <td>1.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>mehmet akinci</td>\n",
       "      <td>95.0</td>\n",
       "      <td>mehmet mehmet</td>\n",
       "      <td>mehmetm88352186</td>\n",
       "      <td>941056302</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.111740e+17</th>\n",
       "      <td>180.0</td>\n",
       "      <td>1554.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>yusuf adiguzel</td>\n",
       "      <td>96.0</td>\n",
       "      <td>Yusuf ADIGÜZEL</td>\n",
       "      <td>ysferc38</td>\n",
       "      <td>711174008</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.578003e+09</th>\n",
       "      <td>41.0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>ahmed ahmed</td>\n",
       "      <td>95.0</td>\n",
       "      <td>Mikaeel Ahmed</td>\n",
       "      <td>AhmedMikaeel</td>\n",
       "      <td>157800302</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.053651e+09</th>\n",
       "      <td>56.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>hakim alizada</td>\n",
       "      <td>92.0</td>\n",
       "      <td>Hakim Alizade</td>\n",
       "      <td>HakimAlizade</td>\n",
       "      <td>105365131</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              followers_count  friends_count  is_org lang      match_name  \\\n",
       "id                                                                          \n",
       "4.158962e+09             17.0          259.0   False   tr   mustafa karan   \n",
       "9.410563e+17              1.0           74.0   False   tr   mehmet akinci   \n",
       "7.111740e+17            180.0         1554.0   False   tr  yusuf adiguzel   \n",
       "1.578003e+09             41.0          171.0   False   en     ahmed ahmed   \n",
       "1.053651e+09             56.0          149.0   False   tr   hakim alizada   \n",
       "\n",
       "              match_ratio            name      screen_name  truncated_id  \\\n",
       "id                                                                         \n",
       "4.158962e+09         92.0     mustafa kar  mustafa07301267     415896221   \n",
       "9.410563e+17         95.0   mehmet mehmet  mehmetm88352186     941056302   \n",
       "7.111740e+17         96.0  Yusuf ADIGÜZEL         ysferc38     711174008   \n",
       "1.578003e+09         95.0   Mikaeel Ahmed     AhmedMikaeel     157800302   \n",
       "1.053651e+09         92.0   Hakim Alizade     HakimAlizade     105365131   \n",
       "\n",
       "             community  \n",
       "id                      \n",
       "4.158962e+09         1  \n",
       "9.410563e+17         1  \n",
       "7.111740e+17         1  \n",
       "1.578003e+09         1  \n",
       "1.053651e+09         1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twu_with_orgs = pd.read_csv(\"../REST/static/filtered_twitter_users.csv\", index_col=\"id\")\n",
    "twu_with_orgs.sample(5,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_orgs = twu_with_orgs[~twu_with_orgs.is_org]\n",
    "orgs = set(twu_with_orgs[twu_with_orgs.is_org].truncated_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nxg = nx.DiGraph()\n",
    "truncate = lambda x: int(str(int(x))[:9])\n",
    "for _, row in cons.iterrows():    \n",
    "    from_ = truncate(row[\"from_user_id\"])\n",
    "    to = truncate(row[\"to_user_id\"])\n",
    "    if from_ in non_orgs.truncated_id and to in non_orgs.truncated_id:\n",
    "        nxg.add_edge(from_, to)\n",
    "nxg = nxg.to_directed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "554"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nxg.nodes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = snap.PUNGraph.New()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in nxg.nodes():\n",
    "    g.AddNode(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f,t in nxg.edges():\n",
    "    g.AddEdge(f, t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "CmtyV = snap.TCnComV()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating communities for nodes that had an edge at least once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Took:', 13.290963888168335)\n",
      "The modularity of the network is 0.619059\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "modularity = snap.CommunityGirvanNewman(g, CmtyV)\n",
    "print(\"Took:\",time.time()-start)\n",
    "print (\"The modularity of the network is %f\" % modularity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Community\n",
      "22726182\n",
      "31348078\n",
      "54890014\n",
      "58895857\n",
      "59087918\n",
      "67326856\n",
      "78625770\n",
      "86759560\n",
      "90942568\n",
      "91697633\n",
      "93257032\n",
      "99132901\n",
      "99529992\n",
      "99947241\n",
      "105561852\n",
      "116874386\n",
      "119125729\n",
      "122483508\n",
      "122663527\n",
      "122762092\n",
      "127227279\n",
      "142105410\n",
      "142638478\n",
      "145593371\n",
      "148740936\n",
      "162000619\n",
      "162417303\n",
      "174239586\n",
      "179632598\n",
      "180698952\n",
      "182798429\n",
      "185153719\n",
      "190175889\n",
      "202960799\n",
      "212853481\n",
      "223989726\n",
      "224170610\n",
      "224265589\n",
      "225548294\n",
      "225925670\n",
      "226497085\n",
      "242772912\n",
      "257963571\n",
      "258835545\n",
      "267598434\n",
      "274019211\n",
      "283868273\n",
      "288988846\n",
      "292138819\n",
      "292878286\n",
      "293464262\n",
      "293910628\n",
      "294182546\n",
      "314054889\n",
      "314777891\n",
      "316015497\n",
      "326711720\n",
      "328485693\n",
      "328899293\n",
      "331175494\n",
      "334509225\n",
      "335134002\n",
      "337061839\n",
      "348415792\n",
      "349249329\n",
      "349791252\n",
      "359872305\n",
      "362783413\n",
      "365925387\n",
      "367753207\n",
      "376124670\n",
      "376196641\n",
      "380937967\n",
      "395971942\n",
      "397429102\n",
      "419545826\n",
      "425411580\n",
      "427576690\n",
      "429101069\n",
      "438540004\n",
      "439156133\n",
      "442653736\n",
      "446303177\n",
      "449856858\n",
      "450064320\n",
      "452056032\n",
      "460049810\n",
      "467503701\n",
      "481423604\n",
      "498656061\n",
      "501589589\n",
      "507737255\n",
      "512309670\n",
      "522595967\n",
      "535239715\n",
      "551235868\n",
      "553444867\n",
      "557654297\n",
      "561229157\n",
      "566067329\n",
      "568512503\n",
      "572763889\n",
      "574782021\n",
      "583011029\n",
      "591258159\n",
      "592015438\n",
      "606426453\n",
      "609944399\n",
      "619082349\n",
      "633057056\n",
      "636286508\n",
      "717060518\n",
      "733590338\n",
      "742645290\n",
      "746961858\n",
      "756048858\n",
      "759973424\n",
      "763402614\n",
      "778109382\n",
      "798758016\n",
      "804561234\n",
      "825818708\n",
      "848094679\n",
      "891024134\n",
      "927466333\n",
      "928454131\n",
      "995806488\n",
      "----------\n",
      "Community\n",
      "9347592\n",
      "14668733\n",
      "32660008\n",
      "34993423\n",
      "47148388\n",
      "52177613\n",
      "68714550\n",
      "71784886\n",
      "85105655\n",
      "90976741\n",
      "96313649\n",
      "97633683\n",
      "98861105\n",
      "99515495\n",
      "113748549\n",
      "118382186\n",
      "140029297\n",
      "143240944\n",
      "144299966\n",
      "153772652\n",
      "155516378\n",
      "159075428\n",
      "159637252\n",
      "160629698\n",
      "162427198\n",
      "166501878\n",
      "168113833\n",
      "169021655\n",
      "170793146\n",
      "171035700\n",
      "174621392\n",
      "187485055\n",
      "189881254\n",
      "192444527\n",
      "195842934\n",
      "199411640\n",
      "201264943\n",
      "202625973\n",
      "204943693\n",
      "205640168\n",
      "227806410\n",
      "229235347\n",
      "230850831\n",
      "239145054\n",
      "239496829\n",
      "243820223\n",
      "246973630\n",
      "248388604\n",
      "251200312\n",
      "253229470\n",
      "263167320\n",
      "269014960\n",
      "271062336\n",
      "293976465\n",
      "294301041\n",
      "300251281\n",
      "300277618\n",
      "301378912\n",
      "310141719\n",
      "313467951\n",
      "315824179\n",
      "315899534\n",
      "316372649\n",
      "318006413\n",
      "319629696\n",
      "320244557\n",
      "323516844\n",
      "326288590\n",
      "327649487\n",
      "331103502\n",
      "334651678\n",
      "340515012\n",
      "342114048\n",
      "342264107\n",
      "345589356\n",
      "349103508\n",
      "355058617\n",
      "380163113\n",
      "396480979\n",
      "409554221\n",
      "411064560\n",
      "413849104\n",
      "437858563\n",
      "442712505\n",
      "447440907\n",
      "471153734\n",
      "473392794\n",
      "477970480\n",
      "485735472\n",
      "493130488\n",
      "501944828\n",
      "505464968\n",
      "529401410\n",
      "593980771\n",
      "595592832\n",
      "611652164\n",
      "624787113\n",
      "630367566\n",
      "633680339\n",
      "708827666\n",
      "712567496\n",
      "715580484\n",
      "753514176\n",
      "753998209\n",
      "755276202\n",
      "765369582\n",
      "769185392\n",
      "786513674\n",
      "786963288\n",
      "811374260\n",
      "850130101\n",
      "886856594\n",
      "918815419\n",
      "928367738\n",
      "930518624\n",
      "939057318\n",
      "939433993\n",
      "944285684\n",
      "966522690\n",
      "967897650\n",
      "----------\n",
      "Community\n",
      "22809165\n",
      "31462811\n",
      "34364681\n",
      "39740301\n",
      "51723397\n",
      "71758894\n",
      "78342853\n",
      "79991662\n",
      "81448875\n",
      "86719278\n",
      "89195407\n",
      "92124041\n",
      "94434738\n",
      "95427945\n",
      "101091897\n",
      "101449077\n",
      "105962645\n",
      "106044999\n",
      "107515144\n",
      "110797114\n",
      "116572924\n",
      "118146998\n",
      "133680108\n",
      "139000702\n",
      "142253498\n",
      "144312972\n",
      "145723363\n",
      "147180161\n",
      "148469186\n",
      "149522762\n",
      "151606317\n",
      "159075864\n",
      "159584851\n",
      "164570071\n",
      "164596213\n",
      "165159065\n",
      "166909100\n",
      "170012983\n",
      "170827894\n",
      "171127878\n",
      "189138605\n",
      "205898019\n",
      "209491402\n",
      "209749537\n",
      "210043256\n",
      "215077402\n",
      "222479855\n",
      "227938071\n",
      "233548506\n",
      "238165726\n",
      "239427912\n",
      "245939291\n",
      "247186119\n",
      "250370642\n",
      "263330786\n",
      "267838228\n",
      "275094798\n",
      "284419963\n",
      "292670498\n",
      "301674726\n",
      "326967793\n",
      "349229868\n",
      "365154925\n",
      "367078062\n",
      "369583895\n",
      "371964113\n",
      "385567588\n",
      "388837812\n",
      "434269864\n",
      "439203857\n",
      "444987253\n",
      "476201697\n",
      "479901964\n",
      "495928885\n",
      "556381215\n",
      "577063332\n",
      "577063420\n",
      "591599255\n",
      "599257209\n",
      "605747177\n",
      "618131962\n",
      "625979067\n",
      "633302673\n",
      "743227844\n",
      "766348868\n",
      "802893386\n",
      "838077084\n",
      "859509014\n",
      "872029765\n",
      "889187688\n",
      "895516783\n",
      "900003228\n",
      "900508574\n",
      "904465758\n",
      "914136576\n",
      "946886773\n",
      "972451742\n",
      "----------\n",
      "Community\n",
      "11711812\n",
      "41118718\n",
      "55220437\n",
      "58197692\n",
      "67543826\n",
      "71555386\n",
      "72787036\n",
      "85659155\n",
      "97239341\n",
      "105170233\n",
      "111705489\n",
      "114893351\n",
      "120289786\n",
      "124118525\n",
      "142080258\n",
      "144098036\n",
      "162310009\n",
      "167588737\n",
      "168580477\n",
      "169590053\n",
      "175288383\n",
      "177500163\n",
      "186791036\n",
      "188474287\n",
      "193020479\n",
      "203458925\n",
      "210618996\n",
      "219547337\n",
      "242477797\n",
      "245400432\n",
      "249090858\n",
      "249755850\n",
      "260354514\n",
      "262198524\n",
      "286783351\n",
      "302222495\n",
      "314535919\n",
      "314595732\n",
      "316369262\n",
      "331002312\n",
      "336105588\n",
      "344339030\n",
      "344826965\n",
      "351220156\n",
      "358143971\n",
      "370287757\n",
      "400579145\n",
      "404173433\n",
      "416178898\n",
      "419092391\n",
      "462741096\n",
      "485742022\n",
      "487861840\n",
      "586372643\n",
      "606948763\n",
      "611495719\n",
      "614648680\n",
      "615555441\n",
      "722521657\n",
      "729427152\n",
      "754775306\n",
      "773231442\n",
      "853472868\n",
      "869852834\n",
      "928991347\n",
      "945462330\n",
      "953664194\n",
      "965098470\n",
      "982366771\n",
      "----------\n",
      "Community\n",
      "11723602\n",
      "37036731\n",
      "98857070\n",
      "113606251\n",
      "159967656\n",
      "166720591\n",
      "174188104\n",
      "175426030\n",
      "178728184\n",
      "179238549\n",
      "181709561\n",
      "195129988\n",
      "203065438\n",
      "214511627\n",
      "218095010\n",
      "248426373\n",
      "248726930\n",
      "265839474\n",
      "273393599\n",
      "276090463\n",
      "287251843\n",
      "306293307\n",
      "334139315\n",
      "347250524\n",
      "363307336\n",
      "367738612\n",
      "367968322\n",
      "376697378\n",
      "377901136\n",
      "380825940\n",
      "385953682\n",
      "390777621\n",
      "403994496\n",
      "425432346\n",
      "428560048\n",
      "462282747\n",
      "464803112\n",
      "465022392\n",
      "475116738\n",
      "512080374\n",
      "534601323\n",
      "539055869\n",
      "553470674\n",
      "572370285\n",
      "574630629\n",
      "576473680\n",
      "579186200\n",
      "586809124\n",
      "594065508\n",
      "712307286\n",
      "736366560\n",
      "741129642\n",
      "773205608\n",
      "791665500\n",
      "806216310\n",
      "827129070\n",
      "849339692\n",
      "891177368\n",
      "931667750\n",
      "----------\n",
      "Community\n",
      "45125220\n",
      "49669079\n",
      "54145022\n",
      "67318883\n",
      "76757587\n",
      "81094202\n",
      "93385894\n",
      "113954902\n",
      "121806922\n",
      "138348596\n",
      "153019444\n",
      "160188040\n",
      "179985645\n",
      "194475532\n",
      "231187264\n",
      "342302461\n",
      "378536512\n",
      "387801894\n",
      "450840581\n",
      "514220222\n",
      "533536556\n",
      "915539516\n",
      "950487481\n",
      "----------\n",
      "Community\n",
      "103194791\n",
      "111450444\n",
      "175334832\n",
      "298020453\n",
      "358094060\n",
      "369714081\n",
      "431102126\n",
      "504378378\n",
      "531079884\n",
      "799977030\n",
      "903302773\n",
      "944226110\n",
      "----------\n",
      "Community\n",
      "18971997\n",
      "86797964\n",
      "96452145\n",
      "99225698\n",
      "108986782\n",
      "122684595\n",
      "214628614\n",
      "258097015\n",
      "281127185\n",
      "281456045\n",
      "447455430\n",
      "541024754\n",
      "----------\n",
      "Community\n",
      "18864998\n",
      "18913663\n",
      "169209299\n",
      "179250667\n",
      "231908931\n",
      "326909005\n",
      "358763677\n",
      "770506495\n",
      "943572432\n",
      "----------\n",
      "Community\n",
      "29551320\n",
      "52171453\n",
      "80393750\n",
      "81829378\n",
      "227981296\n",
      "229517534\n",
      "344090521\n",
      "497860167\n",
      "----------\n",
      "Community\n",
      "71535350\n",
      "139032492\n",
      "186434161\n",
      "437765733\n",
      "479673085\n",
      "538874783\n",
      "805514119\n",
      "----------\n",
      "Community\n",
      "224727919\n",
      "266826023\n",
      "283536919\n",
      "350907157\n",
      "818364804\n",
      "----------\n",
      "Community\n",
      "333528691\n",
      "606482438\n",
      "----------\n",
      "Community\n",
      "253616057\n",
      "740721020\n",
      "----------\n",
      "Community\n",
      "63796949\n",
      "253259284\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "nodes_communities = {} # {node: [communities]}\n",
    "for i, Cmty in enumerate(CmtyV):\n",
    "    print (\"Community\")\n",
    "    for NI in Cmty:\n",
    "        nodes_communities.setdefault(NI, [])\n",
    "        nodes_communities[NI].append(i+2)\n",
    "        print (NI)\n",
    "    print(\"-\"*10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coms = set([i[0] for i in nodes_communities.values()])\n",
    "coms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n"
     ]
    }
   ],
   "source": [
    "print(len(coms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>is_org</th>\n",
       "      <th>lang</th>\n",
       "      <th>match_name</th>\n",
       "      <th>match_ratio</th>\n",
       "      <th>name</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>truncated_id</th>\n",
       "      <th>community</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4.158962e+09</th>\n",
       "      <td>17.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>mustafa karan</td>\n",
       "      <td>92.0</td>\n",
       "      <td>mustafa kar</td>\n",
       "      <td>mustafa07301267</td>\n",
       "      <td>415896221</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9.410563e+17</th>\n",
       "      <td>1.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>mehmet akinci</td>\n",
       "      <td>95.0</td>\n",
       "      <td>mehmet mehmet</td>\n",
       "      <td>mehmetm88352186</td>\n",
       "      <td>941056302</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.111740e+17</th>\n",
       "      <td>180.0</td>\n",
       "      <td>1554.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>yusuf adiguzel</td>\n",
       "      <td>96.0</td>\n",
       "      <td>Yusuf ADIGÜZEL</td>\n",
       "      <td>ysferc38</td>\n",
       "      <td>711174008</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.578003e+09</th>\n",
       "      <td>41.0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>ahmed ahmed</td>\n",
       "      <td>95.0</td>\n",
       "      <td>Mikaeel Ahmed</td>\n",
       "      <td>AhmedMikaeel</td>\n",
       "      <td>157800302</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.053651e+09</th>\n",
       "      <td>56.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>hakim alizada</td>\n",
       "      <td>92.0</td>\n",
       "      <td>Hakim Alizade</td>\n",
       "      <td>HakimAlizade</td>\n",
       "      <td>105365131</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              followers_count  friends_count  is_org lang      match_name  \\\n",
       "id                                                                          \n",
       "4.158962e+09             17.0          259.0   False   tr   mustafa karan   \n",
       "9.410563e+17              1.0           74.0   False   tr   mehmet akinci   \n",
       "7.111740e+17            180.0         1554.0   False   tr  yusuf adiguzel   \n",
       "1.578003e+09             41.0          171.0   False   en     ahmed ahmed   \n",
       "1.053651e+09             56.0          149.0   False   tr   hakim alizada   \n",
       "\n",
       "              match_ratio            name      screen_name  truncated_id  \\\n",
       "id                                                                         \n",
       "4.158962e+09         92.0     mustafa kar  mustafa07301267     415896221   \n",
       "9.410563e+17         95.0   mehmet mehmet  mehmetm88352186     941056302   \n",
       "7.111740e+17         96.0  Yusuf ADIGÜZEL         ysferc38     711174008   \n",
       "1.578003e+09         95.0   Mikaeel Ahmed     AhmedMikaeel     157800302   \n",
       "1.053651e+09         92.0   Hakim Alizade     HakimAlizade     105365131   \n",
       "\n",
       "             community  \n",
       "id                      \n",
       "4.158962e+09         1  \n",
       "9.410563e+17         1  \n",
       "7.111740e+17         1  \n",
       "1.578003e+09         1  \n",
       "1.053651e+09         1  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def assign_com(node):\n",
    "    try:\n",
    "        return str(nodes_communities[node][0])\n",
    "    except KeyError:\n",
    "        if node in orgs:\n",
    "            return \"foci\"  # this is a foci, not a user\n",
    "        else:\n",
    "            return \"1\" # there are no connections for this node\n",
    "    \n",
    "twu_with_orgs[\"community\"] = twu_with_orgs.truncated_id.apply(lambda node: assign_com(node))\n",
    "twu_with_orgs.sample(5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "twu_with_orgs.to_csv(\"../datasets/filtered_twitter_users.csv\", index_label=\"id\")\n",
    "twu_with_orgs.to_csv(\"../REST/static/filtered_twitter_users.csv\", index_label=\"id\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating network diameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../REST/static/networks/twitter_users_graph2.json', 'r') as f:\n",
    "    data = json.load(f)\n",
    "    nxg = json_graph.node_link_graph(data, directed=True)\n",
    "nxg = nxg.to_directed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def networkx_to_snappy(nxg, directed=False):\n",
    "    if directed:\n",
    "        g = snap.PNGraph.New()\n",
    "    else:\n",
    "        g = snap.PUNGraph.New()\n",
    "        \n",
    "    for n in nxg.nodes():\n",
    "        g.AddNode(n)\n",
    "    for f,t in nxg.edges():\n",
    "        g.AddEdge(f, t)\n",
    "        \n",
    "    return g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for node in nxg.nodes():\n",
    "    nxg.nodes[node][\"community\"] = twu_with_orgs.loc[node][\"community\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = nx.node_link_data(nxg)\n",
    "data[\"modularity\"] = modularity\n",
    "data[\"number_of_communities\"] = len(CmtyV)\n",
    "\n",
    "snappy_directed = networkx_to_snappy(nxg, True)\n",
    "snappy_undirected = networkx_to_snappy(nxg, False)\n",
    "\n",
    "diam_directed = snap.GetBfsFullDiam(snappy_directed, 100, True)\n",
    "diam_undirected = snap.GetBfsFullDiam(snappy_undirected, 100, False)\n",
    "\n",
    "data[\"diameter_directed\"]=diam_directed\n",
    "data[\"diameter_undirected\"]=diam_undirected\n",
    "with open('../visualization/twitter_users_graph2.json', 'w') as f:\n",
    "    json.dump(data, f, indent=4)\n",
    "    \n",
    "with open('../REST/static/networks/twitter_users_graph2.json', 'w') as f:\n",
    "    json.dump(data, f, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "722"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nxg.nodes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1280"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nxg.edges())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}