notebooks/calculating_closures.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import psycopg2\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from copy import deepcopy\n",
    "\n",
    "import ast\n",
    "import random\n",
    "import networkx as nx\n",
    "import time, unicodedata\n",
    "import itertools\n",
    "\n",
    "from fuzzywuzzy import fuzz\n",
    "from fuzzywuzzy import process\n",
    "\n",
    "from joblib import Parallel, delayed\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>is_org</th>\n",
       "      <th>lang</th>\n",
       "      <th>match_name</th>\n",
       "      <th>match_ratio</th>\n",
       "      <th>name</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>truncated_id</th>\n",
       "      <th>community</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2.232249e+09</th>\n",
       "      <td>66.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>ibrahim kurban</td>\n",
       "      <td>97.0</td>\n",
       "      <td>İbrahim Kurban</td>\n",
       "      <td>ikrbn6465</td>\n",
       "      <td>223224940</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.893993e+08</th>\n",
       "      <td>34.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>alper kodak</td>\n",
       "      <td>100.0</td>\n",
       "      <td>Alper Kodak</td>\n",
       "      <td>kodakalper1</td>\n",
       "      <td>589399262</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.468747e+08</th>\n",
       "      <td>276.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>halime karayel</td>\n",
       "      <td>100.0</td>\n",
       "      <td>Halime Karayel</td>\n",
       "      <td>Haluket</td>\n",
       "      <td>146874707</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6.664567e+07</th>\n",
       "      <td>79.0</td>\n",
       "      <td>161.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>kutsal koruk</td>\n",
       "      <td>100.0</td>\n",
       "      <td>Kutsal Koruk</td>\n",
       "      <td>kutsalkoruk</td>\n",
       "      <td>66645671</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.565966e+17</th>\n",
       "      <td>48.0</td>\n",
       "      <td>221.0</td>\n",
       "      <td>False</td>\n",
       "      <td>tr</td>\n",
       "      <td>sena gulseren</td>\n",
       "      <td>96.0</td>\n",
       "      <td>Sena Gülseren</td>\n",
       "      <td>senagulseren_</td>\n",
       "      <td>756596649</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              followers_count  friends_count  is_org lang      match_name  \\\n",
       "id                                                                          \n",
       "2.232249e+09             66.0          172.0   False   en  ibrahim kurban   \n",
       "5.893993e+08             34.0           29.0   False   en     alper kodak   \n",
       "1.468747e+08            276.0          160.0   False   tr  halime karayel   \n",
       "6.664567e+07             79.0          161.0   False   tr    kutsal koruk   \n",
       "7.565966e+17             48.0          221.0   False   tr   sena gulseren   \n",
       "\n",
       "              match_ratio            name    screen_name  truncated_id  \\\n",
       "id                                                                       \n",
       "2.232249e+09         97.0  İbrahim Kurban      ikrbn6465     223224940   \n",
       "5.893993e+08        100.0     Alper Kodak    kodakalper1     589399262   \n",
       "1.468747e+08        100.0  Halime Karayel        Haluket     146874707   \n",
       "6.664567e+07        100.0    Kutsal Koruk    kutsalkoruk      66645671   \n",
       "7.565966e+17         96.0   Sena Gülseren  senagulseren_     756596649   \n",
       "\n",
       "              community  \n",
       "id                       \n",
       "2.232249e+09          1  \n",
       "5.893993e+08          1  \n",
       "1.468747e+08          1  \n",
       "6.664567e+07          1  \n",
       "7.565966e+17          1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_users = pd.read_csv(\"../REST/static/filtered_twitter_users.csv\",index_col=\"id\")\n",
    "twitter_users.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "      <th>first_date</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4445</th>\n",
       "      <td>4354816696</td>\n",
       "      <td>113954902</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4118</th>\n",
       "      <td>3299537073</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3893</th>\n",
       "      <td>2940722500</td>\n",
       "      <td>955847747093057537</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4210</th>\n",
       "      <td>3407552944</td>\n",
       "      <td>334139315</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5603</th>\n",
       "      <td>945405496466575360</td>\n",
       "      <td>847741204204634112</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            from_user_id          to_user_id             formation  first_date\n",
       "id                                                                            \n",
       "4445          4354816696           113954902  {'2018.05.24': True}        True\n",
       "4118          3299537073           106086098  {'2018.05.08': True}        True\n",
       "3893          2940722500  955847747093057537  {'2018.05.08': True}        True\n",
       "4210          3407552944           334139315  {'2018.05.24': True}        True\n",
       "5603  945405496466575360  847741204204634112  {'2018.05.08': True}        True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cons = pd.read_csv(\"../REST/static/filtered_twitter_connections.csv\",index_col=\"id\")\n",
    "cons.sample(5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "str2dict = lambda d : ast.literal_eval(d)\n",
    "truncate = lambda x: int(str(int(x))[:9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cons.formation = cons.formation.apply(str2dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def present_in_date(changes_dates, queried_date):\n",
    "    \"\"\"\n",
    "    checking if a connection is present in a queried date\n",
    "    changes_dates: {d1:True, d2:False, d3:True} connection added or removed\n",
    "    queried_date: e.g. \"2018.05.08\"\n",
    "    \"\"\"\n",
    "    str2date = lambda strdate: datetime.strptime(strdate, '%Y.%m.%d')  # 2018.05.08\n",
    "    changes = sorted(changes_dates,key=lambda d: str2date(d))\n",
    "    queried_date = datetime.strptime(queried_date, '%Y.%m.%d')\n",
    "    present = False\n",
    "    for d in changes:\n",
    "        if queried_date < str2date(d):\n",
    "            break\n",
    "        present = changes_dates[d]\n",
    "    return present"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dates(cons):\n",
    "    all_dates = set()\n",
    "    str2date = lambda strdate: datetime.strptime(strdate, '%Y.%m.%d')  # 2018.05.08\n",
    "\n",
    "    for dates in cons.formation.apply(lambda x: list(x)):\n",
    "        for date in dates:\n",
    "            all_dates.add(str2date(date))\n",
    "    return [d.strftime('%Y.%m.%d') for d in sorted(all_dates)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates = get_dates(cons)\n",
    "d1 = dates[0]\n",
    "d2 = dates[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_connections_by_date(cons, date, present=True):\n",
    "    nw = deepcopy(cons)\n",
    "    for_col = nw.formation.apply(lambda dates: present_in_date(dates,date))\n",
    "    return cons[for_col == present]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_new_edges(cons, d1, d2):\n",
    "    return get_connections_by_date(get_connections_by_date(cons, d2), d1, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "took:  0.2541506290435791\n"
     ]
    }
   ],
   "source": [
    "s = time.time()\n",
    "new_edges = calculate_new_edges(cons, d1, d2)\n",
    "print(\"took: \",time.time()-s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match_name</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>match_ratio</th>\n",
       "      <th>name</th>\n",
       "      <th>lang</th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>truncated_id</th>\n",
       "      <th>is_org</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1.289458e+09</th>\n",
       "      <td>suha demirsoy</td>\n",
       "      <td>suha_demirsoy</td>\n",
       "      <td>96</td>\n",
       "      <td>süha demirsoy</td>\n",
       "      <td>tr</td>\n",
       "      <td>18.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>128945774</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.685805e+08</th>\n",
       "      <td>erkan akdemir</td>\n",
       "      <td>srkn_akdemir</td>\n",
       "      <td>96</td>\n",
       "      <td>serkan akdemir</td>\n",
       "      <td>en</td>\n",
       "      <td>69.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>168580477</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9.442261e+08</th>\n",
       "      <td>sumeyra yerinde</td>\n",
       "      <td>sumeyrayerinde</td>\n",
       "      <td>97</td>\n",
       "      <td>Sümeyra Yerinde</td>\n",
       "      <td>tr</td>\n",
       "      <td>110.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>944226110</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.341393e+08</th>\n",
       "      <td>beyza cengiz</td>\n",
       "      <td>Cengizbeyza</td>\n",
       "      <td>100</td>\n",
       "      <td>Beyza Cengiz</td>\n",
       "      <td>tr</td>\n",
       "      <td>642.0</td>\n",
       "      <td>314.0</td>\n",
       "      <td>334139315</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.406830e+07</th>\n",
       "      <td>emrah kaya</td>\n",
       "      <td>emrahkaya</td>\n",
       "      <td>100</td>\n",
       "      <td>Emrah Kaya</td>\n",
       "      <td>en</td>\n",
       "      <td>87334.0</td>\n",
       "      <td>438.0</td>\n",
       "      <td>14068305</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   match_name     screen_name  match_ratio             name  \\\n",
       "id                                                                            \n",
       "1.289458e+09    suha demirsoy   suha_demirsoy           96    süha demirsoy   \n",
       "1.685805e+08    erkan akdemir    srkn_akdemir           96   serkan akdemir   \n",
       "9.442261e+08  sumeyra yerinde  sumeyrayerinde           97  Sümeyra Yerinde   \n",
       "3.341393e+08     beyza cengiz     Cengizbeyza          100     Beyza Cengiz   \n",
       "1.406830e+07       emrah kaya       emrahkaya          100       Emrah Kaya   \n",
       "\n",
       "             lang  followers_count  friends_count  truncated_id  is_org  \n",
       "id                                                                       \n",
       "1.289458e+09   tr             18.0           36.0     128945774   False  \n",
       "1.685805e+08   en             69.0          144.0     168580477   False  \n",
       "9.442261e+08   tr            110.0           94.0     944226110   False  \n",
       "3.341393e+08   tr            642.0          314.0     334139315   False  \n",
       "1.406830e+07   en          87334.0          438.0      14068305   False  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_users = pd.read_csv(\"../REST/static/filtered_twitter_users.csv\", index_col=\"id\")\n",
    "twitter_users.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorizing edges (user-user vs user-foci)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>lang</th>\n",
       "      <th>match_name</th>\n",
       "      <th>match_ratio</th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>truncated_id</th>\n",
       "      <th>is_org</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7.819232e+17</th>\n",
       "      <td>Şehir Gerçekleri</td>\n",
       "      <td>sehirgercekleri</td>\n",
       "      <td>tr</td>\n",
       "      <td>Şehir Üniversitesi (İnsan ve Toplum Bilimleri...</td>\n",
       "      <td>86.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>781923229</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.361548e+09</th>\n",
       "      <td>cefa</td>\n",
       "      <td>cefasehirfm</td>\n",
       "      <td>tr</td>\n",
       "      <td>Dspace Admin</td>\n",
       "      <td>68.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>463.0</td>\n",
       "      <td>236154753</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.847234e+09</th>\n",
       "      <td>Şehir Muzik Kulübü</td>\n",
       "      <td>sehirmusicclub</td>\n",
       "      <td>tr</td>\n",
       "      <td>Şehir Üniversitesi (İnsan ve Toplum Bilimleri...</td>\n",
       "      <td>86.0</td>\n",
       "      <td>243.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>284723384</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8.057471e+17</th>\n",
       "      <td>In-House @ Şehir</td>\n",
       "      <td>inhouse_sehir</td>\n",
       "      <td>tr</td>\n",
       "      <td>Şehir Üniversitesi (İnsan ve Toplum Bilimleri...</td>\n",
       "      <td>86.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>805747110</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.330558e+08</th>\n",
       "      <td>Şehirli Aslanlar</td>\n",
       "      <td>uA_SehirUNI</td>\n",
       "      <td>tr</td>\n",
       "      <td>! Sehir Sanat</td>\n",
       "      <td>69.0</td>\n",
       "      <td>1672.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>533055810</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            name      screen_name lang  \\\n",
       "id                                                       \n",
       "7.819232e+17    Şehir Gerçekleri  sehirgercekleri   tr   \n",
       "2.361548e+09                cefa      cefasehirfm   tr   \n",
       "2.847234e+09  Şehir Muzik Kulübü   sehirmusicclub   tr   \n",
       "8.057471e+17    In-House @ Şehir    inhouse_sehir   tr   \n",
       "5.330558e+08    Şehirli Aslanlar      uA_SehirUNI   tr   \n",
       "\n",
       "                                                     match_name  match_ratio  \\\n",
       "id                                                                             \n",
       "7.819232e+17   Şehir Üniversitesi (İnsan ve Toplum Bilimleri...         86.0   \n",
       "2.361548e+09                                       Dspace Admin         68.0   \n",
       "2.847234e+09   Şehir Üniversitesi (İnsan ve Toplum Bilimleri...         86.0   \n",
       "8.057471e+17   Şehir Üniversitesi (İnsan ve Toplum Bilimleri...         86.0   \n",
       "5.330558e+08                                      ! Sehir Sanat         69.0   \n",
       "\n",
       "              followers_count  friends_count  truncated_id  is_org  \n",
       "id                                                                  \n",
       "7.819232e+17              9.0          120.0     781923229    True  \n",
       "2.361548e+09            235.0          463.0     236154753    True  \n",
       "2.847234e+09            243.0          250.0     284723384    True  \n",
       "8.057471e+17             41.0            8.0     805747110    True  \n",
       "5.330558e+08           1672.0           61.0     533055810    True  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orgs_df = pd.read_csv(\"../REST/static/orgs.csv\", index_col=\"id\")\n",
    "orgs_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "orgs = set(orgs_df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_focal = lambda from_,to, orgs: (0,0) if (from_ in orgs and to in orgs)\\\n",
    "                                        or not(from_ in orgs or to in orgs)\\\n",
    "                                            else ((to,from_) if to in orgs else (from_,to))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "      <th>first_date</th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>47148388</td>\n",
       "      <td>1945992630</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>1945992630</td>\n",
       "      <td>47148388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4388</th>\n",
       "      <td>4217413882</td>\n",
       "      <td>618131962</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>4217413882</td>\n",
       "      <td>618131962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4242</th>\n",
       "      <td>3627550275</td>\n",
       "      <td>2612870692</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>3627550275</td>\n",
       "      <td>2612870692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5725</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>1666891914</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>1666891914</td>\n",
       "      <td>995255575398240256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5120</th>\n",
       "      <td>831060584166719489</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>106086098</td>\n",
       "      <td>831060584166719489</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            from_user_id  to_user_id             formation  first_date  \\\n",
       "id                                                                       \n",
       "69              47148388  1945992630  {'2018.05.24': True}        True   \n",
       "4388          4217413882   618131962  {'2018.05.24': True}        True   \n",
       "4242          3627550275  2612870692  {'2018.05.24': True}        True   \n",
       "5725  995255575398240256  1666891914  {'2018.05.24': True}        True   \n",
       "5120  831060584166719489   106086098  {'2018.05.24': True}        True   \n",
       "\n",
       "            foci             nonfoci  \n",
       "id                                    \n",
       "69    1945992630            47148388  \n",
       "4388  4217413882           618131962  \n",
       "4242  3627550275          2612870692  \n",
       "5725  1666891914  995255575398240256  \n",
       "5120   106086098  831060584166719489  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_edges[\"foci\"] = new_edges.apply(lambda row:is_focal(row['from_user_id'], row['to_user_id'], orgs), axis=1)\n",
    "new_edges[\"nonfoci\"] = new_edges.foci.apply(lambda f: f[1])\n",
    "new_edges[\"foci\"] = new_edges.foci.apply(lambda f: f[0])\n",
    "new_edges[new_edges.foci!=0].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculating membership closures\n",
    "### if a user u1 just got a new connection with foci f1 at time t2, get friends of u1 who had connections with f1 at time t1\n",
    "<img src=\"../misc/member.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "      <th>first_date</th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1120</th>\n",
       "      <td>374146195</td>\n",
       "      <td>2940722500</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>2940722500</td>\n",
       "      <td>374146195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2516</th>\n",
       "      <td>1439588580</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>106086098</td>\n",
       "      <td>1439588580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1711</th>\n",
       "      <td>589516467</td>\n",
       "      <td>4415560576</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>4415560576</td>\n",
       "      <td>589516467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4021</th>\n",
       "      <td>3093872810</td>\n",
       "      <td>174415744</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>174415744</td>\n",
       "      <td>3093872810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2078</th>\n",
       "      <td>802893386</td>\n",
       "      <td>737056442</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>737056442</td>\n",
       "      <td>802893386</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      from_user_id  to_user_id             formation  first_date        foci  \\\n",
       "id                                                                             \n",
       "1120     374146195  2940722500  {'2018.05.08': True}        True  2940722500   \n",
       "2516    1439588580   106086098  {'2018.05.08': True}        True   106086098   \n",
       "1711     589516467  4415560576  {'2018.05.08': True}        True  4415560576   \n",
       "4021    3093872810   174415744  {'2018.05.08': True}        True   174415744   \n",
       "2078     802893386   737056442  {'2018.05.08': True}        True   737056442   \n",
       "\n",
       "         nonfoci  \n",
       "id                \n",
       "1120   374146195  \n",
       "2516  1439588580  \n",
       "1711   589516467  \n",
       "4021  3093872810  \n",
       "2078   802893386  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1 = get_connections_by_date(cons, d1)\n",
    "t1[\"foci\"] = t1.apply(lambda row:is_focal(row['from_user_id'], row['to_user_id'], orgs), axis=1)\n",
    "t1[\"nonfoci\"] = t1.foci.apply(lambda f: f[1])\n",
    "t1[\"foci\"] = t1.foci.apply(lambda f: f[0])\n",
    "t1[t1.foci!=0].sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id_x</th>\n",
       "      <th>to_user_id_x</th>\n",
       "      <th>formation_x</th>\n",
       "      <th>first_date_x</th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci_x</th>\n",
       "      <th>from_user_id_y</th>\n",
       "      <th>to_user_id_y</th>\n",
       "      <th>formation_y</th>\n",
       "      <th>first_date_y</th>\n",
       "      <th>nonfoci_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4569</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>3234065324</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>3234065324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4596</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>3064906390</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>3064906390</td>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>153019444</td>\n",
       "      <td>3064906390</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>153019444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3574</th>\n",
       "      <td>996760635730952194</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>106086098</td>\n",
       "      <td>996760635730952194</td>\n",
       "      <td>106086098</td>\n",
       "      <td>162310009</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>162310009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4281</th>\n",
       "      <td>831060584166719489</td>\n",
       "      <td>609301446</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>609301446</td>\n",
       "      <td>831060584166719489</td>\n",
       "      <td>886280869991895041</td>\n",
       "      <td>609301446</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>886280869991895041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>465062899</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>465062899</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          from_user_id_x  to_user_id_x           formation_x  first_date_x  \\\n",
       "4569  995255575398240256    1222491402  {'2018.05.24': True}          True   \n",
       "4596  995255575398240256    3064906390  {'2018.05.24': True}          True   \n",
       "3574  996760635730952194     106086098  {'2018.05.24': True}          True   \n",
       "4281  831060584166719489     609301446  {'2018.05.24': True}          True   \n",
       "4488  995255575398240256    1222491402  {'2018.05.24': True}          True   \n",
       "\n",
       "            foci           nonfoci_x      from_user_id_y  to_user_id_y  \\\n",
       "4569  1222491402  995255575398240256          3234065324    1222491402   \n",
       "4596  3064906390  995255575398240256           153019444    3064906390   \n",
       "3574   106086098  996760635730952194           106086098     162310009   \n",
       "4281   609301446  831060584166719489  886280869991895041     609301446   \n",
       "4488  1222491402  995255575398240256           465062899    1222491402   \n",
       "\n",
       "               formation_y  first_date_y           nonfoci_y  \n",
       "4569  {'2018.05.08': True}          True          3234065324  \n",
       "4596  {'2018.05.08': True}          True           153019444  \n",
       "3574  {'2018.05.08': True}          True           162310009  \n",
       "4281  {'2018.05.08': True}          True  886280869991895041  \n",
       "4488  {'2018.05.08': True}          True           465062899  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# foci'es present at t1 and t2\n",
    "mutual_foci = new_edges[new_edges.foci!=0].merge(t1[t1.foci!=0], on=\"foci\")\n",
    "mutual_foci.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check for membership closure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def membership_closure(cons, u1, u2):\n",
    "    truncate = lambda x: int(str(int(x))[:9])\n",
    "    u1_friends = set(\n",
    "        cons.apply(lambda row: truncate(row['to_user_id'])if truncate(u1) == truncate(row['from_user_id'])\\\n",
    "                 else (truncate(row['from_user_id']) if truncate(u1) == truncate(row['to_user_id'])\\\n",
    "                   else np.NaN), axis=1).dropna())\n",
    "    return u2 in u1_friends"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took: 395.7770209312439\n",
      "There are 0 membership_closures\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id_x</th>\n",
       "      <th>to_user_id_x</th>\n",
       "      <th>formation_x</th>\n",
       "      <th>first_date_x</th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci_x</th>\n",
       "      <th>from_user_id_y</th>\n",
       "      <th>to_user_id_y</th>\n",
       "      <th>formation_y</th>\n",
       "      <th>first_date_y</th>\n",
       "      <th>nonfoci_y</th>\n",
       "      <th>membership_closure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2702</th>\n",
       "      <td>932632343160020993</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>106086098</td>\n",
       "      <td>932632343160020993</td>\n",
       "      <td>3298803633</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>3298803633</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2674</th>\n",
       "      <td>932632343160020993</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>106086098</td>\n",
       "      <td>932632343160020993</td>\n",
       "      <td>2948587163</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>2948587163</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4478</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>342114048</td>\n",
       "      <td>1222491402</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>342114048</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4376</th>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>737056442</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>737056442</td>\n",
       "      <td>995255575398240256</td>\n",
       "      <td>512080374</td>\n",
       "      <td>737056442</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>512080374</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2616</th>\n",
       "      <td>932632343160020993</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>106086098</td>\n",
       "      <td>932632343160020993</td>\n",
       "      <td>2336600952</td>\n",
       "      <td>106086098</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>2336600952</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          from_user_id_x  to_user_id_x           formation_x  first_date_x  \\\n",
       "2702  932632343160020993     106086098  {'2018.05.24': True}          True   \n",
       "2674  932632343160020993     106086098  {'2018.05.24': True}          True   \n",
       "4478  995255575398240256    1222491402  {'2018.05.24': True}          True   \n",
       "4376  995255575398240256     737056442  {'2018.05.24': True}          True   \n",
       "2616  932632343160020993     106086098  {'2018.05.24': True}          True   \n",
       "\n",
       "            foci           nonfoci_x  from_user_id_y  to_user_id_y  \\\n",
       "2702   106086098  932632343160020993      3298803633     106086098   \n",
       "2674   106086098  932632343160020993      2948587163     106086098   \n",
       "4478  1222491402  995255575398240256       342114048    1222491402   \n",
       "4376   737056442  995255575398240256       512080374     737056442   \n",
       "2616   106086098  932632343160020993      2336600952     106086098   \n",
       "\n",
       "               formation_y  first_date_y   nonfoci_y  membership_closure  \n",
       "2702  {'2018.05.08': True}          True  3298803633               False  \n",
       "2674  {'2018.05.08': True}          True  2948587163               False  \n",
       "4478  {'2018.05.08': True}          True   342114048               False  \n",
       "4376  {'2018.05.08': True}          True   512080374               False  \n",
       "2616  {'2018.05.08': True}          True  2336600952               False  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start = time.time()\n",
    "mutual_foci['membership_closure'] = mutual_foci.apply(lambda row: \n",
    "                                                      membership_closure(t1,\n",
    "                                                                         row['nonfoci_x'],\n",
    "                                                                         row['nonfoci_y']),axis=1)\n",
    "print(\"Took:\", time.time()-start)\n",
    "print(\"There are\", sum(mutual_foci[\"membership_closure\"]), \"membership_closures\")\n",
    "mutual_foci.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculating focal closures\n",
    "### if a user u1 just got a new connection with another user u2 at time t2, and u1 and u2 have been connected to foci f1 at time t1\n",
    "<img src=\"../misc/focal.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "      <th>first_date</th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2416</th>\n",
       "      <td>1241036822</td>\n",
       "      <td>618131962</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>704</th>\n",
       "      <td>233548506</td>\n",
       "      <td>618131962</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3632</th>\n",
       "      <td>2810110713</td>\n",
       "      <td>618131962</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1054</th>\n",
       "      <td>353631855</td>\n",
       "      <td>3421922319</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3093</th>\n",
       "      <td>2255825730</td>\n",
       "      <td>1537318436</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      from_user_id  to_user_id             formation  first_date  foci  \\\n",
       "id                                                                       \n",
       "2416    1241036822   618131962  {'2018.05.24': True}        True     0   \n",
       "704      233548506   618131962  {'2018.05.24': True}        True     0   \n",
       "3632    2810110713   618131962  {'2018.05.24': True}        True     0   \n",
       "1054     353631855  3421922319  {'2018.05.24': True}        True     0   \n",
       "3093    2255825730  1537318436  {'2018.05.24': True}        True     0   \n",
       "\n",
       "      nonfoci  \n",
       "id             \n",
       "2416        0  \n",
       "704         0  \n",
       "3632        0  \n",
       "1054        0  \n",
       "3093        0  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get all new user-user connections at t2\n",
    "new_u2u_edges = new_edges[new_edges.foci==0]\n",
    "new_u2u_edges.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "      <th>first_date</th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>90976741</td>\n",
       "      <td>2847233849</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>2847233849</td>\n",
       "      <td>90976741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2304</th>\n",
       "      <td>1076267664</td>\n",
       "      <td>2940722500</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>2940722500</td>\n",
       "      <td>1076267664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1090</th>\n",
       "      <td>365164239</td>\n",
       "      <td>2940722500</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>2940722500</td>\n",
       "      <td>365164239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1214</th>\n",
       "      <td>411064560</td>\n",
       "      <td>4091594428</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>4091594428</td>\n",
       "      <td>411064560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4830</th>\n",
       "      <td>783600969393303556</td>\n",
       "      <td>2940722500</td>\n",
       "      <td>{'2018.05.08': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>2940722500</td>\n",
       "      <td>783600969393303556</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            from_user_id  to_user_id             formation  first_date  \\\n",
       "id                                                                       \n",
       "226             90976741  2847233849  {'2018.05.08': True}        True   \n",
       "2304          1076267664  2940722500  {'2018.05.08': True}        True   \n",
       "1090           365164239  2940722500  {'2018.05.08': True}        True   \n",
       "1214           411064560  4091594428  {'2018.05.08': True}        True   \n",
       "4830  783600969393303556  2940722500  {'2018.05.08': True}        True   \n",
       "\n",
       "            foci             nonfoci  \n",
       "id                                    \n",
       "226   2847233849            90976741  \n",
       "2304  2940722500          1076267664  \n",
       "1090  2940722500           365164239  \n",
       "1214  4091594428           411064560  \n",
       "4830  2940722500  783600969393303556  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "foci_t1 = t1[t1.foci!=0]\n",
    "foci_t1.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2529427087</td>\n",
       "      <td>{2866935552, 709465725648424960, 7688711934704...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>737056442</td>\n",
       "      <td>{764021907171663872, 798186610583928833, 79882...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>443551180</td>\n",
       "      <td>{723926465679646720, 745389817765203968, 76023...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2207630876</td>\n",
       "      <td>{342114048, 916911351632711680, 70946572564842...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>3892757176</td>\n",
       "      <td>{989792178183471104, 704764994857078784, 70946...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          foci                                            nonfoci\n",
       "18  2529427087  {2866935552, 709465725648424960, 7688711934704...\n",
       "8    737056442  {764021907171663872, 798186610583928833, 79882...\n",
       "3    443551180  {723926465679646720, 745389817765203968, 76023...\n",
       "15  2207630876  {342114048, 916911351632711680, 70946572564842...\n",
       "24  3892757176  {989792178183471104, 704764994857078784, 70946..."
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_with_mut_foci = foci_t1.groupby(\"foci\")['nonfoci'].apply(lambda x: set(x)).reset_index()\n",
    "users_with_mut_foci = users_with_mut_foci[users_with_mut_foci.nonfoci.apply(lambda x: len(x)>1)]\n",
    "users_with_mut_foci.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_fclosure(f, t, users_with_mut_foci):\n",
    "    for _, focal in users_with_mut_foci.iterrows():\n",
    "        if f in focal[\"nonfoci\"] and t in focal[\"nonfoci\"]:\n",
    "            return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "      <th>first_date</th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>32660008</td>\n",
       "      <td>253616057</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14668733</td>\n",
       "      <td>1537318436</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3902</th>\n",
       "      <td>2966940226</td>\n",
       "      <td>113954902</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5497</th>\n",
       "      <td>907489214643818496</td>\n",
       "      <td>618131962</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999</th>\n",
       "      <td>800742744838709248</td>\n",
       "      <td>162310009</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            from_user_id  to_user_id             formation  first_date  foci  \\\n",
       "id                                                                             \n",
       "20              32660008   253616057  {'2018.05.24': True}        True     0   \n",
       "3               14668733  1537318436  {'2018.05.24': True}        True     0   \n",
       "3902          2966940226   113954902  {'2018.05.24': True}        True     0   \n",
       "5497  907489214643818496   618131962  {'2018.05.24': True}        True     0   \n",
       "4999  800742744838709248   162310009  {'2018.05.24': True}        True     0   \n",
       "\n",
       "      nonfoci  \n",
       "id             \n",
       "20          0  \n",
       "3           0  \n",
       "3902        0  \n",
       "5497        0  \n",
       "4999        0  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_u2u_edges.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "took: 0.5408101081848145\n",
      "There are 638 focal closures\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_user_id</th>\n",
       "      <th>to_user_id</th>\n",
       "      <th>formation</th>\n",
       "      <th>first_date</th>\n",
       "      <th>foci</th>\n",
       "      <th>nonfoci</th>\n",
       "      <th>is_focal_closure</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4125</th>\n",
       "      <td>3311816145</td>\n",
       "      <td>32660008</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2627</th>\n",
       "      <td>1537318436</td>\n",
       "      <td>1439588580</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2635</th>\n",
       "      <td>1537318436</td>\n",
       "      <td>2301123770</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4435</th>\n",
       "      <td>4306171395</td>\n",
       "      <td>618131962</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>32660008</td>\n",
       "      <td>791665500</td>\n",
       "      <td>{'2018.05.24': True}</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      from_user_id  to_user_id             formation  first_date  foci  \\\n",
       "id                                                                       \n",
       "4125    3311816145    32660008  {'2018.05.24': True}        True     0   \n",
       "2627    1537318436  1439588580  {'2018.05.24': True}        True     0   \n",
       "2635    1537318436  2301123770  {'2018.05.24': True}        True     0   \n",
       "4435    4306171395   618131962  {'2018.05.24': True}        True     0   \n",
       "24        32660008   791665500  {'2018.05.24': True}        True     0   \n",
       "\n",
       "      nonfoci  is_focal_closure  \n",
       "id                               \n",
       "4125        0              True  \n",
       "2627        0              True  \n",
       "2635        0              True  \n",
       "4435        0             False  \n",
       "24          0              True  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start = time.time()\n",
    "new_u2u_edges[\"is_focal_closure\"] = new_u2u_edges.apply(\n",
    "    lambda row: is_fclosure(\n",
    "        row[\"from_user_id\"], row[\"to_user_id\"], users_with_mut_foci), axis=1)\n",
    "print(\"took:\", time.time()-start)\n",
    "print(\"There are\",sum(new_u2u_edges[\"is_focal_closure\"]), \"focal closures\")\n",
    "new_u2u_edges.sample(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}