FB_featurization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:32px;text-align:center\"> <b>Social network Graph Link Prediction - Facebook Challenge</b> </p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Importing Libraries\n",
    "# please do go through this python notebook: \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import csv\n",
    "import pandas as pd \n",
    "import datetime  \n",
    "import time \n",
    " \n",
    "import numpy as np \n",
    " \n",
    "import matplotlib\n",
    "import matplotlib.pylab as plt\n",
    "import seaborn as sns \n",
    "from matplotlib import rcParams \n",
    "from sklearn.cluster import MiniBatchKMeans, KMeans\n",
    "import math\n",
    "import pickle\n",
    "import os\n",
    " \n",
    "import xgboost as xgb\n",
    "\n",
    "import warnings\n",
    "import networkx as nx\n",
    "import pdb\n",
    "import pickle\n",
    "from pandas import HDFStore,DataFrame\n",
    "from pandas import read_hdf\n",
    "from scipy.sparse.linalg import svds, eigs\n",
    "import gc\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Reading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: \n",
      "Type: DiGraph\n",
      "Number of nodes: 1780722\n",
      "Number of edges: 7550015\n",
      "Average in degree:   4.2399\n",
      "Average out degree:   4.2399\n"
     ]
    }
   ],
   "source": [
    "if os.path.isfile('data/after_eda/train_pos_after_eda.csv'):\n",
    "    train_graph=nx.read_edgelist('data/after_eda/train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n",
    "    print(nx.info(train_graph))\n",
    "else:\n",
    "    print(\"please run the FB_EDA.ipynb or download the files from drive\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Similarity measures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Jaccard Distance:\n",
    "http://www.statisticshowto.com/jaccard-index/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#for followees\n",
    "# code copied from StackOverflow\n",
    "def jaccard_for_followees(a,b):\n",
    "    if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "    sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n",
    "    return sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "#one test case\n",
    "print(jaccard_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "#node 1635354 not in graph \n",
    "print(jaccard_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#for followers\n",
    "# code copied from StackOverflow\n",
    "def jaccard_for_followers(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                 (len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(jaccard_for_followers(273084,470294))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#node 1635354 not in graph \n",
    "print(jaccard_for_followees(669354,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Cosine distance (Otsuka-Ochiai coefficient)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#for followees\n",
    "def cosine_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followees(273084,1635354))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cosine_for_followers(a,b):\n",
    "    try:\n",
    "        \n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                     (math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.02886751345948129\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followers(2,470294))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followers(669354,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Page Rank\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PageRank computes a ranking of the nodes in the graph G based on the structure of the incoming links."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/page_rank.p'):\n",
    "    pr = nx.pagerank(train_graph, alpha=0.85) # alpha is some hyperparameter\n",
    "    pickle.dump(pr,open('data/fea_sample/page_rank.p','wb'))\n",
    "else:\n",
    "    pr = pickle.load(open('data/fea_sample/page_rank.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 1.6556497245737814e-07\n",
      "max 2.7098251341935827e-05\n",
      "mean 5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "print('min',pr[min(pr, key=pr.get)])\n",
    "print('max',pr[max(pr, key=pr.get)])\n",
    "print('mean',float(sum(pr.values())) / len(pr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "#for imputing to nodes which are not there in Train data\n",
    "mean_pr = float(sum(pr.values())) / len(pr)\n",
    "print(mean_pr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shortest path:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting Shortest path between two nodes,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# if has direct edge then deleting that edge and calculating shortest path\n",
    "# https://stackoverflow.com/questions/9430027/networkx-shortest-path-length\n",
    "def compute_shortest_path_length(a,b):\n",
    "    p=-1\n",
    "    try:\n",
    "        p = nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "        return p\n",
    "    except:\n",
    "        return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#testing\n",
    "compute_shortest_path_length(77697, 826021)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#testing\n",
    "compute_shortest_path_length(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking for same weakly conected component (Community)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#getting weekly connected edges from graph \n",
    "wcc=list(nx.weakly_connected_components(train_graph))\n",
    "def belongs_to_same_wcc(a,b):\n",
    "    index = []\n",
    "    if train_graph.has_edge(b,a) or train_graph.has_edge(a,b):\n",
    "        return 1\n",
    "    else:\n",
    "            for i in wcc:\n",
    "                if a in i:\n",
    "                    index= i\n",
    "                    break\n",
    "            if(b in index):\n",
    "                return 1\n",
    "            else:\n",
    "                return 0\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "belongs_to_same_wcc(861, 1659750)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "belongs_to_same_wcc(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adamic/Adar Index:\n",
    "Adamic/Adar measures is defined as inverted sum of degrees of common neighbours for given two vertices.\n",
    "$$A(x,y)=\\sum_{u \\in N(x) \\cap N(y)}\\frac{1}{log(|N(u)|)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#adar index\n",
    "def calc_adar_in(a,b):\n",
    "    sum=0\n",
    "    try:\n",
    "        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n",
    "        if len(n)!=0:\n",
    "            for i in n:\n",
    "                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n",
    "            return sum\n",
    "        else:\n",
    "            return 0\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_adar_in(1,189226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_adar_in(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Does the person follow back?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def follows_back(a,b):\n",
    "    if train_graph.has_edge(b,a):\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "follows_back(1,189226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "follows_back(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Katz Centrality:\n",
    "https://en.wikipedia.org/wiki/Katz_centrality\n",
    "\n",
    "https://www.geeksforgeeks.org/katz-centrality-centrality-measure/\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/katz.p'):\n",
    "    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n",
    "    pickle.dump(katz,open('data/fea_sample/katz.p','wb'))\n",
    "else:\n",
    "    katz = pickle.load(open('data/fea_sample/katz.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 0.0007313532484065916\n",
      "max 0.003394554981699122\n",
      "mean 0.0007483800935562018\n"
     ]
    }
   ],
   "source": [
    "print('min',katz[min(katz, key=katz.get)])\n",
    "print('max',katz[max(katz, key=katz.get)])\n",
    "print('mean',float(sum(katz.values())) / len(katz))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0007483800935562018\n"
     ]
    }
   ],
   "source": [
    "mean_katz = float(sum(katz.values())) / len(katz)\n",
    "print(mean_katz)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HITS Score\n",
    "The HITS algorithm computes two numbers for a node. Authorities estimates the node value based on the incoming links. Hubs estimates the node value based on outgoing links.\n",
    "\n",
    "https://en.wikipedia.org/wiki/HITS_algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "if not os.path.isfile('data/fea_sample/hits.p'):\n",
    "    # returns tuple of 2 dictionary,(hubs,authorities) \n",
    "    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True) \n",
    "    pickle.dump(hits,open('data/fea_sample/hits.p','wb'))\n",
    "else:\n",
    "    hits = pickle.load(open('data/fea_sample/hits.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tuple, dict, dict)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(hits), type(hits[0]), type(hits[1]) # tuple of 2, hub dict, auth dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 0.0\n",
      "max 0.004868653378780953\n",
      "mean 5.615699699344123e-07\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "inf = math.inf\n",
    "Min=inf\n",
    "Max = -1\n",
    "for key, item in hits[0].items():\n",
    "    Min = min(Min, item)\n",
    "    Max = max(Max, item)\n",
    "\n",
    "print('min',Min)\n",
    "print('max',Max)\n",
    "print('mean',float(sum(hits[0].values())) / len(hits[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Featurization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading a sample of Data from both train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "if os.path.isfile('data/after_eda/train_after_eda.csv'):\n",
    "    filename = \"data/after_eda/train_after_eda.csv\"\n",
    "    # you uncomment this line, if you dont know the lentgh of the file name\n",
    "    # here we have hardcoded the number of lines as 15100030\n",
    "    # n_train = sum(1 for line in open(filename)) #number of records in file (excludes header)\n",
    "    n_train =  15100028\n",
    "    s = 100000 #desired sample size\n",
    "    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\n",
    "    #https://stackoverflow.com/a/22259008/4084039"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if os.path.isfile('data/after_eda/train_after_eda.csv'):\n",
    "    filename = \"data/after_eda/test_after_eda.csv\"\n",
    "    # you uncomment this line, if you dont know the lentgh of the file name\n",
    "    # here we have hardcoded the number of lines as 3775008\n",
    "    # n_test = sum(1 for line in open(filename)) #number of records in file (excludes header)\n",
    "    n_test = 3775006\n",
    "    s = 50000 #desired sample size\n",
    "    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\n",
    "    #https://stackoverflow.com/a/22259008/4084039"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows in the train data file: 15100028\n",
      "Number of rows we are going to elimiate in train data are 15000028\n",
      "Number of rows in the test data file: 3775006\n",
      "Number of rows we are going to elimiate in test data are 3725006\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of rows in the train data file:\", n_train)\n",
    "print(\"Number of rows we are going to elimiate in train data are\",len(skip_train))\n",
    "print(\"Number of rows in the test data file:\", n_test)\n",
    "print(\"Number of rows we are going to elimiate in test data are\",len(skip_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our train matrix size  (100002, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>832016</td>\n",
       "      <td>1543415</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link\n",
       "0       273084           1505602               1\n",
       "1       832016           1543415               1"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_train = pd.read_csv('data/after_eda/train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\n",
    "df_final_train['indicator_link'] = pd.read_csv('data/train_y.csv', skiprows=skip_train, names=['indicator_link'])\n",
    "print(\"Our train matrix size \",df_final_train.shape)\n",
    "df_final_train.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our test matrix size  (50002, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>848424</td>\n",
       "      <td>784690</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>483294</td>\n",
       "      <td>1255532</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link\n",
       "0       848424            784690               1\n",
       "1       483294           1255532               1"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_test = pd.read_csv('data/after_eda/test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\n",
    "df_final_test['indicator_link'] = pd.read_csv('data/test_y.csv', skiprows=skip_test, names=['indicator_link'])\n",
    "print(\"Our test matrix size \",df_final_test.shape)\n",
    "df_final_test.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.2 Adding a set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>jaccard_followers</li>\n",
    "<li>jaccard_followees</li>\n",
    "<li>cosine_followers</li>\n",
    "<li>cosine_followees</li>\n",
    "<li>num_followers_s</li>\n",
    "<li>num_followees_s</li>\n",
    "<li>num_followers_d</li>\n",
    "<li>num_followees_d</li>\n",
    "<li>inter_followers</li>\n",
    "<li>inter_followees</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage1.h5'):\n",
    "    #mapping jaccrd followers to train and test data\n",
    "    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping jaccrd followees to train and test data\n",
    "    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "\n",
    "    #mapping jaccrd followers to train and test data\n",
    "    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping jaccrd followees to train and test data\n",
    "    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_features_stage1(df_final):\n",
    "    #calculating no of followers followees for source and destination\n",
    "    #calculating intersection of followers and followees for source and destination\n",
    "    num_followers_s=[]\n",
    "    num_followees_s=[]\n",
    "    num_followers_d=[]\n",
    "    num_followees_d=[]\n",
    "    inter_followers=[]\n",
    "    inter_followees=[]\n",
    "    for i,row in df_final.iterrows():\n",
    "        s1=set(train_graph.predecessors(row['source_node']))\n",
    "        s2=set(train_graph.successors(row['source_node']))\n",
    "    \n",
    "        d1=set(train_graph.predecessors(row['destination_node']))\n",
    "        d2=set(train_graph.successors(row['destination_node']))\n",
    "            \n",
    "        num_followers_s.append(len(s1))\n",
    "        num_followees_s.append(len(s2))\n",
    "\n",
    "        num_followers_d.append(len(d1))\n",
    "        num_followees_d.append(len(d2))\n",
    "\n",
    "        inter_followers.append(len(s1.intersection(d1)))\n",
    "        inter_followees.append(len(s2.intersection(d2)))\n",
    "    \n",
    "    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage1.h5'):\n",
    "    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n",
    "    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n",
    "    df_final_train['inter_followers'], df_final_train['inter_followeed']= compute_features_stage1(df_final_train)\n",
    "    \n",
    "    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n",
    "    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n",
    "    df_final_test['inter_followers'], df_final_test['inter_followeed']= compute_features_stage1(df_final_test)\n",
    "    \n",
    "    hdf = HDFStore('data/fea_sample/storage_sample_stage1.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('data/fea_sample/storage_sample_stage1.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('data/fea_sample/storage_sample_stage1.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.3 Adding new set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>adar index</li>\n",
    "<li>is following back</li>\n",
    "<li>belongs to same weakly connect components</li>\n",
    "<li>shortest path between source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "networkxif not os.path.isfile('data/fea_sample/storage_sample_stage2.h5'):\n",
    "    #mapping adar index on train\n",
    "    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n",
    "    #mapping adar index on test\n",
    "    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping followback or not on train\n",
    "    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping followback or not on test\n",
    "    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping same component of wcc or not on train\n",
    "    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    ##mapping same component of wcc or not on train\n",
    "    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping shortest path on train \n",
    "    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n",
    "    #mapping shortest path on test\n",
    "    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    hdf = HDFStore('data/fea_sample/storage_sample_stage2.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('data/fea_sample/storage_sample_stage2.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('data/fea_sample/storage_sample_stage2.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.4 Adding new set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>Weight Features\n",
    "    <ul>\n",
    "        <li>weight of incoming edges</li>\n",
    "        <li>weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges + weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges * weight of outgoing edges</li>\n",
    "        <li>2*weight of incoming edges + weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges + 2*weight of outgoing edges</li>\n",
    "    </ul>\n",
    "</li>\n",
    "<li>Page Ranking of source</li>\n",
    "<li>Page Ranking of dest</li>\n",
    "<li>katz of source</li>\n",
    "<li>katz of dest</li>\n",
    "<li>hubs of source</li>\n",
    "<li>hubs of dest</li>\n",
    "<li>authorities_s of source</li>\n",
    "<li>authorities_s of dest</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Weight Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation}\n",
    "W = \\frac{1}{\\sqrt{1+|X|}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "it is directed graph so calculated Weighted in and Weighted out differently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████| 1780722/1780722 [00:11<00:00, 152682.24it/s]\n"
     ]
    }
   ],
   "source": [
    "#weight for source and destination of each link\n",
    "Weight_in = {}\n",
    "Weight_out = {}\n",
    "for i in  tqdm(train_graph.nodes()):\n",
    "    s1=set(train_graph.predecessors(i))\n",
    "    w_in = 1.0/(np.sqrt(1+len(s1)))\n",
    "    Weight_in[i]=w_in\n",
    "    \n",
    "    s2=set(train_graph.successors(i))\n",
    "    w_out = 1.0/(np.sqrt(1+len(s2)))\n",
    "    Weight_out[i]=w_out\n",
    "    \n",
    "#for imputing with mean\n",
    "mean_weight_in = np.mean(list(Weight_in.values()))\n",
    "mean_weight_out = np.mean(list(Weight_out.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage3.h5'):\n",
    "    #mapping to pandas train, weight_in for destination node, and weight_out for source node\n",
    "    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "    #mapping to pandas test, weight_in for destination node, and weight_out for source node\n",
    "    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "\n",
    "    #some features engineerings on the in and out weights for Train Data\n",
    "    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n",
    "    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n",
    "    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n",
    "    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n",
    "\n",
    "    #some features engineerings on the in and out weights for Test Data\n",
    "    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n",
    "    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n",
    "    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n",
    "    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage3.h5'):\n",
    "    \n",
    "    #page rank for source and destination in Train and Test\n",
    "    #if anything not there in train graph then adding mean page rank \n",
    "    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "\n",
    "    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    #================================================================================\n",
    "\n",
    "    #Katz centrality score for source and destination in Train and test\n",
    "    #if anything not there in train graph then adding mean katz score\n",
    "    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "\n",
    "    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    #================================================================================\n",
    "\n",
    "    #Hits algorithm score for source and destination in Train and test\n",
    "    #if anything not there in train graph then adding 0\n",
    "    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "\n",
    "    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "    #================================================================================\n",
    "\n",
    "    #Hits algorithm score for source and destination in Train and Test\n",
    "    #if anything not there in train graph then adding 0\n",
    "    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "\n",
    "    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "    #================================================================================\n",
    "\n",
    "    hdf = HDFStore('data/fea_sample/storage_sample_stage3.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('data/fea_sample/storage_sample_stage3.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('data/fea_sample/storage_sample_stage3.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}