FSCNMF_Processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import json \n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict, Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating EdgeList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "adjList = defaultdict(set)\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        paperID = data['id']\n",
    "        references = data.get('references', [])\n",
    "        for referencedPaper in references:\n",
    "            adjList[paperID].add(referencedPaper)\n",
    "            adjList[referencedPaper].add(paperID)\n",
    "            \n",
    "IDsInGraph = list(adjList.keys())\n",
    "IDtoIndex = dict()\n",
    "for i in range(len(IDsInGraph)):\n",
    "    IDtoIndex[IDsInGraph[i]] = i\n",
    "with open(\"./data/fscnmfIDsInGraph.json\", 'w') as f:\n",
    "    json.dump(IDsInGraph, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./data/fscnmfIDsInGraph.json\", 'r') as f:\n",
    "    IDsInGraph = json.load(f)\n",
    "IDtoIndex = dict()\n",
    "for i in range(len(IDsInGraph)):\n",
    "    IDtoIndex[IDsInGraph[i]] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "adjList = {IDtoIndex[key]: [IDtoIndex[v] for v in values] for key, values in adjList.items()}\n",
    "G = nx.from_dict_of_lists(adjList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './data/edgelist.csv'\n",
    "nx.write_edgelist(G, path, comments='#', delimiter=',',  data = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Features for each Paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(lst):\n",
    "    s = sum(lst)\n",
    "    return [float(i)/s for i in lst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "AITopLevelTopics = ['Artificial intelligence', 'Computer vision', 'Data mining',\n",
    "                     'Data science', 'Machine learning', 'Natural language processing',\n",
    "                     'Pattern recognition', 'Speech recognition']\n",
    "topicSet = set(AITopLevelTopics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "featureList = []\n",
    "tmp = []\n",
    "IDsInGraphSet = set(IDsInGraph)\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        if (data['id'] in IDsInGraphSet):\n",
    "            feature = [0] * len(AITopLevelTopics)\n",
    "            featureMap = defaultdict(int)\n",
    "            for fos in data.get('fos', []):\n",
    "                if fos['name'] in topicSet:\n",
    "                    featureMap[fos['name']] = fos['w']\n",
    "            for i in range(len(AITopLevelTopics)):\n",
    "                feature[i] = featureMap.get(AITopLevelTopics[i], 0)\n",
    "            try:\n",
    "                feature = normalize(feature)\n",
    "            except:\n",
    "                feature = feature\n",
    "            feature.insert(0, IDtoIndex[data['id']])\n",
    "            featureList.append(feature)\n",
    "        else:\n",
    "            tmp.append(data['id'])\n",
    "featureList.sort()                                           ## Sort by Node IDs for the FSC NMF package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "with open('./data/features.csv', \"w\") as file:\n",
    "    writer = csv.writer(file)\n",
    "    nodeID = 0\n",
    "    header = ['NodeID', *AITopLevelTopics]\n",
    "    writer.writerow(header)\n",
    "    for row in featureList:\n",
    "        writer.writerow(row)\n",
    "        nodeID += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Label Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average(lis):\n",
    "    return sum(lis) / len(lis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "IDList = []\n",
    "labelList = []\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        paperID = data['id']\n",
    "#         if paperID not in IDtoIndex:\n",
    "#             continue\n",
    "        fosList = defaultdict(int)\n",
    "        for fos in data.get('fos',[]):\n",
    "            if (fos['name'] == 'Data mining' or fos['name'] == 'Data science'):\n",
    "                fosList['Data'] = max(fosList['Data'], fos['w'])\n",
    "            if (fos['name'] == 'Natural language processing' or fos['name'] == 'Speech recognition'):\n",
    "                fosList['NLP'] = max(fosList['NLP'], fos['w'])\n",
    "            if (fos['name'] == 'Computer vision'):\n",
    "                fosList['CV'] = fos['w']\n",
    "        if (len(fosList.keys())):\n",
    "            IDList.append(data['id'])\n",
    "            fosList = [(key, value) for key, value in fosList.items()]\n",
    "            fosList = sorted(fosList, key = lambda x: x[1],    reverse=True)\n",
    "            labelList.append(fosList[0][0])\n",
    "assert len(labelList) == len(IDList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/du0/15CS30003/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:4: DeprecationWarning: 'U' mode is deprecated\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "embeddingDict = dict()                   ## mappping from paperID(not Index) to embedding\n",
    "IndextoID = {val: key for key, val in IDtoIndex.items()}\n",
    "import csv\n",
    "with open('./data/fscNMFEmbeddings.csv', 'rU') as file:\n",
    "    rows = csv.reader(file, delimiter=',')\n",
    "    rows = list(rows)[1 : ]\n",
    "for row in rows:\n",
    "    embeddingDict[IndextoID[int(float(row[0]))]] = [float(f) for f in row[1:]]\n",
    "#     row = list(row)\n",
    "#     embeddingDict[IndextoID[row[0]]] = row[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = []\n",
    "for id in IDList:\n",
    "    if id in embeddingDict:\n",
    "        embeddings.append(embeddingDict[id])\n",
    "    else:\n",
    "        embeddings.append([0] * 32)\n",
    "del embeddingDict\n",
    "assert len(embeddings) == len(labelList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "X = np.asarray(embeddings)\n",
    "del embeddings\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(labelList)\n",
    "Y = le.transform(labelList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.gaussian_process.kernels import RBF\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, BaggingClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "\n",
    "\n",
    "names = [\n",
    " \"Random Forest\", \"Neural Net\", \"AdaBoost\", \"Linear SVC\" ]\n",
    "\n",
    "classifiers = [\n",
    "    RandomForestClassifier(verbose=True, n_jobs = -1),\n",
    "    MLPClassifier(verbose=True, early_stopping=True),\n",
    "    AdaBoostClassifier(),\n",
    "    OneVsRestClassifier(BaggingClassifier(LinearSVC(),n_jobs = -1))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  1\n",
      "Fitting:  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    4.9s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    4.9s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  4\n",
      "Fitting:  5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    5.0s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  5\n",
      "Name Random Forest . Avg Precision:  0.9913456520057522 . Avg Recall:  0.9912923211169847 . Avg F-1 Score:  0.9912997362567912\n",
      "Fitting:  1\n",
      "Iteration 1, loss = 0.37350758\n",
      "Validation score: 0.987753\n",
      "Iteration 2, loss = 0.04625790\n",
      "Validation score: 0.987162\n",
      "Iteration 3, loss = 0.03006978\n",
      "Validation score: 0.987320\n",
      "Iteration 4, loss = 0.02586067\n",
      "Validation score: 0.988934\n",
      "Iteration 5, loss = 0.02415536\n",
      "Validation score: 0.988974\n",
      "Iteration 6, loss = 0.02334396\n",
      "Validation score: 0.988777\n",
      "Iteration 7, loss = 0.02290932\n",
      "Validation score: 0.989210\n",
      "Iteration 8, loss = 0.02260781\n",
      "Validation score: 0.989053\n",
      "Iteration 9, loss = 0.02245857\n",
      "Validation score: 0.989289\n",
      "Iteration 10, loss = 0.02228938\n",
      "Validation score: 0.989564\n",
      "Iteration 11, loss = 0.02218753\n",
      "Validation score: 0.989210\n",
      "Iteration 12, loss = 0.02207763\n",
      "Validation score: 0.989368\n",
      "Iteration 13, loss = 0.02197393\n",
      "Validation score: 0.989092\n",
      "Iteration 14, loss = 0.02193024\n",
      "Validation score: 0.990195\n",
      "Iteration 15, loss = 0.02188653\n",
      "Validation score: 0.989643\n",
      "Iteration 16, loss = 0.02180995\n",
      "Validation score: 0.990037\n",
      "Iteration 17, loss = 0.02172792\n",
      "Validation score: 0.989683\n",
      "Iteration 18, loss = 0.02167630\n",
      "Validation score: 0.990313\n",
      "Iteration 19, loss = 0.02168686\n",
      "Validation score: 0.989919\n",
      "Iteration 20, loss = 0.02159568\n",
      "Validation score: 0.989840\n",
      "Iteration 21, loss = 0.02154821\n",
      "Validation score: 0.990313\n",
      "Iteration 22, loss = 0.02158872\n",
      "Validation score: 0.990076\n",
      "Iteration 23, loss = 0.02154700\n",
      "Validation score: 0.990076\n",
      "Iteration 24, loss = 0.02149023\n",
      "Validation score: 0.989722\n",
      "Iteration 25, loss = 0.02145988\n",
      "Validation score: 0.990155\n",
      "Iteration 26, loss = 0.02143018\n",
      "Validation score: 0.990588\n",
      "Iteration 27, loss = 0.02142795\n",
      "Validation score: 0.989761\n",
      "Iteration 28, loss = 0.02136876\n",
      "Validation score: 0.990155\n",
      "Iteration 29, loss = 0.02139050\n",
      "Validation score: 0.989092\n",
      "Iteration 30, loss = 0.02136536\n",
      "Validation score: 0.989722\n",
      "Iteration 31, loss = 0.02132287\n",
      "Validation score: 0.990352\n",
      "Iteration 32, loss = 0.02126748\n",
      "Validation score: 0.990628\n",
      "Iteration 33, loss = 0.02124707\n",
      "Validation score: 0.989958\n",
      "Iteration 34, loss = 0.02125385\n",
      "Validation score: 0.989328\n",
      "Iteration 35, loss = 0.02122147\n",
      "Validation score: 0.990155\n",
      "Iteration 36, loss = 0.02123771\n",
      "Validation score: 0.989722\n",
      "Iteration 37, loss = 0.02117225\n",
      "Validation score: 0.989683\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  1\n",
      "Fitting:  2\n",
      "Iteration 1, loss = 0.39999103\n",
      "Validation score: 0.985548\n",
      "Iteration 2, loss = 0.04906599\n",
      "Validation score: 0.987280\n",
      "Iteration 3, loss = 0.03052666\n",
      "Validation score: 0.987989\n",
      "Iteration 4, loss = 0.02593937\n",
      "Validation score: 0.988107\n",
      "Iteration 5, loss = 0.02420814\n",
      "Validation score: 0.988856\n",
      "Iteration 6, loss = 0.02332702\n",
      "Validation score: 0.988777\n",
      "Iteration 7, loss = 0.02283361\n",
      "Validation score: 0.989486\n",
      "Iteration 8, loss = 0.02251942\n",
      "Validation score: 0.988895\n",
      "Iteration 9, loss = 0.02230875\n",
      "Validation score: 0.989131\n",
      "Iteration 10, loss = 0.02217905\n",
      "Validation score: 0.988934\n",
      "Iteration 11, loss = 0.02207663\n",
      "Validation score: 0.988934\n",
      "Iteration 12, loss = 0.02188728\n",
      "Validation score: 0.989131\n",
      "Iteration 13, loss = 0.02191681\n",
      "Validation score: 0.988974\n",
      "Iteration 14, loss = 0.02181180\n",
      "Validation score: 0.989171\n",
      "Iteration 15, loss = 0.02171496\n",
      "Validation score: 0.989131\n",
      "Iteration 16, loss = 0.02167657\n",
      "Validation score: 0.989013\n",
      "Iteration 17, loss = 0.02158211\n",
      "Validation score: 0.989407\n",
      "Iteration 18, loss = 0.02155879\n",
      "Validation score: 0.989289\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  2\n",
      "Fitting:  3\n",
      "Iteration 1, loss = 0.38596126\n",
      "Validation score: 0.986257\n",
      "Iteration 2, loss = 0.04823027\n",
      "Validation score: 0.987438\n",
      "Iteration 3, loss = 0.03057965\n",
      "Validation score: 0.987714\n",
      "Iteration 4, loss = 0.02596916\n",
      "Validation score: 0.988974\n",
      "Iteration 5, loss = 0.02410634\n",
      "Validation score: 0.989604\n",
      "Iteration 6, loss = 0.02330822\n",
      "Validation score: 0.989486\n",
      "Iteration 7, loss = 0.02286289\n",
      "Validation score: 0.989446\n",
      "Iteration 8, loss = 0.02252961\n",
      "Validation score: 0.988344\n",
      "Iteration 9, loss = 0.02234969\n",
      "Validation score: 0.989289\n",
      "Iteration 10, loss = 0.02219686\n",
      "Validation score: 0.989840\n",
      "Iteration 11, loss = 0.02204588\n",
      "Validation score: 0.988304\n",
      "Iteration 12, loss = 0.02200896\n",
      "Validation score: 0.988147\n",
      "Iteration 13, loss = 0.02194254\n",
      "Validation score: 0.989801\n",
      "Iteration 14, loss = 0.02181719\n",
      "Validation score: 0.989919\n",
      "Iteration 15, loss = 0.02177696\n",
      "Validation score: 0.989643\n",
      "Iteration 16, loss = 0.02174384\n",
      "Validation score: 0.989564\n",
      "Iteration 17, loss = 0.02165479\n",
      "Validation score: 0.989446\n",
      "Iteration 18, loss = 0.02162389\n",
      "Validation score: 0.989919\n",
      "Iteration 19, loss = 0.02158358\n",
      "Validation score: 0.989879\n",
      "Iteration 20, loss = 0.02156253\n",
      "Validation score: 0.989328\n",
      "Iteration 21, loss = 0.02156311\n",
      "Validation score: 0.990234\n",
      "Iteration 22, loss = 0.02150965\n",
      "Validation score: 0.989368\n",
      "Iteration 23, loss = 0.02146790\n",
      "Validation score: 0.989486\n",
      "Iteration 24, loss = 0.02138426\n",
      "Validation score: 0.989840\n",
      "Iteration 25, loss = 0.02137332\n",
      "Validation score: 0.989368\n",
      "Iteration 26, loss = 0.02131550\n",
      "Validation score: 0.989958\n",
      "Iteration 27, loss = 0.02137899\n",
      "Validation score: 0.989958\n",
      "Iteration 28, loss = 0.02128789\n",
      "Validation score: 0.989683\n",
      "Iteration 29, loss = 0.02129578\n",
      "Validation score: 0.990037\n",
      "Iteration 30, loss = 0.02128014\n",
      "Validation score: 0.989958\n",
      "Iteration 31, loss = 0.02129888\n",
      "Validation score: 0.989958\n",
      "Iteration 32, loss = 0.02121542\n",
      "Validation score: 0.989525\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  3\n",
      "Fitting:  4\n",
      "Iteration 1, loss = 0.41470557\n",
      "Validation score: 0.985469\n",
      "Iteration 2, loss = 0.05052857\n",
      "Validation score: 0.987792\n",
      "Iteration 3, loss = 0.03098027\n",
      "Validation score: 0.988659\n",
      "Iteration 4, loss = 0.02622939\n",
      "Validation score: 0.989525\n",
      "Iteration 5, loss = 0.02449670\n",
      "Validation score: 0.989564\n",
      "Iteration 6, loss = 0.02355285\n",
      "Validation score: 0.990470\n",
      "Iteration 7, loss = 0.02311359\n",
      "Validation score: 0.989210\n",
      "Iteration 8, loss = 0.02279268\n",
      "Validation score: 0.990470\n",
      "Iteration 9, loss = 0.02262271\n",
      "Validation score: 0.990116\n",
      "Iteration 10, loss = 0.02245680\n",
      "Validation score: 0.990234\n",
      "Iteration 11, loss = 0.02235475\n",
      "Validation score: 0.990273\n",
      "Iteration 12, loss = 0.02223164\n",
      "Validation score: 0.989525\n",
      "Iteration 13, loss = 0.02216429\n",
      "Validation score: 0.989840\n",
      "Iteration 14, loss = 0.02208211\n",
      "Validation score: 0.989683\n",
      "Iteration 15, loss = 0.02206355\n",
      "Validation score: 0.989289\n",
      "Iteration 16, loss = 0.02198781\n",
      "Validation score: 0.990549\n",
      "Iteration 17, loss = 0.02185428\n",
      "Validation score: 0.989761\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  4\n",
      "Fitting:  5\n",
      "Iteration 1, loss = 0.40731792\n",
      "Validation score: 0.985115\n",
      "Iteration 2, loss = 0.04967545\n",
      "Validation score: 0.986060\n",
      "Iteration 3, loss = 0.03067572\n",
      "Validation score: 0.987320\n",
      "Iteration 4, loss = 0.02602343\n",
      "Validation score: 0.987714\n",
      "Iteration 5, loss = 0.02412890\n",
      "Validation score: 0.988737\n",
      "Iteration 6, loss = 0.02328074\n",
      "Validation score: 0.987950\n",
      "Iteration 7, loss = 0.02281781\n",
      "Validation score: 0.988226\n",
      "Iteration 8, loss = 0.02245404\n",
      "Validation score: 0.988974\n",
      "Iteration 9, loss = 0.02225689\n",
      "Validation score: 0.988698\n",
      "Iteration 10, loss = 0.02215854\n",
      "Validation score: 0.988541\n",
      "Iteration 11, loss = 0.02201568\n",
      "Validation score: 0.988895\n",
      "Iteration 12, loss = 0.02191261\n",
      "Validation score: 0.989486\n",
      "Iteration 13, loss = 0.02184802\n",
      "Validation score: 0.988974\n",
      "Iteration 14, loss = 0.02172314\n",
      "Validation score: 0.988068\n",
      "Iteration 15, loss = 0.02174953\n",
      "Validation score: 0.988895\n",
      "Iteration 16, loss = 0.02164606\n",
      "Validation score: 0.987911\n",
      "Iteration 17, loss = 0.02161990\n",
      "Validation score: 0.988265\n",
      "Iteration 18, loss = 0.02156832\n",
      "Validation score: 0.989171\n",
      "Iteration 19, loss = 0.02150347\n",
      "Validation score: 0.987950\n",
      "Iteration 20, loss = 0.02148900\n",
      "Validation score: 0.988422\n",
      "Iteration 21, loss = 0.02144187\n",
      "Validation score: 0.987950\n",
      "Iteration 22, loss = 0.02139174\n",
      "Validation score: 0.988777\n",
      "Iteration 23, loss = 0.02136563\n",
      "Validation score: 0.988541\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  5\n",
      "Name Neural Net . Avg Precision:  0.9895316452897991 . Avg Recall:  0.9894493422685631 . Avg F-1 Score:  0.9894540561752031\n",
      "Fitting:  1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  1\n",
      "Fitting:  2\n",
      "count  2\n",
      "Fitting:  3\n",
      "count  3\n",
      "Fitting:  4\n",
      "count  4\n",
      "Fitting:  5\n",
      "count  5\n",
      "Name AdaBoost . Avg Precision:  0.9856940592049384 . Avg Recall:  0.9855302640455106 . Avg F-1 Score:  0.9854933968537871\n",
      "Fitting:  1\n",
      "count  1\n",
      "Fitting:  2\n",
      "count  2\n",
      "Fitting:  3\n",
      "count  3\n",
      "Fitting:  4\n",
      "count  4\n",
      "Fitting:  5\n",
      "count  5\n",
      "Name Linear SVC . Avg Precision:  0.9881540904585124 . Avg Recall:  0.9880852252279514 . Avg F-1 Score:  0.9881050120621259\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn import svm\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "\n",
    "kfold = KFold(n_splits=5, shuffle=True)\n",
    "for name, clf in zip(names, classifiers):\n",
    "    precScores = []\n",
    "    recallScores = []\n",
    "    f1Scores = []\n",
    "    count = 1\n",
    "    for train_index, test_index in kfold.split(X, Y):\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        y_train, y_test = Y[train_index], Y[test_index]\n",
    "        print('Fitting: ', count)\n",
    "        clf.fit(X_train, y_train)\n",
    "        print('count ', count)\n",
    "        y_pred = clf.predict(X_test)\n",
    "        prec, recall, fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
    "        precScores.append(prec)\n",
    "        recallScores.append(recall)\n",
    "        f1Scores.append(fscore)\n",
    "        count += 1\n",
    "    print('Name', name,'. Avg Precision: ', average(precScores), '. Avg Recall: ', average(recallScores), '. Avg F-1 Score: ', average(f1Scores) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "314740"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(IDList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "471633"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(IDsInGraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.sparse as sp\n",
    "\n",
    "\"\"\"\n",
    "Disclaimer: functions defined from lines 15 to 36 in this file come from \n",
    "tkipf/gae original repository on Graph Autoencoders. Moreover, the\n",
    "mask_test_edges function is borrowed from philipjackson's mask_test_edges \n",
    "pull request on this same repository.\n",
    "\"\"\"\n",
    "\n",
    "def sparse_to_tuple(sparse_mx):\n",
    "    if not sp.isspmatrix_coo(sparse_mx):\n",
    "        sparse_mx = sparse_mx.tocoo()\n",
    "    coords = np.vstack((sparse_mx.row, sparse_mx.col)).transpose()\n",
    "    values = sparse_mx.data\n",
    "    shape = sparse_mx.shape\n",
    "    return coords, values, shape\n",
    "\n",
    "def preprocess_graph(adj):\n",
    "    adj = sp.coo_matrix(adj)\n",
    "    adj_ = adj + sp.eye(adj.shape[0])\n",
    "    degree_mat_inv_sqrt = sp.diags(np.power(np.array(adj_.sum(1)), -0.5).flatten())\n",
    "    adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt)\n",
    "    return sparse_to_tuple(adj_normalized)\n",
    "\n",
    "def construct_feed_dict(adj_normalized, adj, features, placeholders):\n",
    "    # Construct feed dictionary\n",
    "    feed_dict = dict()\n",
    "    feed_dict.update({placeholders['features']: features})\n",
    "    feed_dict.update({placeholders['adj']: adj_normalized})\n",
    "    feed_dict.update({placeholders['adj_orig']: adj})\n",
    "    return feed_dict\n",
    "\n",
    "def mask_test_edges(adj, test_percent=1., val_percent=0.):\n",
    "    \"\"\" Randomly removes some edges from original graph to create\n",
    "    test and validation sets for link prediction task\n",
    "    :param adj: complete sparse adjacency matrix of the graph\n",
    "    :param test_percent: percentage of edges in test set\n",
    "    :param val_percent: percentage of edges in validation set\n",
    "    :return: train incomplete adjacency matrix, validation and test sets\n",
    "    \"\"\"\n",
    "    # Remove diagonal elements\n",
    "    adj = adj - sp.dia_matrix((adj.diagonal()[None, :], [0]), shape=adj.shape)\n",
    "    adj.eliminate_zeros()\n",
    "    # Check that diag is zero:\n",
    "    #assert adj.diagonal().sum() == 0\n",
    "\n",
    "    edges_positive, _, _ = sparse_to_tuple(adj)\n",
    "    # Filtering out edges from lower triangle of adjacency matrix\n",
    "    edges_positive = edges_positive[edges_positive[:,1] > edges_positive[:,0],:]\n",
    "    # val_edges, val_edges_false, test_edges, test_edges_false = None, None, None, None\n",
    "\n",
    "    # number of positive (and negative) edges in test and val sets:\n",
    "    num_test = int(np.floor(edges_positive.shape[0] / (100. / test_percent)))\n",
    "    num_val = 0\n",
    "\n",
    "    # sample positive edges for test and val sets:\n",
    "    edges_positive_idx = np.arange(edges_positive.shape[0])\n",
    "    np.random.shuffle(edges_positive_idx)\n",
    "    val_edge_idx = edges_positive_idx[:num_val]\n",
    "    test_edge_idx = edges_positive_idx[num_val:(num_val + num_test)]\n",
    "    test_edges = edges_positive[test_edge_idx] # positive test edges\n",
    "    val_edges = edges_positive[val_edge_idx] # positive val edges\n",
    "    train_edges = np.delete(edges_positive, np.hstack([test_edge_idx, val_edge_idx]), axis = 0) # positive train edges\n",
    "\n",
    "    # the above strategy for sampling without replacement will not work for\n",
    "    # sampling negative edges on large graphs, because the pool of negative\n",
    "    # edges is much much larger due to sparsity, therefore we'll use\n",
    "    # the following strategy:\n",
    "    # 1. sample random linear indices from adjacency matrix WITH REPLACEMENT\n",
    "    # (without replacement is super slow). sample more than we need so we'll\n",
    "    # probably have enough after all the filtering steps.\n",
    "    # 2. remove any edges that have already been added to the other edge lists\n",
    "    # 3. convert to (i,j) coordinates\n",
    "    # 4. swap i and j where i > j, to ensure they're upper triangle elements\n",
    "    # 5. remove any duplicate elements if there are any\n",
    "    # 6. remove any diagonal elements\n",
    "    # 7. if we don't have enough edges, repeat this process until we get enough\n",
    "    positive_idx, _, _ = sparse_to_tuple(adj) # [i,j] coord pairs for all true edges\n",
    "    positive_idx = positive_idx[:,0]*adj.shape[0] + positive_idx[:,1] # linear indices\n",
    "    test_edges_false = np.empty((0,2),dtype='int64')\n",
    "    idx_test_edges_false = np.empty((0,),dtype='int64')\n",
    "\n",
    "    while len(test_edges_false) < len(test_edges):\n",
    "        # step 1:\n",
    "        idx = np.random.choice(adj.shape[0]**2, 2*(num_test - len(test_edges_false)), replace = True)\n",
    "        # step 2:\n",
    "        idx = idx[~np.in1d(idx, positive_idx, assume_unique = True)]\n",
    "        idx = idx[~np.in1d(idx, idx_test_edges_false, assume_unique = True)]\n",
    "        # step 3:\n",
    "        rowidx = idx // adj.shape[0]\n",
    "        colidx = idx % adj.shape[0]\n",
    "        coords = np.vstack((rowidx,colidx)).transpose()\n",
    "        # step 4:\n",
    "        lowertrimask = coords[:,0] > coords[:,1]\n",
    "        coords[lowertrimask] = coords[lowertrimask][:,::-1]\n",
    "        # step 5:\n",
    "        coords = np.unique(coords, axis = 0) # note: coords are now sorted lexicographically\n",
    "        np.random.shuffle(coords) # not anymore\n",
    "        # step 6:\n",
    "        coords = coords[coords[:,0] != coords[:,1]]\n",
    "        # step 7:\n",
    "        coords = coords[:min(num_test, len(idx))]\n",
    "        test_edges_false = np.append(test_edges_false, coords, axis = 0)\n",
    "        idx = idx[:min(num_test, len(idx))]\n",
    "        idx_test_edges_false = np.append(idx_test_edges_false, idx)\n",
    "\n",
    "    val_edges_false = np.empty((0,2), dtype = 'int64')\n",
    "    idx_val_edges_false = np.empty((0,), dtype = 'int64')\n",
    "#     while len(val_edges_false) < len(val_edges):\n",
    "#         # step 1:\n",
    "#         idx = np.random.choice(adj.shape[0]**2, 2*(num_val - len(val_edges_false)), replace = True)\n",
    "#         # step 2:\n",
    "#         idx = idx[~np.in1d(idx, positive_idx, assume_unique = True)]\n",
    "#         idx = idx[~np.in1d(idx, idx_test_edges_false, assume_unique = True)]\n",
    "#         idx = idx[~np.in1d(idx, idx_val_edges_false, assume_unique = True)]\n",
    "#         # step 3:\n",
    "#         rowidx = idx // adj.shape[0]\n",
    "#         colidx = idx % adj.shape[0]\n",
    "#         coords = np.vstack((rowidx,colidx)).transpose()\n",
    "#         # step 4:\n",
    "#         lowertrimask = coords[:,0] > coords[:,1]\n",
    "#         coords[lowertrimask] = coords[lowertrimask][:,::-1]\n",
    "#         # step 5:\n",
    "#         coords = np.unique(coords, axis = 0) # note: coords are now sorted lexicographically\n",
    "#         np.random.shuffle(coords) # not any more\n",
    "#         # step 6:\n",
    "#         coords = coords[coords[:,0] != coords[:,1]]\n",
    "#         # step 7:\n",
    "#         coords = coords[:min(num_val, len(idx))]\n",
    "#         val_edges_false = np.append(val_edges_false, coords, axis = 0)\n",
    "#         idx = idx[:min(num_val, len(idx))]\n",
    "#         idx_val_edges_false = np.append(idx_val_edges_false, idx)\n",
    "\n",
    "    # sanity checks:\n",
    "#     train_edges_linear = train_edges[:,0]*adj.shape[0] + train_edges[:,1]\n",
    "#     test_edges_linear = test_edges[:,0]*adj.shape[0] + test_edges[:,1]\n",
    "#     assert not np.any(np.in1d(idx_test_edges_false, positive_idx))\n",
    "#     assert not np.any(np.in1d(idx_val_edges_false, positive_idx))\n",
    "#     assert not np.any(np.in1d(val_edges[:,0]*adj.shape[0]+val_edges[:,1], train_edges_linear))\n",
    "#     assert not np.any(np.in1d(test_edges_linear, train_edges_linear))\n",
    "#     assert not np.any(np.in1d(val_edges[:,0]*adj.shape[0]+val_edges[:,1], test_edges_linear))\n",
    "\n",
    "    # Re-build adj matrix\n",
    "    data = np.ones(train_edges.shape[0])\n",
    "    adj_train = sp.csr_matrix((data, (train_edges[:, 0], train_edges[:, 1])), shape=adj.shape)\n",
    "    adj_train = adj_train + adj_train.T\n",
    "    return adj_train, val_edges, val_edges_false, test_edges, test_edges_false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of nodes:  471633 . Number of edges:  5464345 . Avg Degree:  23.172021465843144\n",
      "Constructing new graph\n"
     ]
    }
   ],
   "source": [
    "adjList = defaultdict(set)                          # Convert set to list later for node2vec, set: to handle duplicates\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        paperID = data['id']\n",
    "        references = data.get('references', [])\n",
    "        for referencedPaper in references:\n",
    "            adjList[paperID].add(referencedPaper)\n",
    "            adjList[referencedPaper].add(paperID)\n",
    "\n",
    "adjList = {key: list(values) for key, values in adjList.items()}\n",
    "G = nx.from_dict_of_lists(adjList)\n",
    "\n",
    "nnodes = G.number_of_nodes()\n",
    "avgDegree = sum(d for n, d in G.degree()) / float(nnodes)\n",
    "print('Number of nodes: ', nnodes, '. Number of edges: ', G.number_of_edges(), '. Avg Degree: ', avgDegree)\n",
    "\n",
    "adj_sparse = nx.to_scipy_sparse_matrix(G)\n",
    "# Perform train-test split\n",
    "adj_train, val_edges, val_edges_false, test_edges, test_edges_false = mask_test_edges(adj_sparse)\n",
    "print('Constructing new graph')\n",
    "G_train = nx.from_scipy_sparse_matrix(adj_train) # new graph object with only non-hidden edges\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes = list(G.nodes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getEdgeEmbedding(embedding1, embedding2, policy='Hadamard'):\n",
    "    if (policy=='Hadamard'):\n",
    "        return embedding1 * embedding2\n",
    "    elif (policy=='Avg'):\n",
    "        return (embedding1 + embedding2) / 2\n",
    "def average(lis):\n",
    "    return (sum(lis) / len(lis))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "\n",
    "\n",
    "            \n",
    "edges = [*test_edges, *test_edges_false]\n",
    "for edge in edges :\n",
    "\n",
    "    u = nodes[edge[0]]\n",
    "    v = nodes[edge[1]]\n",
    "    if u in embeddingDict:\n",
    "        embedding1 = np.asarray(embeddingDict[u])\n",
    "    else:\n",
    "        embedding1 = np.asarray([0] * 32)\n",
    "    if v in embeddingDict:\n",
    "        embedding2 = np.asarray(embeddingDict[v])\n",
    "    else:\n",
    "        embedding2 = np.asarray([0] * 32)\n",
    "    edgeEmbedding =  getEdgeEmbedding(embedding1, embedding2)\n",
    "    X.append(edgeEmbedding)\n",
    "\n",
    "Y = np.asarray([1] * len(test_edges) + [0] * len(test_edges_false))\n",
    "\n",
    "del embeddingDict    \n",
    "X = np.asarray(X)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.6s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  1\n",
      "Fitting:  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.6s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  2\n",
      "Fitting:  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.7s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  3\n",
      "Fitting:  4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.4s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  4\n",
      "Fitting:  5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.6s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  5\n",
      "Name Random Forest . Avg Precision:  0.740860822956358 . Avg Recall:  0.7387222328644621 . Avg F-1 Score:  0.7381424303229813\n",
      "Fitting:  1\n",
      "Iteration 1, loss = 0.69369339\n",
      "Validation score: 0.500515\n",
      "Iteration 2, loss = 0.69273241\n",
      "Validation score: 0.499485\n",
      "Iteration 3, loss = 0.69249914\n",
      "Validation score: 0.500515\n",
      "Iteration 4, loss = 0.69212476\n",
      "Validation score: 0.734073\n",
      "Iteration 5, loss = 0.69156321\n",
      "Validation score: 0.504289\n",
      "Iteration 6, loss = 0.69083401\n",
      "Validation score: 0.596248\n",
      "Iteration 7, loss = 0.68988860\n",
      "Validation score: 0.732929\n",
      "Iteration 8, loss = 0.68865197\n",
      "Validation score: 0.728354\n",
      "Iteration 9, loss = 0.68690612\n",
      "Validation score: 0.593275\n",
      "Iteration 10, loss = 0.68486385\n",
      "Validation score: 0.625529\n",
      "Iteration 11, loss = 0.68232982\n",
      "Validation score: 0.700332\n",
      "Iteration 12, loss = 0.67935097\n",
      "Validation score: 0.732357\n",
      "Iteration 13, loss = 0.67586542\n",
      "Validation score: 0.731328\n",
      "Iteration 14, loss = 0.67208213\n",
      "Validation score: 0.700446\n",
      "Iteration 15, loss = 0.66814190\n",
      "Validation score: 0.708681\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  1\n",
      "Fitting:  2\n",
      "Iteration 1, loss = 0.69332584\n",
      "Validation score: 0.500858\n",
      "Iteration 2, loss = 0.69278317\n",
      "Validation score: 0.500858\n",
      "Iteration 3, loss = 0.69235896\n",
      "Validation score: 0.505776\n",
      "Iteration 4, loss = 0.69191694\n",
      "Validation score: 0.504632\n",
      "Iteration 5, loss = 0.69137417\n",
      "Validation score: 0.499142\n",
      "Iteration 6, loss = 0.69041634\n",
      "Validation score: 0.688780\n",
      "Iteration 7, loss = 0.68933362\n",
      "Validation score: 0.499142\n",
      "Iteration 8, loss = 0.68763528\n",
      "Validation score: 0.693012\n",
      "Iteration 9, loss = 0.68563661\n",
      "Validation score: 0.542834\n",
      "Iteration 10, loss = 0.68301646\n",
      "Validation score: 0.723550\n",
      "Iteration 11, loss = 0.68002978\n",
      "Validation score: 0.725495\n",
      "Iteration 12, loss = 0.67661811\n",
      "Validation score: 0.688436\n",
      "Iteration 13, loss = 0.67271081\n",
      "Validation score: 0.689466\n",
      "Iteration 14, loss = 0.66823927\n",
      "Validation score: 0.696900\n",
      "Iteration 15, loss = 0.66362294\n",
      "Validation score: 0.663388\n",
      "Iteration 16, loss = 0.65867145\n",
      "Validation score: 0.687979\n",
      "Iteration 17, loss = 0.65386816\n",
      "Validation score: 0.688551\n",
      "Iteration 18, loss = 0.64944170\n",
      "Validation score: 0.689466\n",
      "Iteration 19, loss = 0.64456930\n",
      "Validation score: 0.727096\n",
      "Iteration 20, loss = 0.64011395\n",
      "Validation score: 0.702848\n",
      "Iteration 21, loss = 0.63552184\n",
      "Validation score: 0.689123\n",
      "Iteration 22, loss = 0.63167971\n",
      "Validation score: 0.676999\n",
      "Iteration 23, loss = 0.62780428\n",
      "Validation score: 0.697701\n",
      "Iteration 24, loss = 0.62474312\n",
      "Validation score: 0.719547\n",
      "Iteration 25, loss = 0.62146936\n",
      "Validation score: 0.700560\n",
      "Iteration 26, loss = 0.61881148\n",
      "Validation score: 0.722406\n",
      "Iteration 27, loss = 0.61628519\n",
      "Validation score: 0.719890\n",
      "Iteration 28, loss = 0.61398170\n",
      "Validation score: 0.722521\n",
      "Iteration 29, loss = 0.61279809\n",
      "Validation score: 0.722978\n",
      "Iteration 30, loss = 0.61078816\n",
      "Validation score: 0.720233\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  2\n",
      "Fitting:  3\n",
      "Iteration 1, loss = 0.69326362\n",
      "Validation score: 0.505776\n",
      "Iteration 2, loss = 0.69278284\n",
      "Validation score: 0.500743\n",
      "Iteration 3, loss = 0.69238440\n",
      "Validation score: 0.499257\n",
      "Iteration 4, loss = 0.69174047\n",
      "Validation score: 0.500743\n",
      "Iteration 5, loss = 0.69077389\n",
      "Validation score: 0.732243\n",
      "Iteration 6, loss = 0.68929924\n",
      "Validation score: 0.672538\n",
      "Iteration 7, loss = 0.68697220\n",
      "Validation score: 0.736132\n",
      "Iteration 8, loss = 0.68398032\n",
      "Validation score: 0.643944\n",
      "Iteration 9, loss = 0.68026535\n",
      "Validation score: 0.578520\n",
      "Iteration 10, loss = 0.67607088\n",
      "Validation score: 0.692211\n",
      "Iteration 11, loss = 0.67115752\n",
      "Validation score: 0.715773\n",
      "Iteration 12, loss = 0.66585354\n",
      "Validation score: 0.733501\n",
      "Iteration 13, loss = 0.66023516\n",
      "Validation score: 0.738076\n",
      "Iteration 14, loss = 0.65444226\n",
      "Validation score: 0.738076\n",
      "Iteration 15, loss = 0.64897698\n",
      "Validation score: 0.604484\n",
      "Iteration 16, loss = 0.64298472\n",
      "Validation score: 0.713028\n",
      "Iteration 17, loss = 0.63761851\n",
      "Validation score: 0.736932\n",
      "Iteration 18, loss = 0.63255448\n",
      "Validation score: 0.737847\n",
      "Iteration 19, loss = 0.62783969\n",
      "Validation score: 0.738191\n",
      "Iteration 20, loss = 0.62362044\n",
      "Validation score: 0.734759\n",
      "Iteration 21, loss = 0.61956110\n",
      "Validation score: 0.733959\n",
      "Iteration 22, loss = 0.61595317\n",
      "Validation score: 0.734874\n",
      "Iteration 23, loss = 0.61293845\n",
      "Validation score: 0.706394\n",
      "Iteration 24, loss = 0.61075231\n",
      "Validation score: 0.714400\n",
      "Iteration 25, loss = 0.60836735\n",
      "Validation score: 0.707194\n",
      "Iteration 26, loss = 0.60661238\n",
      "Validation score: 0.732014\n",
      "Iteration 27, loss = 0.60494044\n",
      "Validation score: 0.731671\n",
      "Iteration 28, loss = 0.60270260\n",
      "Validation score: 0.732357\n",
      "Iteration 29, loss = 0.60116191\n",
      "Validation score: 0.732129\n",
      "Iteration 30, loss = 0.59967485\n",
      "Validation score: 0.730870\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  3\n",
      "Fitting:  4\n",
      "Iteration 1, loss = 0.69306236\n",
      "Validation score: 0.502917\n",
      "Iteration 2, loss = 0.69273522\n",
      "Validation score: 0.501658\n",
      "Iteration 3, loss = 0.69226065\n",
      "Validation score: 0.502917\n",
      "Iteration 4, loss = 0.69172524\n",
      "Validation score: 0.501658\n",
      "Iteration 5, loss = 0.69086130\n",
      "Validation score: 0.501658\n",
      "Iteration 6, loss = 0.68983623\n",
      "Validation score: 0.501658\n",
      "Iteration 7, loss = 0.68814444\n",
      "Validation score: 0.501658\n",
      "Iteration 8, loss = 0.68598557\n",
      "Validation score: 0.726867\n",
      "Iteration 9, loss = 0.68313155\n",
      "Validation score: 0.730413\n",
      "Iteration 10, loss = 0.67976501\n",
      "Validation score: 0.674940\n",
      "Iteration 11, loss = 0.67586044\n",
      "Validation score: 0.730299\n",
      "Iteration 12, loss = 0.67178173\n",
      "Validation score: 0.725037\n",
      "Iteration 13, loss = 0.66704498\n",
      "Validation score: 0.730299\n",
      "Iteration 14, loss = 0.66206105\n",
      "Validation score: 0.726295\n",
      "Iteration 15, loss = 0.65693433\n",
      "Validation score: 0.725037\n",
      "Iteration 16, loss = 0.65183170\n",
      "Validation score: 0.724465\n",
      "Iteration 17, loss = 0.64660342\n",
      "Validation score: 0.707537\n",
      "Iteration 18, loss = 0.64173045\n",
      "Validation score: 0.705136\n",
      "Iteration 19, loss = 0.63697011\n",
      "Validation score: 0.658012\n",
      "Iteration 20, loss = 0.63254552\n",
      "Validation score: 0.676656\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  4\n",
      "Fitting:  5\n",
      "Iteration 1, loss = 0.69311288\n",
      "Validation score: 0.498799\n",
      "Iteration 2, loss = 0.69286286\n",
      "Validation score: 0.498799\n",
      "Iteration 3, loss = 0.69256642\n",
      "Validation score: 0.711541\n",
      "Iteration 4, loss = 0.69210141\n",
      "Validation score: 0.733158\n",
      "Iteration 5, loss = 0.69138974\n",
      "Validation score: 0.607229\n",
      "Iteration 6, loss = 0.69034723\n",
      "Validation score: 0.506920\n",
      "Iteration 7, loss = 0.68888036\n",
      "Validation score: 0.597278\n",
      "Iteration 8, loss = 0.68698676\n",
      "Validation score: 0.602882\n",
      "Iteration 9, loss = 0.68475754\n",
      "Validation score: 0.608487\n",
      "Iteration 10, loss = 0.68171665\n",
      "Validation score: 0.651950\n",
      "Iteration 11, loss = 0.67881984\n",
      "Validation score: 0.700675\n",
      "Iteration 12, loss = 0.67508176\n",
      "Validation score: 0.710740\n",
      "Iteration 13, loss = 0.67107593\n",
      "Validation score: 0.734187\n",
      "Iteration 14, loss = 0.66710890\n",
      "Validation score: 0.608487\n",
      "Iteration 15, loss = 0.66252233\n",
      "Validation score: 0.733959\n",
      "Iteration 16, loss = 0.65817041\n",
      "Validation score: 0.691410\n",
      "Iteration 17, loss = 0.65339380\n",
      "Validation score: 0.729612\n",
      "Iteration 18, loss = 0.64905399\n",
      "Validation score: 0.729727\n",
      "Iteration 19, loss = 0.64431775\n",
      "Validation score: 0.695985\n",
      "Iteration 20, loss = 0.64005469\n",
      "Validation score: 0.729269\n",
      "Iteration 21, loss = 0.63578407\n",
      "Validation score: 0.710854\n",
      "Iteration 22, loss = 0.63206507\n",
      "Validation score: 0.703191\n",
      "Iteration 23, loss = 0.62816715\n",
      "Validation score: 0.702962\n",
      "Iteration 24, loss = 0.62553077\n",
      "Validation score: 0.710740\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  5\n",
      "Name Neural Net . Avg Precision:  0.7406036892244252 . Avg Recall:  0.7318137370277269 . Avg F-1 Score:  0.7293814916018915\n",
      "Fitting:  1\n",
      "count  1\n",
      "Fitting:  2\n",
      "count  2\n",
      "Fitting:  3\n",
      "count  3\n",
      "Fitting:  4\n",
      "count  4\n",
      "Fitting:  5\n",
      "count  5\n",
      "Name AdaBoost . Avg Precision:  0.7411017908186526 . Avg Recall:  0.7367274678417475 . Avg F-1 Score:  0.7355382749574038\n",
      "Fitting:  1\n",
      "count  1\n",
      "Fitting:  2\n",
      "count  2\n",
      "Fitting:  3\n",
      "count  3\n",
      "Fitting:  4\n",
      "count  4\n",
      "Fitting:  5\n",
      "count  5\n",
      "Name Linear SVC . Avg Precision:  0.7007325190437851 . Avg Recall:  0.6771590844360442 . Avg F-1 Score:  0.6641931924878092\n"
     ]
    }
   ],
   "source": [
    "\n",
    "kfold = KFold(n_splits=5, shuffle=True)\n",
    "for name, clf in zip(names, classifiers):\n",
    "    precScores = []\n",
    "    recallScores = []\n",
    "    f1Scores = []\n",
    "    count = 1\n",
    "    for train_index, test_index in kfold.split(X, Y):\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        y_train, y_test = Y[train_index], Y[test_index]\n",
    "        print('Fitting: ', count)\n",
    "        clf.fit(X_train, y_train)\n",
    "        print('count ', count)\n",
    "        y_pred = clf.predict(X_test)\n",
    "        prec, recall, fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
    "        precScores.append(prec)\n",
    "        recallScores.append(recall)\n",
    "        f1Scores.append(fscore)\n",
    "        count += 1\n",
    "    print('Name', name,'. Avg Precision: ', average(precScores), '. Avg Recall: ', average(recallScores), '. Avg F-1 Score: ', average(f1Scores) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[143157, 421449],\n",
       "       [ 86911, 145379],\n",
       "       [ 98634, 307101],\n",
       "       ...,\n",
       "       [152374, 389145],\n",
       "       [ 59286, 470841],\n",
       "       [206475, 243476]], dtype=int32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}