LabelClassifcitaion.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "from collections import defaultdict,Counter\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average(lis):\n",
    "    return sum(lis) / len(lis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "IDList = []\n",
    "labelList = []\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        fosList = defaultdict(int)\n",
    "        for fos in data.get('fos',[]):\n",
    "            if (fos['name'] == 'Data mining' or fos['name'] == 'Data science'):\n",
    "                fosList['Data'] = max(fosList['Data'], fos['w'])\n",
    "            if (fos['name'] == 'Natural language processing' or fos['name'] == 'Speech recognition'):\n",
    "                fosList['NLP'] = max(fosList['NLP'], fos['w'])\n",
    "            if (fos['name'] == 'Computer vision'):\n",
    "                fosList['CV'] = fos['w']\n",
    "        if (len(fosList.keys())):\n",
    "            IDList.append(data['id'])\n",
    "            fosList = [(key, value) for key, value in fosList.items()]\n",
    "            fosList = sorted(fosList, key = lambda x: x[1],    reverse=True)\n",
    "            labelList.append(fosList[0][0])\n",
    "assert len(labelList) == len(IDList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'CV': 142790, 'NLP': 57416, 'Data': 117215})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter = Counter(labelList)\n",
    "counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Load embeddings as X\n",
    "embeddingFileName = './data/dblpAbstract_2Thresholded_FT_Embeddings.json'\n",
    "embeddingDict = dict()\n",
    "with open(embeddingFileName, 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        embeddingDict[data['id']] = data['embedding']\n",
    "\n",
    "embeddings = []\n",
    "for id in IDList:\n",
    "    embeddings.append(embeddingDict[id])\n",
    "del embeddingDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/du0/15CS30003/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:6: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "  \n",
      "/home/du0/15CS30003/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "embeddings = [] \n",
    "from gensim import models\n",
    "outFileName = './models/node2vec_USE_2Citation_Embeddings_WL_' + str(8) + '_NN_' + str(4) + '.kv'\n",
    "model = models.keyedvectors.KeyedVectors.load_word2vec_format('./models/node2vec_USE_2Citation_Embeddings.kv')\n",
    "for id in IDList:\n",
    "    if id in model.wv.vocab:\n",
    "        embeddings.append(model.wv[id])\n",
    "    else:\n",
    "        embeddings.append([0] * 128)\n",
    "del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.asarray(embeddings)\n",
    "del embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "samplingDict = dict()\n",
    "samplingDict['CV'] = 30000\n",
    "samplingDict['NLP'] = 30000\n",
    "samplingDict['Data'] = 30000\n",
    "rus = RandomUnderSampler(sampling_strategy = samplingDict,random_state=0)\n",
    "assert X.shape[0] == len(labelList)\n",
    "X,Y= rus.fit_resample(X, labelList)\n",
    "counter = Counter(Y)\n",
    "counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({0: 142790, 1: 117215, 2: 57416})\n"
     ]
    }
   ],
   "source": [
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(labelList)\n",
    "Y = le.transform(labelList)\n",
    "counter = Counter(Y)\n",
    "print(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.gaussian_process.kernels import RBF\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, BaggingClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "\n",
    "\n",
    "names = [\n",
    " \"Random Forest\", \"Neural Net\", \"AdaBoost\", \"Linear SVC\" ]\n",
    "\n",
    "classifiers = [\n",
    "    RandomForestClassifier(verbose=True, n_jobs = -1),\n",
    "    MLPClassifier(verbose=True, early_stopping=True),\n",
    "    AdaBoostClassifier(),\n",
    "    OneVsRestClassifier(BaggingClassifier(LinearSVC(),n_jobs = -1))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   35.9s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   37.8s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   36.9s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   39.5s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   38.2s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name Random Forest . Avg Precision:  0.9054602580821862 . Avg Recall:  0.9067957039648548 . Avg F-1 Score:  0.9055497816508125\n",
      "Fitting:  1\n",
      "Iteration 1, loss = 0.38080415\n",
      "Validation score: 0.899071\n",
      "Iteration 2, loss = 0.29195660\n",
      "Validation score: 0.906041\n",
      "Iteration 3, loss = 0.27632414\n",
      "Validation score: 0.905686\n",
      "Iteration 4, loss = 0.26920996\n",
      "Validation score: 0.909782\n",
      "Iteration 5, loss = 0.26430561\n",
      "Validation score: 0.908522\n",
      "Iteration 6, loss = 0.26180344\n",
      "Validation score: 0.905883\n",
      "Iteration 7, loss = 0.25885140\n",
      "Validation score: 0.911751\n",
      "Iteration 8, loss = 0.25690439\n",
      "Validation score: 0.905923\n",
      "Iteration 9, loss = 0.25470445\n",
      "Validation score: 0.911711\n",
      "Iteration 10, loss = 0.25354121\n",
      "Validation score: 0.912460\n",
      "Iteration 11, loss = 0.25163821\n",
      "Validation score: 0.912735\n",
      "Iteration 12, loss = 0.25070637\n",
      "Validation score: 0.913995\n",
      "Iteration 13, loss = 0.24920658\n",
      "Validation score: 0.908640\n",
      "Iteration 14, loss = 0.24881840\n",
      "Validation score: 0.908167\n",
      "Iteration 15, loss = 0.24858338\n",
      "Validation score: 0.908089\n",
      "Iteration 16, loss = 0.24749621\n",
      "Validation score: 0.913129\n",
      "Iteration 17, loss = 0.24659050\n",
      "Validation score: 0.911239\n",
      "Iteration 18, loss = 0.24666450\n",
      "Validation score: 0.914744\n",
      "Iteration 19, loss = 0.24571000\n",
      "Validation score: 0.912263\n",
      "Iteration 20, loss = 0.24487102\n",
      "Validation score: 0.912893\n",
      "Iteration 21, loss = 0.24418629\n",
      "Validation score: 0.914507\n",
      "Iteration 22, loss = 0.24364814\n",
      "Validation score: 0.914901\n",
      "Iteration 23, loss = 0.24421415\n",
      "Validation score: 0.914507\n",
      "Iteration 24, loss = 0.24291331\n",
      "Validation score: 0.914626\n",
      "Iteration 25, loss = 0.24251789\n",
      "Validation score: 0.911908\n",
      "Iteration 26, loss = 0.24216752\n",
      "Validation score: 0.914507\n",
      "Iteration 27, loss = 0.24217834\n",
      "Validation score: 0.914074\n",
      "Iteration 28, loss = 0.24097771\n",
      "Validation score: 0.914665\n",
      "Iteration 29, loss = 0.24171368\n",
      "Validation score: 0.912814\n",
      "Iteration 30, loss = 0.24142856\n",
      "Validation score: 0.912105\n",
      "Iteration 31, loss = 0.24111215\n",
      "Validation score: 0.915374\n",
      "Iteration 32, loss = 0.23993230\n",
      "Validation score: 0.915098\n",
      "Iteration 33, loss = 0.24034522\n",
      "Validation score: 0.912932\n",
      "Iteration 34, loss = 0.24008851\n",
      "Validation score: 0.914980\n",
      "Iteration 35, loss = 0.23933942\n",
      "Validation score: 0.913877\n",
      "Iteration 36, loss = 0.23966031\n",
      "Validation score: 0.913129\n",
      "Iteration 37, loss = 0.23983184\n",
      "Validation score: 0.916437\n",
      "Iteration 38, loss = 0.23896050\n",
      "Validation score: 0.912972\n",
      "Iteration 39, loss = 0.23844204\n",
      "Validation score: 0.913759\n",
      "Iteration 40, loss = 0.23904326\n",
      "Validation score: 0.915374\n",
      "Iteration 41, loss = 0.23857407\n",
      "Validation score: 0.913484\n",
      "Iteration 42, loss = 0.23788251\n",
      "Validation score: 0.911436\n",
      "Iteration 43, loss = 0.23811949\n",
      "Validation score: 0.914901\n",
      "Iteration 44, loss = 0.23775806\n",
      "Validation score: 0.914665\n",
      "Iteration 45, loss = 0.23763791\n",
      "Validation score: 0.913759\n",
      "Iteration 46, loss = 0.23724735\n",
      "Validation score: 0.916240\n",
      "Iteration 47, loss = 0.23815232\n",
      "Validation score: 0.916476\n",
      "Iteration 48, loss = 0.23725209\n",
      "Validation score: 0.913523\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  1\n",
      "Fitting:  2\n",
      "Iteration 1, loss = 0.38147730\n",
      "Validation score: 0.898598\n",
      "Iteration 2, loss = 0.29333346\n",
      "Validation score: 0.903639\n",
      "Iteration 3, loss = 0.27822924\n",
      "Validation score: 0.908679\n",
      "Iteration 4, loss = 0.27098689\n",
      "Validation score: 0.908994\n",
      "Iteration 5, loss = 0.26620400\n",
      "Validation score: 0.908561\n",
      "Iteration 6, loss = 0.26259545\n",
      "Validation score: 0.907852\n",
      "Iteration 7, loss = 0.26051945\n",
      "Validation score: 0.911515\n",
      "Iteration 8, loss = 0.25863241\n",
      "Validation score: 0.909349\n",
      "Iteration 9, loss = 0.25656065\n",
      "Validation score: 0.911790\n",
      "Iteration 10, loss = 0.25534278\n",
      "Validation score: 0.912657\n",
      "Iteration 11, loss = 0.25410264\n",
      "Validation score: 0.909900\n",
      "Iteration 12, loss = 0.25226569\n",
      "Validation score: 0.912145\n",
      "Iteration 13, loss = 0.25188245\n",
      "Validation score: 0.914586\n",
      "Iteration 14, loss = 0.25036755\n",
      "Validation score: 0.915177\n",
      "Iteration 15, loss = 0.24953199\n",
      "Validation score: 0.914941\n",
      "Iteration 16, loss = 0.24873304\n",
      "Validation score: 0.915098\n",
      "Iteration 17, loss = 0.24849469\n",
      "Validation score: 0.913877\n",
      "Iteration 18, loss = 0.24818543\n",
      "Validation score: 0.910136\n",
      "Iteration 19, loss = 0.24686859\n",
      "Validation score: 0.914232\n",
      "Iteration 20, loss = 0.24640479\n",
      "Validation score: 0.915649\n",
      "Iteration 21, loss = 0.24621280\n",
      "Validation score: 0.913759\n",
      "Iteration 22, loss = 0.24561051\n",
      "Validation score: 0.915846\n",
      "Iteration 23, loss = 0.24551081\n",
      "Validation score: 0.915492\n",
      "Iteration 24, loss = 0.24434105\n",
      "Validation score: 0.914901\n",
      "Iteration 25, loss = 0.24398446\n",
      "Validation score: 0.911199\n",
      "Iteration 26, loss = 0.24333804\n",
      "Validation score: 0.915492\n",
      "Iteration 27, loss = 0.24334285\n",
      "Validation score: 0.915452\n",
      "Iteration 28, loss = 0.24314845\n",
      "Validation score: 0.916594\n",
      "Iteration 29, loss = 0.24241759\n",
      "Validation score: 0.915295\n",
      "Iteration 30, loss = 0.24227906\n",
      "Validation score: 0.916161\n",
      "Iteration 31, loss = 0.24188448\n",
      "Validation score: 0.916634\n",
      "Iteration 32, loss = 0.24187355\n",
      "Validation score: 0.913365\n",
      "Iteration 33, loss = 0.24172338\n",
      "Validation score: 0.913956\n",
      "Iteration 34, loss = 0.24117773\n",
      "Validation score: 0.914862\n",
      "Iteration 35, loss = 0.24091549\n",
      "Validation score: 0.914941\n",
      "Iteration 36, loss = 0.24087936\n",
      "Validation score: 0.914783\n",
      "Iteration 37, loss = 0.24015143\n",
      "Validation score: 0.914192\n",
      "Iteration 38, loss = 0.24072045\n",
      "Validation score: 0.915610\n",
      "Iteration 39, loss = 0.23971277\n",
      "Validation score: 0.911003\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  2\n",
      "Fitting:  3\n",
      "Iteration 1, loss = 0.38013099\n",
      "Validation score: 0.898795\n",
      "Iteration 2, loss = 0.29294155\n",
      "Validation score: 0.904151\n",
      "Iteration 3, loss = 0.27862362\n",
      "Validation score: 0.906907\n",
      "Iteration 4, loss = 0.27069477\n",
      "Validation score: 0.906671\n",
      "Iteration 5, loss = 0.26626448\n",
      "Validation score: 0.908640\n",
      "Iteration 6, loss = 0.26274385\n",
      "Validation score: 0.909427\n",
      "Iteration 7, loss = 0.25981516\n",
      "Validation score: 0.909782\n",
      "Iteration 8, loss = 0.25761133\n",
      "Validation score: 0.911081\n",
      "Iteration 9, loss = 0.25611504\n",
      "Validation score: 0.910806\n",
      "Iteration 10, loss = 0.25407444\n",
      "Validation score: 0.909664\n",
      "Iteration 11, loss = 0.25270852\n",
      "Validation score: 0.911239\n",
      "Iteration 12, loss = 0.25158003\n",
      "Validation score: 0.911554\n",
      "Iteration 13, loss = 0.25088670\n",
      "Validation score: 0.912578\n",
      "Iteration 14, loss = 0.24963827\n",
      "Validation score: 0.909703\n",
      "Iteration 15, loss = 0.24885915\n",
      "Validation score: 0.912853\n",
      "Iteration 16, loss = 0.24761184\n",
      "Validation score: 0.910845\n",
      "Iteration 17, loss = 0.24811618\n",
      "Validation score: 0.913444\n",
      "Iteration 18, loss = 0.24662272\n",
      "Validation score: 0.912775\n",
      "Iteration 19, loss = 0.24616091\n",
      "Validation score: 0.911593\n",
      "Iteration 20, loss = 0.24583149\n",
      "Validation score: 0.913011\n",
      "Iteration 21, loss = 0.24458232\n",
      "Validation score: 0.912893\n",
      "Iteration 22, loss = 0.24464811\n",
      "Validation score: 0.911711\n",
      "Iteration 23, loss = 0.24397351\n",
      "Validation score: 0.913011\n",
      "Iteration 24, loss = 0.24355198\n",
      "Validation score: 0.911908\n",
      "Iteration 25, loss = 0.24346953\n",
      "Validation score: 0.912302\n",
      "Iteration 26, loss = 0.24233883\n",
      "Validation score: 0.913129\n",
      "Iteration 27, loss = 0.24281059\n",
      "Validation score: 0.913050\n",
      "Iteration 28, loss = 0.24237235\n",
      "Validation score: 0.910806\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  3\n",
      "Fitting:  4\n",
      "Iteration 1, loss = 0.38693557\n",
      "Validation score: 0.897535\n",
      "Iteration 2, loss = 0.29567712\n",
      "Validation score: 0.903127\n",
      "Iteration 3, loss = 0.27997529\n",
      "Validation score: 0.905844\n",
      "Iteration 4, loss = 0.27167931\n",
      "Validation score: 0.908443\n",
      "Iteration 5, loss = 0.26656036\n",
      "Validation score: 0.908364\n",
      "Iteration 6, loss = 0.26255098\n",
      "Validation score: 0.908915\n",
      "Iteration 7, loss = 0.25978695\n",
      "Validation score: 0.907498\n",
      "Iteration 8, loss = 0.25731105\n",
      "Validation score: 0.910097\n",
      "Iteration 9, loss = 0.25561617\n",
      "Validation score: 0.909231\n",
      "Iteration 10, loss = 0.25434073\n",
      "Validation score: 0.910569\n",
      "Iteration 11, loss = 0.25331305\n",
      "Validation score: 0.906789\n",
      "Iteration 12, loss = 0.25155359\n",
      "Validation score: 0.911633\n",
      "Iteration 13, loss = 0.25006952\n",
      "Validation score: 0.909703\n",
      "Iteration 14, loss = 0.24965394\n",
      "Validation score: 0.911633\n",
      "Iteration 15, loss = 0.24887578\n",
      "Validation score: 0.903796\n",
      "Iteration 16, loss = 0.24778034\n",
      "Validation score: 0.904032\n",
      "Iteration 17, loss = 0.24666256\n",
      "Validation score: 0.911751\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 18, loss = 0.24691243\n",
      "Validation score: 0.912263\n",
      "Iteration 19, loss = 0.24543926\n",
      "Validation score: 0.910097\n",
      "Iteration 20, loss = 0.24505414\n",
      "Validation score: 0.912578\n",
      "Iteration 21, loss = 0.24460059\n",
      "Validation score: 0.912499\n",
      "Iteration 22, loss = 0.24362876\n",
      "Validation score: 0.911869\n",
      "Iteration 23, loss = 0.24353138\n",
      "Validation score: 0.913799\n",
      "Iteration 24, loss = 0.24315979\n",
      "Validation score: 0.913562\n",
      "Iteration 25, loss = 0.24305361\n",
      "Validation score: 0.912853\n",
      "Iteration 26, loss = 0.24228454\n",
      "Validation score: 0.912932\n",
      "Iteration 27, loss = 0.24230337\n",
      "Validation score: 0.909900\n",
      "Iteration 28, loss = 0.24125927\n",
      "Validation score: 0.910766\n",
      "Iteration 29, loss = 0.24160167\n",
      "Validation score: 0.913995\n",
      "Iteration 30, loss = 0.24139705\n",
      "Validation score: 0.912538\n",
      "Iteration 31, loss = 0.24026356\n",
      "Validation score: 0.914074\n",
      "Iteration 32, loss = 0.24062367\n",
      "Validation score: 0.913759\n",
      "Iteration 33, loss = 0.24101429\n",
      "Validation score: 0.913247\n",
      "Iteration 34, loss = 0.23934614\n",
      "Validation score: 0.912184\n",
      "Iteration 35, loss = 0.23935700\n",
      "Validation score: 0.911081\n",
      "Iteration 36, loss = 0.23939910\n",
      "Validation score: 0.913602\n",
      "Iteration 37, loss = 0.23841546\n",
      "Validation score: 0.913090\n",
      "Iteration 38, loss = 0.23887375\n",
      "Validation score: 0.914507\n",
      "Iteration 39, loss = 0.23827069\n",
      "Validation score: 0.911633\n",
      "Iteration 40, loss = 0.23859898\n",
      "Validation score: 0.911239\n",
      "Iteration 41, loss = 0.23834976\n",
      "Validation score: 0.913484\n",
      "Iteration 42, loss = 0.23808030\n",
      "Validation score: 0.914547\n",
      "Iteration 43, loss = 0.23738707\n",
      "Validation score: 0.910136\n",
      "Iteration 44, loss = 0.23808288\n",
      "Validation score: 0.915019\n",
      "Iteration 45, loss = 0.23718849\n",
      "Validation score: 0.913641\n",
      "Iteration 46, loss = 0.23748764\n",
      "Validation score: 0.915216\n",
      "Iteration 47, loss = 0.23698550\n",
      "Validation score: 0.913168\n",
      "Iteration 48, loss = 0.23638391\n",
      "Validation score: 0.913602\n",
      "Iteration 49, loss = 0.23760892\n",
      "Validation score: 0.914507\n",
      "Iteration 50, loss = 0.23677574\n",
      "Validation score: 0.911711\n",
      "Iteration 51, loss = 0.23627797\n",
      "Validation score: 0.912263\n",
      "Iteration 52, loss = 0.23635704\n",
      "Validation score: 0.914901\n",
      "Iteration 53, loss = 0.23628807\n",
      "Validation score: 0.913602\n",
      "Iteration 54, loss = 0.23597458\n",
      "Validation score: 0.914862\n",
      "Iteration 55, loss = 0.23597441\n",
      "Validation score: 0.914980\n",
      "Iteration 56, loss = 0.23530373\n",
      "Validation score: 0.914822\n",
      "Iteration 57, loss = 0.23530545\n",
      "Validation score: 0.914429\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  4\n",
      "Fitting:  5\n",
      "Iteration 1, loss = 0.38062039\n",
      "Validation score: 0.899543\n",
      "Iteration 2, loss = 0.29219425\n",
      "Validation score: 0.904466\n",
      "Iteration 3, loss = 0.27745224\n",
      "Validation score: 0.906631\n",
      "Iteration 4, loss = 0.27034475\n",
      "Validation score: 0.907695\n",
      "Iteration 5, loss = 0.26537837\n",
      "Validation score: 0.905805\n",
      "Iteration 6, loss = 0.26207469\n",
      "Validation score: 0.907734\n",
      "Iteration 7, loss = 0.25969303\n",
      "Validation score: 0.903796\n",
      "Iteration 8, loss = 0.25738504\n",
      "Validation score: 0.909388\n",
      "Iteration 9, loss = 0.25497629\n",
      "Validation score: 0.910845\n",
      "Iteration 10, loss = 0.25356838\n",
      "Validation score: 0.911436\n",
      "Iteration 11, loss = 0.25193241\n",
      "Validation score: 0.911869\n",
      "Iteration 12, loss = 0.25086471\n",
      "Validation score: 0.907852\n",
      "Iteration 13, loss = 0.24981166\n",
      "Validation score: 0.912617\n",
      "Iteration 14, loss = 0.24903762\n",
      "Validation score: 0.911003\n",
      "Iteration 15, loss = 0.24766903\n",
      "Validation score: 0.910215\n",
      "Iteration 16, loss = 0.24726524\n",
      "Validation score: 0.911593\n",
      "Iteration 17, loss = 0.24662152\n",
      "Validation score: 0.909624\n",
      "Iteration 18, loss = 0.24593375\n",
      "Validation score: 0.910569\n",
      "Iteration 19, loss = 0.24540543\n",
      "Validation score: 0.910018\n",
      "Iteration 20, loss = 0.24458246\n",
      "Validation score: 0.912538\n",
      "Iteration 21, loss = 0.24469797\n",
      "Validation score: 0.910254\n",
      "Iteration 22, loss = 0.24421325\n",
      "Validation score: 0.913484\n",
      "Iteration 23, loss = 0.24314219\n",
      "Validation score: 0.912381\n",
      "Iteration 24, loss = 0.24274429\n",
      "Validation score: 0.911948\n",
      "Iteration 25, loss = 0.24243201\n",
      "Validation score: 0.912066\n",
      "Iteration 26, loss = 0.24178367\n",
      "Validation score: 0.913326\n",
      "Iteration 27, loss = 0.24139552\n",
      "Validation score: 0.912893\n",
      "Iteration 28, loss = 0.24103520\n",
      "Validation score: 0.912893\n",
      "Iteration 29, loss = 0.24096148\n",
      "Validation score: 0.911830\n",
      "Iteration 30, loss = 0.24065998\n",
      "Validation score: 0.911199\n",
      "Iteration 31, loss = 0.24106568\n",
      "Validation score: 0.915137\n",
      "Iteration 32, loss = 0.24025875\n",
      "Validation score: 0.914271\n",
      "Iteration 33, loss = 0.23931064\n",
      "Validation score: 0.913680\n",
      "Iteration 34, loss = 0.23924040\n",
      "Validation score: 0.912145\n",
      "Iteration 35, loss = 0.23975336\n",
      "Validation score: 0.911987\n",
      "Iteration 36, loss = 0.23894563\n",
      "Validation score: 0.913680\n",
      "Iteration 37, loss = 0.23841042\n",
      "Validation score: 0.912341\n",
      "Iteration 38, loss = 0.23768752\n",
      "Validation score: 0.912302\n",
      "Iteration 39, loss = 0.23870322\n",
      "Validation score: 0.913405\n",
      "Iteration 40, loss = 0.23826283\n",
      "Validation score: 0.913129\n",
      "Iteration 41, loss = 0.23744444\n",
      "Validation score: 0.912341\n",
      "Iteration 42, loss = 0.23754479\n",
      "Validation score: 0.914547\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  5\n",
      "Name Neural Net . Avg Precision:  0.9147747891201614 . Avg Recall:  0.9155191381732039 . Avg F-1 Score:  0.9148943158649573\n",
      "Fitting:  1\n",
      "count  1\n",
      "Fitting:  2\n",
      "count  2\n",
      "Fitting:  3\n",
      "count  3\n",
      "Fitting:  4\n",
      "count  4\n",
      "Fitting:  5\n",
      "count  5\n",
      "Name AdaBoost . Avg Precision:  0.8752642574028376 . Avg Recall:  0.8769268649193741 . Avg F-1 Score:  0.8758133612762837\n",
      "Fitting:  1\n",
      "count  1\n",
      "Fitting:  2\n",
      "count  2\n",
      "Fitting:  3\n",
      "count  3\n",
      "Fitting:  4\n",
      "count  4\n",
      "Fitting:  5\n",
      "count  5\n",
      "Name Linear SVC . Avg Precision:  0.9086870896412981 . Avg Recall:  0.9096153074400204 . Avg F-1 Score:  0.9083098634591631\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn import svm\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "\n",
    "kfold = KFold(n_splits=5, shuffle=True)\n",
    "for name, clf in zip(names, classifiers):\n",
    "    precScores = []\n",
    "    recallScores = []\n",
    "    f1Scores = []\n",
    "    count = 1\n",
    "    for train_index, test_index in kfold.split(X, Y):\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        y_train, y_test = Y[train_index], Y[test_index]\n",
    "        print('Fitting: ', count)\n",
    "        clf.fit(X_train, y_train)\n",
    "        print('count ', count)\n",
    "        y_pred = clf.predict(X_test)\n",
    "        prec, recall, fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
    "        precScores.append(prec)\n",
    "        recallScores.append(recall)\n",
    "        f1Scores.append(fscore)\n",
    "        count += 1\n",
    "    print('Name', name,'. Avg Precision: ', average(precScores), '. Avg Recall: ', average(recallScores), '. Avg F-1 Score: ', average(f1Scores) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import models\n",
    "\n",
    "model = models.keyedvectors.KeyedVectors.load_word2vec_format('./models/node2vec_Embeddings.kv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/du0/15CS30003/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "471633"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.wv.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}