AbstractEmbeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from owlready2 import *\n",
    "import re\n",
    "import fasttext \n",
    "import numpy as np\n",
    "from numpy import dot\n",
    "from numpy.linalg import norm\n",
    "import networkx as nx\n",
    "import annoy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "AITopLevelTopics = ['Artificial intelligence', 'Computer vision', 'Data mining',\n",
    "                     'Data science', 'Machine learning', 'Natural language processing',\n",
    "                     'Pattern recognition', 'Speech recognition']\n",
    "with open(\"dblpPaperIDs.json\", 'r') as f:\n",
    "    paperList = json.load(f)\n",
    "papersUnderConsideration = set(paperList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "paperData = dict()\n",
    "with open('dblp_AIpapers_v11.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        \n",
    "        dataDict = dict()\n",
    "        dataDict['title'] = data.get('title', '')\n",
    "        references = list()\n",
    "        for reference in data.get('references',[]):\n",
    "            if reference in papersUnderConsideration:\n",
    "                references.append(reference)\n",
    "        dataDict['references'] = references\n",
    "        dataDict['abstract'] = []\n",
    "        if 'indexed_abstract' in data:\n",
    "            dataDict['abstract'] = [w for w in data['indexed_abstract']['InvertedIndex'].keys() if len(w) > 1]                    \n",
    "        dataDict['fos'] = data.get('fos',[])\n",
    "        # paperData[paperID] = dataDict\n",
    "        tmpDict = dataDict.copy()\n",
    "        tmpDict['id'] = paperID\n",
    "        \n",
    "        paperID = data['id'] \n",
    "            \n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "fasttextModel = fasttext.load_model('crawl-300d-2M-subword.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    with open('./data/dblpAbstract_2Thresholded_FT_Embeddings.json', 'w') as outfile:\n",
    "        for line in file:\n",
    "            data = json.loads(line)\n",
    "            paperID = data['id'] \n",
    "            abstractWordList = data.get('abstract',[])\n",
    "            abstractString = ' '.join(word for word in abstractWordList)\n",
    "            abstractString = abstractString.replace('\\n', ' ').replace('\\r', '')\n",
    "            embedding = fasttextModel.get_sentence_vector(abstractString).tolist()    # while reading use np.asarray to convert to np array\n",
    "            outDict = dict()\n",
    "            outDict['id'] = paperID\n",
    "            outDict['embedding'] = embedding\n",
    "            json.dump(outDict, outfile)\n",
    "            outfile.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!curl -L \"https://tfhub.dev/google/universal-sentence-encoder-large/5?tf-hub-format=compressed\" | tar -zxvC ./module/UnivTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "module_url = \"./module/UnivTrans\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/4\", \"https://tfhub.dev/google/universal-sentence-encoder-large/5\"]\n",
    "model = hub.load(module_url)\n",
    "def embed(inputText):\n",
    "    return model(inputText)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:count: 0\n",
      "INFO:root:count: 10000\n",
      "INFO:root:count: 20000\n",
      "INFO:root:count: 30000\n",
      "INFO:root:count: 40000\n",
      "INFO:root:count: 50000\n",
      "INFO:root:count: 60000\n",
      "INFO:root:count: 70000\n",
      "INFO:root:count: 80000\n",
      "INFO:root:count: 90000\n",
      "INFO:root:count: 100000\n",
      "INFO:root:count: 110000\n",
      "INFO:root:count: 120000\n",
      "INFO:root:count: 130000\n",
      "INFO:root:count: 140000\n",
      "INFO:root:count: 150000\n",
      "INFO:root:count: 160000\n",
      "INFO:root:count: 170000\n",
      "INFO:root:count: 180000\n",
      "INFO:root:count: 190000\n",
      "INFO:root:count: 200000\n",
      "INFO:root:count: 210000\n",
      "INFO:root:count: 220000\n",
      "INFO:root:count: 230000\n",
      "INFO:root:count: 240000\n",
      "INFO:root:count: 250000\n",
      "INFO:root:count: 260000\n",
      "INFO:root:count: 270000\n",
      "INFO:root:count: 280000\n",
      "INFO:root:count: 290000\n",
      "INFO:root:count: 300000\n",
      "INFO:root:count: 310000\n",
      "INFO:root:count: 320000\n",
      "INFO:root:count: 330000\n",
      "INFO:root:count: 340000\n",
      "INFO:root:count: 350000\n",
      "INFO:root:count: 360000\n",
      "INFO:root:count: 370000\n",
      "INFO:root:count: 380000\n",
      "INFO:root:count: 390000\n",
      "INFO:root:count: 400000\n",
      "INFO:root:count: 410000\n",
      "INFO:root:count: 420000\n",
      "INFO:root:count: 430000\n",
      "INFO:root:count: 440000\n",
      "INFO:root:count: 450000\n",
      "INFO:root:count: 460000\n",
      "INFO:root:count: 470000\n",
      "INFO:root:count: 480000\n",
      "INFO:root:count: 490000\n",
      "INFO:root:count: 500000\n",
      "INFO:root:count: 510000\n",
      "INFO:root:count: 520000\n",
      "INFO:root:count: 530000\n",
      "INFO:root:count: 540000\n",
      "INFO:root:count: 550000\n",
      "INFO:root:count: 560000\n",
      "INFO:root:count: 570000\n",
      "INFO:root:count: 580000\n",
      "INFO:root:count: 590000\n",
      "INFO:root:count: 600000\n",
      "INFO:root:count: 610000\n",
      "INFO:root:count: 620000\n",
      "INFO:root:count: 630000\n",
      "INFO:root:count: 640000\n",
      "INFO:root:count: 650000\n",
      "INFO:root:count: 660000\n",
      "INFO:root:count: 670000\n",
      "INFO:root:count: 680000\n",
      "INFO:root:count: 690000\n",
      "INFO:root:count: 700000\n",
      "INFO:root:count: 710000\n",
      "INFO:root:count: 720000\n",
      "INFO:root:count: 730000\n",
      "INFO:root:count: 740000\n",
      "INFO:root:count: 750000\n",
      "INFO:root:count: 760000\n",
      "INFO:root:count: 770000\n",
      "INFO:root:count: 780000\n",
      "INFO:root:count: 790000\n",
      "INFO:root:count: 800000\n",
      "INFO:root:count: 810000\n",
      "INFO:root:count: 820000\n",
      "INFO:root:count: 830000\n",
      "INFO:root:count: 840000\n",
      "INFO:root:count: 850000\n",
      "INFO:root:count: 860000\n",
      "INFO:root:count: 870000\n",
      "INFO:root:count: 880000\n",
      "INFO:root:count: 890000\n",
      "INFO:root:count: 900000\n",
      "INFO:root:count: 910000\n",
      "INFO:root:count: 920000\n",
      "INFO:root:count: 930000\n",
      "INFO:root:count: 940000\n",
      "INFO:root:count: 950000\n",
      "INFO:root:count: 960000\n",
      "INFO:root:count: 970000\n",
      "INFO:root:count: 980000\n",
      "INFO:root:count: 990000\n",
      "INFO:root:count: 1000000\n",
      "INFO:root:count: 1010000\n",
      "INFO:root:count: 1020000\n",
      "INFO:root:count: 1030000\n",
      "INFO:root:count: 1040000\n",
      "INFO:root:count: 1050000\n",
      "INFO:root:count: 1060000\n",
      "INFO:root:count: 1070000\n",
      "INFO:root:count: 1080000\n",
      "INFO:root:count: 1090000\n",
      "INFO:root:count: 1100000\n",
      "INFO:root:count: 1110000\n",
      "INFO:root:count: 1120000\n",
      "INFO:root:count: 1130000\n",
      "INFO:root:count: 1140000\n",
      "INFO:root:count: 1150000\n",
      "INFO:root:count: 1160000\n",
      "INFO:root:count: 1170000\n",
      "INFO:root:count: 1180000\n",
      "INFO:root:count: 1190000\n",
      "INFO:root:count: 1200000\n",
      "INFO:root:count: 1210000\n",
      "INFO:root:count: 1220000\n",
      "INFO:root:count: 1230000\n",
      "INFO:root:count: 1240000\n",
      "INFO:root:count: 1250000\n",
      "INFO:root:count: 1260000\n",
      "INFO:root:count: 1270000\n",
      "INFO:root:count: 1280000\n",
      "INFO:root:count: 1290000\n",
      "INFO:root:count: 1300000\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "with open('dblp_AIpapers_v11.json', 'r') as file:\n",
    "    with open('dblpAbstractUSETransEmbeddings.json', 'w') as outfile:\n",
    "        for line in file:\n",
    "            data = json.loads(line)\n",
    "            paperID = data['id'] \n",
    "            abstractWordList = data.get('abstract',[])\n",
    "            abstractString = ' '.join(word for word in abstractWordList)\n",
    "            abstractString = abstractString.replace('\\n', ' ').replace('\\r', '')\n",
    "            embedding = embed([abstractString])[0].numpy()\n",
    "            outDict = dict()\n",
    "            outDict['id'] = paperID\n",
    "            outDict['embedding'] = embedding.tolist()\n",
    "            json.dump(outDict, outfile)\n",
    "            outfile.write('\\n')\n",
    "            if (count % 10000 == 0):\n",
    "                logger.info('count: {0}'.format(count))\n",
    "            count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as session:\n",
    "    session.run([tf.global_variables_initializer(), tf.tables_initializer()])\n",
    "    abstractEmbeddings = session.run(embed(abstractList))\n",
    "del abstractList[:] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dblpAbstractUSETransEmbeddings.json', 'w') as outfile:\n",
    "    for paperId, embedding in zip(paperIDList, abstractEmbeddings):\n",
    "        outDict = dict()\n",
    "        outDict['id'] = paperID\n",
    "        outDict['embedding'] = embedding.tolist()\n",
    "        json.dump(outDict, outfile)\n",
    "        outfile.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}