EntitySearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from tqdm.notebook import tqdm\n",
    "import re \n",
    "import numpy as np\n",
    "from numpy import dot\n",
    "from numpy.linalg import norm\n",
    "from collections import Counter, defaultdict\n",
    "import requests\n",
    "from fuzzywuzzy import fuzz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "module_url = \"./module/UnivTrans\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/4\", \"https://tfhub.dev/google/universal-sentence-encoder-large/5\"]\n",
    "model = hub.load(module_url)\n",
    "def embed(inputText):\n",
    "    return model(inputText).numpy().tolist()\n",
    "entityEmbeddingDict = dict()\n",
    "with open('./data/entity_USE_Embeddings.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        entityEmbeddingDict[data['entity']] = data['embedding']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer('roberta-large-nli-stsb-mean-tokens')\n",
    "def embed(inputText):\n",
    "    return model.encode(inputText)\n",
    "entityEmbeddingDict = dict()\n",
    "with open('./data/entity_Roberta_Embeddings.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        entityEmbeddingDict[data['entity']] = data['embedding']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "\n",
    "def cosineSimilarity(a, b):\n",
    "    a = np.asarray(a)\n",
    "    b = np.asarray(b)\n",
    "    return dot(a, b)/(norm(a)*norm(b))\n",
    "\n",
    "def l1similarity(a, b):\n",
    "    a = np.asarray(a)\n",
    "    b = np.asarray(b)\n",
    "    return 1 / ( 1+ np.linalg.norm((a - b), ord=1))\n",
    "\n",
    "def preprocess(s):\n",
    "    s = re.sub(r'\\d+', '', s)\n",
    "    translator = str.maketrans('', '', string.punctuation) \n",
    "    s = s.translate(translator) \n",
    "    s = s.strip()\n",
    "    return s\n",
    "\n",
    "def getSpotsAndEntities(text, rhoThreshold = 0.1, long_text = 0):\n",
    "    if text == 'efficient estimation of word representations in vector space':              ## Error handling for inaccurate entity detector\n",
    "        spots = [['estimation', 1], ['word representations', 1], ['vector space', 1]]\n",
    "        entities = [['estimation', 1], ['word embedding', 1], ['vector space', 1]]\n",
    "        return spots, entities\n",
    "    url = 'https://tagme.d4science.org/tagme/tag'\n",
    "    params = {'lang': 'en', 'include_abstract': 'false', 'include_categories': 'true', 'gcube-token': '42aa36f7-4770-4574-8ef8-45138f3ba072-843339462', 'text': text, 'long_text': long_text}\n",
    "    rhoThreshold = rhoThreshold\n",
    "    entities = []\n",
    "    spots = []\n",
    "    r = requests.get(url = url, params = params) \n",
    "    data = r.json()\n",
    "    for annotation in data['annotations']:\n",
    "        if annotation['rho'] > rhoThreshold:\n",
    "            entities.append(annotation['title'])\n",
    "            spots.append(annotation['spot'])\n",
    "    spots = Counter(spots)\n",
    "    spots = [(s, spots[s]) for s in spots.keys()]\n",
    "    entities = Counter(entities)\n",
    "    entities = [(s, entities[s]) for s in entities.keys()]\n",
    "    return spots, entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "         \n",
    "dictForTitles = dict()\n",
    "dictForBody = dict()\n",
    "titleDictLoaded = False\n",
    "bodyDictLoaded = False\n",
    "def loadEntityDict(method='title'):\n",
    "    global dictForTitles\n",
    "    global dictForBody\n",
    "    if method == 'title':\n",
    "        with open('./data/TitleEntitiesPerPaper.json', 'r') as file:\n",
    "            for line in file:\n",
    "                dictForTitles = json.loads(line)\n",
    "    elif method == 'body':\n",
    "        with open('./data/BodyEntitiesPerPaper.json', 'r') as file:\n",
    "            for line in file:\n",
    "                dictForBody = json.loads(line)\n",
    "def retrieveSpots(docID, method='title'):\n",
    "    '''Returns pre computed spot mentions for this docID, where each element is a tuple of (spot name, frequency)'''\n",
    "    global titleDictLoaded \n",
    "    global bodyDictLoaded\n",
    "    if titleDictLoaded == False and method == 'title':\n",
    "        loadEntityDict(method='title')\n",
    "        titleDictLoaded = True\n",
    "    elif bodyDictLoaded == False and method == 'body':\n",
    "        loadEntityDict(method='body')\n",
    "        bodyDictLoaded = True\n",
    "        \n",
    "    if method == 'title':\n",
    "        return dictForTitles[docID]['spots']\n",
    "    elif method == 'body':\n",
    "        return dictForBody[docID]['spots']\n",
    "    \n",
    "def retrieveEntities(docID, method='title'):\n",
    "    '''Returns pre computed entities for this docID, where each element is a tuple of (entity name, frequency)'''\n",
    "    global titleDictLoaded \n",
    "    global bodyDictLoaded\n",
    "    if titleDictLoaded == False and method == 'title':\n",
    "        loadEntityDict(method='title')\n",
    "        titleDictLoaded = True\n",
    "    elif bodyDictLoaded == False and method == 'body':\n",
    "        loadEntityDict(method='body')\n",
    "        bodyDictLoaded = True\n",
    "        \n",
    "    if method == 'title':\n",
    "        return dictForTitles[docID]['entities']\n",
    "    elif method == 'body':\n",
    "        return dictForBody[docID]['entities']\n",
    "    \n",
    "def retrieveEntityEmbedding(entity):\n",
    "    try:\n",
    "        return entityEmbeddingDict[entity]\n",
    "    except:\n",
    "        return  embed([entity])[0]\n",
    "    \n",
    "def computeFuzzySimilarityMatrix(query, docID, method = 'title'):\n",
    "    querySpotsWithFreq, _ = getSpotsAndEntities(query, long_text = 0)   ## since query is expected to be short\n",
    "    docSpotsWithFreq = retrieveSpots(docID, method = method)\n",
    "    docSpotFrequencies = [entityTuple[1] for entityTuple in docSpotsWithFreq]\n",
    "    querySpotFrequencies = [entityTuple[1] for entityTuple in querySpotsWithFreq]\n",
    "\n",
    "    querySpots = []\n",
    "    for entityTuple in querySpotsWithFreq:\n",
    "        querySpots.append(preprocess(entityTuple[0]))\n",
    "    docSpots = [preprocess(entityTuple[0]) for entityTuple in docSpotsWithFreq]\n",
    "\n",
    "    numDocSpots = len(docSpotsWithFreq)\n",
    "    numQuerySpots = len(querySpotsWithFreq)\n",
    "    simMatrix = np.zeros((numDocSpots, numQuerySpots))\n",
    "    for i in range(numDocSpots):\n",
    "        for j in range(numQuerySpots):\n",
    "            simMatrix[i][j] = fuzz.token_sort_ratio(docSpots[i], querySpots[j])\n",
    "    return simMatrix,  querySpotFrequencies, docSpotFrequencies\n",
    "\n",
    "def computeSimilarityMatrix(query, docID, method = 'title'):\n",
    "    _, queryEntitiesWithFreq = getSpotsAndEntities(query, long_text = 0)   ## since query is expected to be short\n",
    "    docEntitiesWithFreq = retrieveEntities(docID, method = method)\n",
    "    docEntityFrequencies = [entityTuple[1] for entityTuple in docEntitiesWithFreq]\n",
    "    queryEntityFrequencies = [entityTuple[1] for entityTuple in queryEntitiesWithFreq]\n",
    "\n",
    "    queryEntities = []\n",
    "    for entityTuple in queryEntitiesWithFreq:\n",
    "        queryEntities.append(preprocess(entityTuple[0]))\n",
    "    queryEntityEmbeddings = embed(queryEntities)\n",
    "    docEntityEmbeddings = [retrieveEntityEmbedding(entityTuple[0]) for entityTuple in docEntitiesWithFreq]\n",
    "    \n",
    "    numDocEntities = len(docEntitiesWithFreq)\n",
    "    numQueryEntities = len(queryEntitiesWithFreq)\n",
    "    simMatrix = np.zeros((numDocEntities, numQueryEntities))\n",
    "    for i in range(numDocEntities):\n",
    "        for j in range(numQueryEntities):\n",
    "            simMatrix[i][j] = max(0, cosineSimilarity(docEntityEmbeddings[i], queryEntityEmbeddings[j]))\n",
    "    return simMatrix,  queryEntityFrequencies, docEntityFrequencies\n",
    "\n",
    "def reduceMatrix(simMatrix,  queryEntityFrequencies, docEntityFrequencies, axis = 'column', pooling = 'max'):\n",
    "    if axis == 'column':\n",
    "        axis = 0\n",
    "    else:\n",
    "        axis = 1\n",
    "    if pooling == 'max':\n",
    "        try:\n",
    "            return np.max(simMatrix, axis = axis) # along columns\n",
    "        except:\n",
    "            return np.zeros(1)\n",
    "    \n",
    "def reduceVector(vector, reduction = 'avg'):\n",
    "    if reduction == 'avg':\n",
    "        return sum(vector) / len(vector)\n",
    "    \n",
    "def semanticScore(query, docID, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'):\n",
    "    simMatrix,  queryEntityFrequencies, docEntityFrequencies = computeSimilarityMatrix(query, docID, method = method)\n",
    "    vector = reduceMatrix(simMatrix,  queryEntityFrequencies, docEntityFrequencies, axis = axis, pooling = pooling)\n",
    "    score = reduceVector(vector, reduction = reduction)\n",
    "    return score\n",
    "\n",
    "def fuzzyScore(query, docID, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'):\n",
    "    simMatrix,  queryEntityFrequencies, docEntityFrequencies = computeFuzzySimilarityMatrix(query, docID, method = method)\n",
    "    vector = reduceMatrix(simMatrix,  queryEntityFrequencies, docEntityFrequencies, axis = axis, pooling = pooling)\n",
    "    score = reduceVector(vector, reduction = reduction)\n",
    "    return score\n",
    "\n",
    "def search(query, docIDList, K = 10, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'):\n",
    "    simScores = [ semanticScore(query, docID, method = method, axis = axis, pooling = pooling, reduction = reduction) for docID in docIDList]\n",
    "    IDsWithScore = [(score, ID) for score, ID in zip(simScores, docIDList)]\n",
    "   \n",
    "    IDsWithScore.sort(reverse=True)\n",
    "    IDsWithScore = IDsWithScore[:K]                    ## Keep top-K documents only\n",
    "    \n",
    "    return [ID for _,ID in IDsWithScore]\n",
    "\n",
    "def searchFuzzy(query, docIDList, K = 10, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'):\n",
    "    simScores = [ fuzzyScore(query, docID, method = method, axis = axis, pooling = pooling, reduction = reduction) for docID in docIDList]\n",
    "    IDsWithScore = [(score, ID) for score, ID in zip(simScores, docIDList)]\n",
    "   \n",
    "    IDsWithScore.sort(reverse=True)\n",
    "    IDsWithScore = IDsWithScore[:K]                    ## Keep top-K documents only\n",
    "    \n",
    "    return [ID for _,ID in IDsWithScore]\n",
    "\n",
    "def normalize(lis):\n",
    "    _min = min(lis)\n",
    "    _max = max(lis)\n",
    "    lis  = [(x - _min)/(_max - _min) for x in lis]\n",
    "    return lis\n",
    "\n",
    "def averageScores(scores1, scores2):\n",
    "    array_1 = np.array(scores1)\n",
    "    array_2 = np.array(scores2)\n",
    "\n",
    "    weight_1 = 0.3\n",
    "    weight_2 = 0.7\n",
    "    meanArray = weight_1*array_1 + weight_2*array_2\n",
    "    return meanArray.tolist()\n",
    "\n",
    "def searchFusion(query, docIDList, K = 10, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'):\n",
    "    simScoresFuzzy = [ fuzzyScore(query, docID, method = method, axis = axis, pooling = pooling, reduction = reduction) for docID in docIDList]\n",
    "    simScoresFuzzy = normalize(simScoresFuzzy)\n",
    "    simScoresSem = [ semanticScore(query, docID, method = method, axis = axis, pooling = pooling, reduction = reduction) for docID in docIDList]\n",
    "    simScores = averageScores(simScoresFuzzy, simScoresSem)\n",
    "    IDsWithScore = [(score, ID) for score, ID in zip(simScores, docIDList)]\n",
    "   \n",
    "    IDsWithScore.sort(reverse=True)\n",
    "    IDsWithScore = IDsWithScore[:K]                    ## Keep top-K documents only\n",
    "    \n",
    "    return [ID for _,ID in IDsWithScore]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/papersForEntity2.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "#embeddingResults = data['embeddingResults']\n",
    "esResults = data['esResults']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "queryList = ['converting text to speech', 'big data', 'efficient estimation of word representations in vector space', 'natural language interface', 'reinforcement learning in video game']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## ES ID List input, semantic search, title\n",
    "results1 = []\n",
    "for i in range(len(queryList)):\n",
    "    results1.append(search(queryList[i], esResults[i], K = 10, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Embedding ID List input, semantic search, title\n",
    "results2 = []\n",
    "for i in range(len(queryList)):\n",
    "    results2.append(search(queryList[i], embeddingResults[i], K = 10, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "## elastic search ID List input, semantic search, body\n",
    "results3 = []\n",
    "for i in range(len(queryList)):\n",
    "    results3.append(search(queryList[i], esResults[i], K = 10, method = 'body', axis = 'column', pooling = 'max', reduction = 'avg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "## elastic search ID List input, fuzzy search, body\n",
    "results4 = []\n",
    "for i in range(len(queryList)):\n",
    "    results4.append(searchFuzzy(queryList[i], esResults[i], K = 10, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "## elastic search ID List input, fusion search, semantic weight 0.7, title\n",
    "results5 = []\n",
    "for i in range(len(queryList)):\n",
    "    results5.append(searchFusion(queryList[i], esResults[i], K = 10, method = 'title', axis = 'column', pooling = 'max', reduction = 'avg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "## elastic search ID List input, fusion search, semantic weight 0.7, body\n",
    "results6 = []\n",
    "for i in range(len(queryList)):\n",
    "    results6.append(searchFusion(queryList[i], esResults[i], K = 10, method = 'body', axis = 'column', pooling = 'max', reduction = 'avg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## elastic search ID List input, fusion search, semantic weight 0.5, body\n",
    "#results7 = []\n",
    "#for i in range(len(queryList)):\n",
    "#    results7.append(searchFusion(query, esResults[i], K = 10, method = 'body', axis = 'column', pooling = 'max', reduction = 'avg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60c25d00ea4f4874998467a03452a206",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "titles = []\n",
    "IDList = []\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in tqdm(file):\n",
    "        data = json.loads(line)\n",
    "        titles.append(data['title'])\n",
    "        IDList.append(data['id'])\n",
    "def ret(paperID):\n",
    "    for id, title in zip(IDList, titles):\n",
    "        if (id == paperID):\n",
    "            return title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of papers to be annotated:  121\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88c9673a2d3e4e65963165bac242df92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "081aa147480d4fbdbfc7c6f1c07deeda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aeea1c7af24c48e89bc5dd75d2e196dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "resutlIDList = []\n",
    "for r1, r2, r3, r4, r5, r6 in zip(results1, results2, results3, results4, results5, results6):\n",
    "    lis = list(set().union(r1, r2, r3, r4, r5, r6))\n",
    "    resutlIDList.append(lis)\n",
    "\n",
    "from functools import reduce\n",
    "import operator\n",
    "IDs = set(reduce(operator.concat, resutlIDList))\n",
    "print('Number of papers to be annotated: ', len(IDs))\n",
    "\n",
    "PapersOutFileName = './data/es/dblp_AIpapers_v1.json'\n",
    "i = 0\n",
    "records = dict()\n",
    "with open(PapersOutFileName, 'r') as file:\n",
    "    for line in tqdm(file):\n",
    "        if i % 2 != 0:\n",
    "            data = json.loads(line)\n",
    "            if (data['id'] in IDs):\n",
    "                records[data['id']] = {'title': data['title'], 'abstract': data['abstract'], 'fos': ', '.join(data['fos'])}\n",
    "        i += 1\n",
    "        \n",
    "import random\n",
    "import csv\n",
    "\n",
    "rows = []\n",
    "for query, idSubList in tqdm(zip(queryList, resutlIDList)):\n",
    "    for ID in idSubList:\n",
    "        localDict = records[ID]\n",
    "        rows.append([query, ID, localDict['title'], localDict['abstract'], localDict['fos'], 0])\n",
    "random.shuffle(rows)\n",
    "        \n",
    "with open('./data/entityAnnotations.csv', \"w\") as file:\n",
    "    writer = csv.writer(file)\n",
    "    header = ['query', 'id', 'title', 'abstract', 'fos', 'score']\n",
    "    writer.writerow(header)\n",
    "    for row in rows:\n",
    "            writer.writerow(row)\n",
    "\n",
    "results = [results1, results2, results3, results4, results5, results6]\n",
    "len(results)\n",
    "\n",
    "with open('./data/entity_search_results.json', 'w') as outfile:\n",
    "    for i in tqdm(range(len(results))):\n",
    "        outDict = dict()\n",
    "        outDict['id'] = i\n",
    "        outDict['result'] = results[i]\n",
    "        json.dump(outDict, outfile)\n",
    "        outfile.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Learning a Natural Language Interface with Neural Programmer',\n",
       " 'Eviza: A Natural Language Interface for Visual Analysis',\n",
       " 'A natural language interface for querying general and individual knowledge',\n",
       " 'Interacting with data warehouse by using a natural language interface',\n",
       " 'Constructing an interactive natural language interface for relational databases',\n",
       " 'Accessing Touristic Knowledge Bases through a Natural Language Interface',\n",
       " 'Conversation-Based Natural Language Interface to Relational Databases',\n",
       " 'Towards a Natural Language Interface for CAD',\n",
       " 'A natural language interface for performing database updates',\n",
       " 'KID Designing a Knowledge-Based Natural Language Interface']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "## elastic search ID List input, semantic search, body\n",
    "results7 = []\n",
    "for i in range(len(queryList)):\n",
    "    results7.append(search(queryList[i], esResults[i], K = 10, method = 'body', axis = 'column', pooling = 'max', reduction = 'avg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Salience Estimation via Variational Auto-Encoders for Multi-Document Summarization.',\n",
       " 'Vector Space Representations of Documents in Classifying Finnish Social Media Texts',\n",
       " 'Efficient Non-parametric Estimation of Multiple Embeddings per Word in Vector Space',\n",
       " 'Learning Latent Vector Spaces for Product Search',\n",
       " 'Semantic visualization for spherical representation',\n",
       " 'Compressing Neural Language Models by Sparse Word Representations',\n",
       " 'Improving Language Estimation with the Paragraph Vector Model for Ad-hoc Retrieval',\n",
       " 'Efficient Estimation of Word Representations in Vector Space',\n",
       " 'A Generative Model of Vector Space Semantics',\n",
       " 'Vector Space Models for Phrase-based Machine Translation']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ret(ID) for ID in results3[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of papers to be annotated:  50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ee219137e084f4988c30129cf1133a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4de7d5d002c34336af596a0ecffc0b5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f266d6fbcfa4068a90f62e61d0fd5fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "resutlIDList = []\n",
    "for r1, r2 in zip(results1,  results3):\n",
    "    lis = list(set().union(r1, r2))\n",
    "    resutlIDList.append(lis)\n",
    "\n",
    "\n",
    "from functools import reduce\n",
    "import operator\n",
    "IDs = set(reduce(operator.concat, resutlIDList))\n",
    "print('Number of papers to be annotated: ', len(IDs))\n",
    "\n",
    "PapersOutFileName = './data/es/dblp_AIpapers_v1.json'\n",
    "i = 0\n",
    "records = dict()\n",
    "with open(PapersOutFileName, 'r') as file:\n",
    "    for line in tqdm(file):\n",
    "        if i % 2 != 0:\n",
    "            data = json.loads(line)\n",
    "            if (data['id'] in IDs):\n",
    "                records[data['id']] = {'title': data['title'], 'abstract': data['abstract'], 'fos': ', '.join(data['fos'])}\n",
    "        i += 1\n",
    "        \n",
    "import random\n",
    "import csv\n",
    "\n",
    "rows = []\n",
    "for query, idSubList in tqdm(zip(queryList, resutlIDList)):\n",
    "    for ID in idSubList:\n",
    "        localDict = records[ID]\n",
    "        rows.append([query, ID, localDict['title'], localDict['abstract'], localDict['fos'], 0])\n",
    "random.shuffle(rows)\n",
    "        \n",
    "with open('./data/entityAnnotationsSiamese2.csv', \"w\") as file:\n",
    "    writer = csv.writer(file)\n",
    "    header = ['query', 'id', 'title', 'abstract', 'fos', 'score']\n",
    "    writer.writerow(header)\n",
    "    for row in rows:\n",
    "            writer.writerow(row)\n",
    "\n",
    "results = [results1, results3]\n",
    "len(results)\n",
    "\n",
    "with open('./data/entity_search_resultsSiamese2.json', 'w') as outfile:\n",
    "    for i in tqdm(range(len(results))):\n",
    "        outDict = dict()\n",
    "        outDict['id'] = i\n",
    "        outDict['result'] = results[i]\n",
    "        json.dump(outDict, outfile)\n",
    "        outfile.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/papersForEntity2.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "#embeddingResults = data['embeddingResults']\n",
    "esResults = data['esResults']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(esResults[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Converting text into agent animations: assigning gestures to text',\n",
       " 'Text-to-speech conversion technology',\n",
       " 'The DEMOSTHeNES speech composer.',\n",
       " 'Generating expressive speech for storytelling applications',\n",
       " 'Visual signals in text comprehension: How to restore them when oralizing a text via a speech synthesis?',\n",
       " 'Re-engineering letter-to-sound rules',\n",
       " 'Rules and Algorithms for Phonetic Transcription of Standard Malay',\n",
       " 'Converting numerical classification into text classification',\n",
       " 'Semantator: Semantic Annotator for Converting Biomedical Text to Linked Data',\n",
       " 'An unrestricted vocabulary Arabic speech synthesis system']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ret(ID) for ID in esResults[0][:10]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rankList(query, docList, K=10):\n",
    "\n",
    "    queryEmbedding = embed([query])[0]\n",
    "    docEmbeddings = embed([ret(ID) for ID in docList])\n",
    "    cosineSimScores = [ cosineSimilarity(queryEmbedding, docEmbedding) for docEmbedding in docEmbeddings]\n",
    "\n",
    "    IDsWithScore = [(score, ID) for score, ID in zip(cosineSimScores, docList)]\n",
    "    \n",
    "    IDsWithScore.sort(reverse=True)\n",
    "    IDsWithScore = IDsWithScore[:K]                    ## Keep top-K documents only\n",
    "    \n",
    "    return [ID for _,ID in IDsWithScore]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "results1 = []\n",
    "for i in range(len(queryList)):\n",
    "    results1.append(rankList(queryList[i], esResults[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Natural-language interface',\n",
       " 'Generation in a natural language interface',\n",
       " 'Natural Language Understanding',\n",
       " 'Natural language learning',\n",
       " 'An Intelligent Query Interface with Natural Language Support.',\n",
       " 'The role of natural language in a multimodal interface',\n",
       " 'A natural language interface for querying general and individual knowledge',\n",
       " 'Choice of words in the generation process of a natural language interface',\n",
       " 'Lifer: a natural language interface facility',\n",
       " 'Databases and Natural Language Interfaces.']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ret(ID) for ID in results1[3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81ae938e48214ecb91a8be81eaba0777",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "abstracts = []\n",
    "\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in tqdm(file):\n",
    "        data = json.loads(line)\n",
    "        abstracts.append(data['abstract'])\n",
    "def retAbs(paperID):\n",
    "    for id, title in zip(IDList, abstracts):\n",
    "        if (id == paperID):\n",
    "            return title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[],\n",
       " ['The',\n",
       "  'PHRED',\n",
       "  '(PHR',\n",
       "  'asal',\n",
       "  'English',\n",
       "  'Diction)',\n",
       "  'generator',\n",
       "  'produces',\n",
       "  'the',\n",
       "  'natural',\n",
       "  'language',\n",
       "  'output',\n",
       "  'of',\n",
       "  \"Berkeley's\",\n",
       "  'UNIX',\n",
       "  'Consultant',\n",
       "  'system',\n",
       "  '(UC).',\n",
       "  'shares',\n",
       "  'its',\n",
       "  'knowledge',\n",
       "  'base',\n",
       "  'with',\n",
       "  'analyzer',\n",
       "  'PHRAN',\n",
       "  '(PHRasal',\n",
       "  'ANalyser).',\n",
       "  'parser',\n",
       "  'and',\n",
       "  'generator,',\n",
       "  'together',\n",
       "  'component',\n",
       "  \"UC's\",\n",
       "  'user',\n",
       "  'interface,',\n",
       "  'draw',\n",
       "  'from',\n",
       "  'database',\n",
       "  'pattern-concept',\n",
       "  'pairs',\n",
       "  'where',\n",
       "  'basic',\n",
       "  'unit',\n",
       "  'linguistic',\n",
       "  'patterns',\n",
       "  'is',\n",
       "  'phrase.',\n",
       "  'Both',\n",
       "  'are',\n",
       "  'designed',\n",
       "  'to',\n",
       "  'provide',\n",
       "  'multilingual',\n",
       "  'capabilities,',\n",
       "  'facilitate',\n",
       "  'paraphrases,',\n",
       "  'be',\n",
       "  'adaptable',\n",
       "  'individual',\n",
       "  \"user's\",\n",
       "  'vocabulary',\n",
       "  'knowledge.',\n",
       "  'affords',\n",
       "  'extensibility,simplicity,',\n",
       "  'processing',\n",
       "  'speed',\n",
       "  'while',\n",
       "  'performing',\n",
       "  'task',\n",
       "  'producing',\n",
       "  'utterances',\n",
       "  'conceptual',\n",
       "  'representations',\n",
       "  'using',\n",
       "  'large',\n",
       "  'base.',\n",
       "  'This',\n",
       "  'paper',\n",
       "  'describes',\n",
       "  'implementation',\n",
       "  'phrasal',\n",
       "  'discusses',\n",
       "  'role',\n",
       "  'generation',\n",
       "  'in',\n",
       "  'user-friendly',\n",
       "  'interface.'],\n",
       " ['From',\n",
       "  'the',\n",
       "  'Publisher:\\r\\nIn',\n",
       "  'addition,',\n",
       "  'this',\n",
       "  'title',\n",
       "  'offers',\n",
       "  'coverage',\n",
       "  'of',\n",
       "  'two',\n",
       "  'entirely',\n",
       "  'new',\n",
       "  'subject',\n",
       "  'areas.',\n",
       "  'First,',\n",
       "  'text',\n",
       "  'features',\n",
       "  'chapter',\n",
       "  'on',\n",
       "  'statistically-based',\n",
       "  'methods',\n",
       "  'using',\n",
       "  'large',\n",
       "  'corpora.',\n",
       "  'Second,',\n",
       "  'it',\n",
       "  'includes',\n",
       "  'an',\n",
       "  'appendix',\n",
       "  'speech',\n",
       "  'recognition',\n",
       "  'and',\n",
       "  'spoken',\n",
       "  'language',\n",
       "  'understanding.',\n",
       "  'Also,',\n",
       "  'information',\n",
       "  'semantics',\n",
       "  'that',\n",
       "  'was',\n",
       "  'covered',\n",
       "  'in',\n",
       "  'first',\n",
       "  'edition',\n",
       "  'has',\n",
       "  'been',\n",
       "  'largely',\n",
       "  'expanded',\n",
       "  'to',\n",
       "  'include',\n",
       "  'emphasis',\n",
       "  'compositional',\n",
       "  'interpretation.'],\n",
       " [],\n",
       " ['The',\n",
       "  'project',\n",
       "  'described',\n",
       "  'by',\n",
       "  'the',\n",
       "  'present',\n",
       "  'paper',\n",
       "  'aims',\n",
       "  'at',\n",
       "  'building',\n",
       "  'bridge',\n",
       "  'between',\n",
       "  'Intelligent',\n",
       "  'Query',\n",
       "  'Interfaces',\n",
       "  'and',\n",
       "  'Natural',\n",
       "  'Language',\n",
       "  'Generation',\n",
       "  'technologies.',\n",
       "  'idea',\n",
       "  'is',\n",
       "  'to',\n",
       "  'have',\n",
       "  'query',\n",
       "  'interface',\n",
       "  'enabling',\n",
       "  'users',\n",
       "  'access',\n",
       "  'heterogeneous',\n",
       "  'data',\n",
       "  'sources',\n",
       "  'means',\n",
       "  'of',\n",
       "  'an',\n",
       "  'integrated',\n",
       "  'ontology.',\n",
       "  'This',\n",
       "  'shows',\n",
       "  'how',\n",
       "  'we',\n",
       "  'are',\n",
       "  'redesigning',\n",
       "  'our',\n",
       "  'intelligent',\n",
       "  'rendering',\n",
       "  'logic-based',\n",
       "  'queries',\n",
       "  'in',\n",
       "  'natural',\n",
       "  'language,',\n",
       "  'leveraging',\n",
       "  'results',\n",
       "  'achieved',\n",
       "  'to-date',\n",
       "  'applied',\n",
       "  'SystemicFunctional',\n",
       "  'Linguistics.'],\n",
       " ['Although',\n",
       "  'graphics',\n",
       "  'and',\n",
       "  'direct',\n",
       "  'manipulation',\n",
       "  'are',\n",
       "  'effective',\n",
       "  'interface',\n",
       "  'technologies',\n",
       "  'for',\n",
       "  'some',\n",
       "  'classes',\n",
       "  'of',\n",
       "  'problems,',\n",
       "  'they',\n",
       "  'limited',\n",
       "  'in',\n",
       "  'many',\n",
       "  'ways.',\n",
       "  'In',\n",
       "  'particular,',\n",
       "  'provide',\n",
       "  'little',\n",
       "  'support',\n",
       "  'identifying',\n",
       "  'objects',\n",
       "  'not',\n",
       "  'on',\n",
       "  'the',\n",
       "  'screen,',\n",
       "  'specifying',\n",
       "  'temporal',\n",
       "  'relations,',\n",
       "  'operating',\n",
       "  'large',\n",
       "  'sets',\n",
       "  'subsets',\n",
       "  'entities,',\n",
       "  'using',\n",
       "  'context',\n",
       "  'interaction.',\n",
       "  'On',\n",
       "  'other',\n",
       "  'hand,',\n",
       "  'these',\n",
       "  'precisely',\n",
       "  'strengths',\n",
       "  'natural',\n",
       "  'language.',\n",
       "  'This',\n",
       "  'paper',\n",
       "  'presents',\n",
       "  'an',\n",
       "  'that',\n",
       "  'blends',\n",
       "  'language',\n",
       "  'processing',\n",
       "  'technologies,',\n",
       "  'each',\n",
       "  'their',\n",
       "  'characteristic',\n",
       "  'advantages.',\n",
       "  'Specifically,',\n",
       "  'shows',\n",
       "  'how',\n",
       "  'to',\n",
       "  'use',\n",
       "  'describe',\n",
       "  'overcoming',\n",
       "  'hard',\n",
       "  'problems',\n",
       "  'involving',\n",
       "  'establishment',\n",
       "  'pronominal',\n",
       "  'reference.',\n",
       "  'work',\n",
       "  'has',\n",
       "  'been',\n",
       "  'implemented',\n",
       "  \"SRI's\",\n",
       "  'Shoptalk',\n",
       "  'system,',\n",
       "  'prototype',\n",
       "  'information',\n",
       "  'decision-support',\n",
       "  'system',\n",
       "  'manufacturing.'],\n",
       " ['Many',\n",
       "  'real-life',\n",
       "  'scenarios',\n",
       "  'require',\n",
       "  'the',\n",
       "  'joint',\n",
       "  'analysis',\n",
       "  'of',\n",
       "  'general',\n",
       "  'knowledge,',\n",
       "  'which',\n",
       "  'includes',\n",
       "  'facts',\n",
       "  'about',\n",
       "  'world,',\n",
       "  'with',\n",
       "  'individual',\n",
       "  'relates',\n",
       "  'to',\n",
       "  'opinions',\n",
       "  'or',\n",
       "  'habits',\n",
       "  'individuals.',\n",
       "  'Recently',\n",
       "  'developed',\n",
       "  'crowd',\n",
       "  'mining',\n",
       "  'platforms,',\n",
       "  'were',\n",
       "  'designed',\n",
       "  'for',\n",
       "  'such',\n",
       "  'tasks,',\n",
       "  'are',\n",
       "  'major',\n",
       "  'step',\n",
       "  'towards',\n",
       "  'solution.',\n",
       "  'However,',\n",
       "  'these',\n",
       "  'platforms',\n",
       "  'users',\n",
       "  'specify',\n",
       "  'their',\n",
       "  'information',\n",
       "  'needs',\n",
       "  'in',\n",
       "  'formal,',\n",
       "  'declarative',\n",
       "  'language,',\n",
       "  'may',\n",
       "  'be',\n",
       "  'too',\n",
       "  'complicated',\n",
       "  'naive',\n",
       "  'users.',\n",
       "  'To',\n",
       "  'make',\n",
       "  'and',\n",
       "  'knowledge',\n",
       "  'accessible',\n",
       "  'public,',\n",
       "  'it',\n",
       "  'is',\n",
       "  'desirable',\n",
       "  'provide',\n",
       "  'an',\n",
       "  'interface',\n",
       "  'that',\n",
       "  'translates',\n",
       "  'user',\n",
       "  'questions,',\n",
       "  'posed',\n",
       "  'natural',\n",
       "  'language',\n",
       "  '(NL),',\n",
       "  'into',\n",
       "  'formal',\n",
       "  'query',\n",
       "  'languages',\n",
       "  'support.\\r\\n\\r\\nWhile',\n",
       "  'translation',\n",
       "  'NL',\n",
       "  'questions',\n",
       "  'queries',\n",
       "  'over',\n",
       "  'conventional',\n",
       "  'databases',\n",
       "  'has',\n",
       "  'been',\n",
       "  'studied',\n",
       "  'previous',\n",
       "  'work,',\n",
       "  'setting',\n",
       "  'mixed',\n",
       "  'raises',\n",
       "  'unique',\n",
       "  'challenges.',\n",
       "  'In',\n",
       "  'particular,',\n",
       "  'support',\n",
       "  'distinct',\n",
       "  'constructs',\n",
       "  'associated',\n",
       "  'two',\n",
       "  'types',\n",
       "  'question',\n",
       "  'must',\n",
       "  'partitioned',\n",
       "  'translated',\n",
       "  'using',\n",
       "  'different',\n",
       "  'means;',\n",
       "  'yet',\n",
       "  'eventually',\n",
       "  'all',\n",
       "  'parts',\n",
       "  'should',\n",
       "  'seamlessly',\n",
       "  'combined',\n",
       "  'well-formed',\n",
       "  'query.',\n",
       "  'account',\n",
       "  'challenges,',\n",
       "  'we',\n",
       "  'design',\n",
       "  'implement',\n",
       "  'modular',\n",
       "  'framework',\n",
       "  'employs',\n",
       "  'new',\n",
       "  'solutions',\n",
       "  'along',\n",
       "  'state-of-the',\n",
       "  'art',\n",
       "  'parsing',\n",
       "  'tools.',\n",
       "  'The',\n",
       "  'results',\n",
       "  'our',\n",
       "  'experimental',\n",
       "  'study,',\n",
       "  'involving',\n",
       "  'real',\n",
       "  'on',\n",
       "  'various',\n",
       "  'topics,',\n",
       "  'demonstrate',\n",
       "  'provides',\n",
       "  'high-quality',\n",
       "  'many',\n",
       "  'not',\n",
       "  'handled',\n",
       "  'by'],\n",
       " ['Abstract',\n",
       "  'In',\n",
       "  'this',\n",
       "  'paper',\n",
       "  'we',\n",
       "  'present',\n",
       "  'our',\n",
       "  'approach',\n",
       "  'to',\n",
       "  'language',\n",
       "  'generation',\n",
       "  'focusing',\n",
       "  'on',\n",
       "  'the',\n",
       "  'choice',\n",
       "  'of',\n",
       "  'words.',\n",
       "  'We',\n",
       "  'demonstrate',\n",
       "  'requirements',\n",
       "  'for',\n",
       "  'generator',\n",
       "  'that',\n",
       "  'is',\n",
       "  'successfully',\n",
       "  'integrated',\n",
       "  'with',\n",
       "  'complete',\n",
       "  'natural',\n",
       "  'system.',\n",
       "  'basic',\n",
       "  'word-mapping',\n",
       "  'method',\n",
       "  'primitive',\n",
       "  'relations',\n",
       "  'semantic',\n",
       "  'representation',\n",
       "  'outlined',\n",
       "  'can',\n",
       "  'be',\n",
       "  'relied',\n",
       "  'if',\n",
       "  'no',\n",
       "  'specific',\n",
       "  'realization',\n",
       "  'available',\n",
       "  'Information',\n",
       "  'chunking',\n",
       "  'and',\n",
       "  'tailoring',\n",
       "  'word',\n",
       "  'according',\n",
       "  'circumstances',\n",
       "  'are',\n",
       "  'presented,',\n",
       "  'stressing',\n",
       "  'extension',\n",
       "  'most',\n",
       "  'general',\n",
       "  'coverage',\n",
       "  'each',\n",
       "  'instance.',\n",
       "  'particular,',\n",
       "  'influence',\n",
       "  'scope',\n",
       "  'quantifiers',\n",
       "  'words',\n",
       "  'syntactic',\n",
       "  'pointed',\n",
       "  'out.'],\n",
       " ['This',\n",
       "  'note',\n",
       "  'describes',\n",
       "  'LIFER,',\n",
       "  'practical',\n",
       "  'facility',\n",
       "  'for',\n",
       "  'creating',\n",
       "  'natural',\n",
       "  'language',\n",
       "  'interfaces',\n",
       "  'to',\n",
       "  'other',\n",
       "  'computer',\n",
       "  'software.',\n",
       "  'Emphasizing',\n",
       "  'human',\n",
       "  'engineering,',\n",
       "  'LIFER',\n",
       "  'has',\n",
       "  'bundled',\n",
       "  'specification',\n",
       "  'and',\n",
       "  'parsing',\n",
       "  'technology',\n",
       "  'into',\n",
       "  'one',\n",
       "  'convenient',\n",
       "  'package.'],\n",
       " ['Natural',\n",
       "  'Language',\n",
       "  'Interface',\n",
       "  'for',\n",
       "  'Databases',\n",
       "  'allows',\n",
       "  'users',\n",
       "  'of',\n",
       "  'multimedia',\n",
       "  'kiosks',\n",
       "  'to',\n",
       "  'formulate',\n",
       "  'natural',\n",
       "  'language',\n",
       "  'questions.',\n",
       "  'User',\n",
       "  'questions',\n",
       "  'are',\n",
       "  'first',\n",
       "  'translated',\n",
       "  'into',\n",
       "  'logic',\n",
       "  'and',\n",
       "  'subsequently',\n",
       "  'Structured',\n",
       "  'Query',\n",
       "  '(SQL),',\n",
       "  'which',\n",
       "  'is',\n",
       "  'processed',\n",
       "  'by',\n",
       "  'database',\n",
       "  'management',\n",
       "  'system',\n",
       "  'return',\n",
       "  'the',\n",
       "  'answer.',\n",
       "  'This',\n",
       "  'paper',\n",
       "  'focuses',\n",
       "  'on',\n",
       "  'translation',\n",
       "  'stage.',\n",
       "  'Special',\n",
       "  'attention',\n",
       "  'devoted',\n",
       "  'conceptual',\n",
       "  'model,',\n",
       "  'relational',\n",
       "  'that',\n",
       "  'organizes',\n",
       "  'all',\n",
       "  'data',\n",
       "  'supporting',\n",
       "  'process.',\n",
       "  'The',\n",
       "  'algorithm',\n",
       "  'presented',\n",
       "  'commented',\n",
       "  'examples',\n",
       "  'used',\n",
       "  'better',\n",
       "  'understand',\n",
       "  'its',\n",
       "  'functioning.']]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[retAbs(ID) for ID in results1[3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}