1_generate_publications_list.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract publication information from the AK API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Microsoft's Academic Knowledge (AK) API provides a free database of academic publications. In a previous step we colected a list of IDs under which the Turing fellows are stored in the AK database (turing_AK_IDs.csv). Here we use the collected IDs to extract up to 50 articles published by each fellow in 2012 or later.  \n",
    "\n",
    "To use the AK database, an access key needs to be obtained. Here we load it from api_key.txt.  \n",
    "\n",
    "Results were stored as publications_eng.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0: Functions and packages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import http.client, urllib.parse, json, re, unicodedata, string\n",
    "import pandas as pd\n",
    "from urllib.request import urlopen\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data\n",
    "(i) read key for AK API from text file  \n",
    "(ii) load information about AK API ID for each fellow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#extract access key to academic knowledge api\n",
    "with open('api_key.txt', 'r') as myfile:\n",
    "    key = myfile.read().replace('\\n', '')\n",
    "\n",
    "ids_list = pd.read_csv('data_files/turing_AK_IDs.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### General functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#function for removing accents from strings\n",
    "def strip_accents(s):\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "def clean_text(text):\n",
    "    text = text.replace(\",\", \"\")\n",
    "    text = strip_accents(text)\n",
    "    return text\n",
    "\n",
    "def save_to_file(my_list, file_name):\n",
    "    \"\"\"\n",
    "    function for saving data to file\n",
    "    input is list which is saved as one row in file called file_name\n",
    "    \"\"\"\n",
    "    f = open('data_files/' + file_name, 'a') \n",
    "    for i in my_list:\n",
    "        f.write(str(i) + ',')\n",
    "    f.write('\\n') # write a line ending\n",
    "    f.close() #close and \"save\" the output file\n",
    "\n",
    "def save_info(data, file_name):\n",
    "    \"\"\"\n",
    "    save data (information) returned by the id_to_paper function (list of dictionaries)   \n",
    "    file_name = where want to save extracted info\n",
    "    \"\"\" \n",
    "    for i in data:\n",
    "        article = []\n",
    "        \n",
    "        for j in i.keys():\n",
    "            if j != 'keywords' and j != 'urls':\n",
    "                text = clean_text(str(i[j]))\n",
    "                article.append(text)\n",
    "            else:\n",
    "                article.extend(i[j])\n",
    "\n",
    "        #save all article related information\n",
    "        save_to_file(article, file_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper functions for extracting data from AK API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_abstract(IA):\n",
    "    \"\"\"\n",
    "    Function for creating abstract out of abstract indexes provided by academic knowledge API\n",
    "    IA is a dictionary with keywords IndexLength and InvertedIndex, IA.IndexLength = length of abstract\n",
    "    IA.InvertedIndex is a dictionary where each word is a key and it provides an associated list of indexes\n",
    "    of where in the abstract it appears\n",
    "    returns abstract as string\n",
    "    \"\"\"\n",
    "    #create list of words of length n where n = length of abstract\n",
    "    abstract = [\"word\"] * IA['IndexLength']\n",
    "\n",
    "    #search through each unique word in the abstract\n",
    "    for word in IA['InvertedIndex'].keys():\n",
    "        \n",
    "        #find positions/indexes in the abstract that this word appears at (can be 1 position or multiple)\n",
    "        for index in IA['InvertedIndex'][word]:\n",
    "        \n",
    "            #place word in correct position in abstract list\n",
    "            abstract[index] = word;\n",
    "\n",
    "    #join all abstract words and return as string\n",
    "    abstract = ' '.join(abstract)\n",
    "    abstract = abstract.rstrip()\n",
    "    abstract = re.sub('[^A-Za-z0-9]+', ' ', abstract)\n",
    "    \n",
    "    return abstract\n",
    "\n",
    "def get_parameters(author_id, year, count):\n",
    "    \n",
    "    expression = \"AND(Composite(AA.AuId=\" + str(author_id) + \"),Y>\" + str(year) + \")\"\n",
    "    att_list = ['Id', 'L', 'AA.AuN', 'AA.AuId', 'AA.AfN','Ti', 'F.FN', 'J.JN', 'E']\n",
    "    attributes = \",\".join(att_list)\n",
    "    \n",
    "    params = urllib.parse.urlencode({\n",
    "        'expr': expression, \n",
    "        'model': 'latest', #unless want to use a different version of the AK API\n",
    "        'count': str(count), #number of results to return\n",
    "        'offset': '0', #index of the first result to retun\n",
    "        'orderby': 'Y:asc', #name of attribute used for sorting  \n",
    "        'attributes': attributes,\n",
    "    })\n",
    "    \n",
    "    return params\n",
    "\n",
    "def get_author_info(authors, author_id):\n",
    "    name, affiliation = \"\", \"\"\n",
    "    for i in authors:\n",
    "        if i['AuId'] == author_id:\n",
    "            name = i['AuN'].replace(\"ł\", \"l\")\n",
    "            if 'AfN' in i.keys():\n",
    "                affiliation = i['AfN']\n",
    "                \n",
    "    return name, affiliation\n",
    "\n",
    "def get_urls(source):\n",
    "    urls = []\n",
    "    for i in source:\n",
    "        urls.append(i['U'])\n",
    "    #make sure the array is always 20 in length\n",
    "    for num in range(20-len(urls)):\n",
    "        urls.append(\"\")\n",
    "        \n",
    "    return urls\n",
    "\n",
    "def get_data(author_id, author_name, data_dict):      \n",
    "    #all information of interest is saved under the key 'entities'\n",
    "    #data_dict['entities'] returns a list of length = number of articles asked for\n",
    "    #within that list, at each index is a dictionary with all the information we want to access about that article\n",
    "    \n",
    "    articles = data_dict[\"entities\"]\n",
    "    final_articles_data = []\n",
    "    \n",
    "    for i in range(len(articles)):\n",
    "        entities = articles[i]\n",
    "        \n",
    "        #make sure paper language is english\n",
    "        if 'L' in entities.keys():\n",
    "            if entities['L'] == 'en':\n",
    "\n",
    "                #author information\n",
    "                if 'AA' in entities.keys():\n",
    "                    name, affiliation = get_author_info(entities['AA'], author_id)\n",
    "\n",
    "                #paper id, title and journal name\n",
    "                article_info = {'Id':\"\", 'Ti':\"\", 'J.JN':\"\"}\n",
    "                for attribute in article_info.keys():\n",
    "                    if attribute in entities.keys():\n",
    "                        article_info[attribute] = entities[attribute]\n",
    "\n",
    "                #fields of study (F) i.e. AK assigned keywords \n",
    "                field_of_study = []\n",
    "                if \"F\" in entities.keys():\n",
    "                    for i in range(len(entities['F'])):\n",
    "                        field_of_study.append(entities[\"F\"][i][\"FN\"])\n",
    "\n",
    "                #entities[\"E\"] returns a string of a dictionary with \"extended attributes\"\n",
    "                #abstract, doi, source URLs\n",
    "                extended = {'IA':\"\", 'DOI':\"\", 'S':[]}    \n",
    "                if \"E\" in entities.keys():\n",
    "                    E = json.loads(entities[\"E\"])\n",
    "                    \n",
    "                    if 'IA' in E.keys():\n",
    "                        extended['IA'] = generate_abstract(E[\"IA\"])\n",
    "\n",
    "                    if 'DOI' in E.keys():\n",
    "                        extended['DOI'] = E['DOI']\n",
    "\n",
    "                    if 'S' in E.keys():\n",
    "                        extended['S'] = get_urls(E['S'])\n",
    "\n",
    "                final_articles_data.append({'author_id':author_id, 'standard_name':author_name, 'ak_name':name, 'affiliation':affiliation,  'paper_id': article_info['Id'], \n",
    "                        'title': article_info['Ti'], 'abstract': extended['IA'], 'journal_name':article_info['J.JN'], 'doi':extended['DOI'], 'urls':extended['S'], \n",
    "                        'keywords':field_of_study})\n",
    "\n",
    "    return final_articles_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for making call to AK API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def id_to_paper(author_id, name, key, year = 2011, count = 20):\n",
    "    \"\"\"\n",
    "    Function for extracting publication/article information\n",
    "    \n",
    "    Need to provide:\n",
    "    - author ID\n",
    "    - cut-off year for publications that interested in (this year is NOT included) i.e. for 2017 put 2016\n",
    "    - count = how many articles want to return\n",
    "    \n",
    "    returns list of dictionaries, one dictionary per article with all info specified in attributes list + supplied info\n",
    "    \"\"\"\n",
    "    \n",
    "    headers = {'Ocp-Apim-Subscription-Key': key}\n",
    "    params = get_parameters(author_id, year, count)\n",
    "\n",
    "    try:\n",
    "        conn = http.client.HTTPSConnection('westus.api.cognitive.microsoft.com')\n",
    "        conn.request(\"GET\", \"/academic/v1.0/evaluate?%s\" % params, \"{body}\", headers)\n",
    "        response = conn.getresponse()\n",
    "        data = response.read()    \n",
    "        \n",
    "        articles_data = []\n",
    "\n",
    "        try:\n",
    "            data_dict=json.loads(data)\n",
    "        except ValueError:\n",
    "            data_dict = {}\n",
    "\n",
    "        #check that something has been returned, if not - ignore\n",
    "        if 'entities' in data_dict.keys():\n",
    "            articles_data = get_data(author_id, name, data_dict)\n",
    "            \n",
    "        conn.close()\n",
    "        \n",
    "        return articles_data\n",
    "    \n",
    "    except Exception as e:\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1: Extract publications information\n",
    "\n",
    "### Set up - file headings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#file where extract publications per author\n",
    "file_name = 'publications_eng.csv'\n",
    "\n",
    "attributes = ['author_id', 'standard_name', 'ak_name','affiliation','paper_id','title','abstract', 'journal_name', 'doi']\n",
    "\n",
    "#anticipate up to 20 URLs and 50 keywords (this is excessive but avoids possible errors - will delete empty columns later)\n",
    "num_urls = 20\n",
    "for i in range(num_urls):\n",
    "    attributes.append('url_' + str(i))\n",
    "    \n",
    "num_keywords = 50\n",
    "for i in range(num_keywords):\n",
    "    attributes.append('keyword_' + str(i))\n",
    "\n",
    "save_to_file(attributes, file_name)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve and save AK data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#number of publications to retrieve\n",
    "num_publications = 50\n",
    "\n",
    "for index, row in ids_list.iterrows():\n",
    "    \n",
    "    #track progress\n",
    "    if index%(int(ids_list.shape[0]/10)) == 0:\n",
    "        print(str(round(index/ids_list.shape[0],2)*100)+\"%\")\n",
    "        \n",
    "    #some fellows have multiple IDs\n",
    "    author_ids = row['id_1':'id_3']\n",
    "    for author_id in author_ids:\n",
    "        if not pd.isnull(author_id):\n",
    "            max_tries = 10\n",
    "            num_tries, found = 0, 0\n",
    "\n",
    "            while num_tries < max_tries and found == 0:\n",
    "                num_tries += 1\n",
    "                data = id_to_paper(int(author_id), row['name'], key,  count = num_publications)\n",
    "                if data != []:\n",
    "                    found = 1\n",
    "\n",
    "            save_info(data, file_name)\n",
    "print('end')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}