3. DBpedia Concepts at Different Confidence Levels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import spotlight\n",
    "from collections import Counter\n",
    "import time\n",
    "\n",
    "input_dir = 'textbooks/'\n",
    "output_dir = 'output/'\n",
    "data_dir = 'data/'\n",
    "metadata_file = 'data/metadata.csv'\n",
    "text_file = 'text.pkl'\n",
    "spotlight_server = 'http://192.168.99.101:2222/rest/annotate'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data and spacy model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "isbns = os.listdir(input_dir)\n",
    "\n",
    "with open(os.path.join(data_dir, text_file), 'rb') as fp:\n",
    "    all_text = pickle.load(fp) \n",
    "    \n",
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load metadata and calculate number of pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISBN</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>subject</th>\n",
       "      <th>imprint</th>\n",
       "      <th>sold_by</th>\n",
       "      <th>start_page</th>\n",
       "      <th>end_page</th>\n",
       "      <th>num_pages</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9781429219617</td>\n",
       "      <td>BIOLOGY OF PLANTS</td>\n",
       "      <td>PETER H RAVEN</td>\n",
       "      <td>BIOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>21</td>\n",
       "      <td>747</td>\n",
       "      <td>863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9781429242301</td>\n",
       "      <td>INTRODUCING PSYCHOLOGY</td>\n",
       "      <td>DANIEL L SCHACTER</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>38</td>\n",
       "      <td>526</td>\n",
       "      <td>616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9781429298643</td>\n",
       "      <td>LIFE: THE SCIENCE OF BIOLOGY</td>\n",
       "      <td>DAVID E SADAVA</td>\n",
       "      <td>BIOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>51</td>\n",
       "      <td>1297</td>\n",
       "      <td>1447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9781429298902</td>\n",
       "      <td>PSYCHOLOGY: A CONCISE INTRODUCTION</td>\n",
       "      <td>RICHARD A GRIGGS</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>WORTH PUBLISHERS</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>22</td>\n",
       "      <td>464</td>\n",
       "      <td>545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9781464126147</td>\n",
       "      <td>MOLECULAR BIOLOGY: PRINCIPLES AND PRACTICE</td>\n",
       "      <td>MICHAEL M COX</td>\n",
       "      <td>BIOLOGY</td>\n",
       "      <td>W. H. FREEMAN</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>30</td>\n",
       "      <td>828</td>\n",
       "      <td>934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9781464135958</td>\n",
       "      <td>WHAT IS LIFE? A GUIDE TO BIOLOGY</td>\n",
       "      <td>JAY PHELAN</td>\n",
       "      <td>BIOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>34</td>\n",
       "      <td>718</td>\n",
       "      <td>773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9781464140815</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>DAVID G MYERS</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>59</td>\n",
       "      <td>751</td>\n",
       "      <td>985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9781464154072</td>\n",
       "      <td>EXPLORING PSYCHOLOGY</td>\n",
       "      <td>DAVID G MYERS</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>WORTH PUBLISHERS</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>59</td>\n",
       "      <td>662</td>\n",
       "      <td>892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9781464171703</td>\n",
       "      <td>ABNORMAL PSYCHOLOGY</td>\n",
       "      <td>RONALD J COMER</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>WORTH PUBLISHERS</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>33</td>\n",
       "      <td>699</td>\n",
       "      <td>852</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            ISBN                                       title  \\\n",
       "0  9781429219617                           BIOLOGY OF PLANTS   \n",
       "1  9781429242301                      INTRODUCING PSYCHOLOGY   \n",
       "2  9781429298643                LIFE: THE SCIENCE OF BIOLOGY   \n",
       "3  9781429298902          PSYCHOLOGY: A CONCISE INTRODUCTION   \n",
       "4  9781464126147  MOLECULAR BIOLOGY: PRINCIPLES AND PRACTICE   \n",
       "5  9781464135958            WHAT IS LIFE? A GUIDE TO BIOLOGY   \n",
       "6  9781464140815                                  PSYCHOLOGY   \n",
       "7  9781464154072                        EXPLORING PSYCHOLOGY   \n",
       "8  9781464171703                         ABNORMAL PSYCHOLOGY   \n",
       "\n",
       "              author     subject           imprint  \\\n",
       "0      PETER H RAVEN     BIOLOGY     FREEMAN/WORTH   \n",
       "1  DANIEL L SCHACTER  PSYCHOLOGY     FREEMAN/WORTH   \n",
       "2     DAVID E SADAVA     BIOLOGY     FREEMAN/WORTH   \n",
       "3   RICHARD A GRIGGS  PSYCHOLOGY  WORTH PUBLISHERS   \n",
       "4      MICHAEL M COX     BIOLOGY     W. H. FREEMAN   \n",
       "5         JAY PHELAN     BIOLOGY     FREEMAN/WORTH   \n",
       "6      DAVID G MYERS  PSYCHOLOGY     FREEMAN/WORTH   \n",
       "7      DAVID G MYERS  PSYCHOLOGY  WORTH PUBLISHERS   \n",
       "8     RONALD J COMER  PSYCHOLOGY  WORTH PUBLISHERS   \n",
       "\n",
       "                      sold_by  start_page  end_page  num_pages  \n",
       "0  Macmillan Higher Education          21       747        863  \n",
       "1  Macmillan Higher Education          38       526        616  \n",
       "2  Macmillan Higher Education          51      1297       1447  \n",
       "3  Macmillan Higher Education          22       464        545  \n",
       "4  Macmillan Higher Education          30       828        934  \n",
       "5  Macmillan Higher Education          34       718        773  \n",
       "6  Macmillan Higher Education          59       751        985  \n",
       "7  Macmillan Higher Education          59       662        892  \n",
       "8  Macmillan Higher Education          33       699        852  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = pd.read_csv(metadata_file, dtype = {'ISBN': 'str'})\n",
    "metadata['num_pages'] = [len(all_text[isbn]) for isbn in metadata['ISBN']]\n",
    "metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parse the text of each textbook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing text for ISBN: 9781429219617\n",
      "Parsing text for ISBN: 9781429242301\n",
      "Parsing text for ISBN: 9781429298643\n",
      "Parsing text for ISBN: 9781429298902\n",
      "Parsing text for ISBN: 9781464126147\n",
      "Parsing text for ISBN: 9781464135958\n",
      "Parsing text for ISBN: 9781464140815\n",
      "Parsing text for ISBN: 9781464154072\n",
      "Parsing text for ISBN: 9781464171703\n"
     ]
    }
   ],
   "source": [
    "all_docs = {}\n",
    "\n",
    "for isbn in isbns:    \n",
    "    print('Parsing text for ISBN: {}'.format(isbn))\n",
    "    start_page = metadata.loc[metadata['ISBN'] == isbn, 'start_page'].values[0]\n",
    "    end_page = metadata.loc[metadata['ISBN'] == isbn, 'end_page'].values[0]\n",
    "    \n",
    "    # Extract the content portion of the textbook, and combine the pages\n",
    "    text = ' '.join(all_text[isbn][(start_page-1):(end_page)])\n",
    "\n",
    "    # Parse the textbook with spacy\n",
    "    all_docs[isbn] = nlp(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use the DBpedia Spotlight server to annotate the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def extract_spotlight_concept(annotation):\n",
    "    return annotation['URI'].split('/resource/')[-1]\n",
    "\n",
    "def spotlight_annotate_text(doc, confidence=0.5, support=1):\n",
    "    # Confidence = confidence score for disambiguation / linking\n",
    "    # Support = number of inlinks to the Wikipedia entry\n",
    "\n",
    "    # Low support with high confidence\n",
    "    annotations = spotlight.annotate(spotlight_server,\n",
    "                                     doc.string,\n",
    "                                     confidence=confidence, \n",
    "                                     support=support)\n",
    "    \n",
    "    # Filter out some annotations\n",
    "    exclude_types = ['DBpedia:TimePeriod,DBpedia:Year']\n",
    "    \n",
    "    filtered_annotations = [ann for ann in annotations \n",
    "                            if ann['types'] not in exclude_types]\n",
    "              \n",
    "    return filtered_annotations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Annotate the text at different confidence levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Annotating for confidence: 0.4\n",
      "Annotating text for ISBN: 9781429219617\n",
      "Annotating text for ISBN: 9781429242301\n",
      "Annotating text for ISBN: 9781429298643\n",
      "Annotating text for ISBN: 9781429298902\n",
      "Annotating text for ISBN: 9781464126147\n",
      "Annotating text for ISBN: 9781464135958\n",
      "Annotating text for ISBN: 9781464140815\n",
      "Annotating text for ISBN: 9781464154072\n",
      "Annotating text for ISBN: 9781464171703\n",
      "Annotating for confidence: 0.6\n",
      "Annotating text for ISBN: 9781429219617\n",
      "Annotating text for ISBN: 9781429242301\n",
      "Annotating text for ISBN: 9781429298643\n",
      "Annotating text for ISBN: 9781429298902\n",
      "Annotating text for ISBN: 9781464126147\n",
      "Annotating text for ISBN: 9781464135958\n",
      "Annotating text for ISBN: 9781464140815\n",
      "Annotating text for ISBN: 9781464154072\n",
      "Annotating text for ISBN: 9781464171703\n",
      "Annotating for confidence: 0.8\n",
      "Annotating text for ISBN: 9781429219617\n",
      "Annotating text for ISBN: 9781429242301\n",
      "Annotating text for ISBN: 9781429298643\n",
      "Annotating text for ISBN: 9781429298902\n",
      "Annotating text for ISBN: 9781464126147\n",
      "Annotating text for ISBN: 9781464135958\n",
      "Annotating text for ISBN: 9781464140815\n",
      "Annotating text for ISBN: 9781464154072\n",
      "Annotating text for ISBN: 9781464171703\n"
     ]
    }
   ],
   "source": [
    "confidence_levels = [0.4, 0.6, 0.8]\n",
    "n_top = 100\n",
    "\n",
    "annotations_df = pd.DataFrame()\n",
    "\n",
    "for confidence in confidence_levels:\n",
    "    print('Annotating for confidence: {}'.format(confidence))\n",
    "    for isbn in isbns:\n",
    "        print('Annotating text for ISBN: {}'.format(isbn))\n",
    "        annotations = spotlight_annotate_text(all_docs[isbn],\n",
    "                                              confidence)\n",
    "        \n",
    "        concepts = [extract_spotlight_concept(ann) for ann in annotations]\n",
    "        freq = Counter(concepts)\n",
    "        common_concepts, counts = zip(*freq.most_common(n_top))\n",
    "        \n",
    "        annotations_df = annotations_df.append(\n",
    "            pd.DataFrame({\n",
    "                          'concept': list(common_concepts),\n",
    "                          'count': list(counts),\n",
    "                          'confidence': confidence,\n",
    "                          'ISBN': isbn\n",
    "                        }))         \n",
    "        time.sleep(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output common concepts to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "concepts_df = pd.merge(annotations_df, metadata, on='ISBN')\n",
    "\n",
    "concepts_df.to_csv(os.path.join(output_dir, 'common_concepts.csv'), \n",
    "                      index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}