1_parsing_xml.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Input: XML from Reference Manager.\n",
    "Output: CSV with dataframe which has columns extracted from xml tags + flag useful.\n",
    "\"\"\"\n",
    "\n",
    "import pandas as pd\n",
    "import xml.etree.cElementTree as ET\n",
    "\n",
    "def get_elem(record, tag):\n",
    "    element = []\n",
    "    if record.iterfind(tag):\n",
    "        for elem in record.iterfind(tag) :\n",
    "            element.append(elem.text.strip().encode('utf-8'))\n",
    "    if element:\n",
    "        return element\n",
    "    else:\n",
    "        return [None]\n",
    "\n",
    "def parse_xml(filename):\n",
    "    data = list()\n",
    "    columns = ['filename','author','fulltitle','subtitle','pages',\n",
    "               'volume','number','keywords','year','abstract']\n",
    "\n",
    "    tree = ET.ElementTree(file=filename)\n",
    "\n",
    "    # Count how many elements of type record we can find\n",
    "    count = 0\n",
    "    for elem in tree.iter(tag='record'):\n",
    "        count += 1\n",
    "    print count\n",
    "\n",
    "    # Parsing fields into Dataframe\n",
    "    for record in tree.iter(tag='record'):\n",
    "        pdf_file = get_elem(record, 'urls/pdf-urls/url')[0]\n",
    "        authors = get_elem(record, 'contributors/authors/author/style')\n",
    "        full_title = get_elem(record, 'titles/title/style')[0]\n",
    "        sub_title = get_elem(record, 'titles/secondary-title/style')[0]\n",
    "        periodical_fulltitle = get_elem(record, 'periodical/full-title/style')[0]\n",
    "        pages = get_elem(record, 'pages')[0]\n",
    "        volume = get_elem(record, 'volume/style')[0]\n",
    "        number = get_elem(record, 'number/style')[0]\n",
    "        keywords = get_elem(record, 'keywords/keyword/style')\n",
    "        year = get_elem(record, 'dates/year')[0]\n",
    "        location = get_elem(record, 'pub-location/style')[0]\n",
    "        publisher = get_elem(record, 'publisher/style')[0]\n",
    "        isbn = get_elem(record, 'isbn/style')[0]\n",
    "        abstract = get_elem(record, 'abstract/style')[0]\n",
    "\n",
    "        data.append([pdf_file, authors, full_title, sub_title, pages, \n",
    "                     volume, number, keywords, year, abstract])\n",
    "    \n",
    "    df = pd.DataFrame(data, columns=columns)   \n",
    "    \n",
    "    # Saving Dataframe as csv\n",
    "    df.to_csv('{0}'.format(filename.replace('.xml','.csv')), index=0)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "658\n",
      "(658, 10)\n",
      "3796\n",
      "(3796, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>author</th>\n",
       "      <th>fulltitle</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>pages</th>\n",
       "      <th>volume</th>\n",
       "      <th>number</th>\n",
       "      <th>keywords</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[=International Agency for Reseach on Cancer (...</td>\n",
       "      <td>Consumption of alcoholic beverages</td>\n",
       "      <td>IARC Monographs on the Evaluation of Carcinoge...</td>\n",
       "      <td>377-504</td>\n",
       "      <td>100 E</td>\n",
       "      <td>None</td>\n",
       "      <td>[basis,1-alk, endometrium,1-alk, endometrium,b...</td>\n",
       "      <td>2012</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[Aass,N., De Mulder,P.H.M., Mickisch,G.H.J., M...</td>\n",
       "      <td>Randomized phase II/III trial of interferon al...</td>\n",
       "      <td>Journal of Clinical Oncology</td>\n",
       "      <td>4172-4178</td>\n",
       "      <td>23</td>\n",
       "      <td>18</td>\n",
       "      <td>[niere,4-med-pall, niere,bereitgestellt, Quell...</td>\n",
       "      <td>2005</td>\n",
       "      <td>Purpose: A randomized phase II/III trial was c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[Abdel-Rahman,O., Fouad,M.]</td>\n",
       "      <td>Efficacy and toxicity of sunitinib for non cle...</td>\n",
       "      <td>Critical Reviews in Oncology/Hematology</td>\n",
       "      <td>238-250</td>\n",
       "      <td>94</td>\n",
       "      <td>2</td>\n",
       "      <td>[niere,4-med-pall, niere,bereitgestellt, Quell...</td>\n",
       "      <td>2015</td>\n",
       "      <td>The randomized phase III trial of sunitinib ve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[Abdellateef,M.]</td>\n",
       "      <td>Laparoscopic Partial Nephrectomy: Expanding Ro...</td>\n",
       "      <td>World Journal of Laparoscopic Surgery</td>\n",
       "      <td>169-173</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>[niere,4-op-lap, niere,bereitgestellt, Quelle,...</td>\n",
       "      <td>2011</td>\n",
       "      <td>Context: The increasing incidence of localized...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[Abel,E Jason, Wood,Christopher G.]</td>\n",
       "      <td>Cytoreductive nephrectomy for metastatic RCC i...</td>\n",
       "      <td>Nature Reviews.Urology</td>\n",
       "      <td>375-383</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>[niere,4-op-pall, niere,bereitgestellt, Quelle...</td>\n",
       "      <td>2009</td>\n",
       "      <td>Metastatic renal cell carcinoma (RCC) has trad...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            filename  \\\n",
       "0  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "1  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "2  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "3  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "4  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "\n",
       "                                              author  \\\n",
       "0  [=International Agency for Reseach on Cancer (...   \n",
       "1  [Aass,N., De Mulder,P.H.M., Mickisch,G.H.J., M...   \n",
       "2                        [Abdel-Rahman,O., Fouad,M.]   \n",
       "3                                   [Abdellateef,M.]   \n",
       "4                [Abel,E Jason, Wood,Christopher G.]   \n",
       "\n",
       "                                           fulltitle  \\\n",
       "0                 Consumption of alcoholic beverages   \n",
       "1  Randomized phase II/III trial of interferon al...   \n",
       "2  Efficacy and toxicity of sunitinib for non cle...   \n",
       "3  Laparoscopic Partial Nephrectomy: Expanding Ro...   \n",
       "4  Cytoreductive nephrectomy for metastatic RCC i...   \n",
       "\n",
       "                                            subtitle      pages volume number  \\\n",
       "0  IARC Monographs on the Evaluation of Carcinoge...    377-504  100 E   None   \n",
       "1                       Journal of Clinical Oncology  4172-4178     23     18   \n",
       "2            Critical Reviews in Oncology/Hematology    238-250     94      2   \n",
       "3              World Journal of Laparoscopic Surgery    169-173      4      3   \n",
       "4                             Nature Reviews.Urology    375-383      6      7   \n",
       "\n",
       "                                            keywords  year  \\\n",
       "0  [basis,1-alk, endometrium,1-alk, endometrium,b...  2012   \n",
       "1  [niere,4-med-pall, niere,bereitgestellt, Quell...  2005   \n",
       "2  [niere,4-med-pall, niere,bereitgestellt, Quell...  2015   \n",
       "3  [niere,4-op-lap, niere,bereitgestellt, Quelle,...  2011   \n",
       "4  [niere,4-op-pall, niere,bereitgestellt, Quelle...  2009   \n",
       "\n",
       "                                            abstract  \n",
       "0                                               None  \n",
       "1  Purpose: A randomized phase II/III trial was c...  \n",
       "2  The randomized phase III trial of sunitinib ve...  \n",
       "3  Context: The increasing incidence of localized...  \n",
       "4  Metastatic renal cell carcinoma (RCC) has trad...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "Here a summary of the data DKG provided:\n",
    "\n",
    "664 pdfs about kidney (Niere.zip)\n",
    "An XML with the metadata about the 664 PDFs (niere-zugeordnete-referenzen-xml.xml)\n",
    "An XML with the metadata about PDFs excluded (niere-ausgeschlossene-xml.xml) \n",
    "\"\"\"\n",
    "\n",
    "classifiedfile = '../metadata/niere-example/niere-zugeordnete-referenzen-xml.xml'\n",
    "excludedfile = '../metadata/niere-example/niere-ausgeschlossene-xml.xml'\n",
    "\n",
    "classified = parse_xml(classifiedfile)\n",
    "print classified.shape\n",
    "\n",
    "excluded = parse_xml(excludedfile)\n",
    "print excluded.shape\n",
    "\n",
    "classified.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4454, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>author</th>\n",
       "      <th>fulltitle</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>pages</th>\n",
       "      <th>volume</th>\n",
       "      <th>number</th>\n",
       "      <th>keywords</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>useful</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[=International Agency for Reseach on Cancer (...</td>\n",
       "      <td>Consumption of alcoholic beverages</td>\n",
       "      <td>IARC Monographs on the Evaluation of Carcinoge...</td>\n",
       "      <td>377-504</td>\n",
       "      <td>100 E</td>\n",
       "      <td>None</td>\n",
       "      <td>[basis,1-alk, endometrium,1-alk, endometrium,b...</td>\n",
       "      <td>2012</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[Aass,N., De Mulder,P.H.M., Mickisch,G.H.J., M...</td>\n",
       "      <td>Randomized phase II/III trial of interferon al...</td>\n",
       "      <td>Journal of Clinical Oncology</td>\n",
       "      <td>4172-4178</td>\n",
       "      <td>23</td>\n",
       "      <td>18</td>\n",
       "      <td>[niere,4-med-pall, niere,bereitgestellt, Quell...</td>\n",
       "      <td>2005</td>\n",
       "      <td>Purpose: A randomized phase II/III trial was c...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[Abdel-Rahman,O., Fouad,M.]</td>\n",
       "      <td>Efficacy and toxicity of sunitinib for non cle...</td>\n",
       "      <td>Critical Reviews in Oncology/Hematology</td>\n",
       "      <td>238-250</td>\n",
       "      <td>94</td>\n",
       "      <td>2</td>\n",
       "      <td>[niere,4-med-pall, niere,bereitgestellt, Quell...</td>\n",
       "      <td>2015</td>\n",
       "      <td>The randomized phase III trial of sunitinib ve...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[Abdellateef,M.]</td>\n",
       "      <td>Laparoscopic Partial Nephrectomy: Expanding Ro...</td>\n",
       "      <td>World Journal of Laparoscopic Surgery</td>\n",
       "      <td>169-173</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>[niere,4-op-lap, niere,bereitgestellt, Quelle,...</td>\n",
       "      <td>2011</td>\n",
       "      <td>Context: The increasing incidence of localized...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>file://R:\\Literatur\\literatur_dialog\\informati...</td>\n",
       "      <td>[Abel,E Jason, Wood,Christopher G.]</td>\n",
       "      <td>Cytoreductive nephrectomy for metastatic RCC i...</td>\n",
       "      <td>Nature Reviews.Urology</td>\n",
       "      <td>375-383</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>[niere,4-op-pall, niere,bereitgestellt, Quelle...</td>\n",
       "      <td>2009</td>\n",
       "      <td>Metastatic renal cell carcinoma (RCC) has trad...</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            filename  \\\n",
       "0  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "1  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "2  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "3  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "4  file://R:\\Literatur\\literatur_dialog\\informati...   \n",
       "\n",
       "                                              author  \\\n",
       "0  [=International Agency for Reseach on Cancer (...   \n",
       "1  [Aass,N., De Mulder,P.H.M., Mickisch,G.H.J., M...   \n",
       "2                        [Abdel-Rahman,O., Fouad,M.]   \n",
       "3                                   [Abdellateef,M.]   \n",
       "4                [Abel,E Jason, Wood,Christopher G.]   \n",
       "\n",
       "                                           fulltitle  \\\n",
       "0                 Consumption of alcoholic beverages   \n",
       "1  Randomized phase II/III trial of interferon al...   \n",
       "2  Efficacy and toxicity of sunitinib for non cle...   \n",
       "3  Laparoscopic Partial Nephrectomy: Expanding Ro...   \n",
       "4  Cytoreductive nephrectomy for metastatic RCC i...   \n",
       "\n",
       "                                            subtitle      pages volume number  \\\n",
       "0  IARC Monographs on the Evaluation of Carcinoge...    377-504  100 E   None   \n",
       "1                       Journal of Clinical Oncology  4172-4178     23     18   \n",
       "2            Critical Reviews in Oncology/Hematology    238-250     94      2   \n",
       "3              World Journal of Laparoscopic Surgery    169-173      4      3   \n",
       "4                             Nature Reviews.Urology    375-383      6      7   \n",
       "\n",
       "                                            keywords  year  \\\n",
       "0  [basis,1-alk, endometrium,1-alk, endometrium,b...  2012   \n",
       "1  [niere,4-med-pall, niere,bereitgestellt, Quell...  2005   \n",
       "2  [niere,4-med-pall, niere,bereitgestellt, Quell...  2015   \n",
       "3  [niere,4-op-lap, niere,bereitgestellt, Quelle,...  2011   \n",
       "4  [niere,4-op-pall, niere,bereitgestellt, Quelle...  2009   \n",
       "\n",
       "                                            abstract  useful  num  \n",
       "0                                               None       1    1  \n",
       "1  Purpose: A randomized phase II/III trial was c...       1    2  \n",
       "2  The randomized phase III trial of sunitinib ve...       1    3  \n",
       "3  Context: The increasing incidence of localized...       1    4  \n",
       "4  Metastatic renal cell carcinoma (RCC) has trad...       1    5  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Add Flag\n",
    "classified['useful'] = 1\n",
    "excluded['useful'] = 0\n",
    "\n",
    "full_df = pd.concat([classified, excluded])\n",
    "\n",
    "# Add ID for records\n",
    "full_df['num'] = range(1, len(full_df) + 1)\n",
    "\n",
    "print full_df.shape\n",
    "\n",
    "full_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save dataframe to be used for classification task\n",
    "full_df.to_csv('../metadata/niere-example/classification_df.csv', index=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all records: 4454\n",
      "    positive records: 658 (14.8% of all records)\n",
      "    negative records: 3796 (85.2% of all records)\n"
     ]
    }
   ],
   "source": [
    "# dataframe stats\n",
    "positive = full_df[full_df.useful == 1]\n",
    "negative = full_df[full_df.useful == 0]\n",
    "\n",
    "print \"\"\"all records: {0}\n",
    "    positive records: {1} ({2:.1f}% of all records)\n",
    "    negative records: {3} ({4:.1f}% of all records)\"\"\".format(len(full_df), \n",
    "                                                              len(positive), float(len(positive))/len(full_df)*100,\n",
    "                                                              len(negative), float(len(negative))/len(full_df)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}