1.clinical-trials.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract indications in clinical trial from clinicaltrials.gov\n",
    "\n",
    "+ [documentation](https://clinicaltrials.gov/ct2/help/how-read-study \"How to read a study record\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import bz2\n",
    "import collections\n",
    "import itertools\n",
    "import os\n",
    "import random\n",
    "import re\n",
    "import urllib.parse\n",
    "import urllib.request\n",
    "import xml.etree.ElementTree\n",
    "import zipfile\n",
    "\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download XML-formatted study records and save randomly sampled studies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # Uncomment this cell to download all trials as XML\n",
    "# query = {'resultsxml': 'true'}\n",
    "# query_str = urllib.parse.urlencode(query)\n",
    "# query_url = 'http://clinicaltrials.gov/search?{}'.format(query_str) \n",
    "# zip_path = 'download/resultsxml.zip'\n",
    "# urllib.request.urlretrieve(query_url, zip_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def zip_reader(path, max_records=None):\n",
    "    \"\"\"\n",
    "    Generate study records from the bulk ClinicalTrials.gov XML zipfile.\n",
    "    \"\"\"\n",
    "    with zipfile.ZipFile(path) as open_zip:\n",
    "        filenames = open_zip.namelist()\n",
    "        for i, filename in enumerate(filenames):\n",
    "            with open_zip.open(filename) as open_xml:\n",
    "                yield filename, xml.etree.ElementTree.parse(open_xml)\n",
    "            if max_records is not None and i + 1 >= max_records:\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# # Uncomment to prepare sample xml files\n",
    "# random.seed(0)\n",
    "# path = 'download/resultsxml.zip'\n",
    "# for filename, tree in zip_reader(path):\n",
    "#     if random.random() < 1e-4:\n",
    "#         sample_path = os.path.join('download', 'sample', filename)\n",
    "#         tree.write(sample_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process study records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# dhimmel/mesh commit\n",
    "commit = '9e16dfdca6c6d32cf8d1dcb4149c86be58a1a029'\n",
    "\n",
    "# Read MeSH descriptor and supplementary terms\n",
    "url = 'https://github.com/dhimmel/mesh/blob/{}/data/descriptor-terms.tsv?raw=true'.format(commit)\n",
    "desc_df = pandas.read_table(url)\n",
    "\n",
    "url = 'https://github.com/dhimmel/mesh/blob/{}/data/supplemental-terms.tsv?raw=true'.format(commit)\n",
    "supp_df = pandas.read_table(url)\n",
    "\n",
    "assert not set(desc_df.TermName) & set(supp_df.TermName)\n",
    "\n",
    "# Create a dictionary of MeSH term names to unique identifiers\n",
    "mesh_name_to_id = dict(zip(desc_df.TermName, desc_df.DescriptorUI))\n",
    "mesh_name_to_id.update(dict(zip(supp_df.TermName, supp_df.SupplementalRecordUI)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "unmatched_terms = collections.Counter()\n",
    "\n",
    "def get_mesh_id(name):\n",
    "    # Match by name\n",
    "    mesh_id = mesh_name_to_id.get(name)\n",
    "    if mesh_id is not None:\n",
    "        return mesh_id\n",
    "    # Match by name with first letter lowercase\n",
    "    first_lower = name[0].lower() + name[1:]\n",
    "    mesh_id = mesh_name_to_id.get(first_lower)\n",
    "    if mesh_id is not None:\n",
    "        return mesh_id\n",
    "    # Return `None` for unmatched\n",
    "    unmatched_terms[name] += 1\n",
    "    return None\n",
    "\n",
    "def get_mesh_ids(names):\n",
    "    mesh_ids = [get_mesh_id(name) for name in names]\n",
    "    return [x for x in mesh_ids if x is not None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_study_xml(tree):\n",
    "    \"\"\"\n",
    "    Extract information from an element tree for a ClinicalTrials.gov XML record.\n",
    "    \"\"\"\n",
    "    study = collections.OrderedDict()\n",
    "    study['nct_id'] = tree.findtext('id_info/nct_id')\n",
    "    study['study_type'] = tree.findtext('study_type')\n",
    "    study['brief_title'] = tree.findtext('brief_title')\n",
    "    brief_summary = tree.findtext('brief_summary/textblock', '')\n",
    "    study['brief_summary'] = re.sub(r' *\\n *', ' ', brief_summary).strip()\n",
    "    study['overall_status'] = tree.findtext('overall_status')\n",
    "    study['start_date'] = tree.findtext('start_date')\n",
    "    study['phase'] = tree.findtext('phase')\n",
    "    study['conditions'] = [x.text for x in tree.findall('condition')]\n",
    "    study['intervention_drugs'] = [x.text for x in tree.findall('intervention[intervention_type=\"Drug\"]/intervention_name')]\n",
    "    study['mesh_conditions'] = get_mesh_ids(x.text for x in tree.findall('condition_browse/mesh_term'))\n",
    "    study['mesh_interventions'] = get_mesh_ids(x.text for x in tree.findall('intervention_browse/mesh_term'))\n",
    "    return study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4min 42s, sys: 4.25 s, total: 4min 47s\n",
      "Wall time: 4min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "studies = list()\n",
    "path = 'download/resultsxml.zip'\n",
    "for filename, tree in zip_reader(path):\n",
    "    study = parse_study_xml(tree)\n",
    "    studies.append(study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unmatched_terms.most_common(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a table of studies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>study_type</th>\n",
       "      <th>brief_title</th>\n",
       "      <th>brief_summary</th>\n",
       "      <th>overall_status</th>\n",
       "      <th>start_date</th>\n",
       "      <th>phase</th>\n",
       "      <th>conditions</th>\n",
       "      <th>intervention_drugs</th>\n",
       "      <th>mesh_conditions</th>\n",
       "      <th>mesh_interventions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>214886</th>\n",
       "      <td>NCT00000105</td>\n",
       "      <td>Observational</td>\n",
       "      <td>Vaccination With Tetanus and KLH to Assess Imm...</td>\n",
       "      <td>The purpose of this study is to learn how the ...</td>\n",
       "      <td>Terminated</td>\n",
       "      <td>July 2002</td>\n",
       "      <td>N/A</td>\n",
       "      <td>[Cancer]</td>\n",
       "      <td>[Montanide ISA51]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[D005620, C032808]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214887</th>\n",
       "      <td>NCT00000104</td>\n",
       "      <td>Observational</td>\n",
       "      <td>Does Lead Burden Alter Neuropsychological Deve...</td>\n",
       "      <td>Inner city children are at an increased risk f...</td>\n",
       "      <td>Completed</td>\n",
       "      <td>None</td>\n",
       "      <td>N/A</td>\n",
       "      <td>[Lead Poisoning]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[D007855]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214888</th>\n",
       "      <td>NCT00000102</td>\n",
       "      <td>Interventional</td>\n",
       "      <td>Congenital Adrenal Hyperplasia: Calcium Channe...</td>\n",
       "      <td>This study will test the ability of extended r...</td>\n",
       "      <td>Completed</td>\n",
       "      <td>None</td>\n",
       "      <td>Phase 1/Phase 2</td>\n",
       "      <td>[Congenital Adrenal Hyperplasia]</td>\n",
       "      <td>[Nifedipine]</td>\n",
       "      <td>[D000312, D000308, D047808, D006965]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id      study_type  \\\n",
       "214886  NCT00000105   Observational   \n",
       "214887  NCT00000104   Observational   \n",
       "214888  NCT00000102  Interventional   \n",
       "\n",
       "                                              brief_title  \\\n",
       "214886  Vaccination With Tetanus and KLH to Assess Imm...   \n",
       "214887  Does Lead Burden Alter Neuropsychological Deve...   \n",
       "214888  Congenital Adrenal Hyperplasia: Calcium Channe...   \n",
       "\n",
       "                                            brief_summary overall_status  \\\n",
       "214886  The purpose of this study is to learn how the ...     Terminated   \n",
       "214887  Inner city children are at an increased risk f...      Completed   \n",
       "214888  This study will test the ability of extended r...      Completed   \n",
       "\n",
       "       start_date            phase                        conditions  \\\n",
       "214886  July 2002              N/A                          [Cancer]   \n",
       "214887       None              N/A                  [Lead Poisoning]   \n",
       "214888       None  Phase 1/Phase 2  [Congenital Adrenal Hyperplasia]   \n",
       "\n",
       "       intervention_drugs                       mesh_conditions  \\\n",
       "214886  [Montanide ISA51]                                    []   \n",
       "214887                 []                             [D007855]   \n",
       "214888       [Nifedipine]  [D000312, D000308, D047808, D006965]   \n",
       "\n",
       "        mesh_interventions  \n",
       "214886  [D005620, C032808]  \n",
       "214887                  []  \n",
       "214888                  []  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "study_df = pandas.DataFrame(studies)\n",
    "study_df = study_df[list(studies[0].keys())]\n",
    "study_df.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Save clinical trials, pipe delimiting plural fields\n",
    "write_df = study_df.copy()\n",
    "plural_columns = ['conditions', 'intervention_drugs', 'mesh_conditions', 'mesh_interventions']\n",
    "for column in plural_columns:\n",
    "    write_df[column] = write_df[column].map(lambda x: '|'.join(x))\n",
    "\n",
    "with bz2.open('data/results.tsv.bz2', 'wt') as write_file:\n",
    "    write_df.to_csv(write_file, sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a table of study, intervention, condition trios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>intervention</th>\n",
       "      <th>condition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>338471</th>\n",
       "      <td>NCT00000114</td>\n",
       "      <td>D014801</td>\n",
       "      <td>D012173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338472</th>\n",
       "      <td>NCT00000114</td>\n",
       "      <td>D014801</td>\n",
       "      <td>D012174</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id intervention condition\n",
       "338471  NCT00000114      D014801   D012173\n",
       "338472  NCT00000114      D014801   D012174"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mesh_rows = list()\n",
    "for study in studies:\n",
    "    nct_id = study['nct_id']\n",
    "    intervention = study['mesh_interventions']\n",
    "    condition = study['mesh_conditions']\n",
    "    for intervention, condition in itertools.product(intervention, condition):\n",
    "        row = nct_id, intervention, condition\n",
    "        mesh_rows.append(row)\n",
    "mesh_df = pandas.DataFrame(mesh_rows, columns=['nct_id', 'intervention', 'condition'])\n",
    "mesh_df = mesh_df.sort_values(['nct_id', 'intervention', 'condition'])\n",
    "mesh_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mesh_df.to_csv('data/mesh-intervention-to-condition.tsv', sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}