symptoms.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute symptom-disease cooccurrence for Hetionet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import io\n",
    "import gzip\n",
    "\n",
    "import pandas\n",
    "import requests\n",
    "import networkx\n",
    "\n",
    "import eutility\n",
    "import cooccurrence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_code</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>mesh_id</th>\n",
       "      <th>mesh_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>hematologic cancer</td>\n",
       "      <td>D019337</td>\n",
       "      <td>Hematologic Neoplasms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:1319</td>\n",
       "      <td>brain cancer</td>\n",
       "      <td>D001932</td>\n",
       "      <td>Brain Neoplasms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DOID:1324</td>\n",
       "      <td>lung cancer</td>\n",
       "      <td>D008175</td>\n",
       "      <td>Lung Neoplasms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DOID:263</td>\n",
       "      <td>kidney cancer</td>\n",
       "      <td>D007680</td>\n",
       "      <td>Kidney Neoplasms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DOID:1793</td>\n",
       "      <td>pancreatic cancer</td>\n",
       "      <td>D010190</td>\n",
       "      <td>Pancreatic Neoplasms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   doid_code           doid_name  mesh_id              mesh_name\n",
       "0  DOID:2531  hematologic cancer  D019337  Hematologic Neoplasms\n",
       "1  DOID:1319        brain cancer  D001932        Brain Neoplasms\n",
       "2  DOID:1324         lung cancer  D008175         Lung Neoplasms\n",
       "3   DOID:263       kidney cancer  D007680       Kidney Neoplasms\n",
       "4  DOID:1793   pancreatic cancer  D010190   Pancreatic Neoplasms"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read mappings for DO Slim terms\n",
    "url = 'https://raw.githubusercontent.com/dhimmel/disease-ontology/72614ade9f1cc5a5317b8f6836e1e464b31d5587/data/xrefs-slim.tsv'\n",
    "disease_df = pandas.read_table(url)\n",
    "disease_df = disease_df.query('resource == \"MSH\"').drop('resource', 1)\n",
    "disease_df = disease_df.rename(columns={'resource_id': 'mesh_id'})\n",
    "\n",
    "# Read MeSH terms to MeSH names\n",
    "url = 'https://raw.githubusercontent.com/dhimmel/mesh/e561301360e6de2140dedeaa7c7e17ce4714eb7f/data/terms.tsv'\n",
    "mesh_df = pandas.read_table(url)\n",
    "disease_df = disease_df.merge(mesh_df)\n",
    "\n",
    "# Manually remove problematic xrefs\n",
    "# https://github.com/obophenotype/human-disease-ontology/issues/45\n",
    "disease_df = disease_df.query(\"mesh_id != 'D003327' and mesh_id != 'D017202'\")\n",
    "disease_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Diseases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7382 articles for Hematologic Neoplasms\n",
      "99586 articles for Brain Neoplasms\n",
      "139299 articles for Lung Neoplasms\n",
      "49515 articles for Kidney Neoplasms\n",
      "46298 articles for Pancreatic Neoplasms\n",
      "85654 articles for Skin Neoplasms\n",
      "83658 articles for Bone Neoplasms\n",
      "22125 articles for Pharyngeal Neoplasms\n",
      "53989 articles for Ovarian Neoplasms\n",
      "188908 articles for Breast Neoplasms\n",
      "49539 articles for Glioma\n",
      "89055 articles for Uterine Neoplasms\n",
      "18514 articles for Adrenal Gland Neoplasms\n",
      "33421 articles for Esophageal Neoplasms\n",
      "12158 articles for Salivary Gland Neoplasms\n",
      "83257 articles for Prostatic Neoplasms\n",
      "64512 articles for Stomach Neoplasms\n",
      "37569 articles for Urinary Bladder Neoplasms\n",
      "14765 articles for Peripheral Nervous System Neoplasms\n",
      "33286 articles for Thyroid Neoplasms\n",
      "97650 articles for Liver Neoplasms\n",
      "50220 articles for Uterine Cervical Neoplasms\n",
      "3507 articles for Vaginal Neoplasms\n",
      "210249 articles for Head and Neck Neoplasms\n",
      "32809 articles for Rectal Neoplasms\n",
      "28761 articles for Eye Neoplasms\n",
      "50799 articles for Colonic Neoplasms\n",
      "19451 articles for Laryngeal Neoplasms\n",
      "225331 articles for Neoplasms, Germ Cell and Embryonal\n",
      "7330 articles for Thymus Neoplasms\n",
      "8568 articles for Myosarcoma\n",
      "4375 articles for Duodenal Neoplasms\n",
      "2161 articles for Ileal Neoplasms\n",
      "90305 articles for Sarcoma\n",
      "2002 articles for Appendiceal Neoplasms\n",
      "3898 articles for Penile Neoplasms\n",
      "3458 articles for Ureteral Neoplasms\n",
      "2526 articles for Tracheal Neoplasms\n",
      "5866 articles for Vulvar Neoplasms\n",
      "1649 articles for Jejunal Neoplasms\n",
      "9852 articles for Peritoneal Neoplasms\n",
      "2469 articles for Vascular Neoplasms\n",
      "9785 articles for Mesothelioma\n",
      "60090 articles for Melanoma\n",
      "2040 articles for Fallopian Tube Neoplasms\n",
      "18463 articles for Testicular Neoplasms\n",
      "5919 articles for Gallbladder Neoplasms\n",
      "15236 articles for Meningeal Neoplasms\n",
      "11129 articles for Bile Duct Neoplasms\n",
      "9591 articles for Mediastinal Neoplasms\n",
      "7736 articles for Spinal Cord Neoplasms\n",
      "6254 articles for Retroperitoneal Neoplasms\n",
      "24975 articles for Crohn Disease\n",
      "39550 articles for Multiple Sclerosis\n",
      "72794 articles for Diabetes Mellitus, Type 2\n",
      "22055 articles for Colitis, Ulcerative\n",
      "50883 articles for Diabetes Mellitus, Type 1\n",
      "76622 articles for Arthritis, Rheumatoid\n",
      "33214 articles for Coronary Artery Disease\n",
      "105366 articles for Obesity\n",
      "13742 articles for Celiac Disease\n",
      "39741 articles for Lupus Erythematosus, Systemic\n",
      "19765 articles for Refractive Errors\n",
      "5304 articles for Liver Cirrhosis, Biliary\n",
      "3492 articles for Vitiligo\n",
      "13479 articles for Macular Degeneration\n",
      "16426 articles for Metabolic Syndrome X\n",
      "88631 articles for Asthma\n",
      "25313 articles for Psoriasis\n",
      "69283 articles for Schizophrenia\n",
      "18093 articles for Migraine Disorders\n",
      "55360 articles for Alzheimer Disease\n",
      "10965 articles for Graves Disease\n",
      "40397 articles for Parkinson Disease\n",
      "11782 articles for Dermatitis, Atopic\n",
      "24524 articles for Bipolar Disorder\n",
      "9401 articles for Spondylitis, Ankylosing\n",
      "8888 articles for Polycystic Ovary Syndrome\n",
      "155557 articles for Hypertension\n",
      "14044 articles for Scleroderma, Systemic\n",
      "6764 articles for Behcet Syndrome\n",
      "3814 articles for Osteitis Deformans\n",
      "18561 articles for Leprosy\n",
      "18785 articles for Intracranial Aneurysm\n",
      "35366 articles for Glaucoma\n",
      "11500 articles for Amyotrophic Lateral Sclerosis\n",
      "2296 articles for Restless Legs Syndrome\n",
      "4319 articles for Mucocutaneous Lymph Node Syndrome\n",
      "18009 articles for Atherosclerosis\n",
      "2125 articles for Alopecia Areata\n",
      "32547 articles for Osteoporosis\n",
      "20300 articles for Hypothyroidism\n",
      "4202 articles for Glomerulonephritis, IGA\n",
      "49443 articles for Alcoholism\n",
      "4464 articles for Creutzfeldt-Jakob Syndrome\n",
      "864 articles for Azoospermia\n",
      "102949 articles for Epilepsy\n",
      "36716 articles for Hepatitis B\n",
      "30665 articles for Pulmonary Disease, Chronic Obstructive\n",
      "12886 articles for Aortic Aneurysm, Abdominal\n",
      "54875 articles for Kidney Failure, Chronic\n",
      "33398 articles for Osteoarthritis\n",
      "2999 articles for Arthritis, Psoriatic\n",
      "6354 articles for Tobacco Use Disorder\n",
      "1918 articles for Glomerulonephritis, Membranous\n",
      "6056 articles for Diabetes, Gestational\n",
      "43489 articles for Malaria\n",
      "13959 articles for Autistic Disorder\n",
      "10108 articles for Cardiomyopathy, Dilated\n",
      "724 articles for Arthritis, Gouty\n",
      "14343 articles for Leiomyoma\n",
      "2309 articles for Cholangitis, Sclerosing\n",
      "2374 articles for Narcolepsy\n",
      "1561 articles for Intervertebral Disc Degeneration\n",
      "9599 articles for Cleft Lip\n",
      "1277 articles for Idiopathic Pulmonary Fibrosis\n",
      "16912 articles for Attention Deficit Disorder with Hyperactivity\n",
      "3143 articles for Tourette Syndrome\n",
      "7893 articles for Aortic Aneurysm, Thoracic\n",
      "63783 articles for Depressive Disorder\n",
      "13894 articles for Rhinitis, Allergic\n",
      "35263 articles for Pancreatitis\n",
      "12217 articles for Nephrolithiasis\n",
      "16409 articles for Periodontitis\n",
      "5256 articles for Barrett Esophagus\n",
      "550 articles for Fuchs' Endothelial Dystrophy\n",
      "3870 articles for Otosclerosis\n",
      "1486 articles for Conduct Disorder\n",
      "2979 articles for Glomerulosclerosis, Focal Segmental\n",
      "25969 articles for Dental Caries\n",
      "105132 articles for Anemia\n",
      "4634 articles for Panic Disorder\n",
      "58396 articles for Acquired Immunodeficiency Syndrome\n"
     ]
    }
   ],
   "source": [
    "rows_out = list()\n",
    "\n",
    "for i, row in disease_df.iterrows():\n",
    "    term_query = '{disease}[MeSH Major Topic]'.format(disease = row.mesh_name.lower())\n",
    "    payload = {'db': 'pubmed', 'term': term_query}\n",
    "    pmids = eutility.esearch_query(payload, retmax = 10000)\n",
    "    row['term_query'] = term_query\n",
    "    row['n_articles'] = len(pmids)\n",
    "    row['pubmed_ids'] = '|'.join(pmids)\n",
    "    rows_out.append(row)\n",
    "    print('{} articles for {}'.format(len(pmids), row.mesh_name))\n",
    "\n",
    "disease_pmids_df = pandas.DataFrame(rows_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "with gzip.open('data/disease-pmids.tsv.gz', 'w') as write_file:\n",
    "    write_file = io.TextIOWrapper(write_file)\n",
    "    disease_pmids_df.to_csv(write_file, sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Symptoms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mesh_id</th>\n",
       "      <th>mesh_name</th>\n",
       "      <th>in_hsdn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>D000006</td>\n",
       "      <td>Abdomen, Acute</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>D000270</td>\n",
       "      <td>Adie Syndrome</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>D000326</td>\n",
       "      <td>Adrenoleukodystrophy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>D000334</td>\n",
       "      <td>Aerophagy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>D000370</td>\n",
       "      <td>Ageusia</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mesh_id             mesh_name  in_hsdn\n",
       "0  D000006        Abdomen, Acute        1\n",
       "1  D000270         Adie Syndrome        0\n",
       "2  D000326  Adrenoleukodystrophy        0\n",
       "3  D000334             Aerophagy        1\n",
       "4  D000370               Ageusia        1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read MeSH Symptoms\n",
    "url = 'https://raw.githubusercontent.com/dhimmel/mesh/e561301360e6de2140dedeaa7c7e17ce4714eb7f/data/symptoms.tsv'\n",
    "symptom_df = pandas.read_table(url)\n",
    "symptom_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8496 articles for Abdomen, Acute\n",
      "313 articles for Adie Syndrome\n",
      "1508 articles for Adrenoleukodystrophy\n",
      "261 articles for Aerophagy\n",
      "222 articles for Ageusia\n",
      "2049 articles for Agnosia\n",
      "849 articles for Agraphia\n",
      "12310 articles for Albuminuria\n",
      "1118 articles for Alcohol Amnestic Disorder\n",
      "846 articles for Alkalosis, Respiratory\n",
      "5803 articles for Amblyopia\n",
      "6330 articles for Amnesia\n",
      "824 articles for Amnesia, Retrograde\n",
      "30785 articles for Angina Pectoris\n",
      "1864 articles for Angina Pectoris, Variant\n",
      "8277 articles for Angina, Unstable\n",
      "933 articles for Anomia\n",
      "4130 articles for Anorexia\n",
      "3055 articles for Olfaction Disorders\n",
      "53502 articles for Anoxia\n",
      "8561 articles for Aphasia\n",
      "1439 articles for Aphasia, Broca\n",
      "825 articles for Aphasia, Wernicke\n",
      "270 articles for Aphonia\n",
      "6310 articles for Apnea\n",
      "2355 articles for Apraxias\n",
      "1544 articles for Articulation Disorders\n",
      "1439 articles for Asthenia\n",
      "6601 articles for Ataxia\n",
      "2930 articles for Ataxia Telangiectasia\n",
      "1284 articles for Athetosis\n",
      "1041 articles for Auditory Perceptual Disorders\n",
      "15025 articles for Back Pain\n",
      "33305 articles for Birth Weight\n",
      "6468 articles for Urinary Bladder, Neurogenic\n",
      "16978 articles for Blindness\n",
      "0 articles for Body Temperature Changes\n",
      "164793 articles for Body Weight\n",
      "5 articles for Body Weight Changes\n",
      "7382 articles for Brain Death\n",
      "4920 articles for Bulimia\n",
      "4023 articles for Cachexia\n",
      "5278 articles for Cardiac Output, Low\n",
      "2347 articles for Catalepsy\n",
      "767 articles for Cataplexy\n",
      "1977 articles for Catatonia\n",
      "564 articles for Causalgia\n",
      "3625 articles for Cerebellar Ataxia\n",
      "981 articles for Cerebrospinal Fluid Otorrhea\n",
      "2619 articles for Cerebrospinal Fluid Rhinorrhea\n",
      "9786 articles for Chest Pain\n",
      "625 articles for Cheyne-Stokes Respiration\n",
      "3746 articles for Chorea\n",
      "361 articles for Choroid Hemorrhage\n",
      "3597 articles for Colic\n",
      "3593 articles for Color Vision Defects\n",
      "10956 articles for Coma\n",
      "1687 articles for Communication Disorders\n",
      "3771 articles for Confusion\n",
      "2178 articles for Consciousness Disorders\n",
      "11088 articles for Constipation\n",
      "12559 articles for Cough\n",
      "609 articles for Cri-du-Chat Syndrome\n",
      "4077 articles for Cyanosis\n",
      "627 articles for De Lange Syndrome\n",
      "23208 articles for Deafness\n",
      "1750 articles for Hearing Loss, Sudden\n",
      "4708 articles for Decerebrate State\n",
      "6161 articles for Delirium\n",
      "39716 articles for Diarrhea\n",
      "6500 articles for Diarrhea, Infantile\n",
      "4370 articles for Diplopia\n",
      "11880 articles for Dizziness\n",
      "21187 articles for Down Syndrome\n",
      "1906 articles for Dysarthria\n",
      "288 articles for Dysgeusia\n",
      "6293 articles for Dyskinesia, Drug-Induced\n",
      "6411 articles for Dyslexia\n",
      "825 articles for Dyslexia, Acquired\n",
      "3213 articles for Dysmenorrhea\n",
      "7353 articles for Dyspepsia\n",
      "15978 articles for Dyspnea\n",
      "322 articles for Dyspnea, Paroxysmal\n",
      "6503 articles for Dystonia\n",
      "614 articles for Earache\n",
      "1075 articles for Ecchymosis\n",
      "211 articles for Echolalia\n",
      "33279 articles for Edema\n",
      "555 articles for Edema, Cardiac\n",
      "607 articles for Emaciation\n",
      "599 articles for Encopresis\n",
      "310 articles for Eructation\n",
      "817 articles for Eye Hemorrhage\n",
      "3146 articles for Eye Manifestations\n",
      "5163 articles for Facial Pain\n",
      "10449 articles for Facial Paralysis\n",
      "1785 articles for Failure to Thrive\n",
      "598 articles for Fasciculation\n",
      "20620 articles for Fatigue\n",
      "1172 articles for Mental Fatigue\n",
      "534 articles for Feminization\n",
      "2763 articles for Fetal Hypoxia\n",
      "2988 articles for Fetal Distress\n",
      "1878 articles for Fetal Macrosomia\n",
      "31658 articles for Fever\n",
      "3742 articles for Fever of Unknown Origin\n",
      "1233 articles for Flatulence\n",
      "1084 articles for Flushing\n",
      "4151 articles for Fragile X Syndrome\n",
      "416 articles for Gagging\n",
      "188 articles for Gerstmann Syndrome\n",
      "2562 articles for Gingival Hemorrhage\n",
      "251 articles for Glossalgia\n",
      "1128 articles for Halitosis\n",
      "9275 articles for Hallucinations\n",
      "22956 articles for Headache\n",
      "13453 articles for Hearing Disorders\n",
      "1745 articles for Hearing Loss, Bilateral\n",
      "481 articles for Hearing Loss, Central\n",
      "3022 articles for Hearing Loss, Conductive\n",
      "158 articles for Hearing Loss, Functional\n",
      "868 articles for Hearing Loss, High-Frequency\n",
      "6101 articles for Hearing Loss, Noise-Induced\n",
      "13197 articles for Hearing Loss, Sensorineural\n",
      "3035 articles for Heart Murmurs\n",
      "1707 articles for Heartburn\n",
      "2030 articles for Hematemesis\n",
      "2493 articles for Hemianopsia\n",
      "10439 articles for Hemiplegia\n",
      "1147 articles for Hemoglobinuria\n",
      "5102 articles for Hemoptysis\n",
      "1904 articles for Oral Hemorrhage\n",
      "939 articles for Hiccup\n",
      "3403 articles for Hirsutism\n",
      "1731 articles for Hoarseness\n",
      "1713 articles for Horner Syndrome\n",
      "9641 articles for Huntington Disease\n",
      "8030 articles for Hyperalgesia\n",
      "7386 articles for Hypercapnia\n",
      "1254 articles for Hyperemesis Gravidarum\n",
      "831 articles for Hyperesthesia\n",
      "3377 articles for Hypergammaglobulinemia\n",
      "3698 articles for Hyperkinesis\n",
      "2475 articles for Hyperphagia\n",
      "2742 articles for Disorders of Excessive Somnolence\n",
      "5160 articles for Hyperventilation\n",
      "2373 articles for Hypesthesia\n",
      "1260 articles for Hyphema\n",
      "4944 articles for Hypotension, Orthostatic\n",
      "8729 articles for Hypothermia\n",
      "1680 articles for Hypoventilation\n",
      "4413 articles for Illusions\n",
      "9308 articles for Sleep Initiation and Maintenance Disorders\n",
      "319 articles for Insulin Coma\n",
      "7081 articles for Intermittent Claudication\n",
      "9452 articles for Jaundice\n",
      "596 articles for Kearns-Sayre Syndrome\n",
      "916 articles for Menkes Kinky Hair Syndrome\n",
      "5025 articles for Language Development Disorders\n",
      "5692 articles for Language Disorders\n",
      "12767 articles for Learning Disorders\n",
      "1144 articles for Lesch-Nyhan Syndrome\n",
      "311 articles for Lipoid Proteinosis of Urbach and Wiethe\n",
      "15869 articles for Memory Disorders\n",
      "238 articles for Meningism\n",
      "47833 articles for Intellectual Disability\n",
      "781 articles for Monoclonal Gammopathy of Undetermined Significance\n",
      "2279 articles for Motion Sickness\n",
      "1119 articles for Mouth Breathing\n",
      "1948 articles for Muscle Cramp\n",
      "777 articles for Muscle Hypertonia\n",
      "2650 articles for Muscle Hypotonia\n",
      "1789 articles for Muscle Rigidity\n",
      "6967 articles for Muscle Spasticity\n",
      "8969 articles for Muscular Atrophy\n",
      "906 articles for Mutism\n",
      "4660 articles for Myoclonus\n",
      "1040 articles for Myotonia\n",
      "2842 articles for Narcolepsy\n",
      "13292 articles for Nausea\n",
      "9009 articles for Neuralgia\n",
      "7314 articles for Neurologic Manifestations\n",
      "1287 articles for Night Blindness\n",
      "132455 articles for Obesity\n",
      "12170 articles for Obesity, Morbid\n",
      "1037 articles for Oliguria\n",
      "7392 articles for Ophthalmoplegia\n",
      "4188 articles for Optical Illusions\n",
      "1484 articles for Oral Manifestations\n",
      "111258 articles for Pain\n",
      "5588 articles for Pain, Intractable\n",
      "28672 articles for Pain, Postoperative\n",
      "269 articles for Pallor\n",
      "18293 articles for Paralysis\n",
      "11424 articles for Paraplegia\n",
      "5291 articles for Paresis\n",
      "5248 articles for Paresthesia\n",
      "5499 articles for Perceptual Disorders\n",
      "1560 articles for Phantom Limb\n",
      "643 articles for Obesity Hypoventilation Syndrome\n",
      "1821 articles for Polyuria\n",
      "2356 articles for Prader-Willi Syndrome\n",
      "1165 articles for Presbycusis\n",
      "20966 articles for Proteinuria\n",
      "9095 articles for Pruritus\n",
      "366 articles for Pruritus Ani\n",
      "314 articles for Pruritus Vulvae\n",
      "3697 articles for Psychomotor Agitation\n",
      "4816 articles for Psychomotor Disorders\n",
      "17476 articles for Psychophysiologic Disorders\n",
      "766 articles for Pupil Disorders\n",
      "4655 articles for Purpura\n",
      "243 articles for Purpura, Hyperglobulinemic\n",
      "3611 articles for Purpura, Schoenlein-Henoch\n",
      "5798 articles for Purpura, Thrombocytopenic\n",
      "3792 articles for Purpura, Thrombotic Thrombocytopenic\n",
      "7154 articles for Quadriplegia\n",
      "393 articles for Hyperacusis\n",
      "4453 articles for Reflex, Abnormal\n",
      "1479 articles for Respiratory Paralysis\n",
      "7110 articles for Respiratory Sounds\n",
      "2744 articles for Restless Legs Syndrome\n",
      "4507 articles for Retinal Hemorrhage\n",
      "417 articles for Rubinstein-Taybi Syndrome\n",
      "4261 articles for Sciatica\n",
      "2517 articles for Scotoma\n",
      "42647 articles for Seizures\n",
      "4095 articles for Sensation Disorders\n",
      "0 articles for Signs and Symptoms, Digestive\n",
      "0 articles for Signs and Symptoms, Respiratory\n",
      "2723 articles for Skin Manifestations\n",
      "12373 articles for Sleep Apnea Syndromes\n",
      "7447 articles for Sleep Deprivation\n",
      "16262 articles for Sleep Disorders\n",
      "781 articles for Sneezing\n",
      "3366 articles for Snoring\n",
      "529 articles for Somnambulism\n",
      "6248 articles for Spasm\n",
      "10007 articles for Speech Disorders\n",
      "3113 articles for Stuttering\n",
      "1910 articles for Supranuclear Palsy, Progressive\n",
      "9376 articles for Syncope\n",
      "1377 articles for Taste Disorders\n",
      "2277 articles for Tetany\n",
      "3905 articles for Thinness\n",
      "1203 articles for Tinea Pedis\n",
      "6152 articles for Tinnitus\n",
      "2362 articles for Toothache\n",
      "3088 articles for Torticollis\n",
      "8147 articles for Tremor\n",
      "1300 articles for Trismus\n",
      "3711 articles for Unconsciousness\n",
      "18321 articles for Urinary Incontinence\n",
      "9276 articles for Urinary Incontinence, Stress\n",
      "8668 articles for Vertigo\n",
      "1893 articles for Virilism\n",
      "22639 articles for Vision Disorders\n",
      "1583 articles for Vitreous Hemorrhage\n",
      "5144 articles for Vocal Cord Paralysis\n",
      "4620 articles for Voice Disorders\n",
      "19894 articles for Vomiting\n",
      "221 articles for Vomiting, Anticipatory\n",
      "422 articles for Waterhouse-Friderichsen Syndrome\n",
      "349 articles for Wolfram Syndrome\n",
      "1908 articles for Hydrops Fetalis\n",
      "362 articles for Pyruvate Dehydrogenase Complex Deficiency Disease\n",
      "2343 articles for Vision, Low\n",
      "23709 articles for Weight Gain\n",
      "25680 articles for Weight Loss\n",
      "1947 articles for Rett Syndrome\n",
      "15002 articles for Abdominal Pain\n",
      "124 articles for Tonic Pupil\n",
      "265 articles for Anisocoria\n",
      "363 articles for Miosis\n",
      "514 articles for Mydriasis\n",
      "786 articles for Mucopolysaccharidosis II\n",
      "167 articles for Cardiac Output, High\n",
      "4732 articles for Purpura, Thrombocytopenic, Idiopathic\n",
      "762 articles for Hypocapnia\n",
      "1306 articles for Akathisia, Drug-Induced\n",
      "15043 articles for Low Back Pain\n",
      "441 articles for Ophthalmoplegia, Chronic Progressive External\n",
      "9826 articles for Pain Threshold\n",
      "897 articles for Microvascular Angina\n",
      "150 articles for Kleine-Levin Syndrome\n",
      "82 articles for WAGR Syndrome\n",
      "3734 articles for Pelvic Pain\n",
      "629 articles for Machado-Joseph Disease\n",
      "268 articles for Brown-Sequard Syndrome\n",
      "2572 articles for Persistent Vegetative State\n",
      "1046 articles for Hypokinesia\n",
      "189 articles for Space Motion Sickness\n",
      "2667 articles for Hyperoxia\n",
      "1228 articles for Gastroparesis\n",
      "22 articles for Sweating Sickness\n",
      "5442 articles for Arthralgia\n",
      "60 articles for Aphasia, Conduction\n",
      "409 articles for Aphasia, Primary Progressive\n",
      "11068 articles for Muscle Weakness\n",
      "1324 articles for Williams Syndrome\n",
      "318 articles for Cafe-au-Lait Spots\n",
      "1557 articles for Syncope, Vasovagal\n",
      "4803 articles for Neck Pain\n",
      "751 articles for Hemifacial Spasm\n",
      "457 articles for Blindness, Cortical\n",
      "2471 articles for Hot Flashes\n",
      "758 articles for Aging, Premature\n",
      "1678 articles for Pseudophakia\n",
      "148 articles for Schnitzler Syndrome\n",
      "129 articles for Neurobehavioral Manifestations\n",
      "3129 articles for Shoulder Pain\n",
      "498 articles for Neurogenic Inflammation\n",
      "20 articles for Chorea Gravidarum\n",
      "45 articles for Hypersomnolence, Idiopathic\n",
      "1410 articles for Sleep Disorders, Circadian Rhythm\n",
      "310 articles for Jet Lag Syndrome\n",
      "12292 articles for Sleep Apnea, Obstructive\n",
      "928 articles for Sleep Apnea, Central\n",
      "45 articles for Nocturnal Paroxysmal Dystonia\n",
      "112 articles for Night Terrors\n",
      "320 articles for Sleep Bruxism\n",
      "706 articles for REM Sleep Behavior Disorder\n",
      "103 articles for Sleep Paralysis\n",
      "493 articles for Nocturnal Myoclonus Syndrome\n",
      "94 articles for Coma, Post-Head Injury\n",
      "4298 articles for Gait Disorders, Neurologic\n",
      "394 articles for Gait Ataxia\n",
      "101 articles for Gait Apraxia\n",
      "316 articles for Amnesia, Transient Global\n",
      "75 articles for Alexia, Pure\n",
      "412 articles for Prosopagnosia\n",
      "179 articles for Apraxia, Ideomotor\n",
      "3000 articles for Postoperative Nausea and Vomiting\n",
      "225 articles for Alcohol Withdrawal Seizures\n",
      "671 articles for Tics\n",
      "221 articles for Amnesia, Anterograde\n",
      "608 articles for Paraparesis\n",
      "325 articles for Paraparesis, Spastic\n",
      "118 articles for Myokymia\n",
      "328 articles for Parasomnias\n",
      "1247 articles for Fetal Weight\n",
      "1610 articles for Spinocerebellar Ataxias\n",
      "245 articles for Amaurosis Fugax\n",
      "495 articles for Photophobia\n",
      "1384 articles for Dyskinesias\n",
      "95 articles for Pseudobulbar Palsy\n",
      "0 articles for Neuromuscular Manifestations\n",
      "991 articles for Somatosensory Disorders\n",
      "348 articles for Korsakoff Syndrome\n",
      "223 articles for Sleep Disorders, Intrinsic\n",
      "293 articles for Dyssomnias\n",
      "129 articles for Sleep Arousal Disorders\n",
      "123 articles for Sleep-Wake Transition Disorders\n",
      "73 articles for REM Sleep Parasomnias\n",
      "0 articles for Urological Manifestations\n",
      "428 articles for Flank Pain\n",
      "156 articles for Chills\n",
      "123 articles for Insomnia, Fatal Familial\n",
      "9079 articles for Hearing Loss\n",
      "142 articles for Metatarsalgia\n",
      "510 articles for Mental Retardation, X-Linked\n",
      "72 articles for Coffin-Lowry Syndrome\n",
      "2706 articles for Jaundice, Obstructive\n",
      "44 articles for Reticulocytosis\n",
      "436 articles for Hearing Loss, Unilateral\n",
      "222 articles for Hearing Loss, Mixed Conductive-Sensorineural\n",
      "110 articles for Synkinesis\n",
      "656 articles for Labor Pain\n",
      "116 articles for Morning Sickness\n",
      "13641 articles for Overweight\n",
      "2618 articles for Mobility Limitation\n",
      "687 articles for Neuralgia, Postherpetic\n",
      "91 articles for Glycogen Storage Disease Type IIb\n",
      "299 articles for Usher Syndromes\n",
      "475 articles for Nocturia\n",
      "224 articles for Dysuria\n",
      "2760 articles for Urinary Bladder, Overactive\n",
      "622 articles for Urinary Incontinence, Urge\n",
      "423 articles for Prostatism\n",
      "463 articles for Hypercalciuria\n",
      "166 articles for Opsoclonus-Myoclonus Syndrome\n",
      "74 articles for Urinoma\n",
      "231 articles for Pain, Referred\n",
      "97 articles for Stupor\n",
      "226 articles for Lethargy\n",
      "8724 articles for Acute Coronary Syndrome\n",
      "66 articles for Deaf-Blind Disorders\n",
      "165 articles for Livedo Reticularis\n",
      "121 articles for Mevalonate Kinase Deficiency\n",
      "48 articles for Systolic Murmurs\n",
      "99 articles for Classical Lissencephalies and Subcortical Band Heterotopias\n",
      "102 articles for Neuroacanthocytosis\n",
      "188 articles for Orthostatic Intolerance\n",
      "209 articles for Postural Orthostatic Tachycardia Syndrome\n",
      "154 articles for Failed Back Surgery Syndrome\n",
      "770 articles for Dysphonia\n",
      "136 articles for Purpura Fulminans\n",
      "1103 articles for Sarcopenia\n",
      "93 articles for Susac Syndrome\n",
      "57 articles for Piriformis Muscle Syndrome\n",
      "35 articles for Alien Hand Syndrome\n",
      "21 articles for Slit Ventricle Syndrome\n",
      "1770 articles for Obesity, Abdominal\n",
      "185 articles for Renal Colic\n",
      "166 articles for Ideal Body Weight\n",
      "65 articles for Primary Progressive Nonfluent Aphasia\n",
      "16 articles for Infantile Apparent Life-Threatening Event\n",
      "27 articles for Post-Exercise Hypotension\n",
      "67 articles for Striae Distensae\n",
      "293 articles for Eye Pain\n",
      "29 articles for Necrolytic Migratory Erythema\n",
      "280 articles for Nociceptive Pain\n",
      "32 articles for Transient Tachypnea of the Newborn\n",
      "65 articles for Tachypnea\n",
      "222 articles for Visceral Pain\n",
      "4756 articles for Chronic Pain\n",
      "1127 articles for Musculoskeletal Pain\n",
      "56 articles for Mastodynia\n",
      "46 articles for Pelvic Girdle Pain\n",
      "146 articles for Breakthrough Pain\n",
      "997 articles for Lower Urinary Tract Symptoms\n",
      "300 articles for Anhedonia\n",
      "65 articles for Polydipsia\n",
      "20 articles for Polydipsia, Psychogenic\n",
      "775 articles for Acute Pain\n",
      "542 articles for Angina, Stable\n",
      "18 articles for Ophthalmoplegic Migraine\n",
      "38 articles for Pudendal Neuralgia\n",
      "83 articles for Dyscalculia\n",
      "13 articles for Alice in Wonderland Syndrome\n",
      "441 articles for Prodromal Symptoms\n",
      "1354 articles for Pediatric Obesity\n",
      "327 articles for Myalgia\n",
      "12 articles for Hypertriglyceridemic Waist\n",
      "388 articles for Cerebrospinal Fluid Leak\n",
      "296 articles for Benign Paroxysmal Positional Vertigo\n",
      "15 articles for Hyperlactatemia\n",
      "0 articles for Allesthesia\n"
     ]
    }
   ],
   "source": [
    "rows_out = list()\n",
    "\n",
    "for i, row in symptom_df.iterrows():\n",
    "    term_query = '{symptom}[MeSH Terms:noexp]'.format(symptom = row.mesh_name.lower())\n",
    "    payload = {'db': 'pubmed', 'term': term_query}\n",
    "    pmids = eutility.esearch_query(payload, retmax = 5000, sleep=2)\n",
    "    row['term_query'] = term_query\n",
    "    row['n_articles'] = len(pmids)\n",
    "    row['pubmed_ids'] = '|'.join(pmids)\n",
    "    rows_out.append(row)\n",
    "    print('{} articles for {}'.format(len(pmids), row.mesh_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mesh_id</th>\n",
       "      <th>mesh_name</th>\n",
       "      <th>in_hsdn</th>\n",
       "      <th>term_query</th>\n",
       "      <th>n_articles</th>\n",
       "      <th>pubmed_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>D000006</td>\n",
       "      <td>Abdomen, Acute</td>\n",
       "      <td>1</td>\n",
       "      <td>abdomen, acute[MeSH Terms:noexp]</td>\n",
       "      <td>8496</td>\n",
       "      <td>25742249|25669229|25650451|25619050|25608417|2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>D000270</td>\n",
       "      <td>Adie Syndrome</td>\n",
       "      <td>0</td>\n",
       "      <td>adie syndrome[MeSH Terms:noexp]</td>\n",
       "      <td>313</td>\n",
       "      <td>25138821|24995781|24625775|24533698|24215593|2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>D000326</td>\n",
       "      <td>Adrenoleukodystrophy</td>\n",
       "      <td>0</td>\n",
       "      <td>adrenoleukodystrophy[MeSH Terms:noexp]</td>\n",
       "      <td>1508</td>\n",
       "      <td>25860611|25583825|25393703|25378668|25297370|2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>D000334</td>\n",
       "      <td>Aerophagy</td>\n",
       "      <td>1</td>\n",
       "      <td>aerophagy[MeSH Terms:noexp]</td>\n",
       "      <td>261</td>\n",
       "      <td>25073665|24796405|24280810|23772202|23772201|2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>D000370</td>\n",
       "      <td>Ageusia</td>\n",
       "      <td>1</td>\n",
       "      <td>ageusia[MeSH Terms:noexp]</td>\n",
       "      <td>222</td>\n",
       "      <td>24999669|24999665|24825557|24782205|24191925|2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mesh_id             mesh_name  in_hsdn  \\\n",
       "0  D000006        Abdomen, Acute        1   \n",
       "1  D000270         Adie Syndrome        0   \n",
       "2  D000326  Adrenoleukodystrophy        0   \n",
       "3  D000334             Aerophagy        1   \n",
       "4  D000370               Ageusia        1   \n",
       "\n",
       "                               term_query  n_articles  \\\n",
       "0        abdomen, acute[MeSH Terms:noexp]        8496   \n",
       "1         adie syndrome[MeSH Terms:noexp]         313   \n",
       "2  adrenoleukodystrophy[MeSH Terms:noexp]        1508   \n",
       "3             aerophagy[MeSH Terms:noexp]         261   \n",
       "4               ageusia[MeSH Terms:noexp]         222   \n",
       "\n",
       "                                          pubmed_ids  \n",
       "0  25742249|25669229|25650451|25619050|25608417|2...  \n",
       "1  25138821|24995781|24625775|24533698|24215593|2...  \n",
       "2  25860611|25583825|25393703|25378668|25297370|2...  \n",
       "3  25073665|24796405|24280810|23772202|23772201|2...  \n",
       "4  24999669|24999665|24825557|24782205|24191925|2...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "symptom_pmids_df = pandas.DataFrame(rows_out)\n",
    "\n",
    "with gzip.open('data/symptom-pmids.tsv.gz', 'w') as write_file:\n",
    "    write_file = io.TextIOWrapper(write_file)\n",
    "    symptom_pmids_df.to_csv(write_file, sep='\\t', index=False)\n",
    "\n",
    "symptom_pmids_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cooccurrence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "symptom_df, symptom_to_pmids = cooccurrence.read_pmids_tsv('data/symptom-pmids.tsv.gz', key='mesh_id')\n",
    "disease_df, disease_to_pmids = cooccurrence.read_pmids_tsv('data/disease-pmids.tsv.gz', key='doid_code')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1759475"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "symptom_pmids = set.union(*symptom_to_pmids.values())\n",
    "len(symptom_pmids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3478558"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disease_pmids = set.union(*disease_to_pmids.values())\n",
    "len(disease_pmids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total articles containing a doid_code: 3478558\n",
      "Total articles containing a mesh_id: 1759475\n",
      "Total articles containing both a doid_code and mesh_id: 363928\n",
      "\n",
      "After removing terms without any cooccurences:\n",
      "+ 133 doid_codes remain\n",
      "+ 426 mesh_ids remain\n",
      "\n",
      "Cooccurrence scores calculated for 56658 doid_code -- mesh_id pairs\n"
     ]
    }
   ],
   "source": [
    "cooc_df = cooccurrence.score_pmid_cooccurrence(disease_to_pmids, symptom_to_pmids, 'doid_code', 'mesh_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_code</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>mesh_id</th>\n",
       "      <th>mesh_name</th>\n",
       "      <th>cooccurrence</th>\n",
       "      <th>expected</th>\n",
       "      <th>enrichment</th>\n",
       "      <th>odds_ratio</th>\n",
       "      <th>p_fisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30318</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>D004314</td>\n",
       "      <td>Down Syndrome</td>\n",
       "      <td>800</td>\n",
       "      <td>35.619601</td>\n",
       "      <td>22.459544</td>\n",
       "      <td>39.918352</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30408</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>D008569</td>\n",
       "      <td>Memory Disorders</td>\n",
       "      <td>1593</td>\n",
       "      <td>76.580532</td>\n",
       "      <td>20.801631</td>\n",
       "      <td>41.885877</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30452</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>D011595</td>\n",
       "      <td>Psychomotor Agitation</td>\n",
       "      <td>334</td>\n",
       "      <td>15.235665</td>\n",
       "      <td>21.922247</td>\n",
       "      <td>35.277329</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30257</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>D000647</td>\n",
       "      <td>Amnesia</td>\n",
       "      <td>307</td>\n",
       "      <td>14.061215</td>\n",
       "      <td>21.833106</td>\n",
       "      <td>34.890099</td>\n",
       "      <td>4.277452e-314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30381</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>D006816</td>\n",
       "      <td>Huntington Disease</td>\n",
       "      <td>255</td>\n",
       "      <td>12.130614</td>\n",
       "      <td>21.021195</td>\n",
       "      <td>32.630035</td>\n",
       "      <td>8.215868e-256</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        doid_code            doid_name  mesh_id              mesh_name  \\\n",
       "30318  DOID:10652  Alzheimer's disease  D004314          Down Syndrome   \n",
       "30408  DOID:10652  Alzheimer's disease  D008569       Memory Disorders   \n",
       "30452  DOID:10652  Alzheimer's disease  D011595  Psychomotor Agitation   \n",
       "30257  DOID:10652  Alzheimer's disease  D000647                Amnesia   \n",
       "30381  DOID:10652  Alzheimer's disease  D006816     Huntington Disease   \n",
       "\n",
       "       cooccurrence   expected  enrichment  odds_ratio       p_fisher  \n",
       "30318           800  35.619601   22.459544   39.918352   0.000000e+00  \n",
       "30408          1593  76.580532   20.801631   41.885877   0.000000e+00  \n",
       "30452           334  15.235665   21.922247   35.277329   0.000000e+00  \n",
       "30257           307  14.061215   21.833106   34.890099  4.277452e-314  \n",
       "30381           255  12.130614   21.021195   32.630035  8.215868e-256  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cooc_df = symptom_df[['mesh_id', 'mesh_name']].drop_duplicates().merge(cooc_df)\n",
    "cooc_df = disease_df[['doid_code', 'doid_name']].drop_duplicates().merge(cooc_df)\n",
    "cooc_df = cooc_df.sort_values(by=['doid_name', 'p_fisher'])\n",
    "cooc_df.to_csv('data/disease-symptom-cooccurrence.tsv', index=False, sep='\\t')\n",
    "cooc_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "import scipy\n",
    "import seaborn\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAECCAYAAADq7fyyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFGxJREFUeJzt3X+MZWV9x/H3guwuyw6j0FmwZVlSlG82bbBC1ZYafqQK\n0h/SmCZNY20lFSQSXJMKqatd+2Mpptu1EX+lsipqbCUQadcSFUsN4MZIaa2Vil9BdHdICI4z4+wO\nyzq7O9M/7h2ZZe6P+XHnnnuffb+SDXOfc+bcLzN3Pvfc5znneVbNzMwgSSrHCVUXIEnqLINdkgpj\nsEtSYQx2SSqMwS5JhTHYJakwL2i1MSJOBG4DzgNmgOuAnwK3A9PAI8D1mTkTEdcA1wJHgO2Zec8K\n1i1JaqLdGfvvANOZ+WrgPcDfAjuBrZl5MbAKuCoizgRuAC4CrgBuiYjVK1e2JKmZlsGemf8KvLX+\n8BxgHLgwMx+ot30ReA3wCmBPZh7OzP3A48D5K1KxJKmltn3smXk0Im4HPgB8ltpZ+qwDwCBwKjDR\noF2S1GULGjzNzDcDAewC1s7ZdCrwE2A/MDCnfYDa2b0kqcvaDZ6+CTgrM28BngWOAg9HxCWZeT9w\nJXAf8BBwc0SsoRb8m6kNrDY1MzMzs2rVqla7SJLmaxucq1pNAhYRJ1O7AuZM4CTgFuC71K6UWQ18\nB7imflXMW6hdFXMCcHNm3t3muWdGRg4s4P+hWkNDA1hn51hn5/RDjWCdnTY0NNA22FuesWfms8Af\nNNh0aYN9d1HrqpEkVcgblCSpMAa7JBXGYJekwrTsY9d8U1NTDA/vbbp948ZNrF7tTbeSqmOwL9Lw\n8F627NjNusEN87YdnPgRH7jx9Zx77ksrqEySagz2JVg3uIH1L/qFqsuQpIbsY5ekwhjsklQYu2Ka\nmB0kHR9fz9jY5M/a9+1rPnAqSb3AYG+i2SDp6JOPcvpZmyuqSpLaM9hbaDRIenDi6YqqkaSFsY9d\nkgpjsEtSYeyK6YJWd6t6p6qkTjPYu6DZQKx3qkpaCQZ7l3i3qqRusY9dkgpjsEtSYQx2SSqMwS5J\nhTHYJakwBrskFcZgl6TCGOySVBiDXZIKY7BLUmEMdkkqjMEuSYVxErAOmj56pOGaqK6TKqmbWgZ7\nRJwEfALYBKwBtgNPAv8GfK++20cy886IuAa4FjgCbM/Me1as6h51aHKUnXeMsW7wqWPaXSdVUje1\nO2N/IzCSmW+KiBcB3wL+CtiZme+f3SkizgRuAC4ETga+FhFfycypFaq7Z7lOqqSqtQv2O4G76l+f\nABymFt4REVcBjwHvAF4J7MnMw8DhiHgcOB94eEWqliQ11XLwNDOfyczJiBigFvLvBh4C3pmZlwBP\nAO8FBoCJOd96ABhcmZIlSa20vSomIjYC/wF8OjM/B9ydmd+sb74beDmwn1q4zxoAxjtcqyRpAdoN\nnp4B3Au8LTO/Wm/+UkS8PTP/E3gNte6Wh4CbI2INsBbYDDzS7smHhgba7VKZ8fH1K/4c00ePMDEx\n0vS5zjnnnEUtdN3LP8+5rLNz+qFGsM5ua9fHvpVal8q2iNhWb3sH8A8RcRh4Cri23l1zK/AgtU8B\nWxcycDoycmDpla+wsbHJFX+OQ5OjbPvY11k3+P152ybHn+LGP7yAs8/eNG/bxo2b5gX+0NBAT/88\nZ1ln5/RDjWCdnbaQN5+WwZ6ZW4AtDTa9usG+u4BdCy1ONc0WuT448TQ77/jWvEsnD078iA/c+HrO\nPfel3SpRUp/xBqUe1iz0JamV4z7Yp6amGB72blFJ5Tjug314eC9bduxm3eCGY9q9W1RSvzrugx28\nW1RSWZzdUZIKY7BLUmEMdkkqjMEuSYUx2CWpMAa7JBXGYJekwhjsklQYg12SCmOwS1JhDHZJKozB\nLkmFcRKwPjN99EjDKYXHx9dzyimnL2opPUllMtj7zKHJUXbeMebKSpKaMtj7kCsrSWrFPnZJKozB\nLkmFMdglqTD2sRei2dUyszZu3OQVM9Jx4rgI9qmpKYaHG4deqzDsJ82ulgGvmJGON8dFsA8P72XL\njt2sG9wwb9vok49y+lmbK6iq87xaRhIcJ8EOzUPv4MTTFVQjSSvHwVNJKozBLkmFMdglqTAGuyQV\npuXgaUScBHwC2ASsAbYDjwK3A9PAI8D1mTkTEdcA1wJHgO2Zec8K1i1JaqLdGfsbgZHMvBh4HfBh\nYCewtd62CrgqIs4EbgAuAq4AbokI74aRpAq0u9zxTuCu+tcnAIeBCzLzgXrbF4HLgaPAnsw8DByO\niMeB84GHO1+yJKmVlsGemc8ARMQAtZB/D/D3c3Y5AAwCpwITDdolSV3W9galiNgIfB74cGb+c0T8\n3ZzNpwI/AfYDA3PaB4DxdsceGhpot0tHjI+v78rz9Krpo0eYmBhp+HM455xzuj6HTLd+78vVD3X2\nQ41gnd3WbvD0DOBe4G2Z+dV68zcj4pLMvB+4ErgPeAi4OSLWAGuBzdQGVlsaGTmwnNoXbGxssivP\n06sOTY6y7WNfZ93g949pr2IOmaGhga793pejH+rshxrBOjttIW8+7c7Yt1LrUtkWEdvqbVuAW+uD\no98B7qpfFXMr8CC1vvitmTm15MrVcc4jIx0/2vWxb6EW5M93aYN9dwG7OlOWJGmpvEFJkgpjsEtS\nYQx2SSqMwS5JhTHYJakwBrskFcZgl6TCGOySVBiDXZIKY7BLUmEMdkkqjMEuSYUx2CWpMAa7JBWm\n7QpK/WRqaorh4b3z2vftm98mSaUqKtiHh/eyZcdu1g1uOKZ99MlHOf2szRVVJUndVVSwQ+OVgg5O\nPF1RNZLUffaxS1Jhijtj18JNHz3SdPxh48ZNrF69ussVSeoEg/04dmhylJ13jLFu8Klj2g9O/IgP\n3Ph6zj33pRVVJmk5DPbjXKMxiVZn8uDZvNTrDHbN0+xMHjybl/qBwa6GGp3JS+oPXhUjSYUx2CWp\nMAa7JBXGYJekwhjsklQYg12SCrOgyx0j4lXA+zLzsoh4OfAF4LH65o9k5p0RcQ1wLXAE2J6Z96xI\nxZKkltoGe0TcBPwRMFlvuhB4f2a+f84+ZwI31LedDHwtIr6SmVOdL1mS1MpCztgfB94AfKb++ELg\nvIi4itpZ+zuAVwJ7MvMwcDgiHgfOBx7ufMmSpFba9rFn5uepda/M+gbwzsy8BHgCeC8wAEzM2ecA\nMNjBOiVJC7SUKQXuzszZEL8b+CDwALVwnzUAjLc70NDQQLtdFmV8fH1Hj6fGTjtt/bJ+d53+va+U\nfqizH2oE6+y2pQT7lyLi7Zn5n8BrqHW3PATcHBFrgLXAZuCRdgcaGTmwhKdvbmxssv1OWraxsckl\n/+6GhgY6/ntfCf1QZz/UCNbZaQt581lMsM/U/3sd8OGIOAw8BVybmZMRcSvwILXuna0OnEpSNRYU\n7Jn5Q+Ci+tffAl7dYJ9dwK5OFidJWjxvUJKkwhjsklQYg12SCmOwS1JhXBpPK2pqaorh4ecWxh4f\nX3/MZakujC11nsGuFTU8vJctO3azbnDDvG0ujC2tDINdK86FsaXuso9dkgpjsEtSYQx2SSqMfezq\niOdf/TJr3775bZJWlsGujmh29cvok49y+lmbK6pKOj4Z7OqYRle/HJx4uqJqpOOXfeySVBiDXZIK\nY1eMFmX66JGGA6IOkkq9w2DXohyaHGXnHWOsG3zqmHYHSaXeYbBr0RwklXqbfeySVBiDXZIKY1eM\nKtNsIBacp11aDoNdlWk2EOs87dLyGOyqlHO1S51nH7skFcZgl6TCGOySVBiDXZIKY7BLUmEMdkkq\nzIIud4yIVwHvy8zLIuIlwO3ANPAIcH1mzkTENcC1wBFge2bes0I1S5JaaHvGHhE3AbcBa+pN7we2\nZubFwCrgqog4E7gBuAi4ArglIrxtUJIqsJCumMeBN1ALcYALMvOB+tdfBF4DvALYk5mHM3N//XvO\n73SxkqT22gZ7Zn6eWvfKrFVzvj4ADAKnAhMN2iVJXbaUKQWm53x9KvATYD8wMKd9ABhvd6ChoYF2\nuyzK+Pj6jh5P1TnttPUdf30sVtXPvxD9UCNYZ7ctJdi/GRGXZOb9wJXAfcBDwM0RsQZYC2ymNrDa\n0sjIgSU8fXNjY5MdPZ6qMzY22fHXx2IMDQ1U+vwL0Q81gnV22kLefBYT7DP1//4ZcFt9cPQ7wF31\nq2JuBR6k1r2zNTOnFlnvgk1NTTE87LqbktTIgoI9M39I7YoXMvMx4NIG++wCdnWwtqaGh/eyZcdu\n1g1uOKbddTclqY+n7XXdTUlqrG+DXeVqtbISuLqS1I7Brp7TbGUlcHUlaSEMdvUkV1aSls5JwCSp\nMAa7JBXGYJekwhjsklQYg12SCmOwS1JhDHZJKozBLkmFMdglqTAGuyQVxmCXpMIY7JJUGINdkgpj\nsEtSYZy2V32l1SIcLsAh1Rjs6ivNFuFwAQ7pOQa7+o6LcEit2ccuSYXxjF1FsO9deo7BriLY9y49\nx2BXMex7l2rsY5ekwnjGrqK16nsH+99VJoNdRWvW9w72v6tcBruKZ9+7jjdLDvaI+G9gov7wCeAW\n4HZgGngEuD4zZ5ZboCRpcZYU7BGxFiAzL5vTthvYmpkPRMRHgauAf+lIlZKkBVvqGfvLgHUR8eX6\nMd4NXJCZD9S3fxG4HINdkrpuqZc7PgPsyMwrgOuAzz5v+yQwuJzCJElLs9Rg/x71MM/Mx4BR4Iw5\n2weAnyyvNEnSUiy1K+Zq4Hzg+oj4eWpBfm9EXJKZ9wNXAve1O8jQ0MCSnnx8fP2Svk96vtNOW9/0\ndbjU12c39UONYJ3dttRg/zjwyYiY7VO/mtpZ+20RsRr4DnBXu4OMjBxY0pOPjU0u6fuk5xsbm2z4\nOhwaGljy67Nb+qFGsM5OW8ibz5KCPTOPAG9qsOnSpRyvkampKYaHG98x2OpOQmmhWt2VOjj4y12u\nRuqcnr1BaXh4L1t27Gbd4IZ520affJTTz9pcQVUqSasZIT9zy3pe9KIXV1SZtDw9G+zQ/I7BgxNP\nV1CNSuRdqSpRTwe71EtadQ+CE4qpdxjs0gK16h50QjH1EoNdWgS7btQPDHbpeaaPHuEHP/jBvMtq\nvRpL/cJgl57n0OQo2z729XldLl6NpX5hsEsNNOpy8Wos9QvXPJWkwhjsklQYg12SCmMfu9QBread\n8cYldZvBLnVAq3lnvHFJ3WawSx3izUvqFQa7tILsolEVDHZpBdlFoyoY7NIKa9RF0+pMHjyb1/IY\n7FIFmp3Jg2fzWj6DXaqIg61aKd6gJEmFMdglqTB2xUg9xksktVwGu9Rjmg2sTo4/xY1/eAFnn70J\ngPHx9T9bDMTA11yVB/uX//2rfOUbj81r3z8+AieeXUFFUvWazQe/845veU282qo82EdGxxg78SXz\n2idPPLmCaqTe5pU0WojKg13SypmammJ42P76443BLhVseHgvW3bsnrd+q903ZTPYpT7X6iqaffv2\nLnpKA8/k+5/BLvW5VtMTjD75KKeftXnB3+OZfBk6GuwRcQLwEeB84KfAWzLz+518DknzNRtUPTjx\n9KK/R/2v02fsvweszsyLIuJVwM56m6Q+0MlZJ2cHbudeb7/UY2lxOh3svwF8CSAzvxERv9rh40ta\nQZ2cdbLZwC3Mv9lqLgN/+Tod7KcC++c8PhoRJ2TmdIefR9IKadZF0+xs/vDhwwCcdNJJx7Q3G7iF\n7txs1epSz0Y1z36y6NQbS5WXmnY62PcDA3Metw311S84kenRb89rn574MYdOeGHD73n2wBiwatnt\nHqu/jlX18x/vxxp/6jG23/Zd1q4/7Zj2iaefYM0pL2zY/sIXn9e0rpMHTm+4rVVX0GLs27eX7bd9\nZV5ds7U1qvnQ5Bjvuea1DT9JdOr5D02O8bG/ecuKDlCvmpmZ6djBIuINwO9m5tUR8WvAX2Tmb3fs\nCSRJbXX6jP1u4LURsaf++OoOH1+S1EZHz9glSdVzoQ1JKozBLkmFMdglqTAGuyQVpuuTgPXbfDL1\nqRHel5mXVV1LIxFxEvAJYBOwBtiemV+otqpjRcSJwG3AecAMcF1m/l+1VTUXERuA/wJ+MzO/V3U9\njUTEfwMT9YdPZOafVllPMxHxLuB3gZOAD2XmpyouaZ6I+BPgzfWHJwMvA87IzP1Nv6kC9ezcRe3v\naBq4JjOz0b5VnLH/bD4Z4M+pzSfTkyLiJmqBtKbqWlp4IzCSmRcDrwM+VHE9jfwOMJ2ZrwbeA9xc\ncT1N1d8o/xF4pupamomItQCZeVn9X6+G+qXAr9f/1i8FfrHSgprIzE/N/iyBh4Ebei3U6y4HTqn/\nHf01Lf6Oqgj2Y+aTAXp5PpnHgTfQ7Fa83nAnsK3+9QnAkQpraSgz/xV4a/3hOcB4ddW0tQP4KDB/\nspTe8TJgXUR8OSLuq3+q7EWXA9+OiH8BvgDsrrielupzW/1SZu6qupYmngUGI2IVMAhMNduximBv\nOJ9MBXW0lZmfpweDcq7MfCYzJyNigFrIv7vqmhrJzKMRcTtwK/BPFZfTUES8mdqnn3vrTb36hv4M\nsCMzrwCuAz7bo39DQ8CFwO9Tr7PactraCvxl1UW0sAdYC3yX2qfKDzbbsYoXw6Lnk1FrEbER+A/g\n05n5uarraSYz30ytf/C2iOjF1cqvpnbn9FeBXwE+FRFnVFxTI9+jHpKZ+RgwCry40ooa+zFwb2Ye\nqY9VHIqIn6u6qEYi4oXAeZl5f9W1tHATsCczg+denw1nEqsi2PcAvwVQn0/mfyuooRj14LkXuCkz\nb6+4nIYi4k31QTSofZycrv/rKZl5SWZeWu9r/R/gjzOz+UoV1bma+thURPw8tU/Bvdh19DVq4z6z\ndZ5C7U2oF10M3Fd1EW2cwnO9HePUBqRPbLRjFUvj9eN8Mr0878JWav1t2yJitq/9ysw8VGFNz3cX\ncHtE3E/txbglM39acU397OPAJyPigfrjq3vxU29m3hMRF0fEQ9ROIt+Wmb36t3Qe0LNX59XtoPZ7\nf5Da39G7MvPZRjs6V4wkFaYXB1wkSctgsEtSYQx2SSqMwS5JhTHYJakwBrskFcZgl6TCGOySVJj/\nBwG0X6gKM3flAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe730979ac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sig_df = cooc_df[cooc_df.p_fisher < 0.05]\n",
    "plt.hist(list(numpy.log(sig_df.enrichment)), bins = 50);"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}