Annotation Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import csv\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Open annotations file "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Open the 'annotations.txt' file and read in any lines starting with 'text_'. This indicates a line with phenotypes for the patients' text file of notes. From this we can extract the patient ID for the given phenotypes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"annotations.txt\",\"r\") as f:\n",
    "    annotations = []\n",
    "    for ln in f:\n",
    "        if ln.startswith(\"text_\"):\n",
    "            annotations.append(ln[5:])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning up the lines read in from annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pheno = [] # will fill nested list with the information from each line in the file\n",
    "\n",
    "for ln in range(len(annotations)):\n",
    "        \n",
    "    line = re.split(r'\\t+', annotations[ln]) # split lines by \\t delimiter\n",
    "    \n",
    "    line_strip = [] \n",
    "        \n",
    "    for i in line:         \n",
    "        # strip away unwanted characters that got retained when reading in the file         \n",
    "        strip1 = i.strip('.txt')\n",
    "        strip2 = strip1.strip('http://purl.obolibrary.org/obo/')\n",
    "        strip = strip2.strip('\\n')        \n",
    "        line_strip.append(strip) \n",
    "        \n",
    "    pheno.append(line_strip) # each list is one line from the file\n",
    "    \n",
    "\n",
    "for i in pheno:\n",
    "    \n",
    "    if len(i) != 8:\n",
    "        \n",
    "        i.insert(5, 'NA') \n",
    "        # ensure all lines have the same number of entries; position 6\n",
    "        # is variably filled so if not, add in 'NA'\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Formatting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotations_df = pd.DataFrame(pheno) # create dataframe from pheno \n",
    "\n",
    "phenotypes = annotations_df.iloc[:,0:2] # get first 3 columns (patient ID and Human Phenotype Ontology (HP) ID)\n",
    "\n",
    "phenotypes = phenotypes.set_axis(['Patient_ID', 'HP_ID'], axis=1, inplace=False) # set column names for use later\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenotypes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tidy up the Human Phenotype ontology (HP) identifiers to HP:0100806 ...\n",
    "\n",
    "phenotypes['HP_ID'] = [x.replace('_',':') for x in phenotypes['HP_ID']]\n",
    "phenotypes = phenotypes.set_index(phenotypes.loc[:,'Patient_ID'], drop=True)\n",
    "\n",
    "# iterate through the unique patient IDs in the index of phenotypes dataframe \n",
    "# saving each set of HP IDs for that patient and formatting to a ; delimited list in new dataframe \n",
    "\n",
    "phenos = pd.DataFrame(columns= ['Patient_ID', 'HP_IDs']) # new dataframe for cleaned IDs \n",
    "\n",
    "for i in list(dict.fromkeys(phenotypes.index.tolist())):\n",
    "    \n",
    "    l = phenotypes.loc[i, 'HP_ID']\n",
    "    \n",
    "    if isinstance(l, str):\n",
    "        s = str(l)\n",
    "    else:\n",
    "        s = str(l.tolist())\n",
    "    \n",
    "    s = s.replace(',', ';').replace('[', '').replace(']', '').replace(\"'\", '').replace(' ', '')\n",
    "\n",
    "    phenos.append({'Patient_ID': i}, ignore_index=True)\n",
    "    phenos.loc[i, 'Patient_ID'] = i\n",
    "    phenos.loc[i, 'HP_IDs'] = s\n",
    "\n",
    "    \n",
    "# tidy up dataframe; set index as patient IDs and drop this column\n",
    "phenos = phenos.set_index('Patient_ID')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenos.shape\n",
    "\n",
    "# note: there may be fewer than 1000 patients here, as some patients in the sample (although exisiting in NOTEEVENTS) \n",
    "# do not actually have notes that can be annotated\n",
    "\n",
    "# if using updated code for your own sample, this should not be an issue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenos.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save as tab delimited file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This should be in the right format for semantic measures library https://www.semantic-measures-library.org/sml/index.php?q=toolkit "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all patients in sample\n",
    "\n",
    "phenos.to_csv('annotated_phenotypes.tsv', sep='\\t', header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first 10 for practice \n",
    "\n",
    "phenos.iloc[0:10,:].to_csv('ten_annotated_phenotypes.tsv', sep='\\t', header=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate patient pairs\n",
    "\n",
    "Want to generate all possible patient pairs so can make similarity comparisons across all patients. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For sample of 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ten_patients = phenos.iloc[0:10,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pairs_from_dataframe(dataframe):\n",
    "    \n",
    "    ids = dataframe.index.tolist()\n",
    "    \n",
    "    pairs = list(itertools.combinations(ids, 2))\n",
    "    \n",
    "    pairs_df = pd.DataFrame.from_records(pairs)\n",
    "    \n",
    "    pairs_df = pairs_df.set_index(pairs_df.iloc[:,0], drop=True)\n",
    "    \n",
    "    pairs_df = pairs_df.drop(0, axis=1)\n",
    "    \n",
    "    return pairs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ten_pairs_df = pairs_from_dataframe(ten_patients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ten_pairs_df.to_csv('ten_patient_pairs.tsv', sep='\\t', header=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For all patients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pairs_df = pairs_from_dataframe(phenos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pairs_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pairs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pairs_df.to_csv('all_patient_pairs.tsv', sep='\\t', header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}