notebooks/snmi_vocab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vocabulary Generator from SNMI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in /home/carlos/anaconda3/envs/spacy/lib/python3.6/site-packages (from -r requirements.txt (line 1)) (1.0.3)\n",
      "Requirement already satisfied: pytz>=2017.2 in /home/carlos/anaconda3/envs/spacy/lib/python3.6/site-packages (from pandas->-r requirements.txt (line 1)) (2019.3)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /home/carlos/.local/lib/python3.6/site-packages (from pandas->-r requirements.txt (line 1)) (2.8.1)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /home/carlos/.local/lib/python3.6/site-packages (from pandas->-r requirements.txt (line 1)) (1.18.2)\n",
      "Requirement already satisfied: six>=1.5 in /home/carlos/.local/lib/python3.6/site-packages (from python-dateutil>=2.6.1->pandas->-r requirements.txt (line 1)) (1.14.0)\n",
      "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.1 is available.\n",
      "You should consider upgrading via the '/home/carlos/anaconda3/envs/spacy/bin/python3.6 -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Install requirements\n",
    "! pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing libraries\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load CSV from Bioportal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download the latest from the [SNMI website](https://bioportal.bioontology.org/ontologies/SNMI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "snmi = pd.read_csv(\"SNMI.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class ID</th>\n",
       "      <th>Preferred Label</th>\n",
       "      <th>Synonyms</th>\n",
       "      <th>Definitions</th>\n",
       "      <th>Obsolete</th>\n",
       "      <th>CUI</th>\n",
       "      <th>Semantic Types</th>\n",
       "      <th>Parents</th>\n",
       "      <th>Associated with</th>\n",
       "      <th>EZ</th>\n",
       "      <th>Has location</th>\n",
       "      <th>Location of</th>\n",
       "      <th>SB</th>\n",
       "      <th>Semantic type UMLS property</th>\n",
       "      <th>SHF</th>\n",
       "      <th>SIC</th>\n",
       "      <th>SMX</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/C-140A7</td>\n",
       "      <td>235m Uranium</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0303352</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T196|...</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/C-14000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T196|...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/T-A1950</td>\n",
       "      <td>Choroid plexus of fourth ventricle</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0152293</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T023</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/T-A1900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C71.7</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/L-61420</td>\n",
       "      <td>Rasahus, NOS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0322617</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T204</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/L-61300</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T204</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/DC-1...</td>\n",
       "      <td>HNSHA due to gamma glutamyl cysteine synthetas...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0272071</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T047</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/DC-1...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T047</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/P1-A...</td>\n",
       "      <td>Repair of perforating laceration of sclera wit...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0197629</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T061</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/P1-A...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T061</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/F-671B0</td>\n",
       "      <td>beta-Carotene 15,15'-dioxygenase</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0053397</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T126|...</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/F-66100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.13.11.21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T126|...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/C-D4163</td>\n",
       "      <td>MITABAN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0702074</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T109|...</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/C-D0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T109|...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(X-10186)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/J-43230</td>\n",
       "      <td>Manufacturers' agent</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0335279</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T097</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/J-43200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T097</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/C-D2165</td>\n",
       "      <td>CLOSTRIDIAL 7-WAY PLUS SOMNUMUNE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0308688</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T116|...</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/C-D0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T116|...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(X-20153)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/F-662E8</td>\n",
       "      <td>2-Hydroxy-3-oxopropionate reductase</td>\n",
       "      <td>Tartronate semialdehyde reductase</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>C0311493</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T126|...</td>\n",
       "      <td>http://purl.bioontology.org/ontology/SNMI/F-66100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.1.1.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://purl.bioontology.org/ontology/STY/T126|...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Class ID  \\\n",
       "0  http://purl.bioontology.org/ontology/SNMI/C-140A7   \n",
       "1  http://purl.bioontology.org/ontology/SNMI/T-A1950   \n",
       "2  http://purl.bioontology.org/ontology/SNMI/L-61420   \n",
       "3  http://purl.bioontology.org/ontology/SNMI/DC-1...   \n",
       "4  http://purl.bioontology.org/ontology/SNMI/P1-A...   \n",
       "5  http://purl.bioontology.org/ontology/SNMI/F-671B0   \n",
       "6  http://purl.bioontology.org/ontology/SNMI/C-D4163   \n",
       "7  http://purl.bioontology.org/ontology/SNMI/J-43230   \n",
       "8  http://purl.bioontology.org/ontology/SNMI/C-D2165   \n",
       "9  http://purl.bioontology.org/ontology/SNMI/F-662E8   \n",
       "\n",
       "                                     Preferred Label  \\\n",
       "0                                       235m Uranium   \n",
       "1                 Choroid plexus of fourth ventricle   \n",
       "2                                       Rasahus, NOS   \n",
       "3  HNSHA due to gamma glutamyl cysteine synthetas...   \n",
       "4  Repair of perforating laceration of sclera wit...   \n",
       "5                   beta-Carotene 15,15'-dioxygenase   \n",
       "6                                            MITABAN   \n",
       "7                               Manufacturers' agent   \n",
       "8                   CLOSTRIDIAL 7-WAY PLUS SOMNUMUNE   \n",
       "9                2-Hydroxy-3-oxopropionate reductase   \n",
       "\n",
       "                            Synonyms  Definitions  Obsolete       CUI  \\\n",
       "0                                NaN          NaN     False  C0303352   \n",
       "1                                NaN          NaN     False  C0152293   \n",
       "2                                NaN          NaN     False  C0322617   \n",
       "3                                NaN          NaN     False  C0272071   \n",
       "4                                NaN          NaN     False  C0197629   \n",
       "5                                NaN          NaN     False  C0053397   \n",
       "6                                NaN          NaN     False  C0702074   \n",
       "7                                NaN          NaN     False  C0335279   \n",
       "8                                NaN          NaN     False  C0308688   \n",
       "9  Tartronate semialdehyde reductase          NaN     False  C0311493   \n",
       "\n",
       "                                      Semantic Types  \\\n",
       "0  http://purl.bioontology.org/ontology/STY/T196|...   \n",
       "1      http://purl.bioontology.org/ontology/STY/T023   \n",
       "2      http://purl.bioontology.org/ontology/STY/T204   \n",
       "3      http://purl.bioontology.org/ontology/STY/T047   \n",
       "4      http://purl.bioontology.org/ontology/STY/T061   \n",
       "5  http://purl.bioontology.org/ontology/STY/T126|...   \n",
       "6  http://purl.bioontology.org/ontology/STY/T109|...   \n",
       "7      http://purl.bioontology.org/ontology/STY/T097   \n",
       "8  http://purl.bioontology.org/ontology/STY/T116|...   \n",
       "9  http://purl.bioontology.org/ontology/STY/T126|...   \n",
       "\n",
       "                                             Parents Associated with  \\\n",
       "0  http://purl.bioontology.org/ontology/SNMI/C-14000             NaN   \n",
       "1  http://purl.bioontology.org/ontology/SNMI/T-A1900             NaN   \n",
       "2  http://purl.bioontology.org/ontology/SNMI/L-61300             NaN   \n",
       "3  http://purl.bioontology.org/ontology/SNMI/DC-1...             NaN   \n",
       "4  http://purl.bioontology.org/ontology/SNMI/P1-A...             NaN   \n",
       "5  http://purl.bioontology.org/ontology/SNMI/F-66100             NaN   \n",
       "6  http://purl.bioontology.org/ontology/SNMI/C-D0000             NaN   \n",
       "7  http://purl.bioontology.org/ontology/SNMI/J-43200             NaN   \n",
       "8  http://purl.bioontology.org/ontology/SNMI/C-D0000             NaN   \n",
       "9  http://purl.bioontology.org/ontology/SNMI/F-66100             NaN   \n",
       "\n",
       "           EZ Has location Location of   SB  \\\n",
       "0         NaN          NaN         NaN  NaN   \n",
       "1         NaN          NaN         NaN  NaN   \n",
       "2         NaN          NaN         NaN  NaN   \n",
       "3         NaN          NaN         NaN  NaN   \n",
       "4         NaN          NaN         NaN  NaN   \n",
       "5  1.13.11.21          NaN         NaN  NaN   \n",
       "6         NaN          NaN         NaN    V   \n",
       "7         NaN          NaN         NaN  NaN   \n",
       "8         NaN          NaN         NaN    V   \n",
       "9    1.1.1.60          NaN         NaN  NaN   \n",
       "\n",
       "                         Semantic type UMLS property  SHF    SIC        SMX  \n",
       "0  http://purl.bioontology.org/ontology/STY/T196|...  NaN    NaN        NaN  \n",
       "1      http://purl.bioontology.org/ontology/STY/T023  NaN  C71.7        NaN  \n",
       "2      http://purl.bioontology.org/ontology/STY/T204  NaN    NaN        NaN  \n",
       "3      http://purl.bioontology.org/ontology/STY/T047  NaN    NaN        NaN  \n",
       "4      http://purl.bioontology.org/ontology/STY/T061  NaN    NaN        NaN  \n",
       "5  http://purl.bioontology.org/ontology/STY/T126|...  NaN    NaN        NaN  \n",
       "6  http://purl.bioontology.org/ontology/STY/T109|...  NaN    NaN  (X-10186)  \n",
       "7      http://purl.bioontology.org/ontology/STY/T097  NaN    NaN        NaN  \n",
       "8  http://purl.bioontology.org/ontology/STY/T116|...  NaN    NaN  (X-20153)  \n",
       "9  http://purl.bioontology.org/ontology/STY/T126|...  NaN    NaN        NaN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Looking at the data\n",
    "snmi.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking at the data, we only need the `Preferred Label` and the `Synonyms` columns to generate a vocabulary text file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = ['Preferred Label','Synonyms']\n",
    "df = snmi[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(109150, 2)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Appending the Synonyms columns to the Preferred Label column\n",
    "df = df['Preferred Label'].append(df['Synonyms'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(218300,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to remove blank values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                              235m Uranium\n",
       "1                        Choroid plexus of fourth ventricle\n",
       "2                                              Rasahus, NOS\n",
       "3         HNSHA due to gamma glutamyl cysteine synthetas...\n",
       "4         Repair of perforating laceration of sclera wit...\n",
       "                                ...                        \n",
       "109131     Metastatic malignant neoplasm to pituitary gland\n",
       "109137        Metastatic malignant neoplasm to skin of knee\n",
       "109140                       Anaemia due to zinc deficiency\n",
       "109142                                             Otoliths\n",
       "109144    Lymphocytic inflammatory cell infiltrate, NOS|...\n",
       "Length: 141274, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_df = df.dropna()\n",
    "cleaned_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can use the `\\W` regular expression (AKA RegEx) to split by non-alphanumeric characters (whitespaces, punctuation,  symbols, etc) and get all unique words for our vocabulary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = cleaned_df.str.split(\"\\W\",expand=True).stack().unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['235m', 'Uranium', 'Choroid', ..., 'Ureteroplasty',\n",
       "       'FLOCCULONODULAR', 'Otoliths'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check\n",
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# making sure there are no blank spaces\n",
    "final_vocab = filter(None, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can save our file to a text (*.txt) file to use in Natural Language Processing (NLP). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = \"../vocab.txt\"\n",
    "with open(filepath, 'w') as file_handler:\n",
    "    for item in final_vocab:\n",
    "        file_handler.write(\"{}\\n\".format(item))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}