SpaCy_BioNER_Transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNm7DOLYsCuBvaSpzershjY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "74cf118985dd4e6aba06243e499423d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8ff0f73562834321b1dbc43289c2e292",
              "IPY_MODEL_7a8cc123f99942f5883490969ef4d4ec",
              "IPY_MODEL_ede2aa1685664251b18fe6238c435a23"
            ],
            "layout": "IPY_MODEL_ae016416aa534ba585291c138247c693"
          }
        },
        "8ff0f73562834321b1dbc43289c2e292": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b76fb812df9f484bba0486f0145e895b",
            "placeholder": "​",
            "style": "IPY_MODEL_5a5ee44931c74369ab6b84b6d548a26d",
            "value": "100%"
          }
        },
        "7a8cc123f99942f5883490969ef4d4ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a459713ac7154bc29865786e1c5da5e6",
            "max": 2999,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a9b75c14558146048d90e2f3d5dc583c",
            "value": 2999
          }
        },
        "ede2aa1685664251b18fe6238c435a23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_909f74a215074500960b698e7c4bc76c",
            "placeholder": "​",
            "style": "IPY_MODEL_92a145d0fff14b2aac2bbeac2f3a7c4e",
            "value": " 2999/2999 [00:02&lt;00:00, 1367.29it/s]"
          }
        },
        "ae016416aa534ba585291c138247c693": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b76fb812df9f484bba0486f0145e895b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a5ee44931c74369ab6b84b6d548a26d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a459713ac7154bc29865786e1c5da5e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9b75c14558146048d90e2f3d5dc583c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "909f74a215074500960b698e7c4bc76c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92a145d0fff14b2aac2bbeac2f3a7c4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/misha345a/SpaCy_BioNER_Transformers/blob/main/SpaCy_BioNER_Transformers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!pip install transformers\n",
        "!pip install --upgrade spacy\n",
        "!pip install spacy-transformers"
      ],
      "metadata": {
        "id": "H5iYAnU10E46"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# check that GPU is enabled\n",
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KnXjFiL-vFDw",
        "outputId": "9e5d285c-657f-43f0-b937-12abaa310d99"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sun May  8 22:14:23 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OmA2xm0Ry2s2"
      },
      "outputs": [],
      "source": [
        "# library imports\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import spacy\n",
        "from spacy import displacy\n",
        "from spacy.tokens import DocBin\n",
        "from spacy.util import filter_spans\n",
        "from datetime import datetime\n",
        "import json\n",
        "import re\n",
        "\n",
        "from tqdm import tqdm\n",
        "from tqdm.notebook import tqdm_notebook"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# initiate tqdm for pandas.apply() functions\n",
        "tqdm_notebook.pandas()\n",
        "\n",
        "# suppress all warnings\n",
        "# warnings.simplefilter('ignore')\n",
        "\n",
        "# expand notebook display options for dataframes\n",
        "pd.set_option('display.max_colwidth', 200)\n",
        "pd.options.display.max_columns = 999\n",
        "pd.options.display.max_rows = 300"
      ],
      "metadata": {
        "id": "keuFcw8fy-KB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Preprocessing"
      ],
      "metadata": {
        "id": "6EjTIidL0uD-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load the dataset\n",
        "df = pd.read_csv('/PATH_TO_DATA.csv', lineterminator='\\n', usecols=range(6))"
      ],
      "metadata": {
        "id": "ygFUaWMey-TL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# there are rows with missing keywords; we need to remove them\n",
        "df.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GUIDWRDey-Vp",
        "outputId": "dd724f4b-5e14-4389-ffc8-bcb3dcba8e81"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 3703 entries, 0 to 3702\n",
            "Data columns (total 6 columns):\n",
            " #   Column             Non-Null Count  Dtype \n",
            "---  ------             --------------  ----- \n",
            " 0   Unnamed: 0         3703 non-null   int64 \n",
            " 1   description        3703 non-null   object\n",
            " 2   medical_specialty  3703 non-null   object\n",
            " 3   sample_name        3703 non-null   object\n",
            " 4   transcription      3679 non-null   object\n",
            " 5   keywords           3023 non-null   object\n",
            "dtypes: int64(1), object(5)\n",
            "memory usage: 173.7+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# drop irrelevant columns and rows w/ missing data\n",
        "df = df[['transcription', 'keywords']].copy(deep=True)\n",
        "df.dropna(how='any', inplace=True)"
      ],
      "metadata": {
        "id": "udTRwKCJy-XC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 2999 rows remaining\n",
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SCLDFmJNy-YF",
        "outputId": "66a38e87-c4a8-4ec2-d1ac-08af05e8925a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2999"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Text Cleaning"
      ],
      "metadata": {
        "id": "-HpN6s7o2lt_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def text_cleaning(text):\n",
        "  text = re.sub(r\"NOTE,:.+\", \"\", text, flags=re.IGNORECASE) # remove transcription disclaimers\n",
        "  text = re.sub(r\"[A-Z]{4,}:\\s?,\", \"\", text)  # remove subheadings (PROCEDURE, IMPRESSION, etc.)\n",
        "  text = re.sub(r\"\\d\\.\", \"\", text) # remove number lists (1., 2.)\n",
        "  text = re.sub(r\"([:.])\\s{0,},\", r\"\\1 \", text) # remove instances of \".,\" or \": ,\"\n",
        "  text = re.sub(r\",$\", \"\", text) # remove trailing commas\n",
        "  text = re.sub(r\"\\s{2,}\", \" \", text) # shorten lengthy spacing \n",
        "  text = text.lower().strip()\n",
        "  return text"
      ],
      "metadata": {
        "id": "ESSmx3c144m1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['keywords'] = df['keywords'].apply(text_cleaning)\n",
        "df['transcription'] = df['transcription'].progress_apply(text_cleaning)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "74cf118985dd4e6aba06243e499423d6",
            "8ff0f73562834321b1dbc43289c2e292",
            "7a8cc123f99942f5883490969ef4d4ec",
            "ede2aa1685664251b18fe6238c435a23",
            "ae016416aa534ba585291c138247c693",
            "b76fb812df9f484bba0486f0145e895b",
            "5a5ee44931c74369ab6b84b6d548a26d",
            "a459713ac7154bc29865786e1c5da5e6",
            "a9b75c14558146048d90e2f3d5dc583c",
            "909f74a215074500960b698e7c4bc76c",
            "92a145d0fff14b2aac2bbeac2f3a7c4e"
          ]
        },
        "id": "BkkymorE5Oqg",
        "outputId": "e05cde53-6159-4cb7-afb8-34a667d43f99"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2999 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "74cf118985dd4e6aba06243e499423d6"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 458
        },
        "id": "XNK1m7dr44pi",
        "outputId": "ac0a99b5-7995-413f-a8bc-21e38a43890f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                                                                                                                                             transcription  \\\n",
              "0  this 23-year-old white female presents with complaint of allergies. she used to have allergies when she lived in seattle but she thinks they are worse here. in the past, she has tried claritin, an...   \n",
              "1  past medical history: he has difficulty climbing stairs, difficulty with airline seats, tying shoes, used to public seating, and lifting objects off the floor. he exercises three times a week at h...   \n",
              "2  i have seen abc today. he is a very pleasant gentleman who is 42 years old, 344 pounds. he is 5'9\". he has a bmi of 5 he has been overweight for ten years since the age of 33, at his highest he wa...   \n",
              "3  2-d m-mode: , left atrial enlargement with left atrial diameter of 7 cm. normal size right and left ventricle. normal lv systolic function with left ventricular ejection fraction of 51%. normal lv...   \n",
              "4  the left ventricular cavity size and wall thickness appear normal. the wall motion and left ventricular systolic function appears hyperdynamic with estimated ejection fraction of 70% to 75%. there...   \n",
              "\n",
              "                                                                                                                                                                                                  keywords  \n",
              "0                                                                       allergy / immunology, allergic rhinitis, allergies, asthma, nasal sprays, rhinitis, nasal, erythematous, allegra, sprays, allergic  \n",
              "1  bariatrics, laparoscopic gastric bypass, weight loss programs, gastric bypass, atkin's diet, weight watcher's, body weight, laparoscopic gastric, weight loss, pounds, months, weight, laparoscopic,...  \n",
              "2  bariatrics, laparoscopic gastric bypass, heart attacks, body weight, pulmonary embolism, potential complications, sleep study, weight loss, gastric bypass, anastomosis, loss, sleep, laparoscopic, ...  \n",
              "3  cardiovascular / pulmonary, 2-d m-mode, doppler, aortic valve, atrial enlargement, diastolic function, ejection fraction, mitral, mitral valve, pericardial effusion, pulmonary valve, regurgitation...  \n",
              "4  cardiovascular / pulmonary, 2-d, doppler, echocardiogram, annular, aortic root, aortic valve, atrial, atrium, calcification, cavity, ejection fraction, mitral, obliteration, outflow, regurgitation...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-995adf6d-4df0-4bb3-a9a6-cc6f8fe14be3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>transcription</th>\n",
              "      <th>keywords</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>this 23-year-old white female presents with complaint of allergies. she used to have allergies when she lived in seattle but she thinks they are worse here. in the past, she has tried claritin, an...</td>\n",
              "      <td>allergy / immunology, allergic rhinitis, allergies, asthma, nasal sprays, rhinitis, nasal, erythematous, allegra, sprays, allergic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>past medical history: he has difficulty climbing stairs, difficulty with airline seats, tying shoes, used to public seating, and lifting objects off the floor. he exercises three times a week at h...</td>\n",
              "      <td>bariatrics, laparoscopic gastric bypass, weight loss programs, gastric bypass, atkin's diet, weight watcher's, body weight, laparoscopic gastric, weight loss, pounds, months, weight, laparoscopic,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>i have seen abc today. he is a very pleasant gentleman who is 42 years old, 344 pounds. he is 5'9\". he has a bmi of 5 he has been overweight for ten years since the age of 33, at his highest he wa...</td>\n",
              "      <td>bariatrics, laparoscopic gastric bypass, heart attacks, body weight, pulmonary embolism, potential complications, sleep study, weight loss, gastric bypass, anastomosis, loss, sleep, laparoscopic, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2-d m-mode: , left atrial enlargement with left atrial diameter of 7 cm. normal size right and left ventricle. normal lv systolic function with left ventricular ejection fraction of 51%. normal lv...</td>\n",
              "      <td>cardiovascular / pulmonary, 2-d m-mode, doppler, aortic valve, atrial enlargement, diastolic function, ejection fraction, mitral, mitral valve, pericardial effusion, pulmonary valve, regurgitation...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>the left ventricular cavity size and wall thickness appear normal. the wall motion and left ventricular systolic function appears hyperdynamic with estimated ejection fraction of 70% to 75%. there...</td>\n",
              "      <td>cardiovascular / pulmonary, 2-d, doppler, echocardiogram, annular, aortic root, aortic valve, atrial, atrium, calcification, cavity, ejection fraction, mitral, obliteration, outflow, regurgitation...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-995adf6d-4df0-4bb3-a9a6-cc6f8fe14be3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-995adf6d-4df0-4bb3-a9a6-cc6f8fe14be3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-995adf6d-4df0-4bb3-a9a6-cc6f8fe14be3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Text Cleaning Example"
      ],
      "metadata": {
        "id": "ESbE7h447I2M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "example = \"DESCRIPTION:,1.  Normal cardiac chambers size.,2.  Normal left ventricular size.,\"\n",
        "print(f\"ORIGINAL: {example}\")\n",
        "print(f\" CLEANED: {text_cleaning(example)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T-cl0RT17II0",
        "outputId": "ff89cacb-404e-4aea-8ecf-e5bcde9844ce"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGINAL: DESCRIPTION:,1.  Normal cardiac chambers size.,2.  Normal left ventricular size.,\n",
            " CLEANED: normal cardiac chambers size. normal left ventricular size.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Keyword Analysis"
      ],
      "metadata": {
        "id": "kpI1eYlE5Pa0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# remove the first term in each keyword set; \n",
        "# this term only defines the specialty and is not an entity\n",
        "df['keywords'] = df['keywords'].apply(lambda x: re.sub(r\"[a-z /]*, \", '', x , 1))"
      ],
      "metadata": {
        "id": "Q54eSaEuy-av"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# count frequency of each keyword\n",
        "keyword_counts = pd.Series(df['keywords'].str.cat().replace(', ', ',').split(',')).value_counts()\n",
        "keyword_counts = keyword_counts.sort_values(ascending=True)"
      ],
      "metadata": {
        "id": "2tlYeaBsy-gu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# number of unique keywords or \"entities\"\n",
        "len(keyword_counts)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n9mFdgh_PhGM",
        "outputId": "6faec2f3-a57e-4fa4-cba8-3b3bf39f6c9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10339"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# plot the counts of the top 25 keywords\n",
        "keyword_counts[-25:].plot(kind='barh')\n",
        "plt.title(\"Top 25 Keywords (Counts)\")\n",
        "plt.xlabel(\"Count\")\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "Jqp2RWBTy-ix",
        "outputId": "d4e51efa-010c-400f-db78-0f315f5d2512"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfIAAAEWCAYAAACdRBVEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdebzd073/8dc7iTkkNVRDaZSgMUWcuOaKpm5ntJRKEVr5aVXQS6/2utdwaakWRdFc16yosSnX1EQiYsgoE0JL1FQlTSIRY/L5/bHWlm929j5nn2Sfc3LOeT8fj/PI3uu71vqu794e1l7r+12fpYjAzMzM2qcubd0AMzMzW3HuyM3MzNoxd+RmZmbtmDtyMzOzdswduZmZWTvmjtzMzKwdc0duZraSJIWkrVag3EaSnpW0Vku0q6VIukPSl9u6HZa4IzfrgCQtLPwtkfRu4f3gOp3jV5Kel7Qgd0ZHlh0PSe8UzntVI3WNlvT9wvt9Jc2VdFg92roKOw24NiLeLSVI+ldJj+TP9U1JYyR9o6UbImm2pEE1Zj8fOKcl22O169bWDTCz+ouI7qXXkmYD34+IP9f5NO8AXweeAwYA90v6S0Q8VsizU0T8pTmVStof+AMwJCLurltr60BSt4j4qE51rQEcBfQrpB0MXA38mPTZLgD2Br4LjKjHeeshIsZLWk9SQ0RMbOv2dHYekZt1IpLWkHSxpNfy38W5QymNgl+R9DNJb+URWtXRe0ScERHPRsSSiHgSGAvsvpLt+xqpEz+81InnNv9K0t8kvSHpytJUtKQZkr5eKL9abvvOkq6T9G85fdM8Q3B8fr+lpH9K6pLfHyvpLzlthKRNCnWGpOMlPQ88n9NOlfR6/gyPKbuGr0h6Oo+oX5V0SpXL/RdgXkS8kssJuBD474i4KiLm5892TEQcm/N0kXS6pJck/UPS9ZJ65GP7SnqlrC0fj7IlnSnpD7nMAkkzJTXkYzcAmwN/yrMnP5G0pqQbJc2RNE/SBEkbF6ofDXy1lu/VWpY7crPO5T+A3UijwJ2AXYHTC8c/BWwIbEoaLQ6XtE1TleaOdQAws+zQI5L+LulOSb2bqObrwA3AwRHxf4X084Ctc5u3ym37r3zsetJoteQrwOsRMQUYA+yb0z8PvADsU3g/NiKWSNoP+AXwbaAX8BJwS1nbDiR1vH0lfQk4Bfgi0Acon47+X+D/RcS6wPbAqCrXuwMwq/B+G2Az4PYq+QGG5L+BwGeB7sBljeQv9w3StfUkjfAvA4iII4C/AV+PiO4R8UvS998jt2kD4Djg3UJdz5D+G7I25o7crHMZDJwdEf+IiDeBs4AjyvL8Z0S8HxFjgHtJHVxTrgSmAg8U0j4P9Aa2BV4D7pHU2O28gaQR77hSQh6lDgVOjoh/RsQC4OdA6d75jcBXJK2X3x9B+jEAqSPfK4+69wF+CexZaNuY/HowcHVETI6I94GfAruX/fD4RT7/u6TP45qImBER7wBnll3Hh6QOf72ImBsRk6tcb0/S1HnJBvnf16vkL7X1woh4ISIW5rYe1sTnWvRoRPxfRCwmfU6NdcQf5jZtFRGLI2JSRLxdOL4gX4O1MXfkZp3LJqQRZ8lLOa1kbu6cqh1fjqQLSCPPb0dhF6aIeCQiPoiIecCJwBbA5xqp6j+B94G7S9P9wEbA2sCkPL07D7g/pxMRr5E6/m9J6gl8GbgpH/sr6T5+P9J95nuA1/IMQ7EjX+YzyR3kHNLIv+TlwutNyt4XP0+Ab5FmBl7KD6pVu90wF1i38H5O/rdXlfzLtTW/7gZsXDn7cv5eeL0IWLORHwE3kH6Y3ZJvIfxS0mqF4+sC82o8r7Ugd+RmnctrwGcK7zfPaSWfkLROI8eXIeksUue5f9lorZIA1Mjxd0gdYA/gttxpvEWazt0uInrmvx7Fh/mA60jT64cAj0fEq4VjY4CDgdVz+hjSlPEngKdynmU+k3z9GwDFeorbRL5Omm4u2XyZi4yYEBEHAJ8E7ibd869kGumWQcks0g+Eb1XJv1xb87k/At4gfX5rF66jK/kHT42W2QozIj6MiLMioi+wB/A1oLgy4XOkWRhrY+7IzTqXm4HTldYvb0i613xjWZ6zJK0uaW/S/7xvq1SRpJ8ChwODImJO2bHtJPWT1FVSd+DXpI7xmcYal6fOv0QaDf+e1PH/D3CRpE/mujeV9K+FYncD/Umj/uvLqhwD/Ah4JL8fnd8/mqeXS5/J0bm9a5Cm7p+MiNlVmvkHYIikvpLWBs4oXPfqkgZL6hERHwJvA0uq1DMe6Clp03ztQXpa/T8lHa30VHgXSXtJGl5o68mStsif68+BW/OT9M+RRthfzT+CTgfWWO6s1b1Buu9eupaBknbIPwjeJk21F6/l88B9zajfWog7crPO5RxgImk0OB2YzLLrgf9OmvJ9jTRFfVxEPFulrp+TRoR/0dK14j/LxzYGbiV1AC+Q7pV/LXdujcpT8V8kjVavJ90H/gvwhKS3gT+THgwr5X8XuIM0dX9nWXVjSFPApY78UdKo9ZFC+T+TpvXvII22t2TpPfhK7bsPuJj0ENtfWP5htiOA2bmtx5Hua1eq5wPgWgoP60XE7cChwDGk7+AN0vfzx5zlatKU9yPAi8B7wAm57Hzgh8BVpB9N7wDLPMXehF+QfuTNy0/af4r04N3bpB9gY/K5kTQAWBgR45tRv7UQFW5pmVknJmlf4MaI+HRbt6W5JP0XsHVEfLfJzKsQSRuRlu3tXAwKs6qTdAfwv2WrC6yNOCCMmbVrktYHvsfyT9+v8vLKgW3buh3NFRGN3ce3VuapdTNrtyQdS3pA7L6IeKSp/GYdkafWzczM2jGPyM3MzNox3yO3VrXhhhtG796927oZZmbtyqRJk96KiIpxAdyRW6vq3bs3Eyd6syQzs+aQVB5B8GOeWjcknZQDW5iZWTvjh906uRy16a9AQ0S81ZxyhchYNVujV5/oddTFzS1mZtauzT5v5XZ8lTQpIhoqHfOIvIOTdLekSXnv4aE5baGkX0uaStrWchPgYUkP5+P7S3pc0mRJt+VQkKW9jc+XNBk4Lf9bOk+f4nszM2sd7sg7vmMiYhegARgmaQNgHVIs6Z0i4mxSKMiBETEwx98+nRQ/uz8pnOePC/XNiYj+EXEuMF9Sv5x+NHBNpQZIGippoqSJixfNb5mrNDPrpPywW8c3TNJB+fVmQB9gMSmudCW7AX2BcWkraFYHHi8cv7Xw+irSZhM/JsWH3rVShRExHBgOaWp9xS7DzMwqcUfegeXY2YOA3SNikaTRwJrAe43c3xbwUER8p8rx4l7Vd5B2fhoFTCrfAauSHTbtwcSVvFdkZmZLeWq9Y+sBzM2d+Lak0XYlC0g7RAE8AewpaStIezNL2rpSoYh4D3gAuIIq0+pmZtay3JF3bPcD3SQ9A5xH6qQrGQ7cL+nhvInDEOBmSdNI0+qNbepwE2mP4gfr1mozM6uZp9Y7sIh4H/hyhUPdy/JdClwKIOkbQN+IGFChvt4V6toLuCYiFkuaTTOXsZmZ2cpxR94JSeoWER9VOhYRI4ARNdZzF7AlsF+t557+6nx6n3ZvrdnNzGq2smu12ytPrbdzko6UNE3SVEk3SNpI0h2SJuS/PXO+M/PxccANkp6QtF2hntGSGiQNkXRZTttY0l257qmS9sjpd0uaBGwNXOYRuJlZ2/GIvB3LHfHpwB4R8Zak9YHLgIsi4lFJm5MeRvtcLtIX2Csi3pV0MvBt4AxJvYBeETFR0vaFU1wCjImIg3IEuNKU/DER8U9JawETJN3R2BPrORDNUICu61WM+W9mZivIHXn7th9wW2lEnDvXQUDfvAYcYL1SZDZgRES8m1//gfSA2hmkDv32KvUfmeteDJSiuVRam161I/c6cjOzluOOvOPpAuyWl4Z9LHfsH68Bj4hXJc2RtCMpmMtxtVTeyNp0MzNrA+7I27dRwF2SLoyIOXlq/UHgBOACAEn9IuKpKuVvBX4C9IiIaRWOjwR+AFxcmFqvdW16RQ4IY2ZWX37YrR2LiJnAucCYvAHKhcAwoCE/APc0jY+0bwcOI02zV3IiMFDSdGAS6R57rWvTzcysFXgbU2tVDQ0NMXHixLZuhplZu+JtTK1FSTpQUt+2boeZWWfke+RWDwcC9wBPN5XRAWHMDDpv8JaW4BG5NVtZEJqHgG8AF0h6StKWbd0+M7POxCNya5YqQWguBO6JiEpr0R0QxsysBXlEbs21XBCapgpExPCIaIiIhq5r92jxBpqZdSYekVur8jpyM7P68ojcmmsUcIikDQDy1PoCYN02bZWZWSfljtyapUoQmluAUyVN8cNuZmaty1Pr1mwRcR1wXVmy15GbmbUBj8hbkKSFbd2GliDpKgeAMTNbNXhE3o5J6hYRH7X2eSPi+yta1gFhzDovB4FpGR6RtwJJ3SWNlDRZ0nRJB+T03pKelXSTpGck3S5p7XzsvyRNkDRD0nDlfUgljZZ0saSJwImSvpDvTU+XdLWkNXK+8yQ9nQO3/CqnbSzprhzIZaqkPXL6j/N5Zkg6qYa2jZbUkF9/KV/XVEkjW/mjNTPr9NyRt473gIMioj8wEPh1qWMGtgEuj4jPAW8DP8zpl0XEgIjYHlgL+FqhvtVz8PzfAtcCh0bEDqQZlh/kJ8oPAraLiB2Bc3K5S4AxEbET0B+YKWkX4GjgX0hbkh4raecm2gaApI2A/wG+les8pNLFSxoqaaKkiYsXzW/Gx2ZmZk1xR946BPxc0jTgz8CmwMb52MsRMS6/vhHYK78eKOnJvIXofsB2hfpuzf9uA7wYEc/l99cB+wDzST8e/lfSN4FF+fh+wBUAEbE4Iubn890VEe9ExELgTmDvJtpWshvwSES8mOusGBzGAWHMzFqO75G3jsHARsAuEfGhpNnAmvlY+T6yIWlN4HKgISJelnRmIT/AO42dLCI+krQr8AXgYOBHpE68uZZr2wrUsQwHhDEzqy+PyFtHD+AfuRMfCHymcGxzSbvn14cDj7K0035LUndSZ1zJLKC3pK3y+yNI67u7Az0i4v+Ak4Gd8vGRwA8AJHWV1AMYCxwoaW1J65Cm5Mc20raiJ4B9JG2R61y/lg/DzMzqxx1567gJaMjT5EcCzxaOzQKOl/QM8AngioiYR7r3PAN4AJhQqdKIeI90f/u2XPcS4EpSlLV78lT+o8CPc5ETSVP204FJQN+ImEy6zz4eeBK4KiKmVGtb2fnfJG2GcmcODnMrZmbWqhSx0rOltoIk9SbtGrZ9K5zrQOC5iGhyz/Ccvzdpj/GfkTr88+rRjoaGhpg4cWI9qjIz6zQkTcoPOS/H98g7jwNJHXNNHXlJRIwARtSav6m17V5Hbta5eO14y/PUehuKiNkrMxqXdLekSZJm5j2/kbRQ0rl5XfcTee34HsA3gAskPSVpy/x3fy4/VtK2ufy1kq4kTZM/KGmIpMvysd6SRuW16SMlbV4sI+lJ4Jcr+bGYmVkzuCNv346JiF2ABmBYXj++DvBEXtf9CHBsRDxGGlWfGhH9IuKvwHDghFz+FNJT8iWfBvaIiB+zrEuB6/La9JtI69KbKuN15GZmLchT6+3bMEkH5debAX2AD0hT6JAeaPtieaH8VPsepIfkSslrFLLcFhGLK5xvd+Cb+fUNLDv6rlaGiBhO+uHAGr36+KEMM7M6ckfeTknaFxgE7B4RiySNJi1b+zCWPsG4mMrfcRdgXkT0q1J9o+vU61jGzMxWkjvy9qsHMDd34tuSoqw1ZgFpWRoR8bakFyUdEhG35XCxO0bE1CbqeAw4jDQaH8zS9eY1c0AYM7P68j3y9ut+oFte430eKThLY24BTs0brGxJ6oi/l9d/zwQOqOGcJwBH5/XpR5DWpZuZWRvyOnJrVV5HbmbWfI2tI/eIvIOQdFJpm9FWONcmkm5vjXOZmVnjPCLvIPJGLA0R8VZbt6Uxa/TqE72Ouritm2FmdeBgL63HI/IORtI6ku7NQV9mSDoD2AR4WNLDOc/+kh6XNFnSbXnJGZJmSzorp08vBIJZPweYmZYDyeyY0z+fg8g8le+vr5sDw8zIx7eTND4fnyapT9t8KmZmnZM78vbpS8BrEbFTjgx3MfAaMDAiBkraEDgdGBQR/YGJLN04BeCtnH4FKRgMwFnAlBzs5WfA9Tn9FOD4vFRtb+DdsrYcB/wmH28AXilvrAPCmJm1HHfk7dN04IuSzpe0d0SU9467AX2BcZKeAo5i2a1T78z/TgJ659d7kZaVERGjgA0krQeMAy6UNAzoWSGO+uPAzyT9O/CZiCjv6ImI4RHREBENXdfusYKXbGZmlXgdeTsUEc9J6g98BThH0siyLAIeiojvVKni/fxvtYAxxXOdJ+nefK5xkv4VeK9w/Pc5xvpXgf+T9P/yD4GKvI7czKy+PCJvhyRtAiyKiBuBC4D+FAK+kNaU7ylpq5x/HUlbN1HtWNLa8lLUuLdy4JgtI2J6RJxP2hd927K2fBZ4ISIuAf4I7FiPazQzs9p4RN4+7UDayWwJ8CHwA1Ic9PslvZbvkw8BbpZUiqF+OvBcI3WeCVydg70sIk3HA5wkaSCwhBQ45j6gV6Hct4EjJH0I/B34eR2uz8zMauTlZ9aqHBDGzKz5vPxsFSepp6QftsJ5viHptBaod0ie7jczs1bmEfkqQFJv4J68lGxl6+pabTvRlpJ3XjslIpocajsgjFnH4YAwrccj8lXfecCWOajKBflvRg7YciikB9AklfYZR9Jl+T54KcjL+ZImA4c0EvRliKTL8ustcsCY6ZLOkbSwhvPsImmMpEmSHpDUS9LBpPXjN+X2r9UaH5iZmSXuyFcNpwF/zUFVngD6ATuR9hu/QFKvxgpncyKif0Tckt9XCvpS9BvgiojYAXi9qcolrQZcChwcEbsAVwPnRsTtpIAzgyOiX6V15A4IY2bWctyRr3r2Am6OiMUR8QYwBhhQQ7lby95XCvpStCdwc359Qw31bwNsDzyUg8ycDny6hnIOCGNm1oK8/Kz9+Ihlf3itWXb8nbL3tQR9qfSARLXzCJgZEbs33dTqHBDGzKy+PCJfNRSDuYwFDpXUVdJGwD7AeOAloK+kNST1BL6wkuccBxyWXw8upFc7zyxgI0m7Q5pql7RdhfabmVkr8oh8FRARcySNyzuK3QdMA6aSRsw/iYi/A0j6AzADeBGYspKnPRH4fY6R/sdCW16udJ6I+CA/2HaJpB6k/3YuJgWJuRa4UtK7wO6V7pObmVnL8PIzA0DSwojovhLlG4AjI2JYY/kcEMbMrPkaW37mEbmtNEnd8hryJnvo6a/Op/dp97ZCq8ysnNd9d0y+R24AFEfjknpLelbStZKek3STpEF5+v95SbtKOlPSDZLGATeUrz83M7PW4Y7cqtkK+DVpt7NtgcNJS+NOAX6W8/QFBjWyXaqZmbUwd+RWzYt5+9LSrmcjIz1QMZ2l69JH1PJgmwPCmJm1HHfkVs37hddLCu+XsPTZivK16xU5IIyZWcvxw27WqhwQxsysvjwiNzMza8c8IrflRMRsUlz10vsh1Y4V0kcDo1u6bWZmtiyPyNuh0pajTeTpnSPFmZlZB+YRubUqB4Qxa10OAtPxeUS+ipN0t6RJkmZKGlpIvyinjcybqyBpF0lTJU0Fji/kXVPSNZKmS5oiaWBOH5Lrf0jSbEk/kvTjnOcJSevnfFtKuj+3Y6ykbXP6tZIukfSYpBdyLHYzM2tF7shXfcdExC5AAzBM0gbAOsDEiNiOtF/5GTnvNcAJEbFTWR3HAxEROwDfAa6TVNqedHvgm6Q9z88FFkXEzsDjwJE5z/Bc7y6kgDCXF+ruRQoU8zXgvEoX4HXkZmYtx1Prq75hkg7KrzcD+pDWct+a024E7sxbjvaMiEdy+g3Al/PrvYBLASLiWUkvAVvnYw9HxAJggaT5wJ9y+nRgR0ndgT2A2ySV2rRGoX1356AxT0vauNIFRMRw0o8B1ujVx7v0mJnVkTvyVZikfYFBpK1BF0kaDaxZIevKdI5NBX7pAsyLiH41lFeVPB/zOnIzs/ry1PqqrQcwN3fi2wK75fQuQOl+9OHAoxExD5gnaa+cPrhQz9jSe0lbA5sDs2ppQES8Dbwo6ZBcXpLKp+7NzKyNuCNftd0PdJP0DOn+8xM5/R1g17y8bD/g7Jx+NPBbSU+x7Oj4cqCLpOmkKfkhEVEcSTdlMPC9/BDdTOCAFb0gMzOrL6V9MMxaR0NDQ0yc2OS25WZmViBpUkQ0VDrmEXknlpePrfSSMUknSVq7Hm0yM7Pm8cNuVg8nkZ6eX9RURgeEMVuWA7bYyvKIvBORdKSkaTlozA05eZ9KAV0knSppQs5/Vk5bR9K9ufwMSYdKGgZsAjws6eE2uCwzs07NI/JOQtJ2wOnAHhHxVo7adiFLA7psC4wAbpe0P2m9+q6kh+ZGSNoH2Ah4LSK+muvsERHzJf0YGBgRb1U591BgKEDX9TZqycs0M+t0PCLvPPYDbit1thHxz5x+d0QsiYingVJAl/3z3xRgMqmT70MKEvNFSedL2jsiagrTFhHDI6IhIhq6rt2jjpdkZmYekVulgC4CfhERvyvPLKk/8BXgHEkjI+Ls8jyNcUAYM7P68oi88xgFHJJjtVPaEKWKB4BjcnhWJG0q6ZOSNiHFYr8RuADon/MvANZtuaabmVk1HpF3EhExU9K5wBhJi0nT5tXyPijpc8DjOb76QuC7wFbABZKWAB8CP8hFhgP3S3otIga25HWYmdmyHBDGWpUDwpiZNZ8DwrQjkhY2cby3pMPreL66BIXJdc2WtGE96jIzs9p4ar396U3aKOX35QckdYuIj1q9Rc3ggDDWWTnwi7UUj8hbgKTvShov6SlJv5PUVdJCSefmYCpPlPbulrSFpMclTZd0TqEOSbogB16ZLunQfOg8YO9c98mShkgaIWkUMFJSd0kjJU3O5Q4o1FkpIAw0IyhMTr9b0iRJM/MacTMzayPuyOssPyR2KLBn3sN7MWn3sHWAJyJiJ+AR4Nhc5DfAFRGxA/B6oapvAv2AnUh7kl8gqRdwGjA2IvpFxEU5b3/g4Ij4PPAecFBE9AcGAr/OPwpKAWH2y204sXCuUlCYr5F+KFAWFKYfsEsOCgNwTETsAjQAw0pPwjfymQyVNFHSxMWLalp6bmZmNXJHXn9fAHYBJuTtRL8AfBb4ALgn55lEmiIH2BO4Ob8ujpL3Am6OiMUR8QYwBhhQ5ZwPFQK8CPi5pGnAn4FNSYFeqgWEgeYFhYHUeU8lbau6WSG9IgeEMTNrOb5HXn8CrouIny6TKJ0SS5cILGbZz35llw68U3g9mBRKdZeI+FDSbGDNJsrXHBRG0r6kGYLdI2KRpNE11P8xB4QxM6svj8jrbyRwsKRPQgq8IukzjeQfBxyWXw8upI8FDs331zcC9gHG03TwlR7AP3InPhAonbs5AWGgSlCYXP/c3IlvC+zWRD1mZtaCPCKvs4h4WtLpwIOSupACpxzfSJETgd9L+nfgj4X0u4DdgamkEftPIuLvkuYAi/PU9rXA3LL6bgL+JGk6MBF4NrerUkCYIY1cR7WgMPcDx0l6BphFml43M7M24oAw1qocEMbMrPkaCwjjEXkH11Jry1e0Xq8jt87Ea8etNfgeeTtSvg48R3kbldNGSto857tW0pWSngR+KalfXrs+TdJdkj6R841W2pJ0vKTnJO2d03tLGpvXok+WtEdO3zenjwCelnS2pJMK7TtX0onLt9zMzFqKR+TtRGEd+B4R8VZ+WO060hPy10k6BrgEODAX+XTOuzgvRTshIsZIOhs4Ayh1wN0iYldJX8npg4B/AF+MiPck9SEtjytN6fQHto+IFyX1Bu4ELs7PAxxGWnde3vahwFCAruttVMdPxczM3JG3H8utA5e0OylwDKQ16L8s5L8td+I9gJ4RMSanXwfcVsh3Z/63uLZ9NeAySaWANlsX8o+PiBdzG2ZLmiNpZ9L68ykRMae84RExnLRDGmv06uOHMszM6sgdecf1TtNZgKVryItr208G3iBFletCihZXrd6rSE+/fwq4ekUaamZmK84defsxCrhL0oURMSdPrT9Gms6+gbQGfWx5oYiYL2mupL0jYixwBClKXGN6AK9ExBJJRwFdG8l7F3A2aRTf5K5sDghjZlZf7sjbiSrrwE8ArpF0KvAmcHSV4kcBV0paG3ihkXwllwN3SDqStG686ug+Ij6Q9DAwLyIWN+uizMxspXkdua2U/JDbZOCQiHi+qfxeR25m1nyNrSP38rMORNLCOtWzr6R7asjXF/gLMLKWTtzMzOrPU+u2wvJuaZ8tT5fUtdo0uwPCWEfnIDDW2jwi74DKR9SSLpM0JL+eLekXkp7Ke4T3l/SApL9KOq5QzXqS7pU0KweX6ZLLX5HLzZR0VuEcs3NwmcnAIa10qWZmnZ5H5J3T3yKin6SLSBuv7EnainQGcGXOsyvQF3iJ9MDbN4Hbgf/Ia9i7AiMl7RgR03KZORHRv/xkDghjZtZyPCLvnEbkf6cDT0bEgoh4E3hfUs98bHxEvJCnyG8G9srp386j7inAdqTOvuTWSieLiOER0RARDV3X7lH3izEz68w8Iu+YPmLZH2lrlh0vBYFZUnhdel/6b6J8OUNI2gI4BRgQEXMlXVtWd5NBaLyO3Mysvjwi75heAvpKWiOPsL+wAnXsKmmLfG/8UOBRYD1SZz1f0sbAl+vWYjMzWyEekXdAEfGypD+Q7nm/SJoGb64JwGXAVsDDwF050tsU4FngZWBcnZpsZmYryAFhrFU5IIyZWfM5IIzVRNIwSc9IuqmZ5fYt7VluZmaty1PrVvRDYFBEvNLMcvsCC0mbuDTKAWGso3NAGGttHpEbAJKuJEVpu0/Sv0m6W9I0SU9I2jHnWb88XVJv4Djg5BxkZu+2uwozs87HHbkBEBHHAa8BA4HewJSI2BH4GXB9znZWeXpEzCYFkbkoIvrlrVKXIWlojgY3cfGi+S1/MWZmnYg7cqtkL9Ie50TEKGADSes1kt4oB4QxM2s5vkdurcoBYczM6ssjcqtkLDAY0hPpwFsR8XYj6QuAddukpWZmnZw7cqvkTGAXSa4OygwAAB4SSURBVNOA84Cjmkj/E3CQH3YzM2t9nlpv5yQ9FhHNXsMtqQE4MiKGldIionfh+A+BSyLi4MLxfwIHltcVEc8BOza3DWZmtvIc2c1a1Rq9+kSvoy5u62aYtRivI7eW4MhuHZikhfnffSWNlnS7pGcl3SRJ+dgASY9JmippvKR1c/578vHP52nxpyRNycd7S5qRj68p6RpJ0/PxgTl9iKQ7Jd0v6XlJv2yrz8HMrLPy1HrHsjNpj/DXSBua7ClpPGmf8EMjYkJeLvZuWblTgOMjYpyk7sB7ZcePByIidpC0LfCgpK3zsX75vO8DsyRdGhEvFwtLGgoMBei63kb1ulYzM8Mj8o5mfES8EhFLgKdIgV22AV6PiAkAEfF2RHxUVm4ccKGkYUDPCsf3Am7M5Z8lbZNa6shHRsT8iHgPeBr4THmjvI7czKzluCPvWN4vvF5MjTMuEXEe8H1gLWBcHnW36DnNzKw+/D/djm8W0EvSgDy1vi5lU+uStoyI6cB0SQOAbUkj+pLS+vFReUp981xv/+Y2xgFhzMzqyyPyDi4iPgAOBS6VNBV4CFizLNtJkmbk9eEfAveVHb8c6CJpOul++5CIeB8zM2tzXn5mraqhoSEmTpzY1s0wM2tXvPysE8rLy/YovL9W0sGNlSkr3zMHhTEzs1WY75F3XPsCC4HHVrB8T+CHpGn1muR168pPzVc0/dX59D7t3hVsktmqyUFgrC15RN7OSDpS0rQc3OUGSV+X9GQO1PJnSRtL6g0cB5xcFv98nxwY5oXi6FzSqZIm5HrPysnnAVvm8hdUy5cDx8ySdD0wA9islT4KMzPDI/J2RdJ2wOnAHhHxlqT1gQB2i4iQ9H3gJxHxb5KuBBZGxK9y2e8BvUhrwrcFRgC3S9of6APsCggYIWkf4DRg+4jol8tXy/e3nH5URDxRpd0OCGNm1kLckbcv+wG3RcRbkDYxkbQDcKukXsDqwIuNlL87T3s/LWnjnLZ//puS33cndcx/KyvbWL6XqnXiuZ3DgeGQYq3XcqFmZlYbd+Tt36XAhRExIu8RfmYjeYtLxlT49xcR8btixjw9T1n+avneqbWxXkduZlZfvkfevowCDpG0AUCeWu8BvJqPH1XIuwBYt4Y6HwCOyTHWkbSppE9WKF8tn5mZtSGPyNuRiJgp6VxgjKTFpGnuM4HbJM0ldfRb5Ox/It0DPwA4oZE6H5T0OeDxvFnaQuC7EfFXSePyDmj3RcSplfKRwrKamVkbcUAYa1UOCGNm1nwOCNPB5X3BN2nk+NmSBq1g3bMlbbjirTMzs5bkqfWOYQhpDfdr5QckdY2I/2qthuTzVZ1ud0AYaw8c4MXaE4/I25Ck8yQdX3h/pqRTGgm88oyk/5E0U9KDktbKgV0agJty8Ja18ij6fEmTSQ/HfRyeVdKAHBRmqqTxktbNI/rLCu24Jz8BX97euyVNyucfWkhfKOnXeVOW3VvsAzMzs+W4I29btwLfLrz/NvAmSwOv9AN2yYFXyOm/jYjtgHnAtyLidmAiMDgi+kVEaYvSORHRPyJuKVUuafV8zhMjYidgEGVbmjbhmIjYhfTDYVjp6XlgHeDJiNgpIh4tLyRpqKSJkiYuXjS/GaczM7OmeGq9DUXEFEmfzPe3NwLmAjtQPfDKixFR2id8EtC7kepvrZC2DfB6REzI538bID+FXothkg7KrzfL7ZpDenL9jmqFHBDGzKzluCNve7cBBwOfInW+n6F64JViQJfFwFqN1FtzkBbgI5adnSnfr5w81T4I2D0iFkkaXcj3XmP3xYscEMbMrL48td72bgUOI3Xmt7FigVdqDf4yC+glaUCue11J3YDZQD9JXSRtRprWL9cDmJs78W2B3Wo4n5mZtTCPyNtYDvKyLvBqRLwOvL4CgVeuBa6U9C6NPGwWER9IOhS4VNJapPvjg4BxpBjtTwPPAJMrFL8fOE7SM6QfBFVjq5uZWetxQJgWkKehP4iIx/L7a4F78oNptZTvCRweETXvBV4o+3+57LxmljsJGB4Ri1amnqY4IIyZWfM1FhDGI/KWsS9pJP3YCpbvCfwQqLkjVxq+KyK+soLnPAm4EVgEsBL1NMrryK098Dpya098j7wZJB2Z13ZPlXSDpK9LelLSFEl/lrRxfijtOODkvK5771x8n7x++4XSmu5c53JrxoHzgC1z+Quq5ctry2dJup4UEGazUiQ2Scfl8k9JelHSw7nMFXkp2MxCPcOATYCHC/k+jugm6ceSZuS/kwrnXm5dewt+/GZmVoFH5DWStB1wOrBHRLyltPNYALtFREj6PvCTiPg3SVcCCyPiV7ns94BewF7AtsAI0oYm+7N0zbiAEXnN+GnA9hHRL5evlu9vOf2o0n7gpaVkEXEl6b75aqTNVC7Ml/IfeR/zrsBISTtGxCWSfgwMLO11XrjuXYCjgX/J535S0hjSUrk+wHci4lhJfwC+RRrVm5lZK3FHXrv9gNtKHV3uDHcAbpXUC1id9MBYNXdHxBLgaUkb57T9qb5mvKixfC+VOvEqfgOMiog/5fffzlHZupF+XPQFpjVSfi/groh4B0DSncDepB8jNa1rz+cbCtB1vY0aOZWZmTWXO/KVcylwYUSMyA+4ndlI3uIacBX+rbZmnLL81fJVXS8uaQhpXfqP8vstgFOAARExNz+Et9ya8WaoaV27A8KYmbUcd+S1GwXcJenCiJiTp9Z7AK/m40cV8i4A1quhzgeA/5Z0U0QslLQp8CHLrwuvlq+qPCV+CrB3ngkgt+kdYH6eFfgyMLrQ5nWBt8qqGgtcK+k80g+Kg4Ajari2ihwQxsysvtyR1yiv9z4XGCNpMWma+0zgNklzSR39Fjn7n0j3wA8ATmikzgcrrRmPiL9KGidpBnBfRJy6AmvLfwSsT3qADWBiRHxf0hTgWeBl0vrxkuHA/ZJei4iBhTZOziP38Tnpqhxatncj5zYzs1bideTWqryO3Mys+RpbR+7lZ2ZmZu2Yp9atVTkgjLUHDghj7YlH5LYMSd+VND4HkvmdpO9Jurhw/FhJF1XJ27XtWm5m1jm5I7eP5QfqDgX2zMFoFpOejv96DiwDKTjM1VXyDq5S79AcTW7i4kXzW/w6zMw6E0+tW9EXgF2ACflJ97WAf5CeyP+a0s5nq0XEdEk/qpJ3OV5HbmbWctyRW5GA6yLip8skSv8C/Iy0bO2axvI2xevIzczqy1PrVjQSOFjSJwEkrS/pMxHxJLAZcDhwc2N526LRZmadmTty+1hEPE3aGOZBSdOAh0jx2AH+AIyLiLk15DUzs1biqXVbRkTcCtxa4dBewEU15jUzs1biEbk1SlJPSc8BA4GpOW1h27bKzMxKPCK3RkXEPGBrSbPrUZ8Dwlh74IAw1p54RN7BSOot6RlJ/yNppqQHJa0laUtJ90uaJGmspG1z/o0k3SFpQv7bM6dvkMvOlHQVS7deLT/fqbncNElnteKlmpkZ7sg7qj7AbyNiO2Ae8C3SOu4TIqK0venlOe9vgIsiYkDOd1VOPwN4NNdxF7B5+Ukk7Z/PtSvQD9hF0j4V8jkgjJlZC/HUesf0YkQ8lV9PAnoDe5C2XC3lWSP/OwjoW0hfT1J3YB/gmwARcW/eqrXc/vlvSn7fndSxP1LM5IAwZmYtxx15x/R+4fViYGNgXg6lWq4LsFtEvFdMLHTsjRHwi4j4Xa0Nc0AYM7P68tR65/A28KKkQwCU7JSPPQicUMooqdTZP0IKAIOkLwOfqFDvA8AxeQSPpE1LAWLMzKx1uCPvPAYD35M0FZgJHJDThwEN+WG1p4HjcvpZwD6SZpKm2P9WXmFEPAj8Hnhc0nTgdmDdlr0MMzMrUoRvWVrraWhoiIkTJ7Z1M8zM2hVJkyKiodIx3yNv5yQNA34ATI6IituItlI7hgANEfGjxvJ5Hbmtqrx23Nord+Tt3w+BQRHxSkufSFK3iPiopc9jZma1c0fejkm6EvgscJ+kW4Atge2B1YAzI+KPeaR8ILAOaWnYr4DVgSNIT7d/JSL+mR9yuxJYG/grcExEzJU0GniKFGv95hyu9fRcxxxgcES80UqXbGZmZfywWzsWEccBr5HioK8DjIqIXfP7CyStk7NuT3pgbQBwLrAoInYGHgeOzHmuB/49InYEppMCwpSsHhENEfFr4FHScrWdgVuAnzTVTgeEMTNrOR6Rdxz7A9+QdEp+vyZLo7E9HBELgAWS5gN/yunTgR0l9QB6RsSYnH4dcFuh7uIOZ58GbpXUizQqf7GphjkgjJlZy3FH3nEI+FZEzFomUfoXlg0Qs6Twfgm1/TfwTuH1pcCFETFC0r7Amc1ppAPCmJnVl6fWO44HgBOUQ7JJ2rnWghExH5grae+cdAQwpkr2HsCr+fVRK9hWMzOrE3fkHcd/kx5ym5aDuPx3M8sfRbqvPo20AcrZVfKdSYrZPgl4awXbamZmdVL3gDCSDgSei4inm1nuG0DfiDivrg1aSZLOBh6JiD83s1xv4J6I2L6JfNfmfLfn7UIvbO5n18x2rdD3Uy8OCGNm1nytHRDmQOAeoOaOIq9PHgGMaGaZuqxpbqyuiPivepyjFhHx/VY4TbO/n3pyQBhbVTkgjLVXTU6tS7pb0iRJMyUNLaQvlHSupKmSnpC0saQ9gG+QpmifkrRl/rs/1zFW0ra5/LWSrpT0JPBLSUMkXZaP9ZY0Ksf/Hilp80plytrZVdKvJM3I5U7I6btIGpPP/0B+2hpJoyVdLGki8B+SXpLUJR9bR9LLklbL5zw4pw+Q9Fi+5vGS1s1tHStpcv7bo4nPU5IukzRL0p+BTxaOjZbUkK/l2nwt0yWdnI9vJenP+fyTJW2Z00+VNCFf91mF+o7MaVMl3VDl++mXv79pku6S9IlCWy7Ky8aeydd+p6TnJZ2T85wt6aTC+c6VdGJT/02ZmVn91DIiPyYHDFkLmCDpjoiYQ1q3/ERE/IekXwLHRsQ5kkaQp4oBJI0EjouI55WeoL4c2C/X/Wlgj4hYrBS4pORS4LqIuE7SMcAlpJHkMmXK2jmUtO92v4j4SNL6klbLdR0QEW9KOpS0jvqYXGb10lSFpP7A54GHga8BD0TEh8rbeUpanbQM69CImCBpPeBd4B/AFyPiPUl9gJuBitMf2UHANkBf0vaiTwNXl+XpB2xampaX1DOn3wScFxF3SVoT6CJpf1Kgl11JT66PkLQPKVjL6fmzekvS+vl7LP9+pgEnRMQYpdsIZwClzvmDiGjInfMfgV2AfwJ/lXRRbvedwMX5R9BhuR3LUPoBOBSg63obNfLRmJlZc9XSkQ+TdFB+vRmp05gDfECaogWYBHyxvKDS9pZ7kB6OKiWvUchyW4UOGWB3UgATgBtYdvRdrcwg4MrSFHnutLYnBUN5KJ+/K/B6ocytZa8PJXXkh5F+cBRtA7weERNy/W/na1wHuEwpMtpiYOsKbSvaB7g5X8NrkkZVyPMC8FlJlwL3Ag9KWpfUud+Vz/9ePv/+pDXkU3LZ7qTvaCfSZ/VW6fMoP4maXj9eutUxHZgZEa/nci8Am0XEU5LmKD0hvzEwJf/IW4bXkZuZtZxGO3KldcKDgN0jYpFSuM418+EPY+mTcour1NUFmBcR/Socg2XXJ9eqOWVE6oB2r6GuEcDPJa1PGnlW6mArORl4g9RxdgHea0b7KsqhUXcC/pW0rei3gWpT1gJ+ERG/WyYx31pYScX15uVr0Uvf91XAEOBTLD+zYGZmLaypEXkPYG7uxLcFdquhzgXkPakj4m1JL0o6JCJuUxoW7xgRU5uo4zHSqPgG0j7aY2s470PA/5P0cGlqHZgFbCRp94h4PE+1bx0RM8sLR8RCSROA35CmnstH/bOAXpIG5Kn1dUlT6z2AVyJiiaSjSKP+xjyS23kd6f74QNKe3h+TtCFpWvsOSbOAGyNigaRXJB0YEXdLWiOf6wHgvyXdlK9hU+BD0g+RuyRdGBFzSlPrLPv9zJc0V9LeETGWxtePV3MXaanaasDhTWV2QBgzs/pq6mG3+4Fukp4BzgOeqKHOW4BTJU3JD2MNBr4naSowEzighjpOAI7O92+PoPpotOgq4G+kddRTgcMj4gPgYOD8nPYUaaq/mluB77LslDsAua5DgUtzXQ+RZicuB47KadvS9IzBXcDzpHvj15PinZfbFBgt6SngRuCnOf0I0q2OaaQfO5+KiAdJPwQelzQduB1YN/9YORcYk9t2Ya6j/Pupdf14RflzeRj4Q5VbHmZm1oLqvo7cOpf8kNtk4JCIeL6p/F5HbmbWfGpkHbkju9WB0hK0GR3tXE2R1Bf4CzCylk7czMzqz5umtBJJXTva1HOODvfZ5pRxQBhblTgIjHUEHpHXTzdJN+XgKbdLWlvSbEnnS5oMHCLpO0oBXmZIOr9UUBWC6+T0jZWCtEzNf6X7+10l/Y9SkJ4H8xp/VD24yzBJT+f0W3Jad0nX5PZMk/StnN5YGy/K5xwpaSOlgDKTC3n6FN+bmVnLc0deP9sAl0fE54C3gR/m9DkR0Z/0tPr5pGA4/YABSnHPYWlwnZ1yvmNz+iXAmJzen/SwIKR14r+NiO2AecC3cvr1wL9HxI6ktd9n5PTTgJ1z+nE57T+B+RGxQ04fJWmTJto4MZ9zDHBGRPwVmJ/X0AMcDVxT/sFIGqoUIW7i4kXza/s0zcysJu7I6+fliBiXX98I7JVfl56AHwCMjog3c9Cam0jBYWD54Dq98+v9gCsAImJx3m4U4MWIeKqYv0pwl1L904CbJH0XKMWUHwT8ttT4iJjbRBuXFK6leH1XkVYYdCU91b/MUrpc9/CIaIiIhq5r9yg/bGZmK8H3yOun/PH/0vtaAtjUElynqBicZTGwVhP5v0rqkL9Oiiu/Qw1takqpvXeQRv6jgEmVIrsVeR25mVl9eUReP5tLKkWQOxx4tOz4eODzkjbMo9fv0HTwlZHAD+DjTWGqDmfzaH2upL1z0hGkNeRdSOFUHwb+nRTApjtpHfzxpfL5fnpjbexCWpO/zPXlULEPkGYOlptWNzOzluWOvH5mAcfn4DmfIE+Jl+Q45aeRgqdMJY1e/9hEnScCA3Ogl0mkjVYaUym4S1fgxlzHFOCSiJgHnAN8Ij/UNhUY2EQb3wF2zUvf9mPZwDE3kabeH2yifWZmVmcOCGM1kbQwIrpXOXYK0CMi/rOpehwQxsys+RoLCON75FaRpOOARRFxfRP57gK2ZOnWtGZm1oo8IrdWtUavPtHrqIvbuhnWyTjwi7V3DtFqQNo7XdK9ObjMDEmH5qA1v8xBYMZL2irnPTNPmSNpdA5sM17Sc6UH6vIDeBdImpCDyvy/trw+M7POyB155/Il4LWI2Ckitiftbgc5MAxwGVBtuNwtInYFTmJpoJnv5bIDSGvQj5W0RXlBB4QxM2s57sg7l+nAF/Poeu9CgJmbC//uXrkod+Z/iwFr9geOVNpu9UlgA1LUuWU4IIyZWcvxw26dSEQ8J6k/8BXgHEkjS4eK2aoULwWhKQasEXBCRDxQaxscEMbMrL48Iu9Eciz1RRFxI3ABKX47pNCqpX8fb0aVDwA/kLRarn9rSevUq71mZtY0j8g7lx1IAWOWAB+SosbdTgoMM4006v5OM+q7ijTNPlmSgDeBAxstYWZmdeXlZx2QpN7APfmBtmL6VcCFeR/xUtpsoCEi3mqNtjkgjJlZ8zkgjAEQEd9v6zZMf3U+vU+7t62bYZ2M15FbR+Z75B1XN0k3SXpG0u2S1s7rwRsAJH0nx19fCJxaKiRpoaSLJM2UNFLSRjl9mKSn83rxWyR1kfR84XgXSX8pvTczs9bhjrzj2ga4PCI+B7wN/LB0ID/0dj4prGo/YICk0r3tdYCJEbEdaeez0prx04CdI2JH4LiIWELal3xwPj4ImBoRb5Y3xOvIzcxajjvyjuvliBiXX98I7FU4NgAYHRFvRsRHpN3L9snHlgC3Vig3DbhJ0neBj3La1cCR+fUxVNnG1OvIzcxajjvyjqv8KcYVfaqxVO6rwG9JS9YmSOoWES8Db0jaD9gVuG8Fz2FmZivID7t1XJtL2j0iHgcOBx4Fvp6PjQcukbQhMJe05OzSfKwLcDBwS6mcpC7AZhHxsKRHgcOA7sA80hK0G4EbImJxU41yQBgzs/ryiLzjmgUcL+kZ4BPAFaUDEfE66Z73w8BUYFJE/DEffgfYVdIM0j30s4GuwI354bgpwCURMS/nH0Hq1CtOq5uZWcvyOnJbhqSFEdG9GfkbgIsiYu8a8y8g/cjoLDYEWmWN/iqgM10rdK7r7UzXCqvm9X4mIiquCvLUuq0wSaeRosMNbipvwaxqQQ06IkkTO8v1dqZrhc51vZ3pWqH9Xa+n1m0ZzRmNR8R5EfGZiHi0JdtkZmbVuSM3MzNrx9yRW2sb3tYNaGWd6Xo707VC57reznSt0M6u1w+7mZmZtWMekZuZmbVj7sjNzMzaMXfk1mokfUnSrLxL2mlt3Z56krSZpIfzDnEzJZ2Y09eX9FDeKe4hSZ9o67bWi6SukqZIuie/30LSk/n7vVXS6m3dxnqR1DPvIvhs3lFw9w7+3Z6c/zueIelmSWt2pO9X0tWS/pEDX5XSKn6fSi7J1z1NUv+2a3ll7sitVUjqSorV/mWgL/AdSX3btlV19RHwbxHRF9iNFFWvLymC3siI6AOMzO87ihOBZwrvzycFB9qKFPr3e23SqpbxG+D+iNgW2Il03R3yu5W0KTAMaIiI7UmRHQ+jY32/1wJfKkur9n1+GeiT/4ZSiJK5qnBHbq1lV+AvEfFCRHxAiuV+QBu3qW4i4vWImJxfLyD9j35T0jVel7NdBxxYuYb2RdKnSRvpXJXfixTS9/acpSNdaw/S7oD/CxARH+QQxR3yu826AWtJ6gasDbxOB/p+I+IR4J9lydW+zwOA6yN5AugpqVfrtLQ27sittWwKvFx4/0pO63Ak9QZ2Bp4ENs6x7QH+DmzcRs2qt4uBn5C2vQXYAJiXt8WFjvX9bgG8CVyTbyVcJWkdOuh3GxGvAr8C/kbqwOcDk+i4329Jte9zlf9/lztyszqS1B24AzgpIt4uHou01rPdr/eU9DXgHxExqa3b0kq6kbbvvSIidiZtLLTMNHpH+W4B8r3hA0g/YDYB1mH5aegOrb19n+7IrbW8CmxWeP/pnNZhSFqN1InfFBF35uQ3StNw+d9/tFX76mhP4BuSZpNukexHuofcM0/FQsf6fl8BXomIJ/P720kde0f8bgEGAS9GxJsR8SFwJ+k776jfb0m173OV/3+XO3JrLROAPvnJ19VJD8+MaOM21U2+R/y/wDMRcWHh0AjgqPz6KOCP5WXbm4j4aUR8OiJ6k77HURExmLQt7sE5W4e4VoCI+DvwsqRtctIXgKfpgN9t9jdgN0lr5/+uS9fbIb/fgmrf5wjgyPz0+m7A/MIU/CrBkd2s1Uj6Cunealfg6og4t42bVDeS9gLGAtNZet/4Z6T75H8ANgdeAr4dEeUP2bRbkvYFTomIr0n6LGmEvj5p3/rvRsT7bdm+epHUj/Rg3+rAC8DRpIFQh/xuJZ0FHEpajTEF+D7pvnCH+H4l3QzsS9qu9A3gDOBuKnyf+cfMZaTbC4uAoyNiYlu0uxp35GZmZu2Yp9bNzMz+f3v382JTGMdx/P2JGgvKgrJUSpIkk43Q+AvEZrYWkpRiQ5bWLCws5j8gKQuylF8rGpmwlNkqC8VCMX0tzr3KWNw749I8x/u1up3znKfnbO7nnlP3+TTMIJckqWEGuSRJDTPIJUlqmEEuSVLDDHJJvZNkW5JbSd4lmU/yIMnOCc4/k+TgpOaT/oRBLqlXBv/7vQs8qqodVTUNXGaye6HPAAa51gSDXFLfHAW+VdXc8EBVLQDPklwddGy/TjILP5+u7w/HJrmR5OTg82KSK0leDq7ZNSjFOQNcSPIqyeF/eG/Sb9aPHiJJTdlD19a13AlgH12f+BbgRZInY8z3sar2JzlLt4vdqSRzwJequjaxVUur5BO5pP/FIeBmVS1V1QfgMXBgjOuGBTjzwPa/tDZp1QxySX3zFphewfjv/PpduGHZ+eF+4kv4FlNrkEEuqW8eAlNJTg8PJNkLfAJmk6xLshU4AjynK8jYnWQqyWa6tq9RPgObJr90aeX8dSmpV6qqkhwHrie5BHwFFoHzwEZgASjg4qCilCS3gTfAe7pmr1HuAXeSHAPOVdXTid+INCbbzyRJapiv1iVJaphBLklSwwxySZIaZpBLktQwg1ySpIYZ5JIkNcwglySpYT8AYmMdlwu8rygAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# number of keywords with less than 3 appearances in the training data\n",
        "len(keyword_counts[keyword_counts < 3])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CREpnh6oU21P",
        "outputId": "20055547-2106-4408-f597-15652d64c48d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7237"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This data has a number of imperfections. Quality assurance is meaningful here to improve the predictabilty of the model.\n",
        "\n",
        "TODO\n",
        "* Define and execute a plan to handle abbreviations"
      ],
      "metadata": {
        "id": "8PaGXsFt86CI"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Annotation"
      ],
      "metadata": {
        "id": "cVS7afXY8iEG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# this list will contain all annotated examples\n",
        "annotated_list = [] \n",
        "\n",
        "def structure_data(text, kw_list):\n",
        "  \"\"\"\n",
        "  Automatically extract the spans of each keyword in text.\n",
        "  Shape the data into JSON annotation format.\n",
        "  \"\"\"\n",
        "  results = []\n",
        "  entities = []\n",
        "\n",
        "  kw_list = kw_list.split(', ')\n",
        "  \n",
        "  # search for instances of keywords within the text (ignoring letter case)\n",
        "  for kw in kw_list:\n",
        "    search = re.finditer(kw, text, flags=re.IGNORECASE)\n",
        "      \n",
        "    # store the start/end character positions\n",
        "    all_instances = [[m.start(), m.end()] for m in search] \n",
        "      \n",
        "    # if the callable_iterator found matches, create an 'entities' list\n",
        "    if len(all_instances)>0:\n",
        "      for i in all_instances:\n",
        "        start = i[0]\n",
        "        end = i[1]\n",
        "        entities.append((start, end, \"CLINICAL\"))\n",
        "          \n",
        "  # add any found entities into a JSON format within collective_dict\n",
        "  if len(entities)>0:\n",
        "    results = [text, {\"entities\": entities}]\n",
        "    annotated_list.append(results)"
      ],
      "metadata": {
        "id": "mNPleWijy-lX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "# the data is currently ordered by specialty\n",
        "# randomly shuffle the entire dataset\n",
        "shuffled = df.sample(frac=1, random_state=42)\n",
        "\n",
        "df.apply(lambda x: structure_data(x['transcription'], x['keywords']), axis=1)"
      ],
      "metadata": {
        "id": "y245IOAt2aXN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# split the annotated data into 80% train, 20% test data\n",
        "train_test_split = int(len(annotated_list)*0.80)\n",
        "\n",
        "TRAIN_DATA = {\"DATA\": annotated_list[:train_test_split]}\n",
        "TEST_DATA = {\"DATA\": annotated_list[train_test_split:]}"
      ],
      "metadata": {
        "id": "flfJWtDb788a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# sneak peek into the training data\n",
        "TRAIN_DATA['DATA'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F2y0LXiuFDHq",
        "outputId": "1ae692be-0c67-4a58-b2fd-fcc2066dc771"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['this 23-year-old white female presents with complaint of allergies. she used to have allergies when she lived in seattle but she thinks they are worse here. in the past, she has tried claritin, and zyrtec. both worked for short time but then seemed to lose effectiveness. she has used allegra also. she used that last summer and she began using it again two weeks ago. it does not appear to be working very well. she has used over-the-counter sprays but no prescription nasal sprays. she does have asthma but doest not require daily medication for this and does not think it is flaring up. her only medication currently is ortho tri-cyclen and the allegra. she has no known medicine allergies. vitals: weight was 130 pounds and blood pressure 124/7,heent: her throat was mildly erythematous without exudate. nasal mucosa was erythematous and swollen. only clear drainage was seen. tms were clear. neck: supple without adenopathy. lungs: clear. allergic rhinitis. plan: she will try zyrtec instead of allegra again. another option will be to use loratadine. she does not think she has prescription coverage so that might be cheaper. samples of nasonex two sprays in each nostril given for three weeks. a prescription was written as well.',\n",
              " {'entities': [(944, 961, 'CLINICAL'),\n",
              "   (57, 66, 'CLINICAL'),\n",
              "   (85, 94, 'CLINICAL'),\n",
              "   (683, 692, 'CLINICAL'),\n",
              "   (498, 504, 'CLINICAL'),\n",
              "   (470, 482, 'CLINICAL'),\n",
              "   (953, 961, 'CLINICAL'),\n",
              "   (470, 475, 'CLINICAL'),\n",
              "   (808, 813, 'CLINICAL'),\n",
              "   (778, 790, 'CLINICAL'),\n",
              "   (825, 837, 'CLINICAL'),\n",
              "   (285, 292, 'CLINICAL'),\n",
              "   (648, 655, 'CLINICAL'),\n",
              "   (1000, 1007, 'CLINICAL'),\n",
              "   (443, 449, 'CLINICAL'),\n",
              "   (476, 482, 'CLINICAL'),\n",
              "   (1155, 1161, 'CLINICAL'),\n",
              "   (944, 952, 'CLINICAL')]}]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Saving the Training Data"
      ],
      "metadata": {
        "id": "_mV3rVhoTjiW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def save_data(filename, data):\n",
        "  \"\"\"\n",
        "  Option to timestamp and save JSON data to local directory.\n",
        "  \"\"\"\n",
        "  # add a timestamp to the filename\n",
        "  now = datetime.now() \n",
        "  datetime_str = now.strftime(\"%m_%d_%Y-%H.%MMT\") # using military time\n",
        "  \n",
        "  split = filename.split('.')\n",
        "  split[0] = split[0]+'--'+datetime_str+'.'\n",
        "  filename = ''.join(split)\n",
        "  \n",
        "  with open (filename, \"w\", encoding=\"utf-8\") as f:\n",
        "    json.dump(data, f, indent=4)\n",
        "\n",
        "# save_data('Training_Data', TRAIN_DATA)\n",
        "# save_data('Test_Data', TEST_DATA)"
      ],
      "metadata": {
        "id": "ILIML6Y7ThYJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Preparing SpaCy Doc Objects"
      ],
      "metadata": {
        "id": "yx-6oqCITwU0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nlp = spacy.blank(\"en\") # load a new spacy model\n",
        "doc_bin = DocBin() # create a DocBin object"
      ],
      "metadata": {
        "id": "HFZfbEWGi3OB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_training(DATA):\n",
        "  \"\"\"\n",
        "  Convert the examples from JSON format into SpaCy Doc objects. \n",
        "  The Doc objects are then stored in a collective DocBin class.\n",
        "  \"\"\"\n",
        "  db = DocBin()\n",
        "  for text, annot in tqdm(DATA):\n",
        "    doc = nlp.make_doc(text)\n",
        "    ents = []\n",
        "\n",
        "    # create span objects\n",
        "    for start, end, label in annot[\"entities\"]:\n",
        "      span = doc.char_span(start, end, label=label, alignment_mode=\"contract\") \n",
        "\n",
        "      # skip if the character indices do not map to a valid span\n",
        "      if span is None:\n",
        "          print(\"Skipping entity.\")\n",
        "      else:\n",
        "          ents.append(span)\n",
        "\n",
        "    # remove duplicate or overlapping span objects (while keeping the first longest span)\n",
        "    filtered_ents = filter_spans(ents)\n",
        "   \n",
        "    # pack Doc objects into DocBin\n",
        "    doc.ents = filtered_ents\n",
        "    db.add(doc) \n",
        "\n",
        "  return db"
      ],
      "metadata": {
        "id": "YS5wWzNuk74S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture \n",
        "\n",
        "TRAIN_DATA_DOC = create_training(TRAIN_DATA['DATA'])\n",
        "TEST_DATA_DOC = create_training(TEST_DATA['DATA'])"
      ],
      "metadata": {
        "id": "jXInHhFto-zP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# export results \n",
        "TRAIN_DATA_DOC.to_disk(\"./TRAIN_DATA/TRAIN_DATA.spacy\")\n",
        "TEST_DATA_DOC.to_disk(\"./TEST_DATA/TEST_DATA.spacy\")"
      ],
      "metadata": {
        "id": "zUw_1luovgcl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## SpaCy Model Training"
      ],
      "metadata": {
        "id": "MK20DBVUlhE1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# auto-fill the config file with any remaining default parameters \n",
        "!python -m spacy init fill-config /content/base_config.cfg config.cfg"
      ],
      "metadata": {
        "id": "KXjTpDbqU9xA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff3d0790-0ff5-44b6-ea0a-f6b5d63b08e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
            "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
            "config.cfg\n",
            "You can now add your data and train your pipeline:\n",
            "python -m spacy train config.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# validate the training and development data\n",
        "!python -m spacy debug data /content/config.cfg"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jmRntNTtT1_5",
        "outputId": "e0b56844-11a2-4f3b-da04-2371afa5a3c4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m\n",
            "============================ Data file validation ============================\u001b[0m\n",
            "Downloading: 100% 28.0/28.0 [00:00<00:00, 28.4kB/s]\n",
            "Downloading: 100% 385/385 [00:00<00:00, 370kB/s]\n",
            "Downloading: 100% 220k/220k [00:00<00:00, 312kB/s]\n",
            "Downloading: 100% 420M/420M [00:06<00:00, 70.1MB/s]\n",
            "Some weights of the model checkpoint at microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.decoder.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "\u001b[38;5;2m✔ Pipeline can be initialized with data\u001b[0m\n",
            "\u001b[38;5;2m✔ Corpus is loadable\u001b[0m\n",
            "\u001b[1m\n",
            "=============================== Training stats ===============================\u001b[0m\n",
            "Language: en\n",
            "Training pipeline: transformer, ner\n",
            "1512 training docs\n",
            "593 evaluation docs\n",
            "\u001b[38;5;3m⚠ 304 training examples also in evaluation data\u001b[0m\n",
            "\u001b[38;5;3m⚠ Low number of examples to train a new pipeline (1512)\u001b[0m\n",
            "\u001b[1m\n",
            "============================== Vocab & Vectors ==============================\u001b[0m\n",
            "\u001b[38;5;4mℹ 429087 total word(s) in the data (11640 unique)\u001b[0m\n",
            "\u001b[38;5;4mℹ No word vectors present in the package\u001b[0m\n",
            "\u001b[1m\n",
            "========================== Named Entity Recognition ==========================\u001b[0m\n",
            "\u001b[38;5;4mℹ 1 label(s)\u001b[0m\n",
            "0 missing value(s) (tokens with '-' label)\n",
            "\u001b[38;5;2m✔ Good amount of examples for all labels\u001b[0m\n",
            "\u001b[38;5;2m✔ Examples without occurrences available for all labels\u001b[0m\n",
            "\u001b[38;5;2m✔ No entities consisting of or starting/ending with whitespace\u001b[0m\n",
            "\u001b[38;5;2m✔ No entities crossing sentence boundaries\u001b[0m\n",
            "\u001b[1m\n",
            "================================== Summary ==================================\u001b[0m\n",
            "\u001b[38;5;2m✔ 6 checks passed\u001b[0m\n",
            "\u001b[38;5;3m⚠ 2 warnings\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# train the model using GPU\n",
        "!python -m spacy train --gpu-id 0 /content/config.cfg --output ./output"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "84MpzioCT2Df",
        "outputId": "e20b2a0f-2b80-4e6b-9d8b-75cc4d93b134"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[38;5;2m✔ Created output directory: output\u001b[0m\n",
            "\u001b[38;5;4mℹ Saving to output directory: output\u001b[0m\n",
            "\u001b[38;5;4mℹ Using GPU: 0\u001b[0m\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "[2022-05-08 22:17:22,454] [INFO] Set up nlp object from config\n",
            "[2022-05-08 22:17:22,463] [INFO] Pipeline: ['transformer', 'ner']\n",
            "[2022-05-08 22:17:22,467] [INFO] Created vocabulary\n",
            "[2022-05-08 22:17:22,468] [INFO] Finished initializing nlp object\n",
            "Some weights of the model checkpoint at microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[2022-05-08 22:17:46,517] [INFO] Initialized pipeline components: ['transformer', 'ner']\n",
            "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "\u001b[38;5;4mℹ Pipeline: ['transformer', 'ner']\u001b[0m\n",
            "\u001b[38;5;4mℹ Initial learn rate: 0.0\u001b[0m\n",
            "E    #       LOSS TRANS...  LOSS NER  ENTS_F  ENTS_P  ENTS_R  SCORE \n",
            "---  ------  -------------  --------  ------  ------  ------  ------\n",
            "  0       0         452.29    632.11    2.42    1.33   13.79    0.02\n",
            "  0     200       50081.50  52695.31   36.18   51.05   28.02    0.36\n",
            "  1     400       12274.31  20842.91   47.76   42.98   53.74    0.48\n",
            "  2     600        9439.03  16325.27   51.12   48.58   53.94    0.51\n",
            "  3     800        6514.53  11406.16   55.49   58.64   52.67    0.55\n",
            "  4    1000        4599.23   7959.94   57.16   52.35   62.94    0.57\n",
            "  5    1200        3613.54   6193.26   58.47   54.83   62.62    0.58\n",
            "  5    1400        2592.87   4414.04   56.80   49.19   67.21    0.57\n",
            "  6    1600        2155.89   3841.03   60.14   55.42   65.74    0.60\n",
            "  7    1800        1777.71   3101.29   61.50   59.17   64.03    0.62\n",
            "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
            "output/model-last\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation"
      ],
      "metadata": {
        "id": "Y_yFSZCGrCGl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m spacy evaluate /content/output/model-best /content/TRAIN_DATA/TRAIN_DATA.spacy --gold-preproc --output ./eval --gpu-id 0"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gaQ3BgPCrEe-",
        "outputId": "085ff8a1-3472-4515-dae8-2bd416f2b9e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[38;5;4mℹ Using GPU: 0\u001b[0m\n",
            "\u001b[1m\n",
            "================================== Results ==================================\u001b[0m\n",
            "\n",
            "TOK     100.00\n",
            "NER P   66.38 \n",
            "NER R   70.68 \n",
            "NER F   68.46 \n",
            "SPEED   17058 \n",
            "\n",
            "\u001b[1m\n",
            "=============================== NER (per type) ===============================\u001b[0m\n",
            "\n",
            "               P       R       F\n",
            "CLINICAL   66.38   70.68   68.46\n",
            "\n",
            "\u001b[38;5;2m✔ Saved results to eval\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Save Model"
      ],
      "metadata": {
        "id": "QvFqoGawJ4dY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x6LOueCQ8vYr",
        "outputId": "6b6b58da-e113-4da6-e0c6-2aa46d3841e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# save model outputs to Drive\n",
        "!cp -r \"/content/output\" \"/content/drive/MyDrive/BioNER_Model\""
      ],
      "metadata": {
        "id": "GaIiQTS19f30"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Testing"
      ],
      "metadata": {
        "id": "CSey6DHnJ7qJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load the trained model\n",
        "nlp_output = spacy.load(\"/content/output/model-best\")"
      ],
      "metadata": {
        "id": "i3_u0yXhSQcz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# example text\n",
        "model_test = df.iloc[3,0]\n",
        "model_test"
      ],
      "metadata": {
        "id": "v4zbASfMU92p",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "7437cbb3-f797-4f95-810c-0ea2cdbc8fd7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2-d m-mode: , left atrial enlargement with left atrial diameter of 7 cm. normal size right and left ventricle. normal lv systolic function with left ventricular ejection fraction of 51%. normal lv diastolic function. no pericardial effusion. normal morphology of aortic valve, mitral valve, tricuspid valve, and pulmonary valve. pa systolic pressure is 36 mmhg. , mild mitral and tricuspid regurgitation. trace aortic and pulmonary regurgitation.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# pass the test instance into the trained pipeline\n",
        "doc = nlp_output(model_test)"
      ],
      "metadata": {
        "id": "Bj1Q9nym_sXF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# customize the label colors\n",
        "colors = {\"CLINICAL\": \"linear-gradient(90deg, #E1D436, #F59710)\"}\n",
        "options = {\"ents\": [\"CLINICAL\"], \"colors\": colors}"
      ],
      "metadata": {
        "id": "HsU8xsUISQhL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# visualize the identified entities\n",
        "displacy.render(doc, style=\"ent\", jupyter=True, options=options)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 192
        },
        "id": "kM6AQ0tNKD6k",
        "outputId": "a90f2f71-e5f1-4d67-8392-1cc793fb9be4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    2-d m-mode\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ": , left \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    atrial enlargement\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              " with left atrial diameter of 7 cm. normal size right and left ventricle. \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    normal lv\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              " \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    systolic function\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              " with left ventricular \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    ejection fraction\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              " of 51%. \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    normal lv\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              " \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    diastolic function\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ". no \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    pericardial effusion\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ". normal morphology of \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    aortic valve\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ", \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    mitral valve\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ", \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    tricuspid valve\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ", and \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    pulmonary valve\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ". pa systolic pressure is 36 mmhg. , mild \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    mitral\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              " and \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    tricuspid\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              " \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    regurgitation\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ". trace aortic and pulmonary \n",
              "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #E1D436, #F59710); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    regurgitation\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CLINICAL</span>\n",
              "</mark>\n",
              ".</div></span>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# print out the identified entities\n",
        "for ent in doc.ents:\n",
        "    if ent.label_ == \"CLINICAL\":\n",
        "        print(ent.text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eCqI6tuf6u90",
        "outputId": "ed4ef908-0590-440b-9f9d-3d7400cab775"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2-d m-mode\n",
            "atrial enlargement\n",
            "normal lv\n",
            "systolic function\n",
            "ejection fraction\n",
            "normal lv\n",
            "diastolic function\n",
            "pericardial effusion\n",
            "aortic valve\n",
            "mitral valve\n",
            "tricuspid valve\n",
            "pulmonary valve\n",
            "mitral\n",
            "tricuspid\n",
            "regurgitation\n",
            "regurgitation\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vXwt4PvMKOXy"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}