ChatGPT_VAERS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98644443",
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import random,os\n",
    "import pandas as pd\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd67ca70",
   "metadata": {},
   "outputs": [],
   "source": [
    "# VAERS\n",
    "dataset = 'VAERS'\n",
    "files = glob('./data/Vaccine-AE-recognition/test/*.bio')\n",
    "random.seed(42)\n",
    "#files = random.sample(files,k=100)\n",
    "len(files)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8eb3e0ba",
   "metadata": {},
   "source": [
    "# ChatGPT API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "834c3aaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "898979f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_output(prompt,GPT):\n",
    "    if GPT == 3.5:\n",
    "        openai.api_key = 'your api key'\n",
    "        model = 'gpt-3.5-turbo-0301'\n",
    "        #model = 'gpt-3.5-turbo'\n",
    "    elif GPT == 4:\n",
    "        openai.api_key = 'your api key'\n",
    "        model = 'gpt-4-0314'\n",
    "\n",
    "    message = openai.ChatCompletion.create(\n",
    "      model=model,\n",
    "      temperature=0,\n",
    "      messages=[\n",
    "            {\"role\": \"user\", \"content\": prompt}\n",
    "        ]\n",
    "    )\n",
    "    return message['choices'][0]['message']['content']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74fd4f01",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(GPT,prompt,prompt_type):\n",
    "    for i,file in enumerate(files):\n",
    "        with open(file,'r') as f_read:\n",
    "            text = ' '.join([line.split('\\t')[0] for line in f_read.read().splitlines()])\n",
    "        file_name = file.split('/')[-1].split('.')[0]\n",
    "\n",
    "        dir_path = f'./output/GPT-{GPT}/{dataset}/{prompt}/'\n",
    "        if not os.path.exists(dir_path):\n",
    "            os.makedirs(dir_path)\n",
    "\n",
    "        success = False\n",
    "        while not success:\n",
    "            try:\n",
    "                output = get_output(prompt.format(text),GPT)\n",
    "                with open(f'./output/GPT-{GPT}/{dataset}/{prompt}/{file_name}.html','w') as f_write:\n",
    "                    f_write.write(output)\n",
    "                    success = True\n",
    "            except Exception as e:\n",
    "                print (e)\n",
    "                pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d312e079",
   "metadata": {},
   "source": [
    "# GPT 3.5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce5d45ed",
   "metadata": {},
   "source": [
    "### baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b532d731",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 3.5\n",
    "prompt_type = 'baseline'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "435ed7f8",
   "metadata": {},
   "source": [
    "### baseline + definition + guideline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60d09c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Entity Definitions\n",
    "Investigation includes typical lab tests or examinations in the report, such as physical examination, oxygen saturation, electromyogram, etc.\n",
    "Nervous adverse event includes typically nervous system-related problems, such as guillain-barré syndrome, ataxia, areflexia, hypoaesthesia, paraesthesia, dizziness, headache and other nervous system disorders.\n",
    "Other adverse event includes medical problems that are assigned to other MedDRA SOCs, including gastrointestinal disorders, cardiac disorders, psychiatric disorders, musculoskeletal and connective tissue disorders, etc.\n",
    "Procedure includes non-medical problem events such as immunization, surgeries such as catheter placement, hospitalization, emergence care, intubation, etc.\n",
    "\n",
    "### Annotation Guidelines\n",
    "Only annotate events that already occurred (i.e., occurred before the diagnosis of GBS).\n",
    "When annotating events related to Flu-GBS, do not include prepositions including modifiers of the event.\n",
    "Separate events in discontinuous segments.\n",
    "When annotating events, more generalized events should not be annotated.\n",
    "When annotating events related to symptom improvement / progress or negation events, the following guideline should be used. In the case where the patient reported a specific adverse event first, and then reported improvement / progress of the adverse event, we should annotate it as an improved symptom. However, we do NOT need to annotate the negation of a symptom which the patient never reported before.\n",
    "Events reported as history (events that did not happen to the reporting patient) should be annotated. Family history is important for risk prediction and may be included as a baseline information (e.g., for statistical analysis).\n",
    "Some VAERS reports have duplicate events reported. For example, the same events / text are repeated twice in the report. The case we are interested in, is the recurrence of some adverse event, i.e., it requires the adverse event appears, then disappear, and then come back. In this case it should definitely be annotated twice. Additionally, we need to annotate the relief/improvement of the event if it is mentioned in the report. When no such information to decide whether it is a recurrence, the principle is that if there are multiple time stamps of the same event, we annotate it twice, if not, we can just keep one record.\n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 3.5\n",
    "prompt_type = 'baseline_definition_guideline'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2613960",
   "metadata": {},
   "source": [
    "### baseline + definition + guideline + error analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ead93ac8",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Entity Definitions\n",
    "Investigation includes typical lab tests or examinations in the report, such as physical examination, oxygen saturation, electromyogram, etc.\n",
    "Nervous adverse event includes typically nervous system-related problems, such as guillain-barré syndrome, ataxia, areflexia, hypoaesthesia, paraesthesia, dizziness, headache and other nervous system disorders.\n",
    "Other adverse event includes medical problems that are assigned to other MedDRA SOCs, including gastrointestinal disorders, cardiac disorders, psychiatric disorders, musculoskeletal and connective tissue disorders, etc.\n",
    "Procedure includes non-medical problem events such as immunization, surgeries such as catheter placement, hospitalization, emergence care, intubation, etc.\n",
    "\n",
    "### Annotation Guidelines\n",
    "Only annotate events that already occurred (i.e., occurred before the diagnosis of GBS).\n",
    "When annotating events related to Flu-GBS, do not include prepositions including modifiers of the event.\n",
    "Separate events in discontinuous segments.\n",
    "When annotating events, more generalized events should not be annotated.\n",
    "When annotating events related to symptom improvement / progress or negation events, the following guideline should be used. In the case where the patient reported a specific adverse event first, and then reported improvement / progress of the adverse event, we should annotate it as an improved symptom. However, we do NOT need to annotate the negation of a symptom which the patient never reported before.\n",
    "Events reported as history (events that did not happen to the reporting patient) should be annotated. Family history is important for risk prediction and may be included as a baseline information (e.g., for statistical analysis).\n",
    "Some VAERS reports have duplicate events reported. For example, the same events / text are repeated twice in the report. The case we are interested in, is the recurrence of some adverse event, i.e., it requires the adverse event appears, then disappear, and then come back. In this case it should definitely be annotated twice. Additionally, we need to annotate the relief/improvement of the event if it is mentioned in the report. When no such information to decide whether it is a recurrence, the principle is that if there are multiple time stamps of the same event, we annotate it twice, if not, we can just keep one record.\n",
    "When annotating events related to hospital admissions, transfers, or discharges, consider them as procedures. Specifically, annotate the words 'hospital', 'rehabilitation center', or any other healthcare facility involved in the patient's care as a procedure.\n",
    "All abnormal symptoms should be considered as adverse events.\n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 3.5\n",
    "prompt_type = 'baseline_definition_guideline_erroranalysis'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9144ec22",
   "metadata": {},
   "source": [
    "### baseline + definition + guideline + error analysis + 1shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3c57bd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Entity Definitions\n",
    "Investigation includes typical lab tests or examinations in the report, such as physical examination, oxygen saturation, electromyogram, etc.\n",
    "Nervous adverse event includes typically nervous system-related problems, such as guillain-barré syndrome, ataxia, areflexia, hypoaesthesia, paraesthesia, dizziness, headache and other nervous system disorders.\n",
    "Other adverse event includes medical problems that are assigned to other MedDRA SOCs, including gastrointestinal disorders, cardiac disorders, psychiatric disorders, musculoskeletal and connective tissue disorders, etc.\n",
    "Procedure includes non-medical problem events such as immunization, surgeries such as catheter placement, hospitalization, emergence care, intubation, etc.\n",
    "\n",
    "### Annotation Guidelines\n",
    "Only annotate events that already occurred (i.e., occurred before the diagnosis of GBS).\n",
    "When annotating events related to Flu-GBS, do not include prepositions including modifiers of the event.\n",
    "Separate events in discontinuous segments.\n",
    "When annotating events, more generalized events should not be annotated.\n",
    "When annotating events related to symptom improvement / progress or negation events, the following guideline should be used. In the case where the patient reported a specific adverse event first, and then reported improvement / progress of the adverse event, we should annotate it as an improved symptom. However, we do NOT need to annotate the negation of a symptom which the patient never reported before.\n",
    "Events reported as history (events that did not happen to the reporting patient) should be annotated. Family history is important for risk prediction and may be included as a baseline information (e.g., for statistical analysis).\n",
    "Some VAERS reports have duplicate events reported. For example, the same events / text are repeated twice in the report. The case we are interested in, is the recurrence of some adverse event, i.e., it requires the adverse event appears, then disappear, and then come back. In this case it should definitely be annotated twice. Additionally, we need to annotate the relief/improvement of the event if it is mentioned in the report. When no such information to decide whether it is a recurrence, the principle is that if there are multiple time stamps of the same event, we annotate it twice, if not, we can just keep one record.\n",
    "When annotating events related to hospital admissions, transfers, or discharges, consider them as procedures. Specifically, annotate the words 'hospital', 'rehabilitation center', or any other healthcare facility involved in the patient's care as a procedure.\n",
    "All abnormal symptoms should be considered as adverse events.\n",
    "\n",
    "### Examples \n",
    "Example Input1: she said the doctor diagnosed her husband with Gillian Barre related to flu vaccine . Seen in ER for C / O numbness in both hands ( fingertips ) and both feet with c / o abd pain ongoing times 5 days .  \n",
    "Example Output1: she said the doctor diagnosed her husband with <span class=\"nervous_AE\">Gillian Barre</span> related to <span class=\"procedure\">flu vaccine</span> . Seen in <span class=\"procedure\">ER</span> for C / O <span class=\"nervous_AE\">numbness in both hands ( fingertips ) and both feet</span> with c / o <span class=\"other_AE\">abd pain</span> ongoing times 5 days .  \n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 3.5\n",
    "prompt_type = 'baseline_definition_guideline_erroranalysis_1shot'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0cee07d",
   "metadata": {},
   "source": [
    "### baseline + definition + guideline + error analysis + 5shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d185eaa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Entity Definitions\n",
    "Investigation includes typical lab tests or examinations in the report, such as physical examination, oxygen saturation, electromyogram, etc.\n",
    "Nervous adverse event includes typically nervous system-related problems, such as guillain-barré syndrome, ataxia, areflexia, hypoaesthesia, paraesthesia, dizziness, headache and other nervous system disorders.\n",
    "Other adverse event includes medical problems that are assigned to other MedDRA SOCs, including gastrointestinal disorders, cardiac disorders, psychiatric disorders, musculoskeletal and connective tissue disorders, etc.\n",
    "Procedure includes non-medical problem events such as immunization, surgeries such as catheter placement, hospitalization, emergence care, intubation, etc.\n",
    "\n",
    "### Annotation Guidelines\n",
    "Only annotate events that already occurred (i.e., occurred before the diagnosis of GBS).\n",
    "When annotating events related to Flu-GBS, do not include prepositions including modifiers of the event.\n",
    "Separate events in discontinuous segments.\n",
    "When annotating events, more generalized events should not be annotated.\n",
    "When annotating events related to symptom improvement / progress or negation events, the following guideline should be used. In the case where the patient reported a specific adverse event first, and then reported improvement / progress of the adverse event, we should annotate it as an improved symptom. However, we do NOT need to annotate the negation of a symptom which the patient never reported before.\n",
    "Events reported as history (events that did not happen to the reporting patient) should be annotated. Family history is important for risk prediction and may be included as a baseline information (e.g., for statistical analysis).\n",
    "Some VAERS reports have duplicate events reported. For example, the same events / text are repeated twice in the report. The case we are interested in, is the recurrence of some adverse event, i.e., it requires the adverse event appears, then disappear, and then come back. In this case it should definitely be annotated twice. Additionally, we need to annotate the relief/improvement of the event if it is mentioned in the report. When no such information to decide whether it is a recurrence, the principle is that if there are multiple time stamps of the same event, we annotate it twice, if not, we can just keep one record.\n",
    "When annotating events related to hospital admissions, transfers, or discharges, consider them as procedures. Specifically, annotate the words 'hospital', 'rehabilitation center', or any other healthcare facility involved in the patient's care as a procedure.\n",
    "All abnormal symptoms should be considered as adverse events.\n",
    "\n",
    "### Examples \n",
    "Example Input1: she said the doctor diagnosed her husband with Gillian Barre related to flu vaccine . Seen in ER for C / O numbness in both hands ( fingertips ) and both feet with c / o abd pain ongoing times 5 days .  \n",
    "Example Output1: she said the doctor diagnosed her husband with <span class=\"nervous_AE\">Gillian Barre</span> related to <span class=\"procedure\">flu vaccine</span> . Seen in <span class=\"procedure\">ER</span> for C / O <span class=\"nervous_AE\">numbness in both hands ( fingertips ) and both feet</span> with c / o <span class=\"other_AE\">abd pain</span> ongoing times 5 days .  \n",
    "Example Input2: He was seen in an emergency room the following day and was admitted to the hospital because he could not walk .  \n",
    "Example Output2: He was seen in an <span class=\"procedure\">emergency room</span> the following day and was admitted to the <span class=\"procedure\">hospital</span> because he <span class=\"nervous_AE\">could not walk</span> .  \n",
    "Example Input3: On 20 September 2009 the patient was diagnosed with GUILLAIN BARRE Syndrome .  \n",
    "Example Output3: On 20 September 2009 the patient was diagnosed with <span class=\"nervous_AE\">GUILLAIN BARRE Syndrome</span> .  \n",
    "Example Input4: No improvement as of yet 4 days of IVIG beginning 1 / 15 / 09 ( Hospitalized ) .  \n",
    "Example Output4: No improvement as of yet 4 days of <span class=\"procedure\">IVIG</span> beginning 1 / 15 / 09 ( <span class=\"procedure\">Hospitalized</span> ) .  \n",
    "Example Input5: Tx with plasmapheresis .  \n",
    "Example Output5: Tx with <span class=\"procedure\">plasmapheresis</span> .  \n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 3.5\n",
    "prompt_type = 'baseline_definition_guideline_erroranalysis_5shot'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07536257",
   "metadata": {},
   "source": [
    "# GPT-4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52bb227a",
   "metadata": {},
   "source": [
    "### baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74f1a8ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 4\n",
    "prompt_type = 'baseline'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8403cdc",
   "metadata": {},
   "source": [
    "### baseline + definition + guidelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e0e61c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Entity Definitions\n",
    "Investigation includes typical lab tests or examinations in the report, such as physical examination, oxygen saturation, electromyogram, etc.\n",
    "Nervous adverse event includes typically nervous system-related problems, such as guillain-barré syndrome, ataxia, areflexia, hypoaesthesia, paraesthesia, dizziness, headache and other nervous system disorders.\n",
    "Other adverse event includes medical problems that are assigned to other MedDRA SOCs, including gastrointestinal disorders, cardiac disorders, psychiatric disorders, musculoskeletal and connective tissue disorders, etc.\n",
    "Procedure includes non-medical problem events such as immunization, surgeries such as catheter placement, hospitalization, emergence care, intubation, etc.\n",
    "\n",
    "### Annotation Guidelines\n",
    "Only annotate events that already occurred (i.e., occurred before the diagnosis of GBS).\n",
    "When annotating events related to Flu-GBS, do not include prepositions including modifiers of the event.\n",
    "Separate events in discontinuous segments.\n",
    "When annotating events, more generalized events should not be annotated.\n",
    "When annotating events related to symptom improvement / progress or negation events, the following guideline should be used. In the case where the patient reported a specific adverse event first, and then reported improvement / progress of the adverse event, we should annotate it as an improved symptom. However, we do NOT need to annotate the negation of a symptom which the patient never reported before.\n",
    "Events reported as history (events that did not happen to the reporting patient) should be annotated. Family history is important for risk prediction and may be included as a baseline information (e.g., for statistical analysis).\n",
    "Some VAERS reports have duplicate events reported. For example, the same events / text are repeated twice in the report. The case we are interested in, is the recurrence of some adverse event, i.e., it requires the adverse event appears, then disappear, and then come back. In this case it should definitely be annotated twice. Additionally, we need to annotate the relief/improvement of the event if it is mentioned in the report. When no such information to decide whether it is a recurrence, the principle is that if there are multiple time stamps of the same event, we annotate it twice, if not, we can just keep one record.\n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 4\n",
    "prompt_type = 'baseline_definition_guideline'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d3a350d",
   "metadata": {},
   "source": [
    "### baseline + definition + guideline + error analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "504387a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Entity Definitions\n",
    "Investigation includes typical lab tests or examinations in the report, such as physical examination, oxygen saturation, electromyogram, etc.\n",
    "Nervous adverse event includes typically nervous system-related problems, such as guillain-barré syndrome, ataxia, areflexia, hypoaesthesia, paraesthesia, dizziness, headache and other nervous system disorders.\n",
    "Other adverse event includes medical problems that are assigned to other MedDRA SOCs, including gastrointestinal disorders, cardiac disorders, psychiatric disorders, musculoskeletal and connective tissue disorders, etc.\n",
    "Procedure includes non-medical problem events such as immunization, surgeries such as catheter placement, hospitalization, emergence care, intubation, etc.\n",
    "\n",
    "### Annotation Guidelines\n",
    "Only annotate events that already occurred (i.e., occurred before the diagnosis of GBS).\n",
    "When annotating events related to Flu-GBS, do not include prepositions including modifiers of the event.\n",
    "Separate events in discontinuous segments.\n",
    "When annotating events, more generalized events should not be annotated.\n",
    "When annotating events related to symptom improvement / progress or negation events, the following guideline should be used. In the case where the patient reported a specific adverse event first, and then reported improvement / progress of the adverse event, we should annotate it as an improved symptom. However, we do NOT need to annotate the negation of a symptom which the patient never reported before.\n",
    "Events reported as history (events that did not happen to the reporting patient) should be annotated. Family history is important for risk prediction and may be included as a baseline information (e.g., for statistical analysis).\n",
    "Some VAERS reports have duplicate events reported. For example, the same events / text are repeated twice in the report. The case we are interested in, is the recurrence of some adverse event, i.e., it requires the adverse event appears, then disappear, and then come back. In this case it should definitely be annotated twice. Additionally, we need to annotate the relief/improvement of the event if it is mentioned in the report. When no such information to decide whether it is a recurrence, the principle is that if there are multiple time stamps of the same event, we annotate it twice, if not, we can just keep one record.\n",
    "When annotating events related to hospital admissions, transfers, or discharges, consider them as procedures. Specifically, annotate the words 'hospital', 'rehabilitation center', or any other healthcare facility involved in the patient's care as a procedure.\n",
    "All abnormal symptoms should be considered as adverse events.\n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 4\n",
    "prompt_type = 'baseline_definition_guideline_erroranalysis'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b325b45b",
   "metadata": {},
   "source": [
    "### baseline + definition + guideline + error analysis + 1shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f2da4bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Entity Definitions\n",
    "Investigation includes typical lab tests or examinations in the report, such as physical examination, oxygen saturation, electromyogram, etc.\n",
    "Nervous adverse event includes typically nervous system-related problems, such as guillain-barré syndrome, ataxia, areflexia, hypoaesthesia, paraesthesia, dizziness, headache and other nervous system disorders.\n",
    "Other adverse event includes medical problems that are assigned to other MedDRA SOCs, including gastrointestinal disorders, cardiac disorders, psychiatric disorders, musculoskeletal and connective tissue disorders, etc.\n",
    "Procedure includes non-medical problem events such as immunization, surgeries such as catheter placement, hospitalization, emergence care, intubation, etc.\n",
    "\n",
    "### Annotation Guidelines\n",
    "Only annotate events that already occurred (i.e., occurred before the diagnosis of GBS).\n",
    "When annotating events related to Flu-GBS, do not include prepositions including modifiers of the event.\n",
    "Separate events in discontinuous segments.\n",
    "When annotating events, more generalized events should not be annotated.\n",
    "When annotating events related to symptom improvement / progress or negation events, the following guideline should be used. In the case where the patient reported a specific adverse event first, and then reported improvement / progress of the adverse event, we should annotate it as an improved symptom. However, we do NOT need to annotate the negation of a symptom which the patient never reported before.\n",
    "Events reported as history (events that did not happen to the reporting patient) should be annotated. Family history is important for risk prediction and may be included as a baseline information (e.g., for statistical analysis).\n",
    "Some VAERS reports have duplicate events reported. For example, the same events / text are repeated twice in the report. The case we are interested in, is the recurrence of some adverse event, i.e., it requires the adverse event appears, then disappear, and then come back. In this case it should definitely be annotated twice. Additionally, we need to annotate the relief/improvement of the event if it is mentioned in the report. When no such information to decide whether it is a recurrence, the principle is that if there are multiple time stamps of the same event, we annotate it twice, if not, we can just keep one record.\n",
    "When annotating events related to hospital admissions, transfers, or discharges, consider them as procedures. Specifically, annotate the words 'hospital', 'rehabilitation center', or any other healthcare facility involved in the patient's care as a procedure.\n",
    "All abnormal symptoms should be considered as adverse events.\n",
    "\n",
    "### Examples \n",
    "Example Input1: she said the doctor diagnosed her husband with Gillian Barre related to flu vaccine . Seen in ER for C / O numbness in both hands ( fingertips ) and both feet with c / o abd pain ongoing times 5 days .  \n",
    "Example Output1: she said the doctor diagnosed her husband with <span class=\"nervous_AE\">Gillian Barre</span> related to <span class=\"procedure\">flu vaccine</span> . Seen in <span class=\"procedure\">ER</span> for C / O <span class=\"nervous_AE\">numbness in both hands ( fingertips ) and both feet</span> with c / o <span class=\"other_AE\">abd pain</span> ongoing times 5 days .  \n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 4\n",
    "prompt_type = 'baseline_definition_guideline_erroranalysis_1shot'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e598ca8",
   "metadata": {},
   "source": [
    "### baseline + definition + guideline + error analysis + 5shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87a6e003",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = '''### Task\n",
    "Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: 'investigations', 'nervous adverse events', 'other adverse events', and 'procedures'. Use HTML <span> tags to highlight these entities. Each <span> should have a class attribute indicating the type of the entity.\n",
    "\n",
    "### Entity Markup Guide\n",
    "Use <span class=\"investigation\"> to denote an investigation.\n",
    "Use <span class=\"nervous_AE\"> to denote a nervous adverse event.\n",
    "Use <span class=\"other_AE\"> to denote an other adverse event.\n",
    "Use <span class=\"procedure\"> to denote a procedure.\n",
    "If no entity found, leave the text as it is.\n",
    "\n",
    "### Entity Definitions\n",
    "Investigation includes typical lab tests or examinations in the report, such as physical examination, oxygen saturation, electromyogram, etc.\n",
    "Nervous adverse event includes typically nervous system-related problems, such as guillain-barré syndrome, ataxia, areflexia, hypoaesthesia, paraesthesia, dizziness, headache and other nervous system disorders.\n",
    "Other adverse event includes medical problems that are assigned to other MedDRA SOCs, including gastrointestinal disorders, cardiac disorders, psychiatric disorders, musculoskeletal and connective tissue disorders, etc.\n",
    "Procedure includes non-medical problem events such as immunization, surgeries such as catheter placement, hospitalization, emergence care, intubation, etc.\n",
    "\n",
    "### Annotation Guidelines\n",
    "Only annotate events that already occurred (i.e., occurred before the diagnosis of GBS).\n",
    "When annotating events related to Flu-GBS, do not include prepositions including modifiers of the event.\n",
    "Separate events in discontinuous segments.\n",
    "When annotating events, more generalized events should not be annotated.\n",
    "When annotating events related to symptom improvement / progress or negation events, the following guideline should be used. In the case where the patient reported a specific adverse event first, and then reported improvement / progress of the adverse event, we should annotate it as an improved symptom. However, we do NOT need to annotate the negation of a symptom which the patient never reported before.\n",
    "Events reported as history (events that did not happen to the reporting patient) should be annotated. Family history is important for risk prediction and may be included as a baseline information (e.g., for statistical analysis).\n",
    "Some VAERS reports have duplicate events reported. For example, the same events / text are repeated twice in the report. The case we are interested in, is the recurrence of some adverse event, i.e., it requires the adverse event appears, then disappear, and then come back. In this case it should definitely be annotated twice. Additionally, we need to annotate the relief/improvement of the event if it is mentioned in the report. When no such information to decide whether it is a recurrence, the principle is that if there are multiple time stamps of the same event, we annotate it twice, if not, we can just keep one record.\n",
    "When annotating events related to hospital admissions, transfers, or discharges, consider them as procedures. Specifically, annotate the words 'hospital', 'rehabilitation center', or any other healthcare facility involved in the patient's care as a procedure.\n",
    "All abnormal symptoms should be considered as adverse events.\n",
    "\n",
    "### Examples \n",
    "Example Input1: she said the doctor diagnosed her husband with Gillian Barre related to flu vaccine . Seen in ER for C / O numbness in both hands ( fingertips ) and both feet with c / o abd pain ongoing times 5 days .  \n",
    "Example Output1: she said the doctor diagnosed her husband with <span class=\"nervous_AE\">Gillian Barre</span> related to <span class=\"procedure\">flu vaccine</span> . Seen in <span class=\"procedure\">ER</span> for C / O <span class=\"nervous_AE\">numbness in both hands ( fingertips ) and both feet</span> with c / o <span class=\"other_AE\">abd pain</span> ongoing times 5 days .  \n",
    "Example Input2: He was seen in an emergency room the following day and was admitted to the hospital because he could not walk .  \n",
    "Example Output2: He was seen in an <span class=\"procedure\">emergency room</span> the following day and was admitted to the <span class=\"procedure\">hospital</span> because he <span class=\"nervous_AE\">could not walk</span> .  \n",
    "Example Input3: On 20 September 2009 the patient was diagnosed with GUILLAIN BARRE Syndrome .  \n",
    "Example Output3: On 20 September 2009 the patient was diagnosed with <span class=\"nervous_AE\">GUILLAIN BARRE Syndrome</span> .  \n",
    "Example Input4: No improvement as of yet 4 days of IVIG beginning 1 / 15 / 09 ( Hospitalized ) .  \n",
    "Example Output4: No improvement as of yet 4 days of <span class=\"procedure\">IVIG</span> beginning 1 / 15 / 09 ( <span class=\"procedure\">Hospitalized</span> ) .  \n",
    "Example Input5: Tx with plasmapheresis .  \n",
    "Example Output5: Tx with <span class=\"procedure\">plasmapheresis</span> .  \n",
    "\n",
    "### Input Text: {}\n",
    "### Output Text:\n",
    "'''\n",
    "GPT = 4\n",
    "prompt_type = 'baseline_definition_guideline_erroranalysis_5shot'\n",
    "run(GPT,prompt,prompt_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "314790b5",
   "metadata": {},
   "source": [
    "# evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0965cdfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "from bs4 import NavigableString, Tag\n",
    "from glob import glob\n",
    "import spacy\n",
    "py_nlp = spacy.load (\"en_core_web_lg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c577f928",
   "metadata": {},
   "outputs": [],
   "source": [
    "def html2bio(html_path):\n",
    "    with open(html_path) as f:\n",
    "        \n",
    "        html = f.read()\n",
    "        \n",
    "        if '***output***' in html.lower():\n",
    "            html = html[html.lower().index('***output***')+len('***output***')+1:]\n",
    "        if 'output:' in html.lower():\n",
    "            html = html[html.lower().index('output:')+len('output:')+1:]\n",
    "        if 'output text' in html.lower():\n",
    "            html = html[html.lower().index('output text')+len('output text')+1:]\n",
    "        if '***Highlighted Text***'  in html.lower():\n",
    "            html = html[html.lower().index('***Highlighted Text***')+len('***Highlighted Text***')+1:]\n",
    "        if '<body>' in html:\n",
    "            html = html[html.index('<body>')+6:html.index('</body>')]\n",
    "        if '<p>' in html:\n",
    "            html = html[html.index('<p>')+3:html.index('</p>')]\n",
    "            \n",
    "        #print (html_path)\n",
    "        #print (html,'\\n')\n",
    "            \n",
    "        # Parse HTML using BeautifulSoup\n",
    "        soup = bs(html, \"html.parser\")\n",
    "\n",
    "        # Extract text under 'p' tags and convert to BIO format\n",
    "        bio_format = []\n",
    "        \n",
    "\n",
    "        for child in soup.children:\n",
    "            if isinstance(child, NavigableString):\n",
    "                for word in child.split():\n",
    "                    bio_format.append(f\"{word}\\tO\\n\")\n",
    "            elif isinstance(child, Tag):\n",
    "                words = py_nlp (child.get_text())\n",
    "                try:\n",
    "                    entity = child.attrs['class'][0]\n",
    "                except:\n",
    "                    entity = 'O'\n",
    "                if len(words) != 0:\n",
    "                    if entity != 'O' and entity in ['investigation','nervous_AE','other_AE','procedure']:\n",
    "                        bio_format.append(f\"{words[0]}\\tB-{entity}\\n\")\n",
    "                        for word in words[1:]:\n",
    "                            bio_format.append(f\"{word}\\tI-{entity}\\n\")\n",
    "                    else:\n",
    "                        bio_format.append(f\"{words[0]}\\tO\\n\")\n",
    "                        for word in words[1:]:\n",
    "                            bio_format.append(f\"{word}\\tO\\n\")\n",
    "    return bio_format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6bb68db",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_performance(GPT,prompt):\n",
    "    all_tags = []\n",
    "    all_tokens = []\n",
    "    gold_tags = []\n",
    "\n",
    "    for file in files:\n",
    "        file_name = file.split('/')[-1].split('.')[0]\n",
    "        with open(file) as f_gold:\n",
    "            lines = f_gold.readlines()\n",
    "            tokens = [line.strip().split(' ')[0] for line in lines]\n",
    "            tags = [line.strip().split(' ')[-1] for line in lines]\n",
    "\n",
    "            prediction = f'./output/GPT{GPT}_output/{dataset}/{prompt}/{file_name}.html'\n",
    "\n",
    "            bio_2 = html2bio(prediction)\n",
    "            all_tokens += tokens\n",
    "\n",
    "            for i,token in enumerate(tokens):\n",
    "                if token != '':\n",
    "                    match = False\n",
    "                    for i2 in range(i,-1,-1):\n",
    "                        try:\n",
    "                            token_2,tag_2 = bio_2[i2].strip().split('\\t')\n",
    "                        except:\n",
    "                            token_2,tag_2 = None, None\n",
    "                        if token_2!=None:\n",
    "                            if token in token_2 or token_2 in token:\n",
    "                                match = True\n",
    "                                break\n",
    "\n",
    "                    if not match:\n",
    "                        tag_2 = 'O'\n",
    "                else:\n",
    "                    tag_2 = ''\n",
    "                #print (token,tags[i],tag_2)\n",
    "\n",
    "                gold_tags.append(tags[i])\n",
    "                all_tags.append(tag_2)\n",
    "    with open('./output/merged_gold_pre.bio','w') as fg:\n",
    "        for i,(token,gold_tag,all_tag) in enumerate(zip(all_tokens,gold_tags,all_tags)):\n",
    "            if token!='':\n",
    "\n",
    "                fg.write(f'{token}\\t{gold_tag}\\t{all_tag}\\n')\n",
    "            else:\n",
    "                fg.write(f'\\n')\n",
    "    !python ./evaluate.py -lf ./output/merged_gold_pre.bio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85ed5387",
   "metadata": {},
   "outputs": [],
   "source": [
    "GPT = 3.5\n",
    "prompt = 'baseline'\n",
    "get_performance(GPT,prompt)\n",
    "prompt = 'baseline_definition_guideline'\n",
    "get_performance(GPT,prompt)\n",
    "prompt = 'baseline_definition_guideline_erroranalysis'\n",
    "get_performance(GPT,prompt)\n",
    "prompt = 'baseline_definition_guideline_erroranalysis_1shot'\n",
    "get_performance(GPT,prompt)\n",
    "prompt = 'baseline_definition_guideline_erroranalysis_5shot'\n",
    "get_performance(GPT,prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e2416e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "GPT = 4\n",
    "prompt = 'baseline'\n",
    "get_performance(GPT,prompt)\n",
    "prompt = 'baseline_definition_guideline'\n",
    "get_performance(GPT,prompt)\n",
    "prompt = 'baseline_definition_guideline_erroranalysis'\n",
    "get_performance(GPT,prompt)\n",
    "prompt = 'baseline_definition_guideline_erroranalysis_1shot'\n",
    "get_performance(GPT,prompt)\n",
    "prompt = 'baseline_definition_guideline_erroranalysis_5shot'\n",
    "get_performance(GPT,prompt)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (pyspark_env)",
   "language": "python",
   "name": "pyspark_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}