Task A and B - EDA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2dab6b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import plotly\n",
    "import plotly.express as px\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import re\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n",
    "from transformers import AutoTokenizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import config as code_config\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca4fcfa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = Path.cwd().joinpath(\"2023_ImageCLEFmed_Mediqa\",\"dataset\",\"TaskB\",\"TaskB-TrainingSet.csv\")\n",
    "validation_path = Path.cwd().joinpath(\"2023_ImageCLEFmed_Mediqa\",\"dataset\",\"TaskB\",\"TaskB-ValidationSet.csv\")\n",
    "augmented_path = Path.cwd().joinpath(\"TaskA-augmented_data.csv\")\n",
    "\n",
    "train_df = pd.read_csv(train_path,index_col=\"ID\")\n",
    "valid_df = pd.read_csv(validation_path,index_col=\"ID\")\n",
    "valid_index = {idx:idx+train_df.shape[0] for idx in valid_df.index}\n",
    "valid_df.rename(mapper=valid_index,inplace=True)\n",
    "augmented_data = pd.read_csv(augmented_path,index_col=\"ID\")\n",
    "augmented_sections = augmented_data[\"section_header\"].unique().tolist()\n",
    "merge_df = pd.concat([train_df,valid_df,augmented_data],axis=0,ignore_index=False)\n",
    "merge_df[\"dialogue_wo_whitespaces\"] = merge_df[\"dialogue\"].apply(lambda x: re.sub(r'[\\r\\n\\s]+',' ',x))\n",
    "merge_df.reset_index(inplace=True)\n",
    "merge_df.rename(mapper={'index':'ID'},axis=1,inplace=True)\n",
    "merge_df_w_augmented_data = merge_df.loc[merge_df[\"section_header\"].isin(augmented_sections)]\n",
    "merge_df_wo_augmented_data = merge_df.loc[~merge_df[\"section_header\"].isin(augmented_sections)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd4e6bf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_df_wo_augmented_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "349600ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "le.fit(merge_df[\"section_header\"])\n",
    "\n",
    "label2idx = {sec:i for i,sec in enumerate(le.classes_)}\n",
    "idx2label = {i:sec for i,sec in enumerate(le.classes_)}\n",
    "\n",
    "with open(\"TaskA_and_B-label2idx.json\",\"w\") as f:\n",
    "    json.dump(label2idx,f,indent=2)\n",
    "    \n",
    "with open(\"TaskA_and_B-idx2label.json\",\"w\") as f:\n",
    "    json.dump(idx2label,f,indent=2)\n",
    "# merge_df[\"label\"] = merge_df[\"section_header\"].apply(lambda x: label2idx[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bbf5d4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "section_header_dist = \\\n",
    "merge_df[\"section_header\"].value_counts(normalize=True).reset_index()\n",
    "section_header_dist.columns = [\"section_header\",\"proportion\"]\n",
    "section_header_cnt = \\\n",
    "merge_df[\"section_header\"].value_counts().reset_index()\n",
    "section_header_cnt.columns = [\"section_header\",\"Count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "478c9763",
   "metadata": {},
   "outputs": [],
   "source": [
    "px.bar(data_frame=section_header_cnt, \\\n",
    "       x='section_header', \\\n",
    "       y='Count', \\\n",
    "       title=\"Section Header Count\").update_layout(xaxis_title=\"Section Header\", \\\n",
    "                                                   yaxis_title=\"Count\", \\\n",
    "                                                   title={'x':0.5,'xanchor': 'center'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82bcda4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "px.bar(data_frame=section_header_dist, \\\n",
    "       x='section_header', \\\n",
    "       y='proportion', \\\n",
    "       title=\"Section Header Proportion\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82156147",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_checkpoint = \"emilyalsentzer/Bio_ClinicalBERT\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint,do_lower_case=True,force_download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75c83f26",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_len_list = []\n",
    "for sentence in merge_df[\"dialogue_wo_whitespaces\"]:\n",
    "    token_list = tokenizer.encode(sentence,add_special_tokens=True)\n",
    "    token_len_list.append(len(token_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c12940b",
   "metadata": {},
   "outputs": [],
   "source": [
    "px.histogram(token_len_list,cumulative=True,histnorm=\"percent\").update_layout(xaxis_title=\"Number of Tokens\", \\\n",
    "                                                   yaxis_title=\"Percentage of IDs\", \\\n",
    "                                                   title={'text':'Cumulative Distribution of number of tokens in every Dialogue', \\\n",
    "                                                          'x':0.5, \\\n",
    "                                                          'xanchor': 'center'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31a37bf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_len_list = []\n",
    "for sentence in merge_df[\"section_text\"]:\n",
    "    token_list = tokenizer.encode(sentence,add_special_tokens=True)\n",
    "    summary_len_list.append(len(token_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a57817b",
   "metadata": {},
   "outputs": [],
   "source": [
    "px.histogram(summary_len_list,cumulative=True,histnorm=\"percent\").update_layout(xaxis_title=\"Number of Tokens\", \\\n",
    "                                                   yaxis_title=\"Percentage of IDs\", \\\n",
    "                                                   title={'text':'Cumulative Distribution of number of tokens in every Summary', \\\n",
    "                                                          'x':0.5, \\\n",
    "                                                          'xanchor': 'center'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db328d7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting min, median, max lengths of the text\n",
    "min(token_len_list), np.median(token_len_list), max(token_len_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6d81c18",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.percentile(token_len_list,q=[0.,25,50,75,80,85,90,95,99,100])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0a22b02",
   "metadata": {},
   "source": [
    "Sentences with length <= 300 account for about 90% of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8558b067",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4054c9f7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(\n",
    "    n_splits=code_config.MULTI_CLASS_N_SPLITS,\n",
    "    shuffle=True,\n",
    "    random_state=code_config.SEED\n",
    ")\n",
    "split_dict = dict()\n",
    "for split, (train_idx, test_idx) in enumerate(\n",
    "    skf.split(merge_df_wo_augmented_data, y=merge_df_wo_augmented_data[\"section_header\"])\n",
    "):\n",
    "    split_dict[split] = dict()\n",
    "    train_df = merge_df_wo_augmented_data.iloc[train_idx,:]\n",
    "    test_df = merge_df_wo_augmented_data.iloc[test_idx,:]\n",
    "    test_counts = test_df[\"section_header\"].value_counts(normalize=True).reset_index()\n",
    "    test_counts.rename({\"section_header\":\"test\"},axis=1,inplace=True)\n",
    "#     print(train_df.head())\n",
    "    train,valid = \\\n",
    "    train_test_split(train_df,test_size=0.2,random_state=code_config.SEED,stratify=train_df[\"section_header\"])\n",
    "    train_counts = train[\"section_header\"].value_counts(normalize=True).reset_index()\n",
    "    train_counts.rename({\"section_header\":\"train\"},axis=1,inplace=True)\n",
    "    valid_counts = valid[\"section_header\"].value_counts(normalize=True).reset_index()\n",
    "    valid_counts.rename({\"section_header\":\"valid\"},axis=1,inplace=True)\n",
    "    \n",
    "    new_df = pd.merge(train_counts,valid_counts,left_on=\"index\",right_on=\"index\",how=\"outer\").fillna(0)\n",
    "    new_df = pd.merge(new_df,test_counts,left_on=\"index\",right_on=\"index\",how=\"outer\").fillna(0)\n",
    "    \n",
    "    split_dict[split][\"train\"] = train[\"ID\"].values.tolist()\n",
    "    split_dict[split][\"valid\"] = valid[\"ID\"].values.tolist()\n",
    "    split_dict[split][\"test\"] = test_df[\"ID\"].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc86ff6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "augmented_train_idx = \\\n",
    "[idx for idx in merge_df_w_augmented_data[\"ID\"] if isinstance(idx,str) and (\"Augmented\" in idx) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60b14870",
   "metadata": {},
   "outputs": [],
   "source": [
    "for split_idx, split in split_dict.items():\n",
    "    split[\"train\"].extend(augmented_train_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2468d9e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_df_w_augmented_data_valid_test = \\\n",
    "merge_df_w_augmented_data.loc[~merge_df_w_augmented_data[\"ID\"].isin(augmented_train_idx),:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fea369af",
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(\n",
    "    n_splits=code_config.MULTI_CLASS_N_SPLITS,\n",
    "    shuffle=True,\n",
    "    random_state=code_config.SEED\n",
    ")\n",
    "new_split_dict = dict()\n",
    "for split, (train_idx, test_idx) in enumerate(\n",
    "    skf.split(merge_df_w_augmented_data_valid_test, y=merge_df_w_augmented_data_valid_test[\"section_header\"])\n",
    "):\n",
    "    new_split_dict[split] = dict()\n",
    "    train_df = merge_df_w_augmented_data_valid_test.iloc[train_idx,:]\n",
    "    test_df = merge_df_w_augmented_data_valid_test.iloc[test_idx,:]\n",
    "    new_split_dict[split][\"valid\"] = train[\"ID\"].values.tolist()\n",
    "    new_split_dict[split][\"test\"] = valid[\"ID\"].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d88de95",
   "metadata": {},
   "outputs": [],
   "source": [
    "for split_idx, split in split_dict.items():\n",
    "    split[\"valid\"].extend(new_split_dict[split_idx][\"valid\"])\n",
    "    split[\"test\"].extend(new_split_dict[split_idx][\"test\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80eeab27",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"taskA_and_B_train_valid_test_split.json\",\"w\") as f:\n",
    "    json.dump(split_dict,f,indent=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}