notebooks/Preprocess_app_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Appendicitis Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we prepare the pediatric appendicitis dataset for model training. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import sys\n",
    "import copy\n",
    "import glob\n",
    "import json\n",
    "import re\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from shutil import copyfile\n",
    "from sklearn.impute import KNNImputer\n",
    "from sklearn.model_selection import GroupShuffleSplit, StratifiedKFold\n",
    "sys.path.insert(0, '../')\n",
    "from datasets.preprocessing import preprocess\n",
    "from datasets.generate_app_data import generate_files\n",
    "from DeepFill.run_preprocessing import deep_fill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Important constants\n",
    "# TODO: fill in relevant directories\n",
    "\n",
    "# Directory with the original, unprocessed ultrasound images\n",
    "IMAGE_DIR = '...'\n",
    "# Directory for saving unprocessed ultrasound images from the training set\n",
    "TRAIN_IMAGE_DIR = '...'\n",
    "# Directory for saving unprocessed ultrasound images from the test set\n",
    "TEST_IMAGE_DIR = '...'\n",
    "# Directory for the temporary output of the DeepFill model\n",
    "DEPFILL_TEMP_DIR = '...'\n",
    "# Directory saving preprocessed ultrasound images\n",
    "PREPROC_IMAGE_DIR = '...'\n",
    "# Excel file with the tabular data\n",
    "CLINICAL_DATA_FILE = '...'\n",
    "# Directory for saving the data dictionaries\n",
    "OUTPUT_DIR = '...'\n",
    "# File with the names of the images containing multiple US snapshots\n",
    "BLACKLIST_FILE = '...'\n",
    "\n",
    "# Type of padding to apply to images: 'constant', 'speckle', 'reflect' or 'resize'\n",
    "PADDING_MODE = 'constant'\n",
    "\n",
    "# Target variable to predict: 'diagnosis', 'treatment' or 'complications'\n",
    "TARGET_LABEL = 'diagnosis'\n",
    "\n",
    "# Feature names to retrieve from the tabular data\n",
    "TABULAR_FEATURES = ['Age', 'Sex', 'Height', 'Weight', 'BMI', 'Alvarado_Score', 'Paedriatic_Appendicitis_Score',\n",
    "                    'Peritonitis', 'Migratory_Pain', 'Lower_Right_Abd_Pain', 'Contralateral_Rebound_Tenderness', \n",
    "                    'Coughing_Pain', 'Psoas_Sign', 'Nausea', 'Loss_of_Appetite', 'Body_Temperature', 'Dysuria', \n",
    "                    'Stool', 'WBC_Count', 'Neutrophil_Percentage', 'CRP', 'Ketones_in_Urine', 'RBC_in_Urine', \n",
    "                    'WBC_in_Urine', 'Appendix_on_US', 'Appendix_Diameter', 'Free_Fluids', \n",
    "                    'Appendix_Wall_Layers', 'Target_Sign', 'Perfusion', 'Perforation', \n",
    "                    'Surrounding_Tissue_Reaction', 'Pathological_Lymph_Nodes', 'Bowel_Wall_Thickening', 'Ileus', \n",
    "                    'Coprostasis', 'Meteorism', 'Enteritis', 'Appendicular_Abscess', \n",
    "                    'Conglomerate_of_Bowel_Loops', 'Gynecological_Findings']\n",
    "\n",
    "# Continuously valued features\n",
    "REAL_VALUED = ['Age', 'Height', 'Weight', 'BMI', 'Body_Temperature', 'WBC_Count', 'Neutrophil_Percentage', \n",
    "               'CRP', 'Appendix_Diameter']\n",
    "\n",
    "# Binary valued features\n",
    "CONCEPTS = ['Appendix_on_US',\n",
    "            'Free_Fluids',\n",
    "            'Appendix_Wall_Layers',\n",
    "            'Target_Sign',\n",
    "            'Surrounding_Tissue_Reaction',\n",
    "            'Pathological_Lymph_Nodes',\n",
    "            'Bowel_Wall_Thickening',\n",
    "            'Coprostasis', \n",
    "            'Meteorism',\n",
    "            'Enteritis', \n",
    "            'Appendix_Diameter',\n",
    "            'Perforation', \n",
    "            'Appendicular_Abscess', \n",
    "            'Conglomerate_of_Bowel_Loops', \n",
    "            'Gynecological_Findings']\n",
    "\n",
    "# Create directories if they do not exist\n",
    "if not os.path.exists(TRAIN_IMAGE_DIR):\n",
    "    os.makedirs(TRAIN_IMAGE_DIR)\n",
    "if not os.path.exists(TEST_IMAGE_DIR):\n",
    "    os.makedirs(TEST_IMAGE_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, an independent test set is reserved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_split(train_image_dir, test_image_dir):\n",
    "    # Verifies that the training and test sets form disjoint sets of subjects\n",
    "    train_image_file_list = glob.glob(train_image_dir + '/*')\n",
    "    train_groups = []\n",
    "    for file in train_image_file_list:\n",
    "        name = file.split('/')[-1]\n",
    "        patient_code = re.split('_| |\\.', name)[0]\n",
    "        train_groups.append(patient_code)\n",
    "    train_groups = np.array(list(map(int, train_groups)))\n",
    "    \n",
    "    test_image_file_list = glob.glob(test_image_dir + '/*')\n",
    "    test_groups = []\n",
    "    for file in test_image_file_list:\n",
    "        name = file.split('/')[-1]\n",
    "        patient_code = re.split('_| |\\.', name)[0]\n",
    "        test_groups.append(patient_code)\n",
    "    test_groups = np.array(list(map(int, test_groups)))\n",
    "    \n",
    "    print(\"\\nNumber of patients in train set: \", len(np.unique(train_groups)))\n",
    "    print(\"Number of patients in test set: \", len(np.unique(test_groups)))\n",
    "    print(\"\\nNumber of images in train set: \", len(train_groups))\n",
    "    print(\"Number of images in test set: \", len(test_groups))\n",
    "    \n",
    "    if len(np.intersect1d(train_groups, test_groups)) == 0:\n",
    "        print(\"\\nPatients in train and test set don't overlap!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the original dataset\n",
    "image_file_list = glob.glob(IMAGE_DIR + '/*')\n",
    "image_names = []\n",
    "groups = []\n",
    "for file in image_file_list:\n",
    "    name = file.split('/')[-1]\n",
    "    patient_code = re.split('_| |\\.', name)[0]\n",
    "    image_names.append(name)\n",
    "    groups.append(patient_code)\n",
    "groups = np.array(list(map(int, groups)))\n",
    "print(\"Total number of patients having US images: \", len(np.unique(groups)))\n",
    "print(\"Total number of images: \", len(image_file_list))\n",
    "\n",
    "# TODO: insert the relevant directory\n",
    "# Load the list form a CSV file containing patient codes for the predefined test set \n",
    "# NOTE: set to None to generate another train-test split\n",
    "test_set_codes = np.genfromtxt('...')\n",
    "\n",
    "# Copy train and test images to new folders\n",
    "if test_set_codes is not None:\n",
    "    for image_idx, group in enumerate(groups):\n",
    "        if group in test_set_codes:\n",
    "            copyfile(os.path.join(IMAGE_DIR, image_names[image_idx]), \n",
    "                     os.path.join(TEST_IMAGE_DIR, image_names[image_idx]))\n",
    "        else:\n",
    "            copyfile(os.path.join(IMAGE_DIR, image_names[image_idx]), \n",
    "                     os.path.join(TRAIN_IMAGE_DIR, image_names[image_idx]))\n",
    "        \n",
    "else:\n",
    "    # Group-stratified split\n",
    "    gss = GroupShuffleSplit(n_splits=1, train_size=.9, random_state=42)\n",
    "    for idx1, idx2 in gss.split(image_names, groups=groups):\n",
    "        train_idx = idx1\n",
    "        test_idx = idx2\n",
    "    \n",
    "    for image_idx, image_name in enumerate(image_names):\n",
    "        if image_idx in train_idx:\n",
    "            copyfile(os.path.join(IMAGE_DIR, image_name), os.path.join(TRAIN_IMAGE_DIR, image_name))\n",
    "        else:\n",
    "            copyfile(os.path.join(IMAGE_DIR, image_name), os.path.join(TEST_IMAGE_DIR, image_name))\n",
    "        \n",
    "# Verify that patients in the splits do not overlap\n",
    "test_split(TRAIN_IMAGE_DIR, TEST_IMAGE_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DeepFill "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before cropping the images to required dimension, automatic filling of markers and annotations can be done. All file extensions will be changed from `.bmp` to `.png`, and all spaces in the file names will be replaced with `'_'`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "deep_fill(\n",
    "    TRAIN_IMAGE_DIR, # Raw dataset directory\n",
    "    os.path.join(DEPFILL_TEMP_DIR, 'train'), # Temporary directory for DeepFill\n",
    "    os.path.join(DEPFILL_TEMP_DIR, 'train_mask'), # Mask directory for DeepFill\n",
    "    os.path.join(PREPROC_IMAGE_DIR, 'deepfilled_train'), # Output directory for DeepFill\n",
    "    \"../DeepFill/preproc/patterns\" # Directory with pattern templates to be removed\n",
    ")\n",
    "\n",
    "deep_fill(\n",
    "    TEST_IMAGE_DIR, # Raw dataset directory\n",
    "    os.path.join(DEPFILL_TEMP_DIR, 'test'), # Temporary directory for DeepFill\n",
    "    os.path.join(DEPFILL_TEMP_DIR, 'test_mask'), # Mask directory for DeepFill\n",
    "    os.path.join(PREPROC_IMAGE_DIR, 'deepfilled_test'), # Output directory for DeepFill\n",
    "    \"../DeepFill/preproc/patterns\" # Directory with pattern templates to be removed\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cropping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we crop the images using the specified padding mode."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_deepfill_train = {\n",
    "    'images': os.path.join(PREPROC_IMAGE_DIR, 'deepfilled_train'), \n",
    "    'target': os.path.join(PREPROC_IMAGE_DIR, PADDING_MODE + '_padding/deepfilled_cropped_train/'), \n",
    "    'padding': PADDING_MODE,\n",
    "    'debug': False,\n",
    "    \"no_black_triangles\": False\n",
    "}\n",
    "config_deepfill_test = {\n",
    "    'images': os.path.join(PREPROC_IMAGE_DIR, 'deepfilled_test'), \n",
    "    'target': os.path.join(PREPROC_IMAGE_DIR, PADDING_MODE + '_padding/deepfilled_cropped_test/'), \n",
    "    'padding': PADDING_MODE,\n",
    "    'debug': False,\n",
    "    'no_black_triangles': False\n",
    "}\n",
    "\n",
    "preprocess(config_deepfill_train)\n",
    "preprocess(config_deepfill_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dictionary File Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, dictionary files are generated for training and test sets. A list of images to be excluded from the dataset can be passed as an argument."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "def build_config(image_dir, target_dir, target_file, blacklist, concepts):\n",
    "    # Build a config file for the dataset\n",
    "    if TARGET_LABEL == 'diagnosis':\n",
    "        label_col_name = 'Diagnosis'\n",
    "        true_label = ['appendicitis']\n",
    "        false_label = ['no appendicitis']\n",
    "    \n",
    "    elif TARGET_LABEL == 'treatment':\n",
    "        label_col_name = 'Management'\n",
    "        true_label = ['primary surgical', 'secondary surgical', 'simultaneous appendectomy']\n",
    "        false_label = ['conservative']\n",
    "    \n",
    "    else:\n",
    "        label_col_name = 'Severity' \n",
    "        true_label = ['complicated']\n",
    "        false_label = ['uncomplicated']\n",
    "    \n",
    "    config = {\n",
    "            'info_file': CLINICAL_DATA_FILE,\n",
    "            'image_dir': image_dir,\n",
    "            'output_dir': target_dir,\n",
    "            'output_file': target_file,\n",
    "            'blacklist': blacklist,\n",
    "            'label': label_col_name ,\n",
    "            'true_label': true_label,\n",
    "            'false_label': false_label,\n",
    "            'concepts': concepts\n",
    "    }\n",
    "    \n",
    "    return config\n",
    "\n",
    "config_gen_deepfilled_cropped_train = build_config(\n",
    "    image_dir = os.path.join(PREPROC_IMAGE_DIR,'constant_padding/deepfilled_cropped_train'), \n",
    "    target_dir = os.path.join(OUTPUT_DIR, TARGET_LABEL),\n",
    "    target_file = 'app_data_train',\n",
    "    blacklist = BLACKLIST_FILE,\n",
    "    concepts=CONCEPTS)\n",
    "\n",
    "config_gen_deepfilled_cropped_test = build_config(\n",
    "    image_dir = os.path.join(PREPROC_IMAGE_DIR,'constant_padding/deepfilled_cropped_test'), \n",
    "    target_dir = os.path.join(OUTPUT_DIR, TARGET_LABEL),\n",
    "    target_file = 'app_data_test',\n",
    "    blacklist = BLACKLIST_FILE,\n",
    "    concepts=CONCEPTS)\n",
    "\n",
    "generate_files(config_gen_deepfilled_cropped_train)\n",
    "\n",
    "generate_files(config_gen_deepfilled_cropped_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imputation of missing values "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tabular variables contain missing values that are imputed using the $k$-nearest neighbors method. To prevent data leakage, validation/test data are not used when fitting the imputer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decompose_dict(f):\n",
    "    # Retrieve information from the data dictionary\n",
    "    keys = []\n",
    "    image_names = []\n",
    "    X = []\n",
    "    y = []\n",
    "    for i in range(len(f)):\n",
    "        keys.append(list(f)[i])\n",
    "        image_names.append(list(f.values())[i][0])\n",
    "        y.append(list(f.values())[i][1])\n",
    "        X.append(list(f.values())[i][2])\n",
    "    X = pd.DataFrame(X, columns=TABULAR_FEATURES)\n",
    "    y = np.array(y)\n",
    "    return keys, image_names, X, y\n",
    "\n",
    "def impute_folds(keys, image_names, X, y, dataset_name):\n",
    "    # Perform imutation for the cross-validation\n",
    "    kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    for fold, (train_ids, val_ids) in enumerate(kfold.split(X, y)):\n",
    "        mapping = {}\n",
    "        X_imp = copy.deepcopy(X)\n",
    "        imputer = KNNImputer(n_neighbors=1)\n",
    "        X_imp.iloc[train_ids] = imputer.fit_transform(X_imp.iloc[train_ids])\n",
    "        X_imp.iloc[val_ids] = imputer.transform(X_imp.iloc[val_ids])\n",
    "        X_imp_norm = copy.deepcopy(X_imp)\n",
    "        # Standardize real-valued variables\n",
    "        for real_var in REAL_VALUED:\n",
    "            mean = np.mean(X_imp[real_var].iloc[train_ids].values)\n",
    "            std = np.std(X_imp[real_var].iloc[train_ids].values)     \n",
    "            X_imp_norm[real_var] = (X_imp[real_var]-mean)/std        \n",
    "        concepts = X_imp_norm[CONCEPTS]\n",
    "        for idx, key in enumerate(keys):\n",
    "            mapping[key] = [image_names[idx], int(y[idx]), X_imp_norm.iloc[idx].values.tolist(),\n",
    "                            concepts.iloc[idx].values.tolist()]\n",
    "        if not os.path.exists(os.path.join(OUTPUT_DIR, TARGET_LABEL, os.path.join('imputed', f'fold{fold}'))):\n",
    "            os.makedirs(os.path.join(OUTPUT_DIR, TARGET_LABEL, os.path.join('imputed', f'fold{fold}')))\n",
    "        with open(os.path.join(OUTPUT_DIR, TARGET_LABEL, 'imputed', f'fold{fold}', dataset_name), 'w') as f:\n",
    "            json.dump(mapping, f)  \n",
    "            \n",
    "def impute_final(keys, image_names, X, y, dataset_name, keys_test, image_names_test, X_test, y_test, \n",
    "                 test_dataset_name):\n",
    "    # Perform imputation for the train-test split\n",
    "    mapping = {}\n",
    "    mapping_test = {}\n",
    "    imputer = KNNImputer(n_neighbors=1)\n",
    "    \n",
    "    X = pd.DataFrame(imputer.fit_transform(X), columns=TABULAR_FEATURES)\n",
    "    X_test = pd.DataFrame(imputer.transform(X_test), columns=TABULAR_FEATURES)\n",
    "    X_norm = copy.deepcopy(X)\n",
    "    X_test_norm = copy.deepcopy(X_test)\n",
    "    # Standardize real-valued variables\n",
    "    for real_var in REAL_VALUED:\n",
    "        mean = np.mean(X[real_var].values)\n",
    "        std = np.std(X[real_var].values)\n",
    "        X_norm[real_var] = (X[real_var]-mean)/std\n",
    "        X_test_norm[real_var] = (X_test[real_var]-mean)/std        \n",
    "    concepts = X_norm[CONCEPTS]\n",
    "    concepts_test = X_test_norm[CONCEPTS]\n",
    "    for idx, key in enumerate(keys):\n",
    "        mapping[key] = [image_names[idx], int(y[idx]), X_norm.iloc[idx].values.tolist(),\n",
    "                        concepts.iloc[idx].values.tolist()]\n",
    "    for idx, key in enumerate(keys_test):\n",
    "        mapping_test[key] = [image_names_test[idx], int(y_test[idx]), X_test_norm.iloc[idx].values.tolist(),\n",
    "                             concepts_test.iloc[idx].values.tolist()]\n",
    "    \n",
    "    if not os.path.exists(os.path.join(OUTPUT_DIR, TARGET_LABEL, 'imputed', 'final')):\n",
    "        os.makedirs(os.path.join(OUTPUT_DIR, TARGET_LABEL, 'imputed', 'final'))\n",
    "    \n",
    "    with open(os.path.join(OUTPUT_DIR, TARGET_LABEL, 'imputed', 'final', dataset_name), 'w') as f:\n",
    "        json.dump(mapping, f) \n",
    "    with open(os.path.join(OUTPUT_DIR, TARGET_LABEL, 'imputed', 'final', test_dataset_name), 'w') as f:\n",
    "        json.dump(mapping_test, f) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'app_data_train'\n",
    "with open(os.path.join(OUTPUT_DIR, TARGET_LABEL, dataset_name)) as f:\n",
    "    data_dict = json.load(f)\n",
    "keys, image_names, X, y = decompose_dict(data_dict)\n",
    "impute_folds(keys, image_names, X, y, dataset_name)\n",
    "test_dataset_name = 'app_data_test'\n",
    "with open(os.path.join(OUTPUT_DIR, TARGET_LABEL, test_dataset_name)) as f:\n",
    "    data_dict_test = json.load(f)\n",
    "keys_test, image_names_test, X_test, y_test = decompose_dict(data_dict_test)\n",
    "impute_final(keys, image_names, X, y, dataset_name, keys_test, image_names_test, X_test, y_test, \n",
    "             test_dataset_name)    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}