COPD Base Model Transfer Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from Parse_TFrecords import *\n",
    "from define_model import *\n",
    "from load_label import *\n",
    "from utilities import *\n",
    "import gc\n",
    "import tensorflow as tf\n",
    "import cv2 as cv\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report, average_precision_score\n",
    "\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "tf.config.set_visible_devices(\n",
    "    gpus[5:], device_type='GPU'\n",
    ")\n",
    "for gpu in gpus[5:]:\n",
    "    tf.config.experimental.set_memory_growth(gpu, True)\n",
    "    \n",
    "print(tf.__version__, gpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = '../Data/Chexpert_cxr.tfrecords'\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "train_dataset = (tf.data.TFRecordDataset(\n",
    "    files, compression_type=None, buffer_size=BATCH_SIZE*10, num_parallel_reads=32)\n",
    ".map(parse_TFrecord_pretrain)\n",
    ".batch(BATCH_SIZE, drop_remainder=True)\n",
    ".shuffle(256))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "archi = 'Xception'\n",
    "\n",
    "checkpoint_filepath = 'checkpoints_new/checkpoint_pretrain_{i}'.format(i=archi)\n",
    "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath=checkpoint_filepath,\n",
    "    save_best_only=True,\n",
    "    save_weights_only=True,\n",
    "    monitor='loss',\n",
    "    mode='min')\n",
    "\n",
    "callback = [tf.keras.callbacks.LearningRateScheduler(scheduler),\n",
    "            model_checkpoint_callback]\n",
    "\n",
    "model = define_model(archi, nodes=6)\n",
    "\n",
    "model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
    "                 optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3), metrics='AUC')\n",
    "\n",
    "model.fit(train_dataset, epochs=5, shuffle=True, callbacks=callback)\n",
    "\n",
    "del model\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine-tune Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "\n",
    "# All\n",
    "# pos = 57925\n",
    "# neg = 212124\n",
    "\n",
    "# Emory\n",
    "# pos = 25294\n",
    "# neg = 120259\n",
    "\n",
    "# MIMIC\n",
    "pos = 32631\n",
    "neg = 91865\n",
    "\n",
    "total = pos+neg\n",
    "\n",
    "weight_for_0 = (1 / neg) * (total / 2.0)\n",
    "weight_for_1 = (1 / pos) * (total / 2.0)\n",
    "\n",
    "class_weight = {0: weight_for_0, 1: weight_for_1}\n",
    "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
    "print('Weight for class 1: {:.2f}'.format(weight_for_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "record_file_train = 'tfrecords/copd_mimic_train.tfrecords'\n",
    "train_dataset = (tf.data.TFRecordDataset(\n",
    "    record_file_train, buffer_size=BATCH_SIZE, compression_type=None, num_parallel_reads=32)\n",
    ".map(parse_TFrecord_train)\n",
    ".shuffle(BATCH_SIZE)\n",
    ".batch(BATCH_SIZE))\n",
    "\n",
    "record_file_val = 'tfrecords/copd_mimic_val.tfrecords'\n",
    "val_dataset = (tf.data.TFRecordDataset(\n",
    "    record_file_val, buffer_size=BATCH_SIZE, compression_type=None, num_parallel_reads=32)\n",
    ".map(parse_TFrecord_train)\n",
    ".batch(BATCH_SIZE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_label(dataset='mimic', split='test', return_demo=False):\n",
    "    \n",
    "    filename = 'tfrecords/copd_{a}_{b}.tfrecords'.format(a=dataset, b=split)\n",
    "      \n",
    "    y = []\n",
    "    img = []\n",
    "    demo = []\n",
    "\n",
    "    #load the test files\n",
    "    raw_dataset = tf.data.TFRecordDataset(filename)\n",
    "    for raw_record in raw_dataset:\n",
    "        example = tf.train.Example()\n",
    "        example.ParseFromString(raw_record.numpy())\n",
    "        \n",
    "        label = example.features.feature['COPD'].int64_list.value[0]\n",
    "        \n",
    "        y.append(label)\n",
    "        \n",
    "        if (return_demo):\n",
    "        \n",
    "            gender = example.features.feature['gender'].int64_list.value[0]\n",
    "            race = example.features.feature['race'].int64_list.value[0]\n",
    "            age = example.features.feature['age'].int64_list.value[0]\n",
    "\n",
    "            demo.append({'Age':age, 'Gender':gender, 'Race':race})\n",
    "            \n",
    "        nparr = np.fromstring(example.features.feature['jpg_bytes'].bytes_list.value[0], np.uint8)\n",
    "        img_np = cv.imdecode(nparr, cv.IMREAD_GRAYSCALE)  \n",
    "        \n",
    "        img.append(np.float32(st.resize(img_np, (256, 256, 1))))\n",
    "\n",
    "        \n",
    "    if (return_demo):\n",
    "        return np.array(y), np.array(demo)\n",
    "    else:\n",
    "        return np.array(y), np.array(img)\n",
    "    \n",
    "y_train, X_train = get_data_label(dataset = 'mimic', split = 'train', return_demo = False)\n",
    "y_val, X_val = get_data_label(dataset = 'mimic', split = 'val', return_demo = False)\n",
    "\n",
    "np.random.shuffle(y_train)\n",
    "np.random.shuffle(y_val)\n",
    "\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(BATCH_SIZE).batch(BATCH_SIZE)\n",
    "val_dataset = tf.data.Dataset.from_tensor_slices((X_val, y_val)).shuffle(BATCH_SIZE).batch(BATCH_SIZE)\n",
    "\n",
    "del y_train, X_train, y_val, X_val\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "archi = 'Xception'\n",
    "checkpoint_filepath = 'checkpoints_mimic/checkpoint_BCE_{i}_shuffle'.format(i=archi)\n",
    "\n",
    "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath=checkpoint_filepath,\n",
    "    save_best_only=True,\n",
    "    save_weights_only=True,\n",
    "    monitor='val_loss',\n",
    "    mode='min')\n",
    "\n",
    "callback = [tf.keras.callbacks.LearningRateScheduler(scheduler),\n",
    "            tf.keras.callbacks.EarlyStopping(mode='min', patience=3, monitor='val_loss'),\n",
    "            model_checkpoint_callback]\n",
    "\n",
    "base_model = load_model_from_pretrain(archi)\n",
    "\n",
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.UpSampling3D(size=(1,1,3)))\n",
    "model.add(base_model)\n",
    "\n",
    "model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
    "                 optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3), metrics='AUC')\n",
    "\n",
    "model.fit(train_dataset, epochs=5, shuffle=True, validation_data=val_dataset, callbacks=callback, class_weight=class_weight)\n",
    "\n",
    "del model\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "record_file_test = 'tfrecords/copd_emory_test.tfrecords'\n",
    "test_dataset = (tf.data.TFRecordDataset(\n",
    "    record_file_test, buffer_size=BATCH_SIZE, compression_type=None, num_parallel_reads=32)\n",
    ".map(parse_TFrecord_test)\n",
    ".batch(BATCH_SIZE))\n",
    "\n",
    "y_test = get_data_label(dataset = 'emory', split = 'test', return_demo = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# checkpoint_filepath = 'checkpoints_mimic/checkpoint_BCE_{i}_shuffle'.format(i=archi)\n",
    "\n",
    "# model = define_model(archi)\n",
    "\n",
    "# model.load_weights(checkpoint_filepath)\n",
    "\n",
    "# y_preds = model.predict(test_dataset)\n",
    "\n",
    "thresh = get_thresh(y_test[0], y_preds, 'Youden')\n",
    "\n",
    "test_CI(y_preds, y_test[0], thresh)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "archis = ['Xception', 'DenseNet121', 'ResNet50V2', 'MobileNetV2']\n",
    "\n",
    "for archi in archis:\n",
    "    checkpoint_filepath = 'checkpoints/checkpoints_merged/checkpoint_BCE_{i}'.format(i=archi)\n",
    "\n",
    "    model = define_model(archi)\n",
    "\n",
    "    model.load_weights(checkpoint_filepath)\n",
    "\n",
    "    y_preds = model.predict(test_dataset)\n",
    "    \n",
    "    thresh = get_thresh(y_test, y_preds, 'Youden')\n",
    "\n",
    "    test_CI(y_preds, y_test, thresh)\n",
    "\n",
    "    gc.collect()\n",
    "\n",
    "    outfile = 'preds/merged/test_preds/mimic_{i}_preds.npy'.format(i=archi)\n",
    "    np.save(outfile, y_preds)"

   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "archis = ['Xception', 'DenseNet121', 'ResNet50V2', 'MobileNetV2']\n",
    "\n",
    "for archi in archis:\n",
    "    checkpoint_filepath = 'checkpoints/checkpoints_merged/checkpoint_BCE_{i}'.format(i=archi)\n",
    "\n",
    "    model = define_model(archi)\n",
    "\n",
    "    model.load_weights(checkpoint_filepath)\n",
    "\n",
    "    y_preds = model.predict(test_dataset)\n",
    "    \n",
    "    thresh = get_thresh(y_test, y_preds, 'Youden')\n",
    "\n",
    "    test_CI(y_preds, y_test, thresh)\n",
    "\n",
    "    gc.collect()\n",
    "\n",
    "    outfile = 'preds/merged/test_preds/emory_{i}_preds.npy'.format(i=archi)\n",
    "    np.save(outfile, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh = get_thresh(y_label, y_preds, 'Youden')\n",
    "\n",
    "test_CI(y_preds, y_label, thresh)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fairness Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "race_list = ['White', 'Black', 'Latino', 'Others', 'Asian']\n",
    "\n",
    "for race_num, race in enumerate(race_list):\n",
    "    print(race)\n",
    "    \n",
    "    idx = []\n",
    "    for i, l in enumerate(y_demo):\n",
    "        if (l['Race']==race_num):\n",
    "            idx.append(i)\n",
    "            \n",
    "    temp_df = y_preds[idx]\n",
    "    temp_label = y_label[idx]\n",
    "    \n",
    "    thresh = get_thresh(temp_label, temp_df, 'Youden')\n",
    "\n",
    "    test_CI(temp_df, temp_label, thresh)\n",
    "\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gender_list = ['Female', 'Male']\n",
    "\n",
    "for gender_num, gender in enumerate(gender_list):\n",
    "    print(gender)\n",
    "    \n",
    "    idx = []\n",
    "    for i, l in enumerate(y_demo):\n",
    "        if (l['Gender']==gender_num):\n",
    "            idx.append(i)\n",
    "\n",
    "    temp_df = y_preds[idx]\n",
    "    temp_label = y_label[idx]\n",
    "    \n",
    "    thresh = get_thresh(temp_label, temp_df, 'Youden')\n",
    "\n",
    "    test_CI(temp_df, temp_label, thresh)\n",
    "\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_list = ['0-40', '40-60', '60-80', '80-']\n",
    "\n",
    "for age_num, age in enumerate(age_list):\n",
    "    print(age)\n",
    "    \n",
    "    idx = []\n",
    "    for i, l in enumerate(y_demo):\n",
    "        if (l['Age']==age_num):\n",
    "            idx.append(i)\n",
    "\n",
    "    temp_df = y_preds[idx]\n",
    "    temp_label = y_label[idx]\n",
    "    \n",
    "    thresh = get_thresh(temp_label, temp_df, 'Youden')\n",
    "\n",
    "    test_CI(temp_df, temp_label, thresh)\n",
    "\n",
    "    gc.collect()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hitienv",
   "language": "python",
   "name": "hitienv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}