Transfer_Learning_NNs_leuko_class.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transfer Learning"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# CHECK THIS OUT:\n",
    "# https://github.com/spmallick/learnopencv/blob/master/Keras-Transfer-Learning/transfer-learning-vgg.ipynb\n",
    "# https://blog.athelas.com/classifying-white-blood-cells-with-convolutional-neural-networks-2ca6da239331\n",
    "# https://github.com/dhruvp/wbc-classification/blob/master/notebooks/binary_training.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "### CONVERT IMAGES TO.PNG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import datetime\n",
    "import math\n",
    "\n",
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "import keras\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.image import ImageDataGenerator, load_img\n",
    "\n",
    "from keras.applications.resnet50 import ResNet50\n",
    "from keras.applications.resnet50 import preprocess_input, decode_predictions\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from keras.applications.vgg19 import VGG19\n",
    "from keras.applications.vgg19 import preprocess_input\n",
    "from keras.applications.inception_v3 import InceptionV3\n",
    "from keras.applications.densenet import DenseNet121\n",
    "from keras.applications.densenet import DenseNet169\n",
    "from keras.applications.densenet import DenseNet201\n",
    "from keras.applications.nasnet import NASNetLarge\n",
    "from keras.applications.nasnet import NASNetMobile\n",
    "from keras.applications.inception_resnet_v2 import InceptionResNetV2\n",
    "from keras.applications.xception import Xception\n",
    "from keras.applications.mobilenet import MobileNet\n",
    "\n",
    "import os\n",
    "import cv2\n",
    "import datetime\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(name):\n",
    "    if name == 'resnet50':\n",
    "        model_choice = ResNet50(weights= 'imagenet', include_top=False, input_shape=(224, 224, 3))\n",
    "        model_name = 'ResNet50'\n",
    "        target_size = (224, 224)\n",
    "        sfvector = (7, 7, 2048) ##shape_feature_vector\n",
    "        reshape_feature_vector = (7* 7* 2048)\n",
    "        optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=False)\n",
    "        optim = \"SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=False)\"\n",
    "        \n",
    "    if name == 'vgg16':   \n",
    "        model_choice = VGG16(weights= 'imagenet', include_top=False, input_shape=(224, 224, 3))\n",
    "        model_name='VGG16'\n",
    "        target_size = (224, 224)\n",
    "        sfvector = (7, 7, 512) ##shape_feature_vector\n",
    "        reshape_feature_vector = (7 * 7 * 512) \n",
    "        optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=5e-4, nesterov=False)\n",
    "        optim = \"SGD(lr=0.01, momentum=0.9, decay=5e-4, nesterov=False)\"\n",
    "        \n",
    "    if name == 'vgg19':\n",
    "        model_choice = VGG19(weights= 'imagenet', include_top=False, input_shape=(224, 224, 3))\n",
    "        model_name='VGG19'\n",
    "        target_size = (224, 224)\n",
    "        sfvector = (7, 7, 512) ##shape_feature_vector\n",
    "        reshape_feature_vector = (7 * 7 * 512) \n",
    "        optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=5e-4, nesterov=False)\n",
    "        optim = \"SGD(lr=0.01, momentum=0.9, decay=5e-4, nesterov=False)\"\n",
    "        \n",
    "    if name == 'inceptionV3':\n",
    "        model_choice = InceptionV3(weights='imagenet', include_top=False, input_shape=(299, 299, 3))\n",
    "        model_name = 'InceptionV3'\n",
    "        target_size = (299, 299)\n",
    "        sfvector = (8, 8, 2048) ##shape_feature_vector\n",
    "        reshape_feature_vector = (8* 8* 2048) \n",
    "        optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=4e-5, nesterov=False)\n",
    "        optim = \"SGD(lr=0.01, momentum=0.9, decay=4e-5, nesterov=False)\"\n",
    "    \n",
    "    if name == 'xception':\n",
    "        model_choice = Xception(weights= 'imagenet', include_top=False, input_shape=(299, 299, 3))\n",
    "        model_name = 'Xception'\n",
    "        target_size = (299, 299)\n",
    "        sfvector = (10, 10, 2048) ##shape_feature_vector\n",
    "        reshape_feature_vector = (10* 10* 2048)\n",
    "        optimizer = keras.optimizers.SGD(lr=0.045, momentum=0.9, decay=1e-5, nesterov=False)\n",
    "        optim = \"SGD(lr=0.045, momentum=0.9, decay=1e-5, nesterov=False)\"\n",
    "     \n",
    "    if name == 'densenet121':\n",
    "        model_choice= DenseNet121(weights= 'imagenet', include_top=False, input_shape=(224, 224, 3))\n",
    "        model_name = 'DenseNet121'\n",
    "        target_size = (224, 224)\n",
    "        sfvector = (7, 7, 1024)\n",
    "        reshape_feature_vector = (7 * 7 * 1024)  \n",
    "        optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=True)\n",
    "        optim = \"SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=True)\"\n",
    "                                   \n",
    "    if name == 'densenet169':\n",
    "        model_choice= DenseNet169(weights= 'imagenet', include_top=False, input_shape=(224, 224, 3))\n",
    "        model_name = 'DenseNet169'\n",
    "        target_size = (224, 224)\n",
    "        sfvector = (7, 7, 1664)\n",
    "        reshape_feature_vector = (7 * 7 * 1664)\n",
    "        optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=True)\n",
    "        optim = \"SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=True)\"\n",
    "    \n",
    "    if name == 'densenet201':\n",
    "        model_choice = DenseNet201(weights= 'imagenet', include_top=False, input_shape=(224, 224, 3))\n",
    "        model_name = 'DenseNet201'\n",
    "        target_size = (224, 224)\n",
    "        sfvector = (7, 7, 1920)\n",
    "        reshape_feature_vector = (7 * 7 * 1920)\n",
    "        optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=True)\n",
    "        optim = \"SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=True)\"\n",
    "   \n",
    "    return model_choice, model_name, reshape_feature_vector, sfvector, target_size, optimizer, optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CHOOSE THE MODEL YOU WANT TO USE\n",
    "## get_model(name); \n",
    "## name = ('resnet50', 'vgg16', 'vgg19', 'inceptionV3', 'xception', 'densenet121', 'densenet169', 'densenet201')\n",
    "model_choice, model_name, reshape_feature_vector, sfvector, target_size, optimizer, optim = get_model('vgg16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 14,714,688\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# NUMBER OF EPOCHS, BATCH_SIZE\n",
    "model_choice.summary()\n",
    "epochs = 100\n",
    "batch_size = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WHERE IS YOUR WORKING DIR? CHANGE TO YOUR SPECS...\n",
    "''' jupyter nbk or floyd hub? mode = ... '''\n",
    "\n",
    "#mode = 'jupyter'\n",
    "#mode = 'floyd'\n",
    "mode = 'ipb'\n",
    "\n",
    "if mode == 'floyd':\n",
    "    BASE_DIR = '/img/'       \n",
    "if mode == 'jupyter':\n",
    "    BASE_DIR = 'C:/Users/Ana Brites/dataleuko3/' \n",
    "if mode == 'ipb':\n",
    "    BASE_DIR = '/home/deep/Documents/Romeu_Beato/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WORK WITH FULL SIZE DATA OR TWEEK WITH SMALL ONE...\n",
    "''' dataset size: dsize = ... '''\n",
    "\n",
    "#dsize = 'small'\n",
    "dsize = 'big'\n",
    "\n",
    "if dsize == 'small':\n",
    "    train_dir = BASE_DIR + 'images/TRAINsmall/'\n",
    "    validation_dir = BASE_DIR +'images/TEST_SIMPLE2small/'\n",
    "    test_dir = BASE_DIR + 'images/TEST_SIMPLEsmall/'\n",
    "    nTrain = 395\n",
    "    nVal = 40\n",
    "    nTest = 40 \n",
    "elif dsize == 'big': \n",
    "    validation_dir = BASE_DIR +'images/TEST_SIMPLE2/'\n",
    "    train_dir = BASE_DIR + 'images/TRAIN/'\n",
    "    test_dir = BASE_DIR + 'images/TEST_SIMPLE/'\n",
    "    nTrain = 4394 ###\n",
    "    nVal = 440 ###\n",
    "    nTest = 440  ###"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate and save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## BUILD GENERATOR AND CLASS INDICES\n",
    "\n",
    "def build(source=None):\n",
    "        datagen = ImageDataGenerator(rescale=1. / 255)\n",
    "            #featurewise_center=True,\n",
    "            #featurewise_std_normalization=True)\n",
    "        data_generator = datagen.flow_from_directory(\n",
    "        source,  # this is the target directory\n",
    "        target_size=(target_size[0], target_size[1]),  # all images will be resized to 150x150\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical',\n",
    "        shuffle=False)\n",
    "        class_dictionary = data_generator.class_indices\n",
    "        return data_generator, class_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4394"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nTrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "440"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nTest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "440"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nVal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 4394 images belonging to 4 classes.\n"
     ]
    }
   ],
   "source": [
    "## Train generator, class dictionary, and train_features and train_labels\n",
    "\n",
    "train_generator = [] \n",
    "\n",
    "train_features = np.zeros(shape=(nTrain, sfvector[0], sfvector[1], sfvector[2]))\n",
    "train_labels = np.zeros(shape=(nTrain, 4))\n",
    "\n",
    "train_generator = build(source=train_dir) \n",
    "\n",
    "i = 0\n",
    "for inputs_batch, labels_batch in train_generator[0]:\n",
    "    features_batch = model_choice.predict(inputs_batch)\n",
    "    train_features[i * batch_size : (i + 1) * batch_size] = features_batch\n",
    "    train_labels[i * batch_size : (i + 1) * batch_size] = labels_batch\n",
    "    i += 1\n",
    "    if i * batch_size >= nTrain :\n",
    "        break\n",
    "\n",
    "train_features = np.reshape(train_features, (nTrain, reshape_feature_vector))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'EOSINOPHIL': 0, 'LYMPHOCYTE': 1, 'MONOCYTE': 2, 'NEUTROPHIL': 3}\n"
     ]
    }
   ],
   "source": [
    "print(train_generator[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 440 images belonging to 4 classes.\n"
     ]
    }
   ],
   "source": [
    "## validation generator, class dictionary, and validation_features and validation_labels\n",
    "\n",
    "validation_generator = [] \n",
    " \n",
    "validation_features = np.zeros(shape=(nVal, sfvector[0], sfvector[1], sfvector[2]))\n",
    "validation_labels = np.zeros(shape=(nVal,4))\n",
    "\n",
    "validation_generator = build(source=validation_dir) \n",
    "\n",
    "i = 0\n",
    "for inputs_batch, labels_batch in validation_generator[0]:\n",
    "    features_batch = model_choice.predict(inputs_batch)\n",
    "    validation_features[i * batch_size : (i + 1) * batch_size] = features_batch\n",
    "    validation_labels[i * batch_size : (i + 1) * batch_size] = labels_batch\n",
    "    i += 1 \n",
    "    if i * batch_size >= nVal:\n",
    "        break\n",
    "\n",
    "validation_features = np.reshape(validation_features, (nVal, sfvector[0] * sfvector[1] * sfvector[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the output as a Numpy array. CHANGE IF NEEDED\n",
    "\n",
    "#save_features = True\n",
    "save_features= False\n",
    "\n",
    "if save_features:\n",
    "    np.save(open('bottleneck_features_train_{}.npy'.format(model_name), 'wb'), train_features) ### 'wb' or 'w'\n",
    "    np.save(open('bottleneck_features_validation_{}.npy'.format(model_name), 'wb'), validation_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOAD DATA OR NOT?\n",
    "\n",
    "#load_features = True\n",
    "load_features = False\n",
    "\n",
    "if load_features:\n",
    "    train_features = np.load(open('bottleneck_features_train_{}.npy'.format(model_name),'rb'))\n",
    "    validation_features = np.load(open('bottleneck_features_validation_{}.npy'.format(model_name),'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(train_features)\n",
    "#print(train_labels)\n",
    "#print(validation_features)\n",
    "#print(validation_labels)\n",
    "#print(validation_features.shape)\n",
    "#print(train_features.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create model and train network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Dense(256, activation='relu', input_dim= sfvector[0] * sfvector[1] * sfvector[2]))\n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(4, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 256)               6422784   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 4)                 1028      \n",
      "=================================================================\n",
      "Total params: 6,423,812\n",
      "Trainable params: 6,423,812\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGD(lr=0.01, momentum=0.9, decay=5e-4, nesterov=False)\n"
     ]
    }
   ],
   "source": [
    "optimizer\n",
    "print(optim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SOME INFORMATION ABOUT THE PARAMETERS OF THE ORIGINAL PAPERS:\n",
    "#DenseNet- keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=True)\n",
    "#Resnet - keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=1e-4, nesterov=False)\n",
    "#Inception V3 - keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=4e-5, nesterov=False)\n",
    "#Xception - keras.optimizers.SGD(lr=0.045, momentum=0.9, decay=1e-5, nesterov=False)\n",
    "#VGG - keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=5e-4, nesterov=False)\n",
    "##########################################################################################\n",
    "\n",
    "optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=5e-4, nesterov=False)\n",
    "\n",
    "optim = optim\n",
    "\n",
    "model.compile(optimizer=optimizer, #optimizers.RMSprop(lr=2e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    " \n",
    "from keras.callbacks import CSVLogger                           \n",
    "csv_logger = CSVLogger('TL_history_{}_{}.csv'.format(model_name, datetime.date.today().strftime(\"%B %d, %Y\")),\n",
    "                       append=True, separator=';')     \n",
    "\n",
    "tic = time.process_time()\n",
    "\n",
    "history = model.fit(train_features,\n",
    "                    train_labels,\n",
    "                    epochs=epochs,\n",
    "                    batch_size=batch_size,\n",
    "                    validation_data=(validation_features,validation_labels),\n",
    "                    callbacks=[csv_logger])                   \n",
    "\n",
    "toc = time.process_time()\n",
    "\n",
    "print(\"---- Total Computation time = \" + str(datetime.timedelta(seconds=math.ceil(toc-tic))) + \" seconds\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "## model to png\n",
    "# from keras.utils import plot_model\n",
    "# plot_model(model, to_file='model_{}_{}.png'.format(models_names[0], datetime.date.today().strftime(\"%B %d, %Y\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save model weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save model weights\n",
    "# model.save_weights('bottleneck_fc_model_{}.h5'.format(models_names[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### History and plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['val_loss', 'val_acc', 'loss', 'acc'])\n"
     ]
    }
   ],
   "source": [
    "print(history.history.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(12,8))\n",
    "plt.plot(range(epochs),history.history['acc'],label='training accuracy', color = \"blue\")\n",
    "plt.plot(range(epochs),history.history['val_acc'],label='validation accuracy', color = \"red\")\n",
    "plt.legend(loc=0)\n",
    "plt.xlabel('epochs')\n",
    "plt.xlim([0,epochs])\n",
    "plt.ylabel('accuracies on dataset')\n",
    "plt.grid(True)\n",
    "plt.title(\"Training and validation accuracy\")\n",
    "plt.text(epochs*.6, max(history.history['acc'])/(1.25),'''No Transfer Learning, \\n \n",
    "            model = {} \\n\n",
    "            day = {}, \\n\n",
    "            optimizer = {}, \\n\n",
    "            dataset = {} train, {} val, ({} test), \\n\n",
    "            input shape = {}, \\n\n",
    "            batch size = {}, \\n\n",
    "            epochs = {}, \\n\n",
    "            training_time = {}'''.format(model_name,\n",
    "                                  datetime.date.today().strftime(\"%B %d, %Y\"),\n",
    "                                  optim,\n",
    "                                  nTrain, nVal, nTest,\n",
    "                                  target_size, #input_shape,\n",
    "                                  batch_size,\n",
    "                                  epochs,\n",
    "                                  str(datetime.timedelta(seconds=math.ceil(toc-tic)))))\n",
    "plt.show()\n",
    "fig.savefig('TL_accuracy_{}_{}_{}_imgs.jpg'.format(model_name,\n",
    "                                                datetime.date.today().strftime(\"%B %d, %Y\"),\n",
    "                                                nTrain+nVal+nTest))\n",
    "plt.close(fig)\n",
    "\n",
    "fig = plt.figure(figsize=(12,8))\n",
    "plt.plot(range(epochs),history.history['loss'],label='training loss')\n",
    "plt.plot(range(epochs),history.history['val_loss'],label='validation loss')\n",
    "plt.legend(loc=0)\n",
    "plt.xlabel('epochs')\n",
    "plt.xlim([0,epochs])\n",
    "plt.ylabel('losses on dataset')\n",
    "plt.grid(True)\n",
    "plt.title(\"Training and validation loss\")\n",
    "plt.text(epochs*.6, max(history.history['loss'])/(2),'''No Transfer Learning, \\n \n",
    "            model = {} \\n\n",
    "            day = {}, \\n\n",
    "            optimizer = {}, \\n\n",
    "            dataset = {} train, {} val, ({} test), \\n\n",
    "            input shape = {}, \\n\n",
    "            batch size = {}, \\n\n",
    "            epochs = {}, \\n\n",
    "            training_time = {}'''.format(model_name,\n",
    "                                  datetime.date.today().strftime(\"%B %d, %Y\"),\n",
    "                                  optim,\n",
    "                                  nTrain, nVal, nTest,\n",
    "                                  target_size, #input_shape,\n",
    "                                  batch_size,\n",
    "                                  epochs,\n",
    "                                  str(datetime.timedelta(seconds=math.ceil(toc-tic)))))\n",
    "\n",
    "plt.show()\n",
    "fig.savefig('TL_loss_{}_{}_{}_imgs.jpg'.format(model_name,\n",
    "                                                datetime.date.today().strftime(\"%B %d, %Y\"),\n",
    "                                                nTrain+nVal+nTest))\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 440 images belonging to 4 classes.\n"
     ]
    }
   ],
   "source": [
    "# TEST AND EVALUATE THE MODEL\n",
    "\n",
    "datagen = ImageDataGenerator(rescale=1. / 255)\n",
    "    #featurewise_center=True,\n",
    "    #featurewise_std_normalization=True)\n",
    "\n",
    "test_features = np.zeros(shape=(nTest, sfvector[0], sfvector[1], sfvector[2]))\n",
    "test_labels = np.zeros(shape=(nTest,4))\n",
    "\n",
    "test_generator = datagen.flow_from_directory(\n",
    "    test_dir,\n",
    "    target_size=(target_size[0], target_size[1]),\n",
    "    batch_size=batch_size,\n",
    "    class_mode='categorical',\n",
    "    shuffle=False)\n",
    "\n",
    "i = 0\n",
    "for inputs_batch, labels_batch in test_generator:\n",
    "    features_batch = model_choice.predict(inputs_batch)\n",
    "    test_features[i * batch_size : (i + 1) * batch_size] = features_batch\n",
    "    test_labels[i * batch_size : (i + 1) * batch_size] = labels_batch\n",
    "    i += 1\n",
    "    if i * batch_size >= nTest:\n",
    "        break\n",
    "\n",
    "test_features = np.reshape(test_features, (nTest,reshape_feature_vector))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting on test data\n",
      "0.25\n"
     ]
    }
   ],
   "source": [
    "print('Predicting on test data')\n",
    "prediction = np.rint(model.predict(test_features))\n",
    "\n",
    "print(accuracy_score(test_labels, prediction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict_classes(validation_features)\n",
    "prob = model.predict(validation_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See wich images were predicted wrongly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc: 25.00%\n",
      "Test Loss 12.088571565801448\n",
      "Test accuracy: 0.25\n",
      "[[0. 1. 0. 0.]]\n",
      "[[1. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "#EVALUATE FIRST IMAGE OF TEST DATASET\n",
    "score = model.evaluate(test_features, test_labels, verbose=0)\n",
    "\n",
    "print(\"%s: %.2f%%\" % (model.metrics_names[1], score[1]*100))\n",
    "\n",
    "print('Test Loss', score[0])\n",
    "print('Test accuracy:', score[1])\n",
    "\n",
    "test_image = test_features[0:1]\n",
    "#print(test_image.shape)\n",
    "print(model.predict(test_image))\n",
    "print(test_labels[0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " PROBABILITIES\n",
      "[[0. 1. 0. 0.]\n",
      " [0. 1. 0. 0.]\n",
      " [0. 1. 0. 0.]\n",
      " ...\n",
      " [0. 1. 0. 0.]\n",
      " [0. 1. 0. 0.]\n",
      " [0. 1. 0. 0.]]\n",
      "\n",
      " RESULTS\n",
      "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1]\n",
      "\n",
      " PRECISION, RECALL, SCORE\n",
      "                     precision    recall  f1-score   support\n",
      "\n",
      "class 0(EOSINOPHIL)       0.00      0.00      0.00       110\n",
      " class 1(LYPHOCYTE)       0.25      1.00      0.40       110\n",
      "  class 2(MONOCYTE)       0.00      0.00      0.00       110\n",
      "class 3(NEUTROPHIL)       0.00      0.00      0.00       110\n",
      "\n",
      "        avg / total       0.06      0.25      0.10       440\n",
      "\n",
      "\n",
      " CONFUSION MATRIX\n",
      "[[  0 110   0   0]\n",
      " [  0 110   0   0]\n",
      " [  0 110   0   0]\n",
      " [  0 110   0   0]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deep/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "## PREDICT FOR TEST DATASET\n",
    "\n",
    "Y_pred = model.predict(test_features)\n",
    "print('\\n PROBABILITIES')\n",
    "print(Y_pred)\n",
    "\n",
    "y_pred = np.argmax(Y_pred, axis=1)\n",
    "\n",
    "print('\\n RESULTS')\n",
    "print(y_pred)\n",
    "  \n",
    "# y_pred = model.predict_classes(test_features) ## == np-argmax\n",
    "#print(y_pred)\n",
    "\n",
    "p=model.predict_proba(test_features) # to predict probability\n",
    "\n",
    "target_names = ['class 0(EOSINOPHIL)', 'class 1(LYPHOCYTE)', 'class 2(MONOCYTE)', 'class 3(NEUTROPHIL)'] ##REVIEW\n",
    "print('\\n PRECISION, RECALL, SCORE')\n",
    "print(classification_report(np.argmax(test_labels,axis=1), y_pred,target_names=target_names))\n",
    "\n",
    "print('\\n CONFUSION MATRIX')\n",
    "print(confusion_matrix(np.argmax(test_labels,axis=1), y_pred))\n",
    "#print(confusion_matrix(np.argmax(test_labels,axis=1), y_pred['EOSINOPHIL', 'MONOCYTE','LIMPHOCYTE', 'NEUTROPHIL']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.25\n"
     ]
    }
   ],
   "source": [
    "print(str(accuracy_score(test_labels, prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deep/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "file2write=open('Test results_{}_#testexamples:{}_{}'.format(model_name,\n",
    "                                                                  len(Y_pred),\n",
    "                                                                  datetime.date.today().strftime(\"%B %d, %Y\")),'w')\n",
    "file2write.write('Predicting on test data' + str(accuracy_score(test_labels, prediction)))\n",
    "file2write.write('EVALUATE FIRST IMAGE OF TEST DATASET')\n",
    "file2write.write(\"%s: %.2f%%\" %(model.metrics_names[1], score[1]*100))\n",
    "file2write.write('Test Loss:{}'.format(score[0]))\n",
    "file2write.write('Test accuracy:{}'.format(score[1]))\n",
    "file2write.write(str(model.predict(test_image)))\n",
    "file2write.write(str(test_labels[0:1]))\n",
    "file2write.write('PREDICT FOR TEST DATASET')\n",
    "file2write.write('\\n PROBABILITIES \\n {}'.format(Y_pred))\n",
    "file2write.write('\\n RESULTS \\n {}'.format(y_pred))\n",
    "file2write.write('\\n PRECISION, RECALL, SCORE \\n' + classification_report(np.argmax(test_labels,axis=1), y_pred,target_names=target_names))\n",
    "file2write.write('\\n CONFUSION MATRIX \\n {}'.format(confusion_matrix(np.argmax(test_labels,axis=1), y_pred)))\n",
    "file2write.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    " keras.backend.clear_session()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}