vanilla CNN - FullDataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# With rgb images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "df = pd.read_csv('sample/Data_Entry_2017.csv')\n",
    "\n",
    "diseases = ['Cardiomegaly','Emphysema','Effusion','Hernia','Nodule','Pneumothorax','Atelectasis','Pleural_Thickening','Mass','Edema','Consolidation','Infiltration','Fibrosis','Pneumonia']\n",
    "#Number diseases\n",
    "for disease in diseases :\n",
    "    df[disease] = df['Finding Labels'].apply(lambda x: 1 if disease in x else 0)\n",
    "\n",
    "# #test to perfect\n",
    "# df = df.drop(df[df['Emphysema']==0][:-127].index.values)\n",
    "    \n",
    "#remove Y after age\n",
    "df['Age']=df['Patient Age'].apply(lambda x: x[:-1]).astype(int)\n",
    "df['Age Type']=df['Patient Age'].apply(lambda x: x[-1:])\n",
    "df.loc[df['Age Type']=='M',['Age']] = df[df['Age Type']=='M']['Age'].apply(lambda x: round(x/12.)).astype(int)\n",
    "df.loc[df['Age Type']=='D',['Age']] = df[df['Age Type']=='D']['Age'].apply(lambda x: round(x/365.)).astype(int)\n",
    "# remove outliers\n",
    "df = df.drop(df['Age'].sort_values(ascending=False).head(16).index)\n",
    "df['Age'] = df['Age']/df['Age'].max()\n",
    "\n",
    "#one hot data\n",
    "# df = df.drop(df.index[4242])\n",
    "df = df.join(pd.get_dummies(df['Patient Gender']))\n",
    "df = df.join(pd.get_dummies(df['View Position']))\n",
    "\n",
    "#random samples\n",
    "df = shuffle(df)\n",
    "\n",
    "#get other data\n",
    "data = df[['Age', 'F', 'M', 'AP', 'PA']]\n",
    "data = np.array(data)\n",
    "\n",
    "labels = df[diseases].as_matrix()\n",
    "files_list = ('sample/images/' + df['Image Index']).tolist()\n",
    "\n",
    "# #test to perfect\n",
    "# labelB = df['Emphysema'].tolist()\n",
    "\n",
    "labelB = (df[diseases].sum(axis=1)>0).tolist()\n",
    "labelB = np.array(labelB, dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/aind2/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n",
      "100%|██████████| 89600/89600 [20:26<00:00, 73.03it/s]\n",
      "100%|██████████| 11200/11200 [02:33<00:00, 73.10it/s]\n",
      "100%|██████████| 11319/11319 [02:38<00:00, 71.35it/s]\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import image                  \n",
    "from tqdm import tqdm\n",
    "\n",
    "def path_to_tensor(img_path, shape):\n",
    "    # loads RGB image as PIL.Image.Image type\n",
    "    img = image.load_img(img_path, target_size=shape)\n",
    "    # convert PIL.Image.Image type to 3D tensor with shape (224, 224, 3)\n",
    "    x = image.img_to_array(img)/255\n",
    "    # convert 3D tensor to 4D tensor with shape (1, 224, 224, 3) and return 4D tensor\n",
    "    return np.expand_dims(x, axis=0)\n",
    "\n",
    "def paths_to_tensor(img_paths, shape):\n",
    "    list_of_tensors = [path_to_tensor(img_path, shape) for img_path in tqdm(img_paths)]\n",
    "    return np.vstack(list_of_tensors)\n",
    "\n",
    "train_labels = labelB[:89600][:, np.newaxis]\n",
    "valid_labels = labelB[89600:100800][:, np.newaxis]\n",
    "test_labels = labelB[100800:][:, np.newaxis]\n",
    "\n",
    "train_data = data[:89600]\n",
    "valid_data = data[89600:100800]\n",
    "test_data = data[100800:]\n",
    "\n",
    "img_shape = (64, 64)\n",
    "train_tensors = paths_to_tensor(files_list[:89600], shape = img_shape)\n",
    "valid_tensors = paths_to_tensor(files_list[89600:100800], shape = img_shape)\n",
    "test_tensors = paths_to_tensor(files_list[100800:], shape = img_shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_13 (Conv2D)           (None, 64, 64, 16)        2368      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_13 (MaxPooling (None, 32, 32, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 32, 32, 32)        12832     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_14 (MaxPooling (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 16, 16, 64)        51264     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_15 (MaxPooling (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_16 (Conv2D)           (None, 4, 4, 128)         204928    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_16 (MaxPooling (None, 2, 2, 128)         0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 100)               51300     \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 322,793\n",
      "Trainable params: 322,793\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, Dropout, Flatten, Dense\n",
    "from keras.models import Sequential\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras import regularizers, initializers, optimizers\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(filters=16, \n",
    "                 kernel_size=7,\n",
    "                 padding='same', \n",
    "                 activation='relu', \n",
    "                 input_shape=train_tensors.shape[1:]))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "\n",
    "model.add(Conv2D(filters=32, \n",
    "                 kernel_size=5,\n",
    "                 padding='same', \n",
    "                 activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "\n",
    "model.add(Conv2D(filters=64, \n",
    "                 kernel_size=5,\n",
    "                 padding='same', \n",
    "                 activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "\n",
    "model.add(Conv2D(filters=128, \n",
    "                 kernel_size=5,\n",
    "                 strides=2,\n",
    "                 padding='same', \n",
    "                 activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def binary_accuracy(y_true, y_pred):\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)))\n",
    "\n",
    "def precision_threshold(threshold = 0.5):\n",
    "    def precision(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(y_pred)\n",
    "        precision_ratio = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision_ratio\n",
    "    return precision\n",
    "\n",
    "def recall_threshold(threshold = 0.5):\n",
    "    def recall(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.clip(y_true, 0, 1))\n",
    "        recall_ratio = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall_ratio\n",
    "    return recall\n",
    "\n",
    "def fbeta_score_threshold(beta = 1, threshold = 0.5):\n",
    "    def fbeta_score(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        beta_value = beta\n",
    "        p = precision_threshold(threshold_value)(y_true, y_pred)\n",
    "        r = recall_threshold(threshold_value)(y_true, y_pred)\n",
    "        bb = beta_value ** 2\n",
    "        fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "        return fbeta_score\n",
    "    return fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer='sgd', loss='binary_crossentropy', \n",
    "              metrics=[precision_threshold(threshold = 0.5), \n",
    "                       recall_threshold(threshold = 0.5), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.5),\n",
    "                      'accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 89600 samples, validate on 11200 samples\n",
      "Epoch 1/20\n",
      "89536/89600 [============================>.] - ETA: 0s - loss: 0.6561 - precision: 0.5974 - recall: 0.4632 - fbeta_score: 0.5386 - acc: 0.6178Epoch 00000: val_loss improved from inf to 0.65672, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 52s - loss: 0.6561 - precision: 0.5974 - recall: 0.4632 - fbeta_score: 0.5387 - acc: 0.6178 - val_loss: 0.6567 - val_precision: 0.5574 - val_recall: 0.7224 - val_fbeta_score: 0.5805 - val_acc: 0.6148\n",
      "Epoch 2/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6416 - precision: 0.6351 - recall: 0.5353 - fbeta_score: 0.6006 - acc: 0.6412Epoch 00001: val_loss improved from 0.65672 to 0.63831, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 51s - loss: 0.6416 - precision: 0.6353 - recall: 0.5353 - fbeta_score: 0.6007 - acc: 0.6412 - val_loss: 0.6383 - val_precision: 0.6061 - val_recall: 0.6122 - val_fbeta_score: 0.6025 - val_acc: 0.6460\n",
      "Epoch 3/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6354 - precision: 0.6417 - recall: 0.5549 - fbeta_score: 0.6115 - acc: 0.6487Epoch 00002: val_loss improved from 0.63831 to 0.63043, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 51s - loss: 0.6354 - precision: 0.6417 - recall: 0.5549 - fbeta_score: 0.6115 - acc: 0.6487 - val_loss: 0.6304 - val_precision: 0.6347 - val_recall: 0.5533 - val_fbeta_score: 0.6106 - val_acc: 0.6556\n",
      "Epoch 4/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6303 - precision: 0.6453 - recall: 0.5640 - fbeta_score: 0.6173 - acc: 0.6537Epoch 00003: val_loss improved from 0.63043 to 0.63033, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 51s - loss: 0.6303 - precision: 0.6455 - recall: 0.5640 - fbeta_score: 0.6174 - acc: 0.6538 - val_loss: 0.6303 - val_precision: 0.6627 - val_recall: 0.4711 - val_fbeta_score: 0.6032 - val_acc: 0.6525\n",
      "Epoch 5/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6267 - precision: 0.6506 - recall: 0.5799 - fbeta_score: 0.6259 - acc: 0.6603Epoch 00004: val_loss improved from 0.63033 to 0.62570, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 52s - loss: 0.6267 - precision: 0.6506 - recall: 0.5800 - fbeta_score: 0.6260 - acc: 0.6603 - val_loss: 0.6257 - val_precision: 0.6350 - val_recall: 0.5960 - val_fbeta_score: 0.6213 - val_acc: 0.6642\n",
      "Epoch 6/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6232 - precision: 0.6519 - recall: 0.5841 - fbeta_score: 0.6287 - acc: 0.6627Epoch 00005: val_loss improved from 0.62570 to 0.62252, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 52s - loss: 0.6232 - precision: 0.6519 - recall: 0.5839 - fbeta_score: 0.6287 - acc: 0.6627 - val_loss: 0.6225 - val_precision: 0.6655 - val_recall: 0.5105 - val_fbeta_score: 0.6197 - val_acc: 0.6628\n",
      "Epoch 7/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6204 - precision: 0.6558 - recall: 0.5892 - fbeta_score: 0.6332 - acc: 0.6668Epoch 00006: val_loss did not improve\n",
      "89600/89600 [==============================] - 51s - loss: 0.6204 - precision: 0.6558 - recall: 0.5893 - fbeta_score: 0.6333 - acc: 0.6669 - val_loss: 0.6237 - val_precision: 0.6176 - val_recall: 0.6639 - val_fbeta_score: 0.6217 - val_acc: 0.6624\n",
      "Epoch 8/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6179 - precision: 0.6574 - recall: 0.5978 - fbeta_score: 0.6364 - acc: 0.6693Epoch 00007: val_loss did not improve\n",
      "89600/89600 [==============================] - 51s - loss: 0.6180 - precision: 0.6574 - recall: 0.5979 - fbeta_score: 0.6364 - acc: 0.6692 - val_loss: 0.6243 - val_precision: 0.6157 - val_recall: 0.6677 - val_fbeta_score: 0.6206 - val_acc: 0.6618\n",
      "Epoch 9/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6151 - precision: 0.6565 - recall: 0.5990 - fbeta_score: 0.6364 - acc: 0.6702Epoch 00008: val_loss improved from 0.62252 to 0.61709, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 51s - loss: 0.6152 - precision: 0.6564 - recall: 0.5988 - fbeta_score: 0.6362 - acc: 0.6701 - val_loss: 0.6171 - val_precision: 0.6748 - val_recall: 0.5154 - val_fbeta_score: 0.6274 - val_acc: 0.6702\n",
      "Epoch 10/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6125 - precision: 0.6618 - recall: 0.6044 - fbeta_score: 0.6417 - acc: 0.6739Epoch 00009: val_loss improved from 0.61709 to 0.61645, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 51s - loss: 0.6125 - precision: 0.6618 - recall: 0.6043 - fbeta_score: 0.6416 - acc: 0.6739 - val_loss: 0.6164 - val_precision: 0.6585 - val_recall: 0.5817 - val_fbeta_score: 0.6353 - val_acc: 0.6746\n",
      "Epoch 11/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6102 - precision: 0.6636 - recall: 0.6092 - fbeta_score: 0.6443 - acc: 0.6762Epoch 00010: val_loss improved from 0.61645 to 0.61598, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 52s - loss: 0.6102 - precision: 0.6636 - recall: 0.6092 - fbeta_score: 0.6443 - acc: 0.6762 - val_loss: 0.6160 - val_precision: 0.6263 - val_recall: 0.6632 - val_fbeta_score: 0.6286 - val_acc: 0.6686\n",
      "Epoch 12/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6075 - precision: 0.6644 - recall: 0.6140 - fbeta_score: 0.6461 - acc: 0.6782Epoch 00011: val_loss improved from 0.61598 to 0.61276, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 53s - loss: 0.6074 - precision: 0.6644 - recall: 0.6140 - fbeta_score: 0.6461 - acc: 0.6782 - val_loss: 0.6128 - val_precision: 0.6400 - val_recall: 0.6476 - val_fbeta_score: 0.6363 - val_acc: 0.6753\n",
      "Epoch 13/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6052 - precision: 0.6678 - recall: 0.6175 - fbeta_score: 0.6498 - acc: 0.6812Epoch 00012: val_loss improved from 0.61276 to 0.60942, saving model to saved_models/bCNN.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 53s - loss: 0.6052 - precision: 0.6678 - recall: 0.6174 - fbeta_score: 0.6498 - acc: 0.6812 - val_loss: 0.6094 - val_precision: 0.6545 - val_recall: 0.5914 - val_fbeta_score: 0.6348 - val_acc: 0.6748\n",
      "Epoch 14/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6021 - precision: 0.6698 - recall: 0.6210 - fbeta_score: 0.6520 - acc: 0.6836Epoch 00013: val_loss did not improve\n",
      "89600/89600 [==============================] - 53s - loss: 0.6022 - precision: 0.6699 - recall: 0.6210 - fbeta_score: 0.6520 - acc: 0.6836 - val_loss: 0.6187 - val_precision: 0.6155 - val_recall: 0.6913 - val_fbeta_score: 0.6249 - val_acc: 0.6645\n",
      "Epoch 15/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.5997 - precision: 0.6727 - recall: 0.6242 - fbeta_score: 0.6547 - acc: 0.6855Epoch 00014: val_loss did not improve\n",
      "89600/89600 [==============================] - 53s - loss: 0.5999 - precision: 0.6724 - recall: 0.6239 - fbeta_score: 0.6544 - acc: 0.6852 - val_loss: 0.6147 - val_precision: 0.6373 - val_recall: 0.6321 - val_fbeta_score: 0.6307 - val_acc: 0.6702\n",
      "Epoch 16/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.5969 - precision: 0.6754 - recall: 0.6313 - fbeta_score: 0.6586 - acc: 0.6883Epoch 00015: val_loss did not improve\n",
      "89600/89600 [==============================] - 53s - loss: 0.5970 - precision: 0.6754 - recall: 0.6312 - fbeta_score: 0.6586 - acc: 0.6883 - val_loss: 0.6095 - val_precision: 0.6442 - val_recall: 0.6437 - val_fbeta_score: 0.6389 - val_acc: 0.6779\n",
      "Epoch 17/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.5937 - precision: 0.6794 - recall: 0.6334 - fbeta_score: 0.6622 - acc: 0.6916Epoch 00016: val_loss did not improve\n",
      "89600/89600 [==============================] - 53s - loss: 0.5937 - precision: 0.6793 - recall: 0.6334 - fbeta_score: 0.6622 - acc: 0.6916 - val_loss: 0.6133 - val_precision: 0.6844 - val_recall: 0.5338 - val_fbeta_score: 0.6403 - val_acc: 0.6790\n",
      "Epoch 00016: early stopping\n",
      "training time: 14.89 minutes\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping\n",
    "import numpy as np\n",
    "\n",
    "epochs = 20\n",
    "batch_size = 32\n",
    "\n",
    "earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')\n",
    "log = CSVLogger('saved_models/log_bCNN_rgb.csv')\n",
    "checkpointer = ModelCheckpoint(filepath='saved_models/bCNN.best.from_scratch.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "model.fit(train_tensors, train_labels, \n",
    "          validation_data=(valid_tensors, valid_labels),\n",
    "          epochs=epochs, batch_size=batch_size, callbacks=[checkpointer, log, earlystop], verbose=1)\n",
    "\n",
    "# Show total training time\n",
    "print(\"training time: %.2f minutes\"%((time.time()-start)/60))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('saved_models/bCNN.best.from_scratch.hdf5')\n",
    "prediction = model.predict(test_tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.672176 %\n",
      "Recall: 0.594230 %\n",
      "Fscore: 0.654993 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.67196751"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.eval(binary_accuracy(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.58734381],\n",
       "       [ 0.57867444],\n",
       "       [ 0.25462931],\n",
       "       [ 0.77830732],\n",
       "       [ 0.25104493],\n",
       "       [ 0.29603228],\n",
       "       [ 0.32766595],\n",
       "       [ 0.3778576 ],\n",
       "       [ 0.70840192],\n",
       "       [ 0.18522891],\n",
       "       [ 0.55603856],\n",
       "       [ 0.60000223],\n",
       "       [ 0.73821551],\n",
       "       [ 0.2869918 ],\n",
       "       [ 0.22979702],\n",
       "       [ 0.4054445 ],\n",
       "       [ 0.32552701],\n",
       "       [ 0.56464356],\n",
       "       [ 0.55663085],\n",
       "       [ 0.58321428],\n",
       "       [ 0.49937385],\n",
       "       [ 0.61920291],\n",
       "       [ 0.76322109],\n",
       "       [ 0.48952124],\n",
       "       [ 0.51417869],\n",
       "       [ 0.26597881],\n",
       "       [ 0.33098736],\n",
       "       [ 0.5749808 ],\n",
       "       [ 0.56771249],\n",
       "       [ 0.27707309]], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.617976 %\n",
      "Recall: 0.735481 %\n",
      "Fscore: 0.638374 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.4\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.712731 %\n",
      "Recall: 0.404833 %\n",
      "Fscore: 0.618630 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.6\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# With gray images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "df = pd.read_csv('sample/Data_Entry_2017.csv')\n",
    "\n",
    "diseases = ['Cardiomegaly','Emphysema','Effusion','Hernia','Nodule','Pneumothorax','Atelectasis','Pleural_Thickening','Mass','Edema','Consolidation','Infiltration','Fibrosis','Pneumonia']\n",
    "#Number diseases\n",
    "for disease in diseases :\n",
    "    df[disease] = df['Finding Labels'].apply(lambda x: 1 if disease in x else 0)\n",
    "\n",
    "# #test to perfect\n",
    "# df = df.drop(df[df['Emphysema']==0][:-127].index.values)\n",
    "    \n",
    "#remove Y after age\n",
    "df['Age']=df['Patient Age'].apply(lambda x: x[:-1]).astype(int)\n",
    "df['Age Type']=df['Patient Age'].apply(lambda x: x[-1:])\n",
    "df.loc[df['Age Type']=='M',['Age']] = df[df['Age Type']=='M']['Age'].apply(lambda x: round(x/12.)).astype(int)\n",
    "df.loc[df['Age Type']=='D',['Age']] = df[df['Age Type']=='D']['Age'].apply(lambda x: round(x/365.)).astype(int)\n",
    "# remove outliers\n",
    "df = df.drop(df['Age'].sort_values(ascending=False).head(16).index)\n",
    "df['Age'] = df['Age']/df['Age'].max()\n",
    "\n",
    "#one hot data\n",
    "# df = df.drop(df.index[4242])\n",
    "df = df.join(pd.get_dummies(df['Patient Gender']))\n",
    "df = df.join(pd.get_dummies(df['View Position']))\n",
    "\n",
    "#random samples\n",
    "df = shuffle(df)\n",
    "\n",
    "#get other data\n",
    "data = df[['Age', 'F', 'M', 'AP', 'PA']]\n",
    "data = np.array(data)\n",
    "\n",
    "labels = df[diseases].as_matrix()\n",
    "files_list = ('sample/images/' + df['Image Index']).tolist()\n",
    "\n",
    "# #test to perfect\n",
    "# labelB = df['Emphysema'].tolist()\n",
    "\n",
    "labelB = (df[diseases].sum(axis=1)>0).tolist()\n",
    "labelB = np.array(labelB, dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/aind2/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n",
      "100%|██████████| 89600/89600 [19:43<00:00, 75.69it/s]\n",
      "100%|██████████| 11200/11200 [02:32<00:00, 73.55it/s]\n",
      "100%|██████████| 11319/11319 [02:33<00:00, 73.74it/s]\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import image                  \n",
    "from tqdm import tqdm\n",
    "\n",
    "def path_to_tensor(img_path, shape):\n",
    "    # loads RGB image as PIL.Image.Image type\n",
    "    img = image.load_img(img_path, grayscale=True, target_size=shape)\n",
    "    # convert PIL.Image.Image type to 3D tensor with shape (224, 224, 1)\n",
    "    x = image.img_to_array(img)/255\n",
    "    # convert 3D tensor to 4D tensor with shape (1, 224, 224, 1) and return 4D tensor\n",
    "    return np.expand_dims(x, axis=0)\n",
    "\n",
    "def paths_to_tensor(img_paths, shape):\n",
    "    list_of_tensors = [path_to_tensor(img_path, shape) for img_path in tqdm(img_paths)]\n",
    "    return np.vstack(list_of_tensors)\n",
    "\n",
    "train_labels = labelB[:89600][:, np.newaxis]\n",
    "valid_labels = labelB[89600:100800][:, np.newaxis]\n",
    "test_labels = labelB[100800:][:, np.newaxis]\n",
    "\n",
    "train_data = data[:89600]\n",
    "valid_data = data[89600:100800]\n",
    "test_data = data[100800:]\n",
    "\n",
    "img_shape = (64, 64)\n",
    "train_tensors = paths_to_tensor(files_list[:89600], shape = img_shape)\n",
    "valid_tensors = paths_to_tensor(files_list[89600:100800], shape = img_shape)\n",
    "test_tensors = paths_to_tensor(files_list[100800:], shape = img_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 64, 64, 16)        800       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 32, 32, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 32, 32, 32)        12832     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 16, 16, 64)        51264     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 4, 4, 128)         204928    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 2, 2, 128)         0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               51300     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 321,225\n",
      "Trainable params: 321,225\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, Dropout, Flatten, Dense\n",
    "from keras.models import Sequential\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras import regularizers, initializers, optimizers\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(filters=16, \n",
    "                 kernel_size=7,\n",
    "                 padding='same', \n",
    "                 activation='relu', \n",
    "                 input_shape=train_tensors.shape[1:]))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "\n",
    "model.add(Conv2D(filters=32, \n",
    "                 kernel_size=5,\n",
    "                 padding='same', \n",
    "                 activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "\n",
    "model.add(Conv2D(filters=64, \n",
    "                 kernel_size=5,\n",
    "                 padding='same', \n",
    "                 activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "\n",
    "model.add(Conv2D(filters=128, \n",
    "                 kernel_size=5,\n",
    "                 strides=2,\n",
    "                 padding='same', \n",
    "                 activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def binary_accuracy(y_true, y_pred):\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)))\n",
    "\n",
    "def precision_threshold(threshold = 0.5):\n",
    "    def precision(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(y_pred)\n",
    "        precision_ratio = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision_ratio\n",
    "    return precision\n",
    "\n",
    "def recall_threshold(threshold = 0.5):\n",
    "    def recall(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.clip(y_true, 0, 1))\n",
    "        recall_ratio = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall_ratio\n",
    "    return recall\n",
    "\n",
    "def fbeta_score_threshold(beta = 1, threshold = 0.5):\n",
    "    def fbeta_score(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        beta_value = beta\n",
    "        p = precision_threshold(threshold_value)(y_true, y_pred)\n",
    "        r = recall_threshold(threshold_value)(y_true, y_pred)\n",
    "        bb = beta_value ** 2\n",
    "        fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "        return fbeta_score\n",
    "    return fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer='sgd', loss='binary_crossentropy', \n",
    "              metrics=[precision_threshold(threshold = 0.5), \n",
    "                       recall_threshold(threshold = 0.5), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.5),\n",
    "                      'accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 89600 samples, validate on 11200 samples\n",
      "Epoch 1/20\n",
      "89472/89600 [============================>.] - ETA: 0s - loss: 0.6632 - precision: 0.5035 - recall: 0.3776 - fbeta_score: 0.4438 - acc: 0.6012Epoch 00000: val_loss improved from inf to 0.64742, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 51s - loss: 0.6632 - precision: 0.5035 - recall: 0.3778 - fbeta_score: 0.4439 - acc: 0.6012 - val_loss: 0.6474 - val_precision: 0.6516 - val_recall: 0.3998 - val_fbeta_score: 0.5670 - val_acc: 0.6283\n",
      "Epoch 2/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.6456 - precision: 0.6291 - recall: 0.5252 - fbeta_score: 0.5922 - acc: 0.6347Epoch 00001: val_loss improved from 0.64742 to 0.63779, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 48s - loss: 0.6456 - precision: 0.6291 - recall: 0.5252 - fbeta_score: 0.5922 - acc: 0.6346 - val_loss: 0.6378 - val_precision: 0.6263 - val_recall: 0.5591 - val_fbeta_score: 0.6054 - val_acc: 0.6468\n",
      "Epoch 3/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.6391 - precision: 0.6361 - recall: 0.5440 - fbeta_score: 0.6047 - acc: 0.6444Epoch 00002: val_loss did not improve\n",
      "89600/89600 [==============================] - 48s - loss: 0.6391 - precision: 0.6361 - recall: 0.5440 - fbeta_score: 0.6048 - acc: 0.6444 - val_loss: 0.6387 - val_precision: 0.6762 - val_recall: 0.4284 - val_fbeta_score: 0.5951 - val_acc: 0.6449\n",
      "Epoch 4/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6341 - precision: 0.6433 - recall: 0.5593 - fbeta_score: 0.6146 - acc: 0.6520Epoch 00003: val_loss improved from 0.63779 to 0.62709, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 48s - loss: 0.6341 - precision: 0.6434 - recall: 0.5594 - fbeta_score: 0.6147 - acc: 0.6520 - val_loss: 0.6271 - val_precision: 0.6313 - val_recall: 0.6086 - val_fbeta_score: 0.6215 - val_acc: 0.6601\n",
      "Epoch 5/20\n",
      "89472/89600 [============================>.] - ETA: 0s - loss: 0.6300 - precision: 0.6482 - recall: 0.5680 - fbeta_score: 0.6209 - acc: 0.6565Epoch 00004: val_loss improved from 0.62709 to 0.62225, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 47s - loss: 0.6299 - precision: 0.6484 - recall: 0.5681 - fbeta_score: 0.6211 - acc: 0.6566 - val_loss: 0.6222 - val_precision: 0.6434 - val_recall: 0.5803 - val_fbeta_score: 0.6238 - val_acc: 0.6628\n",
      "Epoch 6/20\n",
      "89472/89600 [============================>.] - ETA: 0s - loss: 0.6266 - precision: 0.6516 - recall: 0.5729 - fbeta_score: 0.6244 - acc: 0.6599Epoch 00005: val_loss improved from 0.62225 to 0.61933, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 47s - loss: 0.6266 - precision: 0.6516 - recall: 0.5730 - fbeta_score: 0.6244 - acc: 0.6599 - val_loss: 0.6193 - val_precision: 0.6418 - val_recall: 0.5943 - val_fbeta_score: 0.6267 - val_acc: 0.6646\n",
      "Epoch 7/20\n",
      "89536/89600 [============================>.] - ETA: 0s - loss: 0.6234 - precision: 0.6541 - recall: 0.5796 - fbeta_score: 0.6289 - acc: 0.6633Epoch 00006: val_loss did not improve\n",
      "89600/89600 [==============================] - 47s - loss: 0.6234 - precision: 0.6539 - recall: 0.5796 - fbeta_score: 0.6287 - acc: 0.6633 - val_loss: 0.6254 - val_precision: 0.6851 - val_recall: 0.4468 - val_fbeta_score: 0.6094 - val_acc: 0.6534\n",
      "Epoch 8/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.6206 - precision: 0.6558 - recall: 0.5843 - fbeta_score: 0.6316 - acc: 0.6660Epoch 00007: val_loss did not improve\n",
      "89600/89600 [==============================] - 47s - loss: 0.6206 - precision: 0.6558 - recall: 0.5843 - fbeta_score: 0.6316 - acc: 0.6660 - val_loss: 0.6205 - val_precision: 0.6817 - val_recall: 0.4982 - val_fbeta_score: 0.6261 - val_acc: 0.6633\n",
      "Epoch 9/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.6182 - precision: 0.6580 - recall: 0.5909 - fbeta_score: 0.6348 - acc: 0.6679Epoch 00008: val_loss improved from 0.61933 to 0.61392, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 47s - loss: 0.6182 - precision: 0.6580 - recall: 0.5909 - fbeta_score: 0.6348 - acc: 0.6679 - val_loss: 0.6139 - val_precision: 0.6392 - val_recall: 0.6666 - val_fbeta_score: 0.6401 - val_acc: 0.6762\n",
      "Epoch 10/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.6158 - precision: 0.6609 - recall: 0.5961 - fbeta_score: 0.6388 - acc: 0.6712Epoch 00009: val_loss improved from 0.61392 to 0.61250, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 47s - loss: 0.6158 - precision: 0.6608 - recall: 0.5961 - fbeta_score: 0.6387 - acc: 0.6711 - val_loss: 0.6125 - val_precision: 0.6469 - val_recall: 0.6398 - val_fbeta_score: 0.6406 - val_acc: 0.6764\n",
      "Epoch 11/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.6134 - precision: 0.6610 - recall: 0.5980 - fbeta_score: 0.6392 - acc: 0.6727Epoch 00010: val_loss did not improve\n",
      "89600/89600 [==============================] - 47s - loss: 0.6134 - precision: 0.6610 - recall: 0.5980 - fbeta_score: 0.6392 - acc: 0.6727 - val_loss: 0.6131 - val_precision: 0.6434 - val_recall: 0.6591 - val_fbeta_score: 0.6417 - val_acc: 0.6771\n",
      "Epoch 12/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.6111 - precision: 0.6655 - recall: 0.6038 - fbeta_score: 0.6440 - acc: 0.6755Epoch 00011: val_loss improved from 0.61250 to 0.60702, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 47s - loss: 0.6111 - precision: 0.6655 - recall: 0.6038 - fbeta_score: 0.6440 - acc: 0.6755 - val_loss: 0.6070 - val_precision: 0.6684 - val_recall: 0.5950 - val_fbeta_score: 0.6462 - val_acc: 0.6793\n",
      "Epoch 13/20\n",
      "89536/89600 [============================>.] - ETA: 0s - loss: 0.6091 - precision: 0.6655 - recall: 0.6088 - fbeta_score: 0.6460 - acc: 0.6770Epoch 00012: val_loss improved from 0.60702 to 0.60598, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 47s - loss: 0.6091 - precision: 0.6655 - recall: 0.6087 - fbeta_score: 0.6459 - acc: 0.6769 - val_loss: 0.6060 - val_precision: 0.6706 - val_recall: 0.5808 - val_fbeta_score: 0.6443 - val_acc: 0.6776\n",
      "Epoch 14/20\n",
      "89504/89600 [============================>.] - ETA: 0s - loss: 0.6066 - precision: 0.6661 - recall: 0.6114 - fbeta_score: 0.6467 - acc: 0.6792Epoch 00013: val_loss did not improve\n",
      "89600/89600 [==============================] - 47s - loss: 0.6067 - precision: 0.6661 - recall: 0.6114 - fbeta_score: 0.6467 - acc: 0.6792 - val_loss: 0.6065 - val_precision: 0.6767 - val_recall: 0.5648 - val_fbeta_score: 0.6445 - val_acc: 0.6774\n",
      "Epoch 15/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.6043 - precision: 0.6677 - recall: 0.6155 - fbeta_score: 0.6490 - acc: 0.6804Epoch 00014: val_loss did not improve\n",
      "89600/89600 [==============================] - 47s - loss: 0.6043 - precision: 0.6677 - recall: 0.6155 - fbeta_score: 0.6490 - acc: 0.6803 - val_loss: 0.6122 - val_precision: 0.6375 - val_recall: 0.6991 - val_fbeta_score: 0.6449 - val_acc: 0.6806\n",
      "Epoch 16/20\n",
      "89472/89600 [============================>.] - ETA: 0s - loss: 0.6018 - precision: 0.6698 - recall: 0.6196 - fbeta_score: 0.6518 - acc: 0.6833Epoch 00015: val_loss improved from 0.60598 to 0.60483, saving model to saved_models/bCNN_gray.best.from_scratch.hdf5\n",
      "89600/89600 [==============================] - 47s - loss: 0.6017 - precision: 0.6698 - recall: 0.6196 - fbeta_score: 0.6518 - acc: 0.6833 - val_loss: 0.6048 - val_precision: 0.6760 - val_recall: 0.5804 - val_fbeta_score: 0.6481 - val_acc: 0.6796\n",
      "Epoch 17/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.5996 - precision: 0.6730 - recall: 0.6234 - fbeta_score: 0.6555 - acc: 0.6861Epoch 00016: val_loss did not improve\n",
      "89600/89600 [==============================] - 48s - loss: 0.5997 - precision: 0.6730 - recall: 0.6234 - fbeta_score: 0.6555 - acc: 0.6861 - val_loss: 0.6108 - val_precision: 0.6958 - val_recall: 0.5157 - val_fbeta_score: 0.6425 - val_acc: 0.6755\n",
      "Epoch 18/20\n",
      "89536/89600 [============================>.] - ETA: 0s - loss: 0.5973 - precision: 0.6737 - recall: 0.6247 - fbeta_score: 0.6562 - acc: 0.6867Epoch 00017: val_loss did not improve\n",
      "89600/89600 [==============================] - 50s - loss: 0.5972 - precision: 0.6737 - recall: 0.6248 - fbeta_score: 0.6563 - acc: 0.6868 - val_loss: 0.6143 - val_precision: 0.6934 - val_recall: 0.5248 - val_fbeta_score: 0.6437 - val_acc: 0.6756\n",
      "Epoch 19/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.5948 - precision: 0.6778 - recall: 0.6327 - fbeta_score: 0.6612 - acc: 0.6910Epoch 00018: val_loss did not improve\n",
      "89600/89600 [==============================] - 49s - loss: 0.5948 - precision: 0.6779 - recall: 0.6327 - fbeta_score: 0.6613 - acc: 0.6910 - val_loss: 0.6085 - val_precision: 0.6664 - val_recall: 0.6087 - val_fbeta_score: 0.6480 - val_acc: 0.6796\n",
      "Epoch 20/20\n",
      "89568/89600 [============================>.] - ETA: 0s - loss: 0.5929 - precision: 0.6769 - recall: 0.6336 - fbeta_score: 0.6603 - acc: 0.6906Epoch 00019: val_loss did not improve\n",
      "89600/89600 [==============================] - 49s - loss: 0.5929 - precision: 0.6769 - recall: 0.6336 - fbeta_score: 0.6603 - acc: 0.6906 - val_loss: 0.6067 - val_precision: 0.6810 - val_recall: 0.5719 - val_fbeta_score: 0.6497 - val_acc: 0.6806\n",
      "Epoch 00019: early stopping\n",
      "training time: 16.09 minutes\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping\n",
    "import numpy as np\n",
    "\n",
    "epochs = 20\n",
    "batch_size = 32\n",
    "\n",
    "earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')\n",
    "log = CSVLogger('saved_models/log_bCNN_gray.csv')\n",
    "checkpointer = ModelCheckpoint(filepath='saved_models/bCNN_gray.best.from_scratch.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "model.fit(train_tensors, train_labels, \n",
    "          validation_data=(valid_tensors, valid_labels),\n",
    "          epochs=epochs, batch_size=batch_size, callbacks=[checkpointer, log, earlystop], verbose=1)\n",
    "\n",
    "# Show total training time\n",
    "print(\"training time: %.2f minutes\"%((time.time()-start)/60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('saved_models/bCNN_gray.best.from_scratch.hdf5')\n",
    "prediction = model.predict(test_tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.671851 %\n",
      "Recall: 0.572077 %\n",
      "Fscore: 0.649206 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.666622"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.eval(binary_accuracy(K.variable(value=test_labels),\n",
    "                      K.variable(value=prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.627903 %\n",
      "Recall: 0.710935 %\n",
      "Fscore: 0.642921 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.4\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}