IF201812 -Train With Augmentation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train a dataset from Interface 2018/12 with Keras\n",
    "\n",
    "- Unlike small book image dataset, it was little bit harder to fine-tune parameters.\n",
    "- Similar accuracy with fast.ai could be achieved, but spent a lot more effort.\n",
    "\n",
    "Using fast.ai library would be the shortest path to reach the goal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "import numpy as np\n",
    "np.warnings.filterwarnings('ignore')\n",
    "np.random.seed(1001)\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import shutil\n",
    "from pathlib import Path\n",
    "\n",
    "import IPython\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm_notebook\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "matplotlib.style.use('ggplot')\n",
    "%matplotlib inline\n",
    "\n",
    "from keras.applications.vgg16 import VGG16\n",
    "\n",
    "from keras.preprocessing import image\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, Flatten, Dropout, GlobalAveragePooling2D, AveragePooling2D, Conv2D, Softmax\n",
    "from keras import backend as K\n",
    "import keras\n",
    "\n",
    "from skimage.io import imread\n",
    "from scipy.misc import imresize\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from mixup_generator import MixupGenerator\n",
    "from random_eraser import get_random_eraser\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.utils import to_categorical\n",
    "from keras.callbacks import (EarlyStopping, LearningRateScheduler,\n",
    "                             ModelCheckpoint, TensorBoard, ReduceLROnPlateau)\n",
    "\n",
    "### Model\n",
    "def model_imagenet_x(input_shape, num_classes, weights='imagenet', show_layers=False):\n",
    "    # create the base pre-trained model\n",
    "    #base_model = InceptionV3(weights='imagenet', include_top=False, input_shape=input_shape)\n",
    "    base_model = VGG16(weights=weights, include_top=False, input_shape=input_shape)\n",
    "\n",
    "    # add a global spatial average pooling layer & FC\n",
    "    x = base_model.output\n",
    "    x = GlobalAveragePooling2D()(x)\n",
    "    x = Dense(256, activation='relu')(x)\n",
    "\n",
    "    if show_layers:\n",
    "        for i, layer in enumerate(base_model.layers):\n",
    "            print(i, layer.name)\n",
    "\n",
    "    predictions = Dense(num_classes, activation='softmax')(x)\n",
    "\n",
    "    # this is the model we will train\n",
    "    model = Model(inputs=base_model.input, outputs=predictions)\n",
    "\n",
    "    return model, base_model\n",
    "\n",
    "def model_imagenet_x_fastailike(input_shape, num_classes, weights='imagenet', show_layers=False):\n",
    "    # create the base pre-trained model\n",
    "    #base_model = InceptionV3(weights='imagenet', include_top=False, input_shape=input_shape)\n",
    "    base_model = VGG16(weights=weights, include_top=False, input_shape=input_shape)\n",
    "\n",
    "    # add a global spatial average pooling layer & FC\n",
    "    x = base_model.output\n",
    "    x = Conv2D(num_classes, kernel_size=3, padding='valid')(x)\n",
    "    x = AveragePooling2D(pool_size=5)(x)\n",
    "    x = Flatten()(x)\n",
    "    predictions = Softmax()(x)\n",
    "\n",
    "    if show_layers:\n",
    "        for i, layer in enumerate(base_model.layers):\n",
    "            print(i, layer.name)\n",
    "\n",
    "    # this is the model we will train\n",
    "    model = Model(inputs=base_model.input, outputs=predictions)\n",
    "\n",
    "    return model, base_model\n",
    "\n",
    "### Configuration management class\n",
    "class Config:\n",
    "    def __init__(self,\n",
    "                 learning_rate=0.0001,\n",
    "                 batch_size=16,\n",
    "                 shape=[224, 224, 3],\n",
    "                 use_mixup=True,\n",
    "                 use_augmentations=True,\n",
    "                verbose=1):\n",
    "        self.learning_rate = learning_rate\n",
    "        self.batch_size = batch_size\n",
    "        self.shape = shape\n",
    "        self.verbose = verbose\n",
    "        self.use_mixup = use_mixup\n",
    "        self.use_augmentations = use_augmentations\n",
    "\n",
    "### Dataset distribution utility\n",
    "def get_class_distribution(y):\n",
    "    # y_cls can be one of [OH label, index of class, class label name]\n",
    "    # convert OH to index of class\n",
    "    y_cls = [np.argmax(one) for one in y] if len(np.array(y).shape) == 2 else y\n",
    "    # y_cls can be one of [index of class, class label name]\n",
    "    classset = sorted(list(set(y_cls)))\n",
    "    sample_distribution = {cur_cls:len([one for one in y_cls if one == cur_cls]) for cur_cls in classset}\n",
    "    return sample_distribution\n",
    "\n",
    "def get_class_distribution_list(y, num_classes):\n",
    "    dist = get_class_distribution(y)\n",
    "    assert(y[0].__class__ != str) # class index or class OH label only\n",
    "    list_dist = np.zeros((num_classes))\n",
    "    for i in range(num_classes):\n",
    "        if i in dist:\n",
    "            list_dist[i] = dist[i]\n",
    "    return list_dist\n",
    "\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "def balance_class_by_over_sampling(X, y): # Naive: all sample has equal weights\n",
    "    Xidx = [[xidx] for xidx in range(len(X))]\n",
    "    y_cls = [np.argmax(one) for one in y]\n",
    "    classset = sorted(list(set(y_cls)))\n",
    "    sample_distribution = [len([one for one in y_cls if one == cur_cls]) for cur_cls in classset]\n",
    "    nsamples = np.max(sample_distribution)\n",
    "    flat_ratio = {cls:nsamples for cls in classset}\n",
    "    Xidx_resampled, y_cls_resampled = RandomOverSampler(ratio=flat_ratio, random_state=42).fit_sample(Xidx, y_cls)\n",
    "    sampled_index = [idx[0] for idx in Xidx_resampled]\n",
    "    return np.array([X[idx] for idx in sampled_index]), np.array([y[idx] for idx in sampled_index])\n",
    "\n",
    "### Dataset management class\n",
    "class LabeledDataset:\n",
    "    image_suffix = ['.jpg', '.JPG']\n",
    "    def __init__(self, datapath, shape, batch_size):\n",
    "        self.datapath = Path(datapath)\n",
    "        self.shape = shape\n",
    "        self.batch_size = batch_size\n",
    "    def load_image(filename, shape, rescale_factor):\n",
    "        img = imread(filename)\n",
    "        return imresize(img, shape[:2]) * rescale_factor\n",
    "    def load_as_image(self):\n",
    "        # datapath shall contain label/file labeled data files\n",
    "        train_files = sorted([x for x in self.datapath.glob('*/*') if x.suffix in LabeledDataset.image_suffix])\n",
    "        self.X_train = np.array([LabeledDataset.load_image(filename, self.shape, rescale_factor=1/255.)\n",
    "                                    for filename in train_files])\n",
    "        y_train_label = [filename.parent.name for filename in train_files]\n",
    "        self.labels = sorted(list(set(y_train_label)))\n",
    "        self.label2int = {label:i for i, label in enumerate(self.labels)}\n",
    "        self.y_train = to_categorical([self.label2int[label] for label in y_train_label])\n",
    "    def split_train_valid(self, test_size=0.2, random_state=42):\n",
    "        self.cur_X_train, self.cur_X_valid, self.cur_y_train, self.cur_y_valid = train_test_split(\n",
    "            self.X_train, \n",
    "            self.y_train, \n",
    "            test_size=test_size,\n",
    "            random_state=random_state)\n",
    "        self.cur_X_train, self.cur_y_train = \\\n",
    "            balance_class_by_over_sampling(self.cur_X_train, self.cur_y_train)\n",
    "    def load_test_as_image(self, test_datapath):\n",
    "        test_datapath = Path(test_datapath)\n",
    "        test_files = sorted([x for x in test_datapath.glob('*') if x.suffix in LabeledDataset.image_suffix])\n",
    "        self.X_test = np.array([LabeledDataset.load_image(filename, self.shape, rescale_factor=1/255.)\n",
    "                                for filename in test_files])\n",
    "    def create_test_generator(self, IDG_options={}):\n",
    "        test_datagen = ImageDataGenerator(**IDG_options)\n",
    "        y_test_dummy = to_categorical([0 for _ in range(len(self.X_test))])\n",
    "        self.test_gen = test_datagen.flow(self.X_test, y_test_dummy,\n",
    "                                          batch_size=len(self.X_test), shuffle=False) ########## self.batch_size\n",
    "    def create_generator(self, conf, IDG_options={}):\n",
    "        aug_datagen = ImageDataGenerator(**IDG_options)\n",
    "        if conf.use_mixup:\n",
    "            self.train_gen = MixupGenerator(self.cur_X_train, self.cur_y_train, \n",
    "                                            alpha=1.0, batch_size=self.batch_size,\n",
    "                                            datagen=aug_datagen)()\n",
    "        else:\n",
    "            self.train_gen = aug_datagen.flow(self.cur_X_train, self.cur_y_train, \n",
    "                                              batch_size=self.batch_size)\n",
    "        plain_datagen = ImageDataGenerator()\n",
    "        self.valid_gen = plain_datagen.flow(self.cur_X_valid, self.cur_y_valid,\n",
    "                                                              batch_size=self.batch_size, shuffle=False)\n",
    "    def train_steps_per_epoch(self):\n",
    "        return len(self.cur_X_train) // self.batch_size\n",
    "    def valid_steps_per_epoch(self):\n",
    "        return len(self.cur_X_valid) // self.batch_size\n",
    "\n",
    "def reset_generator():\n",
    "    if conf.use_augmentations:\n",
    "        IDG_options={'horizontal_flip': True, 'vertical_flip': True,\n",
    "                     'rotation_range': 40, 'zoom_range': 0.2,\n",
    "                      'preprocessing_function': get_random_eraser(v_l=np.min(d.X_train),                                                           \n",
    "                                                    v_h=np.max(d.X_train))}\n",
    "    else:\n",
    "        IDG_options={}\n",
    "    d.create_generator(conf, IDG_options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2018-10-25 17:06:59--  https://github.com/yasudakn/umaibar/raw/master/interface-201812-umaibar-data-content.zip\n",
      "Resolving github.com (github.com)... 192.30.255.112, 192.30.255.113\n",
      "Connecting to github.com (github.com)|192.30.255.112|:443... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://raw.githubusercontent.com/yasudakn/umaibar/master/interface-201812-umaibar-data-content.zip [following]\n",
      "--2018-10-25 17:07:00--  https://raw.githubusercontent.com/yasudakn/umaibar/master/interface-201812-umaibar-data-content.zip\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.88.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.88.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 21043355 (20M) [application/zip]\n",
      "Saving to: ‘interface-201812-umaibar-data-content.zip.1’\n",
      "\n",
      "interface-201812-um 100%[===================>]  20.07M  4.17MB/s    in 4.8s    \n",
      "\n",
      "2018-10-25 17:07:06 (4.16 MB/s) - ‘interface-201812-umaibar-data-content.zip.1’ saved [21043355/21043355]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "! wget https://github.com/yasudakn/umaibar/raw/master/interface-201812-umaibar-data-content.zip\n",
    "! mkdir -p interface-201812-umaibar-data-content\n",
    "! unzip interface-201812-umaibar-data-content.zip\n",
    "! mv data-content interface-201812-umaibar-data-content\n",
    "! rm interface-201812-umaibar-data-content.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATAROOT = 'interface-201812-umaibar-data-content/data-content'\n",
    "datapath = Path(DATAROOT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf = Config(batch_size=8, shape=[224, 224, 3], \n",
    "              use_mixup=True, use_augmentations=True)\n",
    "\n",
    "d = LabeledDataset(datapath / 'train', conf.shape, conf.batch_size)\n",
    "d.load_as_image()\n",
    "d.split_train_valid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ImageNet Based"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0001"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf.learning_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imagenet_based_evaluation(train_points, model_footer, epochs):\n",
    "    for train_point in train_points:\n",
    "        weight_filename = 'model_weights_%s_imagenet_%s.h5' % (train_point, model_footer)\n",
    "        callbacks = [\n",
    "            ModelCheckpoint(weight_filename, monitor='val_loss', verbose=2, save_best_only=True, save_weights_only=True),\n",
    "            #TensorBoard(log_dir=d.file_with_prefix('logs%s/fold_%d' % (conf.prefix, i)), write_graph=True)\n",
    "        ]\n",
    "        reset_generator()\n",
    "\n",
    "        print('[%s]' % train_point)\n",
    "        model, base_model = model_imagenet_x(input_shape=conf.shape, num_classes=len(d.labels),\n",
    "                                             weights='imagenet')\n",
    "        for layer in base_model.layers:\n",
    "            layer.trainable = False\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=keras.optimizers.Adam(lr=0.001),#, decay=1e-7, epsilon=1e-8),#conf.learning_rate),\n",
    "                      metrics=['accuracy'])\n",
    "        model.summary()\n",
    "        model.fit_generator(d.train_gen, \n",
    "                  steps_per_epoch=d.train_steps_per_epoch(),\n",
    "                  epochs=15,\n",
    "                  validation_data=d.valid_gen,\n",
    "                  validation_steps=d.valid_steps_per_epoch(),\n",
    "                  callbacks=callbacks,\n",
    "                  verbose=conf.verbose)\n",
    "        print('\\n----------------------------------------------')\n",
    "        print('Roughly trained, now fine tune from', train_point, 'for', epochs, 'epochs.')\n",
    "        trainable = False\n",
    "        for layer in base_model.layers:\n",
    "            if layer.name == train_point:\n",
    "                trainable = True\n",
    "            layer.trainable = trainable\n",
    "        model.load_weights(weight_filename)\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "#                      optimizer=keras.optimizers.Adam(lr=0.00001, decay=1e-7, epsilon=1e-8), # not so much good... 0.9 or so\n",
    "#                      optimizer=keras.optimizers.Adam(lr=0.00001), # 0.92\n",
    "#                      optimizer=keras.optimizers.Adam(lr=0.00001, decay=1e-4), # 0.9\n",
    "                      optimizer=keras.optimizers.RMSprop(lr=0.00001), # 0.92\n",
    "#                      optimizer=keras.optimizers.SGD(lr=0.0002, momentum=0.9, decay=1e-7), # 0.92\n",
    "#                      optimizer=keras.optimizers.Adam(lr=0.00001, decay=1e-7), # 0.9\n",
    "                      metrics=['accuracy'])\n",
    "        history = model.fit_generator(d.train_gen, \n",
    "                  steps_per_epoch=d.train_steps_per_epoch(),\n",
    "                  epochs=epochs,\n",
    "                  validation_data=d.valid_gen,\n",
    "                  validation_steps=d.valid_steps_per_epoch(),\n",
    "                  callbacks=callbacks,\n",
    "                  verbose=conf.verbose)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf = Config(batch_size=8, shape=[224, 224, 3], \n",
    "              use_mixup=True, use_augmentations=True)\n",
    "imagenet_based_evaluation(train_points=['block3_conv1'], model_footer='full_aug', epochs=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TBD - Visualization\n",
    "\n",
    "Followings are NOT executed, left just for the future attempts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "model, base_model = model_imagenet_x(input_shape=conf.shape, num_classes=len(d.labels),\n",
    "                                     weights='imagenet')\n",
    "model.load_weights('model_weights_block3_conv1_imagenet_full_aug.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "\n",
    "def imshow_friendly(img):\n",
    "    img_temp = img - np.min(img)\n",
    "    img_temp = img_temp/np.max(img_temp)\n",
    "    friendly = np.uint8(255 * img_temp)\n",
    "    return friendly\n",
    "\n",
    "def visualize_cam(model, model_weight, test_file_index, datapath, \n",
    "                  expected_preds, test_time_aug_param={}):\n",
    "    d.load_test_as_image(datapath)\n",
    "    d.create_test_generator(test_time_aug_param)\n",
    "    model.load_weights(model_weight)\n",
    "    last_conv_layer = model.get_layer('block5_conv3')\n",
    "    cur_X_test, cur_y_test = next(d.test_gen)\n",
    "    x = np.array([cur_X_test[test_file_index]])\n",
    "    preds = model.predict(x)\n",
    "    targ_class = np.argmax(preds[0])\n",
    "    result = calc_soft_acc(expected_preds[test_file_index], preds[0])\n",
    "\n",
    "    output = model.output[:, targ_class]\n",
    "    grads = K.gradients(output, last_conv_layer.output)[0]\n",
    "    pooled_grads = K.mean(grads, axis=(0, 1, 2))\n",
    "    iterate = K.function([model.input], [pooled_grads, last_conv_layer.output[0]])\n",
    "    pooled_grads_value, conv_layer_output_value = iterate([x])\n",
    "    for i in range(int(last_conv_layer.output.shape[3])):\n",
    "        conv_layer_output_value[:, :, i] *= pooled_grads_value[i]\n",
    "    heatmap = np.mean(conv_layer_output_value, axis=-1)\n",
    "    heatmap = np.maximum(heatmap, 0)\n",
    "    heatmap /= np.max(heatmap)\n",
    "    \n",
    "    img = next(d.test_gen)[0][test_file_index]\n",
    "    fig = plt.figure(figsize=(10, 5), dpi=100)\n",
    "    ax = fig.add_subplot(131)\n",
    "    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))\n",
    "    ax.set_axis_off()\n",
    "    ax.matshow(heatmap)\n",
    "    heatmap = np.uint8(255 * heatmap)\n",
    "    heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)\n",
    "    superimposed = ((heatmap*0.5/np.max(heatmap) + img)) / 1.5\n",
    "    ax = fig.add_subplot(132)\n",
    "    ax.set_axis_off()\n",
    "    ax.imshow(imshow_friendly(superimposed))\n",
    "    ax.set_title('%s? %s' % (d.labels[targ_class], 'yes' if result == 1 else 'no'), fontsize=12)\n",
    "    ax = fig.add_subplot(133)\n",
    "    ax.set_axis_off()\n",
    "    ax.imshow(imshow_friendly(img))\n",
    "    fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for i in range(20):\n",
    "    visualize_cam(model, 'model_weights_block4_conv1_imagenet_aug_no_mixup.h5', i, datapath / 'test_difficult',\n",
    "                  test_difficult_expected_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}