Final Submission.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gesture Recognition\n",
    "\n",
    "#### Building 2 types of architectures - \n",
    "#### 1. CNN + RNN architecture \n",
    "#### 2. 3D convolutional network\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "## importing the necessary libraries\n",
    "import numpy as np\n",
    "import os\n",
    "from scipy.misc import imread, imresize\n",
    "import datetime\n",
    "import cv2\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "import random as rn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setting the random seed so that the results don't vary drastically."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(30)\n",
    "rn.seed(30)\n",
    "tf.set_random_seed(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading the folder names for training and validation and Setting the `batch_size`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_doc = np.random.permutation(open('Project_data/train.csv').readlines())\n",
    "val_doc = np.random.permutation(open('Project_data/val.csv').readlines())\n",
    "batch_size =10 #experiment with the batch size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = 120   # X \n",
    "cols = 120   # Y \n",
    "channel = 3  # number of channels in images 3 for color(RGB)\n",
    "frames=15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data resizing - Resizing all the images, so we can have all the images in a specific size\n",
    "def crop_resize_img(img):\n",
    "    if img.shape[0] != img.shape[1]:\n",
    "        img=img[0:120,10:150]\n",
    "    resized_image = imresize(img, (rows,cols))\n",
    "    return resized_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#using percentile to deal with outliers in the data.\n",
    "def normalize_image(img):\n",
    "    #using percentile for normalization for images, as min-max is giving better results.\n",
    "    # normalized_image= img - np.percentile(img,15)/ np.percentile(img,85) - np.percentile(img,15) \n",
    "    normalized_image= (img - np.min(img))/(np.max(img)- np.min(img))\n",
    "    return normalized_image\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocessing the images before feed data to the model in batches.\n",
    "# Hori\n",
    "\n",
    "def fetch_aug_batchdata(source_path, folder_list, batch_num, batch_size, t,validation):\n",
    "    \n",
    "    # intialize variables to store data read from train data\n",
    "    batch_data = np.zeros((batch_size,frames,rows,cols,channel)) # x is the number of images you use for each video, (y,z) is the final size of the input images and 3 is the number of channels RGB\n",
    "    batch_labels = np.zeros((batch_size,5)) # batch_labels is the one hot representation of the output\n",
    "    #print(batch_data)\n",
    "    #print(batch_labels)\n",
    "    # intialize variables for augumented batch data with affine transformation\n",
    "    batch_data_aug,batch_label_aug = batch_data,batch_labels\n",
    "    #print(batch_data_aug)\n",
    "    #print(batch_label_aug)\n",
    "    # intialize variables for augmented batch data with horizontal flip\n",
    "    batch_data_flip,batch_label_flip = batch_data,batch_labels\n",
    "    \n",
    "    #create a list of image numbers you want to use for a particular video using full frames\n",
    "    img_idx = [x for x in range(0, 30,2)] \n",
    "    \n",
    "    for folder in range(batch_size): # iterate over the batch_size\n",
    "        # read all the images in the folder\n",
    "        imgs = sorted(os.listdir(source_path+'/'+ t[folder + (batch_num*batch_size)].split(';')[0])) \n",
    "        # create a random affine to be used in image transformation for buidling agumented data set\n",
    "        dx, dy = np.random.randint(-1.7, 1.8, 2)\n",
    "        M = np.float32([[1, 0, dx], [0, 1, dy]])\n",
    "        \n",
    "        #  Iterate over the frames for each folder to read them in\n",
    "        for idx, item in enumerate(img_idx):             \n",
    "            image = cv2.imread(source_path+'/'+ t[folder + (batch_num*batch_size)].strip().split(';')[0]+'/'+imgs[item], cv2.IMREAD_COLOR)\n",
    "            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)           \n",
    "            # Cropping non symmetric frames\n",
    "            #crop the images and resize them. Note that the images are of 2 different shape \n",
    "            #and the conv3D will throw error if the inputs in a batch have different shapes \n",
    "            resized_image=crop_resize_img(image)\n",
    "            \n",
    "            #Normal data\n",
    "            batch_data[folder,idx,:,:,0] = normalize_image(resized_image[:, : , 0])#normalise and feed in the image\n",
    "            batch_data[folder,idx,:,:,1] = normalize_image(resized_image[:, : , 1])#normalise and feed in the image\n",
    "            batch_data[folder,idx,:,:,2] = normalize_image(resized_image[:, : , 2])#normalise and feed in the image\n",
    "            \n",
    "            x =resized_image.shape[0]\n",
    "            y =resized_image.shape[1]\n",
    "            #Data with affine transformation\n",
    "            batch_data_aug[folder,idx] = (cv2.warpAffine(resized_image, M, (x,y)))\n",
    "            \n",
    "            # Data with horizontal flip\n",
    "            batch_data_flip[folder,idx]= np.flip(resized_image,1)\n",
    "\n",
    "        batch_labels[folder, int(t[folder + (batch_num*batch_size)].strip().split(';')[2])] = 1\n",
    "        batch_label_aug[folder, int(t[folder + (batch_num*batch_size)].strip().split(';')[2])] = 1\n",
    "        \n",
    "        if int(t[folder + (batch_num * batch_size)].strip().split(';')[2])==0:\n",
    "            batch_label_flip[folder, 1] = 1\n",
    "        elif int(t[folder + (batch_num*batch_size)].strip().split(';')[2])==1:\n",
    "            batch_label_flip[folder, 0] = 1                    \n",
    "        else:\n",
    "            batch_label_flip[folder, int(t[folder + (batch_num*batch_size)].strip().split(';')[2])] = 1\n",
    "                  \n",
    "    #adding the augumented data in the main data.\n",
    "    \n",
    "    batch_data_final = np.append(batch_data, batch_data_aug, axis = 0)\n",
    "    batch_data_final = np.append(batch_data_final, batch_data_flip, axis = 0)\n",
    "\n",
    "    batch_label_final = np.append(batch_labels, batch_label_aug, axis = 0) \n",
    "    batch_label_final = np.append(batch_label_final, batch_label_flip, axis = 0)\n",
    "    \n",
    "    if validation:\n",
    "        batch_data_final=batch_data\n",
    "        batch_label_final= batch_labels\n",
    "        \n",
    "    return batch_data_final,batch_label_final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generator\n",
    "The overall structure of the generator has been given. In the generator, you are going to preprocess the images as you have images of 2 different dimensions as well as create a batch of video frames. You have to experiment with `img_idx`, `y`,`z` and normalization such that you get high accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generator(source_path, folder_list, batch_size, validation=False,ablation=None):\n",
    "    print( 'Source path = ', source_path, '; batch size =', batch_size)\n",
    "    if(ablation!=None):\n",
    "        folder_list=folder_list[:ablation]\n",
    "    while True:\n",
    "        t = np.random.permutation(folder_list)\n",
    "        num_batches = len(folder_list)//batch_size # calculate the number of batches\n",
    "        for batch in range(num_batches): # we iterate over the number of batches\n",
    "            # you yield the batch_data and the batch_labels, remember what does yield do\n",
    "            yield fetch_aug_batchdata(source_path, folder_list, batch, batch_size, t,validation)\n",
    "            \n",
    "        \n",
    "        # Code for the remaining data points which are left after full batches\n",
    "        if (len(folder_list) != batch_size*num_batches):\n",
    "            batch_size = len(folder_list) - (batch_size*num_batches)\n",
    "            yield fetch_aug_batchdata(source_path, folder_list, batch, batch_size, t,validation)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# training sequences = 663\n",
      "# validation sequences = 100\n",
      "# epochs = 30\n"
     ]
    }
   ],
   "source": [
    "# Passing the Training and validation folder paths and setting a value for epochs\n",
    "curr_dt_time = datetime.datetime.now()\n",
    "train_path = 'Project_data/train'\n",
    "val_path = 'Project_data/val'\n",
    "num_train_sequences = len(train_doc)\n",
    "print('# training sequences =', num_train_sequences)\n",
    "num_val_sequences = len(val_doc)\n",
    "print('# validation sequences =', num_val_sequences)\n",
    "num_epochs = 30 # the number of epochs\n",
    "print ('# epochs =', num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model\n",
    "using `TimeDistributed` while building a Conv2D + RNN model, the last layer is the softmax. Designing the network in such a way that the model is able to give good accuracy on the least number of parameters so that it can fit in the memory of the webcam."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, GRU, Flatten, TimeDistributed, Flatten, BatchNormalization, Activation, Dropout,LSTM\n",
    "from keras.layers.convolutional import Conv3D,Conv2D, MaxPooling3D,MaxPooling2D\n",
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
    "from keras import optimizers\n",
    "from keras.applications import ResNet50\n",
    "from keras.applications.resnet50 import preprocess_input\n",
    "from keras.layers.recurrent import LSTM, GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#write your model here\n",
    "nb_featuremap = [8,16,32,64]\n",
    "nb_dense = [128,64,5]\n",
    "nb_classes = 5\n",
    "# Input\n",
    "input_shape=(frames,rows,cols,channel)\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(TimeDistributed(Conv2D(nb_featuremap[0], (3, 3), strides=(2, 2),activation='relu', padding='same'), input_shape=input_shape))\n",
    "\n",
    "\n",
    "model.add(TimeDistributed(Conv2D(nb_featuremap[1], (3,3),padding='same', activation='relu')))\n",
    "model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))\n",
    "\n",
    "model.add(TimeDistributed(Conv2D(nb_featuremap[2], (3,3),padding='same', activation='relu')))\n",
    "model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))\n",
    "\n",
    "model.add(TimeDistributed(Conv2D(nb_featuremap[3], (2,2),padding='same', activation='relu')))\n",
    "model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))\n",
    "\n",
    "model.add(TimeDistributed(BatchNormalization()))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(TimeDistributed(Flatten()))\n",
    "\n",
    "model.add(Dense(nb_dense[0], activation='relu'))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Dense(nb_dense[1], activation='relu'))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "## using GRU as the RNN model along with softmax as our last layer.\n",
    "model.add(GRU(128, return_sequences=False))\n",
    "model.add(Dense(nb_classes, activation='softmax')) # using Softmax as last layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compiling the Model and printing the summary of the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_1 (TimeDist (None, 15, 60, 60, 8)     224       \n",
      "_________________________________________________________________\n",
      "time_distributed_2 (TimeDist (None, 15, 60, 60, 16)    1168      \n",
      "_________________________________________________________________\n",
      "time_distributed_3 (TimeDist (None, 15, 30, 30, 16)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_4 (TimeDist (None, 15, 30, 30, 32)    4640      \n",
      "_________________________________________________________________\n",
      "time_distributed_5 (TimeDist (None, 15, 15, 15, 32)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_6 (TimeDist (None, 15, 15, 15, 64)    8256      \n",
      "_________________________________________________________________\n",
      "time_distributed_7 (TimeDist (None, 15, 7, 7, 64)      0         \n",
      "_________________________________________________________________\n",
      "time_distributed_8 (TimeDist (None, 15, 7, 7, 64)      256       \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 15, 7, 7, 64)      0         \n",
      "_________________________________________________________________\n",
      "time_distributed_9 (TimeDist (None, 15, 3136)          0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 15, 128)           401536    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 15, 128)           0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 15, 64)            8256      \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 15, 64)            0         \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (None, 128)               74112     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 5)                 645       \n",
      "=================================================================\n",
      "Total params: 499,093\n",
      "Trainable params: 498,965\n",
      "Non-trainable params: 128\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from keras.optimizers import Adam\n",
    "\n",
    "optimiser =Adam(0.001) #write your optimizer\n",
    "model.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model.summary())\n",
    "\n",
    "# creating the `train_generator` and the `val_generator` which will be used in `.fit_generator`\n",
    "train_generator = generator(train_path, train_doc, batch_size)\n",
    "val_generator = generator(val_path, val_doc, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/disks/user/anaconda3/lib/python3.6/site-packages/keras/callbacks.py:999: UserWarning: `epsilon` argument is deprecated and will be removed, use `min_delta` instead.\n",
      "  warnings.warn('`epsilon` argument is deprecated and '\n"
     ]
    }
   ],
   "source": [
    "model_name = 'model_init' + '_' + str(curr_dt_time).replace(' ','').replace(':','_') + '/'\n",
    "    \n",
    "if not os.path.exists(model_name):\n",
    "    os.mkdir(model_name)\n",
    "        \n",
    "filepath = model_name + 'model-{epoch:05d}-{loss:.5f}-{categorical_accuracy:.5f}-{val_loss:.5f}-{val_categorical_accuracy:.5f}.h5'\n",
    "\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=False, mode='auto', period=1)\n",
    "\n",
    "LR = ReduceLROnPlateau(monitor='val_loss', factor=0.01, patience=5, cooldown=4, verbose=1,mode='auto',epsilon=0.0001)# write the REducelronplateau code here\n",
    "callbacks_list = [checkpoint, LR]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `steps_per_epoch` and `validation_steps` are used by `fit_generator` to decide the number of next() calls it need to make."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "if (num_train_sequences%batch_size) == 0:\n",
    "    steps_per_epoch = int(num_train_sequences/batch_size)\n",
    "else:\n",
    "    steps_per_epoch = (num_train_sequences//batch_size) + 1\n",
    "\n",
    "if (num_val_sequences%batch_size) == 0:\n",
    "    validation_steps = int(num_val_sequences/batch_size)\n",
    "else:\n",
    "    validation_steps = (num_val_sequences//batch_size) + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us now fit the model. This will start training the model and with the help of the checkpoints, you'll be able to save the model at the end of each epoch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  Project_data/val ; batch size = 10\n",
      "Source path =  Project_data/train ; batch size = 10\n",
      "Epoch 1/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/disks/user/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  \"\"\"\n",
      "/mnt/disks/user/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  \"\"\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "67/67 [==============================] - 187s 3s/step - loss: 1.7178 - categorical_accuracy: 0.4162 - val_loss: 1.5197 - val_categorical_accuracy: 0.4700\n",
      "\n",
      "Epoch 00001: saving model to model_init_2019-06-2417_37_04.366939/model-00001-1.72648-0.41126-1.51973-0.47000.h5\n",
      "Epoch 2/5\n",
      "67/67 [==============================] - 38s 561ms/step - loss: 1.6572 - categorical_accuracy: 0.4776 - val_loss: 2.2221 - val_categorical_accuracy: 0.4600\n",
      "\n",
      "Epoch 00002: saving model to model_init_2019-06-2417_37_04.366939/model-00002-1.65718-0.47761-2.22215-0.46000.h5\n",
      "Epoch 3/5\n",
      "67/67 [==============================] - 39s 579ms/step - loss: 1.7473 - categorical_accuracy: 0.4561 - val_loss: 1.9846 - val_categorical_accuracy: 0.4200\n",
      "\n",
      "Epoch 00003: saving model to model_init_2019-06-2417_37_04.366939/model-00003-1.74728-0.45605-1.98459-0.42000.h5\n",
      "Epoch 4/5\n",
      "67/67 [==============================] - 40s 600ms/step - loss: 1.5072 - categorical_accuracy: 0.5058 - val_loss: 2.1926 - val_categorical_accuracy: 0.3700\n",
      "\n",
      "Epoch 00004: saving model to model_init_2019-06-2417_37_04.366939/model-00004-1.50717-0.50580-2.19264-0.37000.h5\n",
      "Epoch 5/5\n",
      "67/67 [==============================] - 39s 579ms/step - loss: 1.4871 - categorical_accuracy: 0.5456 - val_loss: 2.5730 - val_categorical_accuracy: 0.3200\n",
      "\n",
      "Epoch 00005: saving model to model_init_2019-06-2417_37_04.366939/model-00005-1.48714-0.54561-2.57298-0.32000.h5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f77db1334e0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1,callbacks=callbacks_list, validation_data=val_generator,validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0,use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Convolution 3D Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1 (Conv3D)               (None, 15, 120, 120, 8)   3008      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 15, 120, 120, 8)   0         \n",
      "_________________________________________________________________\n",
      "conv2 (Conv3D)               (None, 15, 120, 120, 16)  3472      \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 15, 120, 120, 16)  0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_1 (MaxPooling3 (None, 7, 60, 60, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv3 (Conv3D)               (None, 7, 60, 60, 32)     4640      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 7, 60, 60, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_2 (MaxPooling3 (None, 3, 30, 30, 32)     0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 3, 30, 30, 32)     128       \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 3, 30, 30, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_3 (MaxPooling3 (None, 1, 15, 15, 32)     0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 7200)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               921728    \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 941,557\n",
      "Trainable params: 941,493\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "#write your model here\n",
    "nb_featuremap = [8,16,32,64]\n",
    "nb_dense = [128,64,5]\n",
    "nb_classes = 5\n",
    "# Input\n",
    "input_shape=(frames,rows,cols,channel)\n",
    "\n",
    "model3d = Sequential()\n",
    "model3d.add(Conv3D(nb_featuremap[0], \n",
    "                 kernel_size=(5,5,5),\n",
    "                 input_shape=input_shape,\n",
    "                 padding='same', name=\"conv1\"))\n",
    "model3d.add(Activation('relu'))\n",
    "model3d.add(Conv3D(nb_featuremap[1], \n",
    "                 kernel_size=(3,3,3),\n",
    "                 padding='same',name=\"conv2\"))\n",
    "model3d.add(Activation('relu'))\n",
    "model3d.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "model3d.add(Conv3D(nb_featuremap[2], \n",
    "                 kernel_size=(1,3,3), \n",
    "                 padding='same',name=\"conv3\"))\n",
    "model3d.add(Activation('relu'))\n",
    "model3d.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "model3d.add(BatchNormalization())\n",
    "model3d.add(Dropout(0.25))\n",
    "model3d.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "model3d.add(Flatten())\n",
    "model3d.add(Dense(nb_dense[0], activation='relu'))\n",
    "model3d.add(Dropout(0.25))\n",
    "model3d.add(Dense(nb_dense[1], activation='relu'))\n",
    "#softmax layer\n",
    "model3d.add(Dense(nb_dense[2], activation='softmax'))\n",
    "optimiser = Adam(0.001)\n",
    "model3d.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model3d.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  Project_data/val ; batch size = 10\n",
      "Source path =  Project_data/train ; batch size = 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/disks/user/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  \"\"\"\n",
      "/mnt/disks/user/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  \"\"\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "67/67 [==============================] - 151s 2s/step - loss: 2.4812 - categorical_accuracy: 0.2411 - val_loss: 2.0845 - val_categorical_accuracy: 0.4000\n",
      "\n",
      "Epoch 00001: saving model to model_init_2019-06-2417_37_04.366939/model-00001-2.48451-0.24133-2.08455-0.40000.h5\n",
      "Epoch 2/5\n",
      "67/67 [==============================] - 59s 882ms/step - loss: 2.1009 - categorical_accuracy: 0.2869 - val_loss: 4.0516 - val_categorical_accuracy: 0.3200\n",
      "\n",
      "Epoch 00002: saving model to model_init_2019-06-2417_37_04.366939/model-00002-2.10093-0.28690-4.05157-0.32000.h5\n",
      "Epoch 3/5\n",
      "67/67 [==============================] - 59s 874ms/step - loss: 2.1721 - categorical_accuracy: 0.1990 - val_loss: 2.4504 - val_categorical_accuracy: 0.2700\n",
      "\n",
      "Epoch 00003: saving model to model_init_2019-06-2417_37_04.366939/model-00003-2.17208-0.19900-2.45038-0.27000.h5\n",
      "Epoch 4/5\n",
      "67/67 [==============================] - 59s 879ms/step - loss: 2.1943 - categorical_accuracy: 0.3101 - val_loss: 2.0907 - val_categorical_accuracy: 0.1200\n",
      "\n",
      "Epoch 00004: saving model to model_init_2019-06-2417_37_04.366939/model-00004-2.19432-0.31012-2.09067-0.12000.h5\n",
      "Epoch 5/5\n",
      "67/67 [==============================] - 59s 878ms/step - loss: 1.9624 - categorical_accuracy: 0.3267 - val_loss: 2.7345 - val_categorical_accuracy: 0.2200\n",
      "\n",
      "Epoch 00005: saving model to model_init_2019-06-2417_37_04.366939/model-00005-1.96239-0.32670-2.73453-0.22000.h5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f77acd94ef0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model3d.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1,callbacks=callbacks_list, validation_data=val_generator,validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0,use_multiprocessing=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}