Predicting Invasive Ductal Carcinoma using CNN in Keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import fnmatch\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imagePatches = glob('C:/Users/asus/Documents/Breast cancer classification/**/*.png', recursive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "patternZero = '*class0.png'\n",
    "patternOne = '*class1.png'\n",
    "classZero = fnmatch.filter(imagePatches, patternZero) #saves the file location of all images with file name class0\n",
    "classOne = fnmatch.filter(imagePatches, patternOne) #saves the file location of all images with file name class1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The function process_images takes as input the start and end index. It also reads the pixel values of the images and resizes them so that all images are of the same size. The array x contains the image data and array y contains the corresponding class of the images. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_images(lowerIndex,upperIndex):\n",
    "    \"\"\"\n",
    "    Returns two arrays: \n",
    "        x is an array of resized images\n",
    "        y is an array of labels\n",
    "    \"\"\" \n",
    "    height = 50\n",
    "    width = 50\n",
    "    channels = 3\n",
    "    x = [] #list to store image data\n",
    "    y = [] #list to store corresponding class\n",
    "    for img in imagePatches[lowerIndex:upperIndex]:\n",
    "        full_size_image = cv2.imread(img)\n",
    "        image = (cv2.resize(full_size_image, (width,height), interpolation=cv2.INTER_CUBIC))\n",
    "        x.append(image)\n",
    "        if img in classZero:\n",
    "            y.append(0)\n",
    "        elif img in classOne:\n",
    "            y.append(1)\n",
    "        else:\n",
    "            return\n",
    "    return x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, Y = process_images(0,60000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = np.array(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = X.astype(np.float32) #Casting the array to single precision takes half as much space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X /= 255. #Normalizing, which helps train the model faster and prevents the vanishing/ exploding gradient problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,Y,test_size=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44478"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.count(0) #Checking the number of 0's in the array Y (this denotes number of malignant cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15522"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.count(1) #Checking the number of 1's in the array Y (this denotes number of malignant cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13210"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.count(1) #Checking the number of 1's in the array y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37790"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.count(0) #Checking the number of 1's in the array y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#One-Hot-Encode y_train and y_test\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dealing with class imbalance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reshaping the image data to feed into RandomUnderSampler to deal with class imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_trainShape = X_train.shape[1]*X_train.shape[2]*X_train.shape[3]\n",
    "X_testShape = X_test.shape[1]*X_test.shape[2]*X_test.shape[3]\n",
    "X_trainFlat = X_train.reshape(X_train.shape[0], X_trainShape)\n",
    "X_testFlat = X_test.reshape(X_test.shape[0], X_testShape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The majority class is undersampled to make number of samples of both classes equal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "random_under_sampler = RandomUnderSampler(ratio='majority')\n",
    "X_trainRus, Y_trainRus = random_under_sampler.fit_sample(X_trainFlat, y_train)\n",
    "X_testRus, Y_testRus = random_under_sampler.fit_sample(X_testFlat, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# One-hot-encoding\n",
    "Y_trainRusHot = to_categorical(Y_trainRus, num_classes = 2)\n",
    "Y_testRusHot = to_categorical(Y_testRus, num_classes = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([13210, 13210], dtype=int64))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(Y_trainRus, return_counts=True) #checking the number of samples in each class to make sure RandomUnderSampling worked"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we have equal number of 1's and 0's"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cells below reshapes X_trainRos and X_testRos to the original image shape of 50 x 50 x 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(len(X_trainRus)):\n",
    "    height, width, channels = 50,50,3\n",
    "    X_trainRusReshaped = X_trainRus.reshape(len(X_trainRus),height,width,channels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(len(X_testRus)):\n",
    "    height, width, channels = 50,50,3\n",
    "    X_testRusReshaped = X_testRus.reshape(len(X_testRus),height,width,channels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.preprocessing.image import ImageDataGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 256\n",
    "num_classes = 2\n",
    "epochs = 80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3,3),\n",
    "                 activation='relu',\n",
    "                 input_shape=(50,50,3)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Conv2D(64, (3,3), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(Conv2D(128, (3, 3), activation='relu'))\n",
    "model.add(Conv2D(256, (3, 3), activation='relu'))\n",
    "model.add(Flatten()) #this converts our 3D feature maps to 1D feature vectors for the dense layer below\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dense(num_classes, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compiling the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss=keras.losses.binary_crossentropy,\n",
    "              optimizer=keras.optimizers.Adam(lr=0.00001),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data augmentation on-the-fly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datagen = ImageDataGenerator(\n",
    "    featurewise_center=True,\n",
    "    featurewise_std_normalization=True,\n",
    "    rotation_range=180,\n",
    "    horizontal_flip=True,vertical_flip = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Early stopping to prevent overfitting and model checkpoint to save best model based on minimum val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "early_stopping_monitor = EarlyStopping(monitor='val_loss', patience=3, mode='min')\n",
    "model_checkpoint = ModelCheckpoint('best_model.h5', monitor='val_loss', mode='min', verbose=1, save_best_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fitting the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\lib\\site-packages\\keras_preprocessing\\image.py:1131: UserWarning: This ImageDataGenerator specifies `featurewise_center`, but it hasn't been fit on any training data. Fit it first by calling `.fit(numpy_data)`.\n",
      "  warnings.warn('This ImageDataGenerator specifies '\n",
      "E:\\Anaconda\\lib\\site-packages\\keras_preprocessing\\image.py:1139: UserWarning: This ImageDataGenerator specifies `featurewise_std_normalization`, but it hasn't been fit on any training data. Fit it first by calling `.fit(numpy_data)`.\n",
      "  warnings.warn('This ImageDataGenerator specifies '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104/103 [==============================] - 141s 1s/step - loss: 0.6915 - acc: 0.5022 - val_loss: 0.6883 - val_acc: 0.4999\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.68831, saving model to best_model.h5\n",
      "Epoch 2/80\n",
      "104/103 [==============================] - 32s 309ms/step - loss: 0.6827 - acc: 0.5479 - val_loss: 0.6534 - val_acc: 0.7306\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.68831 to 0.65338, saving model to best_model.h5\n",
      "Epoch 3/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.6096 - acc: 0.7203 - val_loss: 0.5330 - val_acc: 0.7623\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.65338 to 0.53301, saving model to best_model.h5\n",
      "Epoch 4/80\n",
      "104/103 [==============================] - 33s 315ms/step - loss: 0.5164 - acc: 0.7708 - val_loss: 0.4992 - val_acc: 0.7728\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.53301 to 0.49919, saving model to best_model.h5\n",
      "Epoch 5/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4954 - acc: 0.7813 - val_loss: 0.4940 - val_acc: 0.7769\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.49919 to 0.49399, saving model to best_model.h5\n",
      "Epoch 6/80\n",
      "104/103 [==============================] - 32s 309ms/step - loss: 0.4881 - acc: 0.7839 - val_loss: 0.4956 - val_acc: 0.7787\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.49399\n",
      "Epoch 7/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4845 - acc: 0.7864 - val_loss: 0.4922 - val_acc: 0.7821\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.49399 to 0.49216, saving model to best_model.h5\n",
      "Epoch 8/80\n",
      "104/103 [==============================] - 34s 324ms/step - loss: 0.4828 - acc: 0.7887 - val_loss: 0.4896 - val_acc: 0.7834\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.49216 to 0.48955, saving model to best_model.h5\n",
      "Epoch 9/80\n",
      "104/103 [==============================] - 33s 317ms/step - loss: 0.4826 - acc: 0.7892 - val_loss: 0.4883 - val_acc: 0.7832\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.48955 to 0.48829, saving model to best_model.h5\n",
      "Epoch 10/80\n",
      "104/103 [==============================] - 33s 321ms/step - loss: 0.4796 - acc: 0.7895 - val_loss: 0.5128 - val_acc: 0.7609\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.48829\n",
      "Epoch 11/80\n",
      "104/103 [==============================] - 32s 309ms/step - loss: 0.4790 - acc: 0.7893 - val_loss: 0.4851 - val_acc: 0.7853\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.48829 to 0.48515, saving model to best_model.h5\n",
      "Epoch 12/80\n",
      "104/103 [==============================] - 33s 318ms/step - loss: 0.4765 - acc: 0.7914 - val_loss: 0.4859 - val_acc: 0.7845\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.48515\n",
      "Epoch 13/80\n",
      "104/103 [==============================] - 32s 308ms/step - loss: 0.4759 - acc: 0.7912 - val_loss: 0.4836 - val_acc: 0.7873\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.48515 to 0.48360, saving model to best_model.h5\n",
      "Epoch 14/80\n",
      "104/103 [==============================] - 34s 330ms/step - loss: 0.4746 - acc: 0.7929 - val_loss: 0.4843 - val_acc: 0.7853\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.48360\n",
      "Epoch 15/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4729 - acc: 0.7931 - val_loss: 0.4821 - val_acc: 0.7863\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.48360 to 0.48207, saving model to best_model.h5\n",
      "Epoch 16/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4736 - acc: 0.7931 - val_loss: 0.4874 - val_acc: 0.7806\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.48207\n",
      "Epoch 17/80\n",
      "104/103 [==============================] - 32s 308ms/step - loss: 0.4720 - acc: 0.7933 - val_loss: 0.4832 - val_acc: 0.7847\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.48207\n",
      "Epoch 18/80\n",
      "104/103 [==============================] - 32s 309ms/step - loss: 0.4701 - acc: 0.7946 - val_loss: 0.4794 - val_acc: 0.7869\n",
      "\n",
      "Epoch 00018: val_loss improved from 0.48207 to 0.47943, saving model to best_model.h5\n",
      "Epoch 19/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4697 - acc: 0.7944 - val_loss: 0.4787 - val_acc: 0.7873\n",
      "\n",
      "Epoch 00019: val_loss improved from 0.47943 to 0.47866, saving model to best_model.h5\n",
      "Epoch 20/80\n",
      "104/103 [==============================] - 33s 316ms/step - loss: 0.4693 - acc: 0.7939 - val_loss: 0.4767 - val_acc: 0.7888\n",
      "\n",
      "Epoch 00020: val_loss improved from 0.47866 to 0.47673, saving model to best_model.h5\n",
      "Epoch 21/80\n",
      "104/103 [==============================] - 35s 336ms/step - loss: 0.4682 - acc: 0.7954 - val_loss: 0.4772 - val_acc: 0.7876\n",
      "\n",
      "Epoch 00021: val_loss did not improve from 0.47673\n",
      "Epoch 22/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4698 - acc: 0.7944 - val_loss: 0.4765 - val_acc: 0.7880\n",
      "\n",
      "Epoch 00022: val_loss improved from 0.47673 to 0.47652, saving model to best_model.h5\n",
      "Epoch 23/80\n",
      "104/103 [==============================] - 33s 318ms/step - loss: 0.4694 - acc: 0.7934 - val_loss: 0.4766 - val_acc: 0.7874\n",
      "\n",
      "Epoch 00023: val_loss did not improve from 0.47652\n",
      "Epoch 24/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4652 - acc: 0.7973 - val_loss: 0.4761 - val_acc: 0.7881\n",
      "\n",
      "Epoch 00024: val_loss improved from 0.47652 to 0.47605, saving model to best_model.h5\n",
      "Epoch 25/80\n",
      "104/103 [==============================] - 32s 309ms/step - loss: 0.4654 - acc: 0.7957 - val_loss: 0.4727 - val_acc: 0.7914\n",
      "\n",
      "Epoch 00025: val_loss improved from 0.47605 to 0.47265, saving model to best_model.h5\n",
      "Epoch 26/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4644 - acc: 0.7969 - val_loss: 0.4726 - val_acc: 0.7902\n",
      "\n",
      "Epoch 00026: val_loss improved from 0.47265 to 0.47260, saving model to best_model.h5\n",
      "Epoch 27/80\n",
      "104/103 [==============================] - 36s 348ms/step - loss: 0.4621 - acc: 0.7991 - val_loss: 0.4711 - val_acc: 0.7901\n",
      "\n",
      "Epoch 00027: val_loss improved from 0.47260 to 0.47110, saving model to best_model.h5\n",
      "Epoch 28/80\n",
      "104/103 [==============================] - 32s 311ms/step - loss: 0.4605 - acc: 0.7992 - val_loss: 0.4693 - val_acc: 0.7921\n",
      "\n",
      "Epoch 00028: val_loss improved from 0.47110 to 0.46926, saving model to best_model.h5\n",
      "Epoch 29/80\n",
      "104/103 [==============================] - 32s 310ms/step - loss: 0.4618 - acc: 0.7989 - val_loss: 0.4696 - val_acc: 0.7907\n",
      "\n",
      "Epoch 00029: val_loss did not improve from 0.46926\n",
      "Epoch 30/80\n",
      "104/103 [==============================] - 32s 308ms/step - loss: 0.4591 - acc: 0.8000 - val_loss: 0.4691 - val_acc: 0.7908\n",
      "\n",
      "Epoch 00030: val_loss improved from 0.46926 to 0.46914, saving model to best_model.h5\n",
      "Epoch 31/80\n",
      "104/103 [==============================] - 32s 309ms/step - loss: 0.4602 - acc: 0.8014 - val_loss: 0.4676 - val_acc: 0.7913\n",
      "\n",
      "Epoch 00031: val_loss improved from 0.46914 to 0.46760, saving model to best_model.h5\n",
      "Epoch 32/80\n",
      "104/103 [==============================] - 34s 326ms/step - loss: 0.4584 - acc: 0.8000 - val_loss: 0.4674 - val_acc: 0.7916\n",
      "\n",
      "Epoch 00032: val_loss improved from 0.46760 to 0.46736, saving model to best_model.h5\n",
      "Epoch 33/80\n",
      "104/103 [==============================] - 35s 337ms/step - loss: 0.4581 - acc: 0.8015 - val_loss: 0.4643 - val_acc: 0.7951\n",
      "\n",
      "Epoch 00033: val_loss improved from 0.46736 to 0.46429, saving model to best_model.h5\n",
      "Epoch 34/80\n",
      "104/103 [==============================] - 35s 332ms/step - loss: 0.4568 - acc: 0.8006 - val_loss: 0.4641 - val_acc: 0.7910\n",
      "\n",
      "Epoch 00034: val_loss improved from 0.46429 to 0.46407, saving model to best_model.h5\n",
      "Epoch 35/80\n",
      "104/103 [==============================] - 35s 341ms/step - loss: 0.4571 - acc: 0.8012 - val_loss: 0.4648 - val_acc: 0.7958\n",
      "\n",
      "Epoch 00035: val_loss did not improve from 0.46407\n",
      "Epoch 36/80\n",
      "104/103 [==============================] - 40s 387ms/step - loss: 0.4559 - acc: 0.8020 - val_loss: 0.4676 - val_acc: 0.7909\n",
      "\n",
      "Epoch 00036: val_loss did not improve from 0.46407\n",
      "Epoch 37/80\n",
      "104/103 [==============================] - 37s 358ms/step - loss: 0.4523 - acc: 0.8036 - val_loss: 0.4677 - val_acc: 0.7897\n",
      "\n",
      "Epoch 00037: val_loss did not improve from 0.46407\n"
     ]
    }
   ],
   "source": [
    "training = model.fit_generator(datagen.flow(X_trainRusReshaped,Y_trainRusHot,batch_size=batch_size),\n",
    "                    steps_per_epoch=len(X_trainRusReshaped) / batch_size, epochs=epochs,validation_data=(X_testRusReshaped, Y_testRusHot), verbose=1, callbacks=[early_stopping_monitor, model_checkpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plotting the training and validation loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\lib\\site-packages\\keras_preprocessing\\image.py:1131: UserWarning: This ImageDataGenerator specifies `featurewise_center`, but it hasn't been fit on any training data. Fit it first by calling `.fit(numpy_data)`.\n",
      "  warnings.warn('This ImageDataGenerator specifies '\n",
      "E:\\Anaconda\\lib\\site-packages\\keras_preprocessing\\image.py:1139: UserWarning: This ImageDataGenerator specifies `featurewise_std_normalization`, but it hasn't been fit on any training data. Fit it first by calling `.fit(numpy_data)`.\n",
      "  warnings.warn('This ImageDataGenerator specifies '\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmY3GWV6PHvqeqqrq7qfUnSWTuEbARCICGyuAQQDKiA\nGwKDV2ZG0ZnxOt4Zvcq9o46MzvDMHdcZXEAzo+OCiOLgiAJRgihbEgxLEkJCSEhn633vrvXcP95f\nd6r3JV1d3dXn81hPVf2W6tP1mD6823lFVTHGGGNG4st2AMYYY6Y/SxbGGGNGZcnCGGPMqCxZGGOM\nGZUlC2OMMaOyZGGMMWZUliyMmQQi8h8i8vkxXntIRN58up9jzFSyZGGMMWZUliyMMcaMypKFmTW8\n7p9PiMjzItIpIt8Rkbki8isRaReRrSJSlnb9NSKyW0RaRGSbiKxOO3eeiDzr3fdjIDTgZ71NRHZ5\n9z4hImsnGPMHReSAiDSJyAMiMt87LiLyZRGpE5FW73c62zt3tYjs8WI7KiIfn9AXZkwaSxZmtnkX\ncAWwAng78Cvg/wCVuH8PHwUQkRXAj4CPAVXAg8AvRCQoIkHg58B/AuXAT7zPxbv3fGAL8CGgAvgW\n8ICI5I8nUBG5DPgn4HqgGjgM3OOdvhJ4o/d7lALvBRq9c98BPqSqRcDZwG/H83ONGYolCzPb/Kuq\nnlTVo8DjwNOq+kdVjQL3A+d5170X+KWqPqKqceBfgALgYuBCIAB8RVXjqnofsD3tZ3wQ+JaqPq2q\nSVX9LhD17huPPwG2qOqzXny3AReJSA0QB4qAVYCo6l5VPe7dFwfOEpFiVW1W1WfH+XONGcSShZlt\nTqa97h7ifaH3ej7uv+QBUNUUcARY4J07qv2rcB5Oe70E+FuvC6pFRFqARd594zEwhg5c62GBqv4W\n+DfgTuCkiNwlIsXepe8CrgYOi8hjInLROH+uMYNYsjBmaMdwf/QBN0aA+4N/FDgOLPCO9Vqc9voI\n8AVVLU17hFX1R6cZQwTXrXUUQFW/pqrrgTW47qhPeMe3q+q1wBxcd9m94/y5xgxiycKYod0LvFVE\nLheRAPC3uK6kJ4AngQTwURHJE5F3AhvT7r0b+LCIvM4biI6IyFtFpGicMfwQ+FMRWeeNd/wjrtvs\nkIhc4H1+AOgEeoCkN6byJyJS4nWftQHJ0/gejAEsWRgzJFXdB9wM/CvQgBsMf7uqxlQ1BrwTuAVo\nxo1v/Czt3h24cYt/884f8K4dbwy/AT4N/BTXmlkG3OCdLsYlpWZcV1UjblwF4H3AIRFpAz7s/R7G\nnBaxzY+MMcaMxloWxhhjRmXJwhhjzKgsWRhjjBmVJQtjjDGjysvkh4vIZuCrgB/4tqreMeD8l4FL\nvbdhYI6qlnrn3g/8nXfu894q2GFVVlZqTU3NJEZvjDG5b+fOnQ2qWjXadRmbDSUifuBlXB2eWlw5\nhBtVdc8w1/9P4DxV/TMRKQd2ABsABXYC61W1ebift2HDBt2xY8ck/xbGGJPbRGSnqm4Y7bpMdkNt\nBA6o6kFvXvo9wLUjXH8jrnAbwFuAR1S1yUsQjwCbMxirMcaYEWQyWSzAlT3oVesdG0RElgBLOVUd\nc0z3isitIrJDRHbU19dPStDGGGMGy2SykCGODdfndQNwn6r2liUY072qepeqblDVDVVVo3a5GWOM\nmaBMDnDX4gqv9VqIK4w2lBuAvxpw76YB924bbwDxeJza2lp6enrGe+uMEwqFWLhwIYFAINuhGGNy\nUCaTxXZguYgsxVXJvAG4aeBFIrISKMMVZ+v1EPCPabuWXYmr5T8utbW1FBUVUVNTQ/8CoblFVWls\nbKS2tpalS5dmOxxjTA7KWDeUqiaAj+D+8O8F7lXV3SJyu4hck3bpjcA96XsDqGoT8A+4hLMduN07\nNi49PT1UVFTkdKIAEBEqKipmRQvKGJMdGV1noaoP4rajTD/2mQHv/36Ye7fgtqY8LbmeKHrNlt/T\nGJMds34Ft6pyvLWbWMJK/htjzHBmfbKIJVI0dcY4UNdJVywx6Z/f0tLC17/+9XHfd/XVV9PS0jLp\n8RhjzETM+mSRH/CzrKoQn8DB+k5au+OT+vnDJYtkcuSWzIMPPkhpaemkxmKMMRM165MFiSihxr0s\nL44TCvg53NhJQ3t00j7+U5/6FK+88grr1q3jggsu4NJLL+Wmm27inHPOAeC6665j/fr1rFmzhrvu\nuqvvvpqaGhoaGjh06BCrV6/mgx/8IGvWrOHKK6+ku7t70uIzxpixyOgA93TyuV/sZs+xtqFPxjrB\nVw95IaKJJImkEvD7COaNnEvPml/MZ9++ZsRr7rjjDl588UV27drFtm3beOtb38qLL77YN8V1y5Yt\nlJeX093dzQUXXMC73vUuKioq+n3G/v37+dGPfsTdd9/N9ddfz09/+lNuvtl2yjTGTJ1ZkyxG5PND\nynUL5ef5EVLEkylSqoQC/kn9URs3buy3FuJrX/sa999/PwBHjhxh//79g5LF0qVLWbduHQDr16/n\n0KFDkxqTMcaMZtYkixFbAN3N0HwIKpZDfiEAjR1RjrV0Ewr4qamMEPBPTo9dJBLpe71t2za2bt3K\nk08+STgcZtOmTUOulcjPz+977ff7rRvKGDPlbMwCIL/IPUdPdVNVFOazpCJCNJHiQF0H3fGJTa0t\nKiqivb19yHOtra2UlZURDod56aWXeOqppyb0M4wxJtNmTctiRL48CBZCTxsUz+87XFwQYFlVhEON\nXbzW2MXKeUXj/uiKigouueQSzj77bAoKCpg7d27fuc2bN/PNb36TtWvXsnLlSi688MJJ+XWMMWay\nZWzzo6k21OZHe/fuZfXq1WP7gPaT0H4M5q4Bf7DfqRNtPdS39bBmQQm+abxSely/rzHGMD02P5pZ\nQsXuuWfwjKl8vw/FLeAzxpjZyJJFr7yQa1EMkSx6p9BasjDGzFaWLHqJQH4xxNpB+yeFfEsWxphZ\nzpJFulCxSxTRjn6H/T7BL0I0acnCGDM7WbJIFywEpN8UWnDlv4N5PmtZGGNmLUsW6Xx+t+ZimHEL\nK2NujJmtLFkMFCqGZBTi/VdS5+f5iCWU1DinGk+0RDnAV77yFbq6uiZ0rzHGTCZLFgPle1NoB3RF\nBfP8KEp8nOMWliyMMbnAVnAPlJfvptH2tEHhnL7D6TOi8vPGXlwwvUT5FVdcwZw5c7j33nuJRqO8\n4x3v4HOf+xydnZ1cf/311NbWkkwm+fSnP83Jkyc5duwYl156KZWVlTz66KOT/qsaY8xYzZ5k8atP\nwYkXxnZtMgrJOAQjgFuxXYByRjTpkkZvUcF558BVd4z4Ueklyh9++GHuu+8+nnnmGVSVa665ht/9\n7nfU19czf/58fvnLXwKuZlRJSQlf+tKXePTRR6msrJzob22MMZPCuqGG4vMD2le2HFzKEGHcYxbp\nHn74YR5++GHOO+88zj//fF566SX279/POeecw9atW/nkJz/J448/TklJyen/DsYYM4lmT8tilBZA\nP5qCEy9CQSmULgZcsjh6sp2g30dNZWTk+4f7WFVuu+02PvShDw06t3PnTh588EFuu+02rrzySj7z\nmc9M6GcYY0wmWMtiKOI7NYU2rSWRn+cjOs61Fuklyt/ylrewZcsWOjrcor+jR49SV1fHsWPHCIfD\n3HzzzXz84x/n2WefHXSvMcZk0+xpWYxXqBh6WiDeDcEw4NZatPUkUFVkjNVn00uUX3XVVdx0001c\ndNFFABQWFvL973+fAwcO8IlPfAKfz0cgEOAb3/gGALfeeitXXXUV1dXVNsBtjMkqK1E+nGQcTr4I\nRdVQNA+Axs4oR5u7WTWviOA4ZkRNFStRbowZLytRfrr8AQiE+63mzvdbQUFjzOxkyWIk+cUQ74Rk\nAqCvNTHecQtjjJnpcj5ZnFY3W6j/au6AXxARYtOw+myudCcaY6annE4WoVCIxsbGif8hDYTd/txe\nV5SIkO/3EY1Pr2ShqjQ2NhIKhbIdijEmR+X0bKiFCxdSW1tLfX39xD+kqxXiJ6C4C0Ro7IiSSCnd\n9dPrD3MoFGLhwoXZDsMYk6NyOlkEAgGWLl16eh+y+374xS3wp7+GJRfx+f/ew/efPsyez23G5xvb\n9FljjJnpcrobalIsu8x1Re1/CIAllRF64inq2qNZDswYY6ZORpOFiGwWkX0ickBEPjXMNdeLyB4R\n2S0iP0w7nhSRXd7jgUzGOaJQCSy+CA5sBaCmwi3QO9TYmbWQjDFmqmWsG0pE/MCdwBVALbBdRB5Q\n1T1p1ywHbgMuUdVmEZmT9hHdqrouU/GNy5yz4Pl7AKipcHWhDjd2cuEZFdmMyhhjpkwmWxYbgQOq\nelBVY8A9wLUDrvkgcKeqNgOoal0G45m4SCX0tEIiRnVJiIBfONRomxIZY2aPTCaLBcCRtPe13rF0\nK4AVIvIHEXlKRDannQuJyA7v+HUZjHN0EW8/ia4G8vw+FpWFOWzdUMaYWSSTs6GGmio0cMFDHrAc\n2AQsBB4XkbNVtQVYrKrHROQM4Lci8oKqvtLvB4jcCtwKsHjx4smO/5RIlXvubIDi+SypCHOowVoW\nxpjZI5Mti1pgUdr7hcCxIa75L1WNq+qrwD5c8kBVj3nPB4FtwHkDf4Cq3qWqG1R1Q1VV1eT/Br3C\nXsui063XWFIR4XBjp62aNsbMGplMFtuB5SKyVESCwA3AwFlNPwcuBRCRSly31EERKROR/LTjlwB7\nyJb0lgWwpCJMZyxJQ0csayEZY8xUyliyUNUE8BHgIWAvcK+q7haR20XkGu+yh4BGEdkDPAp8QlUb\ngdXADhF5zjt+R/osqimXNmYB/WdEGWPMbJDRFdyq+iDw4IBjn0l7rcDfeI/0a54AzslkbOMSKgFf\nIK0bqnetRRcbasqzGZkxxkwJW8E9FiKudeEli4VlYXxiLQtjzOxhyWKsIpV9YxbBPB8Lygo4bGst\njDGzhCWLsYpU9SULcOMW1rIwxswWlizGKnyqGwrcuIWt4jbGzBaWLMZqiJZFa3ecli6bPmuMyX2W\nLMYqUun244651sQSb/qstS6MMbOBJYuxGrTWwk2ftXELY8xsYMlirPpWcbtxi0XlYUSwGlHGmFnB\nksVY9SWLRgBCAT/VxSFrWRhjZgVLFmMV9jY66jcjKmI75hljZgVLFmM1oBsKoKYybAvzjDGzgiWL\nsQpGIK+gb4AbXMuisTNGW088i4EZY0zmWbIYq776UOlrLdyMqNesdWGMyXGWLMYjMnAVd+9aCxu3\nMMbkNksW4zFgFfeSvrUW1rIwxuQ2SxbjMSBZhIN5zCnK51CDtSyMMbnNksV4hCtcN1Ta3tuu+qy1\nLIwxuc2SxXhEqiAZhWh73yFXfdZaFsaY3GbJYjx611qkTZ+tqYxQ1x6lK5bIUlDGGJN5lizGo7eY\noA1yG2NmGUsW49GXLNJWcXvTZ61GlDEml1myGI++kh+nWhaLvZaF7WthjMlllizGIzy4ZVEcClAR\nCVrLwhiT0yxZjEcgBMGifi0L8GZE2b4WxpgcZslivCKV/WZDQe9aC2tZGGNylyWL8RpQHwpcjahj\nrT30xJNZCsoYYzLLksV4DSj5AW5fC4AjTdYVZYzJTZYsxmtAmXJIrz5rycIYk5ssWYxX2BuzSKX6\nDs0vDQFwvLU7W1EZY0xGWbIYr0gVpBLQ09J3qDKST8AvHG/tyWJgxhiTOZYsxquvPlRj3yGfT5hb\nHOKEJQtjTI6yZDFeQ5T8AKguCVk3lDEmZ1myGK9hksW8kgJrWRhjclZGk4WIbBaRfSJyQEQ+Ncw1\n14vIHhHZLSI/TDv+fhHZ7z3en8k4x6WvPtRQLYseNG1jJGOMyRV5mfpgEfEDdwJXALXAdhF5QFX3\npF2zHLgNuERVm0Vkjne8HPgssAFQYKd3b3Om4h2zcIV77mzsd3hecYhoIkVLV5yySDALgRljTOZk\nsmWxETigqgdVNQbcA1w74JoPAnf2JgFVrfOOvwV4RFWbvHOPAJszGOvY+QMQKh2yZQHYjChjTE7K\nZLJYABxJe1/rHUu3AlghIn8QkadEZPM47kVEbhWRHSKyo76+fuDpzIlUDTFm4ZLFiTYb5DbG5J5M\nJgsZ4tjADv08YDmwCbgR+LaIlI7xXlT1LlXdoKobqqqqTjPccYhU9Zs6C1BdUgBYy8IYk5symSxq\ngUVp7xcCx4a45r9UNa6qrwL7cMljLPdmT6RiUMuiqigfv09sRpQxJidlMllsB5aLyFIRCQI3AA8M\nuObnwKUAIlKJ65Y6CDwEXCkiZSJSBlzpHZsehuiG8vuEOUX51rIwxuSkjM2GUtWEiHwE90feD2xR\n1d0icjuwQ1Uf4FRS2AMkgU+oaiOAiPwDLuEA3K6qTZmKddwiVdDVBKkk+Px9h+eV2CpuY0xuyliy\nAFDVB4EHBxz7TNprBf7Gewy8dwuwJZPxTVi4ElCXMApPjZVUl4TYd6I9e3EZY0yG2AruiRhuFXdx\ngS3MM8bkJEsWE9FXTLD/vhbVJSG6Yknao4ksBGWMMZljyWIihq0P5a21sHELY0yOsWQxEX31oQa3\nLMDWWhhjco8li4koKAPxDUoWvS2L4y22itsYk1ssWUyEzw8F5YO6oeYUhRCxloUxJvdYspioIRbm\nBfN8VBbm25iFMSbnWLKYqEjloG4o8Pa1aLNkYYzJLZYsJipSNWjqLLhkccK2VzXG5BhLFhMVqRzU\nDQWu+qyNWRhjco0li4mKVEFPKyRi/Q7PKwnR3pOgwxbmGWNyyJiShYj8tYgUi/MdEXlWRK7MdHDT\nWu/CvEH7WtjCPGNM7hlry+LPVLUNVyq8CvhT4I6MRTUThIerD2XJwhiTe8aaLHp3rrsa+HdVfY6h\nd7ObPfpWcQ/ci7t3xzwb5DbG5I6xJoudIvIwLlk8JCJFQCpzYc0AfcUE+3dDzSnOB6xlYYzJLWPd\nz+LPgXXAQVXtEpFyXFfU7BWpcM8DWhahgJ+KSNDWWhhjcspYWxYXAftUtUVEbgb+DmjNXFgzQKgU\nfHlDTp+1HfOMMblmrMniG0CXiJwL/G/gMPC9jEU1E4h4JT+GWcVtycIYk0PGmiwS3hao1wJfVdWv\nAkWZC2uGCA9d8mOereI2xuSYsSaLdhG5DXgf8EsR8QOBzIU1Q4ywiru5K05PPJmFoIwxZvKNNVm8\nF4ji1lucABYA/y9jUc0Uw9SHsrUWxphcM6Zk4SWIHwAlIvI2oEdVZ/eYBYxYeRZsXwtjTO4Ya7mP\n64FngPcA1wNPi8i7MxnYjBCphFgHxLr6He7bi7vNxi2MMblhrOss/i9wgarWAYhIFbAVuC9Tgc0I\nfQvzGiC4uO/wPGtZGGNyzFjHLHy9icLTOI57c1dfyY/+XVHhYB4lBQEbszDG5Iyxtix+LSIPAT/y\n3r8XeDAzIc0gfcUEhx63ONZiycIYkxvGlCxU9RMi8i7gElwBwbtU9f6MRjYTRIauPAveWgsbszDG\n5IixtixQ1Z8CP81gLDNP+pjFANUlIV48OrsrohhjcseIyUJE2gEd6hSgqlqckahmimAE8kLDLsxr\n6IgRTSTJz/NnIThjjJk8IyYLVbWSHiMZoT5U74yourYoi8rDUx2ZMcZMKpvRdLpsYZ4xZhawZHG6\nwsPVh+pNFjbIbYyZ+TKaLERks4jsE5EDIvKpIc7fIiL1IrLLe3wg7Vwy7fgDmYzztAzbDeW2V7W1\nFsaYXDDm2VDj5VWmvRO4AqgFtovIA6q6Z8ClP1bVjwzxEd2qui5T8U2aSKWbDaXqxjA8hfl5FOXn\nWTeUMSYnZLJlsRE4oKoHVTUG3IPbDyO3RCoh0eNqRA1gO+YZY3JFJpPFAuBI2vta79hA7xKR50Xk\nPhFZlHY8JCI7ROQpEbluqB8gIrd61+yorx88bjAl+kp+DL0wz/biNsbkgkwmCxni2MA1G78AalR1\nLa4w4XfTzi1W1Q3ATcBXRGTZoA9TvUtVN6jqhqqqqsmKe3z6kkXjoFPVtmOeMSZHZDJZ1ALpLYWF\nwLH0C1S1UVWj3tu7gfVp5455zweBbcB5GYx14sIV7nnIlkUBde1R4snUFAdljDGTK5PJYjuwXESW\nikgQuAHoN6tJRKrT3l4D7PWOl4lIvve6EleTauDA+PQwQjdUdUkIVahvjw46Z4wxM0nGZkOpakJE\nPgI8BPiBLaq6W0RuB3ao6gPAR0XkGiABNAG3eLevBr4lIilcQrtjiFlU08MoxQTBLcybX1owlVEZ\nY8ykyliyAFDVBxlQylxVP5P2+jbgtiHuewI4J5OxTZpAAQSLoGvoMQuwtRbGmJnPVnBPhkjF0N1Q\nxa41Yau4jTEznSWLyRCpGjJZFBfkURDwW8vCGDPjWbKYDJGqIafOigjVttbCGJMDLFlMhvDQ3VBg\nq7iNMbnBksVkiFSdqg81wLySEMdbbMzCGDOzWbKYDJEqSCWgp2XQqeqSECfboyRTQ204aIwxM4Ml\ni8nQt9Zi6FLlyZTS0GEL84wxM5cli8kwwsK8+bZjnjEmB1iymAyFc91zy5FBp+b1LcyzcQtjzMxl\nyWIyVK1yM6IObB10qrqkd2GetSyMMTOXJYvJ4PPD8rfA/ochmeh3qiwcIJjns+mzxpgZzZLFZFm5\n2c2GOvJUv8N9C/MsWRhjZjBLFpNl2WXgD8K+Xw06Na/YFuYZY2Y2SxaTJb8Iat4AL/960ClX8sMG\nuI0xM5cli8m08ipoPAAN+/sdnldSwMnWKClbmGeMmaEsWUymFW9xzwO6oqpLQsSSKZq6YlkIyhhj\nTp8li8lUuhjmnj2oK2qebYJkjJnhLFlMtpVXwWtPQVdT36FqW8VtjJnhLFlMthVXgSZh/yN9h2wV\ntzFmprNkMdnmn+fKf7x8atyiMpJPnk+sZWGMmbEsWUw2nw+WXwkHfgOJmHdImGtrLYwxM5gli0xY\neTVE2+C1J/oOVZeEONzUlcWgjDFm4ixZZMIZmyAvBPtOzYp6w/Iqdh5uZt+J9qyFZYwxE2XJIhOC\nYVj6Jtj3YN9Wq++/eAnhoJ9vPvZKloMzxpjxs2SRKSs3Q8thqH8JgNJwkJs2LuaB545xZKZ3Rx34\nDXTUZTsKY8wUsmSRKSs2u+e01dwfeMMZ+ATu+t3BLAU1CRpfge+/Ex75bLYjMcZMIUsWmVI8H6rX\n9VvNPa8kxLvOX8i9O45Q3z5D9+TescU97/kviHVmNxZjzJSxZJFJK6+CI89Ax6m9uT/0pmXEkym2\n/OHVLAY2QbEu+OP3oWI5xDth7y+yHZExZopYssikFZsBdTvoeZZWRrjqnGq+/+Rh2nri2YttIl78\nqdvg6W1fhrIaeO5H2Y7IGDNFLFlkUvW5UDS/32pugL940zLaown+88nDWQpsAlRh+91QtRpqXg/n\n3ggHH4PW2mxHZoyZApYsMknEzYo68FuIn1q9ffaCEt60oop//8Or9MSTWQxwHI7uhOPPwQV/7n6v\nte8FFJ6/N9uRGWOmgCWLTFtxlevfP/T7fof/ctMyGjpi3LvjSJYCG6dn7oZgIZx7g3tfvhQWX+y6\notQ2dTIm11myyLSlb4RAeFBX1Mal5axfUsa3HjtIPJnKUnBj1NkIu3/mEkV+0anj594ADS/DsWez\nF5sxZkpkNFmIyGYR2SciB0TkU0Ocv0VE6kVkl/f4QNq594vIfu/x/kzGmVGBEJxxqSv9kfZf4CLC\nX25axtGWbn7x3LEsBjgGf/weJGNwwQf6H19znStrsssGuo3JdRlLFiLiB+4ErgLOAm4UkbOGuPTH\nqrrOe3zbu7cc+CzwOmAj8FkRKctUrBm3cjO01cLJF/sdvmzVHFbNK+Ib216Zvvtzp5JubUXNG2DO\n6v7nQiWw6q3w4n19FXaNMbkpky2LjcABVT2oqjHgHuDaMd77FuARVW1S1WbgEWBzhuLMvBWbAelX\nWBBc6+IvNi1jf10HW/eezE5so9n/CLS85ga2h3LuTdDdDPsfmtq4jDFTKpPJYgGQPnpb6x0b6F0i\n8ryI3Ccii8Zzr4jcKiI7RGRHfX39wNPTR+EcWLQR/vAV+O3nobul79Rbz6lmcXmYr297BZ2OA8Xb\nvw1F1bDqbUOfP2OT2+zpuXumMipjzBTLZLKQIY4N/Gv4C6BGVdcCW4HvjuNeVPUuVd2gqhuqqqpO\nK9iMe8e34MzL4Xf/D76yFh77Z+hpI8/v49Y3nsGuIy08ebAx21H213QQDmyF9beAPzD0Nf48WHs9\nvPyQGwg3xuSkTCaLWmBR2vuFQL+RXFVtVNXeIkl3A+vHeu+MU74Urv8efPj3blHbo1+Ar66Fx7/E\nu88ppaoon29sG6F8eSrluoN6Wqcu5u3fAZ8fzh9lfsG5N0Iq7lZ4G2NykmSq60NE8oCXgcuBo8B2\n4CZV3Z12TbWqHvdevwP4pKpe6A1w7wTO9y59Flivqk3D/bwNGzbojh07MvK7ZMTRZ+HRf4QDj0C4\nkieq/wd/tvscPvbmFXxwjeJv3O+mpfY9DkCiG3wB10JZ8w63I1+oODPxxbvhi6tcN9P13x3tavjm\n68GXB7duy0w8xpiMEJGdqrphtOvyMhWAqiZE5CPAQ4Af2KKqu0XkdmCHqj4AfFRErgESQBNwi3dv\nk4j8Ay7BANw+UqKYkRacDzff5woNPvoFLn7lS+wqCBF8PIrv970JXKB0MVSugJo3QuWZrkT47p+7\narb+fDjzzV7i2Nx/DcTp6q0DNXC67HDOvQkeug3qXoI5qyYvDmPMtJCxlsVUm3Eti4EO/QF9/l72\ndkbY8lKAg1rNjVddyrsvXIHIgCGcVAqO7oDd97vE0X7MJY7lV8BZ13mDzqc5hnPXJlei5C+fdOU9\nRtNR51oiF/9PuOJzp/ezjTFTZqwtC0sW09DRlm4+fu9zPHmwkTevnsM/vXMtVUX5Q1+cSkHtM/Di\nz9weEx0n3PE5Z7m1EUvfCDWXQME4lqnU7oRvXwZX/wts/ODY7/vhe+H48/C/XnRjHcaYac+SxQyX\nSilb/vAq//zQPory8/jHd57DW9bMG+WmpBsLOfQ7ePVxeO0pN86BQPXaU8lj3lo33dU3zPyG+/8C\n9j4Af7MsHtiXAAAVE0lEQVR3fGMiu++Hn9wC77sfll029vuMMVljySJHvHyynf/1413sPtbGu9cv\n5LNvP4ui0DDTWAdKRF212Fcfh1d/51ogSW+ltT8IJQvdmEjpYijxngur4Ic3wPnvg7d+cXzBxnvg\niyvcIsR33jW+e40xWWHJIofEEim+9pv9fH3bASL5eVy/YRHvu3AJNZWR8X1QvBtqt7vZVS2veY8j\n7rmzrv+1f/EkzB2qOssofvExeP7H8PGXJ3fAPRO6W1wSXXIJRCqyHY0xWWHJIge9UNvKXY8f5Fcv\nHCeRUjatrOL9F9XwphVV+HxjGIQeSbzbbWTUcthNzz3jTRP7nCPPwHeugGu/Duf9yeDzqZSbZRVt\ncxtD5QVPL+6JiHXBM9+C33/FxeLPdzPKLvgALNwwtgF9Y3KEJYscVtfWww+feY0fPP0a9e1RllSE\ned+FS3jP+kWUhMfYRZUpqvCv690f3IUbobsJuppOPfe0gHol2fNCMP98Vwpl0evcc6Qyc7ElYvDs\nd90q+o6TsPxKlyAObHWVc2Ptbjzngg/AOe+G4DhbbsbMQJYsZoFYIsVDu0/wvScPsf1QM6GAj6vP\nrmZxRZiKSJDySD7lkWDfoywcIM8/BVuYPHM3bP2cq0obLoOCcgiX93/OL3RrMo487XbgS3n7kZef\ncSpxlC52XUVdTa5Y4cDEE+uAqpUuKS3aCNXrXEn4gVJJ1zW27Z9cl9vii+Hyz8CSi05dE22HF34C\nz3wb6nZDfolrGW34c7e+xZgcZcliltl9rJX/fPIwv959gpau+LDXlYYDlEeCVESCVETyqSj0Xhe6\nxFJRGGRBaQGLy8OD13dkSrwbju1yiaN2u5vF1dUw+Lr8YjcFuDfhBApc2ffmQ+68L+BmffUmj0Ub\n3eyw334eGva5PdEv/wwsu3z4riZV9/O3f9tNRU7F3Syyc2+Es66Z/uMwxoyTJYtZLJ5M0dwZo6kr\nRlNHjMbOGM1dMRo7YjR1ukdDR5SmzlPnBv7fYHF5mMtWzeHy1XPYuLSc/LwpXDeh6ooYdtSdSgwF\npcMXM+yoc0nmyDPuceyP3pRhT+UKuPT/wlnXjm88oqMOnv0e7PohNL0CeQWw+u1uh8AzNtlaEpMT\nLFmYMUumlJYulzgaO2IcqGvn0X31/OFAA9FEikjQz+uXV3L5qrlsWlXFnKIhunqmk2QcTrwAtTtc\nS2TNO1x13IlSdcnouR95ZVBaXdn2tde7FsfATaGMmUEsWZjT1h1L8uTBBn6zt47fvlTH8dYeANYu\nLKEiEiSWTBFLpIgmBj4nieTn8fozK9m0cg4XL6sgkp+xMmRTK97j6nI9d48rAplKQNVqKF3kjclU\nuHGacEXa+wo3/pJfmO3ojRnEkoWZVKrK3uPtPLqvjsderqc7liQ/z0fQe7jXfoJ+H/kBH3VtUZ54\npYGuWJKg38fGpeVsWlnFppVzWFYVmbrxkEzqqHdbyh7YCp0NpwbfYx1DXCxQcSbMO8d7rHXjK4Vz\npjxsY9JZsjBZF00k2XGomW376ti2r579de6P6MKyAt60ooqaiggl4QClBQHKIkFKCwLe+yDBvCmY\ntZUpiahLHF2NLnl0NkDDfjjxvOseazl86trCuS55lC2FRI9LNNEOiHW6qbx9rzvdTK+CsuEfwYib\nlpxKuud+r73nQNhNFAgVu8H6vkeJe87GuheTVZYszLRT29zFtn31bNtX39fqGE446GdecYjV84tZ\nM7+Ys6qLWTO/ZPiCijNJd4ubxXXiBfc4/jy0vgaBiPuDn18IQe+RX+iOBQvdrLHu5lPTiLub3WdF\n2yYvtqJqWH0NrLkOFl04fP0wkzMsWZhpTVVpjyZo7YrT0hWnpTvmPcdp7XKvjzR3sed4G0eaTs1s\nqirK70seq6qLKQ7lEfS7rrCA3z2Ceb6+Y+F8P0X5ebnR7TWcZNwNusc6QPwgPjdTS/zes3jHxSWc\naLtLMD1t3uveR6ubwrz/EUhG0xLHO9zaF0scOcmShckZrd1x9h5vY/exNvYca2P3sVYO1HWQSI3t\n/7sBv1AWTluc6K0zKQsHqSwMsqCsgEVlYRaWhSkI2nRYou1uT/Xd9/dPHGdd66YOhytdQhIBxEtG\naa9DpW6qs5kRLFmYnBZNJHm1oZPOaJK4NysrnnSPaCJFPKnEEik6owm3lqR33Yn3urEzRmv34MWL\nVUX5LCorYFF5mEVlYRaVF7B8bhFnVRcTCszCRDJU4hiLcIUb0C9fBhXL3OuKZW6FvpVRmVYsWRgz\nikQyRVNnjCPN3dQ2d3GkqYvXmro40tTNkeYujrf2kPRaL3k+YXV1MWsXlnDuolLWLSplWVUh/tMt\n4DiTRNvh0B8g3nWqvpeq91pPDap3NUHjAbcFcNMr0H68/+cUL3Cbc81dA3PPds+Vy4dfdGkyypKF\nMacpnkxxvKWHvSfaeO5IC8/VtvD8kVbaowkAIkE/5ywsYXV1MQUBP3l+HwGfuGe/kJf22k0v9pPf\n+xzw9U0zzs/zE8n3U1WYP+GxlRav1eQTwe8TRMDvE/wi+LznQJ6Pwmysd4l2uBX5jQdc8mjYDyf3\nQP1Lp2qC+YOuzldv8igod6vw4z1pz94j3uPuCxR4EwEibpZX70SAoPe6aL5ryQxVL8z0sWRhTAak\nUsrBhk6er23huSMt7Kpt5cDJdqKJ1JjHUIYTCvi8rq8wi8tPPS8uD7OwrIC2njiHG7t4rbGLw02d\n7nVTF4cbu4bsUhvKBTVl/Pnrz+CKs+Zmv1WUiEHjfji5280KO7nbPXq3Bu5HXHLIC7lnX54brI93\nDbOuJe2+0sWuG6xyef/n4gUzoxx9b+stfRp0+vRoTbkxpHD5hD7ekoUxU0xVSaSURFKJp1Ikkkoi\nmSKWdGMo0USSaNy9j8bdSveot+K9rTtBbbP74/9aUzdHmrro8FowQ/H7hAWlBSypcMlkSUWYOUUh\nFCWZckktqUoypaj33Nqd4Cc7j1Db3M3i8jB/dkkN79mwaPqtru9scAkgr8C1CvIKXBfVcH/YUynX\n+oh5iSPW6Z5ba10rpnG/9/wKxDtP3efLc3uZ+AOuZZOX9tofcOfmnQMrr3LbEQcKpub3V4X6fW6x\n54GtcPiJ0ceKFl4AH9g6oR9nycKYGUxVae6Ke2MoXRxp7qIoFGCJlxjmlxYQmEC5+UQyxcN7TvLt\nxw/y7GstFIfyuPF1i7nl4hqqS6boj2G2qLrxk94E0lrrph0nY95jwOtYp6svFu903VxnbHJbBq/Y\nDEVzJze2nlY4+JiXIH4DbbXueNVq93PD5d4MtN5p0b7+06QL57jZahNgycIYM6Kdh5vZ8vtX+dWL\nx/GJ8La11Vy2ei7tPW7tS+8MspauOM3ec2t3nKDfRyTfTyQ/j3DQTySYRyQ/j0i+n3Awj8XlYTat\nrGJJRQ7MekpE4dDjsO/XriZY6xF3fP75rsWx+CKXSPx5rkXiC3itEq+F4vO7pNO7gLKnZfDzyT2u\nPL8m3er6MzbBmW+GMy+HkoUZ/xUtWRhjxuRIUxf/8cQhfrz9SL+ur3DQT1k42LcHSmk4SHEoj3gy\nRWcsSVc0QWc0SWcsQVcsSWc04R7eyvyllRHetKKKS1fN4XVLy2f+1GNVN6by8q9c8ji6EziNv5/i\nc2tSShe7xHDmm1130hTPCrNkYYwZl/aeOLXN3X0JYqJ/3A81dLp6YC/X8+QrjUQTKUIBHxedUcGl\nq+bwhuVVLCgtGHf9r554kuOtPRxt7qa9J87ZC0pYWFaQvdX5HXUuefR2X6Xi3uve9wn3Ohh2tbt6\nFyv2PgeLpsWqeEsWxpis64knefJgI4/tq+fRfXUcbuzqO1dSEKCy0O3SWFWYT2VhkMrCfCoK84kl\nkhxt6fYeLkE0dAwe5J1bnM+GJeVsqCljw5JyVlcXTc3WwTnEkoUxZtp5taGTpw82UtcepaHDe7TH\naOiM0tAepa3nVDdYfp6PBaUFLCgrYEFpAfNLC/reh4N+nqttZcehJnYcauZoi6sfFg76OX9xGeuX\nlDGvJITgVSNB8P6HiCBAQdDPirmF1FREZnWCsWRhjJlxookkjR0xAn4flYXBMXcxHW/tZsehZnYc\namL7oWZeOtHGWJe9BPN8LJ9TyMp5RayaV8SqecWsmldEVdHgRZK906Nj3kZffr/M+EKVliyMMbNW\nZzRBe08CRd2aNtwfelVvjRtKe0+Cl0+289IJ73G8jbr2U11dZeEABQG/WxfjJYdYMjVov/rC/Dzm\nl4aYn9768V5Xl4QQYdgdJWOJFAVBP6uri5kzRHKaCmNNFtNsNY4xxpw+N5V39D9vZy8o6fe+qTPG\nSyfa2HeinZdPthNLaN9OkH07Q/pP7RCZSCpHW7o51tLNsdZunq9tpakzNqGYKyJBVlcXc9b8YlZX\nF7G6uphlVYUTWk+TCZYsjDHGUx4JcvGySi5eVjnhz+iKJTjW0sOxlm5OePvWp9cCC/r71wZr7XIl\n+Pceb2fP8Tb+44lDxBKuUGPQ72PFvELWLSplw5Jy1i8py9oMMOuGMsaYaSSRTHGwoZO9x93+LS8e\na+W5I619a2DmFuezfkkZ65eUs2FJGWfNLz6t1se06IYSkc3AVwE/8G1VvWOY694N/AS4QFV3iEgN\nsBfY513ylKp+OJOxGmPMdJDn97FibhEr5hZx7boFACRTyksn2th5uJkdh5rZebiZB19wBRdDAR9v\nXj2Xf7vp/MzGlakPFhE/cCdwBVALbBeRB1R1z4DrioCPAk8P+IhXVHVdpuIzxpiZwu8T1swvYc38\nEv7HRTWAmwHWmzwi+ZlfHZ/JlsVG4ICqHgQQkXuAa4E9A677B+CfgY9nMBZjjMkp1SUFvG1tAW9b\nO39Kfl4mh9kXAEfS3td6x/qIyHnAIlX97yHuXyoifxSRx0TkDUP9ABG5VUR2iMiO+vr6SQvcGGNM\nf5lMFkMN1/eNpouID/gy8LdDXHccWKyq5wF/A/xQRIoHfZjqXaq6QVU3VFVVTVLYxhhjBspksqgF\nFqW9XwgcS3tfBJwNbBORQ8CFwAMiskFVo6raCKCqO4FXgBUZjNUYY8wIMpkstgPLRWSpiASBG4AH\nek+qaquqVqpqjarWAE8B13izoaq8AXJE5AxgOXAwg7EaY4wZQcYGuFU1ISIfAR7CTZ3doqq7ReR2\nYIeqPjDC7W8EbheRBJAEPqyqTZmK1RhjzMhsUZ4xxsxiY12UNz2KjhhjjJnWLFkYY4wZVc50Q4lI\nPXD4ND6iEmiYpHAyyeKcXDMlTpg5sVqcky+TsS5R1VHXHuRMsjhdIrJjLP122WZxTq6ZEifMnFgt\nzsk3HWK1bihjjDGjsmRhjDFmVJYsTrkr2wGMkcU5uWZKnDBzYrU4J1/WY7UxC2OMMaOyloUxxphR\nWbIwxhgzqlmfLERks4jsE5EDIvKpbMczEhE5JCIviMguEZk2tU1EZIuI1InIi2nHykXkERHZ7z2X\nZTNGL6ah4vx7ETnqfae7ROTqbMboxbRIRB4Vkb0isltE/to7Pq2+0xHinI7faUhEnhGR57xYP+cd\nXyoiT3vf6Y+9oqfTMc7/EJFX077TKd9FdFaPWXiVbV8mbetX4MaBW79OF14p9w2qOq0WEonIG4EO\n4HuqerZ37J+BJlW9w0vCZar6yWkY598DHar6L9mMLZ2IVAPVqvqst+3wTuA64Bam0Xc6QpzXM/2+\nUwEiqtohIgHg98Bf4/bL+Zmq3iMi3wSeU9VvTMM4Pwz8t6rel63YZnvLom/rV1WNAb1bv5pxUNXf\nAQOrAl8LfNd7/V3cH5GsGibOaUdVj6vqs97rdmAvbpfJafWdjhDntKNOh/c24D0UuAzo/QM8Hb7T\n4eLMutmeLEbd+nWaUeBhEdkpIrdmO5hRzFXV4+D+qABzshzPSD4iIs973VRZ7y5LJyI1wHnA00zj\n73RAnDANv1MR8YvILqAOeAS3qVqLqia8S6bFv/+Bcapq73f6Be87/bKI5E91XLM9WYy49es0dImq\nng9cBfyV161iTs83gGXAOtx2vl/MbjiniEgh8FPgY6ralu14hjNEnNPyO1XVpKquw+3auRFYPdRl\nUxvVEAEMiFNEzgZuA1YBFwDlwJR3P872ZDHa1q/Tiqoe857rgPtx/4efrk56fdq9fdt1WY5nSKp6\n0vvHmQLuZpp8p15/9U+BH6jqz7zD0+47HSrO6fqd9lLVFmAbbivnUhHp3QRuWv37T4tzs9flp6oa\nBf6dLHynsz1ZjLj163QiIhFvEBERiQBXAi+OfFdWPQC833v9fuC/shjLsHr/+HrewTT4Tr1Bzu8A\ne1X1S2mnptV3Olyc0/Q7rRKRUu91AfBm3BjLo8C7vcumw3c6VJwvpf1HguDGVab8O53Vs6EAvGl9\nX+HU1q9fyHJIQxK3F/n93ts84IfTJVYR+RGwCVdG+STwWeDnwL3AYuA14D3Z3hp3mDg34bpLFDgE\nfKh3XCBbROT1wOPAC0DKO/x/cOMB0+Y7HSHOG5l+3+la3AC2H/cfyfeq6u3ev6t7cF07fwRu9v7r\nfbrF+VugCtd1vgu31XTH8J+Ugdhme7IwxhgzutneDWWMMWYMLFkYY4wZlSULY4wxo7JkYYwxZlSW\nLIwxxozKkoUx04CIbBKR/852HMYMx5KFMcaYUVmyMGYcRORmb7+BXSLyLa/oW4eIfFFEnhWR34hI\nlXftOhF5yiv+dn9vQT0ROVNEtnp7FjwrIsu8jy8UkftE5CUR+YG3WteYacGShTFjJCKrgffiCjqu\nA5LAnwAR4FmvyONjuJXhAN8DPqmqa3GrnHuP/wC4U1XPBS7GFdsDV7X1Y8BZwBnAJRn/pYwZo7zR\nLzHGeC4H1gPbvf/oL8AV80sBP/au+T7wMxEpAUpV9THv+HeBn3j1vRao6v0AqtoD4H3eM6pa673f\nBdTgNr8xJussWRgzdgJ8V1Vv63dQ5NMDrhuphs5IXUvpNYmS2L9PM41YN5QxY/cb4N0iMgf69sRe\ngvt31Fu59Cbg96raCjSLyBu84+8DHvP2e6gVkeu8z8gXkfCU/hbGTID9l4sxY6Sqe0Tk73C7FfqA\nOPBXQCewRkR2Aq24cQ1wJa+/6SWDg8CfesffB3xLRG73PuM9U/hrGDMhVnXWmNMkIh2qWpjtOIzJ\nJOuGMsYYMyprWRhjjBmVtSyMMcaMypKFMcaYUVmyMMYYMypLFsYYY0ZlycIYY8yo/j9hBiyrQ1bE\nfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x16859dd45c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(training.history['loss'])\n",
    "plt.plot(training.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predicting on the test set and printing the confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1761  551]\n",
      " [ 416 1896]]\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "from sklearn import metrics\n",
    "model = load_model('best_model.h5')\n",
    "\n",
    "y_pred_one_hot = model.predict(X_testRusReshaped)\n",
    "y_pred_labels = np.argmax(y_pred_one_hot, axis = 1)\n",
    "\n",
    "y_true_labels = np.argmax(Y_testRusHot,axis=1)\n",
    "\n",
    "confusion_matrix = metrics.confusion_matrix(y_true=y_true_labels, y_pred=y_pred_labels)\n",
    "print(confusion_matrix)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}