Befree_GAD/BeFree-3class.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation using the BeFree corpus \n",
    "\n",
    "### GAD dataset\n",
    "To obtain a large benchmark of Gene Disease Associations along with associated sentences from literature, we used the corpus generated by BeFree system based on Genetic Association Database (GAD)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  ------------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from keras.models import load_model\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "import nltk\n",
    "import xml.etree.ElementTree as ET\n",
    "import pandas as pd\n",
    "import os\n",
    "import string\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "from numpy.random import random_sample\n",
    "import re\n",
    "import pickle\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "from keras.layers import Embedding, Flatten,LSTM\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Dropout, Activation,  Input, merge,Conv1D,MaxPooling1D,GlobalMaxPooling1D,Convolution1D\n",
    "from keras import regularizers\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.layers import Concatenate, concatenate\n",
    "from keras import backend as K\n",
    "from keras.layers import multiply\n",
    "from keras.layers import merge\n",
    "from keras.layers.core import *\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.models import *\n",
    "random_seed=1337"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Define Callback functions to generate Measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def f1(y_true, y_pred):\n",
    "    def recall(y_true, y_pred):\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(y_true, y_pred):\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "    precision = precision(y_true, y_pred)\n",
    "    recall = recall(y_true, y_pred)\n",
    "    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments to reproduce the results of Table 7 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation results for multi-class classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Prerocssed Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/pickles/befree_3class_crawl-300d-2M.pickle', 'rb') as handle:\n",
    "    gene_id_list = pickle.load(handle)\n",
    "    gene_symbol_list = pickle.load(handle)\n",
    "    disease_id_list = pickle.load(handle)    \n",
    "    X_train = pickle.load(handle)\n",
    "    distance1_vectors = pickle.load(handle)\n",
    "    distance2_vectors = pickle.load(handle)\n",
    "    Y_train = pickle.load(handle)\n",
    "    word_list = pickle.load(handle)\n",
    "    word_vectors = pickle.load(handle)\n",
    "    word_dict = pickle.load(handle)\n",
    "    distance1_dict = pickle.load(handle)\n",
    "    distance2_dict = pickle.load(handle)\n",
    "    label_dict = pickle.load(handle)\n",
    "    MAX_SEQUENCE_LENGTH = pickle.load(handle)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Position Embedding Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Pos-Embd (TrigPosEmbedding)  (None, None, 20)          0         \n",
      "=================================================================\n",
      "Total params: 0\n",
      "Trainable params: 0\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(5330, 81, 20)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import keras\n",
    "from keras_pos_embd import TrigPosEmbedding\n",
    "\n",
    "model = keras.models.Sequential()\n",
    "model.add(TrigPosEmbedding(\n",
    "    input_shape=(None,),\n",
    "    output_dim=20,                      # The dimension of embeddings.\n",
    "    mode=TrigPosEmbedding.MODE_EXPAND,  # Use `expand` mode\n",
    "    name='Pos-Embd',\n",
    "))\n",
    "model.compile('adam', keras.losses.mae, {})\n",
    "model.summary()\n",
    "\n",
    "d1_train_embedded=model.predict(distance1_vectors)\n",
    "\n",
    "d1_train_embedded.shape\n",
    "\n",
    "d2_train_embedded=model.predict(distance2_vectors)\n",
    "\n",
    "d2_train_embedded.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Word Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIM=word_vectors.shape[1]\n",
    "embedding_matrix=word_vectors\n",
    "\n",
    "def create_embedding_layer(l2_reg=0.01,use_pretrained=True,is_trainable=False):\n",
    "    \n",
    "    if use_pretrained:        \n",
    "        return Embedding(len(word_dict) ,EMBEDDING_DIM,weights=[embedding_matrix],input_length=MAX_SEQUENCE_LENGTH,trainable=is_trainable,embeddings_regularizer=regularizers.l2(l2_reg))\n",
    "    \n",
    "    else:    \n",
    "        return Embedding(len(word_dict) ,EMBEDDING_DIM,input_length=MAX_SEQUENCE_LENGTH)\n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Attention Mechanism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIM = 2\n",
    "TIME_STEPS = MAX_SEQUENCE_LENGTH\n",
    "def attentionNew(inputs):\n",
    "    inputs = Lambda(lambda x: tf.keras.backend.tanh(x))(inputs)\n",
    "    input_dim = int(inputs.shape[2])\n",
    "    a = Permute((2, 1))(inputs)\n",
    "    a = Dense(TIME_STEPS, activation='softmax')(a)\n",
    "    a_probs = Permute((2, 1))(a)\n",
    "    output_attention_mul = multiply([inputs, a_probs])\n",
    "    output_attention_mul = Lambda(lambda x: tf.keras.backend.tanh(x))(output_attention_mul)\n",
    "    return output_attention_mul\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set parameter for metric calculation, 'macro' for multiclass classification\n",
    "param='macro' \n",
    "def build_model():    \n",
    "    tf.set_random_seed(1337)\n",
    "    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedding_layer=create_embedding_layer(use_pretrained=True,is_trainable=False)\n",
    "    embedded_sequences = embedding_layer(sequence_input)\n",
    "\n",
    "    \n",
    "    pos_embedd_1=Input(shape=(MAX_SEQUENCE_LENGTH,20), dtype='float32')\n",
    "    pos_embedd_2=Input(shape=(MAX_SEQUENCE_LENGTH,20), dtype='float32')\n",
    "    \n",
    "    embedded_sequences = concatenate([embedded_sequences,pos_embedd_1,pos_embedd_2])\n",
    "        \n",
    "    \n",
    "    x = Conv1D(64, 5, activation='relu')(embedded_sequences)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    conv_sequence_w5=GlobalMaxPooling1D()(x)    #x = Flatten()(x)\n",
    "\n",
    "\n",
    "        \n",
    "    x = Conv1D(64, 4, activation='relu')(embedded_sequences)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    conv_sequence_w4=GlobalMaxPooling1D()(x)    #x = Flatten()(x)\n",
    "\n",
    "\n",
    "            \n",
    "    x = Conv1D(64, 3, activation='relu')(embedded_sequences)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    conv_sequence_w3=GlobalMaxPooling1D()(x)    #x = Flatten()(x)\n",
    "\n",
    "    forward = LSTM(100, recurrent_dropout=0.05,return_sequences=True)(embedded_sequences)\n",
    "    backward = LSTM(100, go_backwards=True,recurrent_dropout=0.05,return_sequences=True)(embedded_sequences)\n",
    "    attention_forward=attentionNew(forward)\n",
    "    attention_backward=attentionNew(backward)\n",
    "    lstm_sequence = concatenate([attention_forward,attention_backward])\n",
    "   \n",
    "    lstm_sequence = Flatten()(lstm_sequence)\n",
    "    merge = concatenate([conv_sequence_w5,conv_sequence_w4,conv_sequence_w3,lstm_sequence])\n",
    "    x = Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.1))(merge)\n",
    "    x = Dropout(0.1)(x)\n",
    "    preds = Dense(3, activation='softmax')(x)\n",
    "    model = Model(inputs=[sequence_input, pos_embedd_1,pos_embedd_2],outputs=preds)\n",
    "    opt=keras.optimizers.Adam(lr=0.0001)\n",
    "    model.compile(loss='binary_crossentropy',optimizer=opt,metrics=['acc',f1])\n",
    "    return model\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the Evaluation using 10 fold Cross Validation  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_evaluate_model_lstm_position():\n",
    "    prec_list=[]; reca_list=[]; fscore_list=[] ; fold=0\n",
    "    skf = StratifiedKFold(n_splits=10, random_state=random_seed)\n",
    "    Y = [np.argmax(y, axis=None, out=None) for y in Y_train]\n",
    "    all_histories=[]\n",
    "    for train_index, test_index in skf.split(X_train,Y):     \n",
    "        fold+=1\n",
    "        x_train, X_test = X_train[train_index], X_train[test_index] \n",
    "        pos_train1, pos_test1 = d1_train_embedded[train_index], d1_train_embedded[test_index] \n",
    "        pos_train2, pos_test2 = d2_train_embedded[train_index], d2_train_embedded[test_index] \n",
    "        y_train, y_test = Y_train[train_index], Y_train[test_index]\n",
    "        model = None # Clearing the NN.\n",
    "        model = build_model()\n",
    "        earlystopper = EarlyStopping(monitor='val_loss', patience=3, verbose=1)        \n",
    "        history=model.fit([x_train,pos_train1,pos_train2], y_train, validation_data=([X_test,pos_test1,pos_test2],y_test) ,epochs=50, batch_size=50,verbose=1,callbacks=[TQDMNotebookCallback(),earlystopper])\n",
    "        all_histories.append(history)\n",
    "        predicted = np.argmax(model.predict([X_test,pos_test1,pos_test2]), axis=1)\n",
    "        y_test_to_label = np.argmax(y_test, axis=1)\n",
    "        prec, reca, fscore, sup = precision_recall_fscore_support(y_test_to_label, predicted, average=param)\n",
    "        print(\"Fold {:d}: Precision:{:.2f}% Recall:{:.2f}% Fscore:{:.2f}% \".format(fold,prec*100, reca*100, fscore*100))\n",
    "        prec_list.append(prec), reca_list.append(reca) , fscore_list.append(fscore)\n",
    "    precission=sum(prec_list)/len(prec_list)*100 \n",
    "    recall=sum(reca_list)/len(reca_list)*100 \n",
    "    fscore=sum(fscore_list)/len(fscore_list)*100    \n",
    "    print(\"Final: Precision:{:.2f}% Recall:{:.2f}% Fscore:{:.2f}% \".format(precission, recall, fscore))\n",
    "    return all_histories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 4796 samples, validate on 534 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a43fefb7fd1466fbc10c719519994c2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "622731b50e8b421b8e5099a2140bcfef"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 29s 6ms/step - loss: 2788.2614 - acc: 0.6719 - f1: 0.1560 - val_loss: 2780.1476 - val_acc: 0.6841 - val_f1: 0.1336\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eeeb78ca062147d2b77d8563c97f6c35"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2777.5695 - acc: 0.6937 - f1: 0.2785 - val_loss: 2776.1213 - val_acc: 0.6910 - val_f1: 0.1455\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0d33d10f18e4fa781cbc22e2624b686"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.7472 - acc: 0.7134 - f1: 0.4000 - val_loss: 2775.5687 - val_acc: 0.7029 - val_f1: 0.2816\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14645bc6ff9946f2962864e532301172"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.5088 - acc: 0.7270 - f1: 0.4550 - val_loss: 2775.4846 - val_acc: 0.7341 - val_f1: 0.4532\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69fa674fbc854606b45c265dbb3c6b4c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4662 - acc: 0.7454 - f1: 0.4998 - val_loss: 2775.4611 - val_acc: 0.7484 - val_f1: 0.5141\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99335765ae49479ab2063bd31f67ee6f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4382 - acc: 0.7640 - f1: 0.5460 - val_loss: 2775.4404 - val_acc: 0.7584 - val_f1: 0.5120\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d361f40ab6284572adb3443b68302387"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4150 - acc: 0.7764 - f1: 0.5876 - val_loss: 2775.4409 - val_acc: 0.7765 - val_f1: 0.5600\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "676bc150ccf048cbb367d5ecf335cdfe"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3931 - acc: 0.7947 - f1: 0.6321 - val_loss: 2775.4163 - val_acc: 0.7928 - val_f1: 0.6281\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bed1fbe72fd4b69858e81a675a668eb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3719 - acc: 0.8146 - f1: 0.6806 - val_loss: 2775.4018 - val_acc: 0.7953 - val_f1: 0.6416\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed9bf4bf1b2845b1808e948807e55e92"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3594 - acc: 0.8208 - f1: 0.6977 - val_loss: 2775.3884 - val_acc: 0.8077 - val_f1: 0.6719\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2206796ef7e49199799e9da7300e8a2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3450 - acc: 0.8324 - f1: 0.7225 - val_loss: 2775.3888 - val_acc: 0.8021 - val_f1: 0.6663\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "550556506b944213bdcbe9eee74fe6e8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3279 - acc: 0.8446 - f1: 0.7443 - val_loss: 2775.3815 - val_acc: 0.8115 - val_f1: 0.6832\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ddc86ddd8ef44c3c8f4c683ece3fcc40"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3185 - acc: 0.8533 - f1: 0.7618 - val_loss: 2775.3766 - val_acc: 0.8190 - val_f1: 0.7000\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "353b93b153634583b0cd3b8364ae3741"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3068 - acc: 0.8599 - f1: 0.7748 - val_loss: 2775.3719 - val_acc: 0.8208 - val_f1: 0.7039\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f5b1f091438414aa7996fe3cd1dbfb1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2978 - acc: 0.8669 - f1: 0.7873 - val_loss: 2775.3699 - val_acc: 0.8071 - val_f1: 0.6822\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c0006cb6b784664bdbf68f65bc125c9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2870 - acc: 0.8731 - f1: 0.7979 - val_loss: 2775.3714 - val_acc: 0.8115 - val_f1: 0.6963\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdf11fa7602247ec89ca483ed1a5420d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2763 - acc: 0.8810 - f1: 0.8117 - val_loss: 2775.3632 - val_acc: 0.8315 - val_f1: 0.7270\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7feca9756a7435a95acdb6d7c00ff66"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2669 - acc: 0.8889 - f1: 0.8244 - val_loss: 2775.3706 - val_acc: 0.8196 - val_f1: 0.7159\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f9219e0cf414fb8970bbf24d41fe8f9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2586 - acc: 0.8957 - f1: 0.8356 - val_loss: 2775.3697 - val_acc: 0.8202 - val_f1: 0.7167\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0876c61ae77841d59c22c9ba9190ace8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2514 - acc: 0.8963 - f1: 0.8377 - val_loss: 2775.3671 - val_acc: 0.8265 - val_f1: 0.7250\n",
      "Epoch 00020: early stopping\n",
      "Fold 1: Precision:73.08% Recall:76.96% Fscore:74.08% \n",
      "Train on 4796 samples, validate on 534 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43c1b4f1e3184c41b2edb8fb6d509495"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "529799f757d84fc29efcc6bedeb75657"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 29s 6ms/step - loss: 2788.2390 - acc: 0.6703 - f1: 0.1326 - val_loss: 2780.1317 - val_acc: 0.6704 - val_f1: 0.0288\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebe79a82c4394294819d48e6fccc38e9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2777.5629 - acc: 0.6892 - f1: 0.2528 - val_loss: 2776.1157 - val_acc: 0.7154 - val_f1: 0.4233\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fc030fa8e8f4152bb709dcf9b782951"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.7436 - acc: 0.7118 - f1: 0.3816 - val_loss: 2775.5552 - val_acc: 0.7310 - val_f1: 0.4122\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fce259dd0d04f5cbcc3c57fa5a7e2d5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.5067 - acc: 0.7243 - f1: 0.4422 - val_loss: 2775.4944 - val_acc: 0.7310 - val_f1: 0.3845\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ce67a045bf94b2ba446180621616e7c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4662 - acc: 0.7403 - f1: 0.4797 - val_loss: 2775.4785 - val_acc: 0.7397 - val_f1: 0.5241\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c5489542be54e9da2a082af4f09488f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 25s 5ms/step - loss: 2775.4418 - acc: 0.7536 - f1: 0.5289 - val_loss: 2775.4559 - val_acc: 0.7491 - val_f1: 0.4622\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0684794f78c4affa2e990bafd987531"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 25s 5ms/step - loss: 2775.4152 - acc: 0.7785 - f1: 0.5836 - val_loss: 2775.4391 - val_acc: 0.7672 - val_f1: 0.5426\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62cc6c77c3d2456a85bae5cca9dfe7b0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3880 - acc: 0.7987 - f1: 0.6378 - val_loss: 2775.4198 - val_acc: 0.7728 - val_f1: 0.5840\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "864dd2adb8c448b797f7e7105cecf3bf"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3686 - acc: 0.8156 - f1: 0.6810 - val_loss: 2775.4244 - val_acc: 0.7871 - val_f1: 0.6332\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9a125c2970647659fe6b5c0c9786cf6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3548 - acc: 0.8260 - f1: 0.7073 - val_loss: 2775.3961 - val_acc: 0.7871 - val_f1: 0.6378\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37a682606c7b4750bfc5006304b4d859"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3362 - acc: 0.8430 - f1: 0.7412 - val_loss: 2775.3927 - val_acc: 0.7953 - val_f1: 0.6565\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd4a2e11236c4ac392b5d714288c87f4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3260 - acc: 0.8537 - f1: 0.7614 - val_loss: 2775.3849 - val_acc: 0.7953 - val_f1: 0.6640\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5aacf8a34b2a45d9826e6f581d09eb86"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3151 - acc: 0.8562 - f1: 0.7678 - val_loss: 2775.3794 - val_acc: 0.8015 - val_f1: 0.6748\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58547d7cdf2f4597a8ec610e6a1e60c3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 25s 5ms/step - loss: 2775.3019 - acc: 0.8696 - f1: 0.7907 - val_loss: 2775.3770 - val_acc: 0.8002 - val_f1: 0.6761\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce6f52ea62f54c04b4de73f70a235f47"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2895 - acc: 0.8780 - f1: 0.8062 - val_loss: 2775.3838 - val_acc: 0.8102 - val_f1: 0.6882\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f3d037f5b9b46c29445f9beabd8f5c8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2823 - acc: 0.8792 - f1: 0.8087 - val_loss: 2775.3756 - val_acc: 0.8015 - val_f1: 0.6791\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "488a36a9b787455ab6ec446fafc7a0e8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2725 - acc: 0.8837 - f1: 0.8157 - val_loss: 2775.3747 - val_acc: 0.8115 - val_f1: 0.6991\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b8e82f1380147828da59f687386b6e6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2591 - acc: 0.8953 - f1: 0.8353 - val_loss: 2775.3750 - val_acc: 0.8090 - val_f1: 0.6953\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0eb560d305844e2ae393d47439d8fc1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2497 - acc: 0.9019 - f1: 0.8456 - val_loss: 2775.3721 - val_acc: 0.8215 - val_f1: 0.7157\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2edd754c4f294f35b0f0d1a53bdca7a3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2411 - acc: 0.9090 - f1: 0.8575 - val_loss: 2775.3840 - val_acc: 0.8090 - val_f1: 0.6910\n",
      "Epoch 21/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "870ec28a88ba49e0afb3e7ca1da0cad3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2344 - acc: 0.9111 - f1: 0.8612 - val_loss: 2775.3686 - val_acc: 0.8196 - val_f1: 0.7169\n",
      "Epoch 22/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36b93776f9ed41488335f0044b186fe6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2235 - acc: 0.9188 - f1: 0.8735 - val_loss: 2775.3791 - val_acc: 0.8065 - val_f1: 0.6943\n",
      "Epoch 23/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f46a92f449749c48e415bef90158481"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2174 - acc: 0.9208 - f1: 0.8767 - val_loss: 2775.3630 - val_acc: 0.8202 - val_f1: 0.7177\n",
      "Epoch 24/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8be0b63993444165b25a1fb3b2b5bd9e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2081 - acc: 0.9279 - f1: 0.8879 - val_loss: 2775.3652 - val_acc: 0.8221 - val_f1: 0.7217\n",
      "Epoch 25/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a384423a80ba4bff94abbc39f9774d55"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2004 - acc: 0.9327 - f1: 0.8954 - val_loss: 2775.3719 - val_acc: 0.8190 - val_f1: 0.7117\n",
      "Epoch 26/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6bf4652ed7f48b2bf7233040bf9bb64"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.1943 - acc: 0.9361 - f1: 0.9009 - val_loss: 2775.3651 - val_acc: 0.8196 - val_f1: 0.7138\n",
      "Epoch 00026: early stopping\n",
      "Fold 2: Precision:73.94% Recall:72.32% Fscore:73.05% \n",
      "Train on 4796 samples, validate on 534 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb9096256a3044189eb27aa0168d51a4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86399f2ea7874d5c95a6476921d7adf3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 30s 6ms/step - loss: 2788.2404 - acc: 0.6699 - f1: 0.1895 - val_loss: 2780.1379 - val_acc: 0.6710 - val_f1: 0.0255\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "413a8a06b1e24fa392efa89f7f695f90"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2777.5649 - acc: 0.6979 - f1: 0.3194 - val_loss: 2776.1221 - val_acc: 0.6998 - val_f1: 0.2642\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04e2e390de534fcca1730ef06c1cb721"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.7433 - acc: 0.7160 - f1: 0.4057 - val_loss: 2775.5589 - val_acc: 0.7154 - val_f1: 0.4038\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bafbc187e0e74bc296a061b53cec97b6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.5006 - acc: 0.7280 - f1: 0.4458 - val_loss: 2775.4940 - val_acc: 0.7166 - val_f1: 0.4024\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3050312284443a0847b332ba94be977"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4617 - acc: 0.7427 - f1: 0.4931 - val_loss: 2775.4753 - val_acc: 0.7253 - val_f1: 0.4523\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95615f63057b4922905d17acc9194b43"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4370 - acc: 0.7617 - f1: 0.5389 - val_loss: 2775.4511 - val_acc: 0.7447 - val_f1: 0.4938\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ecd06529c794ae2be8f357f0a9366d4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4111 - acc: 0.7801 - f1: 0.5895 - val_loss: 2775.4379 - val_acc: 0.7672 - val_f1: 0.5578\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efb6f9a8c6da4ab1826d4184b9877244"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3842 - acc: 0.8024 - f1: 0.6472 - val_loss: 2775.4194 - val_acc: 0.7809 - val_f1: 0.6011\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34effd38136349b4b1d5172d60cd4961"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3665 - acc: 0.8173 - f1: 0.6868 - val_loss: 2775.4037 - val_acc: 0.7915 - val_f1: 0.6344\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6e7f0b5daa94929a611036df41159fb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3453 - acc: 0.8337 - f1: 0.7220 - val_loss: 2775.4038 - val_acc: 0.7803 - val_f1: 0.6251\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d265cfcc5bbc4b2383b7bdcf9d6796ec"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3333 - acc: 0.8418 - f1: 0.7397 - val_loss: 2775.3883 - val_acc: 0.7984 - val_f1: 0.6584\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b0418f05c4c4482b4b0738f1032a6b1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3195 - acc: 0.8519 - f1: 0.7591 - val_loss: 2775.3869 - val_acc: 0.8134 - val_f1: 0.6899\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "edb3758f7ec2478c80cf6650e0f53ce1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3043 - acc: 0.8610 - f1: 0.7763 - val_loss: 2775.3849 - val_acc: 0.8084 - val_f1: 0.6835\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d13b506247c644d08147a5769cd8795f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2966 - acc: 0.8662 - f1: 0.7856 - val_loss: 2775.3826 - val_acc: 0.8121 - val_f1: 0.6951\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fed93b14a76b491f837e027f20b6b73d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2871 - acc: 0.8743 - f1: 0.8000 - val_loss: 2775.3734 - val_acc: 0.8121 - val_f1: 0.6945\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6c8c34af2364685a0e0e5651cc71514"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2747 - acc: 0.8841 - f1: 0.8163 - val_loss: 2775.3717 - val_acc: 0.8134 - val_f1: 0.6979\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33c82301472445199017ca2f31de9e40"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2627 - acc: 0.8958 - f1: 0.8358 - val_loss: 2775.3708 - val_acc: 0.8134 - val_f1: 0.6995\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41333e1c3aea4ea997a81dbf8934f6d9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2521 - acc: 0.9010 - f1: 0.8444 - val_loss: 2775.3735 - val_acc: 0.8134 - val_f1: 0.7019\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bac966fb5b02487480a858eb7d4789b4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2460 - acc: 0.9028 - f1: 0.8473 - val_loss: 2775.3671 - val_acc: 0.8146 - val_f1: 0.7062\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c08c1df986ac4e1098369bc750b58145"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2345 - acc: 0.9130 - f1: 0.8638 - val_loss: 2775.3681 - val_acc: 0.8271 - val_f1: 0.7282\n",
      "Epoch 21/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33f4cdc1c84149bea4731b5a815bd8fd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2274 - acc: 0.9127 - f1: 0.8636 - val_loss: 2775.3682 - val_acc: 0.8096 - val_f1: 0.6998\n",
      "Epoch 22/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef53b3efdc6e41d88aa15f61e643e11b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2194 - acc: 0.9220 - f1: 0.8788 - val_loss: 2775.3649 - val_acc: 0.8177 - val_f1: 0.7145\n",
      "Epoch 23/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e2eef4ffef14abdbe6876fd677bb850"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2121 - acc: 0.9233 - f1: 0.8807 - val_loss: 2775.3655 - val_acc: 0.8221 - val_f1: 0.7201\n",
      "Epoch 24/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ddd2d7753a1d4cee9b02c0e3f8aabd8f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2022 - acc: 0.9316 - f1: 0.8938 - val_loss: 2775.3685 - val_acc: 0.8221 - val_f1: 0.7201\n",
      "Epoch 25/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64ba651da5bf4754b42d78542b405b01"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.1968 - acc: 0.9339 - f1: 0.8978 - val_loss: 2775.3670 - val_acc: 0.8290 - val_f1: 0.7338\n",
      "Epoch 00025: early stopping\n",
      "Fold 3: Precision:74.29% Recall:75.00% Fscore:74.44% \n",
      "Train on 4796 samples, validate on 534 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6a9b0bc9e134477893e51f24ea93c16"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ab11acc0b9c4864907e663cbc92b312"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 30s 6ms/step - loss: 2788.2324 - acc: 0.6701 - f1: 0.1458 - val_loss: 2780.1326 - val_acc: 0.6679 - val_f1: 0.4700\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa186f54118d49fa911fb5cca7c39e57"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2777.5623 - acc: 0.6938 - f1: 0.2952 - val_loss: 2776.1166 - val_acc: 0.7160 - val_f1: 0.2946\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e715ff19c28a4b08b4da8c184c6e7318"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.7414 - acc: 0.7172 - f1: 0.3934 - val_loss: 2775.5566 - val_acc: 0.7247 - val_f1: 0.3567\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5816733081248e0a3a64881f901ef26"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.5018 - acc: 0.7356 - f1: 0.4624 - val_loss: 2775.4873 - val_acc: 0.7235 - val_f1: 0.4457\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce057f10833c4f769f920b5e22c1f63b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4638 - acc: 0.7439 - f1: 0.4989 - val_loss: 2775.4688 - val_acc: 0.7372 - val_f1: 0.4291\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cde9e0e0ed7f479b8ab4b7e265eca997"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4376 - acc: 0.7613 - f1: 0.5382 - val_loss: 2775.4512 - val_acc: 0.7478 - val_f1: 0.5174\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9547c2489c64d36b4b9daa52196cf83"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.4143 - acc: 0.7766 - f1: 0.5839 - val_loss: 2775.4384 - val_acc: 0.7572 - val_f1: 0.5113\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dae98a96470b44549d1d00c1c2009c21"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3921 - acc: 0.7961 - f1: 0.6325 - val_loss: 2775.4236 - val_acc: 0.7709 - val_f1: 0.5716\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ea063533413468c91e1232f225ae513"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3677 - acc: 0.8165 - f1: 0.6795 - val_loss: 2775.4152 - val_acc: 0.7728 - val_f1: 0.5929\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "299f5d9c6efa4031a40fc29676cb7695"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3500 - acc: 0.8331 - f1: 0.7169 - val_loss: 2775.4127 - val_acc: 0.7772 - val_f1: 0.6136\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a194542d72344ff5837faa056275f6a7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3398 - acc: 0.8365 - f1: 0.7267 - val_loss: 2775.4072 - val_acc: 0.7840 - val_f1: 0.6328\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94784348b1df420484a92e6dac2ef33d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3223 - acc: 0.8532 - f1: 0.7588 - val_loss: 2775.4027 - val_acc: 0.7878 - val_f1: 0.6442\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80c45092bc6d43c28a43e19ccdbfd6f6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.3086 - acc: 0.8584 - f1: 0.7700 - val_loss: 2775.3996 - val_acc: 0.7903 - val_f1: 0.6527\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "920a79eaafb84a668e2bf6f6c8adf9f7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2987 - acc: 0.8682 - f1: 0.7877 - val_loss: 2775.3974 - val_acc: 0.7915 - val_f1: 0.6597\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5359e3da2544e3f9ab3904e8695f551"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2931 - acc: 0.8713 - f1: 0.7937 - val_loss: 2775.3936 - val_acc: 0.7884 - val_f1: 0.6556\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e93d9d9835049a2bae4d82ca8322dcb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2777 - acc: 0.8830 - f1: 0.8145 - val_loss: 2775.4034 - val_acc: 0.7978 - val_f1: 0.6737\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9102fd0c517543e0881d7855646124d0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2679 - acc: 0.8900 - f1: 0.8262 - val_loss: 2775.3913 - val_acc: 0.7996 - val_f1: 0.6721\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28885b2c6b6b488a840f60272c847e47"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2557 - acc: 0.8976 - f1: 0.8375 - val_loss: 2775.3902 - val_acc: 0.8021 - val_f1: 0.6808\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a96c69b54bde43e592556a177e6963c0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2485 - acc: 0.9019 - f1: 0.8458 - val_loss: 2775.3960 - val_acc: 0.7940 - val_f1: 0.6695\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14ac4e71283843d7a92ac2808bf740cd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2395 - acc: 0.9077 - f1: 0.8546 - val_loss: 2775.3909 - val_acc: 0.8059 - val_f1: 0.6884\n",
      "Epoch 21/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f74fa5bfe9364934bc12284002c63301"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4796/4796 [==============================] - 26s 5ms/step - loss: 2775.2289 - acc: 0.9135 - f1: 0.8645 - val_loss: 2775.4115 - val_acc: 0.7940 - val_f1: 0.6726\n",
      "Epoch 00021: early stopping\n",
      "Fold 4: Precision:69.09% Recall:70.66% Fscore:68.47% \n",
      "Train on 4797 samples, validate on 533 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88d9b95477234f6cb76e6b2631ac390e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "659bd0db8d184debbf4f3bd580f03334"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 30s 6ms/step - loss: 2788.2322 - acc: 0.6744 - f1: 0.1758 - val_loss: 2780.1332 - val_acc: 0.6873 - val_f1: 0.3562\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a429c8c7f73a4061bfb0c7182b7a4a3b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2777.5608 - acc: 0.7004 - f1: 0.3485 - val_loss: 2776.1246 - val_acc: 0.6954 - val_f1: 0.3780\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3f40c94844442939b388b4695507dff"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.7440 - acc: 0.7157 - f1: 0.4142 - val_loss: 2775.5688 - val_acc: 0.7211 - val_f1: 0.3641\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0c53f62f5ae431593254df6055654fb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.5037 - acc: 0.7343 - f1: 0.4685 - val_loss: 2775.4964 - val_acc: 0.7311 - val_f1: 0.4445\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95b6c4ed37bf4cdb8f4fd732fdddd3c7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4594 - acc: 0.7512 - f1: 0.5152 - val_loss: 2775.4730 - val_acc: 0.7323 - val_f1: 0.5041\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c600e3cb980b4232b24a23587d102515"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4309 - acc: 0.7674 - f1: 0.5621 - val_loss: 2775.4502 - val_acc: 0.7367 - val_f1: 0.5051\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a305293be97a4b379a68f640216882b1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4040 - acc: 0.7851 - f1: 0.6096 - val_loss: 2775.4416 - val_acc: 0.7667 - val_f1: 0.5796\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf3981c6f6fd49899b91b3495f89fd46"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3803 - acc: 0.8062 - f1: 0.6623 - val_loss: 2775.4194 - val_acc: 0.7767 - val_f1: 0.6027\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88a677654fe44d6994c83a2ecf3e9efa"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3611 - acc: 0.8223 - f1: 0.7007 - val_loss: 2775.4115 - val_acc: 0.7830 - val_f1: 0.6296\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5586585dfe9c49b7be9a54cee46a484b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3445 - acc: 0.8343 - f1: 0.7255 - val_loss: 2775.4032 - val_acc: 0.7780 - val_f1: 0.6266\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99c4e7c420f6474e8caec9a4f093746f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3318 - acc: 0.8425 - f1: 0.7423 - val_loss: 2775.3989 - val_acc: 0.7974 - val_f1: 0.6603\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eacf5528f6fd47f59012f7daf7238190"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3158 - acc: 0.8549 - f1: 0.7646 - val_loss: 2775.3960 - val_acc: 0.8036 - val_f1: 0.6780\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a1a8bc89cc04a46aeacfb721873ff4e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3048 - acc: 0.8619 - f1: 0.7781 - val_loss: 2775.3929 - val_acc: 0.7855 - val_f1: 0.6480\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6114937ad86142f2a87eed09da823ca2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2954 - acc: 0.8659 - f1: 0.7860 - val_loss: 2775.3989 - val_acc: 0.7992 - val_f1: 0.6755\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b21c6567a0d6451f80b813a2d95307a0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2851 - acc: 0.8749 - f1: 0.8014 - val_loss: 2775.3855 - val_acc: 0.7980 - val_f1: 0.6759\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffd6e41268b24df0848aaa9ac736186b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2721 - acc: 0.8833 - f1: 0.8152 - val_loss: 2775.3854 - val_acc: 0.8074 - val_f1: 0.6924\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38e140ac87b0457baf2fd4eb09d6d3a9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2620 - acc: 0.8924 - f1: 0.8309 - val_loss: 2775.3880 - val_acc: 0.8099 - val_f1: 0.6958\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b754deb9ea940158f72671be98294fe"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2544 - acc: 0.8972 - f1: 0.8386 - val_loss: 2775.3924 - val_acc: 0.8055 - val_f1: 0.6904\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5707737b886649539393f8ac5e1ec4e7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2406 - acc: 0.9049 - f1: 0.8512 - val_loss: 2775.3909 - val_acc: 0.7974 - val_f1: 0.6780\n",
      "Epoch 00019: early stopping\n",
      "Fold 5: Precision:68.60% Recall:67.62% Fscore:67.85% \n",
      "Train on 4797 samples, validate on 533 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7e7597135144272adab8ccd3cf1645f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c2101337d38454eb4ee0311b4f20538"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 30s 6ms/step - loss: 2788.2341 - acc: 0.6699 - f1: 0.1602 - val_loss: 2780.1381 - val_acc: 0.6923 - val_f1: 0.2104\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65b65db1cb01435fa9248dbfdde87f90"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2777.5696 - acc: 0.6963 - f1: 0.2987 - val_loss: 2776.1273 - val_acc: 0.7079 - val_f1: 0.3870\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10c72adbd027473dae7aba054efd327b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.7502 - acc: 0.7153 - f1: 0.4045 - val_loss: 2775.5692 - val_acc: 0.7117 - val_f1: 0.4115\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5f3f4cb614b4f59ae8594fbeb26c946"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.5107 - acc: 0.7282 - f1: 0.4580 - val_loss: 2775.5059 - val_acc: 0.7198 - val_f1: 0.3989\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4914c422b1e4b579711cbbb62c21f59"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4714 - acc: 0.7428 - f1: 0.4980 - val_loss: 2775.4936 - val_acc: 0.7173 - val_f1: 0.4711\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81e056714ead4d79b4ad306dc811129b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4541 - acc: 0.7522 - f1: 0.5326 - val_loss: 2775.4723 - val_acc: 0.7348 - val_f1: 0.4716\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7de89e9dcde247f79dbae2cbff3717ed"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4317 - acc: 0.7677 - f1: 0.5641 - val_loss: 2775.4578 - val_acc: 0.7430 - val_f1: 0.4905\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32ce766174224c37ac27187c87f9981b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4096 - acc: 0.7835 - f1: 0.6010 - val_loss: 2775.4444 - val_acc: 0.7555 - val_f1: 0.5438\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b1e7619b58d44c4b9a3676010db9105"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3829 - acc: 0.8056 - f1: 0.6550 - val_loss: 2775.4332 - val_acc: 0.7680 - val_f1: 0.5810\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af3720f750364e979c6cb8747c34750f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3605 - acc: 0.8233 - f1: 0.6980 - val_loss: 2775.4187 - val_acc: 0.7730 - val_f1: 0.6059\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49b642f2e63947ba986c9ec61796bcc8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3452 - acc: 0.8384 - f1: 0.7287 - val_loss: 2775.4099 - val_acc: 0.7755 - val_f1: 0.6227\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cef31bf01e8f477a884b9d717ae437c4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3274 - acc: 0.8496 - f1: 0.7519 - val_loss: 2775.4043 - val_acc: 0.7880 - val_f1: 0.6479\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c2a8880198748e7849c1d44b38a4dfe"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3154 - acc: 0.8571 - f1: 0.7677 - val_loss: 2775.3977 - val_acc: 0.7917 - val_f1: 0.6589\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "644cf36c46654b1882885988563ccb75"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3030 - acc: 0.8651 - f1: 0.7826 - val_loss: 2775.3981 - val_acc: 0.7905 - val_f1: 0.6595\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "acf1504d00994dd5aac08903e6a1f571"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2903 - acc: 0.8758 - f1: 0.8003 - val_loss: 2775.3938 - val_acc: 0.7936 - val_f1: 0.6653\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3080e1b069b94d08b239e6f82db19d86"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2806 - acc: 0.8790 - f1: 0.8074 - val_loss: 2775.3942 - val_acc: 0.8030 - val_f1: 0.6830\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "919b11d9307944358c2c7aca92716d49"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2687 - acc: 0.8885 - f1: 0.8232 - val_loss: 2775.3950 - val_acc: 0.8043 - val_f1: 0.6825\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "764bb677abc74303961237ae00e40c73"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2606 - acc: 0.8936 - f1: 0.8323 - val_loss: 2775.3917 - val_acc: 0.7955 - val_f1: 0.6721\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5311fbc7b2114cc2bae7fae73e8dbeda"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2537 - acc: 0.8994 - f1: 0.8416 - val_loss: 2775.3963 - val_acc: 0.7867 - val_f1: 0.6600\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85f50616530e46c0b6eb0373d7b155c5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2383 - acc: 0.9095 - f1: 0.8579 - val_loss: 2775.3916 - val_acc: 0.8043 - val_f1: 0.6898\n",
      "Epoch 21/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a145b28367c4dd9b7edbbd201bed4e5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2298 - acc: 0.9157 - f1: 0.8684 - val_loss: 2775.3956 - val_acc: 0.8111 - val_f1: 0.6997\n",
      "Epoch 22/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0df2dfb3832d4df8891dee419d6b2611"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2181 - acc: 0.9201 - f1: 0.8754 - val_loss: 2775.3933 - val_acc: 0.8093 - val_f1: 0.6999\n",
      "Epoch 23/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d11ff1d379b34c96ae7bf66d7710e4fd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2117 - acc: 0.9272 - f1: 0.8867 - val_loss: 2775.3989 - val_acc: 0.8086 - val_f1: 0.6994\n",
      "Epoch 00023: early stopping\n",
      "Fold 6: Precision:71.73% Recall:75.43% Fscore:73.01% \n",
      "Train on 4797 samples, validate on 533 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d114b2f5db624c3cbdfeecb6dcbbaab5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fde1f3a8754c49f48b38c675bc30c808"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 31s 6ms/step - loss: 2788.2345 - acc: 0.6748 - f1: 0.1952 - val_loss: 2780.1410 - val_acc: 0.6679 - val_f1: 0.0074\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb8ba5d489ac40b9a3a5489eaa0a2804"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2777.5716 - acc: 0.6946 - f1: 0.3243 - val_loss: 2776.1297 - val_acc: 0.6961 - val_f1: 0.2077\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cb106e8519a413e8a1d6647fbfb5bf3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.7504 - acc: 0.7196 - f1: 0.4187 - val_loss: 2775.5602 - val_acc: 0.7348 - val_f1: 0.4241\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a6b4a27d0ac46df8d93fa4589278289"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.5112 - acc: 0.7313 - f1: 0.4641 - val_loss: 2775.4892 - val_acc: 0.7311 - val_f1: 0.4172\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39eecb5f94044aef92a1832c48097b33"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4687 - acc: 0.7487 - f1: 0.5100 - val_loss: 2775.4708 - val_acc: 0.7442 - val_f1: 0.5093\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be90550fc3d14602847dd90c2e4ad12d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4457 - acc: 0.7569 - f1: 0.5399 - val_loss: 2775.4425 - val_acc: 0.7630 - val_f1: 0.5551\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc2b931f0e6944c9b9c317a88a447da6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.4193 - acc: 0.7772 - f1: 0.5851 - val_loss: 2775.4277 - val_acc: 0.7630 - val_f1: 0.5528\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "118cba4c504742d3aad2a6dcb63effe0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3926 - acc: 0.7982 - f1: 0.6360 - val_loss: 2775.4060 - val_acc: 0.7924 - val_f1: 0.6280\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b0c00213a2642a0acbda95ca57aeb4f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3723 - acc: 0.8145 - f1: 0.6771 - val_loss: 2775.3950 - val_acc: 0.8055 - val_f1: 0.6578\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6166ea464e5e4d4f95f69d632a7c5493"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3545 - acc: 0.8276 - f1: 0.7087 - val_loss: 2775.3853 - val_acc: 0.8074 - val_f1: 0.6731\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f0f8dc8a3e0427f9fa80b558058985c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3361 - acc: 0.8416 - f1: 0.7377 - val_loss: 2775.3879 - val_acc: 0.8011 - val_f1: 0.6689\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f133e6f87cd4188bd7d7cea10dbffda"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3273 - acc: 0.8464 - f1: 0.7497 - val_loss: 2775.3728 - val_acc: 0.8155 - val_f1: 0.6947\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52a73eba5a724976a51cc1f2adcd277b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.3083 - acc: 0.8588 - f1: 0.7718 - val_loss: 2775.3829 - val_acc: 0.8080 - val_f1: 0.6855\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d57daba499842379864c974ab686fab"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2984 - acc: 0.8677 - f1: 0.7879 - val_loss: 2775.3709 - val_acc: 0.8174 - val_f1: 0.7030\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b58d85299934aa09fa8357f96185729"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2842 - acc: 0.8755 - f1: 0.8010 - val_loss: 2775.3668 - val_acc: 0.8130 - val_f1: 0.6989\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d7a4311a1e84dbbaf91041bf7409bd0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2759 - acc: 0.8821 - f1: 0.8137 - val_loss: 2775.3684 - val_acc: 0.8211 - val_f1: 0.7136\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3028e31f31a24171b25976845af0f443"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2651 - acc: 0.8914 - f1: 0.8283 - val_loss: 2775.3663 - val_acc: 0.8168 - val_f1: 0.7072\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce18df1ee41a4a808a5dab1000c05876"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2570 - acc: 0.8933 - f1: 0.8320 - val_loss: 2775.3656 - val_acc: 0.8236 - val_f1: 0.7204\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d6748ad41ed41659c886ee4dd973bb4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2439 - acc: 0.9063 - f1: 0.8532 - val_loss: 2775.3645 - val_acc: 0.8280 - val_f1: 0.7300\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7ed5bfbb8f94cd7bd901b2029027c0b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2338 - acc: 0.9104 - f1: 0.8597 - val_loss: 2775.3652 - val_acc: 0.8299 - val_f1: 0.7330\n",
      "Epoch 21/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "508a3db6f8dd408e94c60beee4ef7f9d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2283 - acc: 0.9133 - f1: 0.8647 - val_loss: 2775.3759 - val_acc: 0.8236 - val_f1: 0.7219\n",
      "Epoch 22/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7aa07f1b577e4a79abcbf25c89e561c4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4797/4797 [==============================] - 26s 5ms/step - loss: 2775.2201 - acc: 0.9177 - f1: 0.8718 - val_loss: 2775.3682 - val_acc: 0.8218 - val_f1: 0.7217\n",
      "Epoch 00022: early stopping\n",
      "Fold 7: Precision:74.72% Recall:73.49% Fscore:74.02% \n",
      "Train on 4798 samples, validate on 532 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a331edb6213c4e018cfc85e1f3f8a110"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7563feb8ea1e49389eb2c9692d7dba1c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 31s 6ms/step - loss: 2788.2423 - acc: 0.6742 - f1: 0.1564 - val_loss: 2780.1352 - val_acc: 0.6723 - val_f1: 0.0321\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c981101105d94123a66b313d1fab3fbd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2777.5653 - acc: 0.6979 - f1: 0.2860 - val_loss: 2776.1165 - val_acc: 0.7099 - val_f1: 0.3603\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c737eee602245f5967e8a825c9af896"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.7442 - acc: 0.7161 - f1: 0.4018 - val_loss: 2775.5578 - val_acc: 0.7243 - val_f1: 0.3644\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e16e4b6678b42c1a43a0d11c299b7d0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.5055 - acc: 0.7289 - f1: 0.4491 - val_loss: 2775.4914 - val_acc: 0.7306 - val_f1: 0.4434\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df7e4f0377c84660876d1d3ad5437445"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.4688 - acc: 0.7439 - f1: 0.4962 - val_loss: 2775.4777 - val_acc: 0.7356 - val_f1: 0.4833\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "018664c96c7e4f75a782ccba4e601c6f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.4461 - acc: 0.7561 - f1: 0.5319 - val_loss: 2775.4669 - val_acc: 0.7381 - val_f1: 0.4949\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d257bfb9157f48fb99c4b5ae4c3b027e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.4294 - acc: 0.7685 - f1: 0.5623 - val_loss: 2775.4573 - val_acc: 0.7519 - val_f1: 0.5423\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "549a8543fa2241ec8589d114fab68db8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.4071 - acc: 0.7859 - f1: 0.6051 - val_loss: 2775.4408 - val_acc: 0.7594 - val_f1: 0.5511\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e858fc0866204f71a7f36f3925174d9f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3869 - acc: 0.8017 - f1: 0.6432 - val_loss: 2775.4436 - val_acc: 0.7538 - val_f1: 0.5641\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ff20d0dc3594acca1929067ce0511f7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3730 - acc: 0.8121 - f1: 0.6735 - val_loss: 2775.4226 - val_acc: 0.7751 - val_f1: 0.5952\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ff6145fc6574694a32ee742298eee59"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3534 - acc: 0.8256 - f1: 0.7021 - val_loss: 2775.4157 - val_acc: 0.7776 - val_f1: 0.6117\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b458e76f76714c9c97657b9a6d2223f8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3384 - acc: 0.8400 - f1: 0.7335 - val_loss: 2775.4180 - val_acc: 0.7744 - val_f1: 0.6210\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "240aea72096e4c89b36e7c978dddbae2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3279 - acc: 0.8475 - f1: 0.7500 - val_loss: 2775.4281 - val_acc: 0.7776 - val_f1: 0.6316\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a544df892bca4e68a656c3613ca80047"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3137 - acc: 0.8595 - f1: 0.7713 - val_loss: 2775.4037 - val_acc: 0.7882 - val_f1: 0.6510\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5968f0490e3c447993df09e4c5825fdc"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3051 - acc: 0.8655 - f1: 0.7838 - val_loss: 2775.4175 - val_acc: 0.7807 - val_f1: 0.6427\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2842c521032f4bdf86c9691f4cb36518"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2927 - acc: 0.8734 - f1: 0.7969 - val_loss: 2775.4026 - val_acc: 0.7838 - val_f1: 0.6494\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57f419856177498ca878773e53e57017"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2811 - acc: 0.8792 - f1: 0.8074 - val_loss: 2775.3954 - val_acc: 0.7895 - val_f1: 0.6588\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29280f67c867477c8895340aaa39e2bd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2713 - acc: 0.8861 - f1: 0.8197 - val_loss: 2775.3931 - val_acc: 0.7964 - val_f1: 0.6696\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "854e47d7aa3d43baad4edd0c6af74168"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2627 - acc: 0.8920 - f1: 0.8293 - val_loss: 2775.3932 - val_acc: 0.7951 - val_f1: 0.6706\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0417a4b487741fead5a3a3500fa0eae"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2532 - acc: 0.8960 - f1: 0.8364 - val_loss: 2775.3945 - val_acc: 0.7945 - val_f1: 0.6701\n",
      "Epoch 21/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e50acde065246ab921362caf42ec05e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2407 - acc: 0.9052 - f1: 0.8516 - val_loss: 2775.4079 - val_acc: 0.7920 - val_f1: 0.6708\n",
      "Epoch 00021: early stopping\n",
      "Fold 8: Precision:67.82% Recall:70.06% Fscore:67.82% \n",
      "Train on 4798 samples, validate on 532 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "174fc59b37cc42ebb9fefe7531dd881f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b490ca619c48421989ef08267d5a1e38"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 31s 7ms/step - loss: 2788.2327 - acc: 0.6677 - f1: 0.1232 - val_loss: 2780.1385 - val_acc: 0.6692 - val_f1: 0.0147\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f0a7913011e464a99953e8b58b37304"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2777.5731 - acc: 0.6909 - f1: 0.2455 - val_loss: 2776.1264 - val_acc: 0.7143 - val_f1: 0.2810\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89d5a925faf74ac9b0830347cfe865d5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.7507 - acc: 0.7125 - f1: 0.3785 - val_loss: 2775.5592 - val_acc: 0.7193 - val_f1: 0.3391\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44a91f59f595488fa5979ca8c9a5da3b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.5098 - acc: 0.7286 - f1: 0.4458 - val_loss: 2775.4799 - val_acc: 0.7325 - val_f1: 0.4133\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fae31f4ce2a743489a24b44edd4a6aef"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.4671 - acc: 0.7442 - f1: 0.4878 - val_loss: 2775.4529 - val_acc: 0.7481 - val_f1: 0.4671\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5c7945fb9fc4c7f9692e8558134bffa"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.4332 - acc: 0.7631 - f1: 0.5475 - val_loss: 2775.4293 - val_acc: 0.7625 - val_f1: 0.5331\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea46e9d458f74facacae20e9ca061526"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.4068 - acc: 0.7834 - f1: 0.6060 - val_loss: 2775.4191 - val_acc: 0.7813 - val_f1: 0.6031\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d63959343db4afb94dadd54e68b1e72"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3847 - acc: 0.8019 - f1: 0.6557 - val_loss: 2775.4116 - val_acc: 0.7794 - val_f1: 0.6207\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5915c1b3a2f490583497acc37f4eb69"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3660 - acc: 0.8161 - f1: 0.6884 - val_loss: 2775.3970 - val_acc: 0.7895 - val_f1: 0.6317\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d163d7b35e6648dfaff8c06d4bd24192"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3505 - acc: 0.8276 - f1: 0.7141 - val_loss: 2775.3868 - val_acc: 0.7982 - val_f1: 0.6587\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "320dcac5dc9f43afb24c246fa5ff9445"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3367 - acc: 0.8406 - f1: 0.7380 - val_loss: 2775.3802 - val_acc: 0.8008 - val_f1: 0.6672\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d43a20a5755547729338b26d608443fb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3240 - acc: 0.8506 - f1: 0.7585 - val_loss: 2775.3766 - val_acc: 0.7989 - val_f1: 0.6678\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee00043f9fac40bab2181a34bf53fc26"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3135 - acc: 0.8572 - f1: 0.7700 - val_loss: 2775.3851 - val_acc: 0.8008 - val_f1: 0.6724\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "925eb729e78a46f9be32ac761f0ba354"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.3032 - acc: 0.8648 - f1: 0.7847 - val_loss: 2775.3858 - val_acc: 0.8033 - val_f1: 0.6799\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb75ce4c51914a1897a0a0d296580c90"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2934 - acc: 0.8699 - f1: 0.7930 - val_loss: 2775.3714 - val_acc: 0.8058 - val_f1: 0.6851\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db6057764c444f57b0054f3326eab9b1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2806 - acc: 0.8818 - f1: 0.8130 - val_loss: 2775.3724 - val_acc: 0.8045 - val_f1: 0.6829\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4672975549274759a417eca6cabed7ba"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2696 - acc: 0.8878 - f1: 0.8224 - val_loss: 2775.3673 - val_acc: 0.8102 - val_f1: 0.6926\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "330504518bb04dd3acd57024da95edd6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2622 - acc: 0.8936 - f1: 0.8330 - val_loss: 2775.3694 - val_acc: 0.8114 - val_f1: 0.6964\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8c320c6b7b54b0e861fda825e96bf7a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2534 - acc: 0.8986 - f1: 0.8405 - val_loss: 2775.3688 - val_acc: 0.8020 - val_f1: 0.6807\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "936d93f371d342baa0374833ab94728c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4798/4798 [==============================] - 26s 5ms/step - loss: 2775.2421 - acc: 0.9068 - f1: 0.8539 - val_loss: 2775.3688 - val_acc: 0.8152 - val_f1: 0.7048\n",
      "Epoch 00020: early stopping\n",
      "Fold 9: Precision:69.19% Recall:71.34% Fscore:70.08% \n",
      "Train on 4799 samples, validate on 531 samples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "411f44e5c3c649a7982ccb59e071ceb5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c110e93863f64f49998354bc007748cb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 32s 7ms/step - loss: 2788.2246 - acc: 0.6729 - f1: 0.1592 - val_loss: 2780.1273 - val_acc: 0.6755 - val_f1: 0.0641\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b67128a50c046f1a0f6f242e80d6bd1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2777.5600 - acc: 0.6958 - f1: 0.2648 - val_loss: 2776.1169 - val_acc: 0.6937 - val_f1: 0.4572\n",
      "Epoch 3/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bfb413e543947cfbfe2e4e8f3edc94f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.7427 - acc: 0.7131 - f1: 0.3922 - val_loss: 2775.5531 - val_acc: 0.7219 - val_f1: 0.3783\n",
      "Epoch 4/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4e1d6ae81b345ed8de4f83da0644670"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.5033 - acc: 0.7292 - f1: 0.4439 - val_loss: 2775.4873 - val_acc: 0.7294 - val_f1: 0.4166\n",
      "Epoch 5/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76654dbca1014f4691e1bd90aab946b4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.4620 - acc: 0.7428 - f1: 0.4976 - val_loss: 2775.4666 - val_acc: 0.7401 - val_f1: 0.4515\n",
      "Epoch 6/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc7d1c2f5e224a18892d4b737c9120a6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.4387 - acc: 0.7585 - f1: 0.5323 - val_loss: 2775.4471 - val_acc: 0.7552 - val_f1: 0.5216\n",
      "Epoch 7/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a26492c889048e995bac6b7d8aac718"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.4089 - acc: 0.7840 - f1: 0.5989 - val_loss: 2775.4266 - val_acc: 0.7772 - val_f1: 0.5809\n",
      "Epoch 8/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7389f378ee8a4ae88958aba91da86fba"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.3825 - acc: 0.8026 - f1: 0.6506 - val_loss: 2775.4071 - val_acc: 0.7859 - val_f1: 0.6206\n",
      "Epoch 9/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb38795c54aa418c913e8f743c08551f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.3664 - acc: 0.8163 - f1: 0.6866 - val_loss: 2775.3964 - val_acc: 0.7910 - val_f1: 0.6405\n",
      "Epoch 10/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b635603ba824484bac5765cba62514ce"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.3500 - acc: 0.8273 - f1: 0.7115 - val_loss: 2775.3944 - val_acc: 0.8110 - val_f1: 0.6822\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c6d2fd1431b4ee3932aaf5666a27af9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.3331 - acc: 0.8388 - f1: 0.7359 - val_loss: 2775.3836 - val_acc: 0.8192 - val_f1: 0.6997\n",
      "Epoch 12/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b114368a58a462695fe5c603e46f52e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.3217 - acc: 0.8510 - f1: 0.7590 - val_loss: 2775.3794 - val_acc: 0.8104 - val_f1: 0.6881\n",
      "Epoch 13/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2dbd83fae05b4f2da00f406cafdc7387"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.3075 - acc: 0.8609 - f1: 0.7759 - val_loss: 2775.3888 - val_acc: 0.8067 - val_f1: 0.6873\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5791f9350cb6494a8eccc19a815fd1a3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2952 - acc: 0.8707 - f1: 0.7926 - val_loss: 2775.3732 - val_acc: 0.8167 - val_f1: 0.7052\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a887593c1e214ac1b0368eda204e713f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2827 - acc: 0.8773 - f1: 0.8046 - val_loss: 2775.3690 - val_acc: 0.8198 - val_f1: 0.7137\n",
      "Epoch 16/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f695dc88d6854af1b565269fd1bd845c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2752 - acc: 0.8796 - f1: 0.8095 - val_loss: 2775.3647 - val_acc: 0.8293 - val_f1: 0.7256\n",
      "Epoch 17/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f0547e3883b4e7c8043a30afd666d7e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2653 - acc: 0.8868 - f1: 0.8218 - val_loss: 2775.3653 - val_acc: 0.8236 - val_f1: 0.7145\n",
      "Epoch 18/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe0a0af8c9244392b706f496fd284e18"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2591 - acc: 0.8902 - f1: 0.8271 - val_loss: 2775.3638 - val_acc: 0.8267 - val_f1: 0.7266\n",
      "Epoch 19/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a01e76e65ab349acb7dff995bdc9ac58"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2481 - acc: 0.9003 - f1: 0.8432 - val_loss: 2775.3673 - val_acc: 0.8205 - val_f1: 0.7158\n",
      "Epoch 20/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5c7f6198f024040a225d83bb7f60c59"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2367 - acc: 0.9085 - f1: 0.8569 - val_loss: 2775.3664 - val_acc: 0.8186 - val_f1: 0.7106\n",
      "Epoch 21/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02bf409ba2de48508eb150ce59278fae"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2299 - acc: 0.9140 - f1: 0.8660 - val_loss: 2775.3607 - val_acc: 0.8293 - val_f1: 0.7272\n",
      "Epoch 22/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21365f6d609e4bea90c0c3bceac471ca"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2241 - acc: 0.9147 - f1: 0.8675 - val_loss: 2775.3678 - val_acc: 0.8236 - val_f1: 0.7232\n",
      "Epoch 23/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7f225049da243ef97c68eb7296e4434"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2139 - acc: 0.9232 - f1: 0.8809 - val_loss: 2775.3577 - val_acc: 0.8286 - val_f1: 0.7279\n",
      "Epoch 24/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70ff9b910e7c4f61b4bcfc4c37d05cc6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.2012 - acc: 0.9311 - f1: 0.8930 - val_loss: 2775.3562 - val_acc: 0.8286 - val_f1: 0.7322\n",
      "Epoch 25/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "402c42c9a2514efdb64fbc7dea47d6ce"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.1979 - acc: 0.9331 - f1: 0.8963 - val_loss: 2775.3538 - val_acc: 0.8324 - val_f1: 0.7383\n",
      "Epoch 26/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "780dca1a4142490297ddb307b77a2108"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.1873 - acc: 0.9385 - f1: 0.9050 - val_loss: 2775.3615 - val_acc: 0.8223 - val_f1: 0.7211\n",
      "Epoch 27/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a90a3a94af84a52bd83183e65f610cd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.1814 - acc: 0.9399 - f1: 0.9074 - val_loss: 2775.3570 - val_acc: 0.8311 - val_f1: 0.7326\n",
      "Epoch 28/50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6a8e03856154ce68084415fac169f20"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4799/4799 [==============================] - 26s 5ms/step - loss: 2775.1760 - acc: 0.9433 - f1: 0.9126 - val_loss: 2775.3573 - val_acc: 0.8280 - val_f1: 0.7320\n",
      "Epoch 00028: early stopping\n",
      "Fold 10: Precision:73.75% Recall:73.54% Fscore:73.46% \n",
      "Final: Precision:71.62% Recall:72.64% Fscore:71.63% \n"
     ]
    }
   ],
   "source": [
    "all_histories=train_and_evaluate_model_lstm_position()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}