toxic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mukesh/miniconda3/envs/statoil/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "from string import punctuation\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv1D, BatchNormalization, Activation\n",
    "from keras.layers import Embedding, Input, Dense, Dropout, Lambda, MaxPooling1D\n",
    "from keras.optimizers import SGD\n",
    "from keras.models import Model\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#len(ALPHABET)=68\n",
    "ALPHABET = 'abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:’\"/|_#$%ˆ&*˜‘+=<>()[]{} '\n",
    "FEATURE_LEN = 1024 #maxlen\n",
    "path = '../data/'\n",
    "TRAIN_DATA_FILE=path+'train.csv'\n",
    "TEST_DATA_FILE=path+'test.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_char_dict():\n",
    "    char_dict={}\n",
    "    for i,c in enumerate(ALPHABET):\n",
    "        char_dict[c]=i+1\n",
    "    return char_dict\n",
    "\n",
    "def char2vec(text, max_length=FEATURE_LEN):\n",
    "    char_dict = get_char_dict()\n",
    "    data=np.zeros(max_length)\n",
    "    \n",
    "    for i in range(0, len(text)):\n",
    "        if i >= max_length:\n",
    "            return data\n",
    "        \n",
    "        elif text[i] in char_dict:\n",
    "            data[i] = char_dict[text[i]]\n",
    "        \n",
    "        else:\n",
    "            data[i]=68\n",
    "    return data\n",
    "    \n",
    "\n",
    "def conv_shape(conv):\n",
    "    return conv.get_shape().as_list()[1:]\n",
    "\n",
    "replace_ip=re.compile(r'([0-9]+)(?:\\.[0-9]+){3}',)\n",
    "def text_to_wordlist(text, remove_stopwords=True, stem_words=False):\n",
    "    # Clean the text, with the option to remove stopwords and to stem words.\n",
    "    \n",
    "    # Convert words to lower case and split them\n",
    "    text = text.lower().split()\n",
    "    \n",
    "    \n",
    "    \n",
    "    # Optionally, remove stop words\n",
    "    if remove_stopwords:\n",
    "        stops = set(stopwords.words(\"english\"))\n",
    "        text = [w for w in text if not w in stops]\n",
    "    \n",
    "    text = \" \".join(text)\n",
    "    \n",
    "    \n",
    "    #Replace IP address\n",
    "    text=replace_ip.sub('',text)\n",
    "    \n",
    "    \n",
    "    # Optionally, shorten words to their stems\n",
    "    if stem_words:\n",
    "        text = text.split()\n",
    "        stemmer = SnowballStemmer('english')\n",
    "        stemmed_words = [stemmer.stem(word) for word in text]\n",
    "        text = \" \".join(stemmed_words)\n",
    "    \n",
    "    # Return a list of words\n",
    "    return(text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(TRAIN_DATA_FILE)\n",
    "test_df = pd.read_csv(TEST_DATA_FILE)\n",
    "\n",
    "list_sentences_train = train_df[\"comment_text\"].fillna(\"NA\").values\n",
    "list_classes = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "y = train_df[list_classes].values\n",
    "list_sentences_test = test_df[\"comment_text\"].fillna(\"NA\").values\n",
    "data=[]\n",
    "for text in list_sentences_train:\n",
    "    data.append(char2vec(text_to_wordlist(text)))\n",
    "data=np.array(data)\n",
    "\n",
    "test_data = []\n",
    "for text in list_sentences_test:\n",
    "    test_data.append(char2vec(text_to_wordlist(text)))\n",
    "test_data=np.array(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(153164, 1024)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ConvolutionalBlock(input_shape, num_filters):\n",
    "    model=Sequential()\n",
    "\n",
    "    #1st conv layer\n",
    "    model.add(Conv1D(filters = num_filters, kernel_size = 3, strides = 1, padding = \"same\", input_shape = input_shape))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Activation(\"relu\"))\n",
    "\n",
    "    #2nd conv layer\n",
    "    model.add(Conv1D(filters = num_filters, kernel_size = 3, strides = 1, padding = \"same\"))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Activation(\"relu\"))\n",
    "\n",
    "    return model\n",
    "\n",
    "#https://www.tensorflow.org/api_docs/python/tf/nn/top_k\n",
    "def top_kmax(x):\n",
    "    x=tf.transpose(x, [0, 2, 1])\n",
    "    k_max = tf.nn.top_k(x, k=top_k)\n",
    "    return tf.reshape(k_max[0], (-1, num_filters[-1]*top_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vdcnn_model(num_filters, num_classes, sequence_max_length, num_chars, embedding_size, top_k, learning_rate=0.001):\n",
    "    \n",
    "    inputs=Input(shape=(sequence_max_length, ), dtype='int32', name='input')\n",
    "    \n",
    "    embedded_seq = Embedding(num_chars, embedding_size, input_length=sequence_max_length)(inputs)\n",
    "    embedded_seq = BatchNormalization()(embedded_seq)\n",
    "    #1st Layer\n",
    "    conv = Conv1D(filters=64, kernel_size=3, strides=2, padding=\"same\")(embedded_seq)\n",
    "    \n",
    "    #ConvBlocks\n",
    "    for i in range(len(num_filters)):\n",
    "        conv = ConvolutionalBlock(conv_shape(conv), num_filters[i])(conv)\n",
    "        conv = MaxPooling1D(pool_size=3, strides=2, padding=\"same\")(conv)\n",
    "        \n",
    "    def _top_k(x):\n",
    "        x = tf.transpose(x, [0, 2, 1])\n",
    "        k_max = tf.nn.top_k(x, k=top_k)\n",
    "        return tf.reshape(k_max[0], (-1, num_filters[-1] * top_k))\n",
    "    \n",
    "    k_max = Lambda(_top_k, output_shape=(num_filters[-1] * top_k,))(conv)\n",
    "    \n",
    "    #fully connected layers\n",
    "    # in original paper they didn't used dropouts\n",
    "    fc1=Dense(512, activation='relu', kernel_initializer='he_normal')(k_max)\n",
    "    fc1=Dropout(0.3)(fc1)\n",
    "    fc2=Dense(512, activation='relu', kernel_initializer='he_normal')(fc1)\n",
    "    fc2=Dropout(0.3)(fc2)\n",
    "    out=Dense(num_classes, activation='sigmoid')(fc2)\n",
    "    \n",
    "    #optimizer\n",
    "    #sgd = SGD(lr=learning_rate, decay=1e-6, momentum=0.9, nesterov=False)\n",
    "    \n",
    "    model = Model(inputs=inputs, outputs=out)\n",
    "    model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input (InputLayer)           (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 1024, 16)          1104      \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 1024, 16)          64        \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 512, 64)           3136      \n",
      "_________________________________________________________________\n",
      "sequential_1 (Sequential)    (None, 512, 64)           25216     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 256, 64)           0         \n",
      "_________________________________________________________________\n",
      "sequential_2 (Sequential)    (None, 256, 128)          75008     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 128, 128)          0         \n",
      "_________________________________________________________________\n",
      "sequential_3 (Sequential)    (None, 128, 256)          297472    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1 (None, 64, 256)           0         \n",
      "_________________________________________________________________\n",
      "sequential_4 (Sequential)    (None, 64, 512)           1184768   \n",
      "_________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1 (None, 32, 512)           0         \n",
      "_________________________________________________________________\n",
      "lambda_1 (Lambda)            (None, 1536)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 512)               786944    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 6)                 3078      \n",
      "=================================================================\n",
      "Total params: 2,639,446\n",
      "Trainable params: 2,635,574\n",
      "Non-trainable params: 3,872\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "num_filters = [64, 128, 256, 512]\n",
    "model=vdcnn_model(num_filters=num_filters, num_classes=6,num_chars=69, sequence_max_length=FEATURE_LEN,embedding_size=16,top_k=3)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def limit_mem():\n",
    "    import keras.backend as K\n",
    "    K.get_session().close()\n",
    "    cfg = K.tf.ConfigProto()\n",
    "    cfg.gpu_options.allow_growth = True\n",
    "    K.set_session(K.tf.Session(config=cfg))\n",
    "    print('gpu memory cleaned')\n",
    "    \n",
    "def preds(k):\n",
    "    from datetime import datetime\n",
    "    y_temp = np.zeros((len(test_data), len(list_classes)))\n",
    "    y_pred = np.zeros((len(test_data), len(list_classes)))\n",
    "    i=0;\n",
    "    from sklearn.model_selection import KFold\n",
    "    kf = KFold(n_splits=k, random_state=2)\n",
    "\n",
    "    for train_index, test_index in kf.split(data):\n",
    "        limit_mem()\n",
    "        start=datetime.now()\n",
    "        print('fold====================>>>>>>>>>>',i+1)\n",
    "        X_train, X_test = data[train_index], data[test_index]\n",
    "        y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "        model = None\n",
    "        num_filters = [64, 128, 256, 512]\n",
    "        model=vdcnn_model(num_filters=num_filters, num_classes=6,num_chars=69, sequence_max_length=FEATURE_LEN,embedding_size=16,top_k=3)\n",
    "\n",
    "        early_stopping =EarlyStopping(monitor='val_loss', patience=5)\n",
    "        bst_model_path = 'cv10_best_weights'+str(i+1) + '.h5'\n",
    "        model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "        hist = model.fit(X_train, y_train, \\\n",
    "                validation_data=(X_test, y_test), \\\n",
    "                epochs=200, batch_size=256,callbacks=[early_stopping, model_checkpoint])\n",
    "\n",
    "        bst_val_score = min(hist.history['val_loss'])\n",
    "        print('bst_val_score',bst_val_score)\n",
    "\n",
    "        model.load_weights(bst_model_path)\n",
    "        #model.fit(data, y,epochs=2, batch_size=256, shuffle=True,)\n",
    "        \n",
    "        y_temp = model.predict([test_data], batch_size=256, verbose=1)\n",
    "        y_pred+=y_temp\n",
    "        end=datetime.now()\n",
    "        print(\" \")\n",
    "        print('time taken for this fold', end-start)\n",
    "        i+=1\n",
    "    y_test_pred=y_pred/k\n",
    "    return y_test_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 1\n",
      "Train on 143613 samples, validate on 15958 samples\n",
      "Epoch 1/200\n",
      "143613/143613 [==============================] - 265s 2ms/step - loss: 0.1147 - acc: 0.9710 - val_loss: 0.1356 - val_acc: 0.9603\n",
      "Epoch 2/200\n",
      "143613/143613 [==============================] - 237s 2ms/step - loss: 0.0678 - acc: 0.9784 - val_loss: 0.0707 - val_acc: 0.9788\n",
      "Epoch 3/200\n",
      "143613/143613 [==============================] - 214s 1ms/step - loss: 0.0618 - acc: 0.9796 - val_loss: 0.0643 - val_acc: 0.9798\n",
      "Epoch 4/200\n",
      "143613/143613 [==============================] - 214s 1ms/step - loss: 0.0578 - acc: 0.9803 - val_loss: 0.0590 - val_acc: 0.9798\n",
      "Epoch 5/200\n",
      "143613/143613 [==============================] - 213s 1ms/step - loss: 0.0552 - acc: 0.9809 - val_loss: 0.0596 - val_acc: 0.9806\n",
      "Epoch 6/200\n",
      "143613/143613 [==============================] - 213s 1ms/step - loss: 0.0535 - acc: 0.9813 - val_loss: 0.0660 - val_acc: 0.9797\n",
      "Epoch 7/200\n",
      "143613/143613 [==============================] - 213s 1ms/step - loss: 0.0520 - acc: 0.9817 - val_loss: 0.0582 - val_acc: 0.9808\n",
      "Epoch 8/200\n",
      "143613/143613 [==============================] - 212s 1ms/step - loss: 0.0505 - acc: 0.9820 - val_loss: 0.0601 - val_acc: 0.9807\n",
      "Epoch 9/200\n",
      "143613/143613 [==============================] - 212s 1ms/step - loss: 0.0496 - acc: 0.9822 - val_loss: 0.0632 - val_acc: 0.9806\n",
      "Epoch 10/200\n",
      "143613/143613 [==============================] - 212s 1ms/step - loss: 0.0487 - acc: 0.9823 - val_loss: 0.0606 - val_acc: 0.9808\n",
      "Epoch 11/200\n",
      "143613/143613 [==============================] - 212s 1ms/step - loss: 0.0476 - acc: 0.9826 - val_loss: 0.0605 - val_acc: 0.9797\n",
      "Epoch 12/200\n",
      "143613/143613 [==============================] - 212s 1ms/step - loss: 0.0468 - acc: 0.9829 - val_loss: 0.0608 - val_acc: 0.9802\n",
      "bst_val_score 0.058247977309559267\n",
      "153164/153164 [==============================] - 76s 498us/step\n",
      " \n",
      "time taken for this fold 0:45:14.805579\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 2\n",
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 222s 2ms/step - loss: 0.1200 - acc: 0.9698 - val_loss: 0.1105 - val_acc: 0.9684\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0715 - acc: 0.9783 - val_loss: 0.1574 - val_acc: 0.9615\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0649 - acc: 0.9792 - val_loss: 0.0645 - val_acc: 0.9785\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0582 - acc: 0.9803 - val_loss: 0.0638 - val_acc: 0.9787\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0556 - acc: 0.9809 - val_loss: 0.0653 - val_acc: 0.9798\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0533 - acc: 0.9814 - val_loss: 0.0599 - val_acc: 0.9797\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0517 - acc: 0.9817 - val_loss: 0.0597 - val_acc: 0.9791\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0506 - acc: 0.9820 - val_loss: 0.0569 - val_acc: 0.9801\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0491 - acc: 0.9822 - val_loss: 0.0727 - val_acc: 0.9797\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0481 - acc: 0.9826 - val_loss: 0.0597 - val_acc: 0.9797\n",
      "Epoch 11/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0469 - acc: 0.9828 - val_loss: 0.0609 - val_acc: 0.9801\n",
      "Epoch 12/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0462 - acc: 0.9830 - val_loss: 0.0597 - val_acc: 0.9803\n",
      "Epoch 13/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0455 - acc: 0.9831 - val_loss: 0.0603 - val_acc: 0.9797\n",
      "bst_val_score 0.056901069800744435\n",
      "153164/153164 [==============================] - 76s 496us/step\n",
      " \n",
      "time taken for this fold 0:47:35.189056\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 3\n",
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 225s 2ms/step - loss: 0.1151 - acc: 0.9706 - val_loss: 0.0787 - val_acc: 0.9748\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0686 - acc: 0.9783 - val_loss: 0.0613 - val_acc: 0.9799\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0620 - acc: 0.9795 - val_loss: 0.0586 - val_acc: 0.9799\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0577 - acc: 0.9806 - val_loss: 0.0583 - val_acc: 0.9802\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0554 - acc: 0.9808 - val_loss: 0.0577 - val_acc: 0.9796\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0532 - acc: 0.9813 - val_loss: 0.0560 - val_acc: 0.9811\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0515 - acc: 0.9816 - val_loss: 0.0571 - val_acc: 0.9798\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0503 - acc: 0.9820 - val_loss: 0.0596 - val_acc: 0.9801\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0492 - acc: 0.9822 - val_loss: 0.0595 - val_acc: 0.9796\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0481 - acc: 0.9824 - val_loss: 0.0542 - val_acc: 0.9811\n",
      "Epoch 11/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0470 - acc: 0.9827 - val_loss: 0.0912 - val_acc: 0.9766\n",
      "Epoch 12/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0460 - acc: 0.9830 - val_loss: 0.0537 - val_acc: 0.9809\n",
      "Epoch 13/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0456 - acc: 0.9832 - val_loss: 0.0775 - val_acc: 0.9788\n",
      "Epoch 14/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0446 - acc: 0.9832 - val_loss: 0.0563 - val_acc: 0.9808\n",
      "Epoch 15/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0443 - acc: 0.9834 - val_loss: 0.0620 - val_acc: 0.9797\n",
      "Epoch 16/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0441 - acc: 0.9834 - val_loss: 0.0593 - val_acc: 0.9804\n",
      "Epoch 17/200\n",
      "143614/143614 [==============================] - 212s 1ms/step - loss: 0.0431 - acc: 0.9837 - val_loss: 0.0695 - val_acc: 0.9811\n",
      "bst_val_score 0.053729575833573844\n",
      "153164/153164 [==============================] - 76s 498us/step\n",
      " \n",
      "time taken for this fold 1:01:56.463782\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 4\n",
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 228s 2ms/step - loss: 0.1151 - acc: 0.9705 - val_loss: 0.0794 - val_acc: 0.9763\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0682 - acc: 0.9783 - val_loss: 0.0663 - val_acc: 0.9795\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0616 - acc: 0.9796 - val_loss: 0.0821 - val_acc: 0.9767\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0578 - acc: 0.9803 - val_loss: 0.0570 - val_acc: 0.9807\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0553 - acc: 0.9809 - val_loss: 0.0585 - val_acc: 0.9809\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0530 - acc: 0.9813 - val_loss: 0.0752 - val_acc: 0.9781\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0515 - acc: 0.9815 - val_loss: 0.0573 - val_acc: 0.9814\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0505 - acc: 0.9820 - val_loss: 0.0567 - val_acc: 0.9811\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0493 - acc: 0.9821 - val_loss: 0.0564 - val_acc: 0.9806\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0483 - acc: 0.9824 - val_loss: 0.0881 - val_acc: 0.9745\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0473 - acc: 0.9826 - val_loss: 0.0856 - val_acc: 0.9795\n",
      "Epoch 12/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0465 - acc: 0.9828 - val_loss: 0.0681 - val_acc: 0.9805\n",
      "Epoch 13/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0457 - acc: 0.9829 - val_loss: 0.1153 - val_acc: 0.9796\n",
      "Epoch 14/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0451 - acc: 0.9831 - val_loss: 0.0615 - val_acc: 0.9802\n",
      "bst_val_score 0.056414422745740364\n",
      "153164/153164 [==============================] - 82s 533us/step\n",
      " \n",
      "time taken for this fold 0:51:33.760633\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 5\n",
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 231s 2ms/step - loss: 0.1124 - acc: 0.9708 - val_loss: 0.1134 - val_acc: 0.9756\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0684 - acc: 0.9783 - val_loss: 0.0669 - val_acc: 0.9791\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0624 - acc: 0.9793 - val_loss: 0.0838 - val_acc: 0.9772\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0584 - acc: 0.9803 - val_loss: 0.0599 - val_acc: 0.9796\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0557 - acc: 0.9808 - val_loss: 0.0579 - val_acc: 0.9811\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0536 - acc: 0.9813 - val_loss: 0.0713 - val_acc: 0.9780\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0522 - acc: 0.9815 - val_loss: 0.0568 - val_acc: 0.9805\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0508 - acc: 0.9819 - val_loss: 0.0607 - val_acc: 0.9810\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0496 - acc: 0.9820 - val_loss: 0.0686 - val_acc: 0.9792\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0484 - acc: 0.9823 - val_loss: 0.0657 - val_acc: 0.9811\n",
      "Epoch 11/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0474 - acc: 0.9827 - val_loss: 0.0600 - val_acc: 0.9809\n",
      "Epoch 12/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0468 - acc: 0.9827 - val_loss: 0.0603 - val_acc: 0.9800\n",
      "bst_val_score 0.056759927017225424\n",
      "153164/153164 [==============================] - 83s 540us/step\n",
      " \n",
      "time taken for this fold 0:44:39.765910\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 6\n",
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 234s 2ms/step - loss: 0.1200 - acc: 0.9696 - val_loss: 0.0720 - val_acc: 0.9777\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0695 - acc: 0.9782 - val_loss: 0.0737 - val_acc: 0.9784\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0631 - acc: 0.9793 - val_loss: 0.0748 - val_acc: 0.9785\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0587 - acc: 0.9802 - val_loss: 0.0586 - val_acc: 0.9810\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0560 - acc: 0.9808 - val_loss: 0.0582 - val_acc: 0.9806\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0537 - acc: 0.9812 - val_loss: 0.0555 - val_acc: 0.9811\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0523 - acc: 0.9816 - val_loss: 0.0614 - val_acc: 0.9810\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0506 - acc: 0.9821 - val_loss: 0.0656 - val_acc: 0.9795\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0494 - acc: 0.9821 - val_loss: 0.0662 - val_acc: 0.9796\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0480 - acc: 0.9824 - val_loss: 0.0588 - val_acc: 0.9809\n",
      "Epoch 11/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0474 - acc: 0.9826 - val_loss: 0.0597 - val_acc: 0.9789\n",
      "bst_val_score 0.0554629157862807\n",
      "153164/153164 [==============================] - 82s 536us/step\n",
      " \n",
      "time taken for this fold 0:41:03.546180\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 7\n",
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 237s 2ms/step - loss: 0.1154 - acc: 0.9698 - val_loss: 0.1020 - val_acc: 0.9757\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0686 - acc: 0.9784 - val_loss: 0.0760 - val_acc: 0.9780\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0621 - acc: 0.9793 - val_loss: 0.0619 - val_acc: 0.9796\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0582 - acc: 0.9802 - val_loss: 0.0600 - val_acc: 0.9798\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0555 - acc: 0.9807 - val_loss: 0.0569 - val_acc: 0.9811\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0535 - acc: 0.9811 - val_loss: 0.0730 - val_acc: 0.9803\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0518 - acc: 0.9818 - val_loss: 0.0611 - val_acc: 0.9810\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0504 - acc: 0.9819 - val_loss: 0.0681 - val_acc: 0.9813\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0491 - acc: 0.9822 - val_loss: 0.0654 - val_acc: 0.9796\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0482 - acc: 0.9824 - val_loss: 0.0653 - val_acc: 0.9805\n",
      "bst_val_score 0.0569391775181279\n",
      "153164/153164 [==============================] - 83s 544us/step\n",
      " \n",
      "time taken for this fold 0:37:37.349243\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 8\n",
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 240s 2ms/step - loss: 0.1098 - acc: 0.9707 - val_loss: 0.1868 - val_acc: 0.9271\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0682 - acc: 0.9786 - val_loss: 0.0666 - val_acc: 0.9788\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0618 - acc: 0.9796 - val_loss: 0.0862 - val_acc: 0.9777\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0579 - acc: 0.9804 - val_loss: 0.0759 - val_acc: 0.9756\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0555 - acc: 0.9809 - val_loss: 0.0707 - val_acc: 0.9789\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0535 - acc: 0.9812 - val_loss: 0.0568 - val_acc: 0.9809\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0520 - acc: 0.9815 - val_loss: 0.0624 - val_acc: 0.9799\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0504 - acc: 0.9819 - val_loss: 0.0558 - val_acc: 0.9811\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0492 - acc: 0.9821 - val_loss: 0.0632 - val_acc: 0.9797\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0483 - acc: 0.9825 - val_loss: 0.0581 - val_acc: 0.9805\n",
      "Epoch 11/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0475 - acc: 0.9826 - val_loss: 0.1012 - val_acc: 0.9632\n",
      "Epoch 12/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0469 - acc: 0.9828 - val_loss: 0.0642 - val_acc: 0.9793\n",
      "Epoch 13/200\n",
      "143614/143614 [==============================] - 213s 1ms/step - loss: 0.0459 - acc: 0.9830 - val_loss: 0.0633 - val_acc: 0.9785\n",
      "bst_val_score 0.05578766349942835\n",
      "153164/153164 [==============================] - 83s 541us/step\n",
      " \n",
      "time taken for this fold 0:48:23.735653\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 243s 2ms/step - loss: 0.1110 - acc: 0.9708 - val_loss: 0.0704 - val_acc: 0.9780\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0685 - acc: 0.9782 - val_loss: 0.0712 - val_acc: 0.9791\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 216s 2ms/step - loss: 0.0618 - acc: 0.9796 - val_loss: 0.0682 - val_acc: 0.9774\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0589 - acc: 0.9802 - val_loss: 0.0726 - val_acc: 0.9800\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0563 - acc: 0.9808 - val_loss: 0.0551 - val_acc: 0.9812\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0541 - acc: 0.9810 - val_loss: 0.0575 - val_acc: 0.9808\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0526 - acc: 0.9814 - val_loss: 0.0609 - val_acc: 0.9799\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0512 - acc: 0.9817 - val_loss: 0.0608 - val_acc: 0.9786\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0500 - acc: 0.9820 - val_loss: 0.0965 - val_acc: 0.9795\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0493 - acc: 0.9821 - val_loss: 0.0548 - val_acc: 0.9816\n",
      "Epoch 11/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0480 - acc: 0.9824 - val_loss: 0.0553 - val_acc: 0.9813\n",
      "Epoch 12/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0473 - acc: 0.9825 - val_loss: 0.0889 - val_acc: 0.9696\n",
      "Epoch 13/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0464 - acc: 0.9829 - val_loss: 0.0937 - val_acc: 0.9791\n",
      "Epoch 14/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0456 - acc: 0.9829 - val_loss: 0.0645 - val_acc: 0.9797\n",
      "Epoch 15/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0448 - acc: 0.9830 - val_loss: 0.0628 - val_acc: 0.9789\n",
      "bst_val_score 0.05478952087061801\n",
      "153164/153164 [==============================] - 84s 546us/step\n",
      " \n",
      "time taken for this fold 0:55:39.237994\n",
      "gpu memory cleaned\n",
      "fold====================>>>>>>>>>> 10\n",
      "Train on 143614 samples, validate on 15957 samples\n",
      "Epoch 1/200\n",
      "143614/143614 [==============================] - 246s 2ms/step - loss: 0.1167 - acc: 0.9702 - val_loss: 0.0845 - val_acc: 0.9732\n",
      "Epoch 2/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0683 - acc: 0.9784 - val_loss: 0.0886 - val_acc: 0.9765\n",
      "Epoch 3/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0617 - acc: 0.9797 - val_loss: 0.0635 - val_acc: 0.9790\n",
      "Epoch 4/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0577 - acc: 0.9805 - val_loss: 0.0642 - val_acc: 0.9789\n",
      "Epoch 5/200\n",
      "143614/143614 [==============================] - 215s 1ms/step - loss: 0.0551 - acc: 0.9811 - val_loss: 0.0650 - val_acc: 0.9790\n",
      "Epoch 6/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0531 - acc: 0.9814 - val_loss: 0.0700 - val_acc: 0.9782\n",
      "Epoch 7/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0514 - acc: 0.9818 - val_loss: 0.0610 - val_acc: 0.9797\n",
      "Epoch 8/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0499 - acc: 0.9821 - val_loss: 0.0668 - val_acc: 0.9792\n",
      "Epoch 9/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0487 - acc: 0.9824 - val_loss: 0.0782 - val_acc: 0.9772\n",
      "Epoch 10/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0475 - acc: 0.9826 - val_loss: 0.0613 - val_acc: 0.9797\n",
      "Epoch 11/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0466 - acc: 0.9830 - val_loss: 0.0624 - val_acc: 0.9790\n",
      "Epoch 12/200\n",
      "143614/143614 [==============================] - 214s 1ms/step - loss: 0.0457 - acc: 0.9832 - val_loss: 0.0683 - val_acc: 0.9776\n",
      "bst_val_score 0.06096092736404656\n",
      "153164/153164 [==============================] - 78s 511us/step\n",
      " \n",
      "time taken for this fold 0:44:50.697559\n"
     ]
    }
   ],
   "source": [
    "y_test=preds(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "statoil",
   "language": "python",
   "name": "statoil"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}