toxic-comment-classifier-dnn-keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Toxic Comment Classifier DNN \n",
    "\n",
    "This notebook will focus on the use of Deep Neural Networks to tackle the problem of tox comment classification. Starting from the work done in the `toxic-comment-classifier-classical-model.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(159571, 8)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "from  sklearn.model_selection import train_test_split\n",
    "from keras.utils import plot_model\n",
    "\n",
    "data = pd.read_csv('./data/train.csv')\n",
    "print(data.shape)\n",
    "X = data['comment_text'].values\n",
    "y = data[data.columns[2:]].values\n",
    "\n",
    "X_train, X_test, y_train, y_test =  train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "tokenizer = Tokenizer()\n",
    "\n",
    "tokenizer.fit_on_texts(X_train)\n",
    "tokenized_train = tokenizer.texts_to_sequences(X_train)\n",
    "tokenized_test = tokenizer.texts_to_sequences(X_test)\n",
    "word_index = tokenizer.word_index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grandma Terri Should Burn in Trash \n",
      "Grandma Terri is trash. I hate Grandma Terri. F%%K her to HELL! 71.74.76.40\n",
      "[12927, 8296, 56, 3980, 10, 4414, 12927, 8296, 8, 4414, 7, 398, 12927, 8296, 871, 1369, 184, 2, 866, 1697, 2609, 1738, 1336]\n",
      "[12927  8296    56  3980    10  4414 12927  8296     8  4414     7   398\n",
      " 12927  8296   871  1369   184     2   866  1697  2609  1738  1336     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0]\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "max_len = 70\n",
    "\n",
    "padded_train = pad_sequences(tokenized_train, maxlen=max_len, padding='post')\n",
    "padded_test = pad_sequences(tokenized_test, maxlen=max_len, padding='post')\n",
    "print(X_train[0])\n",
    "print(tokenized_train[0])\n",
    "print(padded_train[0])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We're going to use a pretrained Word2Vector as the basis of our embedded vocabulary. The pretrained Word2Vec model is going to be Facebook's [fastText](https://s3-us-west-1.amazonaws.com/fasttext-vectors/wiki.en.vec). For more info please find it [here](https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "embedding_dim = 300\n",
    "\n",
    "def process_pretrained_word_vec(line):\n",
    "    values = line.rstrip().rsplit(' ', embedding_dim)\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    return (word, coefs)\n",
    "    \n",
    "with open('./wiki.en.vec', encoding='utf8') as f:\n",
    "    embedding = dict(map(process_pretrained_word_vec, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = np.zeros((len(word_index) + 1, embedding_dim))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embedding.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [-0.065334  , -0.093031  , -0.017571  , ...,  0.16642   ,\n",
       "        -0.13079   ,  0.035397  ],\n",
       "       [-0.21341   ,  0.15353   ,  0.05288   , ..., -0.025937  ,\n",
       "        -0.072507  ,  0.14989001],\n",
       "       ...,\n",
       "       [ 0.20563   ,  0.18877   , -0.61066997, ...,  0.43869999,\n",
       "        -0.19874001,  0.32304999],\n",
       "       [-0.25375   , -0.24808   , -0.17106   , ...,  0.28101999,\n",
       "         0.30978999,  0.233     ],\n",
       "       [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "         0.        ,  0.        ]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_9 (Embedding)      (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_9 (MaxPooling1 (None, 23, 512)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_9 (Glob (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 50)                25650     \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 55,755,916\n",
      "Trainable params: 55,754,892\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n",
      "None\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_10 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_13 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "conv1d_14 (Conv1D)           (None, 70, 256)           655616    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_10 (MaxPooling (None, 23, 256)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_10 (Glo (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 50)                12850     \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 56,397,708\n",
      "Trainable params: 56,397,196\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n",
      "None\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_11 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_15 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "conv1d_16 (Conv1D)           (None, 70, 256)           655616    \n",
      "_________________________________________________________________\n",
      "conv1d_17 (Conv1D)           (None, 70, 64)            81984     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_11 (MaxPooling (None, 23, 64)            0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_11 (Glo (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 64)                256       \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 50)                3250      \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 56,469,324\n",
      "Trainable params: 56,469,196\n",
      "Non-trainable params: 128\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, Conv1D, MaxPooling1D, GlobalMaxPooling1D, BatchNormalization, Dense, Dropout\n",
    "from keras.preprocessing.sequence import TimeseriesGenerator\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "cnn_models = []\n",
    "cnn_params = [\n",
    "    {\n",
    "        'cnn1D_layers_filters': [512]\n",
    "    },\n",
    "    {\n",
    "        'cnn1D_layers_filters': [512, 256]\n",
    "    },\n",
    "    {\n",
    "        'cnn1D_layers_filters': [512, 256, 64]\n",
    "    }\n",
    "]\n",
    "for cnn_param in cnn_params:\n",
    "    \n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(Embedding(len(embedding_matrix),\n",
    "                        embedding_dim, weights=[embedding_matrix],\n",
    "                        input_length=max_len, trainable=True)\n",
    "             )\n",
    "\n",
    "    # Add Convolutional layer(s)\n",
    "    for filters in cnn_param['cnn1D_layers_filters']:\n",
    "        model.add(Conv1D(filters=filters, kernel_size=5, padding='same', activation='relu'))\n",
    "        \n",
    "    model.add(MaxPooling1D(3))\n",
    "    model.add(GlobalMaxPooling1D())\n",
    "    model.add(BatchNormalization())\n",
    "    # Add fully connected layers\n",
    "    model.add(Dense(50, activation='relu'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(6, activation='sigmoid'))\n",
    "\n",
    "    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    print(model.summary())\n",
    "    cnn_models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_9 (Embedding)      (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_9 (MaxPooling1 (None, 23, 512)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_9 (Glob (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 50)                25650     \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 55,755,916\n",
      "Trainable params: 55,754,892\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/1\n",
      "127656/127656 [==============================] - 66s 513us/step - loss: 0.0386 - acc: 0.9849\n",
      "0.9821650781544098\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_9 (Embedding)      (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_9 (MaxPooling1 (None, 23, 512)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_9 (Glob (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 50)                25650     \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 55,755,916\n",
      "Trainable params: 55,754,892\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/2\n",
      "127656/127656 [==============================] - 66s 514us/step - loss: 0.0297 - acc: 0.9881\n",
      "Epoch 2/2\n",
      "127656/127656 [==============================] - 66s 513us/step - loss: 0.0232 - acc: 0.9907\n",
      "0.9766720735209405\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_9 (Embedding)      (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_9 (MaxPooling1 (None, 23, 512)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_9 (Glob (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 50)                25650     \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 55,755,916\n",
      "Trainable params: 55,754,892\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/5\n",
      "127656/127656 [==============================] - 66s 514us/step - loss: 0.0183 - acc: 0.9928\n",
      "Epoch 2/5\n",
      "127656/127656 [==============================] - 66s 514us/step - loss: 0.0151 - acc: 0.9942\n",
      "Epoch 3/5\n",
      "127656/127656 [==============================] - 66s 514us/step - loss: 0.0129 - acc: 0.9951\n",
      "Epoch 4/5\n",
      "127656/127656 [==============================] - 65s 513us/step - loss: 0.0114 - acc: 0.9958\n",
      "Epoch 5/5\n",
      "127656/127656 [==============================] - 65s 512us/step - loss: 0.0098 - acc: 0.9964\n",
      "0.9716734617442241\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_10 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_13 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "conv1d_14 (Conv1D)           (None, 70, 256)           655616    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_10 (MaxPooling (None, 23, 256)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_10 (Glo (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 50)                12850     \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 56,397,708\n",
      "Trainable params: 56,397,196\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/1\n",
      "127656/127656 [==============================] - 73s 571us/step - loss: 0.0665 - acc: 0.9764\n",
      "0.9800540408174648\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_10 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_13 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "conv1d_14 (Conv1D)           (None, 70, 256)           655616    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_10 (MaxPooling (None, 23, 256)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_10 (Glo (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 50)                12850     \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 56,397,708\n",
      "Trainable params: 56,397,196\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "127656/127656 [==============================] - 71s 560us/step - loss: 0.0454 - acc: 0.9828\n",
      "Epoch 2/2\n",
      "127656/127656 [==============================] - 71s 560us/step - loss: 0.0375 - acc: 0.9853\n",
      "0.9811202454045903\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_10 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_13 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "conv1d_14 (Conv1D)           (None, 70, 256)           655616    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_10 (MaxPooling (None, 23, 256)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_10 (Glo (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 50)                12850     \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 56,397,708\n",
      "Trainable params: 56,397,196\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/5\n",
      "127656/127656 [==============================] - 72s 560us/step - loss: 0.0303 - acc: 0.9877\n",
      "Epoch 2/5\n",
      "127656/127656 [==============================] - 72s 560us/step - loss: 0.0255 - acc: 0.9899\n",
      "Epoch 3/5\n",
      "127656/127656 [==============================] - 72s 560us/step - loss: 0.0215 - acc: 0.9915\n",
      "Epoch 4/5\n",
      "127656/127656 [==============================] - 72s 561us/step - loss: 0.0181 - acc: 0.9928\n",
      "Epoch 5/5\n",
      "127656/127656 [==============================] - 72s 562us/step - loss: 0.0153 - acc: 0.9941\n",
      "0.9743238867162961\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_11 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_15 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "conv1d_16 (Conv1D)           (None, 70, 256)           655616    \n",
      "_________________________________________________________________\n",
      "conv1d_17 (Conv1D)           (None, 70, 64)            81984     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_11 (MaxPooling (None, 23, 64)            0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_11 (Glo (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 64)                256       \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 50)                3250      \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 56,469,324\n",
      "Trainable params: 56,469,196\n",
      "Non-trainable params: 128\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/1\n",
      "127656/127656 [==============================] - 76s 595us/step - loss: 0.0661 - acc: 0.9774\n",
      "0.9717092070622764\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_11 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_15 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "conv1d_16 (Conv1D)           (None, 70, 256)           655616    \n",
      "_________________________________________________________________\n",
      "conv1d_17 (Conv1D)           (None, 70, 64)            81984     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_11 (MaxPooling (None, 23, 64)            0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_11 (Glo (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 64)                256       \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 50)                3250      \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 56,469,324\n",
      "Trainable params: 56,469,196\n",
      "Non-trainable params: 128\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/2\n",
      "127656/127656 [==============================] - 74s 582us/step - loss: 0.0475 - acc: 0.9823\n",
      "Epoch 2/2\n",
      "127656/127656 [==============================] - 74s 580us/step - loss: 0.0395 - acc: 0.9847\n",
      "0.9774988142661806\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_11 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "conv1d_15 (Conv1D)           (None, 70, 512)           768512    \n",
      "_________________________________________________________________\n",
      "conv1d_16 (Conv1D)           (None, 70, 256)           655616    \n",
      "_________________________________________________________________\n",
      "conv1d_17 (Conv1D)           (None, 70, 64)            81984     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_11 (MaxPooling (None, 23, 64)            0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_11 (Glo (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 64)                256       \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 50)                3250      \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 56,469,324\n",
      "Trainable params: 56,469,196\n",
      "Non-trainable params: 128\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/5\n",
      "127656/127656 [==============================] - 74s 579us/step - loss: 0.0333 - acc: 0.9869\n",
      "Epoch 2/5\n",
      "127656/127656 [==============================] - 75s 587us/step - loss: 0.0289 - acc: 0.9885\n",
      "Epoch 3/5\n",
      "127656/127656 [==============================] - 75s 586us/step - loss: 0.0253 - acc: 0.9900\n",
      "Epoch 4/5\n",
      "127656/127656 [==============================] - 74s 578us/step - loss: 0.0221 - acc: 0.9914\n",
      "Epoch 5/5\n",
      "127656/127656 [==============================] - 74s 579us/step - loss: 0.0196 - acc: 0.9924\n",
      "0.9722849820657183\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "epochs = [1,2,5]\n",
    "cnn_trained_models = []\n",
    "for model in cnn_models:\n",
    "    for epoch in epochs:\n",
    "        print(model.summary())\n",
    "        model.fit(padded_train, y_train, epochs=epoch)\n",
    "        y_pred = model.predict(padded_test)\n",
    "        print(roc_auc_score(y_test, y_pred))\n",
    "        cnn_trained_models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def d_zip(*args):\n",
    "    x, y, z = args\n",
    "    d = round(len(x) / len(y))\n",
    "    return list(zip(x, y*d, z*d))\n",
    "    \n",
    "d_zip([1,2,3], ['a'], ['x','y'])\n",
    "cnn_scores = d_zip([0.9821650781544098,\n",
    " 0.9766720735209405,\n",
    " 0.9716734617442241,\n",
    " 0.9800540408174648,\n",
    " 0.9811202454045903,\n",
    " 0.9743238867162961,\n",
    " 0.9774988142661806,\n",
    " 0.9722849820657183],epochs, cnn_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_model(cnn_models[0], show_shapes=True, to_file='model_cnn.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.8660351 , 0.0452344 , 0.05817014, 0.9524812 , 0.03342581,\n",
       "        0.02150893]], dtype=float32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pad_sequences(tokenizer.texts_to_sequences([\"I will kill you \"]), maxlen=max_len, padding='post')\n",
    "model.predict([test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.9821650781544098, 1, {'cnn1D_layers_filters': [512]}),\n",
       " (0.9811202454045903, 2, {'cnn1D_layers_filters': [512, 256]}),\n",
       " (0.9800540408174648, 1, {'cnn1D_layers_filters': [512]}),\n",
       " (0.9774988142661806, 1, {'cnn1D_layers_filters': [512]}),\n",
       " (0.9766720735209405, 2, {'cnn1D_layers_filters': [512, 256]}),\n",
       " (0.9743238867162961, 5, {'cnn1D_layers_filters': [512, 256, 64]}),\n",
       " (0.9722849820657183, 2, {'cnn1D_layers_filters': [512, 256]}),\n",
       " (0.9716734617442241, 5, {'cnn1D_layers_filters': [512, 256, 64]})]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(cnn_scores, key=lambda x: x[0], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.981859497387775\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "y_pred = model.predict(padded_test)\n",
    "print(roc_auc_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0.9822522540504516"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_12 (Embedding)     (None, 70, 300)           54959400  \n",
      "_________________________________________________________________\n",
      "lstm_layer (LSTM)            (None, 70, 60)            86640     \n",
      "_________________________________________________________________\n",
      "conv1d_18 (Conv1D)           (None, 70, 512)           154112    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_12 (MaxPooling (None, 23, 512)           0         \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_12 (Glo (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_12 (Batc (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 50)                25650     \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 6)                 306       \n",
      "=================================================================\n",
      "Total params: 55,228,156\n",
      "Trainable params: 55,227,132\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import LSTM\n",
    "model_rnn = Sequential()\n",
    "\n",
    "model_rnn.add(Embedding(len(embedding_matrix),\n",
    "                    embedding_dim, weights=[embedding_matrix],\n",
    "                    input_length=max_len, trainable=True)\n",
    "         )\n",
    "\n",
    "model_rnn.add(LSTM(60, return_sequences=True, name='lstm_layer'))\n",
    "model_rnn.add(Conv1D(filters=512, kernel_size=5, padding='same', activation='relu'))\n",
    "model_rnn.add(MaxPooling1D(3))\n",
    "model_rnn.add(GlobalMaxPooling1D())\n",
    "model_rnn.add(BatchNormalization())\n",
    "# Add fully connected layers\n",
    "model_rnn.add(Dense(50, activation='relu'))\n",
    "model_rnn.add(Dropout(0.3))\n",
    "model_rnn.add(Dense(6, activation='sigmoid'))\n",
    "\n",
    "model_rnn.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "print(model_rnn.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_model(model_rnn, show_shapes=True, to_file='model_rnn.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "127656/127656 [==============================] - 392s 3ms/step - loss: 0.0652 - acc: 0.9771\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f2bcd75beb8>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_rnn.fit(padded_train, y_train, epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9828070415809732\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "y_pred = model_rnn.predict(padded_test)\n",
    "print(roc_auc_score(y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}