jigsaw-toxic-comment-classification-challenge/work/09-bidirectional-GRU-and-GloVe100.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from preprocess import preprocess  # local file. restart kernel if this changed, it won't be re-imported otherwise\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "## Read the first few rows during crude developing:\n",
    "#train = pd.read_csv('../data/train.csv', nrows=10000).fillna(' ')  # train has 159571 comments\n",
    "#test = pd.read_csv('../data/test.csv', nrows=1000).fillna(' ')\n",
    "\n",
    "## These lines load all data:\n",
    "train = pd.read_csv('../data/train.csv').fillna(' ')\n",
    "test = pd.read_csv('../data/test.csv').fillna(' ')\n",
    "\n",
    "[train, test, train_text, test_text, all_text, class_names] = preprocess(train, test)\n",
    "\n",
    "# Remove 80% of the train set's comments that are all-ok to counter the imbalancedness a bit\n",
    "#delete_idx = (train[class_names].sum(axis = 1) == 0) & (np.random.rand(len(train)) > 0.2)\n",
    "#keep_idx = ~delete_idx  # elem-wise NOT for pd.Series\n",
    "#train = train[keep_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# I want a fixed 5000 observations in the test set. Adapt the fraction accordingly:\n",
    "test_fraction = 5000 / len(train)\n",
    "\n",
    "train, dev = train_test_split(train, test_size=test_fraction, random_state=42)\n",
    "# Reset indices because they were shuffled and I need them from 0 to len()\n",
    "train.index = list(range(len(train)))\n",
    "dev.index = list(range(len(dev)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# either:\n",
    "#max_len_train = train['comment_text'].str.split().apply(len).max()\n",
    "#max_len_test = test['comment_text'].str.split().apply(len).max()\n",
    "#max_len = max(max_len_train, max_len_test)\n",
    "\n",
    "# or:\n",
    "max_len = 200  # that might work to have shorter LSTM cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Dense, Input, Dropout, LSTM, GRU, Activation, Bidirectional\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing import sequence\n",
    "from keras.initializers import glorot_uniform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_glove_vecs(glove_file):\n",
    "    with open(glove_file, 'r') as f:\n",
    "        words = set()\n",
    "        word_to_vec_map = {}\n",
    "        for line in f:\n",
    "            line = line.strip().split()\n",
    "            curr_word = line[0]\n",
    "            words.add(curr_word)\n",
    "            word_to_vec_map[curr_word] = np.array(line[1:], dtype=np.float64)\n",
    "        \n",
    "        i = 1\n",
    "        words_to_index = {}\n",
    "        index_to_words = {}\n",
    "        for w in sorted(words):\n",
    "            words_to_index[w] = i\n",
    "            index_to_words[i] = w\n",
    "            i = i + 1\n",
    "    return words_to_index, index_to_words, word_to_vec_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_to_index, index_to_word, word_to_vec_map = read_glove_vecs('../data/glove.6B.100d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def sentences_to_indices(X, word_to_index, max_len):\n",
    "    \"\"\"\n",
    "    Converts an array of sentences (strings) into an array of indices corresponding to words in the sentences.\n",
    "    The output shape should be such that it can be given to `Embedding()` (described in Figure 4). \n",
    "    \n",
    "    Arguments:\n",
    "    X -- array of sentences (strings), of shape (m, 1)\n",
    "    word_to_index -- a dictionary containing the each word mapped to its index\n",
    "    max_len -- maximum number of words in a sentence. You can assume every sentence in X is no longer than this. \n",
    "    \n",
    "    Returns:\n",
    "    X_indices -- array of indices corresponding to words in the sentences from X, of shape (m, max_len)\n",
    "    \"\"\"\n",
    "    \n",
    "    m = X.shape[0]\n",
    "\n",
    "    X_indices = np.zeros((m, max_len))\n",
    "    \n",
    "    for i in range(m):\n",
    "        \n",
    "        # Convert the ith training sentence in lower case and split is into words. You get a list of words.\n",
    "        sentence_words = re.sub(r\"\\W\", \" \", X[i]).lower().split()\n",
    "\n",
    "        n_words = len(sentence_words)\n",
    "        \n",
    "        if n_words > max_len:  # use the *beginning* of the comment if it's too long\n",
    "            sentence_words = sentence_words[0:max_len]\n",
    "            n_words = max_len\n",
    "\n",
    "        ctr = 0\n",
    "        # Loop over the words of sentence_words\n",
    "        for w in sentence_words:\n",
    "            j = max_len - n_words + ctr \n",
    "            \n",
    "            if w in word_to_index:\n",
    "                # pad sequences from the left, i.e. start so the \"last\" word in the comment is the max_len'th entry.\n",
    "                X_indices[i, j] = word_to_index[w]\n",
    "                \n",
    "            ctr += 1\n",
    "            \n",
    "            if j >= max_len:  \n",
    "                # adding this clause, you can have sentences longer than max_len and crop them\n",
    "                break\n",
    "            \n",
    "    return X_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pretrained_embedding_layer(word_to_vec_map, word_to_index):\n",
    "    \"\"\"\n",
    "    Creates a Keras Embedding() layer and loads in pre-trained GloVe 50-dimensional vectors.\n",
    "    \n",
    "    Arguments:\n",
    "    word_to_vec_map -- dictionary mapping words to their GloVe vector representation.\n",
    "    word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)\n",
    "\n",
    "    Returns:\n",
    "    embedding_layer -- pretrained layer Keras instance\n",
    "    \"\"\"\n",
    "    \n",
    "    vocab_len = len(word_to_index) + 1                  # adding 1 to fit Keras embedding (requirement)\n",
    "    emb_dim = word_to_vec_map[\"cucumber\"].shape[0]      # define dimensionality of your GloVe word vectors (e.g. 50)\n",
    "    \n",
    "    # Initialize the embedding matrix as a numpy array of zeros of shape (vocab_len, dimensions of word vectors = emb_dim)\n",
    "    emb_matrix = np.zeros((vocab_len, emb_dim))\n",
    "    \n",
    "    # Set each row \"index\" of the embedding matrix to be the word vector representation of the \"index\"th word of the vocabulary\n",
    "    for word, index in word_to_index.items():\n",
    "        emb_matrix[index, :] = word_to_vec_map[word]\n",
    "\n",
    "    # Define Keras embedding layer with the correct output/input sizes, make it trainable. Use Embedding(...). Make sure to set trainable=False. \n",
    "    embedding_layer = Embedding(vocab_len, emb_dim, trainable = False)#, mask_zero = True)\n",
    "\n",
    "    # Build the embedding layer, it is required before setting the weights of the embedding layer. Do not modify the \"None\".\n",
    "    embedding_layer.build((None,))\n",
    "    \n",
    "    # Set the weights of the embedding layer to the embedding matrix. Your layer is now pretrained.\n",
    "    embedding_layer.set_weights([emb_matrix])\n",
    "    \n",
    "    return embedding_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def myModel(input_shape, word_to_vec_map, word_to_index):\n",
    "    \"\"\"\n",
    "    Function creating the Emojify-v2 model's graph.\n",
    "    \n",
    "    Arguments:\n",
    "    input_shape -- shape of the input, usually (max_len,)\n",
    "    word_to_vec_map -- dictionary mapping every word in a vocabulary into its 50-dimensional vector representation\n",
    "    word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)\n",
    "\n",
    "    Returns:\n",
    "    model -- a model instance in Keras\n",
    "    \"\"\"\n",
    "    \n",
    "    # Define sentence_indices as the input of the graph, it should be of shape input_shape and dtype 'int32' (as it contains indices).\n",
    "    sentence_indices = Input(shape = input_shape, dtype = 'int32')\n",
    "    \n",
    "    # Create the embedding layer pretrained with GloVe Vectors (≈1 line)\n",
    "    embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
    "    \n",
    "    # Propagate sentence_indices through your embedding layer, you get back the embeddings\n",
    "    X = embedding_layer(sentence_indices)   \n",
    "\n",
    "    # Propagate the embeddings through an LSTM layer with 128-dimensional hidden state\n",
    "    # Be careful, the returned output should be a batch of sequences.\n",
    "    #X = LSTM(units = 64, return_sequences = True)(X)\n",
    "    X = Bidirectional(GRU(units = 32, activation = 'relu', return_sequences = True))(X)\n",
    "    # Add dropout with a probability of 0.5\n",
    "    X = Dropout(rate = 0.5)(X)\n",
    "    # Propagate X trough another LSTM layer with 128-dimensional hidden state\n",
    "    # Be careful, the returned output should be a single hidden state, not a batch of sequences.\n",
    "    #X = LSTM(units = 64)(X)\n",
    "    X = GRU(units = 64, activation = 'relu')(X)\n",
    "    # Add dropout with a probability of 0.5\n",
    "    X = Dropout(rate = 0.5)(X)\n",
    "    # Propagate X through a Dense layer with softmax activation to get back a batch of 5-dimensional vectors.\n",
    "    X = Dense(units = 6, activation = 'sigmoid')(X)\n",
    "    # Add a softmax activation\n",
    "    #X = Activation(\"softmax\")(X)\n",
    "    \n",
    "    # Create Model instance which converts sentence_indices into X.\n",
    "    model = Model(inputs = sentence_indices, outputs = X)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 200)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 200, 100)          40000100  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 200, 64)           25536     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 200, 64)           0         \n",
      "_________________________________________________________________\n",
      "gru_2 (GRU)                  (None, 64)                24768     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 6)                 390       \n",
      "=================================================================\n",
      "Total params: 40,050,794\n",
      "Trainable params: 50,694\n",
      "Non-trainable params: 40,000,100\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = myModel((max_len,), word_to_vec_map, word_to_index)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X_train_indices = sentences_to_indices(train['comment_text'], word_to_index, max_len)\n",
    "Y_train = train[class_names].values\n",
    "X_dev_indices = sentences_to_indices(dev['comment_text'], word_to_index, max_len)\n",
    "Y_dev = dev[class_names].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# If you want gradient clipping, put optimizer=adam_gradclip in the model.compile() call\n",
    "\n",
    "#from keras import optimizers\n",
    "#\n",
    "#adam_gradclip = optimizers.Adam(clipnorm = 1, clipvalue = 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model.compile(loss = 'categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# if you want to save a model's configuration and weights, and train it further tomorrow:\n",
    "\n",
    "#from keras.models import load_model\n",
    "#model = load_model(\"09-model-glove100.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss history:\n",
    "\n",
    "### Glove 100 w/o dropout:\n",
    "\n",
    "#### Epoch 1\n",
    "\n",
    "Epoch 1/1\n",
    "154571/154571 [==============================] - 950s 6ms/step - loss: 0.0585 - acc: 0.9797\n",
    "CPU times: user 1h 8min 49s, sys: 16min 9s, total: 1h 24min 59s\n",
    "Wall time: 15min 51s\n",
    "\n",
    "#### Epoch 2\n",
    "\n",
    "Epoch 1/1\n",
    "154571/154571 [==============================] - 938s 6ms/step - loss: 0.0454 - acc: 0.9828\n",
    "CPU times: user 1h 7min 34s, sys: 16min 17s, total: 1h 23min 51s\n",
    "Wall time: 15min 38s\n",
    "\n",
    "### Glove 100 w/ dropout:\n",
    "\n",
    "#### Epoch 1 (batch size 1024)\n",
    "\n",
    "Epoch 1/1\n",
    "154571/154571 [==============================] - 1291s 8ms/step - loss: 0.1799 - acc: 0.9402\n",
    "CPU times: user 1h 17min 50s, sys: 33min 57s, total: 1h 51min 48s\n",
    "Wall time: 21min 33s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "154571/154571 [==============================] - 1232s 8ms/step - loss: 0.1320 - acc: 0.9587\n",
      "Epoch 2/2\n",
      " 67584/154571 [============>.................] - ETA: 19:13 - loss: 0.0655 - acc: 0.9781"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "model.fit(X_train_indices, Y_train, epochs = 2, batch_size = 512, shuffle=True)\n",
    "model.save(\"09-model-glove100.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "m = 5000\n",
    "\n",
    "pred_train = model.predict(X_train_indices[0:m, :])  # train performance on first 5k samples only...\n",
    "aucs = np.zeros((len(class_names)))\n",
    "\n",
    "for y_i in range(len(class_names)):\n",
    "    y = Y_train[0:m, y_i]\n",
    "    yhat = pred_train[0:m, y_i]\n",
    "    fpr, tpr, thresholds = metrics.roc_curve(y, yhat, pos_label = 1)\n",
    "    this_auc = metrics.auc(fpr, tpr)\n",
    "    aucs[y_i] = this_auc\n",
    "    \n",
    "np.mean(aucs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Glove 100 w/ dropout (batch size 1024):\n",
    "\n",
    "- 1 epoch: 0.9418021863936903\n",
    "\n",
    "Glove 100 w/o dropout:\n",
    "\n",
    "- 1 epoch: 0.9855825964913917\n",
    "- 2: 0.9883302986173689"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute dev set mean AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "\n",
    "# model.evaluate(X_dev_indices, Y_dev, batch_size = len(train))\n",
    "pred_dev = model.predict(X_dev_indices)\n",
    "\n",
    "aucs = np.zeros((len(class_names)))\n",
    "\n",
    "for y_i in range(len(class_names)):\n",
    "    y = Y_dev[:, y_i]\n",
    "    yhat = pred_dev[:, y_i]\n",
    "    fpr, tpr, thresholds = metrics.roc_curve(y, yhat, pos_label = 1)\n",
    "    this_auc = metrics.auc(fpr, tpr)\n",
    "    aucs[y_i] = this_auc\n",
    "    \n",
    "np.mean(aucs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Glove 100 w/ dropout (batch size 1024):\n",
    "\n",
    "- 1 epoch: 0.9327199504007933\n",
    "\n",
    "Glove 100 w/o dropout:\n",
    "\n",
    "- 1 epoch: 0.9832965881854069\n",
    "- 2: 0.986526510845655  # 0.9724 on kaggle\n",
    "\n",
    "Glove 50:\n",
    "\n",
    "- 1 epoch: 0.9760979734073979\n",
    "- 2: 0.9833688361886418  # 0.9726 on kaggle\n",
    "- 3: 0.9849738642393575  # 0.975\n",
    "- 4: 0.9856226244722991  # 0.9755\n",
    "- 5: 0.9869049877287438  # 0.9762\n",
    "- 6: 0.9860198630049598\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sanity check train predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = model.predict(X_train_indices[0:7, :])  # the 7th comment (in unshuffled train) is a very toxic doggo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6.5171067e-03, 6.5184315e-05, 9.1334747e-04, 1.2875482e-04,\n",
       "        7.1950658e-04, 1.6628247e-04],\n",
       "       [5.0020735e-03, 4.8525042e-05, 7.0083264e-04, 7.4038879e-05,\n",
       "        6.8845449e-04, 1.5631037e-04],\n",
       "       [3.9328416e-03, 4.6791298e-05, 7.2026328e-04, 8.8051842e-05,\n",
       "        5.9658178e-04, 1.3134754e-04],\n",
       "       [5.8707823e-03, 9.0390487e-05, 8.9317624e-04, 1.4250874e-04,\n",
       "        9.9516171e-04, 1.5783354e-04],\n",
       "       [5.2551061e-02, 2.2301429e-03, 1.5775675e-02, 3.2121548e-03,\n",
       "        1.6967883e-02, 2.7074148e-03],\n",
       "       [3.8935333e-03, 5.3284726e-05, 6.1131467e-04, 9.4763054e-05,\n",
       "        5.8495841e-04, 1.0770443e-04],\n",
       "       [7.8439724e-01, 6.9102101e-02, 4.8785472e-01, 5.9137352e-02,\n",
       "        6.1871827e-01, 4.9431138e-02]], dtype=float32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create submission csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_indices = sentences_to_indices(test['comment_text'], word_to_index, max_len)\n",
    "pred = model.predict(X_test_indices)\n",
    "submit = pd.DataFrame(test['id'].values, columns=['id'])\n",
    "preds = pd.DataFrame(pred, columns=class_names)\n",
    "submit = pd.concat([submit, preds], axis = 1)\n",
    "#submit\n",
    "submit.to_csv('../data/submission_09_glove100.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "3 + 3"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}