02.TextClassificationWithRNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification - sentence level Attentional RNN\n",
    "    In this post, the model is based on recurrent neural network and attention based LSTM/GRU encoder.\n",
    "    The attention network is implemented on top of LSTM for the classification task."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from collections import defaultdict\n",
    "import re\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import sys\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import Dense, Input, Flatten\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding, Concatenate, Dropout, LSTM, GRU, Bidirectional\n",
    "from keras.models import Model, Sequential\n",
    "\n",
    "from keras.engine.topology import Layer, InputSpec\n",
    "from keras import initializers\n",
    "\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (1) Parameter setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 1000\n",
    "MAX_NB_WORDS = 20000\n",
    "EMBEDDING_DIM = 100\n",
    "VALIDATION_SPLIT = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (2) Data input\n",
    "Data are from [IMDB Dataset](https://www.kaggle.com/c/word2vec-nlp-tutorial/data)\n",
    "All the review of movies are classfied as positive (sentiment = 1) or negative (sentiment = 0)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25000, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5814_8</td>\n",
       "      <td>1</td>\n",
       "      <td>With all this stuff going down at the moment w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2381_9</td>\n",
       "      <td>1</td>\n",
       "      <td>\\The Classic War of the Worlds\\\" by Timothy Hi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7759_3</td>\n",
       "      <td>0</td>\n",
       "      <td>The film starts with a manager (Nicholas Bell)...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  sentiment                                             review\n",
       "0  5814_8          1  With all this stuff going down at the moment w...\n",
       "1  2381_9          1  \\The Classic War of the Worlds\\\" by Timothy Hi...\n",
       "2  7759_3          0  The film starts with a manager (Nicholas Bell)..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train = pd.read_csv('data/labeledTrainData.tsv',sep='\\t')\n",
    "print(data_train.shape)\n",
    "data_train[0:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (3) Data preprocess -- remove some characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_str(string):\n",
    "    \"\"\"\n",
    "    Tokenization/string cleaning for dataset\n",
    "    Every dataset is lower cased except\n",
    "    \"\"\"\n",
    "    string = re.sub(r\"\\\\\",\"\",string)\n",
    "    string = re.sub(r\"\\'\",\"\",string)\n",
    "    string = re.sub(r\"\\\"\",\"\",string)\n",
    "    return string.strip().lower()\n",
    "\n",
    "texts=[]\n",
    "labels=[]\n",
    "\n",
    "for idx in range(data_train.review.shape[0]):\n",
    "    text = BeautifulSoup(data_train.review[idx],'lxml')\n",
    "    texts.append(clean_str(text.get_text()))\n",
    "    labels.append(data_train.sentiment[idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (3) Data preprocess -- data and label\n",
    "Use Keras function to process the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 81501 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "tokenizer=Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(texts)\n",
    "sequences = tokenizer.texts_to_sequences(texts)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sequences is a list of list, which contains 25000 reviews. Each review is a list of its words. Then pads each sequence to the same length."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data tensor:  (25000, 1000)\n",
      "Shape of label tensor:  (25000, 2)\n"
     ]
    }
   ],
   "source": [
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "labels = to_categorical(np.asarray(labels))\n",
    "labels = np.asarray(labels, dtype = np.float32)\n",
    "print('Shape of data tensor: ', data.shape)\n",
    "print('Shape of label tensor: ',labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (3) Data preprocess -- train data and validation data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of negative and positive reviews in training and validation set\n",
      "[  9996.  10004.]\n",
      "[ 2504.  2496.]\n"
     ]
    }
   ],
   "source": [
    "indices = np.arange(data.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "data = data[indices]\n",
    "labels = labels[indices]\n",
    "num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])\n",
    "\n",
    "x_train = data[:-num_validation_samples]\n",
    "y_train = labels[:-num_validation_samples]\n",
    "x_val = data[-num_validation_samples:]\n",
    "y_val = labels[-num_validation_samples:]\n",
    "\n",
    "print ('Number of negative and positive reviews in training and validation set')\n",
    "print(y_train.sum(axis=0))\n",
    "print(y_val.sum(axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (4) Create the BiLSTM model\n",
    "![BiLSTM](https://raw.githubusercontent.com/WistariaDing/TextClassification/master/picture/bi_lstm1.jpg)\n",
    "LSTM only preserves information of the past because the only inputs it has seen are from the past\n",
    "\n",
    "Using bidirectional LSTM will run your inputs in two ways, one from past to future and one from future to past. \n",
    "\n",
    "    The difference between LSTM and BiLSTM is that:\n",
    "    LSTM runs forwards while BiLSTM runs both forwards and backwards which is able to preserve information from both past and future."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In Keras, there are several merge modes, that is how the forward and backward outputs should be combines before being passed on to the next layer.\n",
    "The default mode is to concatenate, and this is the method often used in studies fo bidirectional LSTMs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 400000 word vectors in Glove 6B 100d.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 1000, 100)         8150200   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 200)               160800    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 402       \n",
      "=================================================================\n",
      "Total params: 8,311,402\n",
      "Trainable params: 8,311,402\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "## Use pre-trained wordToVec\n",
    "embeddings_index = {}\n",
    "f=open('data/glove.6B/glove.6B.100d.txt',encoding='utf8')\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:],dtype='float32')\n",
    "    embeddings_index[word]=coefs\n",
    "f.close()\n",
    "\n",
    "print('Total %s word vectors in Glove 6B 100d.' % len(embeddings_index))\n",
    "\n",
    "\n",
    "embedding_matrix = np.random.random((len(word_index)+1,EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i]=embedding_vector\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(len(word_index) +1,\n",
    "                    EMBEDDING_DIM,\n",
    "                    weights = [embedding_matrix],\n",
    "                    input_length=MAX_SEQUENCE_LENGTH,\n",
    "                    trainable = True))\n",
    "model.add(Bidirectional(LSTM(100)))\n",
    "model.add(Dense(2,activation='softmax'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['acc'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (5) Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model fitting - Bidirectional LSTM\n",
      "----------The fitting process is ignored here.\n"
     ]
    }
   ],
   "source": [
    "print(\"model fitting - Bidirectional LSTM\")\n",
    "print(\"----------The fitting process is ignored here.\")\n",
    "model.fit(x_train, y_train, \n",
    "          validation_data=(x_val,y_val),\n",
    "          epochs=10,batch_size=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (6) Attention model\n",
    "    CNTK has no reverse funtion when the return_sequences = True in LSTM. So choose tensorflow as the backend of Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AttLayer(Layer):\n",
    "    def __init__(self, **kwargs):\n",
    "        self.init = initializers.get('normal')\n",
    "        super(AttLayer, self).__init__(** kwargs)\n",
    "    \n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape)==3\n",
    "        self.W = self.init((input_shape[-1],1))\n",
    "        self.trainable_weights=[self.W]\n",
    "        super(AttLayer,self).build(input_shape)\n",
    "    def call(self,x,mask=None):\n",
    "        eij = K.tanh(K.dot(x,self.W))\n",
    "#        print('W shape:',self.W.shape)\n",
    "#        print('eij shape: ', eij.shape)\n",
    "        ai = K.exp(eij)\n",
    "#        print('aij shape: ',ai.shape)\n",
    "        weights = ai/K.sum(ai,axis=1)\n",
    "#        print('weigths shape: ',weights.shape)\n",
    "#        print('x shape: ',x.shape)\n",
    "        weighted_input = x*weights\n",
    "#        print('input shape: ', weighted_input.shape)\n",
    "        output=K.sum(weighted_input,axis=1)\n",
    "        \n",
    "        return output\n",
    "    \n",
    "    def compute_output_shape(self,input_shape):\n",
    "        return (input_shape[0],input_shape[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 1000)              0         \n",
      "_________________________________________________________________\n",
      "embedding_2 (Embedding)      (None, 1000, 100)         8150200   \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 1000, 200)         120600    \n",
      "_________________________________________________________________\n",
      "att_layer_1 (AttLayer)       (None, 200)               200       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 402       \n",
      "=================================================================\n",
      "Total params: 8,271,402\n",
      "Trainable params: 8,271,402\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "embedding_layer = Embedding(len(word_index) +1,\n",
    "                    EMBEDDING_DIM,\n",
    "                    weights = [embedding_matrix],\n",
    "                    input_length=MAX_SEQUENCE_LENGTH,\n",
    "                    trainable=True)\n",
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,),dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "l_gru = Bidirectional(GRU(100,return_sequences=True))(embedded_sequences)\n",
    "l_att = AttLayer()(l_gru)\n",
    "preds = Dense(2, activation='softmax')(l_att)\n",
    "model = Model(sequence_input,preds)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['acc'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model fitting - Attentional LSTM\n",
      "----------The fitting process is ignored here.------------------------\n"
     ]
    }
   ],
   "source": [
    "print(\"model fitting - Attentional LSTM\")\n",
    "print(\"----------The fitting process is ignored here.------------------------\")\n",
    "model.fit(x_train, y_train, \n",
    "          validation_data=(x_val,y_val),\n",
    "          epochs=1,batch_size=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}