sentiment analysis in pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<spacy.lang.en.English at 0x7f752cfab748>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#importing the required libraries\n",
    "import torch\n",
    "import torchtext\n",
    "import spacy\n",
    "spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#arguments specify the processing of data like which type of tokenization to be used etc.\n",
    "TEXT = torchtext.data.Field(lower=True,tokenize='spacy')\n",
    "Label = torchtext.data.LabelField(tensor_type=torch.FloatTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train , test = torchtext.datasets.IMDB.splits(TEXT,Label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000 25000\n"
     ]
    }
   ],
   "source": [
    "print(len(train),len(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I am reducing the size of my train set that I will use because of non-availability of GPU and time constraint\n",
    "train,unused_train=train.split(split_ratio = 0.1)\n",
    "test,unused_test = test.split(split_ratio = 0.04)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2500 1000\n"
     ]
    }
   ],
   "source": [
    "print(len(train),len(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now we will create the validation set and for that we will use test set, not train set\n",
    "train,validation = train.split(split_ratio=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2000 1000 500\n"
     ]
    }
   ],
   "source": [
    "print(len(train),len(test),len(validation))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#building the vocabulary/ the lookup table for our words and using the pretrained word vector embeddings(glove model)\n",
    "#keep in mind the ram access of your python version to avoid memory error\n",
    "TEXT.build_vocab(train,vectors='glove.6B.100d',max_size=25000)\n",
    "Label.build_vocab(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create iterators with batches to feed into the network for training,validation,testing \n",
    "#using bucket iterators which puts examples of similar length into same bucket means by the amount of <pad> required for them\n",
    "batch_size = 8\n",
    "\n",
    "train_iter, valid_iter, test_iter = torchtext.data.BucketIterator.splits((train, validation, test), batch_size=batch_size, sort_key=lambda x: len(x.text), repeat=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now define the model, here we will use LSTM which is a modified version of RNN as well as dropout as our regularizer\n",
    "#LSTM will be bidirectional \n",
    "\n",
    "class lstm_arch(torch.nn.Module):\n",
    "    \n",
    "    def __init__(self,embed_dim,vocab_size,hid_dim,out_dim,dropout_prob): \n",
    "        super().__init__()\n",
    "        self.embedding = torch.nn.Embedding(vocab_size,embed_dim)           #prepare the lookup table for word embeddings\n",
    "        self.rnn = torch.nn.LSTM(embed_dim,hid_dim,bias=True,num_layers=2,bidirectional=True,dropout=dropout_prob)  #LSTM 2 layered and bidirectional\n",
    "        self.fc = torch.nn.Linear(hid_dim*2,out_dim)          #fully connected layer for output\n",
    "        self.dropout = torch.nn.Dropout(p = dropout_prob)\n",
    "    \n",
    "    def forward(self,feed_data):\n",
    "        embed_out = self.dropout(self.embedding(feed_data))\n",
    "        rnn_out,(rnn_hid,rnn_cell) = self.rnn(embed_out)\n",
    "        hidden = self.dropout(torch.cat((rnn_hid[-2,:,:], rnn_hid[-1,:,:]), dim=1))\n",
    "        return self.fc(hidden.squeeze(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating an instance of our lstm_arch class or the specific model to train and test\n",
    "model = lstm_arch(vocab_size=len(TEXT.vocab),embed_dim=100,hid_dim=256,out_dim=1,dropout_prob=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "        [-0.0382, -0.2449,  0.7281,  ..., -0.1459,  0.8278,  0.2706],\n",
       "        ...,\n",
       "        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "        [ 0.0156,  0.3192,  0.2856,  ...,  0.5062, -0.0928,  0.5417],\n",
       "        [ 0.4981, -0.6555,  0.1981,  ...,  0.4539, -0.6788,  0.2721]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#now loading the values of pretrained embeddings into the embedding layer of our model\n",
    "pretrained = TEXT.vocab.vectors\n",
    "model.embedding.weight.data.copy_(pretrained)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(),betas=(0.7,0.995),lr=0.005)  #optimizer for our model used adam here\n",
    "criterion = torch.nn.BCEWithLogitsLoss()                   #criterion to evaluate loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function to train our model we calculate both loss and accuracy of the model\n",
    "def train(model, iterator, optimizer, criterion,epoch):\n",
    "    \n",
    "    model.train()\n",
    "    for i in range(epoch):\n",
    "        epoch_loss = 0\n",
    "        epoch_acc = 0\n",
    "        for batch in iterator:\n",
    "        \n",
    "            optimizer.zero_grad()\n",
    "        \n",
    "            predictions = model(batch.text).squeeze(1)\n",
    "        \n",
    "            loss = criterion(predictions, batch.label)\n",
    "        \n",
    "            rounded_predictions = torch.round(torch.nn.functional.sigmoid(predictions))\n",
    "            correct = (rounded_predictions == batch.label).float() \n",
    "            acc = correct.sum()/len(correct)\n",
    "        \n",
    "            loss.backward()\n",
    "        \n",
    "            optimizer.step()\n",
    "        \n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "        print('loss: ',epoch_loss / len(iterator),' accuracy: ' ,epoch_acc / len(iterator))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function for evaluation of model's performance\n",
    "def evaluate(model, iterator, optimizer, criterion,epoch):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for i in range(epoch):\n",
    "            epoch_loss = 0\n",
    "            epoch_acc = 0\n",
    "            for batch in iterator:\n",
    "\n",
    "                predictions = model(batch.text).squeeze(1)\n",
    "                loss = criterion(predictions, batch.label)\n",
    "            \n",
    "                rounded_predictions = torch.round(torch.nn.functional.sigmoid(predictions))\n",
    "                correct = (rounded_predictions == batch.label).float() \n",
    "                acc = correct.sum()/len(correct)\n",
    "        \n",
    "                epoch_loss += loss.item()\n",
    "                epoch_acc += acc.item()\n",
    "        \n",
    "            print('loss: ',epoch_loss / len(iterator),' accuracy: ' ,epoch_acc / len(iterator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 :\n",
      "loss:  0.716819109916687  accuracy:  0.51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ujjawal/anaconda3/lib/python3.6/site-packages/torchtext/data/field.py:322: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  return Variable(arr, volatile=not train)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:  0.6988581352763705  accuracy:  0.47023809523809523\n",
      "\n",
      "\n",
      "epoch 2 :\n",
      "loss:  0.7548297755718231  accuracy:  0.4965\n",
      "loss:  0.6881475155315702  accuracy:  0.5376984126984127\n",
      "\n",
      "\n",
      "epoch 3 :\n",
      "loss:  0.7458459858894348  accuracy:  0.542\n",
      "loss:  0.7193611689976284  accuracy:  0.501984126984127\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#here losses are a bit high and accuracy is low compared to the one where we train the model with 25000 training examples and validate on 5000 examples taken from test set and finally we evaluate on 20000 test examples\n",
    "for i in range(3):\n",
    "    print('epoch %d :'%(i+1))\n",
    "    train(model,train_iter,optimizer,criterion,1)\n",
    "    evaluate(model,valid_iter,optimizer,criterion,1)\n",
    "    print('\\n')\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test results: \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ujjawal/anaconda3/lib/python3.6/site-packages/torchtext/data/field.py:322: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  return Variable(arr, volatile=not train)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:  0.7121250720024109  accuracy:  0.533\n"
     ]
    }
   ],
   "source": [
    "#checking the results on the test set we can see that our model generalizes well enough just for more accuracy use all data\n",
    "print('test results: ')\n",
    "evaluate(model,test_iter,optimizer,criterion,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}