sentiment_analyzer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Max/anaconda3/envs/allennlp/lib/python3.6/site-packages/sklearn/utils/linear_assignment_.py:21: DeprecationWarning: The linear_assignment_ module is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import allennlp\n",
    "from allennlp.data.dataset_readers import StanfordSentimentTreeBankDatasetReader \n",
    "import os\n",
    "import torch\n",
    "import pdb\n",
    "from typing import Dict\n",
    "import torch.optim as optim\n",
    "from allennlp.training.trainer import Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = '/Users/Max/Downloads/trees/'\n",
    "EMBEDDING_DIM = 300\n",
    "HIDDEN_DIM = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8544it [00:01, 6507.61it/s]\n",
      "1101it [00:00, 4411.34it/s]\n"
     ]
    }
   ],
   "source": [
    "reader = StanfordSentimentTreeBankDatasetReader()\n",
    "train_ds = reader.read(os.path.join(input_dir, 'train.txt'))\n",
    "dev_ds = reader.read(os.path.join(input_dir, 'dev.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## initialize embeddings and indexer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9645/9645 [00:00<00:00, 79603.42it/s]\n"
     ]
    }
   ],
   "source": [
    "from allennlp.data.vocabulary import Vocabulary \n",
    "vocab = Vocabulary.from_instances(train_ds + dev_ds, min_count = {'tokens':3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.modules.token_embedders.embedding import Embedding\n",
    "from allennlp.modules.text_field_embedders.basic_text_field_embedder import BasicTextFieldEmbedder\n",
    "token_embedding = Embedding(num_embeddings = vocab.get_vocab_size('tokens'), embedding_dim = 300)\n",
    "\n",
    "word_embedder = BasicTextFieldEmbedder({'tokens':token_embedding})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.models import Model\n",
    "from allennlp.modules.seq2vec_encoders import Seq2VecEncoder, PytorchSeq2VecWrapper\n",
    "from allennlp.training.metrics.categorical_accuracy import CategoricalAccuracy\n",
    "from allennlp.nn.util import get_text_field_mask\n",
    "\n",
    "#@Model.register(\"lstm_classifier\")\n",
    "class LstmClassifier(Model):\n",
    "    def __init__(self,\n",
    "                 embedder: BasicTextFieldEmbedder,\n",
    "                 encoder: Seq2VecEncoder,\n",
    "                 vocab: Vocabulary) -> None:\n",
    "        super().__init__(vocab)\n",
    "        \n",
    "        self.word_embedder = embedder\n",
    "        self.vocab = vocab\n",
    "        self.encoder = encoder\n",
    "        \n",
    "        self.hidden_to_tag = torch.nn.Linear(in_features = self.encoder.get_output_dim(), out_features = self.vocab.get_vocab_size('labels'))\n",
    "        self.loss_function = torch.nn.CrossEntropyLoss()\n",
    "        self.accuracy = CategoricalAccuracy()\n",
    "        \n",
    "    def forward(self,\n",
    "                tokens: Dict[str, torch.Tensor],\n",
    "                label: torch.Tensor = None) -> torch.Tensor:\n",
    "        \n",
    "        mask = get_text_field_mask(tokens)\n",
    "        embedded = self.word_embedder(tokens)\n",
    "        encoded = self.encoder(embedded, mask)\n",
    "        logits = self.hidden_to_tag(encoded)\n",
    "        \n",
    "        output = {'logits': logits}\n",
    "        if label is not None:\n",
    "            self.accuracy(logits, label)\n",
    "            output['loss'] = self.loss_function(logits, label)\n",
    "        return output\n",
    "    \n",
    "    def get_metrics(self, reset = False) -> Dict[str, float]:\n",
    "        return {'accuracy': self.accuracy.get_metric(reset)}\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## init model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_encoder = PytorchSeq2VecWrapper(torch.nn.LSTM(EMBEDDING_DIM, HIDDEN_DIM, batch_first=True))\n",
    "model = LstmClassifier(word_embedder, lstm_encoder, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize the trainer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr = 1e-4, weight_decay = 1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.data.iterators import BucketIterator\n",
    "iterator = BucketIterator(batch_size = 32, sorting_keys=[(\"tokens\", \"num_tokens\")])\n",
    "iterator.index_with(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(model=model,\n",
    "                  optimizer=optimizer,\n",
    "                  iterator=iterator,\n",
    "                  train_dataset=train_ds,\n",
    "                  validation_dataset=dev_ds,\n",
    "                  patience=10,\n",
    "                  num_epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "accuracy: 0.2666, loss: 1.5767 ||: 100%|██████████| 267/267 [00:22<00:00, 11.73it/s]\n",
      "accuracy: 0.2579, loss: 1.5715 ||: 100%|██████████| 35/35 [00:00<00:00, 65.49it/s]\n",
      "accuracy: 0.2782, loss: 1.5632 ||: 100%|██████████| 267/267 [00:22<00:00, 11.81it/s]\n",
      "accuracy: 0.2616, loss: 1.5686 ||: 100%|██████████| 35/35 [00:00<00:00, 73.22it/s]\n",
      "accuracy: 0.3164, loss: 1.5293 ||: 100%|██████████| 267/267 [00:21<00:00, 12.39it/s]\n",
      "accuracy: 0.3406, loss: 1.4824 ||: 100%|██████████| 35/35 [00:00<00:00, 70.10it/s]\n",
      "accuracy: 0.4002, loss: 1.3745 ||: 100%|██████████| 267/267 [00:22<00:00, 12.13it/s]\n",
      "accuracy: 0.3678, loss: 1.4006 ||: 100%|██████████| 35/35 [00:00<00:00, 58.45it/s]\n",
      "accuracy: 0.4930, loss: 1.1875 ||: 100%|██████████| 267/267 [00:22<00:00, 12.05it/s]\n",
      "accuracy: 0.3869, loss: 1.4110 ||: 100%|██████████| 35/35 [00:00<00:00, 51.34it/s]\n",
      "accuracy: 0.5679, loss: 1.0363 ||: 100%|██████████| 267/267 [00:23<00:00, 11.55it/s]\n",
      "accuracy: 0.3806, loss: 1.4598 ||: 100%|██████████| 35/35 [00:00<00:00, 70.49it/s]\n",
      "accuracy: 0.6246, loss: 0.9214 ||: 100%|██████████| 267/267 [00:22<00:00, 11.76it/s]\n",
      "accuracy: 0.3933, loss: 1.5078 ||: 100%|██████████| 35/35 [00:00<00:00, 69.66it/s]\n",
      "accuracy: 0.6651, loss: 0.8185 ||: 100%|██████████| 267/267 [00:22<00:00, 11.77it/s]\n",
      "accuracy: 0.3706, loss: 1.6537 ||: 100%|██████████| 35/35 [00:00<00:00, 70.06it/s]\n",
      "accuracy: 0.7104, loss: 0.7288 ||: 100%|██████████| 267/267 [00:23<00:00, 12.52it/s]\n",
      "accuracy: 0.3624, loss: 1.7685 ||: 100%|██████████| 35/35 [00:00<00:00, 67.46it/s]\n",
      "accuracy: 0.7447, loss: 0.6604 ||: 100%|██████████| 267/267 [00:22<00:00, 12.99it/s]\n",
      "accuracy: 0.3697, loss: 1.9572 ||: 100%|██████████| 35/35 [00:00<00:00, 65.28it/s]\n",
      "accuracy: 0.7702, loss: 0.6016 ||: 100%|██████████| 267/267 [00:21<00:00, 12.20it/s]\n",
      "accuracy: 0.3533, loss: 1.9647 ||: 100%|██████████| 35/35 [00:00<00:00, 53.82it/s]\n",
      "accuracy: 0.7982, loss: 0.5466 ||: 100%|██████████| 267/267 [00:22<00:00, 11.94it/s]\n",
      "accuracy: 0.3697, loss: 2.3639 ||: 100%|██████████| 35/35 [00:00<00:00, 56.49it/s]\n",
      "accuracy: 0.8181, loss: 0.4989 ||: 100%|██████████| 267/267 [00:21<00:00, 12.30it/s]\n",
      "accuracy: 0.3697, loss: 2.4844 ||: 100%|██████████| 35/35 [00:00<00:00, 73.17it/s]\n",
      "accuracy: 0.8282, loss: 0.4674 ||: 100%|██████████| 267/267 [00:22<00:00, 10.14it/s]\n",
      "accuracy: 0.3615, loss: 2.7376 ||: 100%|██████████| 35/35 [00:00<00:00, 69.92it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'best_epoch': 3,\n",
       " 'peak_cpu_memory_MB': 325.926912,\n",
       " 'training_duration': '0:04:58.924841',\n",
       " 'training_start_epoch': 0,\n",
       " 'training_epochs': 12,\n",
       " 'epoch': 12,\n",
       " 'training_accuracy': 0.8181179775280899,\n",
       " 'training_loss': 0.4988881808318449,\n",
       " 'training_cpu_memory_MB': 325.926912,\n",
       " 'validation_accuracy': 0.36966394187102636,\n",
       " 'validation_loss': 2.4844228676387243,\n",
       " 'best_validation_accuracy': 0.3678474114441417,\n",
       " 'best_validation_loss': 1.4005519798823765}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python allennlp",
   "language": "python",
   "name": "allennlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}