pytorch/17_lstm.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "model-zoo.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Hk4wNbNytnB"
      },
      "source": [
        "# RNN with LSTM cell on IMDB\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LFpHDxd6HHLy"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lEpof2LG7NA3"
      },
      "source": [
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import torchtext \n",
        "import time\n",
        "import random\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PXz_dHMTHHL0"
      },
      "source": [
        "### Settings and Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9zWGjESt2QYa"
      },
      "source": [
        "# Device\n",
        "DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "\n",
        "# Hyperparameters\n",
        "RANDOM_SEED = 123\n",
        "torch.manual_seed(RANDOM_SEED)\n",
        "VOCABULARY_SIZE = 20000\n",
        "LEARNING_RATE = 1e-4\n",
        "BATCH_SIZE = 128\n",
        "NUM_EPOCHS = 10\n",
        "EMBEDDING_DIM = 128\n",
        "HIDDEN_DIM = 256\n",
        "OUTPUT_DIM = 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rNDyyWr5iNMq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c51a0b1d-39d8-4df4-9815-0da9c87a2aa8"
      },
      "source": [
        "TEXT = torchtext.legacy.data.Field(tokenize=\"basic_english\",\n",
        "                  include_lengths=True) \n",
        "LABEL = torchtext.legacy.data.LabelField(dtype=torch.float)\n",
        "\n",
        "\n",
        "train_data, test_data = torchtext.legacy.datasets.IMDB.splits(TEXT, LABEL)\n",
        "train_data, valid_data = train_data.split(random_state=random.seed(RANDOM_SEED),\n",
        "                                          split_ratio=0.8)\n",
        "\n",
        "\n",
        "print(f'Num Train: {len(train_data)}')\n",
        "print(f'Num Valid: {len(valid_data)}')\n",
        "print(f'Num Test: {len(test_data)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Num Train: 20000\n",
            "Num Valid: 5000\n",
            "Num Test: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QCWc_a0Iz5Q5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "36e875e9-f8dc-4a06-8ef9-8418c21c25de"
      },
      "source": [
        "TEXT.build_vocab(train_data, max_size=VOCABULARY_SIZE)\n",
        "LABEL.build_vocab(train_data)\n",
        "\n",
        "\n",
        "print(f'Vocabulary size: {len(TEXT.vocab)}')\n",
        "print(f'Number of classes: {len(LABEL.vocab)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Vocabulary size: 20002\n",
            "Number of classes: 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I7kX5qj0Lu7X"
      },
      "source": [
        "train_loader, valid_loader, test_loader = torchtext.legacy.data.BucketIterator.splits(\n",
        "    (train_data, valid_data, test_data), \n",
        "    batch_size=BATCH_SIZE,\n",
        "    sort_within_batch=True,\n",
        "    device=DEVICE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yGrbTQyo__7B",
        "outputId": "a559742a-9533-4fea-ee74-9a7e133c9590"
      },
      "source": [
        "print('Train')\n",
        "for batch in train_loader:\n",
        "    print(f'Text matrix size: {batch.text[0].size()}')\n",
        "    print(f'Target vector size: {batch.label.size()}')\n",
        "    break\n",
        "    \n",
        "\n",
        "print('\\nValid:')\n",
        "for batch in valid_loader:\n",
        "    print(f'Text matrix size: {batch.text[0].size()}')\n",
        "    print(f'Target vector size: {batch.label.size()}')\n",
        "    break\n",
        "    \n",
        "    \n",
        "print('\\nTest:')\n",
        "for batch in test_loader:\n",
        "    print(f'Text matrix size: {batch.text[0].size()}')\n",
        "    print(f'Target vector size: {batch.label.size()}')\n",
        "    break"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train\n",
            "Text matrix size: torch.Size([133, 128])\n",
            "Target vector size: torch.Size([128])\n",
            "\n",
            "Valid:\n",
            "Text matrix size: torch.Size([58, 128])\n",
            "Target vector size: torch.Size([128])\n",
            "\n",
            "Test:\n",
            "Text matrix size: torch.Size([42, 128])\n",
            "Target vector size: torch.Size([128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F1a6OD4-HHL3"
      },
      "source": [
        "### Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZO_OO1tzADCq"
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class RNN(nn.Module):\n",
        "    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):\n",
        "        \n",
        "        super().__init__()\n",
        "        self.embedding = nn.Embedding(input_dim, embedding_dim)\n",
        "        self.rnn = nn.LSTM(embedding_dim, hidden_dim)\n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "        \n",
        "    def forward(self, text, text_length):\n",
        "        #[sentence len, batch size] => [sentence len, batch size, embedding size]\n",
        "        embedded = self.embedding(text)\n",
        "        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, text_length.cpu())\n",
        "        \n",
        "        #[sentence len, batch size, embedding size] => \n",
        "        #  output: [sentence len, batch size, hidden size]\n",
        "        #  hidden: [1, batch size, hidden size]\n",
        "        packed_output, (hidden, cell) = self.rnn(packed)\n",
        "        return self.fc(hidden.squeeze(0)).view(-1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LAcdBjxCBmDg"
      },
      "source": [
        "INPUT_DIM = len(TEXT.vocab)\n",
        "torch.manual_seed(RANDOM_SEED)\n",
        "model = RNN(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)\n",
        "model = model.to(DEVICE)\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rzFIKC5mSCNs"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f_693ihY2769",
        "outputId": "8489258b-dbdb-4fa9-9f65-6f050fbceae9"
      },
      "source": [
        "def compute_binary_accuracy(model, data_loader):\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    with torch.no_grad():\n",
        "        for batch_idx, batch_data in enumerate(data_loader):\n",
        "            text, text_lengths = batch_data.text\n",
        "            logits = model(text, text_lengths)\n",
        "            predicted_labels = (torch.sigmoid(logits) > 0.5).long()\n",
        "            num_examples += batch_data.label.size(0)\n",
        "            correct_pred += (predicted_labels == batch_data.label.long()).sum()\n",
        "        return correct_pred.float()/num_examples * 100\n",
        "\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    model.train()\n",
        "    \n",
        "    for batch_idx, batch_data in enumerate(train_loader):\n",
        "        \n",
        "      text, text_lengths = batch_data.text\n",
        "        \n",
        "      # FORWARD AND BACK PROP\n",
        "      logits = model(text, text_lengths)\n",
        "      cost = F.binary_cross_entropy_with_logits(logits, batch_data.label)\n",
        "      optimizer.zero_grad()\n",
        "        \n",
        "      cost.backward()\n",
        "        \n",
        "      # UPDATE MODEL PARAMETERS\n",
        "      optimizer.step()\n",
        "        \n",
        "      # Logging\n",
        "      if not batch_idx % 50:\n",
        "        print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
        "                   %(epoch+1, NUM_EPOCHS, batch_idx, \n",
        "                     len(train_loader), cost))\n",
        "\n",
        "    model.eval()\n",
        "    with torch.set_grad_enabled(False):\n",
        "        print('Epoch: %03d/%03d | Train: %.3f%% ' %(\n",
        "              epoch+1, NUM_EPOCHS, \n",
        "              compute_binary_accuracy(model, train_loader)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/010 | Batch 0000/0157 | Cost: 0.6964\n",
            "Epoch: 001/010 | Batch 0050/0157 | Cost: 0.6892\n",
            "Epoch: 001/010 | Batch 0100/0157 | Cost: 0.6807\n",
            "Epoch: 001/010 | Batch 0150/0157 | Cost: 0.6720\n",
            "Epoch: 001/010 | Train: 59.390% \n",
            "Epoch: 002/010 | Batch 0000/0157 | Cost: 0.6640\n",
            "Epoch: 002/010 | Batch 0050/0157 | Cost: 0.6537\n",
            "Epoch: 002/010 | Batch 0100/0157 | Cost: 0.6171\n",
            "Epoch: 002/010 | Batch 0150/0157 | Cost: 0.6121\n",
            "Epoch: 002/010 | Train: 70.565% \n",
            "Epoch: 003/010 | Batch 0000/0157 | Cost: 0.5752\n",
            "Epoch: 003/010 | Batch 0050/0157 | Cost: 0.5931\n",
            "Epoch: 003/010 | Batch 0100/0157 | Cost: 0.5470\n",
            "Epoch: 003/010 | Batch 0150/0157 | Cost: 0.6280\n",
            "Epoch: 003/010 | Train: 74.690% \n",
            "Epoch: 004/010 | Batch 0000/0157 | Cost: 0.4507\n",
            "Epoch: 004/010 | Batch 0050/0157 | Cost: 0.5037\n",
            "Epoch: 004/010 | Batch 0100/0157 | Cost: 0.4900\n",
            "Epoch: 004/010 | Batch 0150/0157 | Cost: 0.4947\n",
            "Epoch: 004/010 | Train: 77.825% \n",
            "Epoch: 005/010 | Batch 0000/0157 | Cost: 0.4872\n",
            "Epoch: 005/010 | Batch 0050/0157 | Cost: 0.4827\n",
            "Epoch: 005/010 | Batch 0100/0157 | Cost: 0.4060\n",
            "Epoch: 005/010 | Batch 0150/0157 | Cost: 0.6321\n",
            "Epoch: 005/010 | Train: 70.150% \n",
            "Epoch: 006/010 | Batch 0000/0157 | Cost: 0.5942\n",
            "Epoch: 006/010 | Batch 0050/0157 | Cost: 0.4998\n",
            "Epoch: 006/010 | Batch 0100/0157 | Cost: 0.5098\n",
            "Epoch: 006/010 | Batch 0150/0157 | Cost: 0.4472\n",
            "Epoch: 006/010 | Train: 78.180% \n",
            "Epoch: 007/010 | Batch 0000/0157 | Cost: 0.4297\n",
            "Epoch: 007/010 | Batch 0050/0157 | Cost: 0.4417\n",
            "Epoch: 007/010 | Batch 0100/0157 | Cost: 0.4799\n",
            "Epoch: 007/010 | Batch 0150/0157 | Cost: 0.3895\n",
            "Epoch: 007/010 | Train: 80.680% \n",
            "Epoch: 008/010 | Batch 0000/0157 | Cost: 0.4339\n",
            "Epoch: 008/010 | Batch 0050/0157 | Cost: 0.4090\n",
            "Epoch: 008/010 | Batch 0100/0157 | Cost: 0.3958\n",
            "Epoch: 008/010 | Batch 0150/0157 | Cost: 0.3853\n",
            "Epoch: 008/010 | Train: 82.975% \n",
            "Epoch: 009/010 | Batch 0000/0157 | Cost: 0.4012\n",
            "Epoch: 009/010 | Batch 0050/0157 | Cost: 0.4358\n",
            "Epoch: 009/010 | Batch 0100/0157 | Cost: 0.4218\n",
            "Epoch: 009/010 | Batch 0150/0157 | Cost: 0.4041\n",
            "Epoch: 009/010 | Train: 84.995% \n",
            "Epoch: 010/010 | Batch 0000/0157 | Cost: 0.3539\n",
            "Epoch: 010/010 | Batch 0050/0157 | Cost: 0.3640\n",
            "Epoch: 010/010 | Batch 0100/0157 | Cost: 0.4492\n",
            "Epoch: 010/010 | Batch 0150/0157 | Cost: 0.4770\n",
            "Epoch: 010/010 | Train: 84.475% \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7sQBgaPtHHL7"
      },
      "source": [
        "### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1S-3zI94Bu8J"
      },
      "source": [
        "import spacy\n",
        "nlp = spacy.load('en')\n",
        "\n",
        "\n",
        "def predict_sentiment(model, sentence):\n",
        "    model.eval()\n",
        "    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]\n",
        "    indexed = [TEXT.vocab.stoi[t] for t in tokenized]\n",
        "    length = [len(indexed)]\n",
        "    tensor = torch.LongTensor(indexed).to(DEVICE)\n",
        "    tensor = tensor.unsqueeze(1)\n",
        "    length_tensor = torch.LongTensor(length)\n",
        "    prediction = torch.sigmoid(model(tensor, length_tensor))\n",
        "    return prediction.item()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B0MYkRbBQs5_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1ae36cda-ec64-4d8c-a7ef-cb0a906be6a7"
      },
      "source": [
        "print('Probability positive:')\n",
        "predict_sentiment(model, \"I really love this movie. This movie is so great!\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Probability positive:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7884787917137146"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    }
  ]
}