PyTorch/rnn_two_hidden_layer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN - 2 Hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = ds.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True)\n",
    "test_dataset = ds.MNIST(root='./data', train=False, transform=transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\harshil\\Anaconda3\\lib\\site-packages\\torchvision\\datasets\\mnist.py:53: UserWarning: train_data has been renamed data\n",
      "  warnings.warn(\"train_data has been renamed data\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([60000, 28, 28])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\harshil\\Anaconda3\\lib\\site-packages\\torchvision\\datasets\\mnist.py:43: UserWarning: train_labels has been renamed targets\n",
      "  warnings.warn(\"train_labels has been renamed targets\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([60000])\n"
     ]
    }
   ],
   "source": [
    "print(train_dataset.train_data.size())\n",
    "print(train_dataset.train_labels.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\harshil\\Anaconda3\\lib\\site-packages\\torchvision\\datasets\\mnist.py:58: UserWarning: test_data has been renamed data\n",
      "  warnings.warn(\"test_data has been renamed data\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10000, 28, 28])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\harshil\\Anaconda3\\lib\\site-packages\\torchvision\\datasets\\mnist.py:48: UserWarning: test_labels has been renamed targets\n",
      "  warnings.warn(\"test_labels has been renamed targets\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10000])\n"
     ]
    }
   ],
   "source": [
    "print(test_dataset.test_data.size())\n",
    "print(test_dataset.test_labels.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make dataset iterable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 100\n",
    "n_iters = 3000\n",
    "num_epochs = int(n_iters / (len(train_dataset) / batch_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create model class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNModel(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n",
    "        super(RNNModel, self).__init__()\n",
    "        \n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.layer_dim = layer_dim\n",
    "        \n",
    "        self.rnn = nn.RNN(input_dim, hidden_dim, layer_dim, batch_first=True, nonlinearity='relu')\n",
    "        \n",
    "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        h0= torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n",
    "        out, hn = self.rnn(x, h0.detach())\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate model class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dim = 28\n",
    "hidden_dim = 100\n",
    "layer_dim = 2\n",
    "output_dim = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RNNModel(input_dim, hidden_dim, layer_dim, output_dim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate loss class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate optimizer class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_dim = 28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 500: Loss 2.2959275245666504, Accuracy 15\n",
      "Iteration 1000: Loss 2.3010785579681396, Accuracy 12\n",
      "Iteration 1500: Loss 2.2832164764404297, Accuracy 16\n",
      "Iteration 2000: Loss 2.1377947330474854, Accuracy 25\n",
      "Iteration 2500: Loss 1.0977602005004883, Accuracy 63\n",
      "Iteration 3000: Loss 0.6976373791694641, Accuracy 76\n"
     ]
    }
   ],
   "source": [
    "iter = 0\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        model.train()\n",
    "        \n",
    "        images = images.view(-1, seq_dim, input_dim).requires_grad_()\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        iter += 1\n",
    "        if iter % 500 == 0:\n",
    "            model.eval()\n",
    "            correct = 0\n",
    "            total = 0\n",
    "            for images, labels in test_loader:\n",
    "                images = images.view(-1, seq_dim, input_dim)\n",
    "                outputs = model(images)\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                total += labels.size(0)\n",
    "                correct += (predicted == labels).sum()\n",
    "            accuracy = 100 * correct / total\n",
    "            print('Iteration {}: Loss {}, Accuracy {}'.format(iter, loss.item(), accuracy))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}