RNN/RNN-from-scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    },
    "orig_nbformat": 2,
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.7.10 64-bit"
    },
    "colab": {
      "name": "RNN-from-scratch.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Gladiator07/Natural-Language-Processing/blob/main/RNN/RNN-from-scratch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bA7BqE1T--fw"
      },
      "source": [
        "## RNN from scratch in PyTorch"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jXxH69fj--f1"
      },
      "source": [
        "![](https://miro.medium.com/max/840/1*o65pRKyHxhw7m8LgMbVERg.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zgJpe05n--f2"
      },
      "source": [
        "First, let’s build the computation graph for a single-layer RNN. Again, we are not concerned with the math for now, I just want to show you the PyTorch operations needed to build your RNN models."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SfcjiJkz--f3"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import os\n",
        "import numpy as np"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rh2670Dm--f5"
      },
      "source": [
        "class SingleRNN(nn.Module):\n",
        "    def __init__(self, n_inputs, n_neurons):\n",
        "        super(SingleRNN, self).__init__()\n",
        "\n",
        "        self.Wx = torch.randn(n_inputs, n_neurons)  # 4 x 1\n",
        "        self.Wy = torch.randn(n_neurons, n_neurons) # 1 x 1\n",
        "\n",
        "        self.b = torch.zeros(1, n_neurons) # 1 x 4\n",
        "    \n",
        "    def forward(self, X0, X1):\n",
        "        self.Y0 = torch.tanh(torch.mm(X0, self.Wx) + self.b) # 4 x 1\n",
        "\n",
        "        self.Y1 = torch.tanh(torch.mm(self.Y0, self.Wy) + \n",
        "                             torch.mm(X1, self.Wx) + self.b) # 4 x 1\n",
        "        \n",
        "        return self.Y0, self.Y1"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WgQU_uon--f5"
      },
      "source": [
        "The forward function computes two outputs — one for each time step (two overall). Note that we are using tanh as the non-linearity (activation function) via torch.tanh(...)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kINT0p8e--f6"
      },
      "source": [
        "This is how the data is fed into RNN:\n",
        "\n",
        "![](https://miro.medium.com/max/764/1*xCj9h3f2kekfqN_dMCpcag.png)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qzh88ne3--f6"
      },
      "source": [
        "N_INPUT = 4\n",
        "N_NEURONS = 1\n",
        "\n",
        "\n",
        "X0_batch = torch.tensor([[0,1,2,0], [3,4,5,0], \n",
        "                         [6,7,8,0], [9,0,1,0]],\n",
        "                        dtype = torch.float) #t=0 => 4 X 4\n",
        "\n",
        "X1_batch = torch.tensor([[9,8,7,0], [0,0,0,0], \n",
        "                         [6,5,4,0], [3,2,1,0]],\n",
        "                        dtype = torch.float) #t=1 => 4 X 4\n",
        "model = SingleRNN(N_INPUT, N_NEURONS)\n",
        "\n",
        "Y0_val, Y1_val = model(X0_batch, X1_batch)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bylAMyBE--f7",
        "outputId": "c3227c17-712e-46c3-d7c2-6c088cb102bb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(Y0_val)\n",
        "print(Y1_val)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[-0.9544],\n",
            "        [-0.0408],\n",
            "        [ 0.9465],\n",
            "        [ 1.0000]])\n",
            "tensor([[1.0000],\n",
            "        [0.0377],\n",
            "        [0.9998],\n",
            "        [0.9925]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cVRFKsmj--f8"
      },
      "source": [
        "### Increasing neurons in RNN Layer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SM1Hl4Aw--f8"
      },
      "source": [
        "![](https://miro.medium.com/max/840/1*KLBXIeszx_cqkYs3-EXHwg.png)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I5sPxDDt--f9"
      },
      "source": [
        "class BasicRNN(nn.Module):\n",
        "    def __init__(self, n_inputs, n_neurons):\n",
        "        super(BasicRNN, self).__init__()\n",
        "\n",
        "        self.Wx = torch.randn(n_inputs, n_neurons) # n_inputs x n_neurons\n",
        "        self.Wy = torch.randn(n_neurons, n_neurons) # n_neurons x n_neurons\n",
        "\n",
        "        self.b = torch.zeros(1, n_neurons) # 1 x neurons\n",
        "\n",
        "    def forward(self, X0, X1):\n",
        "        self.Y0 = torch.tanh(torch.mm(X0, self.Wx) + self.b) # batch_size x n_neurons\n",
        "        self.Y1 = torch.tanh(torch.mm(self.Y0, self.Wy) + torch.mm(X1, self.Wx) + self.b) # batch_size x n_neurons\n",
        "\n",
        "        return self.Y0, self.Y1    "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z8z6APCy--f9"
      },
      "source": [
        "N_INPUT = 3 # number of features in input\n",
        "N_NEURONS = 5 # number of units in layer\n",
        "\n",
        "X0_batch = torch.tensor([[0,1,2], [3,4,5], \n",
        "                         [6,7,8], [9,0,1]],\n",
        "                        dtype = torch.float) #t=0 => 4 X 3\n",
        "\n",
        "X1_batch = torch.tensor([[9,8,7], [0,0,0], \n",
        "                         [6,5,4], [3,2,1]],\n",
        "                        dtype = torch.float) #t=1 => 4 X 3\n",
        "\n",
        "model = BasicRNN(N_INPUT, N_NEURONS)\n",
        "\n",
        "Y0_val, Y1_val = model(X0_batch, X1_batch)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iuSjfkYL--f-",
        "outputId": "561b8313-a4ef-42f1-b4d5-ba9c95bb42d1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(Y0_val)\n",
        "print(Y1_val)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[-0.4270, -0.2270,  0.1532, -0.5908,  0.9927],\n",
            "        [-0.8519, -0.8407,  1.0000, -1.0000, -0.9086],\n",
            "        [-0.9686, -0.9765,  1.0000, -1.0000, -1.0000],\n",
            "        [ 0.9845, -0.4702,  1.0000, -1.0000, -1.0000]])\n",
            "tensor([[-0.8298, -0.9995,  1.0000, -1.0000, -1.0000],\n",
            "        [-0.7453,  0.9513,  0.6089,  0.1575, -0.8982],\n",
            "        [-0.9818,  0.1597,  1.0000, -1.0000, -1.0000],\n",
            "        [-0.7212,  0.9855,  1.0000, -1.0000, -1.0000]])\n"
          ]
        }
      ]
    }
  ]
}