Sentiment_Analyser_App_with_LSTM_network.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sentiment Analyser App with LSTM network.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "XY4JaXJnT2hK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "\n",
        "#Get system variables for using later in GPU and file locations\n",
        "from os.path import exists\n",
        "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
        "platform='{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
        "cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n",
        "accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'\n",
        "\n",
        "import torch"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7z3ppsktRiyx",
        "colab_type": "code",
        "outputId": "d1218d65-43ff-4882-c5ff-befbf8346c93",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "\n",
        "#check if cuda is available\n",
        "train_on_GPU=torch.cuda.is_available();\n",
        "\n",
        "if not train_on_GPU:\n",
        "    print('Not available')\n",
        "else:\n",
        "    print('Available')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Available\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xt6RWs4MThFH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#download the required data and place it in appropriate directory\n",
        "!mkdir -p data\n",
        "!wget -cq https://github.com/udacity/deep-learning-v2-pytorch/raw/master/sentiment-rnn/data/labels.txt -O data/labels.txt\n",
        "!wget -cq https://github.com/udacity/deep-learning-v2-pytorch/raw/master/sentiment-rnn/data/reviews.txt -O data/reviews.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rdf7nkr6Tmuj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "#read data//Open the first movie review and recursively search through the whole file\n",
        "\n",
        "with open('data/reviews.txt','r') as f:\n",
        "  reviews=f.read()\n",
        "with open('data/labels.txt','r') as f:\n",
        "  labels=f.read()\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pW--rzrTVB0P",
        "colab_type": "text"
      },
      "source": [
        "#Data Preprocessing:\n",
        "\n",
        "\n",
        "*   Remove Punctuation\n",
        "*   Split text into each review using \\n as delimiter\n",
        "*   COmbine all the reviews into one large string"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pOTgVOm6U77W",
        "colab_type": "code",
        "outputId": "15918578-c71f-4dff-8bc5-6b24eaef7906",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from string import punctuation\n",
        "\n",
        "print(punctuation) #the characters we are getting rid off\n",
        "\n",
        "# get rid of punctuation\n",
        "reviews = reviews.lower() # lowercase, standardize\n",
        "all_text = ''.join([c for c in reviews if c not in punctuation])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wkaiItmNWWj6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# split by new lines and spaces\n",
        "reviews_split = all_text.split('\\n')\n",
        "all_text = ' '.join(reviews_split)\n",
        "\n",
        "# creating a list of words\n",
        "words = all_text.split() #tokenize the string"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kiauQpsbTXZz",
        "colab_type": "text"
      },
      "source": [
        "#Encoding the words\n",
        "\n",
        "The embedding lookup requires that we pass in integers to our network. The easiest way to do this is to create dictionaries that map the words in the vocabulary to integers. Then we can convert each of our reviews into integers so they can be passed into the network.The fastest way to do this is by using word2vec which converts words into vectors;which helps in the lookup of words."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T2JmhvlkTXZ0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import Counter\n",
        "\n",
        "## Build a dictionary that maps words to integers\n",
        "counts = Counter(words)\n",
        "vocab = sorted(counts, key=counts.get, reverse=True)\n",
        "vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n",
        "\n",
        "## use the dict to tokenize each review in reviews_split\n",
        "## store the tokenized reviews in reviews_ints\n",
        "reviews_ints = []\n",
        "for review in reviews_split:\n",
        "    reviews_ints.append([vocab_to_int[word] for word in review.split()])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OMPQPN2uXjpY",
        "colab_type": "code",
        "outputId": "ac84c670-c81c-4b86-bffd-af0cb48b71b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "# stats about vocabulary\n",
        "print('Unique words: ', len((vocab_to_int)))  # should ~ 74000+\n",
        "print()\n",
        "\n",
        "# print tokens in first review\n",
        "print('Tokenized review: \\n', reviews_ints[:1])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique words:  74072\n",
            "\n",
            "Tokenized review: \n",
            " [[21025, 308, 6, 3, 1050, 207, 8, 2138, 32, 1, 171, 57, 15, 49, 81, 5785, 44, 382, 110, 140, 15, 5194, 60, 154, 9, 1, 4975, 5852, 475, 71, 5, 260, 12, 21025, 308, 13, 1978, 6, 74, 2395, 5, 613, 73, 6, 5194, 1, 24103, 5, 1983, 10166, 1, 5786, 1499, 36, 51, 66, 204, 145, 67, 1199, 5194, 19869, 1, 37442, 4, 1, 221, 883, 31, 2988, 71, 4, 1, 5787, 10, 686, 2, 67, 1499, 54, 10, 216, 1, 383, 9, 62, 3, 1406, 3686, 783, 5, 3483, 180, 1, 382, 10, 1212, 13583, 32, 308, 3, 349, 341, 2913, 10, 143, 127, 5, 7690, 30, 4, 129, 5194, 1406, 2326, 5, 21025, 308, 10, 528, 12, 109, 1448, 4, 60, 543, 102, 12, 21025, 308, 6, 227, 4146, 48, 3, 2211, 12, 8, 215, 23]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MVoOlR93XsM_"
      },
      "source": [
        "#Encoding the labels\n",
        "\n",
        "Our labels are \"positive\" or \"negative\". To use these labels in our network, we need to convert them to 0 and 1.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uc_PeiK7TXZ9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 1=positive, 0=negative label conversion\n",
        "labels_split = labels.split('\\n')\n",
        "encoded_labels = np.array([1 if label == 'positive' else 0 for label in labels_split])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XRNI3mzITXaB",
        "colab_type": "text"
      },
      "source": [
        "# Removing Outliers\n",
        "\n",
        "1. Getting rid of extremely long or short reviews; the outliers\n",
        "2. Padding/truncating the remaining data so that we have reviews of the same length."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7dIxZ9m1Yv8v",
        "colab_type": "code",
        "outputId": "438dfb09-d7d9-49c6-fb96-7829c775bc40",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "# outlier review stats\n",
        "review_lens = Counter([len(x) for x in reviews_ints])\n",
        "print(\"0-length reviews: {}\".format(review_lens[0]))\n",
        "print(\"Max rev length: {}\".format(max(review_lens)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0-length reviews: 1\n",
            "Max rev length: 2514\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UL4SQnPGY65E",
        "colab_type": "code",
        "outputId": "e329abb5-1406-4ae1-8b0d-1e6ba972a512",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "print('No. of revs before removing outliers: ', len(reviews_ints))\n",
        "\n",
        "## remove any reviews/labels with zero length from the reviews_ints list.\n",
        "\n",
        "# get indices of any reviews with length 0\n",
        "non_zero_idx = [ii for ii, review in enumerate(reviews_ints) if len(review) != 0]\n",
        "\n",
        "# remove 0-length reviews and their labels\n",
        "reviews_ints = [reviews_ints[ii] for ii in non_zero_idx]\n",
        "encoded_labels = np.array([encoded_labels[ii] for ii in non_zero_idx])\n",
        "\n",
        "print('No. of revs after removing outliers: ', len(reviews_ints))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No. of revs before removing outliers:  25001\n",
            "No. of revs after removing outliers:  25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7gctkQH-TXaJ",
        "colab_type": "text"
      },
      "source": [
        "# Padding sequences\n",
        "\n",
        "To deal with both short and very long reviews, we'll pad or truncate all our reviews to a specific length. For reviews shorter than some `seq_length`, we'll pad with 0s. For reviews longer than `seq_length`, we can truncate them to the first `seq_length` words. A good `seq_length`, in this case, is 200.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PZr-qHqAZqw5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def pad_features(reviews_ints, seq_length):\n",
        "    ''' Return features of review_ints, where each review is padded with 0's \n",
        "        or truncated to the input seq_length.\n",
        "    '''\n",
        "    \n",
        "    # getting the correct rows x cols shape\n",
        "    features = np.zeros((len(reviews_ints), seq_length), dtype=int)\n",
        "\n",
        "    # for each review, I grab that review and \n",
        "    for i, row in enumerate(reviews_ints):\n",
        "        features[i, -len(row):] = np.array(row)[:seq_length]\n",
        "    \n",
        "    return features"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gc239B8wZvEq",
        "colab_type": "code",
        "outputId": "2dd49aae-e911-4915-dc72-f6575de65e2b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 521
        }
      },
      "source": [
        "# Testing the implementation!\n",
        "\n",
        "seq_length = 200\n",
        "\n",
        "features = pad_features(reviews_ints, seq_length=seq_length)\n",
        "\n",
        "## test statements\n",
        "assert len(features)==len(reviews_ints), \"Your features should have as many rows as reviews.\"\n",
        "assert len(features[0])==seq_length, \"Each feature row should contain seq_length values.\"\n",
        "\n",
        "# print first 10 values of the first 30 batches \n",
        "print(features[:30,:10])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [22382    42 46418    15   706 17139  3389    47    77    35]\n",
            " [ 4505   505    15     3  3342   162  8312  1652     6  4819]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   54    10    14   116    60   798   552    71   364     5]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    1   330   578    34     3   162   748  2731     9   325]\n",
            " [    9    11 10171  5305  1946   689   444    22   280   673]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    1   307 10399  2069  1565  6202  6528  3288 17946 10628]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   21   122  2069  1565   515  8181    88     6  1325  1182]\n",
            " [    1    20     6    76    40     6    58    81    95     5]\n",
            " [   54    10    84   329 26230 46427    63    10    14   614]\n",
            " [   11    20     6    30  1436 32317  3769   690 15100     6]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   40    26   109 17952  1422     9     1   327     4   125]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   10   499     1   307 10399    55    74     8    13    30]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9TN4PatIaAG3",
        "colab_type": "text"
      },
      "source": [
        "#$ Training $$ Validation $ And Test "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cecm1_DpaUmP",
        "colab_type": "code",
        "outputId": "cdd88ce9-4da4-4aa2-bc4b-f5bba9371879",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "split_frac = 0.8\n",
        "\n",
        "## split data into training, validation, and test data (features and labels, x and y)\n",
        "\n",
        "split_idx = int(len(features)*split_frac)\n",
        "train_x, remaining_x = features[:split_idx], features[split_idx:]\n",
        "train_y, remaining_y = encoded_labels[:split_idx], encoded_labels[split_idx:]\n",
        "\n",
        "test_idx = int(len(remaining_x)*0.5)\n",
        "val_x, test_x = remaining_x[:test_idx], remaining_x[test_idx:]\n",
        "val_y, test_y = remaining_y[:test_idx], remaining_y[test_idx:]\n",
        "\n",
        "## print out the shapes of your resultant feature data\n",
        "print(\"\\t\\t\\tFeature Shapes:\")\n",
        "print(\"Train set: \\t\\t{}\".format(train_x.shape), \n",
        "      \"\\nValidation set: \\t{}\".format(val_x.shape),\n",
        "      \"\\nTest set: \\t\\t{}\".format(test_x.shape))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\t\t\tFeature Shapes:\n",
            "Train set: \t\t(20000, 200) \n",
            "Validation set: \t(2500, 200) \n",
            "Test set: \t\t(2500, 200)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eJ1D36CTTXaY",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "# ***DataLoaders and Batching***\n",
        "\n",
        "After creating training, test, and validation data, we can create DataLoaders for this data by following two steps:\n",
        "1. Create a known format for accessing our data, using TensorDataset which takes in an input set of data and a target set of data with the same first dimension, and creates a dataset.\n",
        "2. Create DataLoaders and batch our training, validation, and test Tensor datasets.\n",
        "\n",
        "```\n",
        "train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))\n",
        "train_loader = DataLoader(train_data, batch_size=batch_size)\n",
        "```\n",
        "\n",
        "This is an alternative to creating a generator function for batching our data into full batches."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3K1jNMT2TXaZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "\n",
        "# create Tensor datasets\n",
        "train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))\n",
        "valid_data = TensorDataset(torch.from_numpy(val_x), torch.from_numpy(val_y))\n",
        "test_data = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y))\n",
        "\n",
        "# dataloaders\n",
        "batch_size = 50\n",
        "\n",
        "# make sure to SHUFFLE your data\n",
        "train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)\n",
        "valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size)\n",
        "test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vtg43hXBbWLS",
        "colab_type": "code",
        "outputId": "e150d8d0-6c85-4d4b-9914-ceb4164b7d32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "# obtain one batch of training data\n",
        "dataiter = iter(train_loader)\n",
        "sample_x, sample_y = dataiter.next()\n",
        "\n",
        "print('Sample input size: ', sample_x.size()) # batch_size, seq_length\n",
        "print('Sample input: \\n', sample_x)\n",
        "print()\n",
        "print('Sample label size: ', sample_y.size()) # batch_size\n",
        "print('Sample label: \\n', sample_y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sample input size:  torch.Size([50, 200])\n",
            "Sample input: \n",
            " tensor([[    0,     0,     0,  ...,     9, 25402, 14099],\n",
            "        [    0,     0,     0,  ...,    85,   121, 27012],\n",
            "        [    0,     0,     0,  ...,    55,    50,  1890],\n",
            "        ...,\n",
            "        [    0,     0,     0,  ...,     8,    55,  1478],\n",
            "        [    0,     0,     0,  ...,   105,   447,   209],\n",
            "        [    0,     0,     0,  ...,    32,    11,   270]])\n",
            "\n",
            "Sample label size:  torch.Size([50])\n",
            "Sample label: \n",
            " tensor([0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0,\n",
            "        0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0,\n",
            "        1, 0])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SNJFfSITTXaj",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "# Sentiment Network with PyTorch\n",
        "\n",
        "\n",
        "<img src=\"https://github.com/udacity/deep-learning-v2-pytorch/blob/master/sentiment-rnn/assets/network_diagram.png?raw=1\" width=40%>\n",
        "\n",
        "The layers are as follows:\n",
        "1. An embedding layer that converts our word tokens (integers) into embeddings of a specific size.\n",
        "2. An LSTM layer defined by a hidden_state size and number of layers\n",
        "3. A fully-connected output layer that maps the LSTM layer outputs to a desired output_size\n",
        "4. A sigmoid activation layer which turns all outputs into a value 0-1; return **only the last sigmoid output** as the output of this network.\n",
        "\n",
        "## The Embedding Layer\n",
        "\n",
        "We need to add an [embedding layer](https://pytorch.org/docs/stable/nn.html#embedding) because there are 74000+ words in our vocabulary. It is massively inefficient to one-hot encode that many classes. So, instead of one-hot encoding, we can have an embedding layer and use that layer as a lookup table. You could train an embedding layer using Word2Vec, then load it here. But, it's fine to just make a new layer, using it for only dimensionality reduction, and let the network learn the weights.\n",
        "\n",
        "## The LSTM Layer(s)\n",
        "\n",
        "We'll create an LSTM to use in our recurrent network, which takes in an input_size, a hidden_dim, a number of layers, a dropout probability (for dropout between multiple layers), and a batch_first parameter.\n",
        "\n",
        "Most of the time, you're network will have better performance with more layers; between 2-3. Adding more layers allows the network to learn really complex relationships. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7B_0wrGSbyEa",
        "colab_type": "code",
        "outputId": "be554148-d7af-44af-bd16-a08571c6766d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# First checking if GPU is available\n",
        "train_on_gpu=torch.cuda.is_available()\n",
        "\n",
        "if(train_on_gpu):\n",
        "    print('Training on GPU.')\n",
        "else:\n",
        "    print('No GPU available, training on CPU.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on GPU.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6n6WsVCPb0NN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class SentimentRNN(nn.Module):\n",
        "    \"\"\"\n",
        "    The RNN model that will be used to perform Sentiment analysis.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5):\n",
        "        \"\"\"\n",
        "        Initialize the model by setting up the layers.\n",
        "        \"\"\"\n",
        "        super(SentimentRNN, self).__init__()\n",
        "\n",
        "        self.output_size = output_size\n",
        "        self.n_layers = n_layers\n",
        "        self.hidden_dim = hidden_dim\n",
        "        \n",
        "        # embedding and LSTM layers\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, \n",
        "                            dropout=drop_prob, batch_first=True)\n",
        "        \n",
        "        # dropout layer\n",
        "        self.dropout = nn.Dropout(0.3)\n",
        "        \n",
        "        # linear and sigmoid layers\n",
        "        self.fc = nn.Linear(hidden_dim, output_size)\n",
        "        self.sig = nn.Sigmoid()\n",
        "        \n",
        "\n",
        "    def forward(self, x, hidden):\n",
        "        \"\"\"\n",
        "        Perform a forward pass of our model on some input and hidden state.\n",
        "        \"\"\"\n",
        "        batch_size = x.size(0)\n",
        "\n",
        "        # embeddings and lstm_out\n",
        "        x = x.long()\n",
        "        embeds = self.embedding(x)\n",
        "        lstm_out, hidden = self.lstm(embeds, hidden)\n",
        "    \n",
        "        # stack up lstm outputs\n",
        "        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)\n",
        "        \n",
        "        # dropout and fully-connected layer\n",
        "        out = self.dropout(lstm_out)\n",
        "        out = self.fc(out)\n",
        "        # sigmoid function\n",
        "        sig_out = self.sig(out)\n",
        "        \n",
        "        # reshape to be batch_size first\n",
        "        sig_out = sig_out.view(batch_size, -1)\n",
        "        sig_out = sig_out[:, -1] # get last batch of labels\n",
        "        \n",
        "        # return last sigmoid output and hidden state\n",
        "        return sig_out, hidden\n",
        "    \n",
        "    \n",
        "    def init_hidden(self, batch_size):\n",
        "        ''' Initializes hidden state '''\n",
        "        # Create two new tensors with sizes n_layers x batch_size x hidden_dim,\n",
        "        # initialized to zero, for hidden state and cell state of LSTM\n",
        "        weight = next(self.parameters()).data\n",
        "        \n",
        "        if (train_on_gpu):\n",
        "            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),\n",
        "                  weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())\n",
        "        else:\n",
        "            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_(),\n",
        "                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_())\n",
        "        \n",
        "        return hidden\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KMlSrLTOTXaq",
        "colab_type": "text"
      },
      "source": [
        "## Instantiate the network\n",
        "\n",
        "Here, we'll instantiate the network. First up, defining the hyperparameters.\n",
        "\n",
        "* `vocab_size`: Size of our vocabulary or the range of values for our input, word tokens.\n",
        "* `output_size`: Size of our desired output; the number of class scores we want to output (pos/neg).\n",
        "* `embedding_dim`: Number of columns in the embedding lookup table; size of our embeddings.\n",
        "* `hidden_dim`: Number of units in the hidden layers of our LSTM cells. Usually larger is better performance wise. Common values are 128, 256, 512, etc.\n",
        "* `n_layers`: Number of LSTM layers in the network. Typically between 1-3\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LFbk1Zdnb9it",
        "colab_type": "code",
        "outputId": "e81f1ec2-0b03-45d2-9690-484287ef5377",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 134
        }
      },
      "source": [
        "# Instantiate the model w/ hyperparams\n",
        "vocab_size = len(vocab_to_int)+1 # +1 for the 0 padding + our word tokens\n",
        "output_size = 1\n",
        "embedding_dim = 400\n",
        "hidden_dim = 256\n",
        "n_layers = 2\n",
        "\n",
        "net = SentimentRNN(vocab_size, output_size, embedding_dim, hidden_dim, n_layers)\n",
        "\n",
        "print(net)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SentimentRNN(\n",
            "  (embedding): Embedding(74073, 400)\n",
            "  (lstm): LSTM(400, 256, num_layers=2, batch_first=True, dropout=0.5)\n",
            "  (dropout): Dropout(p=0.3, inplace=False)\n",
            "  (fc): Linear(in_features=256, out_features=1, bias=True)\n",
            "  (sig): Sigmoid()\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6cGwLc4STXas",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "# Training\n",
        "\n",
        ">We'll use a new kind of cross entropy loss, which is designed to work with a single Sigmoid output. BCELoss or **Binary Cross Entropy Loss**, applies cross entropy loss to a single value between 0 and 1.\n",
        "\n",
        "We also have some data and training hyparameters:\n",
        "\n",
        "* `lr`: Learning rate for our optimizer.\n",
        "* `epochs`: Number of times to iterate through the training dataset.\n",
        "* `clip`: The maximum gradient value to clip at (to prevent exploding gradients)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zbquM_w9cCde",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# loss and optimization functions\n",
        "lr=0.001\n",
        "\n",
        "criterion = nn.BCELoss()\n",
        "optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fH5lQoNDcn5U",
        "colab_type": "code",
        "outputId": "76800aed-04cd-4aee-d783-73845a7699e1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        }
      },
      "source": [
        "# training params\n",
        "\n",
        "epochs = 4 # 3-4 is approx where I noticed the validation loss stop decreasing\n",
        "\n",
        "counter = 0\n",
        "print_every = 100\n",
        "clip=5 # gradient clipping\n",
        "\n",
        "# move model to GPU, if available\n",
        "if(train_on_gpu):\n",
        "    net.cuda()\n",
        "\n",
        "net.train()\n",
        "# train for some number of epochs\n",
        "for e in range(epochs):\n",
        "    # initialize hidden state\n",
        "    h = net.init_hidden(batch_size)\n",
        "\n",
        "    # batch loop\n",
        "    for inputs, labels in train_loader:\n",
        "        counter += 1\n",
        "\n",
        "        if(train_on_gpu):\n",
        "            inputs, labels = inputs.cuda(), labels.cuda()\n",
        "\n",
        "        # Creating new variables for the hidden state, otherwise\n",
        "        # we'd backprop through the entire training history\n",
        "        h = tuple([each.data for each in h])\n",
        "\n",
        "        # zero accumulated gradients\n",
        "        net.zero_grad()\n",
        "\n",
        "        # get the output from the model\n",
        "        output, h = net(inputs, h)\n",
        "\n",
        "        # calculate the loss and perform backprop\n",
        "        loss = criterion(output.squeeze(), labels.float())\n",
        "        loss.backward()\n",
        "        # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
        "        nn.utils.clip_grad_norm_(net.parameters(), clip)\n",
        "        optimizer.step()\n",
        "\n",
        "        # loss stats\n",
        "        if counter % print_every == 0:\n",
        "            # Get validation loss\n",
        "            val_h = net.init_hidden(batch_size)\n",
        "            val_losses = []\n",
        "            net.eval()\n",
        "            for inputs, labels in valid_loader:\n",
        "\n",
        "                # Creating new variables for the hidden state, otherwise\n",
        "                # we'd backprop through the entire training history\n",
        "                val_h = tuple([each.data for each in val_h])\n",
        "\n",
        "                if(train_on_gpu):\n",
        "                    inputs, labels = inputs.cuda(), labels.cuda()\n",
        "\n",
        "                output, val_h = net(inputs, val_h)\n",
        "                val_loss = criterion(output.squeeze(), labels.float())\n",
        "\n",
        "                val_losses.append(val_loss.item())\n",
        "\n",
        "            net.train()\n",
        "            print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
        "                  \"Step: {}...\".format(counter),\n",
        "                  \"Loss: {:.6f}...\".format(loss.item()),\n",
        "                  \"Val Loss: {:.6f}\".format(np.mean(val_losses)))"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1/4... Step: 100... Loss: 0.651402... Val Loss: 0.639391\n",
            "Epoch: 1/4... Step: 200... Loss: 0.666441... Val Loss: 0.650215\n",
            "Epoch: 1/4... Step: 300... Loss: 0.578398... Val Loss: 0.588607\n",
            "Epoch: 1/4... Step: 400... Loss: 0.488081... Val Loss: 0.542887\n",
            "Epoch: 2/4... Step: 500... Loss: 0.473556... Val Loss: 0.514368\n",
            "Epoch: 2/4... Step: 600... Loss: 0.474001... Val Loss: 0.537137\n",
            "Epoch: 2/4... Step: 700... Loss: 0.387720... Val Loss: 0.493411\n",
            "Epoch: 2/4... Step: 800... Loss: 0.420487... Val Loss: 0.441201\n",
            "Epoch: 3/4... Step: 900... Loss: 0.369323... Val Loss: 0.481768\n",
            "Epoch: 3/4... Step: 1000... Loss: 0.473760... Val Loss: 0.483815\n",
            "Epoch: 3/4... Step: 1100... Loss: 0.395666... Val Loss: 0.489816\n",
            "Epoch: 3/4... Step: 1200... Loss: 0.540236... Val Loss: 0.591278\n",
            "Epoch: 4/4... Step: 1300... Loss: 0.250454... Val Loss: 0.522298\n",
            "Epoch: 4/4... Step: 1400... Loss: 0.252285... Val Loss: 0.518782\n",
            "Epoch: 4/4... Step: 1500... Loss: 0.240984... Val Loss: 0.515339\n",
            "Epoch: 4/4... Step: 1600... Loss: 0.380001... Val Loss: 0.558095\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aXWzqfQbTXax",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "## Testing\n",
        "\n",
        "There are a few ways to test your network.\n",
        "\n",
        "* **Test data performance:** First, we'll see how our trained model performs on all of our defined test_data, above. We'll calculate the average loss and accuracy over the test data.\n",
        "\n",
        "* **Inference on user-generated data:** Second, we'll see if we can input just one example review at a time (without a label), and see what the trained model predicts. Looking at new, user input data like this, and predicting an output label, is called **inference**."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WvU2mNikcxtS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "32ecee3f-ea86-4353-eb02-13d38752f781"
      },
      "source": [
        "# Get test data loss and accuracy\n",
        "\n",
        "test_losses = [] # track loss\n",
        "num_correct = 0\n",
        "\n",
        "# init hidden state\n",
        "h = net.init_hidden(batch_size)\n",
        "\n",
        "net.eval()\n",
        "# iterate over test data\n",
        "for inputs, labels in test_loader:\n",
        "\n",
        "    # Creating new variables for the hidden state, otherwise\n",
        "    # we'd backprop through the entire training history\n",
        "    h = tuple([each.data for each in h])\n",
        "\n",
        "    if(train_on_gpu):\n",
        "        inputs, labels = inputs.cuda(), labels.cuda()\n",
        "    \n",
        "    # get predicted outputs\n",
        "    output, h = net(inputs, h)\n",
        "    \n",
        "    # calculate loss\n",
        "    test_loss = criterion(output.squeeze(), labels.float())\n",
        "    test_losses.append(test_loss.item())\n",
        "    \n",
        "    # convert output probabilities to predicted class (0 or 1)\n",
        "    pred = torch.round(output.squeeze())  # rounds to the nearest integer\n",
        "    \n",
        "    # compare predictions to true label\n",
        "    correct_tensor = pred.eq(labels.float().view_as(pred))\n",
        "    correct = np.squeeze(correct_tensor.numpy()) if not train_on_gpu else np.squeeze(correct_tensor.cpu().numpy())\n",
        "    num_correct += np.sum(correct)\n",
        "\n",
        "\n",
        "# -- stats! -- ##\n",
        "# avg test loss\n",
        "print(\"Test loss: {:.3f}\".format(np.mean(test_losses)))\n",
        "\n",
        "# accuracy over all test data\n",
        "test_acc = num_correct/len(test_loader.dataset)\n",
        "print(\"Test accuracy: {:.3f}\".format(test_acc))"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.538\n",
            "Test accuracy: 0.799\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rDGFubAMTXa2",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "#Example on a review"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fasc8JxCdKiV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# positive review\n",
        "test_review_neg = 'The best movie I have seen; acting was terrific and it was value for money. This movie had the best acting and the dialogue was awesome.'\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "chKNxfUfdkOU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "db660e92-c72d-4310-f9c6-3a1ad6c871fd"
      },
      "source": [
        "from string import punctuation\n",
        "\n",
        "def tokenize_review(test_review):\n",
        "    test_review = test_review.lower() # lowercase\n",
        "    # get rid of punctuation\n",
        "    test_text = ''.join([c for c in test_review if c not in punctuation])\n",
        "\n",
        "    # splitting by spaces\n",
        "    test_words = test_text.split()\n",
        "\n",
        "    # tokens\n",
        "    test_ints = []\n",
        "    test_ints.append([vocab_to_int[word] for word in test_words])\n",
        "\n",
        "    return test_ints\n",
        "\n",
        "# test code and generate tokenized review\n",
        "test_ints = tokenize_review(test_review_neg)\n",
        "print(test_ints)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1, 117, 18, 10, 28, 108, 113, 14, 1279, 2, 8, 14, 1085, 16, 273, 11, 18, 68, 1, 117, 113, 2, 1, 410, 14, 1167]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1rhmxBPadn4Z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "256cd314-0e59-4c00-f4d9-d9b74a75a900"
      },
      "source": [
        "# test sequence padding\n",
        "seq_length=200\n",
        "features = pad_features(test_ints, seq_length)\n",
        "\n",
        "print(features)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[   0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "     0    0    0    0    0    0    1  117   18   10   28  108  113   14\n",
            "  1279    2    8   14 1085   16  273   11   18   68    1  117  113    2\n",
            "     1  410   14 1167]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0vq266AydpXS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict(net, test_review, sequence_length=200):\n",
        "    \n",
        "    net.eval()\n",
        "    \n",
        "    # tokenize review\n",
        "    test_ints = tokenize_review(test_review)\n",
        "    \n",
        "    # pad tokenized sequence\n",
        "    seq_length=sequence_length\n",
        "    features = pad_features(test_ints, seq_length)\n",
        "    \n",
        "    # convert to tensor to pass into your model\n",
        "    feature_tensor = torch.from_numpy(features)\n",
        "    \n",
        "    batch_size = feature_tensor.size(0)\n",
        "    \n",
        "    # initialize hidden state\n",
        "    h = net.init_hidden(batch_size)\n",
        "    \n",
        "    if(train_on_gpu):\n",
        "        feature_tensor = feature_tensor.cuda()\n",
        "    \n",
        "    # get the output from the model\n",
        "    output, h = net(feature_tensor, h)\n",
        "    \n",
        "    # convert output probabilities to predicted class (0 or 1)\n",
        "    pred = torch.round(output.squeeze()) \n",
        "    # printing output value, before rounding\n",
        "    print('Pred value, pre-rounding: {:.6f}'.format(output.item()))\n",
        "    \n",
        "    # print custom response\n",
        "    if(pred.item()==1):\n",
        "        print(\"Pos rev!\")\n",
        "    else:\n",
        "        print(\"Neg rev!\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qkg0ucXaeAAR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "42f3d5e8-9582-4d13-d4b9-13bfaa211421"
      },
      "source": [
        "# call function\n",
        "# trying negative and positive reviews!\n",
        "seq_length=200\n",
        "predict(net, test_review_neg, seq_length)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Pred value, pre-rounding: 0.073534\n",
            "Neg rev!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EIih6vVZeGFH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# neg test review\n",
        "test_review_pos = 'This movie had the worst acting and the dialogue was so bad. I vomited upon leaving.'\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ncVcS5veTcc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "d8e3c3cc-9a13-46dd-9af5-d292b132d0ae"
      },
      "source": [
        "# call function\n",
        "# try negative and positive reviews!\n",
        "seq_length=200\n",
        "predict(net, test_review_pos, seq_length)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Pred value, pre-rounding: 0.004821\n",
            "Neg rev!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J8Y5mG87eamD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.save(net.state_dict(), 'sentiment.pth')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "04G-OfKoebbT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "457705b6-6a8f-4a77-804d-b64e12250dd3"
      },
      "source": [
        "!ls"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "data  sample_data  sentiment.pth\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bDu2rtuJ2khL",
        "colab_type": "text"
      },
      "source": [
        "# Converted PyTorch Model to ONNX file\n",
        "\n",
        "![alt text](https://azurecomcdn.azureedge.net/mediahandler/acomblog/media/Default/blog/13189eea-5f0f-4ef9-83d1-92679b3a60f6.png)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R5HZR4_oeuiv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "outputId": "298ad6cd-e624-4400-e4a4-0b1dc427e379"
      },
      "source": [
        "from torch.autograd import Variable\n",
        "import torch.onnx\n",
        "\n",
        "# Load the trained model from file\n",
        "net_save = SentimentRNN(vocab_size, output_size, embedding_dim, hidden_dim, n_layers)\n",
        "\n",
        "trained_model = net_save\n",
        "trained_model.load_state_dict(torch.load('sentiment.pth'))\n",
        "\n",
        "# Export the trained model to ONNX\n",
        "dummy_input = Variable(torch.randn(1, 1, 28, 28)) # one black and white 28 x 28 picture will be the input to the model\n",
        "torch.onnx.export(trained_model, dummy_input, \"sentiment.onnx\")"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-32-a9d25627b347>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;31m# Export the trained model to ONNX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mdummy_input\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# one black and white 28 x 28 picture will be the input to the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0monnx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrained_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdummy_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"sentiment.onnx\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/onnx/__init__.py\u001b[0m in \u001b[0;36mexport\u001b[0;34m(model, args, f, export_params, verbose, training, input_names, output_names, aten, export_raw_ir, operator_export_type, opset_version, _retain_param_name, do_constant_folding, example_outputs, strip_doc_string, dynamic_axes)\u001b[0m\n\u001b[1;32m    130\u001b[0m                         \u001b[0moperator_export_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopset_version\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_retain_param_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m                         \u001b[0mdo_constant_folding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexample_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m                         strip_doc_string, dynamic_axes)\n\u001b[0m\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/onnx/utils.py\u001b[0m in \u001b[0;36mexport\u001b[0;34m(model, args, f, export_params, verbose, training, input_names, output_names, aten, export_raw_ir, operator_export_type, opset_version, _retain_param_name, do_constant_folding, example_outputs, strip_doc_string, dynamic_axes)\u001b[0m\n\u001b[1;32m     62\u001b[0m             \u001b[0moperator_export_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moperator_export_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopset_version\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mopset_version\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m             \u001b[0m_retain_param_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_retain_param_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdo_constant_folding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdo_constant_folding\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m             example_outputs=example_outputs, strip_doc_string=strip_doc_string, dynamic_axes=dynamic_axes)\n\u001b[0m\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/onnx/utils.py\u001b[0m in \u001b[0;36m_export\u001b[0;34m(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, export_type, example_outputs, propagate, opset_version, _retain_param_name, do_constant_folding, strip_doc_string, dynamic_axes)\u001b[0m\n\u001b[1;32m    327\u001b[0m                                                         \u001b[0moutput_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperator_export_type\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m                                                         \u001b[0mexample_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpropagate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 329\u001b[0;31m                                                         _retain_param_name, do_constant_folding)\n\u001b[0m\u001b[1;32m    330\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m         \u001b[0;31m# TODO: Don't allocate a in-memory string for the protobuf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/onnx/utils.py\u001b[0m in \u001b[0;36m_model_to_graph\u001b[0;34m(model, args, verbose, training, input_names, output_names, operator_export_type, example_outputs, propagate, _retain_param_name, do_constant_folding, _disable_torch_constant_prop)\u001b[0m\n\u001b[1;32m    211\u001b[0m             model.graph, tuple(in_vars), False, propagate)\n\u001b[1;32m    212\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m         \u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_trace_and_get_graph_from_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    214\u001b[0m         \u001b[0mstate_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_unique_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m         \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/onnx/utils.py\u001b[0m in \u001b[0;36m_trace_and_get_graph_from_model\u001b[0;34m(model, args, training)\u001b[0m\n\u001b[1;32m    169\u001b[0m     \u001b[0;31m# training mode was.)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mset_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m         \u001b[0mtrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_trace_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_force_outplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    172\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    173\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0morig_state_dict_keys\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0m_unique_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py\u001b[0m in \u001b[0;36mget_trace_graph\u001b[0;34m(f, args, kwargs, _force_outplace, return_inputs)\u001b[0m\n\u001b[1;32m    254\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m         \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 256\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mLegacyTracedModule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_force_outplace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m    321\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    322\u001b[0m             \u001b[0mtrace_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_unflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_trace_inputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0min_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0min_desc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 323\u001b[0;31m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mtrace_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    324\u001b[0m             \u001b[0mout_vars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_flatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tracer_exit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    543\u001b[0m                 \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 545\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    547\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    529\u001b[0m         \u001b[0mtracing_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_traced_module_stack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    530\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 531\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    532\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    533\u001b[0m             \u001b[0mtracing_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: forward() missing 1 required positional argument: 'hidden'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A6uP4PiL23d7",
        "colab_type": "text"
      },
      "source": [
        "## Converted ONNX to Tensorflow then download"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3p3d408n228b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##git clone git@github.com:onnx/onnx-tensorflow.git && cd onnx-tensorflow\n",
        "### pip install -e .\n",
        "\n",
        "import onnx\n",
        "import warnings\n",
        "from onnx_tf.backend import prepare\n",
        "\n",
        "model = onnx.load('sentiment.onnx') # Load the ONNX file\n",
        "tf_rep = prepare(model) # Import the ONNX model to Tensorflow\n",
        "tf_rep.export_graph(\"tb_pb/sentiment.pb\")"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}