Sentiment analysis pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis with variable length sequences in pytorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Load Data\n",
    "2. Simple Data Analysis\n",
    "3. Build Vocabulary and tokenize\n",
    "4. Unpadded dataset and dataloader\n",
    "5. Padded dataset and dataloader\n",
    "6. Simple GRU model\n",
    "6. GRU model with concat pooling\n",
    "7. Max Pooling and Average Pooling\n",
    "8. Train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import os, sys\n",
    "import re\n",
    "import string\n",
    "import pathlib\n",
    "import random\n",
    "from collections import Counter, OrderedDict\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import nltk\n",
    "import spacy\n",
    "from tqdm import tqdm, tqdm_notebook, tnrange\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from sklearn.model_selection import StratifiedShuffleSplit, train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity='all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.6.4 | packaged by conda-forge | (default, Dec 23 2017, 16:31:06) \n",
      "[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]\n",
      "Pandas version: 0.22.0\n",
      "Pytorch version: 0.3.1\n",
      "Spacy version: 2.0.8\n"
     ]
    }
   ],
   "source": [
    "print('Python version:',sys.version)\n",
    "print('Pandas version:',pd.__version__)\n",
    "print('Pytorch version:', torch.__version__)\n",
    "print('Spacy version:', spacy.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_root = pathlib.Path('./data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 8836: expected 4 fields, saw 5\\n'\n",
      "b'Skipping line 535882: expected 4 fields, saw 7\\n'\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1578612, 4)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ItemID</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>SentimentSource</th>\n",
       "      <th>SentimentText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>is so sad for my APL frie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>I missed the New Moon trail...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>omg its already 7:30 :O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>.. Omgaga. Im sooo  im gunna CRy. I'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>i think mi bf is cheating on me!!!   ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ItemID  Sentiment SentimentSource  \\\n",
       "0       1          0    Sentiment140   \n",
       "1       2          0    Sentiment140   \n",
       "2       3          1    Sentiment140   \n",
       "3       4          0    Sentiment140   \n",
       "4       5          0    Sentiment140   \n",
       "\n",
       "                                       SentimentText  \n",
       "0                       is so sad for my APL frie...  \n",
       "1                     I missed the New Moon trail...  \n",
       "2                            omg its already 7:30 :O  \n",
       "3            .. Omgaga. Im sooo  im gunna CRy. I'...  \n",
       "4           i think mi bf is cheating on me!!!   ...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(data_root/'Sentiment Analysis Dataset.csv', error_bad_lines=False)\n",
    "df.shape\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    790177\n",
       "0    788435\n",
       "Name: Sentiment, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Sentiment.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Simple Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgoAAAFACAYAAADd6lTCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGwhJREFUeJzt3X+w3fVd5/HnyyAttVIoDSwmVNBmXGm1FLIQ7W63FjeE+iPMTpmluyvZymycLnX9sVul7o7Rtox1dewu3cpuViKho6X0h0vUYJql1a4OUEJBKMWaK1W4BktoKFKppanv/eN8sh5uz+fec1JO7s3N8zFz5ny/7+/n8/187h+Z88r3Z6oKSZKkUb5usScgSZKWLoOCJEnqMihIkqQug4IkSeoyKEiSpC6DgiRJ6jIoSJKkLoOCJEnqMihIkqSu4xZ7AkvFi170ojrzzDMXexqSJB0Rd91112NVtXKhdgaF5swzz2TPnj2LPQ1Jko6IJH8xTjtPPUiSpC6DgiRJ6jIoSJKkLoOCJEnqmmpQSPITSe5P8skk703y3CRnJbkjyd4k70tyfGv7nLY+07afObSft7T6p5NcNFTf0GozSa4aqo8cQ5IkTWZqQSHJKuDfA2ur6mXACuAy4BeBd1bVGuBx4IrW5Qrg8ap6CfDO1o4kZ7d+LwU2AL+aZEWSFcC7gYuBs4HXt7bMM4YkSZrAtE89HAeckOQ44HnAI8BrgA+07duBS9ryxrZO235hkrT6jVX1par6DDADnN8+M1X1YFU9DdwIbGx9emNIkqQJTC0oVNVfAr8MPMQgIDwB3AV8vqoOtmazwKq2vAp4uPU92NqfMlyf06dXP2WeMSRJ0gSmeerhZAZHA84Cvgn4BganCeaqQ106256t+qg5bk6yJ8me/fv3j2oiSdIxbZqnHr4X+ExV7a+qLwMfAr4bOKmdigBYDexry7PAGQBt+wuAA8P1OX169cfmGeMZqmprVa2tqrUrVy74FEtJko450wwKDwHrkjyvXTdwIfAp4KPA61qbTcDNbXlHW6dt/0hVVatf1u6KOAtYA3wcuBNY0+5wOJ7BBY87Wp/eGJIkaQJTe9dDVd2R5APAJ4CDwN3AVuB3gRuTvL3VrmtdrgPek2SGwZGEy9p+7k9yE4OQcRC4sqq+ApDkTcAuBndUbKuq+9u+frozxhF33ptvWKyhpWfVXb90+WJPQdIimOpLoapqC7BlTvlBBncszG37t8Clnf1cDVw9or4T2DmiPnIMSceOh976HYs9BelZ8eKfvW9Rx/fJjJIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkrqmFhSSfFuSe4Y+f53kx5O8MMnuJHvb98mtfZJck2Qmyb1Jzh3a16bWfm+STUP185Lc1/pckyStPnIMSZI0makFhar6dFWdU1XnAOcBTwG/BVwF3FpVa4Bb2zrAxcCa9tkMXAuDH31gC3ABcD6wZeiH/9rW9lC/Da3eG0OSJE3gSJ16uBD4s6r6C2AjsL3VtwOXtOWNwA01cDtwUpLTgYuA3VV1oKoeB3YDG9q2E6vqtqoq4IY5+xo1hiRJmsCRCgqXAe9ty6dV1SMA7fvUVl8FPDzUZ7bV5qvPjqjPN8YzJNmcZE+SPfv37z/MP02SpOVr6kEhyfHADwLvX6jpiFodRn1sVbW1qtZW1dqVK1dO0lWSpGPCkTiicDHwiar6bFv/bDttQPt+tNVngTOG+q0G9i1QXz2iPt8YkiRpAkciKLyevz/tALADOHTnwibg5qH65e3uh3XAE+20wS5gfZKT20WM64FdbduTSda1ux0un7OvUWNIkqQJHDfNnSd5HvDPgB8ZKr8DuCnJFcBDwKWtvhN4LTDD4A6JNwBU1YEkbwPubO3eWlUH2vIbgeuBE4Bb2me+MSRJ0gSmGhSq6inglDm1zzG4C2Ju2wKu7OxnG7BtRH0P8LIR9ZFjSJKkyfhkRkmS1GVQkCRJXQYFSZLUZVCQJEldBgVJktRlUJAkSV0GBUmS1GVQkCRJXQYFSZLUZVCQJEldBgVJktRlUJAkSV0GBUmS1GVQkCRJXQYFSZLUZVCQJEldBgVJktRlUJAkSV0GBUmS1GVQkCRJXQYFSZLUZVCQJEldBgVJktRlUJAkSV1TDQpJTkrygSR/kuSBJN+V5IVJdifZ275Pbm2T5JokM0nuTXLu0H42tfZ7k2waqp+X5L7W55okafWRY0iSpMlM+4jCfwN+r6r+IfBy4AHgKuDWqloD3NrWAS4G1rTPZuBaGPzoA1uAC4DzgS1DP/zXtraH+m1o9d4YkiRpAlMLCklOBF4FXAdQVU9X1eeBjcD21mw7cElb3gjcUAO3AyclOR24CNhdVQeq6nFgN7ChbTuxqm6rqgJumLOvUWNIkqQJTPOIwrcA+4FfT3J3kl9L8g3AaVX1CED7PrW1XwU8PNR/ttXmq8+OqDPPGM+QZHOSPUn27N+///D/UkmSlqlpBoXjgHOBa6vqFcDfMP8pgIyo1WHUx1ZVW6tqbVWtXbly5SRdJUk6JkwzKMwCs1V1R1v/AIPg8Nl22oD2/ehQ+zOG+q8G9i1QXz2izjxjSJKkCUwtKFTVXwEPJ/m2VroQ+BSwAzh058Im4Oa2vAO4vN39sA54op022AWsT3Jyu4hxPbCrbXsyybp2t8Plc/Y1agxJkjSB46a8/x8FfiPJ8cCDwBsYhJObklwBPARc2truBF4LzABPtbZU1YEkbwPubO3eWlUH2vIbgeuBE4Bb2gfgHZ0xJEnSBKYaFKrqHmDtiE0XjmhbwJWd/WwDto2o7wFeNqL+uVFjSJKkyfhkRkmS1GVQkCRJXQYFSZLUZVCQJEldBgVJktRlUJAkSV0GBUmS1GVQkCRJXQYFSZLUZVCQJEldBgVJktRlUJAkSV0GBUmS1GVQkCRJXQYFSZLUZVCQJEldBgVJktRlUJAkSV0GBUmS1GVQkCRJXQYFSZLUZVCQJEldBgVJktRlUJAkSV1TDQpJ/jzJfUnuSbKn1V6YZHeSve375FZPkmuSzCS5N8m5Q/vZ1NrvTbJpqH5e2/9M65v5xpAkSZM5EkcUvqeqzqmqtW39KuDWqloD3NrWAS4G1rTPZuBaGPzoA1uAC4DzgS1DP/zXtraH+m1YYAxJkjSBxTj1sBHY3pa3A5cM1W+ogduBk5KcDlwE7K6qA1X1OLAb2NC2nVhVt1VVATfM2deoMSRJ0gSmHRQK+HCSu5JsbrXTquoRgPZ9aquvAh4e6jvbavPVZ0fU5xvjGZJsTrInyZ79+/cf5p8oSdLyddyU9//KqtqX5FRgd5I/madtRtTqMOpjq6qtwFaAtWvXTtRXkqRjwVSPKFTVvvb9KPBbDK4x+Gw7bUD7frQ1nwXOGOq+Gti3QH31iDrzjCFJkiYwtaCQ5BuSfOOhZWA98ElgB3DozoVNwM1teQdwebv7YR3wRDttsAtYn+TkdhHjemBX2/ZkknXtbofL5+xr1BiSJGkCY516SPLKqvqjhWpznAb8Vrtj8TjgN6vq95LcCdyU5ArgIeDS1n4n8FpgBngKeANAVR1I8jbgztburVV1oC2/EbgeOAG4pX0A3tEZQ5IkTWDcaxTeBZw7Ru3/q6oHgZePqH8OuHBEvYArO/vaBmwbUd8DvGzcMSRJ0mTmDQpJvgv4bmBlkp8c2nQisGKaE5MkSYtvoSMKxwPPb+2+caj+18DrpjUpSZK0NMwbFKrqD4A/SHJ9Vf3FEZqTJElaIsa9RuE5SbYCZw73qarXTGNSkiRpaRg3KLwf+B/ArwFfmd50JEnSUjJuUDhYVddOdSaSJGnJGfeBS7+d5N8lOb29wvmF7a2OkiRpGRv3iMKhpxy+eahWwLc8u9ORJElLyVhBoarOmvZEJEnS0jPWqYckz0vyn9udDyRZk+T7pzs1SZK02Ma9RuHXgacZPKURBm9ufPtUZiRJkpaMcYPCt1bVfwG+DFBVXwQytVlJkqQlYdyg8HSSExhcwEiSbwW+NLVZSZKkJWHcux62AL8HnJHkN4BXAv9mWpOSJElLw7h3PexO8glgHYNTDj9WVY9NdWaSJGnRjXvqAWAVg1dLHw+8Ksk/n86UJEnSUjHWEYUk24DvBO4H/q6VC/jQlOYlSZKWgHGvUVhXVWdPdSaSJGnJGffUw21JDAqSJB1jxj2isJ1BWPgrBrdFBqiq+s6pzUySJC26cYPCNuCHgPv4+2sUJEnSMjduUHioqnZMdSaSJGnJGTco/EmS3wR+m6EnMlaVdz1IkrSMjXsx4wkMAsJ64AfaZ6y3RyZZkeTuJL/T1s9KckeSvUnel+T4Vn9OW59p288c2sdbWv3TSS4aqm9otZkkVw3VR44hSZImM1ZQqKo3jPj88Jhj/BjwwND6LwLvrKo1wOPAFa1+BfB4Vb0EeGdrR7vb4jLgpcAG4Fdb+FgBvBu4GDgbeP3QnRm9MSRJ0gTmDQpJfqp9vyvJNXM/C+08yWrg+4Bfa+sBXgN8oDXZDlzSlje2ddr2C1v7jcCNVfWlqvoMMAOc3z4zVfVgVT0N3AhsXGAMSZI0gYWuUTh0JGDPYe7/vwI/BXxjWz8F+HxVHWzrswweDU37fhigqg4meaK1XwXcPrTP4T4Pz6lfsMAYkiRpAvMGhar67bb4VFW9f3hbkkvn65vk+4FHq+quJK8+VB41zALbevVRR0Pmaz9qjpuBzQAvfvGLRzWRJOmYNu7FjG8ZszbslcAPJvlzBqcFXsPgCMNJSQ4FlNXAvrY8C5wB0La/ADgwXJ/Tp1d/bJ4xnqGqtlbV2qpau3LlygX+HEmSjj0LXaNwcZJ3AavmXJ9wPXBwvr5V9ZaqWl1VZzK4GPEjVfWvgI8Cr2vNNgE3t+UdbZ22/SNVVa1+Wbsr4ixgDfBx4E5gTbvD4fg2xo7WpzeGJEmawELXKOxjcH3CDwJ3DdWfBH7iMMf8aeDGJG8H7gaua/XrgPckmWFwJOEygKq6P8lNwKcYhJMrq+orAEneBOxi8PrrbVV1/wJjSJKkCSx0jcIfA3+c5Der6suHO0hV/T7w+235QQZ3LMxt87fAyOsequpq4OoR9Z3AzhH1kWNIkqTJjPtkxvOT/Bzwza3PoZdCfcu0JiZJkhbfuEHhOganGu4CvjK96UiSpKVk3KDwRFXdMtWZSJKkJWfcoPDRJL8EfIhnvhTqE1OZlSRJWhLGDQoXtO+1Q7Vi8GwESZK0TI0VFKrqe6Y9EUmStPSM9WTGJKcluS7JLW397CS+kVGSpGVu3Ec4X8/gwUbf1Nb/FPjxaUxIkiQtHeMGhRdV1U3A38Hg7Y54m6QkScveuEHhb5KcQnsLY5J1wBNTm5UkSVoSxr3r4ScZvJzpW5P8EbCSv3/pkiRJWqYWenvkP0ryD9rzEv4p8DMMnqPwYQaveZYkScvYQqce/ifwdFv+buA/Ae8GHge2TnFekiRpCVjo1MOKqjrQlv8FsLWqPgh8MMk9052aJElabAsdUViR5FCYuBD4yNC2ca9vkCRJR6mFfuzfC/xBkseALwL/FyDJS/CuB0mSlr15g0JVXZ3kVuB04MNVVW3T1wE/Ou3JSZKkxbXg6YOqun1E7U+nMx1JkrSUjPvAJUmSdAwyKEiSpC6DgiRJ6jIoSJKkLoOCJEnqMihIkqSuqQWFJM9N8vEkf5zk/iQ/3+pnJbkjyd4k70tyfKs/p63PtO1nDu3rLa3+6SQXDdU3tNpMkquG6iPHkCRJk5nmEYUvAa+pqpcD5wAbkqwDfhF4Z1WtYfByqSta+yuAx6vqJcA7WzuSnA1cBrwU2AD8apIVSVYweEHVxcDZwOtbW+YZQ5IkTWBqQaEGvtBWv759CngN8IFW3w5c0pY3tnXa9guTpNVvrKovVdVngBng/PaZqaoHq+pp4EZgY+vTG0OSJE1gqtcotP/53wM8CuwG/gz4fFUdbE1mgVVteRXwMEDb/gRwynB9Tp9e/ZR5xpAkSROYalCoqq9U1TnAagZHAL59VLP2nc62Z6v+VZJsTrInyZ79+/ePaiJJ0jHtiNz1UFWfB34fWAecNPTq6tXAvrY8C5wB0La/ADgwXJ/Tp1d/bJ4x5s5ra1Wtraq1K1eu/Fr+REmSlqVp3vWwMslJbfkE4HuBB4CPAq9rzTYBN7flHW2dtv0j7W2VO4DL2l0RZwFrgI8DdwJr2h0OxzO44HFH69MbQ5IkTWDBt0d+DU4Htre7E74OuKmqfifJp4Abk7wduBu4rrW/DnhPkhkGRxIuA6iq+5PcBHwKOAhcWVVfAUjyJmAXsALYVlX3t339dGcMSZI0gakFhaq6F3jFiPqDDK5XmFv/W+DSzr6uBq4eUd8J7Bx3DEmSNBmfzChJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKlrakEhyRlJPprkgST3J/mxVn9hkt1J9rbvk1s9Sa5JMpPk3iTnDu1rU2u/N8mmofp5Se5rfa5JkvnGkCRJk5nmEYWDwH+oqm8H1gFXJjkbuAq4tarWALe2dYCLgTXtsxm4FgY/+sAW4ALgfGDL0A//ta3toX4bWr03hiRJmsDUgkJVPVJVn2jLTwIPAKuAjcD21mw7cElb3gjcUAO3AyclOR24CNhdVQeq6nFgN7ChbTuxqm6rqgJumLOvUWNIkqQJHJFrFJKcCbwCuAM4raoegUGYAE5tzVYBDw91m221+eqzI+rMM8bceW1OsifJnv379x/unydJ0rI19aCQ5PnAB4Efr6q/nq/piFodRn1sVbW1qtZW1dqVK1dO0lWSpGPCVINCkq9nEBJ+o6o+1MqfbacNaN+PtvoscMZQ99XAvgXqq0fU5xtDkiRNYJp3PQS4Dnigqn5laNMO4NCdC5uAm4fql7e7H9YBT7TTBruA9UlObhcxrgd2tW1PJlnXxrp8zr5GjSFJkiZw3BT3/Urgh4D7ktzTaj8DvAO4KckVwEPApW3bTuC1wAzwFPAGgKo6kORtwJ2t3Vur6kBbfiNwPXACcEv7MM8YkiRpAlMLClX1h4y+jgDgwhHtC7iys69twLYR9T3Ay0bUPzdqDEmSNBmfzChJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKnLoCBJkroMCpIkqcugIEmSugwKkiSpy6AgSZK6DAqSJKlrakEhybYkjyb55FDthUl2J9nbvk9u9SS5JslMknuTnDvUZ1NrvzfJpqH6eUnua32uSZL5xpAkSZOb5hGF64ENc2pXAbdW1Rrg1rYOcDGwpn02A9fC4Ecf2AJcAJwPbBn64b+2tT3Ub8MCY0iSpAlNLShU1ceAA3PKG4HtbXk7cMlQ/YYauB04KcnpwEXA7qo6UFWPA7uBDW3biVV1W1UVcMOcfY0aQ5IkTehIX6NwWlU9AtC+T231VcDDQ+1mW22++uyI+nxjfJUkm5PsSbJn//79h/1HSZK0XC2VixkzolaHUZ9IVW2tqrVVtXblypWTdpckadk70kHhs+20Ae370VafBc4Yarca2LdAffWI+nxjSJKkCR3poLADOHTnwibg5qH65e3uh3XAE+20wS5gfZKT20WM64FdbduTSda1ux0un7OvUWNIkqQJHTetHSd5L/Bq4EVJZhncvfAO4KYkVwAPAZe25juB1wIzwFPAGwCq6kCStwF3tnZvrapDF0i+kcGdFScAt7QP84whSZImNLWgUFWv72y6cETbAq7s7GcbsG1EfQ/wshH1z40aQ5IkTW6pXMwoSZKWIIOCJEnqMihIkqQug4IkSeoyKEiSpC6DgiRJ6jIoSJKkLoOCJEnqMihIkqQug4IkSeoyKEiSpC6DgiRJ6jIoSJKkLoOCJEnqMihIkqQug4IkSeoyKEiSpC6DgiRJ6jIoSJKkLoOCJEnqMihIkqQug4IkSeoyKEiSpK5lGxSSbEjy6SQzSa5a7PlIknQ0WpZBIckK4N3AxcDZwOuTnL24s5Ik6eizLIMCcD4wU1UPVtXTwI3AxkWekyRJR53lGhRWAQ8Prc+2miRJmsBxiz2BKcmIWn1Vo2QzsLmtfiHJp6c6K03Li4DHFnsSy11+edNiT0FLk//+pm3LqJ+0Z8U3j9NouQaFWeCMofXVwL65japqK7D1SE1K05FkT1WtXex5SMci//0tf8v11MOdwJokZyU5HrgM2LHIc5Ik6aizLI8oVNXBJG8CdgErgG1Vdf8iT0uSpKPOsgwKAFW1E9i52PPQEeHpI2nx+O9vmUvVV13jJ0mSBCzfaxQkSdKzwKAgSZK6DAo6qvlOD2lxJNmW5NEkn1zsuWi6DAo6avlOD2lRXQ9sWOxJaPoMCjqa+U4PaZFU1ceAA4s9D02fQUFHM9/pIUlTZlDQ0Wysd3pIkg6fQUFHs7He6SFJOnwGBR3NfKeHJE2ZQUFHrao6CBx6p8cDwE2+00M6MpK8F7gN+LYks0muWOw5aTp8hLMkSeryiIIkSeoyKEiSpC6DgiRJ6jIoSJKkLoOCJEnqMihI+pol+cIEbX8uyX+c1v4lPbsMCpIkqcugIGkqkvxAkjuS3J3k/yQ5bWjzy5N8JMneJP92qM+bk9yZ5N4kPz9in6cn+ViSe5J8Msk/OSJ/jHQMMyhImpY/BNZV1SsYvAL8p4a2fSfwfcB3AT+b5JuSrAfWMHh9+DnAeUleNWef/xLYVVXnAC8H7pny3yAd845b7AlIWrZWA+9LcjpwPPCZoW03V9UXgS8m+SiDcPCPgfXA3a3N8xkEh48N9bsT2Jbk64H/XVUGBWnKPKIgaVreBfz3qvoO4EeA5w5tm/vs+GLw2vBfqKpz2uclVXXdMxpVfQx4FfCXwHuSXD696UsCg4Kk6XkBgx90gE1ztm1M8twkpwCvZnCkYBfww0meD5BkVZJThzsl+Wbg0ar6X8B1wLlTnL8kPPUg6dnxvCSzQ+u/Avwc8P4kfwncDpw1tP3jwO8CLwbeVlX7gH1Jvh24LQnAF4B/DTw61O/VwJuTfLlt94iCNGW+PVKSJHV56kGSJHUZFCRJUpdBQZIkdRkUJElSl0FBkiR1GRQkSVKXQUGSJHX9P7r5PllIE9HFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(8,5))\n",
    "ax = sns.barplot(x=df.Sentiment.unique(),y=df.Sentiment.value_counts());\n",
    "ax.set(xlabel='Labels');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Build Vocabulary and tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load spacy tokenizer\n",
    "nlp = spacy.load('en',disable=['parser', 'tagger', 'ner'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                     is so sad for my APL friend............."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp(df.SentimentText.values[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Progress: 100%|██████████| 1578612/1578612 [00:01<00:00, 1251595.62it/s]\n"
     ]
    }
   ],
   "source": [
    "# remove leading and trailing spaces\n",
    "df['SentimentText'] = df.SentimentText.progress_apply(lambda x: x.strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Use Counter to calculate the frequency of the unique words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9bc6bef4a634fb4b62d38eab21ca44f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1578612), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "words = Counter()\n",
    "for sent in tqdm_notebook(df.SentimentText.values):\n",
    "    words.update(w.text.lower() for w in nlp(sent))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Total unique words and top 20 most frequently occuring words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "773040"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[('i', 991632),\n",
       " ('!', 897920),\n",
       " ('.', 804192),\n",
       " ('to', 561225),\n",
       " (' ', 530969),\n",
       " ('the', 519913),\n",
       " (',', 480979),\n",
       " ('a', 378304),\n",
       " ('my', 313303),\n",
       " ('and', 302087),\n",
       " ('it', 301873),\n",
       " ('you', 298312),\n",
       " ('is', 244439),\n",
       " ('?', 235671),\n",
       " ('...', 219279),\n",
       " ('in', 215078),\n",
       " ('for', 214961),\n",
       " ('of', 182941),\n",
       " (\"'s\", 179227),\n",
       " ('that', 173790)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)\n",
    "words.most_common(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Sort the words according to their frequency in the corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " '!',\n",
       " '.',\n",
       " 'to',\n",
       " ' ',\n",
       " 'the',\n",
       " ',',\n",
       " 'a',\n",
       " 'my',\n",
       " 'and',\n",
       " 'it',\n",
       " 'you',\n",
       " 'is',\n",
       " '?',\n",
       " '...',\n",
       " 'in',\n",
       " 'for',\n",
       " 'of',\n",
       " \"'s\",\n",
       " 'that']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = sorted(words, key=words.get, reverse=True)\n",
    "words[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Add _PAD and _UNK token at the begining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['_PAD', '_UNK', 'i', '!', '.', 'to', ' ', 'the', ',', 'a']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = ['_PAD','_UNK'] + words\n",
    "words[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Construct a mapping of words to indicies and vice versa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = {o:i for i,o in enumerate(words)}\n",
    "idx2word = {i:o for i,o in enumerate(words)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indexer(s): return [word2idx[w.text.lower()] for w in nlp(s)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize and calculate tweet length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 46s, sys: 259 ms, total: 2min 47s\n",
      "Wall time: 2min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df['sentimentidx'] = df.SentimentText.apply(indexer)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ItemID</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>SentimentSource</th>\n",
       "      <th>SentimentText</th>\n",
       "      <th>sentimentidx</th>\n",
       "      <th>lengths</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>is so sad for my APL friend.............</td>\n",
       "      <td>[14, 26, 132, 18, 10, 241549, 266, 6621]</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>I missed the New Moon trailer...</td>\n",
       "      <td>[2, 272, 7, 90, 812, 1274, 16]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>omg its already 7:30 :O</td>\n",
       "      <td>[247, 82, 217, 4573, 1012]</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>.. Omgaga. Im sooo  im gunna CRy. I've been at...</td>\n",
       "      <td>[37, 241550, 4, 2, 73, 440, 6, 2, 73, 1454, 55...</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>i think mi bf is cheating on me!!!       T_T</td>\n",
       "      <td>[2, 93, 1815, 1342, 14, 5521, 23, 24, 3, 3, 3,...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ItemID  Sentiment SentimentSource  \\\n",
       "0       1          0    Sentiment140   \n",
       "1       2          0    Sentiment140   \n",
       "2       3          1    Sentiment140   \n",
       "3       4          0    Sentiment140   \n",
       "4       5          0    Sentiment140   \n",
       "\n",
       "                                       SentimentText  \\\n",
       "0           is so sad for my APL friend.............   \n",
       "1                   I missed the New Moon trailer...   \n",
       "2                            omg its already 7:30 :O   \n",
       "3  .. Omgaga. Im sooo  im gunna CRy. I've been at...   \n",
       "4       i think mi bf is cheating on me!!!       T_T   \n",
       "\n",
       "                                        sentimentidx  lengths  \n",
       "0           [14, 26, 132, 18, 10, 241549, 266, 6621]        8  \n",
       "1                     [2, 272, 7, 90, 812, 1274, 16]        7  \n",
       "2                         [247, 82, 217, 4573, 1012]        5  \n",
       "3  [37, 241550, 4, 2, 73, 440, 6, 2, 73, 1454, 55...       35  \n",
       "4  [2, 93, 1815, 1342, 14, 5521, 23, 24, 3, 3, 3,...       13  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['lengths'] = df.sentimentidx.apply(len)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Plot the frequency distribution of the lengths of the tokenized tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgsAAAFACAYAAAAyKD/8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG8hJREFUeJzt3XuQpXV95/H3JyCKt4BhIIRLIHF0RTcBHIHSmDUhwkBtRLOaAJbMKgm5wK5W3FrRpILlpUqTqJHdhATDRDCOQDRGNjuK46WiWWVguCwX0TBBAgMERocABoMBv/vH+fXmMJz+zemmz3T3mfer6tR5nu9z+/3m6Zn+zHNNVSFJkjSbH1jsBkiSpKXNsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqWv3xW7AUrHPPvvUIYccstjNkCRpp7j66qu/VVUrxpnXsNAccsghbNq0abGbIUnSTpHkH8ad19MQkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5Ikqct3Qyxj6zbe3p1+6tEH76SWSJKmmUcWJElSl2FBkiR1GRYkSVKXYUGSJHUZFiRJUpd3QywS72SQJC0XHlmQJEldEwsLSQ5K8sUkNye5KckbW/3tSe5Mcl37nDi0zFuTbE7yjSTHD9VXt9rmJGcP1Q9NsjHJLUkuSbJHqz+5jW9u0w+ZVD8lSZp2kzyy8Ajw5qp6HnAMcGaSw9q0D1TV4e2zHqBNOxl4PrAa+KMkuyXZDfhD4ATgMOCUofW8t61rJXAfcHqrnw7cV1XPBj7Q5pMkSfMwsbBQVXdX1TVt+EHgZuCAziInARdX1cNV9U1gM3BU+2yuqlur6nvAxcBJSQL8LPDxtvyFwCuH1nVhG/44cGybX5IkzdFOuWahnQY4AtjYSmcluT7J2iR7t9oBwB1Di21ptdnqPwT8U1U9sl39Metq0+9v82/frjOSbEqyaevWrU+oj5IkTauJh4UkTwc+Abypqh4AzgN+HDgcuBt438ysIxavedR763psoer8qlpVVatWrFjR7YckSbuqiYaFJE9iEBQ+WlV/CVBV91TVo1X1feBDDE4zwODIwEFDix8I3NWpfwvYK8nu29Ufs642/QeBbQvbO0mSdg2TvBsiwAXAzVX1/qH6/kOzvQq4sQ1fBpzc7mQ4FFgJXAlcBaxsdz7sweAiyMuqqoAvAq9uy68BPjW0rjVt+NXAF9r8kiRpjib5UKaXAK8DbkhyXau9jcHdDIczOC1wG/CrAFV1U5JLga8xuJPizKp6FCDJWcDlwG7A2qq6qa3vLcDFSd4FXMsgnNC+P5JkM4MjCidPsJ+SJE21iYWFqvpbRl87sL6zzLuBd4+orx+1XFXdyr+dxhiu/wvwmrm0V5IkjeYTHCVJUpdhQZIkdRkWJElSl2FBkiR1GRYkSVKXYUGSJHUZFiRJUpdhQZIkdRkWJElSl2FBkiR1GRYkSVKXYUGSJHUZFiRJUpdhQZIkdRkWJElSl2FBkiR1GRYkSVKXYUGSJHUZFiRJUpdhQZIkdRkWJElSl2FBkiR1GRYkSVKXYUGSJHUZFiRJUtfui90ATc66jbd3p5969ME7qSWSpOXMIwuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnL5ywsUTt6RoIkSTuLRxYkSVKXYUGSJHUZFiRJUpdhQZIkdRkWJElSl2FBkiR1GRYkSVLXxMJCkoOSfDHJzUluSvLGVn9Wkg1Jbmnfe7d6kpybZHOS65McObSuNW3+W5KsGaq/MMkNbZlzk6S3DUmSNHeTPLLwCPDmqnoecAxwZpLDgLOBz1fVSuDzbRzgBGBl+5wBnAeDX/zAOcDRwFHAOUO//M9r884st7rVZ9uGJEmao4mFhaq6u6quacMPAjcDBwAnARe22S4EXtmGTwIuqoErgL2S7A8cD2yoqm1VdR+wAVjdpj2zqr5aVQVctN26Rm1DkiTN0U65ZiHJIcARwEZgv6q6GwaBAti3zXYAcMfQYltarVffMqJOZxvbt+uMJJuSbNq6det8uydJ0lSbeFhI8nTgE8CbquqB3qwjajWP+tiq6vyqWlVVq1asWDGXRSVJ2mVMNCwkeRKDoPDRqvrLVr6nnUKgfd/b6luAg4YWPxC4awf1A0fUe9uQJElzNMm7IQJcANxcVe8fmnQZMHNHwxrgU0P109pdEccA97dTCJcDxyXZu13YeBxweZv2YJJj2rZO225do7YhSZLmaJKvqH4J8DrghiTXtdrbgPcAlyY5HbgdeE2bth44EdgMPAS8HqCqtiV5J3BVm+8dVbWtDf868GFgT+DT7UNnG5IkaY4mFhaq6m8ZfV0BwLEj5i/gzFnWtRZYO6K+CXjBiPq3R21DkiTNnU9wlCRJXYYFSZLUZViQJEldhgVJktRlWJAkSV2GBUmS1GVYkCRJXYYFSZLUZViQJEldhgVJktRlWJAkSV2GBUmS1GVYkCRJXYYFSZLUZViQJEldhgVJktRlWJAkSV2GBUmS1GVYkCRJXYYFSZLUZViQJEldhgVJktRlWJAkSV2GBUmS1GVYkCRJXYYFSZLUZViQJEldhgVJktS1+2I3QEvXuo23d6efevTBO6klkqTF5JEFSZLUZViQJEldhgVJktQ1VlhI8oJJN0SSJC1N4x5Z+OMkVyb5jSR7TbRFkiRpSRkrLFTVTwGvBQ4CNiVZl+TlE22ZJElaEsa+ZqGqbgF+G3gL8B+Ac5N8PckvTKpxkiRp8Y17zcJPJPkAcDPws8DPV9Xz2vAHJtg+SZK0yMZ9KNP/BD4EvK2qvjtTrKq7kvz2RFomSZKWhHHDwonAd6vqUYAkPwA8paoeqqqPTKx1kiRp0Y17zcLngD2Hxp/aapIkacqNGxaeUlXfmRlpw0/tLZBkbZJ7k9w4VHt7kjuTXNc+Jw5Ne2uSzUm+keT4ofrqVtuc5Oyh+qFJNia5JcklSfZo9Se38c1t+iFj9lGSJI0wblj45yRHzowkeSHw3c78AB8GVo+of6CqDm+f9W19hwEnA89vy/xRkt2S7Ab8IXACcBhwSpsX4L1tXSuB+4DTW/104L6qejaDiy/fO2YfJUnSCOOGhTcBf5Hky0m+DFwCnNVboKq+BGwbc/0nARdX1cNV9U1gM3BU+2yuqlur6nvAxcBJScLgToyPt+UvBF45tK4L2/DHgWPb/JIkaR7GusCxqq5K8u+A5wIBvl5V/zrPbZ6V5DRgE/DmqroPOAC4YmieLa0GcMd29aOBHwL+qaoeGTH/ATPLVNUjSe5v839r+4YkOQM4A+Dgg33dsiRJo8zlRVIvAn4COILB6YDT5rG984AfBw4H7gbe1+qj/udf86j31vX4YtX5VbWqqlatWLGi125JknZZYx1ZSPIRBr/krwMebeUCLprLxqrqnqF1fgj46za6hcGjpGccCNzVhkfVvwXslWT3dnRheP6ZdW1Jsjvwg4x/OkSSJG1n3OcsrAIOq6qR/0MfV5L9q+ruNvoqYOZOicuAdUneD/wIsBK4ksFRgpVJDgXuZHAR5KlVVUm+CLyawXUMa4BPDa1rDfDVNv0LT7TdkiTtysYNCzcCP8zg1MFYknwMeBmwT5ItwDnAy5IczuCoxG3ArwJU1U1JLgW+BjwCnDn0AKizgMuB3YC1VXVT28RbgIuTvAu4Frig1S8APpJkM4MjCieP22ZJkvR444aFfYCvJbkSeHimWFWvmG2BqjplRPmCEbWZ+d8NvHtEfT2wfkT9VgZ3S2xf/xfgNbNtR5Ikzc24YeHtk2yEJElausa9dfJvkvwosLKqPpfkqQxOC2gXtm7j7d3ppx7t7aiSNA3GfUX1rzB4wNGftNIBwF9NqlGSJGnpGPc5C2cCLwEeAKiqW4B9J9UoSZK0dIwbFh5uj1sGoD2/wNsRJUnaBYx7gePfJHkbsGeSlwO/AfyvyTVLO8OOrjmQJAnGP7JwNrAVuIHBsxHWA789qUZJkqSlY9y7Ib4PfKh9JEnSLmTcd0N8kxHXKFTVjy14iyRJ0pIyl3dDzHgKgyckPmvhmyNJkpaasa5ZqKpvD33urKo/AH52wm2TJElLwLinIY4cGv0BBkcanjGRFkmSpCVl3NMQ7xsafoTBGyN/ccFbI0mSlpxx74b4mUk3RJIkLU3jnob4zd70qnr/wjRHkiQtNXO5G+JFwGVt/OeBLwF3TKJRkiRp6Rg3LOwDHFlVDwIkeTvwF1X1y5NqmCRJWhrGfdzzwcD3hsa/Bxyy4K2RJElLzrhHFj4CXJnkkwye5Pgq4KKJtUqSJC0Z494N8e4knwZe2kqvr6prJ9csSZK0VIx7GgLgqcADVfVBYEuSQyfUJkmStISMFRaSnAO8BXhrKz0J+PNJNUqSJC0d4x5ZeBXwCuCfAarqLnzcsyRJu4Rxw8L3qqpor6lO8rTJNUmSJC0l44aFS5P8CbBXkl8BPgd8aHLNkiRJS8W4d0P8fpKXAw8AzwV+p6o2TLRlkiRpSdhhWEiyG3B5Vf0cYECQJGkXs8PTEFX1KPBQkh/cCe2RJElLzLhPcPwX4IYkG2h3RABU1X+dSKskSdKSMW5Y+N/tI0mSdjHdsJDk4Kq6vaou3FkNkiRJS8uOrln4q5mBJJ+YcFskSdIStKOwkKHhH5tkQyRJ0tK0o7BQswxLkqRdxI4ucPzJJA8wOMKwZxumjVdVPXOirZMkSYuuGxaqared1RBJkrQ0jftuCEmStIsyLEiSpC7DgiRJ6jIsSJKkLsOCJEnqmlhYSLI2yb1JbhyqPSvJhiS3tO+9Wz1Jzk2yOcn1SY4cWmZNm/+WJGuG6i9MckNb5twk6W1DkiTNzySPLHwYWL1d7Wzg81W1Evh8Gwc4AVjZPmcA58HgFz9wDnA0cBRwztAv//PavDPLrd7BNiRJ0jxMLCxU1ZeAbduVTwJmXkp1IfDKofpFNXAFsFeS/YHjgQ1Vta2q7gM2AKvbtGdW1VerqoCLtlvXqG1IkqR52NnXLOxXVXcDtO99W/0A4I6h+ba0Wq++ZUS9t43HSXJGkk1JNm3dunXenZIkaZotlQscM6JW86jPSVWdX1WrqmrVihUr5rq4JEm7hJ0dFu5ppxBo3/e2+hbgoKH5DgTu2kH9wBH13jYkSdI87OywcBkwc0fDGuBTQ/XT2l0RxwD3t1MIlwPHJdm7Xdh4HHB5m/ZgkmPaXRCnbbeuUduQJEnzsKO3Ts5bko8BLwP2SbKFwV0N7wEuTXI6cDvwmjb7euBEYDPwEPB6gKraluSdwFVtvndU1cxFk7/O4I6LPYFPtw+dbUiSpHmYWFioqlNmmXTsiHkLOHOW9awF1o6obwJeMKL+7VHbkCRJ87NULnCUJElLlGFBkiR1GRYkSVKXYUGSJHUZFiRJUpdhQZIkdRkWJElSl2FBkiR1GRYkSVLXxJ7guKtbt/H2xW6CJEkLwiMLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSunZf7AZoeq3beHt3+qlHH7yTWiJJeiI8siBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6FiUsJLktyQ1JrkuyqdWelWRDklva996tniTnJtmc5PokRw6tZ02b/5Yka4bqL2zr39yWzc7vpSRJ02Exjyz8TFUdXlWr2vjZwOeraiXw+TYOcAKwsn3OAM6DQbgAzgGOBo4CzpkJGG2eM4aWWz357kiSNJ2W0mmIk4AL2/CFwCuH6hfVwBXAXkn2B44HNlTVtqq6D9gArG7TnllVX62qAi4aWpckSZqjxQoLBXw2ydVJzmi1/arqboD2vW+rHwDcMbTsllbr1beMqD9OkjOSbEqyaevWrU+wS5IkTafdF2m7L6mqu5LsC2xI8vXOvKOuN6h51B9frDofOB9g1apVI+eRJGlXtyhHFqrqrvZ9L/BJBtcc3NNOIdC+722zbwEOGlr8QOCuHdQPHFGXJEnzsNPDQpKnJXnGzDBwHHAjcBkwc0fDGuBTbfgy4LR2V8QxwP3tNMXlwHFJ9m4XNh4HXN6mPZjkmHYXxGlD65IkSXO0GKch9gM+2e5m3B1YV1WfSXIVcGmS04Hbgde0+dcDJwKbgYeA1wNU1bYk7wSuavO9o6q2teFfBz4M7Al8un0kSdI87PSwUFW3Aj85ov5t4NgR9QLOnGVda4G1I+qbgBc84cZKkqQldeukJElaggwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKlr98VugHZd6zbe3p1+6tEH76SWSJJ6PLIgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSugwLkiSpy7AgSZK6DAuSJKnLsCBJkroMC5IkqcuwIEmSunZf7AZMSpLVwAeB3YA/rar3LHKTNEfrNt6+w3lOPfrgndASSdq1TeWRhSS7AX8InAAcBpyS5LDFbZUkScvTVIYF4Chgc1XdWlXfAy4GTlrkNkmStCxN62mIA4A7hsa3AEcvUls0QTs6VeFpCkl64qY1LGRErR43U3IGcEYb/U6SbyzAtvcBvrUA61mqllX/Xju32ZdV3+bB/i1v09y/ae4bLN3+/ei4M05rWNgCHDQ0fiBw1/YzVdX5wPkLueEkm6pq1UKucymZ5v5Nc9/A/i1309y/ae4bTEf/pvWahauAlUkOTbIHcDJw2SK3SZKkZWkqjyxU1SNJzgIuZ3Dr5NqqummRmyVJ0rI0lWEBoKrWA+sXYdMLelpjCZrm/k1z38D+LXfT3L9p7htMQf9S9bjr/iRJkv6/ab1mQZIkLRDDgiRJ6jIsLKAkq5N8I8nmJGcvdnueiCQHJflikpuT3JTkja3+9iR3JrmufU5c7LbOV5LbktzQ+rGp1Z6VZEOSW9r33ovdzvlI8tyhfXRdkgeSvGk5778ka5Pcm+TGodrI/ZWBc9vfxeuTHLl4Ld+xWfr2e0m+3tr/ySR7tfohSb47tA//ePFaPp5Z+jfrz2KSt7Z9940kxy9Oq8c3S/8uGerbbUmua/Vlt//AaxYWTHsfxd8BL2fwnIergFOq6muL2rB5SrI/sH9VXZPkGcDVwCuBXwS+U1W/v6gNXABJbgNWVdW3hmq/C2yrqve0wLd3Vb1lsdq4ENrP5p0MnmL6epbp/kvy08B3gIuq6gWtNnJ/tV88/wU4kUG/P1hVS/YprrP07TjgC+3urvcCtL4dAvz1zHzLwSz9ezsjfhbbe3w+xuCx/T8CfA54TlU9ulMbPQej+rfd9PcB91fVO5bj/gOPLCykqXofRVXdXVXXtOEHgZsZPEZ72p0EXNiGL2QQkJa7Y4G/r6p/WOyGPBFV9SVg23bl2fbXSQz+4a6qugLYqwXgJWlU36rqs1X1SBu9gsHD5ZalWfbdbE4CLq6qh6vqm8BmBv++Llm9/iUJg/9kfWynNmqBGRYWzqj3UUzFL9eWhI8ANrbSWe3Q6Nrlepi+KeCzSa7O4NHfAPtV1d0wCEzAvovWuoVzMo/9h2pa9h/Mvr+m7e/jG4BPD40fmuTaJH+T5KWL1agFMOpncdr23UuBe6rqlqHastt/hoWFM9b7KJabJE8HPgG8qaoeAM4Dfhw4HLgbeN8iNu+JeklVHcngVeZntkOJUyWDJ5i+AviLVpqm/dczNX8fk/wW8Ajw0Va6Gzi4qo4AfhNYl+SZi9W+J2C2n8Wp2XfNKTw2rC/L/WdYWDhjvY9iOUnyJAZB4aNV9ZcAVXVPVT1aVd8HPsQSPzzYU1V3te97gU8y6Ms9M4er2/e9i9fCBXECcE1V3QPTtf+a2fbXVPx9TLIG+I/Aa6tdYNYOz3+7DV8N/D3wnMVr5fx0fhanYt8BJNkd+AXgkpnact1/hoWFM1Xvo2jn2S4Abq6q9w/Vh8/7vgq4cftll4MkT2sXbpLkacBxDPpyGbCmzbYG+NTitHDBPOZ/NdOy/4bMtr8uA05rd0Ucw+DisrsXo4HzlWQ18BbgFVX10FB9RbtolSQ/BqwEbl2cVs5f52fxMuDkJE9OciiD/l25s9u3QH4O+HpVbZkpLNf9N7WPe97ZpvB9FC8BXgfcMHPLD/A24JQkhzM4LHgb8KuL07wnbD/gk4NMxO7Auqr6TJKrgEuTnA7cDrxmEdv4hCR5KoO7c4b30e8u1/2X5GPAy4B9kmwBzgHew+j9tZ7BnRCbgYcY3AWyZM3St7cCTwY2tJ/TK6rq14CfBt6R5BHgUeDXqmrciwcXxSz9e9mon8WquinJpcDXGJx+OXMp3wkBo/tXVRfw+OuFYBnuP/DWSUmStAOehpAkSV2GBUmS1GVYkCRJXYYFSZLUZViQJEldhgVpF5Xkh4befPePeewbAPdY4G29IckPzzLtz5NM7B0cSY5szyyYGX9XkjdNanvSNPI5C9Iuqj1F7nCY/Q2AC+gNwDXAP05o/T1HAi8APrMI25amgkcWJD1Gkrcl+Y02/D+SfLYNH5/kw234hCRfTXJNkkvaUzBJ8qL2cpyrk3w6yX5JfolBKLlkLkctkpyd5Mr2oqHfabVnJ7kxyQVJbmrbeEqbdkyb9ytJfq9ta0/gd4DXtvFXt9X/+9bOW5OcuXB/etJ0MixI2t6XGLwpDwb/K9+rPeP+p4AvJ9kXOBs4tr2I63rgjUmeDHwQ+E9V9ULgz4F3VtUlwHXAL1XV4e0V7l1JTgQOBo5mEDRenOTFbfJzgT+oqucD3+XfXkv9Z8AvV9WLaS8jqqrvAu9g8H6Tw6vq423e5zB4uuUxDJ6mt9vc/5ikXYenISRt7yrgRUn2Ar7D4JHJRzAIEB8BXgwcBnylPYZ4D+BvgecBzwc+1+q7MXgp0Hwcx+AlWNe28acz+AV/L7C5qm5o9auBQ5LsA+xRVTPvEFjH4Ln8s/nrFlruTbINWMHinCKRlgXDgqTHqKqHk9wFnAb8H+DvgGMZvFb375I8H/hMVb1ueLkkRwDXV9VLH7fSuQvwrvZ8/eFtPBt4eKj0KIN/x0a91rhn1DokzcLTEJJG+RLw39r3l4EzGfwvHuArwH9ob8ybeYPnSgYv/jkgyVGtvkcLFgAPAs+Yw/YvB04fuhbiwHb0YKSq2gr8a5JVrXTy0OS5blvSdgwLkkb5MoM3c26sqjuBf201quoe4HQGFyz+Xwbh4TlV9TDwauD9rX4tg2sOYHA9wZ92LnD80yRb2ufLVbUe+DhwRZIbgEsZnIroeQPwZ0m+AnwfuL/VvwD8ZJJrhy5wlDQHvnVS0lRI8vSq+k4b/i3gWVX15kVuljQVPE8naVq8Isl/Z/Dv2m3Af17U1khTxCMLkiSpy2sWJElSl2FBkiR1GRYkSVKXYUGSJHUZFiRJUtf/A9nAYOgan7ecAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(8,5))\n",
    "ax = sns.distplot(df.lengths.values,kde=False);\n",
    "ax.set(xlabel='Tweet Length', ylabel='Frequency');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Unpadded dataset and dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VectorizeData(Dataset):\n",
    "    def __init__(self, df_path):\n",
    "        self.df = pd.read_csv(df_path, error_bad_lines=False)\n",
    "        self.df['SentimentText'] = self.df.SentimentText.apply(lambda x: x.strip())\n",
    "        self.df['sentimentidx'] = self.df.SentimentText.apply(indexer)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.df.shape[0]\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        X = self.df.sentimentidx[idx]\n",
    "        y = self.df.Sentiment[idx]\n",
    "        return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 8836: expected 4 fields, saw 5\\n'\n",
      "b'Skipping line 535882: expected 4 fields, saw 7\\n'\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1578612"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = VectorizeData(data_root/'Sentiment Analysis Dataset.csv')\n",
    "len(ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's check the first 4 samples out of our custom dataset. It gives out the first 4 indexed sentiment and corresponding labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0             [14, 26, 132, 18, 10, 241549, 266, 6621]\n",
       " 1                       [2, 272, 7, 90, 812, 1274, 16]\n",
       " 2                           [247, 82, 217, 4573, 1012]\n",
       " 3    [37, 241550, 4, 2, 73, 440, 6, 2, 73, 1454, 55...\n",
       " Name: sentimentidx, dtype: object, 0    0\n",
       " 1    0\n",
       " 2    1\n",
       " 3    0\n",
       " Name: Sentiment, dtype: int64)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Minor Details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### If the data that comes out of the pytorch dataset is unpadded (if samples are of different lengths) then pytorch dataloader returns a python list instead of pytorch tensor with samples truncated to minimum length of the sample in the batch.\n",
    "\n",
    "##### First three samples are truncated to 5 (shortest tweet length in the batch) and returned as python list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total batches 526204\n"
     ]
    }
   ],
   "source": [
    "dl = DataLoader(dataset=ds, batch_size=3) \n",
    "print('Total batches', len(dl))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Each element in the xs list contains a batch (size 3) i.e. one element from each of the first 3 tweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Samples in batch 5\n",
      "<class 'list'>\n",
      "[\n",
      "  14\n",
      "   2\n",
      " 247\n",
      "[torch.LongTensor of size 3]\n",
      ", \n",
      "  26\n",
      " 272\n",
      "  82\n",
      "[torch.LongTensor of size 3]\n",
      ", \n",
      " 132\n",
      "   7\n",
      " 217\n",
      "[torch.LongTensor of size 3]\n",
      ", \n",
      "   18\n",
      "   90\n",
      " 4573\n",
      "[torch.LongTensor of size 3]\n",
      ", \n",
      "   10\n",
      "  812\n",
      " 1012\n",
      "[torch.LongTensor of size 3]\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "it = iter(dl)\n",
    "xs,ys = next(it)\n",
    "print('Samples in batch', len(xs))\n",
    "print(type(xs))\n",
    "print(xs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Concatenate to form a torch tensor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "   14     2   247\n",
      "   26   272    82\n",
      "  132     7   217\n",
      "   18    90  4573\n",
      "   10   812  1012\n",
      "[torch.LongTensor of size 5x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(torch.stack(xs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Padded dataset and dataloader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Pad the dataset and calculate the lengths of the tweets. In the below code you can see the output of padded dataset and dataloader.  \n",
    "##### Now samples are of equal lengths and output of dataloader is LongTensor.  \n",
    "##### Note: I have taken the max length as 10 and padded the tweets that are shorter than 10 with zeros (to the right) and truncated otherwise. We will require lengths of the tweets for the next step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VectorizeData(Dataset):\n",
    "    def __init__(self, df_path, maxlen=10):\n",
    "        self.maxlen = maxlen\n",
    "        self.df = pd.read_csv(df_path, error_bad_lines=False)\n",
    "        self.df['SentimentText'] = self.df.SentimentText.apply(lambda x: x.strip())\n",
    "        print('Indexing...')\n",
    "        self.df['sentimentidx'] = self.df.SentimentText.progress_apply(indexer)\n",
    "        print('Calculating lengths')\n",
    "        self.df['lengths'] = self.df.sentimentidx.progress_apply(lambda x: self.maxlen if len(x) > self.maxlen else len(x))\n",
    "        print('Padding')\n",
    "        self.df['sentimentpadded'] = self.df.sentimentidx.progress_apply(self.pad_data)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.df.shape[0]\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        X = self.df.sentimentpadded[idx]\n",
    "        lens = self.df.lengths[idx]\n",
    "        y = self.df.Sentiment[idx]\n",
    "        return X,y,lens\n",
    "    \n",
    "    def pad_data(self, s):\n",
    "        padded = np.zeros((self.maxlen,), dtype=np.int64)\n",
    "        if len(s) > self.maxlen: padded[:] = s[:self.maxlen]\n",
    "        else: padded[:len(s)] = s\n",
    "        return padded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 8836: expected 4 fields, saw 5\\n'\n",
      "b'Skipping line 535882: expected 4 fields, saw 7\\n'\n",
      "Progress:   0%|          | 609/1578612 [00:00<04:19, 6089.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Progress: 100%|██████████| 1578612/1578612 [02:53<00:00, 9099.97it/s] \n",
      "Progress:  13%|█▎        | 209357/1578612 [00:00<00:01, 970058.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating lengths\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Progress: 100%|██████████| 1578612/1578612 [00:01<00:00, 1146873.17it/s]\n",
      "Progress:   4%|▎         | 57227/1578612 [00:00<00:05, 265988.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Padding\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Progress: 100%|██████████| 1578612/1578612 [00:04<00:00, 340997.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 1s, sys: 565 ms, total: 3min 1s\n",
      "Wall time: 3min 1s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "ds = VectorizeData(data_root/'Sentiment Analysis Dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0    [14, 26, 132, 18, 10, 241549, 266, 6621, 0, 0]\n",
      "1           [2, 272, 7, 90, 812, 1274, 16, 0, 0, 0]\n",
      "2         [247, 82, 217, 4573, 1012, 0, 0, 0, 0, 0]\n",
      "3       [37, 241550, 4, 2, 73, 440, 6, 2, 73, 1454]\n",
      "4       [2, 93, 1815, 1342, 14, 5521, 23, 24, 3, 3]\n",
      "Name: sentimentpadded, dtype: object, 0    0\n",
      "1    0\n",
      "2    1\n",
      "3    0\n",
      "4    0\n",
      "Name: Sentiment, dtype: int64, 0     8\n",
      "1     7\n",
      "2     5\n",
      "3    10\n",
      "4    10\n",
      "Name: lengths, dtype: int64)\n"
     ]
    }
   ],
   "source": [
    "print(ds[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total batches 526204\n"
     ]
    }
   ],
   "source": [
    "dl = DataLoader(ds, batch_size=3)\n",
    "print('Total batches', len(dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "it = iter(dl)\n",
    "xs,ys,lens =  next(it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.LongTensor'>\n",
      "\n",
      "\n",
      "Columns 0 to 5 \n",
      " 1.4000e+01  2.6000e+01  1.3200e+02  1.8000e+01  1.0000e+01  2.4155e+05\n",
      " 2.0000e+00  2.7200e+02  7.0000e+00  9.0000e+01  8.1200e+02  1.2740e+03\n",
      " 2.4700e+02  8.2000e+01  2.1700e+02  4.5730e+03  1.0120e+03  0.0000e+00\n",
      "\n",
      "Columns 6 to 9 \n",
      " 2.6600e+02  6.6210e+03  0.0000e+00  0.0000e+00\n",
      " 1.6000e+01  0.0000e+00  0.0000e+00  0.0000e+00\n",
      " 0.0000e+00  0.0000e+00  0.0000e+00  0.0000e+00\n",
      "[torch.LongTensor of size 3x10]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(type(xs))\n",
    "print(xs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels: \n",
      " 0\n",
      " 0\n",
      " 1\n",
      "[torch.LongTensor of size 3]\n",
      "\n",
      "Lengths: \n",
      " 8\n",
      " 7\n",
      " 5\n",
      "[torch.LongTensor of size 3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Labels:',ys)\n",
    "print('Lengths:',lens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Simple GRU model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Simple GRU model](data/imgs/Slide1.JPG \"Simple GRU model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(words)\n",
    "embedding_dim = 4\n",
    "n_hidden = 5\n",
    "n_out = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleGRU(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):\n",
    "        super().__init__()\n",
    "        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out\n",
    "        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)\n",
    "        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)\n",
    "        self.out = nn.Linear(self.n_hidden, self.n_out)\n",
    "        \n",
    "    def forward(self, seq, lengths, gpu=True):\n",
    "        print('Sequence shape',seq.shape)\n",
    "        print('Lengths',lengths)\n",
    "        bs = seq.size(1) # batch size\n",
    "        print('batch size', bs)\n",
    "        self.h = self.init_hidden(bs, gpu) # initialize hidden state of GRU\n",
    "        print('Inititial hidden state shape', self.h.shape)\n",
    "        embs = self.emb(seq)\n",
    "        embs = pack_padded_sequence(embs, lengths) # unpad\n",
    "        gru_out, self.h = self.gru(embs, self.h) # gru returns hidden state of all timesteps as well as hidden state at last timestep\n",
    "        gru_out, lengths = pad_packed_sequence(gru_out) # pad the sequence to the max length in the batch\n",
    "        print('GRU output(all timesteps)', gru_out.shape)\n",
    "        print(gru_out)\n",
    "        print('GRU last timestep output')\n",
    "        print(gru_out[-1])\n",
    "        print('Last hidden state', self.h)\n",
    "        # since it is as classification problem, we will grab the last hidden state\n",
    "        outp = self.out(self.h[-1]) # self.h[-1] contains hidden state of last timestep\n",
    "        return F.log_softmax(outp, dim=-1)\n",
    "    \n",
    "    def init_hidden(self, batch_size, gpu):\n",
    "        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())\n",
    "        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = SimpleGRU(vocab_size, embedding_dim, n_hidden, n_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SimpleGRU(\n",
      "  (emb): Embedding(773042, 4)\n",
      "  (gru): GRU(4, 5)\n",
      "  (out): Linear(in_features=5, out_features=2, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Function to sort the batch according to tweet lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_batch(X, y, lengths):\n",
    "    lengths, indx = lengths.sort(dim=0, descending=True)\n",
    "    X = X[indx]\n",
    "    y = y[indx]\n",
    "    return X.transpose(0,1), y, lengths # transpose (batch x seq) to (seq x batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run the model for 1 batch and check the output from different operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "dl = DataLoader(ds, batch_size=3)\n",
    "it = iter(dl)\n",
    "xs,ys,lens =  next(it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequence shape torch.Size([10, 3])\n",
      "Lengths [8 7 5]\n",
      "batch size 3\n",
      "Inititial hidden state shape torch.Size([1, 3, 5])\n",
      "GRU output(all timesteps) torch.Size([8, 3, 5])\n",
      "Variable containing:\n",
      "(0 ,.,.) = \n",
      "  0.2030 -0.1839  0.1656  0.3169 -0.1318\n",
      "  0.3057 -0.0472 -0.0504  0.1198 -0.2683\n",
      "  0.2297 -0.0398 -0.2159  0.1139 -0.1195\n",
      "\n",
      "(1 ,.,.) = \n",
      " -0.0421 -0.3075  0.4247  0.3247 -0.1539\n",
      "  0.0342 -0.2529  0.2466  0.2147 -0.0439\n",
      " -0.1966 -0.1550  0.1997  0.0225  0.0199\n",
      "\n",
      "(2 ,.,.) = \n",
      " -0.2481 -0.2520  0.2310  0.1879  0.2104\n",
      "  0.0481 -0.2240  0.1004  0.1697  0.0725\n",
      " -0.1446 -0.2211  0.3108  0.0659 -0.2115\n",
      "\n",
      "(3 ,.,.) = \n",
      " -0.5341 -0.2932  0.4882  0.0314  0.3191\n",
      "  0.4127 -0.2569  0.1696  0.3845  0.0228\n",
      " -0.1979 -0.3493  0.4340  0.2279 -0.1458\n",
      "\n",
      "(4 ,.,.) = \n",
      " -0.1381 -0.3548  0.5372  0.1559 -0.0239\n",
      "  0.2906 -0.3783  0.3775  0.4882 -0.0338\n",
      " -0.2305 -0.3701  0.5152  0.1853  0.1007\n",
      "\n",
      "(5 ,.,.) = \n",
      "  0.2070 -0.4003  0.5654  0.3212 -0.0549\n",
      "  0.3796 -0.2345 -0.1808  0.4134 -0.0265\n",
      "  0.0000  0.0000  0.0000  0.0000  0.0000\n",
      "\n",
      "(6 ,.,.) = \n",
      " -0.0022 -0.3838  0.6476  0.2513  0.1293\n",
      "  0.0661 -0.3815  0.3207  0.4140  0.0072\n",
      "  0.0000  0.0000  0.0000  0.0000  0.0000\n",
      "\n",
      "(7 ,.,.) = \n",
      "  0.1062 -0.4258  0.6330  0.3214  0.0947\n",
      "  0.0000  0.0000  0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  0.0000  0.0000\n",
      "[torch.FloatTensor of size 8x3x5]\n",
      "\n",
      "GRU last timestep output\n",
      "Variable containing:\n",
      " 0.1062 -0.4258  0.6330  0.3214  0.0947\n",
      " 0.0000  0.0000  0.0000  0.0000  0.0000\n",
      " 0.0000  0.0000  0.0000  0.0000  0.0000\n",
      "[torch.FloatTensor of size 3x5]\n",
      "\n",
      "Last hidden state Variable containing:\n",
      "(0 ,.,.) = \n",
      "  0.1062 -0.4258  0.6330  0.3214  0.0947\n",
      "  0.0661 -0.3815  0.3207  0.4140  0.0072\n",
      " -0.2305 -0.3701  0.5152  0.1853  0.1007\n",
      "[torch.FloatTensor of size 1x3x5]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "xs,ys,lens = sort_batch(xs,ys,lens)\n",
    "outp = m(xs,lens.cpu().numpy(), gpu=False) # last non zero values from gru is same as hidden output by gru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "-1.0445 -0.4337\n",
       "-0.9878 -0.4659\n",
       "-1.1302 -0.3900\n",
       "[torch.FloatTensor of size 3x2]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       " -0.4337\n",
       " -0.4659\n",
       " -0.3900\n",
       " [torch.FloatTensor of size 3], Variable containing:\n",
       "  1\n",
       "  1\n",
       "  1\n",
       " [torch.LongTensor of size 3])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.max(outp, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ys.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use F.nll_loss function. This function takes preditions as **(batch_size x number_of_classes)** and **(target as batch_size)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0.8074\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.nll_loss(outp, Variable(ys))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function to train the model. I will use gpu for training. If you don't have gpu then take subset of the data as it will take quite some time to train the model on cpu."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(model, train_dl, val_dl, loss_fn, opt, epochs=3):\n",
    "    num_batch = len(train_dl)\n",
    "    for epoch in tnrange(epochs):      \n",
    "        y_true_train = list()\n",
    "        y_pred_train = list()\n",
    "        total_loss_train = 0\n",
    "        \n",
    "        if val_dl:\n",
    "            y_true_val = list()\n",
    "            y_pred_val = list()\n",
    "            total_loss_val = 0\n",
    "        \n",
    "        t = tqdm_notebook(iter(train_dl), leave=False, total=num_batch)\n",
    "        for X,y, lengths in t:\n",
    "            t.set_description(f'Epoch {epoch}')\n",
    "            X,y,lengths = sort_batch(X,y,lengths)\n",
    "            X = Variable(X.cuda())\n",
    "            y = Variable(y.cuda())\n",
    "            lengths = lengths.numpy()\n",
    "            \n",
    "            opt.zero_grad()\n",
    "            pred = model(X, lengths, gpu=True)\n",
    "            loss = loss_fn(pred, y)\n",
    "            loss.backward()\n",
    "            opt.step()\n",
    "            \n",
    "            t.set_postfix(loss=loss.data[0])\n",
    "            pred_idx = torch.max(pred, dim=1)[1]\n",
    "            \n",
    "            y_true_train += list(y.cpu().data.numpy())\n",
    "            y_pred_train += list(pred_idx.cpu().data.numpy())\n",
    "            total_loss_train += loss.data[0]\n",
    "            \n",
    "        train_acc = accuracy_score(y_true_train, y_pred_train)\n",
    "        train_loss = total_loss_train/len(train_dl)\n",
    "        print(f' Epoch {epoch}: Train loss: {train_loss} acc: {train_acc}')\n",
    "        \n",
    "        if val_dl:\n",
    "            for X,y,lengths in tqdm_notebook(valdl, leave=False):\n",
    "                X, y,lengths = sort_batch(X,y,lengths)\n",
    "                X = Variable(X.cuda())\n",
    "                y = Variable(y.cuda())\n",
    "                pred = model(X, lengths.numpy())\n",
    "                loss = loss_fn(pred, y)\n",
    "                pred_idx = torch.max(pred, 1)[1]\n",
    "                y_true_val += list(y.cpu().data.numpy())\n",
    "                y_pred_val += list(pred_idx.cpu().data.numpy())\n",
    "                total_loss_val += loss.data[0]\n",
    "            valacc = accuracy_score(y_true_val, y_pred_val)\n",
    "            valloss = total_loss_val/len(valdl)\n",
    "            print(f'Val loss: {valloss} acc: {valacc}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### I will rewrite the model without print statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleGRU(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):\n",
    "        super().__init__()\n",
    "        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out\n",
    "        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)\n",
    "        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)\n",
    "        self.out = nn.Linear(self.n_hidden, self.n_out)\n",
    "        \n",
    "    def forward(self, seq, lengths, gpu=True):\n",
    "        bs = seq.size(1) # batch size\n",
    "        self.h = self.init_hidden(bs, gpu) # initialize hidden state of GRU\n",
    "        embs = self.emb(seq)\n",
    "        embs = pack_padded_sequence(embs, lengths) # unpad\n",
    "        gru_out, self.h = self.gru(embs, self.h) # gru returns hidden state of all timesteps as well as hidden state at last timestep\n",
    "        gru_out, lengths = pad_packed_sequence(gru_out) # pad the sequence to the max length in the batch\n",
    "        # since it is as classification problem, we will grab the last hidden state\n",
    "        outp = self.out(self.h[-1]) # self.h[-1] contains hidden state of last timestep\n",
    "        return F.log_softmax(outp, dim=-1)\n",
    "    \n",
    "    def init_hidden(self, batch_size, gpu):\n",
    "        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())\n",
    "        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(ds, batch_size=512)\n",
    "m = SimpleGRU(vocab_size, embedding_dim, n_hidden, n_out).cuda()\n",
    "opt = optim.Adam(m.parameters(), 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b80056fcfa04d57a35e591696e73abb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fa5318f748444079ce77639ab3efb43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 0: Train loss: 0.5220140588268077 acc: 0.7359186424529903\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13a411f19224411eaa2ea09b8da0db90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 1: Train loss: 0.4639615505431258 acc: 0.7782165598639817\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a086935ceed4f52840b1ac1b23a0df8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 2: Train loss: 0.41349101777825065 acc: 0.8098969221062554\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98695f9b7ba042d3977d2f36bf1c8e4c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 3: Train loss: 0.3724046204959063 acc: 0.8322919121354709\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fit(model=m, train_dl=train_dl, val_dl=None, loss_fn=F.nll_loss, opt=opt, epochs=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. GRU model with concat pooling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![GRU model with concat pooling](data/imgs/Slide2.JPG \"GRU model with concat pooling\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ##### Concat Pooling in simple terms means taking max and average of output of all timesteps and then concatenating them along with the last hidden state before passing it is output layer.  \n",
    "> ##### Refer Concat Pooling section in this [paper](https://arxiv.org/abs/1801.06146)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### We can pass the output of GRU to Adaptive Max pooling and  Adaptive Avg pooling functions of pytorch. But there is a problem with this method. \n",
    "##### Since GRU output is padded to longest length sample, the average taken by F.adaptive_avg_pool1d() maybe lower than the actual because the zero padding will also be accounted.\n",
    "##### Similarly the max pooling taken by F.adaptive_max_pool1d() maybe higher than the actual because if the hidden state contains negative value then with zero padding, zero will be taken as max instead of negative value (refer output below)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConcatPoolingGRU(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):\n",
    "        super().__init__()\n",
    "        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out\n",
    "        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)\n",
    "        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)\n",
    "        self.out = nn.Linear(self.n_hidden*3, self.n_out)\n",
    "        \n",
    "    def forward(self, seq, lengths, gpu=True):\n",
    "        self.h = self.init_hidden(seq.size(1), gpu)\n",
    "        embs = self.emb(seq)\n",
    "        embs = pack_padded_sequence(embs, lengths)\n",
    "        gru_out, self.h = self.gru(embs, self.h)\n",
    "        gru_out, lengths = pad_packed_sequence(gru_out)\n",
    "        avg_pool = F.adaptive_avg_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)\n",
    "        print('Adaptive avg pooling', avg_pool)\n",
    "        # adaptive avg pooling by hand\n",
    "        # taking the sum along the batch axis and dividing by the corresponding lengths to get the actual mean\n",
    "        avg_pool_byhand = torch.sum(gru_out, dim=0)/Variable(torch.FloatTensor(lengths).view(-1,1)) \n",
    "        print('By hand Adaptive avg pooling', avg_pool_byhand)\n",
    "        max_pool = F.adaptive_max_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)\n",
    "        print('Adaptive max pooling', max_pool)\n",
    "        # adaptive max pooling by hand\n",
    "        # collect all the non padded elements of the batch and then take max of them\n",
    "        max_pool_byhand = torch.cat([torch.max(i[:l], dim=0)[0].view(1,-1) for i,l in zip(gru_out.permute(1,0,2), lengths)], dim=0) \n",
    "        print('By hand Adaptive max pooling', max_pool_byhand)\n",
    "\n",
    "#         outp = self.out(torch.cat([self.h[-1],avg_pool,max_pool],dim=1))\n",
    "        outp = self.out(torch.cat([self.h[-1],avg_pool_byhand,max_pool_byhand],dim=1))\n",
    "        return F.log_softmax(outp, dim=-1)\n",
    "    \n",
    "    def init_hidden(self, batch_size, gpu):\n",
    "        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())\n",
    "        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ConcatPoolingGRU(\n",
      "  (emb): Embedding(773042, 4)\n",
      "  (gru): GRU(4, 5)\n",
      "  (out): Linear(in_features=15, out_features=2, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "m = ConcatPoolingGRU(vocab_size, embedding_dim, n_hidden, n_out)\n",
    "print(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run the model for 1 batch and check the output from different operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "dl = DataLoader(ds, batch_size=3)\n",
    "it = iter(dl)\n",
    "xs,ys,lens =  next(it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adaptive avg pooling Variable containing:\n",
      " 0.4374 -0.1042  0.0401  0.0711  0.2725\n",
      " 0.4153 -0.1695  0.0579 -0.1884  0.0968\n",
      " 0.0339 -0.1235  0.0869  0.1116  0.1094\n",
      "[torch.FloatTensor of size 3x5]\n",
      "\n",
      "By hand Adaptive avg pooling Variable containing:\n",
      " 0.4374 -0.1042  0.0401  0.0711  0.2725\n",
      " 0.4746 -0.1937  0.0662 -0.2153  0.1106\n",
      " 0.0542 -0.1977  0.1390  0.1786  0.1751\n",
      "[torch.FloatTensor of size 3x5]\n",
      "\n",
      "Adaptive max pooling Variable containing:\n",
      " 0.6257  0.3014  0.3932  0.3380  0.5303\n",
      " 0.7005  0.0000  0.3749  0.2389  0.5936\n",
      " 0.3532  0.1311  0.2898  0.5618  0.4185\n",
      "[torch.FloatTensor of size 3x5]\n",
      "\n",
      "By hand Adaptive max pooling Variable containing:\n",
      " 0.6257  0.3014  0.3932  0.3380  0.5303\n",
      " 0.7005 -0.0575  0.3749  0.2389  0.5936\n",
      " 0.3532  0.1311  0.2898  0.5618  0.4185\n",
      "[torch.FloatTensor of size 3x5]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "xs,ys,lens = sort_batch(xs,ys,lens)\n",
    "outp = m(xs,lens.cpu().numpy(), gpu=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "-0.6811 -0.7053\n",
       "-0.6732 -0.7135\n",
       "-0.8236 -0.5778\n",
       "[torch.FloatTensor of size 3x2]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       " -0.6811\n",
       " -0.6732\n",
       " -0.5778\n",
       " [torch.FloatTensor of size 3], Variable containing:\n",
       "  0\n",
       "  0\n",
       "  1\n",
       " [torch.LongTensor of size 3])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.max(outp, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### I will rewrite the model without print statements  \n",
    "#### Model with custom max and avg pooling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConcatPoolingGRU(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):\n",
    "        super().__init__()\n",
    "        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out\n",
    "        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)\n",
    "        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)\n",
    "        self.out = nn.Linear(self.n_hidden*3, self.n_out)\n",
    "        \n",
    "    def forward(self, seq, lengths, gpu=True):\n",
    "        self.h = self.init_hidden(seq.size(1), gpu)\n",
    "        embs = self.emb(seq)\n",
    "        embs = pack_padded_sequence(embs, lengths)\n",
    "        gru_out, self.h = self.gru(embs, self.h)\n",
    "        gru_out, lengths = pad_packed_sequence(gru_out)\n",
    "        # adaptive avg pooling by hand\n",
    "        # taking the sum along the batch axis and dividing by the corresponding lengths to get the actual mean\n",
    "        avg_pool_byhand = torch.sum(gru_out, dim=0)/Variable(torch.FloatTensor(lengths).cuda().view(-1,1)) \n",
    "        # adaptive max pooling by hand\n",
    "        # collect all the non padded elements of the batch and then take max of them\n",
    "        max_pool_byhand = torch.cat([torch.max(i[:l], dim=0)[0].view(1,-1) for i,l in zip(gru_out.permute(1,0,2), lengths)], dim=0) \n",
    "\n",
    "        outp = self.out(torch.cat([self.h[-1],avg_pool_byhand,max_pool_byhand],dim=1))\n",
    "        return F.log_softmax(outp, dim=-1)\n",
    "    \n",
    "    def init_hidden(self, batch_size, gpu):\n",
    "        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())\n",
    "        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca0e3e9981b346e4ae8cafb6a2705a33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "757536eba7c142b2a29044ce682648df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 0: Train loss: 0.5220152462812094 acc: 0.7369264898531115\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50b564190a19447b9f43ca31523fd927",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 1: Train loss: 0.4618328406189202 acc: 0.7803595817084882\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "987dae46fa084a9784b48c089a87d757",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 2: Train loss: 0.40983702267499594 acc: 0.8119322544108368\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cc7afebb28649f887ad0d57531e9096",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 3: Train loss: 0.3674998400894739 acc: 0.8345464243271937\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_dl = DataLoader(ds, batch_size=512)\n",
    "m = ConcatPoolingGRU(vocab_size, embedding_dim, n_hidden, n_out).cuda()\n",
    "opt = optim.Adam(m.parameters(), 1e-2)\n",
    "\n",
    "fit(model=m, train_dl=train_dl, val_dl=None, loss_fn=F.nll_loss, opt=opt, epochs=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model with pytorch Adaptive max and avg pooling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConcatPoolingGRUAdaptive(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):\n",
    "        super().__init__()\n",
    "        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out\n",
    "        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)\n",
    "        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)\n",
    "        self.out = nn.Linear(self.n_hidden*3, self.n_out)\n",
    "        \n",
    "    def forward(self, seq, lengths, gpu=True):\n",
    "        self.h = self.init_hidden(seq.size(1), gpu)\n",
    "        embs = self.emb(seq)\n",
    "        embs = pack_padded_sequence(embs, lengths)\n",
    "        gru_out, self.h = self.gru(embs, self.h)\n",
    "        gru_out, lengths = pad_packed_sequence(gru_out)        \n",
    "        \n",
    "        avg_pool = F.adaptive_avg_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)\n",
    "        max_pool = F.adaptive_max_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)\n",
    "\n",
    "        outp = self.out(torch.cat([self.h[-1],avg_pool,max_pool],dim=1))             \n",
    "        return F.log_softmax(outp, dim=-1)\n",
    "    \n",
    "    def init_hidden(self, batch_size, gpu):\n",
    "        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())\n",
    "        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e314d86b783041499259a6d4d80f0a06",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0dcbb57f8fdc494abf22f792bdd1a18b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 0: Train loss: 0.5223443690595615 acc: 0.7380686324441978\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2abedc8431bc498ebdccfe1ea498a183",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 1: Train loss: 0.4630386214002716 acc: 0.779187666126952\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61f0aac9421f493a9bc02fe5b3f27472",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 2: Train loss: 0.40990094730362353 acc: 0.8115103648014839\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2337d5e6d06448c7ae117dbbc11a8bbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3084), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch 3: Train loss: 0.3680681557661519 acc: 0.834021912920971\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_dl = DataLoader(ds, batch_size=512)\n",
    "m = ConcatPoolingGRUAdaptive(vocab_size, embedding_dim, n_hidden, n_out).cuda()\n",
    "opt = optim.Adam(m.parameters(), 1e-2)\n",
    "\n",
    "fit(model=m, train_dl=train_dl, val_dl=None, loss_fn=F.nll_loss, opt=opt, epochs=4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}