CNN_LSTM_PyTorch_BeamSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D-3mE4Nue1dq"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import punctuation\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt\n",
    "import string \n",
    "import os\n",
    "import re\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aRsO6IiQe1d4"
   },
   "source": [
    "## Import GloVe Embedding as Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OFz8UfwNe1d7"
   },
   "outputs": [],
   "source": [
    "# create embedding dictionary for GloVe embedding (300d)\n",
    "embedding_dict = dict()\n",
    "f = open('glove.6B.300d.txt')\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coef = np.asarray(values[1:],dtype='float32')\n",
    "    embedding_dict[word] = coef # save word as key and coefficients as the value in the dictionary\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OQHpQdoNe1eB"
   },
   "outputs": [],
   "source": [
    "def preprocess(file):\n",
    "    processed = [] # preprocess the file\n",
    "    lines = file.read().split('\\n')\n",
    "    for line in lines:\n",
    "        line = line.lower() # lower case\n",
    "        line = line.replace('#','<num>') # replace the token '#' with '<num>'\n",
    "        line = re.sub(\">.<\",' point ',line) # replace the token '>.<' with ' point '\n",
    "        text = re.sub(r\"[^a-z?.!,'<>]+\",\" \",line) # replace other tokens with a space\n",
    "        text = text.rstrip().strip() # strip white space\n",
    "        text = 'bos ' + text + ' eos' # beginning and end tokens for each sentence\n",
    "        processed.append(text)\n",
    "    return processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "L39HXLPXe1eH"
   },
   "outputs": [],
   "source": [
    "pf_train_in = os.getcwd()+'/train/train.article.txt'\n",
    "pf_train_sum = os.getcwd()+'/train/train.title.txt'\n",
    "text_train_in = open(pf_train_in,'r') # load in the data\n",
    "text_train_sum = open(pf_train_sum,'r') # load in the summaries\n",
    "\n",
    "# preprocess the text\n",
    "lines_train_in = preprocess(text_train_in)\n",
    "lines_train_sum = preprocess(text_train_sum)\n",
    "# save input text and summaries as a DataFrame\n",
    "TRAIN = pd.DataFrame({'Input':lines_train_in,'Sum1':lines_train_sum})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Noemuh62e1eL",
    "outputId": "2cbe4d0c-34be-4d9a-9cd1-87d75341a36c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Input</th>\n",
       "      <th>Sum1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bos australia 's current account deficit shrun...</td>\n",
       "      <td>bos australian current account deficit narrows...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bos at least two people were killed in a suspe...</td>\n",
       "      <td>bos at least two dead in southern philippines ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bos australian shares closed down &lt;num point n...</td>\n",
       "      <td>bos australian stocks close down &lt;num point nu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bos south korea 's nuclear envoy kim sook urge...</td>\n",
       "      <td>bos envoy urges north korea to restart nuclear...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bos south korea on monday announced sweeping t...</td>\n",
       "      <td>bos skorea announces tax cuts to stimulate eco...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Input  \\\n",
       "0  bos australia 's current account deficit shrun...   \n",
       "1  bos at least two people were killed in a suspe...   \n",
       "2  bos australian shares closed down <num point n...   \n",
       "3  bos south korea 's nuclear envoy kim sook urge...   \n",
       "4  bos south korea on monday announced sweeping t...   \n",
       "\n",
       "                                                Sum1  \n",
       "0  bos australian current account deficit narrows...  \n",
       "1  bos at least two dead in southern philippines ...  \n",
       "2  bos australian stocks close down <num point nu...  \n",
       "3  bos envoy urges north korea to restart nuclear...  \n",
       "4  bos skorea announces tax cuts to stimulate eco...  "
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TRAIN.head() # display text and summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EyiBz9xKe1eV",
    "outputId": "e6d2a1b6-0096-4dee-cd70-5d0ca58422b8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN has 3803958 articles\n"
     ]
    }
   ],
   "source": [
    "print(\"TRAIN has {} articles\".format(TRAIN.shape[0])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3y3mR0tOe1ea"
   },
   "outputs": [],
   "source": [
    "TRAIN_in_len = []\n",
    "TRAIN_sum_len = [] \n",
    "\n",
    "# calculate the length of the summary and the input texts\n",
    "for line in lines_train_in:\n",
    "    w = 0\n",
    "    for word in line.split(' '):\n",
    "        w += 1\n",
    "    TRAIN_in_len.append(w)\n",
    "    \n",
    "for line in lines_train_sum:\n",
    "    w = 0\n",
    "    for word in line.split(' '):\n",
    "        w += 1\n",
    "    TRAIN_sum_len.append(w)\n",
    "TRAIN_in_len = np.array(TRAIN_in_len)\n",
    "TRAIN_sum_len = np.array(TRAIN_sum_len) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hCAA8hYIe1ee",
    "outputId": "6f2f8a82-9ac3-4f2b-8d9a-92dab45a7c78"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t Min\t 25%\t Avg\t 75%\t Max\n",
      "\n",
      "TRAIN Input\t 3.0\t29.0\t34.0\t39.0\t130.0\n",
      "TRAIN Summary\t 3.0\t9.0\t10.0\t12.0\t76.0\n"
     ]
    }
   ],
   "source": [
    "print(\"\\t\\t Min\\t 25%\\t Avg\\t 75%\\t Max\")\n",
    "print(\"\\nTRAIN Input\\t {}\\t{}\\t{}\\t{}\\t{}\".format(np.percentile(TRAIN_in_len,0),np.percentile(TRAIN_in_len,25),np.percentile(TRAIN_in_len,50),np.percentile(TRAIN_in_len,75),np.percentile(TRAIN_in_len,100)))\n",
    "print(\"TRAIN Summary\\t {}\\t{}\\t{}\\t{}\\t{}\".format(np.percentile(TRAIN_sum_len,0),np.percentile(TRAIN_sum_len,25),np.percentile(TRAIN_sum_len,50),np.percentile(TRAIN_sum_len,75),np.percentile(TRAIN_sum_len,100)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oXh-KU-ze1eo"
   },
   "outputs": [],
   "source": [
    "lines_train_in = TRAIN.iloc[:,0].values\n",
    "lines_train_sum = TRAIN.iloc[:,1].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p5xTZfM_e1eu",
    "outputId": "0e9214f3-f468-4001-de60-562dd69468ed"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1.00000e+00, 0.00000e+00, 6.00000e+00, 1.10310e+04, 2.80080e+04,\n",
       "        6.83570e+04, 7.09900e+04, 1.77520e+05, 1.82373e+05, 3.94482e+05,\n",
       "        3.33967e+05, 5.86758e+05, 6.08298e+05, 3.47617e+05, 3.88372e+05,\n",
       "        1.78841e+05, 1.81985e+05, 7.74680e+04, 7.44690e+04, 3.00340e+04,\n",
       "        2.85920e+04, 1.15240e+04, 1.05650e+04, 4.07200e+03, 3.79200e+03,\n",
       "        2.01600e+03, 8.32000e+02, 7.68000e+02, 3.11000e+02, 3.61000e+02,\n",
       "        1.42000e+02, 1.49000e+02, 5.80000e+01, 7.70000e+01, 3.20000e+01,\n",
       "        3.80000e+01, 1.80000e+01, 2.50000e+01, 6.00000e+00, 1.00000e+00,\n",
       "        0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00,\n",
       "        0.00000e+00, 1.00000e+00, 0.00000e+00, 0.00000e+00, 1.00000e+00]),\n",
       " array([  3.  ,   5.54,   8.08,  10.62,  13.16,  15.7 ,  18.24,  20.78,\n",
       "         23.32,  25.86,  28.4 ,  30.94,  33.48,  36.02,  38.56,  41.1 ,\n",
       "         43.64,  46.18,  48.72,  51.26,  53.8 ,  56.34,  58.88,  61.42,\n",
       "         63.96,  66.5 ,  69.04,  71.58,  74.12,  76.66,  79.2 ,  81.74,\n",
       "         84.28,  86.82,  89.36,  91.9 ,  94.44,  96.98,  99.52, 102.06,\n",
       "        104.6 , 107.14, 109.68, 112.22, 114.76, 117.3 , 119.84, 122.38,\n",
       "        124.92, 127.46, 130.  ]),\n",
       " <a list of 50 Patch objects>)"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFMJJREFUeJzt3X+MXeV95/H3pzhQki61AYOoTdZEtbohaBPICNzNqspCBQaimD+CRDYqVtaSpYjsptuuGtP8gZpsJEe7Cg1S4hUKLqZKQyhNipWYuJZDVFUCgmmy/AzrKWFhCsVODJQWNZT0u3/cx9mb4c7cZxzLdwa/X9LVPed7nnOeZ449/nDOee4lVYUkST1+YdIDkCQtHYaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuyyY9gKPt9NNPrzVr1kx6GJK0pDz44IM/rKqV49q94UJjzZo17Nu3b9LDkKQlJcn/7Wnn7SlJUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd26QiPJ8iR3Jvl+kseT/HqSU5PsSbK/va9obZPkpiTTSR5KcsHQcTa29vuTbByqvzvJw22fm5Kk1Uf2IUmajN5PhH8O+GZVfSDJicCbgd8H9lbV1iRbgC3Ax4HLgbXtdRGwDbgoyanADcAUUMCDSXZW1QutzWbgPmAXsB64ux1zVB+akDVbvjGy/tTWK4/xSCRNwtgrjSSnAL8B3AJQVa9W1YvABmBHa7YDuKotbwBuq4H7gOVJzgIuA/ZU1aEWFHuA9W3bKVV1b1UVcNusY43qQ5I0AT23p94GHAT+KMl3k3wxyVuAM6vqOYD2fkZrvwp4Zmj/mVabrz4zos48fUiSJqAnNJYBFwDbqup84B8Z3CaaS0bU6gjq3ZJsTrIvyb6DBw8uZFdJ0gL0hMYMMFNV97f1OxmEyPPt1hLt/cBQ+7OH9l8NPDumvnpEnXn6+BlVdXNVTVXV1MqVY7/ZV5J0hMaGRlX9HfBMkl9rpUuAx4CdwOEZUBuBu9ryTuDaNotqHfBSu7W0G7g0yYo2C+pSYHfb9nKSdW3W1LWzjjWqD0nSBPTOnvrPwJfazKkngQ8zCJw7kmwCngaubm13AVcA08ArrS1VdSjJp4AHWrtPVtWhtvwR4FbgZAazpu5u9a1z9CFJmoAMJiy9cUxNTZX/E6af31xTa+filFtpaUvyYFVNjWvnJ8IlSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd26QiPJU0keTvK9JPta7dQke5Lsb+8rWj1JbkoyneShJBcMHWdja78/ycah+rvb8afbvpmvD0nSZCzkSuM/VNW7qmqqrW8B9lbVWmBvWwe4HFjbXpuBbTAIAOAG4CLgQuCGoRDY1toe3m/9mD4kSRPw89ye2gDsaMs7gKuG6rfVwH3A8iRnAZcBe6rqUFW9AOwB1rdtp1TVvVVVwG2zjjWqD0nSBPSGRgF/keTBJJtb7cyqeg6gvZ/R6quAZ4b2nWm1+eozI+rz9SFJmoBlne3eU1XPJjkD2JPk+/O0zYhaHUG9WwuyzQBvfetbF7KrJGkBuq40qurZ9n4A+BqDZxLPt1tLtPcDrfkMcPbQ7quBZ8fUV4+oM08fs8d3c1VNVdXUypUre34kSdIRGBsaSd6S5F8dXgYuBR4BdgKHZ0BtBO5qyzuBa9ssqnXAS+3W0m7g0iQr2gPwS4HdbdvLSda1WVPXzjrWqD4kSRPQc3vqTOBrbRbsMuBPquqbSR4A7kiyCXgauLq13wVcAUwDrwAfBqiqQ0k+BTzQ2n2yqg615Y8AtwInA3e3F8DWOfqQJE3A2NCoqieBd46o/wi4ZES9gOvmONZ2YPuI+j7gvN4+JEmT4SfCJUndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndukMjyQlJvpvk6239nCT3J9mf5CtJTmz1k9r6dNu+ZugY17f6E0kuG6qvb7XpJFuG6iP7kCRNxkKuND4GPD60/hngxqpaC7wAbGr1TcALVfWrwI2tHUnOBa4B3gGsB77QgugE4PPA5cC5wAdb2/n6kCRNQFdoJFkNXAl8sa0HuBi4szXZAVzVlje0ddr2S1r7DcDtVfXjqvoBMA1c2F7TVfVkVb0K3A5sGNOHJGkCeq80/hD4PeBf2vppwItV9VpbnwFWteVVwDMAbftLrf1P67P2mas+Xx+SpAkYGxpJ3gccqKoHh8sjmtaYbUerPmqMm5PsS7Lv4MGDo5pIko6CniuN9wDvT/IUg1tHFzO48lieZFlrsxp4ti3PAGcDtO2/DBwars/aZ676D+fp42dU1c1VNVVVUytXruz4kSRJR2LZuAZVdT1wPUCS9wL/rao+lORPgQ8wCJKNwF1tl51t/d62/VtVVUl2An+S5LPArwBrge8wuKJYm+Qc4G8ZPCz/j22fe+boQwu0Zss3Rtaf2nrlMR6JpKXs5/mcxseB30kyzeD5wy2tfgtwWqv/DrAFoKoeBe4AHgO+CVxXVT9pzyw+CuxmMDvrjtZ2vj4kSRMw9kpjWFV9G/h2W36Swcyn2W3+Cbh6jv0/DXx6RH0XsGtEfWQfWjq8wpHeWPxEuCSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG4L+sJCaS5zfTGhpDcWrzQkSd0MDUlSN0NDktTN0JAkdfNB+HHOB9iSFsIrDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUbGxpJfjHJd5L87ySPJvmDVj8nyf1J9if5SpITW/2ktj7dtq8ZOtb1rf5EksuG6utbbTrJlqH6yD4kSZPRc6XxY+Diqnon8C5gfZJ1wGeAG6tqLfACsKm13wS8UFW/CtzY2pHkXOAa4B3AeuALSU5IcgLweeBy4Fzgg60t8/QhSZqAsaFRA//QVt/UXgVcDNzZ6juAq9ryhrZO235JkrT67VX146r6ATANXNhe01X1ZFW9CtwObGj7zNWHJGkCup5ptCuC7wEHgD3A3wAvVtVrrckMsKotrwKeAWjbXwJOG67P2meu+mnz9CFJmoCu0Kiqn1TVu4DVDK4M3j6qWXvPHNuOVv11kmxOsi/JvoMHD45qIkk6ChY0e6qqXgS+DawDlic5/IWHq4Fn2/IMcDZA2/7LwKHh+qx95qr/cJ4+Zo/r5qqaqqqplStXLuRHkiQtQM/sqZVJlrflk4HfBB4H7gE+0JptBO5qyzvbOm37t6qqWv2aNrvqHGAt8B3gAWBtmyl1IoOH5TvbPnP1IUmagJ6vRj8L2NFmOf0CcEdVfT3JY8DtSf478F3gltb+FuCPk0wzuMK4BqCqHk1yB/AY8BpwXVX9BCDJR4HdwAnA9qp6tB3r43P0IUmagLGhUVUPAeePqD/J4PnG7Po/AVfPcaxPA58eUd8F7OrtQ5I0GX4iXJLUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUbWxoJDk7yT1JHk/yaJKPtfqpSfYk2d/eV7R6ktyUZDrJQ0kuGDrWxtZ+f5KNQ/V3J3m47XNTkszXhyRpMnquNF4Dfreq3g6sA65Lci6wBdhbVWuBvW0d4HJgbXttBrbBIACAG4CLgAuBG4ZCYFtre3i/9a0+Vx+SpAkYGxpV9VxV/XVbfhl4HFgFbAB2tGY7gKva8gbgthq4D1ie5CzgMmBPVR2qqheAPcD6tu2Uqrq3qgq4bdaxRvUhSZqABT3TSLIGOB+4Hzizqp6DQbAAZ7Rmq4BnhnababX56jMj6szThyRpArpDI8kvAX8G/HZV/f18TUfU6gjq3ZJsTrIvyb6DBw8uZFdJ0gJ0hUaSNzEIjC9V1Vdb+fl2a4n2fqDVZ4Czh3ZfDTw7pr56RH2+Pn5GVd1cVVNVNbVy5cqeH0mSdAR6Zk8FuAV4vKo+O7RpJ3B4BtRG4K6h+rVtFtU64KV2a2k3cGmSFe0B+KXA7rbt5STrWl/XzjrWqD4kSROwrKPNe4DfAh5O8r1W+31gK3BHkk3A08DVbdsu4ApgGngF+DBAVR1K8inggdbuk1V1qC1/BLgVOBm4u72Ypw9J0gSMDY2q+itGP3cAuGRE+wKum+NY24HtI+r7gPNG1H80qg9J0mT4iXBJUree21NaQtZs+cakh/BzmW/8T2298hiORNIohsYS9UYOB0mLl7enJEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSt7GhkWR7kgNJHhmqnZpkT5L97X1FqyfJTUmmkzyU5IKhfTa29vuTbByqvzvJw22fm5Jkvj4kSZPTc6VxK7B+Vm0LsLeq1gJ72zrA5cDa9toMbINBAAA3ABcBFwI3DIXAttb28H7rx/QhSZqQsaFRVX8JHJpV3gDsaMs7gKuG6rfVwH3A8iRnAZcBe6rqUFW9AOwB1rdtp1TVvVVVwG2zjjWqD0nShBzpM40zq+o5gPZ+RquvAp4ZajfTavPVZ0bU5+tDkjQhR/tBeEbU6gjqC+s02ZxkX5J9Bw8eXOjukqRORxoaz7dbS7T3A60+A5w91G418OyY+uoR9fn6eJ2qurmqpqpqauXKlUf4I0mSxjnS0NgJHJ4BtRG4a6h+bZtFtQ54qd1a2g1cmmRFewB+KbC7bXs5ybo2a+raWcca1YckaUKWjWuQ5MvAe4HTk8wwmAW1FbgjySbgaeDq1nwXcAUwDbwCfBigqg4l+RTwQGv3yao6/HD9IwxmaJ0M3N1ezNOHJGlCxoZGVX1wjk2XjGhbwHVzHGc7sH1EfR9w3oj6j0b1IUmaHD8RLknqZmhIkrqNvT0lLRZrtnxjZP2prVce45FIxy9DQ0ueYSIdO96ekiR180pjkZvrv6IlaRIMjUXAYJC0VHh7SpLUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUje/sPAY8osJj62Fnm///xvSeF5pSJK6GRqSpG6GhiSpm6EhSepmaEiSui362VNJ1gOfA04AvlhVWyc8pLGcJbU0OdtKGm9RX2kkOQH4PHA5cC7wwSTnTnZUknT8WuxXGhcC01X1JECS24ENwGMTHZXE3FcmXoHojWyxh8Yq4Jmh9RngogmNReribS69kS320MiIWr2uUbIZ2NxW/yHJE/Mc83Tgh0dhbJPi+CfrqI8/nzmaRxvL8z85i33s/7qn0WIPjRng7KH11cCzsxtV1c3AzT0HTLKvqqaOzvCOPcc/WY5/spby+Jfy2Ict6gfhwAPA2iTnJDkRuAbYOeExSdJxa1FfaVTVa0k+CuxmMOV2e1U9OuFhSdJxa1GHBkBV7QJ2HcVDdt3GWsQc/2Q5/slayuNfymP/qVS97rmyJEkjLfZnGpKkReS4CY0k65M8kWQ6yZZJj2ecJGcnuSfJ40keTfKxVj81yZ4k+9v7ikmPdT5JTkjy3SRfb+vnJLm/jf8rbYLDopRkeZI7k3y//Tn8+lI6/0n+a/u780iSLyf5xcV8/pNsT3IgySNDtZHnOwM3td/nh5JcMLmR/3Sso8b/P9rfn4eSfC3J8qFt17fxP5HkssmMeuGOi9BYol9H8hrwu1X1dmAdcF0b8xZgb1WtBfa29cXsY8DjQ+ufAW5s438B2DSRUfX5HPDNqvo3wDsZ/BxL4vwnWQX8F2Cqqs5jMJHkGhb3+b8VWD+rNtf5vhxY216bgW3HaIzzuZXXj38PcF5V/Vvg/wDXA7Tf5WuAd7R9vtD+nVr0jovQYOjrSKrqVeDw15EsWlX1XFX9dVt+mcE/WKsYjHtHa7YDuGoyIxwvyWrgSuCLbT3AxcCdrcmiHX+SU4DfAG4BqKpXq+pFltD5ZzDR5eQky4A3A8+xiM9/Vf0lcGhWea7zvQG4rQbuA5YnOevYjHS0UeOvqr+oqtfa6n0MPmsGg/HfXlU/rqofANMM/p1a9I6X0Bj1dSSrJjSWBUuyBjgfuB84s6qeg0GwAGdMbmRj/SHwe8C/tPXTgBeHfokW85/D24CDwB+122tfTPIWlsj5r6q/Bf4n8DSDsHgJeJClc/4Pm+t8L8Xf6f8E3N2Wl+L4geMnNLq+jmQxSvJLwJ8Bv11Vfz/p8fRK8j7gQFU9OFwe0XSx/jksAy4AtlXV+cA/skhvRY3S7v1vAM4BfgV4C4NbOrMt1vM/zlL6u0SSTzC45fylw6URzRbt+IcdL6HR9XUki02SNzEIjC9V1Vdb+fnDl+Ht/cCkxjfGe4D3J3mKwe3AixlceSxvt0tgcf85zAAzVXV/W7+TQYgslfP/m8APqupgVf0z8FXg37F0zv9hc53vJfM7nWQj8D7gQ/X/P+OwZMY/2/ESGkvu60ja/f9bgMer6rNDm3YCG9vyRuCuYz22HlV1fVWtrqo1DM73t6rqQ8A9wAdas8U8/r8Dnknya610CYOv5F8S55/Bbal1Sd7c/i4dHv+SOP9D5jrfO4Fr2yyqdcBLh29jLSYZ/E/kPg68v6peGdq0E7gmyUlJzmHwQP87kxjjglXVcfECrmAwe+FvgE9Mejwd4/33DC5XHwK+115XMHgusBfY395PnfRYO36W9wJfb8tvY/DLMQ38KXDSpMc3z7jfBexrfwZ/DqxYSucf+APg+8AjwB8DJy3m8w98mcHzl39m8F/im+Y63wxu73y+/T4/zGCW2GIc/zSDZxeHf4f/11D7T7TxPwFcPunx9778RLgkqdvxcntKknQUGBqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnq9v8AxKu275zoJvYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(TRAIN_in_len,bins=50) # plot histogram of the length of input text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qZy7XXWCe1e2",
    "outputId": "10f91594-37f2-41c3-e522-6354c50a8559"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2.250000e+02, 3.429000e+03, 3.949560e+05, 4.589540e+05,\n",
       "        1.272175e+06, 5.745740e+05, 6.951070e+05, 1.677130e+05,\n",
       "        1.638550e+05, 3.454500e+04, 2.826100e+04, 4.896000e+03,\n",
       "        2.401000e+03, 2.043000e+03, 3.800000e+02, 2.890000e+02,\n",
       "        6.000000e+01, 4.500000e+01, 1.200000e+01, 1.100000e+01,\n",
       "        2.000000e+00, 8.000000e+00, 2.000000e+00, 3.000000e+00,\n",
       "        1.000000e+00, 1.000000e+00, 1.000000e+00, 1.000000e+00,\n",
       "        0.000000e+00, 2.000000e+00, 0.000000e+00, 1.000000e+00,\n",
       "        0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
       "        0.000000e+00, 0.000000e+00, 0.000000e+00, 1.000000e+00,\n",
       "        0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
       "        0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00,\n",
       "        0.000000e+00, 4.000000e+00]),\n",
       " array([ 3.  ,  4.46,  5.92,  7.38,  8.84, 10.3 , 11.76, 13.22, 14.68,\n",
       "        16.14, 17.6 , 19.06, 20.52, 21.98, 23.44, 24.9 , 26.36, 27.82,\n",
       "        29.28, 30.74, 32.2 , 33.66, 35.12, 36.58, 38.04, 39.5 , 40.96,\n",
       "        42.42, 43.88, 45.34, 46.8 , 48.26, 49.72, 51.18, 52.64, 54.1 ,\n",
       "        55.56, 57.02, 58.48, 59.94, 61.4 , 62.86, 64.32, 65.78, 67.24,\n",
       "        68.7 , 70.16, 71.62, 73.08, 74.54, 76.  ]),\n",
       " <a list of 50 Patch objects>)"
      ]
     },
     "execution_count": 11,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAD8CAYAAACyyUlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFINJREFUeJzt3X2MneWZ3/Hvr/ZCSLbEvDgRtVFNFCsbEm0SMiJOU61S2IKBKOYPIoFWxUotWY1ImzQrbUxXKmrSlUCtll2kBAkFNrCKIJTNFish8VpAtFIVXsYhy2tYT4HCFBYPa0PSRQ1L9uof53ZzMGdm8NyDzxn8/UhH53mu536e+5oZh1+elzmTqkKSpB7/aNwNSJJWPsNEktTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3QwTSVK31eNu4Eg5+eSTa8OGDeNuQ5JWlD179rxQVWsXG3fUhMmGDRuYnp4edxuStKIk+V9vZJyXuSRJ3QwTSVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdTNMJEndjprfgJ90G3Z8b2T9qSsvOMKdSNLh88xEktTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3RYNkyQ3JNmX5OGh2n9J8tMkDyb58yRrhrZdnmQmyeNJzh2qb261mSQ7huqnJbk3yd4k305yTKsf29Zn2vYNi80hSRqPN3Jm8k1g8yG13cAHq+o3gb8GLgdIcjpwMfCBts/Xk6xKsgr4GnAecDpwSRsLcBVwdVVtBA4A21p9G3Cgqt4LXN3GzTvHYX7dkqRltGiYVNVfAvsPqf1FVb3aVu8B1rflLcAtVfWLqnoSmAHObK+Zqnqiql4BbgG2JAlwFnBb2/9G4MKhY93Ylm8Dzm7j55tDkjQmy3HP5F8D32/L64BnhrbNttp89ZOAF4eC6WD9Ncdq219q4+c71usk2Z5kOsn03Nzckr44SdLiusIkye8DrwLfOlgaMayWUF/KsV5frLquqqaqamrt2rWjhkiSlsGSP+gxyVbgU8DZVXXwP+azwKlDw9YDz7blUfUXgDVJVrezj+HxB481m2Q18E4Gl9sWmkOSNAZLOjNJshn4MvDpqnp5aNNO4OL2JNZpwEbgPuB+YGN7cusYBjfQd7YQuhu4qO2/Fbh96Fhb2/JFwF1t/HxzSJLGZNEzkyQ3A58ETk4yC1zB4OmtY4Hdg3vi3FNV/6aqHklyK/Aog8tfl1XVL9txPg/sAlYBN1TVI22KLwO3JPnPwAPA9a1+PfCnSWYYnJFcDLDQHJKk8civrlC9tU1NTdX09PS425iXf89E0iRKsqeqphYb52/AS5K6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKnbomGS5IYk+5I8PFQ7McnuJHvb+wmtniTXJJlJ8mCSM4b22drG702ydaj+0SQPtX2uSZKlziFJGo83cmbyTWDzIbUdwJ1VtRG4s60DnAdsbK/twLUwCAbgCuBjwJnAFQfDoY3ZPrTf5qXMIUkan0XDpKr+Eth/SHkLcGNbvhG4cKh+Uw3cA6xJcgpwLrC7qvZX1QFgN7C5bTu+qn5UVQXcdMixDmcOSdKYLPWeybur6jmA9v6uVl8HPDM0brbVFqrPjqgvZQ5J0pgs9w34jKjVEupLmeP1A5PtSaaTTM/NzS1yWEnSUi01TJ4/eGmpve9r9Vng1KFx64FnF6mvH1FfyhyvU1XXVdVUVU2tXbv2sL5ASdIbt9Qw2QkcfCJrK3D7UP3S9sTVJuCldolqF3BOkhPajfdzgF1t28+TbGpPcV16yLEOZw5J0pisXmxAkpuBTwInJ5ll8FTWlcCtSbYBTwOfacPvAM4HZoCXgc8CVNX+JF8F7m/jvlJVB2/qf47BE2PHAd9vLw53DknS+CwaJlV1yTybzh4xtoDL5jnODcANI+rTwAdH1P/2cOeQJI2HvwEvSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpW1eYJPn3SR5J8nCSm5O8LclpSe5NsjfJt5Mc08Ye29Zn2vYNQ8e5vNUfT3LuUH1zq80k2TFUHzmHJGk8lhwmSdYB/w6YqqoPAquAi4GrgKuraiNwANjWdtkGHKiq9wJXt3EkOb3t9wFgM/D1JKuSrAK+BpwHnA5c0saywBySpDHovcy1GjguyWrg7cBzwFnAbW37jcCFbXlLW6dtPztJWv2WqvpFVT0JzABnttdMVT1RVa8AtwBb2j7zzSFJGoMlh0lV/W/gvwJPMwiRl4A9wItV9WobNgusa8vrgGfavq+28ScN1w/ZZ776SQvMIUkag57LXCcwOKs4DfgnwDsYXJI6VB3cZZ5ty1Uf1eP2JNNJpufm5kYNkSQtg57LXL8NPFlVc1X198B3gH8GrGmXvQDWA8+25VngVIC2/Z3A/uH6IfvMV39hgTleo6quq6qpqppau3Ztx5cqSVpIT5g8DWxK8vZ2H+Ns4FHgbuCiNmYrcHtb3tnWadvvqqpq9Yvb016nARuB+4D7gY3tya1jGNyk39n2mW8OSdIY9NwzuZfBTfAfAw+1Y10HfBn4UpIZBvc3rm+7XA+c1OpfAna04zwC3MogiH4AXFZVv2z3RD4P7AIeA25tY1lgDknSGGTwf/Tf+qampmp6enrcbcxrw47vjaw/deUFR7gTSfqVJHuqamqxcf4GvCSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6rFx+iSeQHQ0qaJJ6ZSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkbl1hkmRNktuS/DTJY0k+nuTEJLuT7G3vJ7SxSXJNkpkkDyY5Y+g4W9v4vUm2DtU/muShts81SdLqI+eQJI1H75nJHwM/qKrfAD4EPAbsAO6sqo3AnW0d4DxgY3ttB66FQTAAVwAfA84ErhgKh2vb2IP7bW71+eaQJI3BksMkyfHAbwHXA1TVK1X1IrAFuLENuxG4sC1vAW6qgXuANUlOAc4FdlfV/qo6AOwGNrdtx1fVj6qqgJsOOdaoOSRJY9BzZvIeYA74kyQPJPlGkncA766q5wDa+7va+HXAM0P7z7baQvXZEXUWmEOSNAY9YbIaOAO4tqo+AvwdC19uyohaLaH+hiXZnmQ6yfTc3Nzh7CpJOgw9YTILzFbVvW39Ngbh8ny7REV73zc0/tSh/dcDzy5SXz+izgJzvEZVXVdVU1U1tXbt2iV9kZKkxS05TKrqb4Bnkryvlc4GHgV2AgefyNoK3N6WdwKXtqe6NgEvtUtUu4BzkpzQbryfA+xq236eZFN7iuvSQ441ag5J0hj0/qXFfwt8K8kxwBPAZxkE1K1JtgFPA59pY+8AzgdmgJfbWKpqf5KvAve3cV+pqv1t+XPAN4HjgO+3F8CV88whSRqDDB6Ueuubmpqq6enpcbcxr/n+DO/h8s/2SlpOSfZU1dRi4/wNeElSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3QwTSVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLUrfdvwOswLdef55WkSeKZiSSpm2EiSepmmEiSuhkmkqRuhokkqVt3mCRZleSBJN9t66cluTfJ3iTfTnJMqx/b1mfa9g1Dx7i81R9Pcu5QfXOrzSTZMVQfOYckaTyW48zkC8BjQ+tXAVdX1UbgALCt1bcBB6rqvcDVbRxJTgcuBj4AbAa+3gJqFfA14DzgdOCSNnahOSRJY9AVJknWAxcA32jrAc4CbmtDbgQubMtb2jpt+9lt/Bbglqr6RVU9CcwAZ7bXTFU9UVWvALcAWxaZQ5I0Br1nJn8E/B7wD239JODFqnq1rc8C69ryOuAZgLb9pTb+/9cP2We++kJzSJLGYMlhkuRTwL6q2jNcHjG0Ftm2XPVRPW5PMp1kem5ubtQQSdIy6Pk4lU8An05yPvA24HgGZyprkqxuZw7rgWfb+FngVGA2yWrgncD+ofpBw/uMqr+wwByvUVXXAdcBTE1NjQycN4sfmyLpaLLkM5Oquryq1lfVBgY30O+qqt8B7gYuasO2Are35Z1tnbb9rqqqVr+4Pe11GrARuA+4H9jYntw6ps2xs+0z3xySpDF4M37P5MvAl5LMMLi/cX2rXw+c1OpfAnYAVNUjwK3Ao8APgMuq6pftrOPzwC4GT4vd2sYuNIckaQyW5VODq+qHwA/b8hMMnsQ6dMz/BT4zz/5/APzBiPodwB0j6iPnkCSNh78BL0nqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqduSwyTJqUnuTvJYkkeSfKHVT0yyO8ne9n5CqyfJNUlmkjyY5IyhY21t4/cm2TpU/2iSh9o+1yTJQnNIksaj58zkVeB3q+r9wCbgsiSnAzuAO6tqI3BnWwc4D9jYXtuBa2EQDMAVwMeAM4ErhsLh2jb24H6bW32+OSRJY7DkMKmq56rqx23558BjwDpgC3BjG3YjcGFb3gLcVAP3AGuSnAKcC+yuqv1VdQDYDWxu246vqh9VVQE3HXKsUXNIksZgWe6ZJNkAfAS4F3h3VT0Hg8AB3tWGrQOeGdptttUWqs+OqLPAHIf2tT3JdJLpubm5pX55kqRFdIdJkl8H/gz4YlX9bKGhI2q1hPobVlXXVdVUVU2tXbv2cHaVJB2GrjBJ8msMguRbVfWdVn6+XaKive9r9Vng1KHd1wPPLlJfP6K+0BySpDHoeZorwPXAY1X1h0ObdgIHn8jaCtw+VL+0PdW1CXipXaLaBZyT5IR24/0cYFfb9vMkm9pclx5yrFFzSJLGYHXHvp8A/hXwUJKftNp/AK4Ebk2yDXga+EzbdgdwPjADvAx8FqCq9if5KnB/G/eVqtrflj8HfBM4Dvh+e7HAHJKkMcjgQam3vqmpqZqenj5i823Y8b0jNtcb8dSVF4y7BUkrUJI9VTW12LieMxOtIPOFmyEjaTn4cSqSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkrqtHncDGq8NO743sv7UlRcc4U4krWSGiUYyZCQdjhV9mSvJ5iSPJ5lJsmPc/UjS0WrFhkmSVcDXgPOA04FLkpw+3q4k6ei0YsMEOBOYqaonquoV4BZgy5h7kqSj0kq+Z7IOeGZofRb42JFuYr57C29Vy/n1ev9FeutYyWGSEbV6zYBkO7C9rf6fJI+35ZOBF97E3pbLSuhzyT3mqmXuZGEr4XsJK6PPldAj2Ody+advZNBKDpNZ4NSh9fXAs8MDquo64LpDd0wyXVVTb257/VZCnyuhR7DP5bQSegT7PNJW8j2T+4GNSU5LcgxwMbBzzD1J0lFpxZ6ZVNWrST4P7AJWATdU1SNjbkuSjkorNkwAquoO4I4l7Pq6S18TaiX0uRJ6BPtcTiuhR7DPIypVtfgoSZIWsJLvmUiSJsRRFSaT+vErSW5Isi/Jw0O1E5PsTrK3vZ8wzh5bT6cmuTvJY0keSfKFSes1yduS3Jfkr1qP/6nVT0tyb+vx2+2hjbFLsirJA0m+29Ynrs8kTyV5KMlPkky32sT8zIf6XJPktiQ/bf9GPz5JfSZ5X/seHnz9LMkXJ6nHHkdNmEz4x698E9h8SG0HcGdVbQTubOvj9irwu1X1fmATcFn7Hk5Sr78AzqqqDwEfBjYn2QRcBVzdejwAbBtjj8O+ADw2tD6pff6Lqvrw0COsk/QzP+iPgR9U1W8AH2LwfZ2YPqvq8fY9/DDwUeBl4M8nqccuVXVUvICPA7uG1i8HLh93X0P9bAAeHlp/HDilLZ8CPD7uHkf0fDvwLye1V+DtwI8ZfDLCC8DqUf8Wxtjfegb/8TgL+C6DX8SdxD6fAk4+pDZRP3PgeOBJ2n3gSe1zqK9zgP8xyT0e7uuoOTNh9MevrBtTL2/Eu6vqOYD2/q4x9/MaSTYAHwHuZcJ6bZeOfgLsA3YD/xN4sapebUMm5Wf/R8DvAf/Q1k9iMvss4C+S7GmfKgET9jMH3gPMAX/SLht+I8k7mLw+D7oYuLktT2qPh+VoCpNFP35Fb0ySXwf+DPhiVf1s3P0cqqp+WYNLCesZfCDo+0cNO7JdvVaSTwH7qmrPcHnE0En4N/qJqjqDwSXiy5L81rgbGmE1cAZwbVV9BPg7JvRyUbsP9mngv427l+V0NIXJoh+/MmGeT3IKQHvfN+Z+AEjyawyC5FtV9Z1Wnsheq+pF4IcM7u+sSXLw96om4Wf/CeDTSZ5i8InXZzE4U5m0PqmqZ9v7PgbX+M9k8n7ms8BsVd3b1m9jEC6T1icMQvnHVfV8W5/EHg/b0RQmK+3jV3YCW9vyVgb3J8YqSYDrgceq6g+HNk1Mr0nWJlnTlo8DfpvBjdi7gYvasLF/P6vq8qpaX1UbGPxbvKuqfocJ6zPJO5L844PLDK71P8wE/cwBqupvgGeSvK+VzgYeZcL6bC7hV5e4YDJ7PHzjvmlzJF/A+cBfM7iG/vvj7meor5uB54C/Z/D/sLYxuH5+J7C3vZ84AX3+cwaXXR4EftJe509Sr8BvAg+0Hh8G/mOrvwe4D5hhcHnh2HF/P4d6/iTw3Unss/XzV+31yMH/3UzSz3yo1w8D0+1n/9+BEyatTwYPhfwt8M6h2kT1uNSXvwEvSep2NF3mkiS9SQwTSVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdft/lJPjAWoZN7IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(TRAIN_sum_len,bins=50) # plot histogram of the length of summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7YqRGGyne1e9",
    "outputId": "5d7f36a4-66a2-40a5-f1b4-0298dc25c84a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3326008\n"
     ]
    }
   ],
   "source": [
    "# remove outliers\n",
    "a = np.where(np.logical_and(TRAIN_in_len>=20,np.logical_and(TRAIN_in_len<=50,np.logical_and(TRAIN_sum_len>=6,TRAIN_sum_len<=14))))\n",
    "print(len(a[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kDrDeX1Re1fM"
   },
   "outputs": [],
   "source": [
    "data_indices = np.random.randint(len(a[0]),size=1000000) # randomly choose 1 million samples\n",
    "indices = a[0][data_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tRZ1eTAxe1fT"
   },
   "outputs": [],
   "source": [
    "data_text = lines_train_in[indices]\n",
    "data_sum = lines_train_sum[indices]\n",
    "output_len = TRAIN_sum_len[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SP-4VR-Ue1fY",
    "outputId": "35ae7e61-5298-4a3f-baae-0f11012136d5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-3.0522e-02, -2.2719e-01,  3.8781e-01, -7.0417e-01, -1.1544e-01,\n",
       "       -7.0090e-01, -3.9910e-02,  4.2745e-02,  1.5057e-01, -4.0904e-01,\n",
       "       -3.8437e-01,  6.8181e-02,  2.9802e-01, -2.8310e-01,  5.3504e-01,\n",
       "        1.3696e+00, -4.4908e-02,  7.3455e-01,  2.7652e-01, -9.7095e-02,\n",
       "       -2.4187e-01, -2.0424e-01,  1.5428e-01, -5.6790e-01, -2.5146e-01,\n",
       "        2.8584e-01,  6.3807e-01,  1.8123e-01,  2.7460e-02,  3.0633e-02,\n",
       "        4.8657e-02, -2.3110e-01, -3.5313e-01,  3.5028e-01, -1.9633e-01,\n",
       "        5.9194e-02, -2.8627e-01, -8.3223e-02,  1.6305e-01,  3.4971e-01,\n",
       "       -3.3111e-01,  3.4556e-01, -6.1087e-01, -1.5548e-01,  2.6479e-02,\n",
       "       -1.5457e-01,  7.0944e-01, -3.0296e-01, -1.8202e-01, -1.5534e-01,\n",
       "        3.9994e-01, -1.7475e-01,  3.1277e-01, -3.2295e-01, -1.7056e-01,\n",
       "        4.1496e-01,  3.5370e-01, -5.5457e-01,  2.7422e-01,  3.0380e-01,\n",
       "       -6.8106e-02,  1.4468e-01, -3.2422e-01, -1.1255e-01,  5.4311e-02,\n",
       "        2.1412e-01,  2.3971e-01, -4.2925e-01, -1.1484e-01,  6.8309e-02,\n",
       "        4.6927e-01,  2.7818e-01,  2.7761e-01,  6.7662e-01, -4.3719e-02,\n",
       "        3.0930e-01,  4.7784e-01,  4.0585e-01,  4.7442e-01, -4.5702e-01,\n",
       "        3.8474e-01,  3.7459e-01, -8.0396e-01,  1.2715e-01, -2.7424e-01,\n",
       "       -1.1502e-01, -2.6622e-01,  5.9803e-01, -4.0461e-01, -6.9133e-01,\n",
       "        2.7395e-01,  1.1549e-01,  2.0535e-01, -7.8427e-02,  2.3155e-01,\n",
       "        4.9158e-01, -4.5234e-01, -5.7944e-02,  2.7459e-01, -4.3238e-02,\n",
       "       -3.9403e-01,  3.9582e-01, -2.4819e-01,  2.7267e-01,  5.8850e-03,\n",
       "       -1.5230e-01, -3.2904e-01,  7.7212e-02, -3.1884e-01, -4.9338e-01,\n",
       "       -4.2691e-01, -6.9709e-02, -2.2209e-01, -1.6935e-01,  1.2959e-01,\n",
       "       -6.7211e-01, -1.2006e-01,  1.8683e-01, -3.0736e-01, -6.7685e-01,\n",
       "       -3.2302e-03, -1.4633e-01, -1.1515e-02,  4.7606e-02, -3.5942e-01,\n",
       "        2.1525e-01, -3.6572e-01, -4.3253e-01,  8.5577e-01,  5.3071e-01,\n",
       "        1.7728e-01,  6.2575e-01, -2.7721e-01, -2.3528e-01,  4.1617e-01,\n",
       "       -6.3685e-01, -1.3228e-01, -8.4596e-02,  3.9318e-01, -3.4730e-01,\n",
       "       -8.5430e-02, -2.5523e-02, -5.5561e-01, -4.1545e-01, -1.0443e-02,\n",
       "        1.5979e-01, -7.3388e-01,  2.0613e-01, -4.7352e-02,  1.6043e-01,\n",
       "       -2.8636e-01,  2.1679e-01,  2.5871e-01, -4.6480e-01, -1.5189e-01,\n",
       "       -3.5660e-01,  2.0509e-01, -1.4258e-01,  7.5137e-01, -2.9748e-01,\n",
       "        1.4892e-01, -9.7381e-02, -1.1261e-01, -1.2699e-01,  5.6518e-01,\n",
       "       -2.0014e-01, -4.3796e-01,  3.2039e-01, -1.2375e-01,  1.6236e-01,\n",
       "       -5.9564e-01, -5.9208e-01,  1.3845e-01,  2.6548e-01, -4.8460e-01,\n",
       "       -4.5490e-01,  3.6372e-01,  6.1271e-01, -7.9227e-02, -2.0203e-01,\n",
       "       -7.0302e-02, -2.9754e-01,  9.4895e-01,  2.4119e-01, -6.7992e-01,\n",
       "        1.8321e-01,  8.0144e-01,  2.3200e-01, -9.2892e-02,  2.9012e-01,\n",
       "        5.2748e-01,  4.0973e-01, -6.9563e-01,  1.1502e-01,  8.5671e-01,\n",
       "        5.3983e-01, -4.9115e-01, -2.1865e-01,  4.1167e-01, -2.8302e-01,\n",
       "        6.7575e-01, -8.7271e-03,  5.4358e-02,  2.6724e-01,  6.4988e-02,\n",
       "        1.8340e-01,  6.8273e-01, -1.4183e-01, -4.4524e-01,  4.1151e-01,\n",
       "       -6.3538e-02,  1.6092e-01,  1.5135e-01, -1.5512e-01,  3.3532e-01,\n",
       "        2.3471e-01, -3.2777e-01, -5.1048e-01, -4.1558e-01,  4.6170e-01,\n",
       "       -1.0669e-01,  3.4845e-01, -3.2336e-01,  1.4570e-01, -2.1846e-02,\n",
       "       -1.8653e-01, -4.6824e-02, -4.7711e-01, -8.4673e-02, -4.9193e-01,\n",
       "        1.2038e-01, -8.1942e-04, -6.2167e-02, -6.8141e-02,  2.1541e-01,\n",
       "        6.2880e-02, -1.8996e-01,  1.4871e-02,  5.9061e-01, -3.8892e-04,\n",
       "        5.8776e-01, -6.7990e-02, -1.5828e-02,  2.5458e-01, -3.1814e-01,\n",
       "        9.0046e-02,  2.7234e-01,  1.7847e-01,  1.5347e-01,  6.0593e-02,\n",
       "       -2.9186e-01, -1.5807e-01, -9.1611e-02, -4.9164e-01,  1.2379e-01,\n",
       "        2.5897e-01, -5.2763e-02, -6.4442e-01,  5.6134e-02,  5.2372e-01,\n",
       "       -4.3609e-01,  5.3946e-01, -2.7729e-01,  2.9321e-01,  4.9666e-02,\n",
       "        3.9158e-01, -4.4257e-01,  7.5404e-02,  2.1156e-01,  1.0542e+00,\n",
       "        3.6859e-01,  3.2406e-01, -2.9011e-01, -2.6407e-01, -4.5163e-01,\n",
       "       -2.2718e-02, -1.1600e+00, -5.4053e-02, -2.2611e-01, -6.4435e-03,\n",
       "       -1.8577e-01, -3.2979e-01, -9.4983e-02,  2.8727e-02, -7.1909e-01,\n",
       "       -4.6893e-01,  1.7047e-01,  2.6780e-01, -8.0399e-01,  4.4505e-01,\n",
       "        1.9638e-01, -6.1000e-02,  1.3899e-01, -5.3471e-01,  4.0549e-02,\n",
       "        4.4949e-01,  5.2472e-01, -4.1621e-01, -6.4880e-01,  7.2649e-02],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_dict['pad']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nqpltrple1fh",
    "outputId": "daf59468-4fe7-469c-b555-8c8a7cd908a8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93755 words found\n",
      "1839 OOV words\n",
      "95594 words in the vocabulary\n"
     ]
    }
   ],
   "source": [
    "words = [] # create word list for the words in the input text and summaries\n",
    "for line in data_text:\n",
    "    for word in line.split(' '):\n",
    "        words.append(word)\n",
    "for line in data_sum:\n",
    "    for word in line.split(' '):\n",
    "        words.append(word)\n",
    "counts = Counter(words)\n",
    "vocab = sorted(counts, key=counts.get, reverse=True)\n",
    "embed_dim = 300\n",
    "words_found = 0\n",
    "words_OOV = 0\n",
    "word2idx = {}\n",
    "word2idx['<pad>'] = 0\n",
    "# convert words to index by creating word2idx dictionary\n",
    "for index, word in enumerate(vocab):\n",
    "    word2idx[word] = index+1\n",
    "# create weight embedding matrix for GloVe embedding\n",
    "weight_embedding = np.zeros([len(vocab)+1,embed_dim],dtype='float32')\n",
    "for i, word in enumerate(vocab):\n",
    "    try:\n",
    "        # if word found in pretrained GloVe embedding dictionary\n",
    "        weight_embedding[i,:] = embedding_dict[word]\n",
    "        words_found += 1\n",
    "    except:\n",
    "        # if word not found, initialize as random vector\n",
    "        weight_embedding[i,:] = np.random.normal(scale=0.6, size=[embed_dim,])\n",
    "        words_OOV += 1\n",
    "\n",
    "print(\"{} words found\".format(words_found))\n",
    "print(\"{} OOV words\".format(words_OOV))\n",
    "idx2word = dict((v,k) for k,v in word2idx.items())\n",
    "print(\"{} words in the vocabulary\".format(len(vocab)))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qpuZXnpDe1fq",
    "outputId": "03c8b306-5d22-4e74-9255-5fe60d66c4bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tData Shapes:\n",
      "Train set: \t\t(800000, 50) \n",
      "Test set: \t\t(200000, 50)\n",
      "\n",
      "\n",
      "\t\t\tFeature Shapes:\n",
      "Train set: \t\t(800000, 14) \n",
      "Test set: \t\t(200000, 14)\n"
     ]
    }
   ],
   "source": [
    "text_ints = []\n",
    "summary_ints = []\n",
    "# convert input text and summaries from words to integers\n",
    "for each in data_text:\n",
    "    sentence = []\n",
    "    for word in each.split():\n",
    "        sentence.extend([word2idx[word]])\n",
    "    text_ints.append(sentence)\n",
    "for each in data_sum:\n",
    "    sentence = []\n",
    "    for word in each.split():\n",
    "        sentence.extend([word2idx[word]])\n",
    "    summary_ints.append(sentence)\n",
    "\n",
    "# pad input with 'pad' for sequence length = 50\n",
    "# pad summaries with 'pad' for sequence length = 14\n",
    "text_len = 50 \n",
    "summary_len = 14\n",
    "text_feature = np.zeros((len(text_ints),text_len),dtype=int)\n",
    "summary_feature = np.zeros((len(summary_ints),summary_len), dtype=int)\n",
    "for i, row in enumerate(text_ints):\n",
    "    text_feature[i,:len(row)] = np.array(row)[:text_len]\n",
    "for i, row in enumerate(summary_ints):\n",
    "    summary_feature[i,:len(row)] = np.array(row)[:summary_len]\n",
    "\n",
    "# train test split\n",
    "split_frac = 0.8 \n",
    "split_index = int(len(text_feature)*0.8)\n",
    "train_text, test_text = text_feature[:split_index],text_feature[split_index:]\n",
    "train_summary, test_summary = summary_feature[:split_index],summary_feature[split_index:]\n",
    "\n",
    "\n",
    "print(\"\\t\\t\\tData Shapes:\")\n",
    "print(\"Train set: \\t\\t{}\".format(train_text.shape), \n",
    "      \"\\nTest set: \\t\\t{}\".format(test_text.shape))\n",
    "\n",
    "print(\"\\n\\n\\t\\t\\tFeature Shapes:\")\n",
    "print(\"Train set: \\t\\t{}\".format(train_summary.shape),\n",
    "      \"\\nTest set: \\t\\t{}\".format(test_summary.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mGhY9bhEe1gJ",
    "outputId": "7836d56a-deae-40f9-8b2d-f2b31f033f53"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Saved!\n"
     ]
    }
   ],
   "source": [
    "# save the data an .npz file\n",
    "np.savez('data1M300d.npz',train_text=train_text,test_text=test_text,\n",
    "         train_summary=train_summary,test_summary=test_summary,\n",
    "         word2idx = word2idx,idx2word=idx2word, embed_matrix=weight_embedding, output_length = output_len)\n",
    "print(\"Data Saved!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "Ovnv3XZpe1gQ",
    "outputId": "0a90de3f-97de-4704-fde4-a139cd163890"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data\n",
      "Data Loaded!\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import punctuation\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt\n",
    "import string \n",
    "import os\n",
    "import re\n",
    "import queue\n",
    "%matplotlib inline\n",
    "\n",
    "# load in the data\n",
    "print(\"Loading data\")\n",
    "data = np.load('data100k300d.npz')\n",
    "train_text = data['train_text']\n",
    "test_text = data['test_text']\n",
    "train_summary = data['train_summary']\n",
    "test_summary = data['test_summary']\n",
    "word2idx = data['word2idx']\n",
    "word2idx = dict(word2idx.item())\n",
    "idx2word = data['idx2word']\n",
    "idx2word = dict(idx2word.item())\n",
    "weight_embedding = data['embed_matrix']\n",
    "weight_embedding = torch.from_numpy(weight_embedding)\n",
    "output_length = data['output_length']\n",
    "split_index = int(len(test_text)*0.1)\n",
    "val_text, test_text  = test_text[:split_index],test_text[split_index:]\n",
    "val_summary, test_summary  = test_summary[:split_index],test_summary[split_index:]\n",
    "print('Data Loaded!')\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\") # device to run the code on GPU\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sM9G8YV8e1ge"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "def create_emb(weight_matrix, non_trainable=False):\n",
    "    # create embedding matrix with the pretrained GloVe embedding matrix\n",
    "    emb_layer = torch.nn.Embedding(weight_matrix.shape[0],weight_matrix.shape[1])\n",
    "    emb_layer.load_state_dict({'weight': weight_matrix})\n",
    "    if non_trainable:\n",
    "        emb_layer.weight.requires_grad = False # turn off training for the embedding vector\n",
    "    return emb_layer\n",
    "\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self,kernel_size,filter_size,dropout,num_hidden,layers,weight_matrix,embed_dim,device):\n",
    "        super(Encoder,self).__init__()\n",
    "        self.layers = layers\n",
    "        self.kernel_size = kernel_size\n",
    "        self.filter_size = filter_size\n",
    "        self.num_hidden = num_hidden\n",
    "        self.device = device\n",
    "        self.embed_dim = embed_dim\n",
    "        \n",
    "        # convolutional layers\n",
    "        self.conv1 = torch.nn.Conv1d(embed_dim,self.filter_size,self.kernel_size[0],stride=1,padding=0)\n",
    "        self.conv2 = torch.nn.Conv1d(embed_dim,self.filter_size,self.kernel_size[1],stride=1,padding=1)\n",
    "        self.conv3 = torch.nn.Conv1d(embed_dim,self.filter_size,self.kernel_size[2],stride=1,padding=2)\n",
    "        self.dropout = torch.nn.Dropout(dropout)\n",
    "        self.embedding = create_emb(weight_matrix,True)\n",
    "        # encoder LSTM\n",
    "        self.lstm = torch.nn.LSTM(input_size = self.embed_dim,hidden_size = self.num_hidden,\n",
    "                                  num_layers = self.layers,batch_first=True,dropout=0.5,\n",
    "                                 bidirectional=True)\n",
    "    def forward(self,x,hidden):\n",
    "        x = self.embedding(x) # embed the input text\n",
    "        \n",
    "        # convolutional layers\n",
    "        #x1 = torch.tanh(self.dropout(self.conv1(x)))\n",
    "        #x2 = torch.tanh(self.dropout(self.conv2(x)))\n",
    "        #x3 = torch.tanh(self.dropout(self.conv3(x)))\n",
    "        # apply dropout\n",
    "        lstm_in = self.dropout((x))\n",
    "        # encoder LSTM\n",
    "        output,(h_hidden,c_hidden) = self.lstm(lstm_in,hidden)\n",
    "        # hidden state of the encoder (in case of multilayer LSTM)\n",
    "        h = torch.cat((h_hidden[-1,:,:],h_hidden[-2,:,:]),1)\n",
    "        c = torch.cat((c_hidden[-1,:,:],c_hidden[-2,:,:]),1)\n",
    "\n",
    "        hidden = (h.unsqueeze(0),c.unsqueeze(0))\n",
    "       \n",
    "        return output, hidden\n",
    "    \n",
    "    def init_hidden(self,batch_size):\n",
    "        # initialize the hidden state as the zero tensor\n",
    "        weight = next(self.parameters()).data\n",
    "        hidden = (weight.new(self.layers*2,batch_size, self.num_hidden).zero_(),\n",
    "                  weight.new(self.layers*2, batch_size, self.num_hidden).zero_())\n",
    "        return hidden\n",
    "        \n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iOzAegfHe1gs"
   },
   "outputs": [],
   "source": [
    "\n",
    "# code for the attentional decoder\n",
    "class AttentionDecoder(nn.Module):\n",
    "    def __init__(self,num_hidden,dropout,vocab_size,layers,weight_matrix,embed_dims,device):\n",
    "        super(AttentionDecoder,self).__init__()\n",
    "        self.num_hidden = num_hidden\n",
    "        self.dropout = dropout\n",
    "        self.layers = layers\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embed_dims = embed_dims\n",
    "        self.device = device\n",
    "        self.dropout_layer = torch.nn.Dropout(self.dropout)\n",
    "        \n",
    "        # Linear layers for the attention mechanism\n",
    "        self.V = torch.nn.Linear(self.num_hidden,1)\n",
    "        \n",
    "        # layer to get the pointer probability\n",
    "        self.generator = torch.nn.Linear(2*self.num_hidden+self.embed_dims,1)\n",
    "        # output layer to vocab\n",
    "        self.output_layer = torch.nn.Linear(2*self.num_hidden,self.vocab_size)\n",
    "        # softmax function\n",
    "        self.softmax = torch.nn.Softmax(dim=1)\n",
    "        # decoder LSTM\n",
    "        self.lstm = torch.nn.LSTM(input_size = self.num_hidden+self.embed_dims,hidden_size = self.num_hidden,\n",
    "                                  num_layers = self.layers,batch_first=True,dropout=0.5,\n",
    "                                 bidirectional=False)\n",
    "        # embedding matrix\n",
    "        self.embedding = create_emb(weight_matrix,True)\n",
    "        self.sig = torch.nn.Sigmoid()\n",
    "        self.device = device\n",
    "        \n",
    "        \n",
    "    def forward(self,x,enc_out,hidden,text,batch_size):\n",
    "        # decoder\n",
    "        # Decoder Input Shape: [batch_size]\n",
    "        x = self.embedding(x).unsqueeze(1)\n",
    "        # Decoder Embedded Shape: [batch_size,1,embed_dim]\n",
    "        x = self.dropout_layer(x)\n",
    "        \n",
    "        # Bahdanau Attention\n",
    "        dec_a = hidden[0].permute(1,0,2)\n",
    "        enc_score = self.V(torch.tanh(enc_out + dec_a)) # attention score\n",
    "        # Attention Score Shape: [batch_size,input_seq_len,1]\n",
    "        enc_weight = self.softmax(enc_score) # attention weight\n",
    "        # Attention Weight Shape: [batch_size,input_seq_len,1]\n",
    "        enc_context = torch.mul(enc_weight,enc_out) # find the context vector\n",
    "        # Attention Context Shape: [batch_size,input_seq_len,decoder_num_hidden]\n",
    "        enc_context = enc_context.sum(1)\n",
    "        # Attention Context Shape: [batch_size,decoder_num_hidden]\n",
    "        enc_context.unsqueeze_(1)\n",
    "        # Attention Context Shape: [batch_size,1,decoder_num_hidden]     \n",
    "        \n",
    "        d_in = torch.cat((x,enc_context),2)\n",
    "        # Decoder Input Shape: [batch_size,1,decoder_num_hidden+embed_dims]  \n",
    "        \n",
    "        # run the decoder LSTM\n",
    "        d_output, hidden = self.lstm(d_in,hidden)\n",
    "        # Decoder Output Shape: [batch_size,1,decoder_num_hidden]  \n",
    "        # Decoder Hidden Shape: [1,batch_size,decoder_num_hidden]  \n",
    "\n",
    "        # concatenate output with the encoder context tensor\n",
    "        output = torch.cat((d_output.squeeze(1),enc_context.squeeze(1)),1)\n",
    "        # Decoder Output Shape: [batch_size,2*decoder_num_hidden]  \n",
    "        \n",
    "        output_generator = torch.cat((enc_context.squeeze(1),d_output.squeeze(1),x.squeeze(1)),1)\n",
    "        # Generator Input Shape: [batch_size,2*decoder_num_hidden + embed_dims]  \n",
    "        \n",
    "        p_gen = self.sig(self.generator(output_generator).squeeze(1))\n",
    "        \n",
    "        # pointer-generator\n",
    "        p_pointer = 1 - p_gen\n",
    "        pointer_prob = torch.zeros([batch_size,self.vocab_size],device=self.device)\n",
    "        for i in range(batch_size):\n",
    "            pointer_prob[i,text[i,:]] = enc_weight[i,:,0] # pointer probability weights are the attention scores\n",
    "        generator_prob = self.output_layer(output) # output layer to get vocabulary probability\n",
    "        output_probability = torch.mul(p_pointer.unsqueeze(1),pointer_prob) + torch.mul(p_gen.unsqueeze(1),generator_prob)\n",
    "        \n",
    "        return output_probability, hidden\n",
    "    \n",
    "    def init_hidden(self,batch_size):\n",
    "        # initialize the hidden state as the zero tensor\n",
    "        weight = next(self.parameters()).data\n",
    "        hidden = (weight.new(self.layers, batch_size, 2*self.num_hidden).zero_(),\n",
    "                  weight.new(self.layers, batch_size, 2*self.num_hidden).zero_())\n",
    "        return hidden\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oAgfyC4ae1gz"
   },
   "outputs": [],
   "source": [
    "# hyperparameters of the mode\n",
    "kernel_size = [1,3,5] # kernel size for CNN\n",
    "filter_size = 100 # number of kernels used in CNN\n",
    "dropout = 0.5 # dropout probability\n",
    "num_hidden = 256 # number of hidden units in LSTMs\n",
    "enc_layers = 1 # number of layers in the LSTM encoder\n",
    "batch_size = 32\n",
    "vocab_size = len(word2idx) # length of vocab\n",
    "dec_layers = 1 # number of layers in the LSTM decoder\n",
    "embed_dims = 300 # embedding dimensions\n",
    "\n",
    "# create a beam search node to store the running sequences for the beam search decoder\n",
    "# and records the hidden state associated with the sequence, the probability and the loss\n",
    "class BeamNode(object):\n",
    "    def __init__(self,hidden_state,seq,prob,length,loss):\n",
    "        self.hidden = hidden_state\n",
    "        self.seq = seq\n",
    "        self.prob = prob\n",
    "        self.len = length\n",
    "        self.loss = loss\n",
    "        self.s = self.score()\n",
    "        \n",
    "    def score(self):\n",
    "        return self.prob/float(self.len-1+1e-6) # calculates the score of a sequence\n",
    "\n",
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self,kernel_size,filter_size,dropout,num_hidden,enc_layers,weight_embedding,\n",
    "                vocab_size,dec_layers,embed_dims,device):\n",
    "        super(Seq2Seq, self).__init__()\n",
    "        self.vocab_size = vocab_size\n",
    "        self.device = device\n",
    "        self.encoder = Encoder(kernel_size,filter_size,dropout,num_hidden,enc_layers,weight_embedding,embed_dims,device)\n",
    "        self.decoder = AttentionDecoder(2*num_hidden,dropout,vocab_size,dec_layers,weight_embedding,embed_dims,device)\n",
    "    \n",
    "    def forward(self,x,target,e_hidden,criterion,batch_size):\n",
    "        # training the decoder\n",
    "        loss = 0\n",
    "        prediction = target[:,0].unsqueeze(1) # records the running sequences generated by the decoder\n",
    "        enc_output,enc_hidden = self.encoder(x,e_hidden) # run the encoder\n",
    "        d_hidden = enc_hidden # the decoder input hidden state is the encoder output hidden state\n",
    "        dec_input = target[:,0] # the decoder input starts of as the 'bos' token\n",
    "        for t in range(1,target.shape[1]):\n",
    "            # run the decoder\n",
    "            logits, d_hidden = self.decoder(dec_input,enc_output,d_hidden,x,batch_size)\n",
    "            dec_input = target[:,t] # teacher forcing turned on \n",
    "            loss += criterion(logits,target[:,t]) # calculate the loss function\n",
    "            # add to get the running prediction output by the decoder\n",
    "            prediction = torch.cat((prediction,torch.argmax(logits,dim=0).unsqueeze(1)),0)\n",
    "        return loss, prediction\n",
    "    \n",
    "    def inference_greedy(self,x,target,e_hidden,criterion,batch_size):\n",
    "        loss = 0\n",
    "        prediction = target[:,0].unsqueeze(1) # records the running sequences generated by the decoder\n",
    "        enc_output,enc_hidden = self.encoder(x,e_hidden) # run the encoder\n",
    "        d_hidden = enc_hidden # the decoder input hidden state is the encoder output hidden state\n",
    "        dec_input = target[:,0] # the decoder input starts of as the 'bos' token\n",
    "        for t in range(1,target.shape[1]):\n",
    "            # run the decoder\n",
    "            logits, d_hidden = self.decoder(dec_input,enc_output,d_hidden,x,batch_size)\n",
    "            # the input to the decoder at the next step is the argument with the largest probability\n",
    "            dec_input = torch.argmax(logits,dim=1)\n",
    "            loss += criterion(logits,target[:,t]) # calculate the loss\n",
    "            # add to get the running prediction output by the decoder\n",
    "            prediction = torch.cat((prediction,torch.argmax(logits,dim=0).unsqueeze(1)),0)\n",
    "        return loss, prediction\n",
    "    \n",
    "    def inference_beam(self,x,target,e_hidden,criterion,beam_width,batch_size):\n",
    "        decoded = [] \n",
    "        losses = 0\n",
    "        enc_output,enc_hidden = self.encoder(x,e_hidden) # run the encoder\n",
    "        for i in range(batch_size):\n",
    "            # for each sentence in the batch\n",
    "            prediction = target[i,0].view([1]).unsqueeze(1) # running prediction tensor\n",
    "            dec_hidden = enc_hidden[0].permute(1,0,2)[i,:,:].unsqueeze(0)\n",
    "            dec_input = target[i,0].view([1])\n",
    "            d_hidden = (enc_hidden[0][:,i,:].unsqueeze(0),enc_hidden[1][:,i,:].unsqueeze(0))\n",
    "            first_node = BeamNode(d_hidden,dec_input,0,1,0) # the first node is the 'bos' token\n",
    "            nodes = queue.PriorityQueue(maxsize=beam_width) # create a priority queue to store the beam search nodes\n",
    "            nodes.put((-first_node.score(),first_node)) # place the first node in the queue\n",
    "            for t in range(1,target.shape[1]):\n",
    "                # go through each of the words in the target\n",
    "                candidatenodes = [] # stores the candidate nodes for a one step look ahead\n",
    "                candidatescore = [] # stores the candidate scores for a one step look ahead\n",
    "                donenodes = [] # stores the sequences that have been completed - contain 'eos' token\n",
    "\n",
    "                # This runs through the 10 sequences in the queue and does a one step look ahead\n",
    "                # to find 100 possible candidates. The top 10 candidates are then added to the queue\n",
    "                # and the next word of the decoder is computed\n",
    "                \n",
    "                for _ in range(nodes.qsize()):\n",
    "                    # for each sequence in the queue\n",
    "                    sc,nodex = nodes.get() # get the node from the queue\n",
    "                    seq = nodex.seq.view([-1,1]) # the sequence in the node\n",
    "                    dec_input = seq[-1,:] # the last word of the sequence is the input to the decoder\n",
    "                    hidden = nodex.hidden # the hidden state associated with the sequence\n",
    "                    d_hidden = (hidden[0],hidden[1]) # the hidden state of the decoder\n",
    "\n",
    "                    if dec_input == word2idx['eos']:\n",
    "                        # if the sequence is completed, then it is added to 'donenodes' list\n",
    "                        donenodes.append((sc,nodex))\n",
    "                    else:\n",
    "                        # run the decoder\n",
    "                        logits, d_hidden = self.decoder(dec_input,enc_output[i,:,:].unsqueeze(0),d_hidden,x,batch_size=1)\n",
    "                        # calculate the loss\n",
    "                        loss = criterion(logits,target[i,t].view([1]))\n",
    "                        # pick the top 10 logits values\n",
    "                        log_p, index = torch.topk(logits,beam_width)\n",
    "                        for k in range(beam_width):\n",
    "                            # create a beam node for each of the top 10 to get the candidate nodes\n",
    "                            node = BeamNode(d_hidden, torch.cat([nodex.seq,index[0,k].unsqueeze(0)]),nodex.prob+log_p[0][k],nodex.len+1,nodex.loss+loss)\n",
    "                            score = -node.score()\n",
    "                            candidatenodes.append([score,node])\n",
    "                # put the finished sequences first in the queue first and then fill with candidate nodes\n",
    "                for score,n in donenodes:\n",
    "                    nodes.put((score,n))\n",
    "                for score,n in candidatenodes[nodes.qsize():beam_width]:\n",
    "                    nodes.put((score,n))\n",
    "            _,output_node = nodes.get()\n",
    "            \n",
    "            # the output of the beam search decoder is the sequence with the lowest score\n",
    "            decoded.append(output_node.seq)\n",
    "            losses+=output_node.loss\n",
    "            del nodes\n",
    "        return losses/batch_size,decoded\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YD6zWGZ_e1g4"
   },
   "outputs": [],
   "source": [
    "def get_batches(x, y,batch_size=100):\n",
    "    n_batches = len(x)//batch_size\n",
    "    x, y = x[:n_batches*batch_size], y[:n_batches*batch_size]\n",
    "    for ii in range(0, len(x), batch_size):\n",
    "        yield x[ii:ii+batch_size], y[ii:ii+batch_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 8741
    },
    "colab_type": "code",
    "id": "SP_-cvZae1g9",
    "outputId": "c29646aa-4f4f-42ac-aab1-b2febf1f79fe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/15  \tStep: 0  \tLoss: 141.9948 \n",
      "Epoch: 1/15  \tStep: 50  \tLoss: 92.8735 \n",
      "Epoch: 1/15  \tStep: 100  \tLoss: 84.0220 \n",
      "Epoch: 1/15  \tStep: 150  \tLoss: 78.2484 \n",
      "Epoch: 1/15  \tStep: 200  \tLoss: 81.2634 \n",
      "Epoch: 1/15  \tStep: 250  \tLoss: 76.2267 \n",
      "Epoch: 1/15  \tStep: 300  \tLoss: 83.6968 \n",
      "Epoch: 1/15  \tStep: 350  \tLoss: 84.6046 \n",
      "Epoch: 1/15  \tStep: 400  \tLoss: 85.4649 \n",
      "Epoch: 1/15  \tStep: 450  \tLoss: 78.7258 \n",
      "Epoch: 1/15  \tStep: 500  \tLoss: 78.5601 \n",
      "Epoch: 1/15  \tStep: 550  \tLoss: 88.6967 \n",
      "Epoch: 1/15  \tStep: 600  \tLoss: 67.9000 \n",
      "Epoch: 1/15  \tStep: 650  \tLoss: 79.5246 \n",
      "Epoch: 1/15  \tStep: 700  \tLoss: 74.8440 \n",
      "Epoch: 1/15  \tStep: 750  \tLoss: 75.3816 \n",
      "Epoch: 1/15  \tStep: 800  \tLoss: 74.4047 \n",
      "Epoch: 1/15  \tStep: 850  \tLoss: 78.3814 \n",
      "Epoch: 1/15  \tStep: 900  \tLoss: 78.0796 \n",
      "Epoch: 1/15  \tStep: 950  \tLoss: 72.6038 \n",
      "Epoch: 1/15  \tStep: 1000  \tLoss: 77.3561 \n",
      "Epoch: 1/15  \tStep: 1050  \tLoss: 81.0590 \n",
      "Epoch: 1/15  \tStep: 1100  \tLoss: 72.0654 \n",
      "Epoch: 1/15  \tStep: 1150  \tLoss: 69.8134 \n",
      "Epoch: 1/15  \tStep: 1200  \tLoss: 73.8085 \n",
      "Epoch: 1/15  \tStep: 1250  \tLoss: 66.5266 \n",
      "Epoch: 1/15  \tStep: 1300  \tLoss: 67.6565 \n",
      "Epoch: 1/15  \tStep: 1350  \tLoss: 78.6387 \n",
      "Epoch: 1/15  \tStep: 1400  \tLoss: 69.2062 \n",
      "Epoch: 1/15  \tStep: 1450  \tLoss: 66.6248 \n",
      "Epoch: 1/15  \tStep: 1500  \tLoss: 61.8506 \n",
      "Epoch: 1/15  \tStep: 1550  \tLoss: 69.7987 \n",
      "Epoch: 1/15  \tStep: 1600  \tLoss: 70.8397 \n",
      "Epoch: 1/15  \tStep: 1650  \tLoss: 71.6094 \n",
      "Epoch: 1/15  \tStep: 1700  \tLoss: 68.3355 \n",
      "Epoch: 1/15  \tStep: 1750  \tLoss: 69.0275 \n",
      "Epoch: 1/15  \tStep: 1800  \tLoss: 68.8657 \n",
      "Epoch: 1/15  \tStep: 1850  \tLoss: 73.1294 \n",
      "Epoch: 1/15  \tStep: 1900  \tLoss: 66.6112 \n",
      "Epoch: 1/15  \tStep: 1950  \tLoss: 61.5314 \n",
      "Epoch: 1/15  \tStep: 2000  \tLoss: 76.6162 \n",
      "Epoch: 1/15  \tStep: 2050  \tLoss: 69.8317 \n",
      "Epoch: 1/15  \tStep: 2100  \tLoss: 69.1603 \n",
      "Epoch: 1/15  \tStep: 2150  \tLoss: 64.5025 \n",
      "Epoch: 1/15  \tStep: 2200  \tLoss: 76.5575 \n",
      "Epoch: 1/15  \tStep: 2250  \tLoss: 67.1616 \n",
      "Epoch: 1/15  \tStep: 2300  \tLoss: 67.2063 \n",
      "Epoch: 1/15  \tStep: 2350  \tLoss: 69.9872 \n",
      "Epoch: 1/15  \tStep: 2400  \tLoss: 64.7164 \n",
      "Epoch: 1/15  \tStep: 2450  \tLoss: 66.8009 \n",
      "Time to train epoch: 1390.2508704662323 s\n",
      "Epoch: 2/15  \tStep: 2500  \tLoss: 61.5868 \n",
      "Epoch: 2/15  \tStep: 2550  \tLoss: 61.5923 \n",
      "Epoch: 2/15  \tStep: 2600  \tLoss: 61.4054 \n",
      "Epoch: 2/15  \tStep: 2650  \tLoss: 59.3951 \n",
      "Epoch: 2/15  \tStep: 2700  \tLoss: 56.3940 \n",
      "Epoch: 2/15  \tStep: 2750  \tLoss: 62.5829 \n",
      "Epoch: 2/15  \tStep: 2800  \tLoss: 69.8529 \n",
      "Epoch: 2/15  \tStep: 2850  \tLoss: 66.8564 \n",
      "Epoch: 2/15  \tStep: 2900  \tLoss: 61.5209 \n",
      "Epoch: 2/15  \tStep: 2950  \tLoss: 65.5651 \n",
      "Epoch: 2/15  \tStep: 3000  \tLoss: 61.9655 \n",
      "Epoch: 2/15  \tStep: 3050  \tLoss: 64.7906 \n",
      "Epoch: 2/15  \tStep: 3100  \tLoss: 60.9439 \n",
      "Epoch: 2/15  \tStep: 3150  \tLoss: 66.6128 \n",
      "Epoch: 2/15  \tStep: 3200  \tLoss: 68.8399 \n",
      "Epoch: 2/15  \tStep: 3250  \tLoss: 58.5416 \n",
      "Epoch: 2/15  \tStep: 3300  \tLoss: 60.8023 \n",
      "Epoch: 2/15  \tStep: 3350  \tLoss: 65.3559 \n",
      "Epoch: 2/15  \tStep: 3400  \tLoss: 60.1391 \n",
      "Epoch: 2/15  \tStep: 3450  \tLoss: 58.2548 \n",
      "Epoch: 2/15  \tStep: 3500  \tLoss: 66.2106 \n",
      "Epoch: 2/15  \tStep: 3550  \tLoss: 58.0417 \n",
      "Epoch: 2/15  \tStep: 3600  \tLoss: 66.0394 \n",
      "Epoch: 2/15  \tStep: 3650  \tLoss: 55.4318 \n",
      "Epoch: 2/15  \tStep: 3700  \tLoss: 61.3859 \n",
      "Epoch: 2/15  \tStep: 3750  \tLoss: 62.0833 \n",
      "Epoch: 2/15  \tStep: 3800  \tLoss: 65.4148 \n",
      "Epoch: 2/15  \tStep: 3850  \tLoss: 58.8403 \n",
      "Epoch: 2/15  \tStep: 3900  \tLoss: 53.0558 \n",
      "Epoch: 2/15  \tStep: 3950  \tLoss: 51.9432 \n",
      "Epoch: 2/15  \tStep: 4000  \tLoss: 62.1214 \n",
      "Epoch: 2/15  \tStep: 4050  \tLoss: 61.8627 \n",
      "Epoch: 2/15  \tStep: 4100  \tLoss: 67.2190 \n",
      "Epoch: 2/15  \tStep: 4150  \tLoss: 58.7158 \n",
      "Epoch: 2/15  \tStep: 4200  \tLoss: 58.5384 \n",
      "Epoch: 2/15  \tStep: 4250  \tLoss: 56.4200 \n",
      "Epoch: 2/15  \tStep: 4300  \tLoss: 62.1489 \n",
      "Epoch: 2/15  \tStep: 4350  \tLoss: 62.6805 \n",
      "Epoch: 2/15  \tStep: 4400  \tLoss: 56.8217 \n",
      "Epoch: 2/15  \tStep: 4450  \tLoss: 53.9855 \n",
      "Epoch: 2/15  \tStep: 4500  \tLoss: 67.2691 \n",
      "Epoch: 2/15  \tStep: 4550  \tLoss: 61.3577 \n",
      "Epoch: 2/15  \tStep: 4600  \tLoss: 65.0251 \n",
      "Epoch: 2/15  \tStep: 4650  \tLoss: 57.7213 \n",
      "Epoch: 2/15  \tStep: 4700  \tLoss: 58.2334 \n",
      "Epoch: 2/15  \tStep: 4750  \tLoss: 55.2441 \n",
      "Epoch: 2/15  \tStep: 4800  \tLoss: 62.9576 \n",
      "Epoch: 2/15  \tStep: 4850  \tLoss: 59.0677 \n",
      "Epoch: 2/15  \tStep: 4900  \tLoss: 54.4195 \n",
      "Epoch: 2/15  \tStep: 4950  \tLoss: 57.1507 \n",
      "Time to train epoch: 1395.4119021892548 s\n",
      "Epoch: 3/15  \tStep: 5000  \tLoss: 47.9496 \n",
      "Epoch: 3/15  \tStep: 5050  \tLoss: 54.1838 \n",
      "Epoch: 3/15  \tStep: 5100  \tLoss: 57.0756 \n",
      "Epoch: 3/15  \tStep: 5150  \tLoss: 48.8872 \n",
      "Epoch: 3/15  \tStep: 5200  \tLoss: 56.2471 \n",
      "Epoch: 3/15  \tStep: 5250  \tLoss: 48.1642 \n",
      "Epoch: 3/15  \tStep: 5300  \tLoss: 55.9353 \n",
      "Epoch: 3/15  \tStep: 5350  \tLoss: 47.5778 \n",
      "Epoch: 3/15  \tStep: 5400  \tLoss: 57.6248 \n",
      "Epoch: 3/15  \tStep: 5450  \tLoss: 52.9975 \n",
      "Epoch: 3/15  \tStep: 5500  \tLoss: 60.6621 \n",
      "Epoch: 3/15  \tStep: 5550  \tLoss: 58.1180 \n",
      "Epoch: 3/15  \tStep: 5600  \tLoss: 56.8843 \n",
      "Epoch: 3/15  \tStep: 5650  \tLoss: 55.8090 \n",
      "Epoch: 3/15  \tStep: 5700  \tLoss: 52.2141 \n",
      "Epoch: 3/15  \tStep: 5750  \tLoss: 55.9491 \n",
      "Epoch: 3/15  \tStep: 5800  \tLoss: 48.2623 \n",
      "Epoch: 3/15  \tStep: 5850  \tLoss: 53.1551 \n",
      "Epoch: 3/15  \tStep: 5900  \tLoss: 49.7450 \n",
      "Epoch: 3/15  \tStep: 5950  \tLoss: 53.2256 \n",
      "Epoch: 3/15  \tStep: 6000  \tLoss: 46.5316 \n",
      "Epoch: 3/15  \tStep: 6050  \tLoss: 50.4424 \n",
      "Epoch: 3/15  \tStep: 6100  \tLoss: 56.9590 \n",
      "Epoch: 3/15  \tStep: 6150  \tLoss: 62.3200 \n",
      "Epoch: 3/15  \tStep: 6200  \tLoss: 51.2502 \n",
      "Epoch: 3/15  \tStep: 6250  \tLoss: 51.3755 \n",
      "Epoch: 3/15  \tStep: 6300  \tLoss: 53.9280 \n",
      "Epoch: 3/15  \tStep: 6350  \tLoss: 53.7987 \n",
      "Epoch: 3/15  \tStep: 6400  \tLoss: 54.4468 \n",
      "Epoch: 3/15  \tStep: 6450  \tLoss: 54.6213 \n",
      "Epoch: 3/15  \tStep: 6500  \tLoss: 52.8924 \n",
      "Epoch: 3/15  \tStep: 6550  \tLoss: 54.9243 \n",
      "Epoch: 3/15  \tStep: 6600  \tLoss: 54.8490 \n",
      "Epoch: 3/15  \tStep: 6650  \tLoss: 48.0397 \n",
      "Epoch: 3/15  \tStep: 6700  \tLoss: 53.9179 \n",
      "Epoch: 3/15  \tStep: 6750  \tLoss: 49.0103 \n",
      "Epoch: 3/15  \tStep: 6800  \tLoss: 54.3341 \n",
      "Epoch: 3/15  \tStep: 6850  \tLoss: 54.0755 \n",
      "Epoch: 3/15  \tStep: 6900  \tLoss: 52.6949 \n",
      "Epoch: 3/15  \tStep: 6950  \tLoss: 52.9086 \n",
      "Epoch: 3/15  \tStep: 7000  \tLoss: 49.4204 \n",
      "Epoch: 3/15  \tStep: 7050  \tLoss: 49.2787 \n",
      "Epoch: 3/15  \tStep: 7100  \tLoss: 58.7345 \n",
      "Epoch: 3/15  \tStep: 7150  \tLoss: 56.0164 \n",
      "Epoch: 3/15  \tStep: 7200  \tLoss: 53.0144 \n",
      "Epoch: 3/15  \tStep: 7250  \tLoss: 52.5297 \n",
      "Epoch: 3/15  \tStep: 7300  \tLoss: 53.5634 \n",
      "Epoch: 3/15  \tStep: 7350  \tLoss: 53.9478 \n",
      "Epoch: 3/15  \tStep: 7400  \tLoss: 57.6950 \n",
      "Epoch: 3/15  \tStep: 7450  \tLoss: 45.8577 \n",
      "Time to train epoch: 1391.0922195911407 s\n",
      "Epoch: 4/15  \tStep: 7500  \tLoss: 43.7981 \n",
      "Epoch: 4/15  \tStep: 7550  \tLoss: 46.6195 \n",
      "Epoch: 4/15  \tStep: 7600  \tLoss: 48.4205 \n",
      "Epoch: 4/15  \tStep: 7650  \tLoss: 50.1331 \n",
      "Epoch: 4/15  \tStep: 7700  \tLoss: 40.8550 \n",
      "Epoch: 4/15  \tStep: 7750  \tLoss: 48.7927 \n",
      "Epoch: 4/15  \tStep: 7800  \tLoss: 47.9812 \n",
      "Epoch: 4/15  \tStep: 7850  \tLoss: 43.8029 \n",
      "Epoch: 4/15  \tStep: 7900  \tLoss: 40.9684 \n",
      "Epoch: 4/15  \tStep: 7950  \tLoss: 43.1610 \n",
      "Epoch: 4/15  \tStep: 8000  \tLoss: 48.1470 \n",
      "Epoch: 4/15  \tStep: 8050  \tLoss: 50.4416 \n",
      "Epoch: 4/15  \tStep: 8100  \tLoss: 46.2378 \n",
      "Epoch: 4/15  \tStep: 8150  \tLoss: 43.8071 \n",
      "Epoch: 4/15  \tStep: 8200  \tLoss: 50.8006 \n",
      "Epoch: 4/15  \tStep: 8250  \tLoss: 47.9546 \n",
      "Epoch: 4/15  \tStep: 8300  \tLoss: 42.8460 \n",
      "Epoch: 4/15  \tStep: 8350  \tLoss: 47.8221 \n",
      "Epoch: 4/15  \tStep: 8400  \tLoss: 47.4401 \n",
      "Epoch: 4/15  \tStep: 8450  \tLoss: 47.2622 \n",
      "Epoch: 4/15  \tStep: 8500  \tLoss: 50.4553 \n",
      "Epoch: 4/15  \tStep: 8550  \tLoss: 52.6891 \n",
      "Epoch: 4/15  \tStep: 8600  \tLoss: 49.9526 \n",
      "Epoch: 4/15  \tStep: 8650  \tLoss: 43.5348 \n",
      "Epoch: 4/15  \tStep: 8700  \tLoss: 46.4994 \n",
      "Epoch: 4/15  \tStep: 8750  \tLoss: 47.3171 \n",
      "Epoch: 4/15  \tStep: 8800  \tLoss: 46.9303 \n",
      "Epoch: 4/15  \tStep: 8850  \tLoss: 44.4473 \n",
      "Epoch: 4/15  \tStep: 8900  \tLoss: 47.2664 \n",
      "Epoch: 4/15  \tStep: 8950  \tLoss: 43.0861 \n",
      "Epoch: 4/15  \tStep: 9000  \tLoss: 51.1738 \n",
      "Epoch: 4/15  \tStep: 9050  \tLoss: 45.7596 \n",
      "Epoch: 4/15  \tStep: 9100  \tLoss: 43.5695 \n",
      "Epoch: 4/15  \tStep: 9150  \tLoss: 47.8967 \n",
      "Epoch: 4/15  \tStep: 9200  \tLoss: 48.7376 \n",
      "Epoch: 4/15  \tStep: 9250  \tLoss: 43.1155 \n",
      "Epoch: 4/15  \tStep: 9300  \tLoss: 45.8506 \n",
      "Epoch: 4/15  \tStep: 9350  \tLoss: 44.5421 \n",
      "Epoch: 4/15  \tStep: 9400  \tLoss: 49.9936 \n",
      "Epoch: 4/15  \tStep: 9450  \tLoss: 47.2509 \n",
      "Epoch: 4/15  \tStep: 9500  \tLoss: 43.6004 \n",
      "Epoch: 4/15  \tStep: 9550  \tLoss: 53.3544 \n",
      "Epoch: 4/15  \tStep: 9600  \tLoss: 49.6292 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4/15  \tStep: 9650  \tLoss: 50.0437 \n",
      "Epoch: 4/15  \tStep: 9700  \tLoss: 50.1892 \n",
      "Epoch: 4/15  \tStep: 9750  \tLoss: 48.8464 \n",
      "Epoch: 4/15  \tStep: 9800  \tLoss: 48.1849 \n",
      "Epoch: 4/15  \tStep: 9850  \tLoss: 53.3362 \n",
      "Epoch: 4/15  \tStep: 9900  \tLoss: 50.3443 \n",
      "Epoch: 4/15  \tStep: 9950  \tLoss: 45.4564 \n",
      "Time to train epoch: 1389.3618381023407 s\n",
      "Epoch: 5/15  \tStep: 10000  \tLoss: 45.8945 \n",
      "Epoch: 5/15  \tStep: 10050  \tLoss: 39.8304 \n",
      "Epoch: 5/15  \tStep: 10100  \tLoss: 39.1575 \n",
      "Epoch: 5/15  \tStep: 10150  \tLoss: 36.6136 \n",
      "Epoch: 5/15  \tStep: 10200  \tLoss: 38.2140 \n",
      "Epoch: 5/15  \tStep: 10250  \tLoss: 39.2443 \n",
      "Epoch: 5/15  \tStep: 10300  \tLoss: 39.4646 \n",
      "Epoch: 5/15  \tStep: 10350  \tLoss: 41.5406 \n",
      "Epoch: 5/15  \tStep: 10400  \tLoss: 36.8862 \n",
      "Epoch: 5/15  \tStep: 10450  \tLoss: 33.6260 \n",
      "Epoch: 5/15  \tStep: 10500  \tLoss: 40.2368 \n",
      "Epoch: 5/15  \tStep: 10550  \tLoss: 44.9388 \n",
      "Epoch: 5/15  \tStep: 10600  \tLoss: 43.1830 \n",
      "Epoch: 5/15  \tStep: 10650  \tLoss: 49.1766 \n",
      "Epoch: 5/15  \tStep: 10700  \tLoss: 40.2547 \n",
      "Epoch: 5/15  \tStep: 10750  \tLoss: 40.8649 \n",
      "Epoch: 5/15  \tStep: 10800  \tLoss: 42.2797 \n",
      "Epoch: 5/15  \tStep: 10850  \tLoss: 38.0541 \n",
      "Epoch: 5/15  \tStep: 10900  \tLoss: 40.1395 \n",
      "Epoch: 5/15  \tStep: 10950  \tLoss: 40.3692 \n",
      "Epoch: 5/15  \tStep: 11000  \tLoss: 44.5697 \n",
      "Epoch: 5/15  \tStep: 11050  \tLoss: 39.7881 \n",
      "Epoch: 5/15  \tStep: 11100  \tLoss: 39.3386 \n",
      "Epoch: 5/15  \tStep: 11150  \tLoss: 39.9388 \n",
      "Epoch: 5/15  \tStep: 11200  \tLoss: 40.8485 \n",
      "Epoch: 5/15  \tStep: 11250  \tLoss: 42.1399 \n",
      "Epoch: 5/15  \tStep: 11300  \tLoss: 46.2358 \n",
      "Epoch: 5/15  \tStep: 11350  \tLoss: 41.2852 \n",
      "Epoch: 5/15  \tStep: 11400  \tLoss: 45.5443 \n",
      "Epoch: 5/15  \tStep: 11450  \tLoss: 42.3479 \n",
      "Epoch: 5/15  \tStep: 11500  \tLoss: 43.8931 \n",
      "Epoch: 5/15  \tStep: 11550  \tLoss: 45.8839 \n",
      "Epoch: 5/15  \tStep: 11600  \tLoss: 37.6800 \n",
      "Epoch: 5/15  \tStep: 11650  \tLoss: 43.3571 \n",
      "Epoch: 5/15  \tStep: 11700  \tLoss: 42.1325 \n",
      "Epoch: 5/15  \tStep: 11750  \tLoss: 46.2262 \n",
      "Epoch: 5/15  \tStep: 11800  \tLoss: 42.3666 \n",
      "Epoch: 5/15  \tStep: 11850  \tLoss: 42.3808 \n",
      "Epoch: 5/15  \tStep: 11900  \tLoss: 41.3052 \n",
      "Epoch: 5/15  \tStep: 11950  \tLoss: 40.4191 \n",
      "Epoch: 5/15  \tStep: 12000  \tLoss: 39.8315 \n",
      "Epoch: 5/15  \tStep: 12050  \tLoss: 41.4045 \n",
      "Epoch: 5/15  \tStep: 12100  \tLoss: 45.2425 \n",
      "Epoch: 5/15  \tStep: 12150  \tLoss: 46.2727 \n",
      "Epoch: 5/15  \tStep: 12200  \tLoss: 41.7126 \n",
      "Epoch: 5/15  \tStep: 12250  \tLoss: 43.9675 \n",
      "Epoch: 5/15  \tStep: 12300  \tLoss: 49.8363 \n",
      "Epoch: 5/15  \tStep: 12350  \tLoss: 44.1541 \n",
      "Epoch: 5/15  \tStep: 12400  \tLoss: 41.6076 \n",
      "Epoch: 5/15  \tStep: 12450  \tLoss: 40.7554 \n",
      "Time to train epoch: 1392.859228849411 s\n",
      "Epoch: 6/15  \tStep: 12500  \tLoss: 37.1053 \n",
      "Epoch: 6/15  \tStep: 12550  \tLoss: 33.4836 \n",
      "Epoch: 6/15  \tStep: 12600  \tLoss: 40.7216 \n",
      "Epoch: 6/15  \tStep: 12650  \tLoss: 35.3126 \n",
      "Epoch: 6/15  \tStep: 12700  \tLoss: 39.1728 \n",
      "Epoch: 6/15  \tStep: 12750  \tLoss: 34.4072 \n",
      "Epoch: 6/15  \tStep: 12800  \tLoss: 38.8405 \n",
      "Epoch: 6/15  \tStep: 12850  \tLoss: 37.3263 \n",
      "Epoch: 6/15  \tStep: 12900  \tLoss: 35.7503 \n",
      "Epoch: 6/15  \tStep: 12950  \tLoss: 37.4115 \n",
      "Epoch: 6/15  \tStep: 13000  \tLoss: 37.2949 \n",
      "Epoch: 6/15  \tStep: 13050  \tLoss: 39.3024 \n",
      "Epoch: 6/15  \tStep: 13100  \tLoss: 41.2988 \n",
      "Epoch: 6/15  \tStep: 13150  \tLoss: 36.1038 \n",
      "Epoch: 6/15  \tStep: 13200  \tLoss: 38.1023 \n",
      "Epoch: 6/15  \tStep: 13250  \tLoss: 34.7294 \n",
      "Epoch: 6/15  \tStep: 13300  \tLoss: 41.1164 \n",
      "Epoch: 6/15  \tStep: 13350  \tLoss: 35.5510 \n",
      "Epoch: 6/15  \tStep: 13400  \tLoss: 42.0032 \n",
      "Epoch: 6/15  \tStep: 13450  \tLoss: 38.9654 \n",
      "Epoch: 6/15  \tStep: 13500  \tLoss: 40.8102 \n",
      "Epoch: 6/15  \tStep: 13550  \tLoss: 40.4171 \n",
      "Epoch: 6/15  \tStep: 13600  \tLoss: 37.5774 \n",
      "Epoch: 6/15  \tStep: 13650  \tLoss: 40.6940 \n",
      "Epoch: 6/15  \tStep: 13700  \tLoss: 41.8243 \n",
      "Epoch: 6/15  \tStep: 13750  \tLoss: 36.8290 \n",
      "Epoch: 6/15  \tStep: 13800  \tLoss: 36.5383 \n",
      "Epoch: 6/15  \tStep: 13850  \tLoss: 37.5197 \n",
      "Epoch: 6/15  \tStep: 13900  \tLoss: 38.0899 \n",
      "Epoch: 6/15  \tStep: 13950  \tLoss: 43.5207 \n",
      "Epoch: 6/15  \tStep: 14000  \tLoss: 37.7373 \n",
      "Epoch: 6/15  \tStep: 14050  \tLoss: 36.0488 \n",
      "Epoch: 6/15  \tStep: 14100  \tLoss: 36.8793 \n",
      "Epoch: 6/15  \tStep: 14150  \tLoss: 35.0205 \n",
      "Epoch: 6/15  \tStep: 14200  \tLoss: 42.3298 \n",
      "Epoch: 6/15  \tStep: 14250  \tLoss: 40.6636 \n",
      "Epoch: 6/15  \tStep: 14300  \tLoss: 45.3798 \n",
      "Epoch: 6/15  \tStep: 14350  \tLoss: 36.0655 \n",
      "Epoch: 6/15  \tStep: 14400  \tLoss: 40.8903 \n",
      "Epoch: 6/15  \tStep: 14450  \tLoss: 44.5451 \n",
      "Epoch: 6/15  \tStep: 14500  \tLoss: 42.7751 \n",
      "Epoch: 6/15  \tStep: 14550  \tLoss: 44.7621 \n",
      "Epoch: 6/15  \tStep: 14600  \tLoss: 36.9989 \n",
      "Epoch: 6/15  \tStep: 14650  \tLoss: 43.2069 \n",
      "Epoch: 6/15  \tStep: 14700  \tLoss: 40.0011 \n",
      "Epoch: 6/15  \tStep: 14750  \tLoss: 39.7938 \n",
      "Epoch: 6/15  \tStep: 14800  \tLoss: 45.0223 \n",
      "Epoch: 6/15  \tStep: 14850  \tLoss: 37.7290 \n",
      "Epoch: 6/15  \tStep: 14900  \tLoss: 40.0047 \n",
      "Epoch: 6/15  \tStep: 14950  \tLoss: 43.5026 \n",
      "Time to train epoch: 1391.8433828353882 s\n",
      "Epoch: 7/15  \tStep: 15000  \tLoss: 32.6065 \n",
      "Epoch: 7/15  \tStep: 15050  \tLoss: 33.3177 \n",
      "Epoch: 7/15  \tStep: 15100  \tLoss: 34.3832 \n",
      "Epoch: 7/15  \tStep: 15150  \tLoss: 36.5538 \n",
      "Epoch: 7/15  \tStep: 15200  \tLoss: 37.3338 \n",
      "Epoch: 7/15  \tStep: 15250  \tLoss: 36.5434 \n",
      "Epoch: 7/15  \tStep: 15300  \tLoss: 29.7220 \n",
      "Epoch: 7/15  \tStep: 15350  \tLoss: 33.1910 \n",
      "Epoch: 7/15  \tStep: 15400  \tLoss: 31.1437 \n",
      "Epoch: 7/15  \tStep: 15450  \tLoss: 36.1737 \n",
      "Epoch: 7/15  \tStep: 15500  \tLoss: 35.3616 \n",
      "Epoch: 7/15  \tStep: 15550  \tLoss: 35.1888 \n",
      "Epoch: 7/15  \tStep: 15600  \tLoss: 33.9930 \n",
      "Epoch: 7/15  \tStep: 15650  \tLoss: 35.1181 \n",
      "Epoch: 7/15  \tStep: 15700  \tLoss: 35.3272 \n",
      "Epoch: 7/15  \tStep: 15750  \tLoss: 32.1207 \n",
      "Epoch: 7/15  \tStep: 15800  \tLoss: 33.8928 \n",
      "Epoch: 7/15  \tStep: 15850  \tLoss: 32.3234 \n",
      "Epoch: 7/15  \tStep: 15900  \tLoss: 33.1212 \n",
      "Epoch: 7/15  \tStep: 15950  \tLoss: 34.4273 \n",
      "Epoch: 7/15  \tStep: 16000  \tLoss: 31.5770 \n",
      "Epoch: 7/15  \tStep: 16050  \tLoss: 36.3870 \n",
      "Epoch: 7/15  \tStep: 16100  \tLoss: 37.7597 \n",
      "Epoch: 7/15  \tStep: 16150  \tLoss: 37.7584 \n",
      "Epoch: 7/15  \tStep: 16200  \tLoss: 33.8620 \n",
      "Epoch: 7/15  \tStep: 16250  \tLoss: 32.2006 \n",
      "Epoch: 7/15  \tStep: 16300  \tLoss: 37.5176 \n",
      "Epoch: 7/15  \tStep: 16350  \tLoss: 36.6044 \n",
      "Epoch: 7/15  \tStep: 16400  \tLoss: 35.6592 \n",
      "Epoch: 7/15  \tStep: 16450  \tLoss: 34.7924 \n",
      "Epoch: 7/15  \tStep: 16500  \tLoss: 39.9451 \n",
      "Epoch: 7/15  \tStep: 16550  \tLoss: 32.5452 \n",
      "Epoch: 7/15  \tStep: 16600  \tLoss: 35.7597 \n",
      "Epoch: 7/15  \tStep: 16650  \tLoss: 38.0619 \n",
      "Epoch: 7/15  \tStep: 16700  \tLoss: 36.3649 \n",
      "Epoch: 7/15  \tStep: 16750  \tLoss: 34.4160 \n",
      "Epoch: 7/15  \tStep: 16800  \tLoss: 36.3304 \n",
      "Epoch: 7/15  \tStep: 16850  \tLoss: 34.6245 \n",
      "Epoch: 7/15  \tStep: 16900  \tLoss: 36.7544 \n",
      "Epoch: 7/15  \tStep: 16950  \tLoss: 39.5682 \n",
      "Epoch: 7/15  \tStep: 17000  \tLoss: 40.6691 \n",
      "Epoch: 7/15  \tStep: 17050  \tLoss: 37.0825 \n",
      "Epoch: 7/15  \tStep: 17100  \tLoss: 33.8319 \n",
      "Epoch: 7/15  \tStep: 17150  \tLoss: 37.1025 \n",
      "Epoch: 7/15  \tStep: 17200  \tLoss: 33.3728 \n",
      "Epoch: 7/15  \tStep: 17250  \tLoss: 32.6179 \n",
      "Epoch: 7/15  \tStep: 17300  \tLoss: 40.9145 \n",
      "Epoch: 7/15  \tStep: 17350  \tLoss: 35.6780 \n",
      "Epoch: 7/15  \tStep: 17400  \tLoss: 37.1254 \n",
      "Epoch: 7/15  \tStep: 17450  \tLoss: 38.0608 \n",
      "Time to train epoch: 1394.4274117946625 s\n",
      "Epoch: 8/15  \tStep: 17500  \tLoss: 38.1626 \n",
      "Epoch: 8/15  \tStep: 17550  \tLoss: 27.4935 \n",
      "Epoch: 8/15  \tStep: 17600  \tLoss: 31.7106 \n",
      "Epoch: 8/15  \tStep: 17650  \tLoss: 34.8720 \n",
      "Epoch: 8/15  \tStep: 17700  \tLoss: 33.2034 \n",
      "Epoch: 8/15  \tStep: 17750  \tLoss: 33.3304 \n",
      "Epoch: 8/15  \tStep: 17800  \tLoss: 34.5221 \n",
      "Epoch: 8/15  \tStep: 17850  \tLoss: 34.3710 \n",
      "Epoch: 8/15  \tStep: 17900  \tLoss: 28.8366 \n",
      "Epoch: 8/15  \tStep: 17950  \tLoss: 33.3878 \n",
      "Epoch: 8/15  \tStep: 18000  \tLoss: 33.6650 \n",
      "Epoch: 8/15  \tStep: 18050  \tLoss: 35.2090 \n",
      "Epoch: 8/15  \tStep: 18100  \tLoss: 35.8092 \n",
      "Epoch: 8/15  \tStep: 18150  \tLoss: 33.2558 \n",
      "Epoch: 8/15  \tStep: 18200  \tLoss: 32.8046 \n",
      "Epoch: 8/15  \tStep: 18250  \tLoss: 33.9961 \n",
      "Epoch: 8/15  \tStep: 18300  \tLoss: 34.6656 \n",
      "Epoch: 8/15  \tStep: 18350  \tLoss: 29.3782 \n",
      "Epoch: 8/15  \tStep: 18400  \tLoss: 32.8575 \n",
      "Epoch: 8/15  \tStep: 18450  \tLoss: 35.5259 \n",
      "Epoch: 8/15  \tStep: 18500  \tLoss: 36.5164 \n",
      "Epoch: 8/15  \tStep: 18550  \tLoss: 33.2551 \n",
      "Epoch: 8/15  \tStep: 18600  \tLoss: 35.4190 \n",
      "Epoch: 8/15  \tStep: 18650  \tLoss: 37.1746 \n",
      "Epoch: 8/15  \tStep: 18700  \tLoss: 31.4805 \n",
      "Epoch: 8/15  \tStep: 18750  \tLoss: 33.6026 \n",
      "Epoch: 8/15  \tStep: 18800  \tLoss: 38.7962 \n",
      "Epoch: 8/15  \tStep: 18850  \tLoss: 33.4530 \n",
      "Epoch: 8/15  \tStep: 18900  \tLoss: 32.8440 \n",
      "Epoch: 8/15  \tStep: 18950  \tLoss: 33.0675 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8/15  \tStep: 19000  \tLoss: 32.0367 \n",
      "Epoch: 8/15  \tStep: 19050  \tLoss: 36.9760 \n",
      "Epoch: 8/15  \tStep: 19100  \tLoss: 31.3112 \n",
      "Epoch: 8/15  \tStep: 19150  \tLoss: 38.8018 \n",
      "Epoch: 8/15  \tStep: 19200  \tLoss: 33.4501 \n",
      "Epoch: 8/15  \tStep: 19250  \tLoss: 33.9018 \n",
      "Epoch: 8/15  \tStep: 19300  \tLoss: 38.3106 \n",
      "Epoch: 8/15  \tStep: 19350  \tLoss: 35.8445 \n",
      "Epoch: 8/15  \tStep: 19400  \tLoss: 34.4174 \n",
      "Epoch: 8/15  \tStep: 19450  \tLoss: 36.9865 \n",
      "Epoch: 8/15  \tStep: 19500  \tLoss: 35.6074 \n",
      "Epoch: 8/15  \tStep: 19550  \tLoss: 35.2710 \n",
      "Epoch: 8/15  \tStep: 19600  \tLoss: 33.1237 \n",
      "Epoch: 8/15  \tStep: 19650  \tLoss: 37.5752 \n",
      "Epoch: 8/15  \tStep: 19700  \tLoss: 36.7445 \n",
      "Epoch: 8/15  \tStep: 19750  \tLoss: 36.8472 \n",
      "Epoch: 8/15  \tStep: 19800  \tLoss: 37.6922 \n",
      "Epoch: 8/15  \tStep: 19850  \tLoss: 32.5437 \n",
      "Epoch: 8/15  \tStep: 19900  \tLoss: 35.2253 \n",
      "Epoch: 8/15  \tStep: 19950  \tLoss: 33.9279 \n",
      "Time to train epoch: 1395.2207427024841 s\n",
      "Epoch: 9/15  \tStep: 20000  \tLoss: 34.8348 \n",
      "Epoch: 9/15  \tStep: 20050  \tLoss: 32.4806 \n",
      "Epoch: 9/15  \tStep: 20100  \tLoss: 24.4278 \n",
      "Epoch: 9/15  \tStep: 20150  \tLoss: 29.9343 \n",
      "Epoch: 9/15  \tStep: 20200  \tLoss: 33.2204 \n",
      "Epoch: 9/15  \tStep: 20250  \tLoss: 32.6381 \n",
      "Epoch: 9/15  \tStep: 20300  \tLoss: 31.4737 \n",
      "Epoch: 9/15  \tStep: 20350  \tLoss: 33.4586 \n",
      "Epoch: 9/15  \tStep: 20400  \tLoss: 30.4289 \n",
      "Epoch: 9/15  \tStep: 20450  \tLoss: 32.7224 \n",
      "Epoch: 9/15  \tStep: 20500  \tLoss: 30.8082 \n",
      "Epoch: 9/15  \tStep: 20550  \tLoss: 31.7961 \n",
      "Epoch: 9/15  \tStep: 20600  \tLoss: 32.2233 \n",
      "Epoch: 9/15  \tStep: 20650  \tLoss: 29.6262 \n",
      "Epoch: 9/15  \tStep: 20700  \tLoss: 31.6511 \n",
      "Epoch: 9/15  \tStep: 20750  \tLoss: 30.3263 \n",
      "Epoch: 9/15  \tStep: 20800  \tLoss: 31.8142 \n",
      "Epoch: 9/15  \tStep: 20850  \tLoss: 32.6881 \n",
      "Epoch: 9/15  \tStep: 20900  \tLoss: 33.1550 \n",
      "Epoch: 9/15  \tStep: 20950  \tLoss: 29.3122 \n",
      "Epoch: 9/15  \tStep: 21000  \tLoss: 29.9492 \n",
      "Epoch: 9/15  \tStep: 21050  \tLoss: 36.1098 \n",
      "Epoch: 9/15  \tStep: 21100  \tLoss: 32.8509 \n",
      "Epoch: 9/15  \tStep: 21150  \tLoss: 32.9018 \n",
      "Epoch: 9/15  \tStep: 21200  \tLoss: 31.7498 \n",
      "Epoch: 9/15  \tStep: 21250  \tLoss: 30.7400 \n",
      "Epoch: 9/15  \tStep: 21300  \tLoss: 30.2240 \n",
      "Epoch: 9/15  \tStep: 21350  \tLoss: 30.7221 \n",
      "Epoch: 9/15  \tStep: 21400  \tLoss: 33.4805 \n",
      "Epoch: 9/15  \tStep: 21450  \tLoss: 31.6679 \n",
      "Epoch: 9/15  \tStep: 21500  \tLoss: 29.5098 \n",
      "Epoch: 9/15  \tStep: 21550  \tLoss: 30.4235 \n",
      "Epoch: 9/15  \tStep: 21600  \tLoss: 30.6533 \n",
      "Epoch: 9/15  \tStep: 21650  \tLoss: 29.5466 \n",
      "Epoch: 9/15  \tStep: 21700  \tLoss: 31.5583 \n",
      "Epoch: 9/15  \tStep: 21750  \tLoss: 33.3129 \n",
      "Epoch: 9/15  \tStep: 21800  \tLoss: 31.6964 \n",
      "Epoch: 9/15  \tStep: 21850  \tLoss: 36.4422 \n",
      "Epoch: 9/15  \tStep: 21900  \tLoss: 35.9258 \n",
      "Epoch: 9/15  \tStep: 21950  \tLoss: 39.4901 \n",
      "Epoch: 9/15  \tStep: 22000  \tLoss: 33.9806 \n",
      "Epoch: 9/15  \tStep: 22050  \tLoss: 36.2622 \n",
      "Epoch: 9/15  \tStep: 22100  \tLoss: 34.4724 \n",
      "Epoch: 9/15  \tStep: 22150  \tLoss: 37.1060 \n",
      "Epoch: 9/15  \tStep: 22200  \tLoss: 35.0539 \n",
      "Epoch: 9/15  \tStep: 22250  \tLoss: 30.7744 \n",
      "Epoch: 9/15  \tStep: 22300  \tLoss: 36.0371 \n",
      "Epoch: 9/15  \tStep: 22350  \tLoss: 33.5986 \n",
      "Epoch: 9/15  \tStep: 22400  \tLoss: 33.2515 \n",
      "Epoch: 9/15  \tStep: 22450  \tLoss: 38.7176 \n",
      "Time to train epoch: 1395.963205575943 s\n",
      "Epoch: 10/15  \tStep: 22500  \tLoss: 28.8901 \n",
      "Epoch: 10/15  \tStep: 22550  \tLoss: 30.7754 \n",
      "Epoch: 10/15  \tStep: 22600  \tLoss: 32.1351 \n",
      "Epoch: 10/15  \tStep: 22650  \tLoss: 30.1881 \n",
      "Epoch: 10/15  \tStep: 22700  \tLoss: 29.7652 \n",
      "Epoch: 10/15  \tStep: 22750  \tLoss: 28.1270 \n",
      "Epoch: 10/15  \tStep: 22800  \tLoss: 30.9793 \n",
      "Epoch: 10/15  \tStep: 22850  \tLoss: 25.8568 \n",
      "Epoch: 10/15  \tStep: 22900  \tLoss: 27.6449 \n",
      "Epoch: 10/15  \tStep: 22950  \tLoss: 25.9527 \n",
      "Epoch: 10/15  \tStep: 23000  \tLoss: 32.1576 \n",
      "Epoch: 10/15  \tStep: 23050  \tLoss: 28.6480 \n",
      "Epoch: 10/15  \tStep: 23100  \tLoss: 26.6636 \n",
      "Epoch: 10/15  \tStep: 23150  \tLoss: 25.5227 \n",
      "Epoch: 10/15  \tStep: 23200  \tLoss: 29.0539 \n",
      "Epoch: 10/15  \tStep: 23250  \tLoss: 28.3834 \n",
      "Epoch: 10/15  \tStep: 23300  \tLoss: 35.1066 \n",
      "Epoch: 10/15  \tStep: 23350  \tLoss: 29.1947 \n",
      "Epoch: 10/15  \tStep: 23400  \tLoss: 29.7439 \n",
      "Epoch: 10/15  \tStep: 23450  \tLoss: 29.7443 \n",
      "Epoch: 10/15  \tStep: 23500  \tLoss: 31.0666 \n",
      "Epoch: 10/15  \tStep: 23550  \tLoss: 28.9842 \n",
      "Epoch: 10/15  \tStep: 23600  \tLoss: 30.7941 \n",
      "Epoch: 10/15  \tStep: 23650  \tLoss: 29.3686 \n",
      "Epoch: 10/15  \tStep: 23700  \tLoss: 36.3448 \n",
      "Epoch: 10/15  \tStep: 23750  \tLoss: 31.0845 \n",
      "Epoch: 10/15  \tStep: 23800  \tLoss: 33.5855 \n",
      "Epoch: 10/15  \tStep: 23850  \tLoss: 31.3108 \n",
      "Epoch: 10/15  \tStep: 23900  \tLoss: 33.0431 \n",
      "Epoch: 10/15  \tStep: 23950  \tLoss: 33.3370 \n",
      "Epoch: 10/15  \tStep: 24000  \tLoss: 30.8641 \n",
      "Epoch: 10/15  \tStep: 24050  \tLoss: 32.9998 \n",
      "Epoch: 10/15  \tStep: 24100  \tLoss: 34.1334 \n",
      "Epoch: 10/15  \tStep: 24150  \tLoss: 32.3061 \n",
      "Epoch: 10/15  \tStep: 24200  \tLoss: 30.4422 \n",
      "Epoch: 10/15  \tStep: 24250  \tLoss: 28.8745 \n",
      "Epoch: 10/15  \tStep: 24300  \tLoss: 31.8183 \n",
      "Epoch: 10/15  \tStep: 24350  \tLoss: 32.1608 \n",
      "Epoch: 10/15  \tStep: 24400  \tLoss: 33.8553 \n",
      "Epoch: 10/15  \tStep: 24450  \tLoss: 37.4647 \n",
      "Epoch: 10/15  \tStep: 24500  \tLoss: 27.3689 \n",
      "Epoch: 10/15  \tStep: 24550  \tLoss: 32.8399 \n",
      "Epoch: 10/15  \tStep: 24600  \tLoss: 34.2321 \n",
      "Epoch: 10/15  \tStep: 24650  \tLoss: 34.0553 \n",
      "Epoch: 10/15  \tStep: 24700  \tLoss: 28.6881 \n",
      "Epoch: 10/15  \tStep: 24750  \tLoss: 29.8948 \n",
      "Epoch: 10/15  \tStep: 24800  \tLoss: 34.5022 \n",
      "Epoch: 10/15  \tStep: 24850  \tLoss: 31.2414 \n",
      "Epoch: 10/15  \tStep: 24900  \tLoss: 33.6408 \n",
      "Epoch: 10/15  \tStep: 24950  \tLoss: 33.8357 \n",
      "Time to train epoch: 1393.1510875225067 s\n",
      "Epoch: 11/15  \tStep: 25000  \tLoss: 25.5173 \n",
      "Epoch: 11/15  \tStep: 25050  \tLoss: 29.2379 \n",
      "Epoch: 11/15  \tStep: 25100  \tLoss: 28.5955 \n",
      "Epoch: 11/15  \tStep: 25150  \tLoss: 32.5226 \n",
      "Epoch: 11/15  \tStep: 25200  \tLoss: 30.2937 \n",
      "Epoch: 11/15  \tStep: 25250  \tLoss: 27.9219 \n",
      "Epoch: 11/15  \tStep: 25300  \tLoss: 26.5874 \n",
      "Epoch: 11/15  \tStep: 25350  \tLoss: 28.1939 \n",
      "Epoch: 11/15  \tStep: 25400  \tLoss: 23.0917 \n",
      "Epoch: 11/15  \tStep: 25450  \tLoss: 27.1462 \n",
      "Epoch: 11/15  \tStep: 25500  \tLoss: 30.5935 \n",
      "Epoch: 11/15  \tStep: 25550  \tLoss: 29.4175 \n",
      "Epoch: 11/15  \tStep: 25600  \tLoss: 24.1106 \n",
      "Epoch: 11/15  \tStep: 25650  \tLoss: 30.8532 \n",
      "Epoch: 11/15  \tStep: 25700  \tLoss: 29.6059 \n",
      "Epoch: 11/15  \tStep: 25750  \tLoss: 28.7734 \n",
      "Epoch: 11/15  \tStep: 25800  \tLoss: 28.8399 \n",
      "Epoch: 11/15  \tStep: 25850  \tLoss: 29.8239 \n",
      "Epoch: 11/15  \tStep: 25900  \tLoss: 28.6740 \n",
      "Epoch: 11/15  \tStep: 25950  \tLoss: 27.9146 \n",
      "Epoch: 11/15  \tStep: 26000  \tLoss: 27.6615 \n",
      "Epoch: 11/15  \tStep: 26050  \tLoss: 28.2708 \n",
      "Epoch: 11/15  \tStep: 26100  \tLoss: 31.9622 \n",
      "Epoch: 11/15  \tStep: 26150  \tLoss: 28.8245 \n",
      "Epoch: 11/15  \tStep: 26200  \tLoss: 31.6303 \n",
      "Epoch: 11/15  \tStep: 26250  \tLoss: 30.5622 \n",
      "Epoch: 11/15  \tStep: 26300  \tLoss: 26.5745 \n",
      "Epoch: 11/15  \tStep: 26350  \tLoss: 30.8710 \n",
      "Epoch: 11/15  \tStep: 26400  \tLoss: 28.6407 \n",
      "Epoch: 11/15  \tStep: 26450  \tLoss: 26.7193 \n",
      "Epoch: 11/15  \tStep: 26500  \tLoss: 32.6489 \n",
      "Epoch: 11/15  \tStep: 26550  \tLoss: 33.6721 \n",
      "Epoch: 11/15  \tStep: 26600  \tLoss: 28.3066 \n",
      "Epoch: 11/15  \tStep: 26650  \tLoss: 34.9989 \n",
      "Epoch: 11/15  \tStep: 26700  \tLoss: 31.7010 \n",
      "Epoch: 11/15  \tStep: 26750  \tLoss: 33.5194 \n",
      "Epoch: 11/15  \tStep: 26800  \tLoss: 32.1451 \n",
      "Epoch: 11/15  \tStep: 26850  \tLoss: 31.8747 \n",
      "Epoch: 11/15  \tStep: 26900  \tLoss: 32.4157 \n",
      "Epoch: 11/15  \tStep: 26950  \tLoss: 30.5799 \n",
      "Epoch: 11/15  \tStep: 27000  \tLoss: 31.6817 \n",
      "Epoch: 11/15  \tStep: 27050  \tLoss: 28.6369 \n",
      "Epoch: 11/15  \tStep: 27100  \tLoss: 29.6676 \n",
      "Epoch: 11/15  \tStep: 27150  \tLoss: 30.6696 \n",
      "Epoch: 11/15  \tStep: 27200  \tLoss: 29.8842 \n",
      "Epoch: 11/15  \tStep: 27250  \tLoss: 31.9356 \n",
      "Epoch: 11/15  \tStep: 27300  \tLoss: 30.9800 \n",
      "Epoch: 11/15  \tStep: 27350  \tLoss: 33.0437 \n",
      "Epoch: 11/15  \tStep: 27400  \tLoss: 31.6808 \n",
      "Epoch: 11/15  \tStep: 27450  \tLoss: 31.7012 \n",
      "Time to train epoch: 1391.4950563907623 s\n",
      "Epoch: 12/15  \tStep: 27500  \tLoss: 24.4389 \n",
      "Epoch: 12/15  \tStep: 27550  \tLoss: 25.3340 \n",
      "Epoch: 12/15  \tStep: 27600  \tLoss: 26.5772 \n",
      "Epoch: 12/15  \tStep: 27650  \tLoss: 27.9844 \n",
      "Epoch: 12/15  \tStep: 27700  \tLoss: 26.5997 \n",
      "Epoch: 12/15  \tStep: 27750  \tLoss: 26.1067 \n",
      "Epoch: 12/15  \tStep: 27800  \tLoss: 27.0012 \n",
      "Epoch: 12/15  \tStep: 27850  \tLoss: 22.4621 \n",
      "Epoch: 12/15  \tStep: 27900  \tLoss: 26.7751 \n",
      "Epoch: 12/15  \tStep: 27950  \tLoss: 29.7954 \n",
      "Epoch: 12/15  \tStep: 28000  \tLoss: 27.5117 \n",
      "Epoch: 12/15  \tStep: 28050  \tLoss: 25.5165 \n",
      "Epoch: 12/15  \tStep: 28100  \tLoss: 28.4637 \n",
      "Epoch: 12/15  \tStep: 28150  \tLoss: 25.6688 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12/15  \tStep: 28200  \tLoss: 31.5977 \n",
      "Epoch: 12/15  \tStep: 28250  \tLoss: 29.6908 \n",
      "Epoch: 12/15  \tStep: 28300  \tLoss: 25.0247 \n",
      "Epoch: 12/15  \tStep: 28350  \tLoss: 30.5297 \n",
      "Epoch: 12/15  \tStep: 28400  \tLoss: 27.4503 \n",
      "Epoch: 12/15  \tStep: 28450  \tLoss: 29.7671 \n",
      "Epoch: 12/15  \tStep: 28500  \tLoss: 29.3487 \n",
      "Epoch: 12/15  \tStep: 28550  \tLoss: 30.1749 \n",
      "Epoch: 12/15  \tStep: 28600  \tLoss: 31.2096 \n",
      "Epoch: 12/15  \tStep: 28650  \tLoss: 30.9037 \n",
      "Epoch: 12/15  \tStep: 28700  \tLoss: 28.4752 \n",
      "Epoch: 12/15  \tStep: 28750  \tLoss: 27.5750 \n",
      "Epoch: 12/15  \tStep: 28800  \tLoss: 28.3155 \n",
      "Epoch: 12/15  \tStep: 28850  \tLoss: 25.8614 \n",
      "Epoch: 12/15  \tStep: 28900  \tLoss: 28.5366 \n",
      "Epoch: 12/15  \tStep: 28950  \tLoss: 31.2969 \n",
      "Epoch: 12/15  \tStep: 29000  \tLoss: 24.9656 \n",
      "Epoch: 12/15  \tStep: 29050  \tLoss: 30.2415 \n",
      "Epoch: 12/15  \tStep: 29100  \tLoss: 27.5489 \n",
      "Epoch: 12/15  \tStep: 29150  \tLoss: 32.7421 \n",
      "Epoch: 12/15  \tStep: 29200  \tLoss: 32.1441 \n",
      "Epoch: 12/15  \tStep: 29250  \tLoss: 29.9636 \n",
      "Epoch: 12/15  \tStep: 29300  \tLoss: 31.7101 \n",
      "Epoch: 12/15  \tStep: 29350  \tLoss: 32.4742 \n",
      "Epoch: 12/15  \tStep: 29400  \tLoss: 28.9952 \n",
      "Epoch: 12/15  \tStep: 29450  \tLoss: 26.5773 \n",
      "Epoch: 12/15  \tStep: 29500  \tLoss: 30.4195 \n",
      "Epoch: 12/15  \tStep: 29550  \tLoss: 28.8280 \n",
      "Epoch: 12/15  \tStep: 29600  \tLoss: 32.1835 \n",
      "Epoch: 12/15  \tStep: 29650  \tLoss: 29.6699 \n",
      "Epoch: 12/15  \tStep: 29700  \tLoss: 30.0111 \n",
      "Epoch: 12/15  \tStep: 29750  \tLoss: 32.2644 \n",
      "Epoch: 12/15  \tStep: 29800  \tLoss: 30.5252 \n",
      "Epoch: 12/15  \tStep: 29850  \tLoss: 32.3342 \n",
      "Epoch: 12/15  \tStep: 29900  \tLoss: 28.8950 \n",
      "Epoch: 12/15  \tStep: 29950  \tLoss: 26.8491 \n",
      "Time to train epoch: 1387.2016594409943 s\n",
      "Epoch: 13/15  \tStep: 30000  \tLoss: 24.1520 \n",
      "Epoch: 13/15  \tStep: 30050  \tLoss: 23.7706 \n",
      "Epoch: 13/15  \tStep: 30100  \tLoss: 26.0249 \n",
      "Epoch: 13/15  \tStep: 30150  \tLoss: 27.1397 \n",
      "Epoch: 13/15  \tStep: 30200  \tLoss: 25.6632 \n",
      "Epoch: 13/15  \tStep: 30250  \tLoss: 27.0008 \n",
      "Epoch: 13/15  \tStep: 30300  \tLoss: 28.9002 \n",
      "Epoch: 13/15  \tStep: 30350  \tLoss: 31.1014 \n",
      "Epoch: 13/15  \tStep: 30400  \tLoss: 26.7170 \n",
      "Epoch: 13/15  \tStep: 30450  \tLoss: 25.5365 \n",
      "Epoch: 13/15  \tStep: 30500  \tLoss: 26.3773 \n",
      "Epoch: 13/15  \tStep: 30550  \tLoss: 28.0017 \n",
      "Epoch: 13/15  \tStep: 30600  \tLoss: 24.7525 \n",
      "Epoch: 13/15  \tStep: 30650  \tLoss: 27.5579 \n",
      "Epoch: 13/15  \tStep: 30700  \tLoss: 26.5033 \n",
      "Epoch: 13/15  \tStep: 30750  \tLoss: 26.7388 \n",
      "Epoch: 13/15  \tStep: 30800  \tLoss: 26.6237 \n",
      "Epoch: 13/15  \tStep: 30850  \tLoss: 32.2354 \n",
      "Epoch: 13/15  \tStep: 30900  \tLoss: 25.1013 \n",
      "Epoch: 13/15  \tStep: 30950  \tLoss: 28.9391 \n",
      "Epoch: 13/15  \tStep: 31000  \tLoss: 25.7231 \n",
      "Epoch: 13/15  \tStep: 31050  \tLoss: 28.3832 \n",
      "Epoch: 13/15  \tStep: 31100  \tLoss: 26.9401 \n",
      "Epoch: 13/15  \tStep: 31150  \tLoss: 27.0074 \n",
      "Epoch: 13/15  \tStep: 31200  \tLoss: 30.6952 \n",
      "Epoch: 13/15  \tStep: 31250  \tLoss: 27.8703 \n",
      "Epoch: 13/15  \tStep: 31300  \tLoss: 29.7411 \n",
      "Epoch: 13/15  \tStep: 31350  \tLoss: 28.8311 \n",
      "Epoch: 13/15  \tStep: 31400  \tLoss: 29.6168 \n",
      "Epoch: 13/15  \tStep: 31450  \tLoss: 24.8834 \n",
      "Epoch: 13/15  \tStep: 31500  \tLoss: 28.4559 \n",
      "Epoch: 13/15  \tStep: 31550  \tLoss: 28.3179 \n",
      "Epoch: 13/15  \tStep: 31600  \tLoss: 29.3643 \n",
      "Epoch: 13/15  \tStep: 31650  \tLoss: 31.2025 \n",
      "Epoch: 13/15  \tStep: 31700  \tLoss: 26.3611 \n",
      "Epoch: 13/15  \tStep: 31750  \tLoss: 28.5163 \n",
      "Epoch: 13/15  \tStep: 31800  \tLoss: 29.5166 \n",
      "Epoch: 13/15  \tStep: 31850  \tLoss: 29.6965 \n",
      "Epoch: 13/15  \tStep: 31900  \tLoss: 29.6522 \n",
      "Epoch: 13/15  \tStep: 31950  \tLoss: 30.0586 \n",
      "Epoch: 13/15  \tStep: 32000  \tLoss: 31.5963 \n",
      "Epoch: 13/15  \tStep: 32050  \tLoss: 34.4759 \n",
      "Epoch: 13/15  \tStep: 32100  \tLoss: 26.6089 \n",
      "Epoch: 13/15  \tStep: 32150  \tLoss: 30.2800 \n",
      "Epoch: 13/15  \tStep: 32200  \tLoss: 29.5127 \n",
      "Epoch: 13/15  \tStep: 32250  \tLoss: 29.1727 \n",
      "Epoch: 13/15  \tStep: 32300  \tLoss: 28.7551 \n",
      "Epoch: 13/15  \tStep: 32350  \tLoss: 30.7539 \n",
      "Epoch: 13/15  \tStep: 32400  \tLoss: 31.5837 \n",
      "Epoch: 13/15  \tStep: 32450  \tLoss: 29.9519 \n",
      "Time to train epoch: 1388.0199773311615 s\n",
      "Epoch: 14/15  \tStep: 32500  \tLoss: 27.3129 \n",
      "Epoch: 14/15  \tStep: 32550  \tLoss: 24.4328 \n",
      "Epoch: 14/15  \tStep: 32600  \tLoss: 22.8349 \n",
      "Epoch: 14/15  \tStep: 32650  \tLoss: 25.5630 \n",
      "Epoch: 14/15  \tStep: 32700  \tLoss: 23.7710 \n",
      "Epoch: 14/15  \tStep: 32750  \tLoss: 24.9832 \n",
      "Epoch: 14/15  \tStep: 32800  \tLoss: 25.0178 \n",
      "Epoch: 14/15  \tStep: 32850  \tLoss: 22.2417 \n",
      "Epoch: 14/15  \tStep: 32900  \tLoss: 26.5171 \n",
      "Epoch: 14/15  \tStep: 32950  \tLoss: 25.6803 \n",
      "Epoch: 14/15  \tStep: 33000  \tLoss: 27.0876 \n",
      "Epoch: 14/15  \tStep: 33050  \tLoss: 30.4562 \n",
      "Epoch: 14/15  \tStep: 33100  \tLoss: 25.4819 \n",
      "Epoch: 14/15  \tStep: 33150  \tLoss: 25.0393 \n",
      "Epoch: 14/15  \tStep: 33200  \tLoss: 25.5542 \n",
      "Epoch: 14/15  \tStep: 33250  \tLoss: 25.3483 \n",
      "Epoch: 14/15  \tStep: 33300  \tLoss: 23.8099 \n",
      "Epoch: 14/15  \tStep: 33350  \tLoss: 29.5425 \n",
      "Epoch: 14/15  \tStep: 33400  \tLoss: 26.9999 \n",
      "Epoch: 14/15  \tStep: 33450  \tLoss: 26.0171 \n",
      "Epoch: 14/15  \tStep: 33500  \tLoss: 26.4003 \n",
      "Epoch: 14/15  \tStep: 33550  \tLoss: 27.9764 \n",
      "Epoch: 14/15  \tStep: 33600  \tLoss: 26.5306 \n",
      "Epoch: 14/15  \tStep: 33650  \tLoss: 27.9565 \n",
      "Epoch: 14/15  \tStep: 33700  \tLoss: 28.7853 \n",
      "Epoch: 14/15  \tStep: 33750  \tLoss: 23.9668 \n",
      "Epoch: 14/15  \tStep: 33800  \tLoss: 30.2396 \n",
      "Epoch: 14/15  \tStep: 33850  \tLoss: 25.3840 \n",
      "Epoch: 14/15  \tStep: 33900  \tLoss: 26.9128 \n",
      "Epoch: 14/15  \tStep: 33950  \tLoss: 26.4973 \n",
      "Epoch: 14/15  \tStep: 34000  \tLoss: 27.9762 \n",
      "Epoch: 14/15  \tStep: 34050  \tLoss: 26.0676 \n",
      "Epoch: 14/15  \tStep: 34100  \tLoss: 28.9704 \n",
      "Epoch: 14/15  \tStep: 34150  \tLoss: 26.5459 \n",
      "Epoch: 14/15  \tStep: 34200  \tLoss: 29.5589 \n",
      "Epoch: 14/15  \tStep: 34250  \tLoss: 28.8704 \n",
      "Epoch: 14/15  \tStep: 34300  \tLoss: 27.4922 \n",
      "Epoch: 14/15  \tStep: 34350  \tLoss: 27.8066 \n",
      "Epoch: 14/15  \tStep: 34400  \tLoss: 28.5940 \n",
      "Epoch: 14/15  \tStep: 34450  \tLoss: 29.4860 \n",
      "Epoch: 14/15  \tStep: 34500  \tLoss: 32.4860 \n",
      "Epoch: 14/15  \tStep: 34550  \tLoss: 26.9829 \n",
      "Epoch: 14/15  \tStep: 34600  \tLoss: 27.6572 \n",
      "Epoch: 14/15  \tStep: 34650  \tLoss: 26.4364 \n",
      "Epoch: 14/15  \tStep: 34700  \tLoss: 25.1614 \n",
      "Epoch: 14/15  \tStep: 34750  \tLoss: 28.7564 \n",
      "Epoch: 14/15  \tStep: 34800  \tLoss: 30.6158 \n",
      "Epoch: 14/15  \tStep: 34850  \tLoss: 26.8947 \n",
      "Epoch: 14/15  \tStep: 34900  \tLoss: 30.0908 \n",
      "Epoch: 14/15  \tStep: 34950  \tLoss: 32.4512 \n",
      "Time to train epoch: 1390.87633228302 s\n",
      "Epoch: 15/15  \tStep: 35000  \tLoss: 24.9314 \n",
      "Epoch: 15/15  \tStep: 35050  \tLoss: 22.7728 \n",
      "Epoch: 15/15  \tStep: 35100  \tLoss: 23.7340 \n",
      "Epoch: 15/15  \tStep: 35150  \tLoss: 29.0916 \n",
      "Epoch: 15/15  \tStep: 35200  \tLoss: 27.2470 \n",
      "Epoch: 15/15  \tStep: 35250  \tLoss: 23.8407 \n",
      "Epoch: 15/15  \tStep: 35300  \tLoss: 28.1953 \n",
      "Epoch: 15/15  \tStep: 35350  \tLoss: 26.5202 \n",
      "Epoch: 15/15  \tStep: 35400  \tLoss: 26.5909 \n",
      "Epoch: 15/15  \tStep: 35450  \tLoss: 23.7264 \n",
      "Epoch: 15/15  \tStep: 35500  \tLoss: 27.2295 \n",
      "Epoch: 15/15  \tStep: 35550  \tLoss: 26.8149 \n",
      "Epoch: 15/15  \tStep: 35600  \tLoss: 25.7042 \n",
      "Epoch: 15/15  \tStep: 35650  \tLoss: 24.2477 \n",
      "Epoch: 15/15  \tStep: 35700  \tLoss: 25.3375 \n",
      "Epoch: 15/15  \tStep: 35750  \tLoss: 29.3553 \n",
      "Epoch: 15/15  \tStep: 35800  \tLoss: 26.9456 \n",
      "Epoch: 15/15  \tStep: 35850  \tLoss: 26.1833 \n",
      "Epoch: 15/15  \tStep: 35900  \tLoss: 25.0712 \n",
      "Epoch: 15/15  \tStep: 35950  \tLoss: 24.7301 \n",
      "Epoch: 15/15  \tStep: 36000  \tLoss: 24.9450 \n",
      "Epoch: 15/15  \tStep: 36050  \tLoss: 25.6033 \n",
      "Epoch: 15/15  \tStep: 36100  \tLoss: 23.4097 \n",
      "Epoch: 15/15  \tStep: 36150  \tLoss: 22.6849 \n",
      "Epoch: 15/15  \tStep: 36200  \tLoss: 24.9501 \n",
      "Epoch: 15/15  \tStep: 36250  \tLoss: 27.2635 \n",
      "Epoch: 15/15  \tStep: 36300  \tLoss: 27.8164 \n",
      "Epoch: 15/15  \tStep: 36350  \tLoss: 25.3506 \n",
      "Epoch: 15/15  \tStep: 36400  \tLoss: 30.7506 \n",
      "Epoch: 15/15  \tStep: 36450  \tLoss: 25.5982 \n",
      "Epoch: 15/15  \tStep: 36500  \tLoss: 24.0611 \n",
      "Epoch: 15/15  \tStep: 36550  \tLoss: 30.8325 \n",
      "Epoch: 15/15  \tStep: 36600  \tLoss: 27.7024 \n",
      "Epoch: 15/15  \tStep: 36650  \tLoss: 26.1731 \n",
      "Epoch: 15/15  \tStep: 36700  \tLoss: 26.6230 \n",
      "Epoch: 15/15  \tStep: 36750  \tLoss: 26.9729 \n",
      "Epoch: 15/15  \tStep: 36800  \tLoss: 28.4669 \n",
      "Epoch: 15/15  \tStep: 36850  \tLoss: 30.5442 \n",
      "Epoch: 15/15  \tStep: 36900  \tLoss: 30.0399 \n",
      "Epoch: 15/15  \tStep: 36950  \tLoss: 26.8399 \n",
      "Epoch: 15/15  \tStep: 37000  \tLoss: 29.5042 \n",
      "Epoch: 15/15  \tStep: 37050  \tLoss: 25.5962 \n",
      "Epoch: 15/15  \tStep: 37100  \tLoss: 25.9259 \n",
      "Epoch: 15/15  \tStep: 37150  \tLoss: 27.5241 \n",
      "Epoch: 15/15  \tStep: 37200  \tLoss: 27.9428 \n",
      "Epoch: 15/15  \tStep: 37250  \tLoss: 27.1771 \n",
      "Epoch: 15/15  \tStep: 37300  \tLoss: 23.5102 \n",
      "Epoch: 15/15  \tStep: 37350  \tLoss: 26.2686 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15/15  \tStep: 37400  \tLoss: 26.6001 \n",
      "Epoch: 15/15  \tStep: 37450  \tLoss: 25.4084 \n",
      "Time to train epoch: 1387.3100702762604 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "epochs = 15\n",
    "learning_rate = 0.001\n",
    "# initialize model\n",
    "model = Seq2Seq(kernel_size,filter_size,dropout,num_hidden,enc_layers,weight_embedding,\n",
    "                vocab_size,dec_layers,embed_dims,device).to(device)\n",
    "# Adam Optimizer\n",
    "optimizer = torch.optim.Adam(model.parameters(),lr=learning_rate)\n",
    "# cross entropy loss function with the padding ignored\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
    "counter = 0\n",
    "loss = []\n",
    "beam_width = 10 # beam search width is 10\n",
    "for e in range(epochs):\n",
    "    start_time = time.time() # start timer\n",
    "    # shuffle the data at the beginning of epoch\n",
    "    p = np.random.permutation(train_summary.shape[0])\n",
    "    train_text = train_text[p,:]\n",
    "    train_summary = train_summary[p,:]\n",
    "    # initialize encoder initial hidden state\n",
    "    e_hidden = model.encoder.init_hidden(batch_size)\n",
    "    \n",
    "    for x,y in get_batches(train_text,train_summary,batch_size):\n",
    "        model.train()\n",
    "        # convert inputs to PyTorch tensor\n",
    "        x = torch.from_numpy(x).to(device)\n",
    "        y = torch.from_numpy(y).to(device)\n",
    "        \n",
    "        e_hidden = tuple([each.data for each in e_hidden])\n",
    "        optimizer.zero_grad() # zero the gradients of the model\n",
    "        # train the model\n",
    "        l,prediction = model(x,y,e_hidden,criterion,batch_size)\n",
    "        loss.append(l.item()) # add loss function\n",
    "        l.backward()\n",
    "        nn.utils.clip_grad_norm_(model.parameters(),2) # gradient clip with norm = 2\n",
    "        optimizer.step() # step the optimizer to apply the gradient\n",
    "        \n",
    "        if counter%50 == 0:\n",
    "            print(\"Epoch: {}/{} \".format(e+1, epochs),\n",
    "                      \"\\tStep: {} \".format(counter),\n",
    "                      \"\\tLoss: {:.4f} \".format(l.item()))\n",
    "        counter += 1\n",
    "    print(\"Time to train epoch: {0} s\".format(time.time()-start_time))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0llxEThJe1hD"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beam Test: 881.8145661354065 s\n",
      "Val Beam Search Loss: 21.2239\n"
     ]
    }
   ],
   "source": [
    "# beam search decoder\n",
    "with torch.no_grad():\n",
    "    beam_time = time.time() # start timer\n",
    "    loss_beam = [] \n",
    "    beam_predict = [] # save beam search decoder outputs\n",
    "    summary_validation = [] # save validation target summaries\n",
    "    text_validation = [] # save validation input text\n",
    "    model.eval()\n",
    "    # initialize the encoder hidden state\n",
    "    val_hidden = model.encoder.init_hidden(batch_size)\n",
    "    for x_val, y_val in get_batches(val_text,val_summary,batch_size):\n",
    "        # convert data to PyTorch tensor\n",
    "        x_val = torch.from_numpy(x_val).to(device)\n",
    "        y_val = torch.from_numpy(y_val).to(device)\n",
    "        val_hidden = tuple([each.data for each in val_hidden])\n",
    "        # run the beam search decoder\n",
    "        val_loss, prediction = model.inference_beam(x,y,val_hidden,criterion,beam_width,batch_size=32)\n",
    "        loss_beam.append(val_loss.item())\n",
    "        beam_predict.append(prediction)\n",
    "        summary_validation.append(y_val)\n",
    "        text_validation.append(x_val)\n",
    "    model.train()\n",
    "    print(\"Beam Test: {0} s\".format(time.time()-beam_time))\n",
    "    print(\"Val Beam Search Loss: {:.4f}\".format(np.mean(loss_beam)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "553D7W2Eb55Q"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Greedy Test: 8.562105417251587 s\n",
      "Val Greedy Loss: 39.8127\n"
     ]
    }
   ],
   "source": [
    "# Run the greedy decoder\n",
    "with torch.no_grad():\n",
    "    greedy_time = time.time() # start timer\n",
    "    loss_greedy = []\n",
    "    greedy_predict = []\n",
    "    model.eval()\n",
    "    # initialize the encoder hidden states\n",
    "    val_hidden = model.encoder.init_hidden(batch_size=32)\n",
    "    for x_val, y_val in get_batches(val_text,val_summary,batch_size=32):\n",
    "        # convert data to PyTorch tensor\n",
    "        x_val = torch.from_numpy(x_val).to(device) \n",
    "        y_val = torch.from_numpy(y_val).to(device)\n",
    "        val_hidden = tuple([each.data for each in val_hidden])\n",
    "        # run the greedy decoder\n",
    "        val_loss, prediction = model.inference_greedy(x,y,val_hidden,criterion,batch_size=32)\n",
    "        loss_greedy.append(val_loss.item())\n",
    "        greedy_predict.append(prediction)\n",
    "        \n",
    "    model.train()\n",
    "    print(\"Greedy Test: {0} s\".format(time.time()-greedy_time))\n",
    "    print(\"Val Greedy Loss: {:.4f}\".format(np.mean(loss_greedy)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GjLJcMWVe1hO"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Saved!\n"
     ]
    }
   ],
   "source": [
    "np.savez('AbSumOut.npz',greedy_predict=greedy_predict,beam_predict=beam_predict, text_validation=text_validation,summary_validation=summary_validation,idx2word=idx2word)\n",
    "print(\"Data Saved!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b3994PRlb55U"
   },
   "outputs": [],
   "source": [
    "# save the results of the beam search inference method\n",
    "beam = []\n",
    "file = open(\"beam.txt\",\"w\")\n",
    "for i in beam_predict:\n",
    "    for p in i:\n",
    "        array = torch.Tensor.cpu(p).numpy()\n",
    "        beam_string = ''\n",
    "        for q in array:\n",
    "            beam_string += idx2word[q] + ' ' \n",
    "        file.write(beam_string+'\\n')\n",
    "        beam.append(beam_string)\n",
    "file.close()\n",
    "\n",
    "# save the validation summaries for the input text used in the beam search decoder\n",
    "summary = []\n",
    "file = open(\"validation_summary.txt\",\"w\")\n",
    "for i in summary_validation:\n",
    "    for p in i:\n",
    "        array = torch.Tensor.cpu(p).numpy()\n",
    "        summary_string = ''\n",
    "        for q in array:\n",
    "            if q != 0:\n",
    "                summary_string += idx2word[q] + ' ' \n",
    "        file.write(summary_string+'\\n')\n",
    "        summary.append(summary_string)\n",
    "file.close()\n",
    "\n",
    "# Save the validation input text as a .txt file\n",
    "text = []\n",
    "file = open(\"validation_text.txt\",\"w\")\n",
    "for i in text_validation:\n",
    "    for p in i:\n",
    "        array = torch.Tensor.cpu(p).numpy()\n",
    "        text_string = ''\n",
    "        for q in array:\n",
    "            if q != 0:\n",
    "                text_string += idx2word[q] + ' ' \n",
    "        file.write(text_string+'\\n')\n",
    "        text.append(text_string)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Bz_F6Vfqb55Y"
   },
   "outputs": [],
   "source": [
    "## Plotting the error function\n",
    "import matplotlib.pyplot as plt\n",
    "import pylab\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0ZgCbqJAb55c",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEACAYAAABCl1qQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmYVNW19/HvogEB4TYKKgZUUFSCoEEb1BhR0RiIqAkRHDBOqNHHMYoDGa650UTjEF+NA3FANBoQ9eYVvY5xCJo40CgaEQ2DEhARh0hEBVTW/WN33VNVXdVV1VTVqar+fZ6nnjrDPnVWH5pavfc+Z29zd0RERFrSLu4ARESk8ilZiIhITkoWIiKSk5KFiIjkpGQhIiI5KVmIiEhOShYiIpKTkoWIiOSkZCEiIjkpWYiISE7t4w6gWHr27Ol9+/aNOwwRkaoyZ86cD9x9s1zlaiZZ9O3bl8bGxrjDEBGpKma2JJ9yaoYSEZGclCxERCQnJQsREclJyUJERHJSshARkZyULEREJCclCxERyalmnrOYOxe6d2/dse3bQ9eu0atbt9T1Ql7dukHHjsX92URE4lYzyeKrr2DVqtYf/+GHxYulvh523hl22SV6DRoEnTsX7xwiIuVk7h53DEUxZEiDP/VU657gXrcOPv0UVq/O/Prkk+z7MpX98svm52jXDnbcMTWB7LILbLklmG3gDy8i0kpmNsfdG3KVq5maRV1d65uhiskdVqyAV15Jfb35JsyfH17Tp0fle/aE734XJk6EwYPji1tEpCU1U7NoaGjwSh4bas0amDeveRL5+OOozKhRcN55sO++qm2ISHnkW7NQsoiROyxYANdfD7fcAp99FrY3NMD558OYMaHGJCJSKvkmC906GyMz2GEHuOYa+Oc/4Ze/DM1SjY0wblzYd8MNURIREYmLkkWF6NEDfv5zWLIkJIhtt4XFi+G002CnneCDD+KOUETaMiWLCtOlC5x6KvzjHzBjBgwYAG+/Db/6VdyRiUhbpmRRoerqYOzYcOeUWejXeOutuKMSkbZKyaLC7bILHH00fPEF/OxncUcjIm2VkkUVuPjiMITIH/8IL78cdzQi0hYpWVSBbbaBM84IyxdcEG8sItI2KVlUiUmTwphTjz8eXiIi5aRkUSV69AgJA0LtYv36eOMRkbZFyaKKnHkm9OkT+i2mTYs7GhFpS5QsqkjnzuEpbwh3Rq1dG288ItJ2KFlUmWOOCU90v/023Hhj3NGISFuhZFFl6urgssvC8iWXbNiETyIi+VKyqEIHHQTDh4fZ/a68Mu5oRKQtULKoQmZw6aVh+Xe/U+1CREpPyaJKffObsN9+IVFcf33rPuPzz2HhwjB/uYhIS5QsqthPfxrer746zCGej8ZGOO442Hln6NYNtt9eTVkikpuSRRUbMQKGDQtzXdxyS+7y770HBxwAt98Of/97VKO4777Sxiki1U/JooqZRbWLK67I/dzFueeGZqsRI+D552HlSmjfHubMUb+HiLRMyaLKjR4NgwbBO+/AHXdkL/fkk3DXXdCpE9x8M+y+O2y2WaiZrF8PzzxTvphFpPooWVS5du3gJz8Jy7/5DXz5ZfMya9eG2fcgTN267bbRvv32C+9PPVXaOEWkuilZ1IBx46B/f1i0KEzFmu6KK8I0rQMGwMSJqftGjAjvShYi0hIlixpQVxfNc/HrX6eOSPv66+FJbwjDg3TsmHrsnnuGbXPnwkcflSdeEak+ShY14phjwoi08+bBL34RJkvaaafwWrsWfvhD2Hff5sd17hwShjv85S/ljlpEqoWSRY3o2BHOOy8sX3wxXHddqFV06QJjxoRnMbJRv4WI5NI+7gAyMbPvAQcBmwPXu/tjMYdUFU48ER59NDygN2JE9BxGetNTuv32C7URJQsRycbcvTwnMpsCjAZWuvugpO0jgWuAOuAWd78sad8mwJXuPiHX5zc0NHhjY2PxA28D1q6F7t1hzZrw7MVmm8UdkYiUi5nNcfeGXOXK2Qw1FRiZvMHM6oDrgVHAQOBIMxuYVORnTfulhDbaCPbaKyw/pjqciGRQtmTh7rOA9PtthgEL3X2xu68DpgOHWvAb4GF3f6lcMbZliX6Lo48OHeWHHRZqGSIiEH8Hd29gadL6sqZtZwAHAIeZ2SnZDjazk82s0cwa33///dJGWuOOPTYkjI03Dk+D33cf3Hln3FGJSKWIO1lYhm3u7te6+27ufoq7T852sLvf5O4N7t6wmRraN0ifPmFIkFWr4Le/DdteeCHa/8c/wgknhGHNC7V8ebgbS3OGi1SvuJPFMmCrpPU+wPKYYhHCA34HHhiWn38+vLvD+efDbbfBtGmp5S+5BL71rZYHIjzqKDjnHDglax1RRCpd3MliNrC9mfUzs47AEcDMmGNq8wYMCHNd/POf8O678NZboWkKYMqUqNxHH4Vk8de/woMPZv+8xMN+U6eWLOSqsWhRmLhKNxJItSlbsjCzacBzwI5mtszMJrj7l8DpwKPAfGCGu88rV0ySWV1deD4DQlNU8oi0f/1rGGcKwii2iaalJ5/M/nlf+1q0/MUXuc//yCMwuanx8b33Qs2mVpx0Ejz3HHznO3FHIlKYct4NdaS7b+nuHdy9j7vf2rT9IXffwd23c/dflSseadnuu4f355+PksXGG4f3224LX+DJEy498UR+X+qzZ0fLL74YhiVJ7gdZuRJGjQqj5P7+99CrF1x0Ucuf+dVXpUsob7yRX4LL17//XbzPEimnuJuhpEIlkkVyzeKXvwzvt98O//M/8Oqr0KMHbLopLFkCixc3/5z160PtICHxlPhXX4VzXHcdXHZZSAyffx51rkPUx3HxxfD226FJLJPDDoNttmk+tez69aGv5YEHCvrR/89998HXvw7f+17rjs/EMt3SIVIN3L0mXrvttptL8axY4Q7uG20U3rt0cV+3zn377cN64vXjH7v/4Adh+fe/b/45772XWv7b33Zftcp94sTU7eB+8cXuQ4Y03578Svfmm9G+l15K3Xf//dG+WbMKvwYHHZT9vK01dGjxP1NkQwCNnsd3rGoWktEWW0DfvlGfxJ57QocOoRYwaFCYP2O33eCss6I5Mf77v+HZZ2HduuhzErWB+vrw/uKLMHw4XHll83O+8ELUkZ5NYt7whIceipb/9a/UfUuWRMvDh8P8+S1/dkvnyqeZ6667YJ99mseRTDULqVZKFpLVHntEy8OHh/cxY+Dvf4cFC6CxMTT/7L9/2Pfoo7D33lEfw+LF4Y4qgIaG0NG9ahW88krYltzxDaE5K9dT4598krq+POlG6wULwhf8yy+HL/fPPkstO2dO5s9ctSo0c6VLThb9+0e3Bzc2hqlp0x19NMyaBZdfnj3+5GRx2WWaQ0SqR9UnCzM72MxuWtXSjf7SKol+CwhJIJsddoBzz4VvfCOsP/FEuF12u+3guOPCti23TP28MWOa30GV3ueQyQcfpE4dm5xcTjkl1HZ23RVmzoTVq1OPvfvu1PUPPwzzlvfrF1433wx/+EO0PzlZLF4c+i4efBCGDoWTTw5f/McfHxJTctn0TuxPPoH//M/QWZ6cLCZNCg86lsLbb4dzKhlJ0eTTVlUNL/VZFN9zz4W29Q4d3D/9NHf5jz8O5Tt2dD/zzNS+hokT3S+7LFq/4Qb3zz9PLTN4cMv9FeBeX+/ep4/7RRe5n366+8iRmcudeqr7CSc0375ihfvateH8gwZlPjbR9zJ8eO54wP2119wXLozWJ0xwX7kyui6//GW0b889U4/t3j0qd+ed7rfe2vp/r3XrouWttgqff/jhLR+zapX7+vWtP6dUP/Lss4j9S75YLyWL4vviC/exY91/9rP8j9lxx+hLMPlL8aqr3J96Klr/xz9C+eQyHTuG98QXXT6vjTfOv2whr/Xr3ffaK7+yF12UefuiRe7Ll7vvvHO0ra6ueblLL3V//vlofc2awv+tJk8Oxz70UOp17do1+zGNjaHMMccUfj6pHfkmi6pvhpLSad8eZswIt67ma+jQ8P7xx6nbt9wy7Pva18J7//7Nj010jDfkHFk/kk/TVWt89lnzzvRs/uu/Mm+/665wM8Crr0bbMn3mpEmp/UPJc6hD6OMZOjT0CUEYjsUsdaytxG3GEyakduyvXh36Rq67DrbeOupDgujBxzvugN/9LpppUSQTJQspqkSySNerV3iob/58ePrpqO2+V6/mZYcMiZa33rroIeblk0/yTxbZfPFF6/oMEvWCd98NMRx6aOhUHzkSxo+Hxx8P5U48sfmx774b7mJLNmlSePhx6dLQp/PrX4c+peQkduaZ4Q61TB39uSxZEu6KW7o0d1mpYvlUP6rhpWaoyvC3v0VNIA0N0fL8+ZnLv/GG+4EHpjbL3HFHaLKaNSu1CSfbq2tX9wEDMu/r1i1qGivktd12hR9TrNf++7t///v5ld11V/dzzy3eue++Ozw7k9xH9cEHYd/xx2f+N0z0/ei/YHUiz2aosk2rWmqaVrUyfP45/Md/hDuWbrkl+uv3k0+ga9fMx7z+Ouy0U7T+2GPw7W+H5a23bv4X6x//CJ06hTuqIDzbsP/+4e4fCNPC1teHz7z+eujdW883tEbiq6Ffv6jG4R6a1x57DG69NTRVJq5tXV3qnWpSHfKdVrV9OYKRtqNz5/BMxt/+Fr7AFywICSRbooAwwm2yzTePlpOHCkk44ojoiwzC/OGJcasgjD+1zTati18ihx8eJsBKbpq64QY47bSwPHo0jB1b+OeuXh2arpL/QJDKpz4LKbp77oHXXgtt5/37w+DBLZdPTxbJ81glPw2eYAbt2oV2dgjPeCQni/TPg9CmLoWZMSPMVZIskSig+XMs+TZS7LRT6Ph/7rkNi0/KS8lCim7TTcMDeflKr3X07Bktjx6d/bhEh+zee+dOFldckX88Ennxxez7nn0WrrkmWl+/Pjzl/tprLU+Glbgj689/Lk6MUh5KFhK79kmNoR07hlfC1KnhtWRJmCP8nnuifR06RM1NdXWp29Nl2pbs+98PzVkDBhQafds1ZQqcfXbqtu7dQ02ye/cwL0l6bSNRG5TqU/XJQsN91JbOnVPXe/SAY48NHd1PPhmGI88k/dmEQg0YAO+/H9rk85EYPFGyGzUqNBcm+jXefDM8z5Gg/7LVpeqThbs/4O4n1yeGNZWq1qlT647b0GTxta+FGk6+7e6JuT0kt3vvDTcqrFmTuv2qq+D+++OJSQpX9clCakt6zSJf+dxZc+ed2fclnqDO9w6d5E74ZAcckN/xbU2vXmGAyXTf+15q06JULiULqSitTRa77hrmtliwIHuZ8eOj5eQ+jPPOi4YY2WILeOutluekOPTQMNLu5MlhCI7kPpdp00Jn+gsvtO7nqGXnnpt5u+5Uqw5KFlJRWtsMBaGNPNOYU8kSX1i/+EW0LX3YjL59Qwft2WfDN78Zba+vD0lg+vSw/qMfhXGafvzjqEzPnjBxYkg6ya66qpCfpG159134059ad+xnn4Vh66X0lCykomxIssjH5ZfDvHlhvKSE5Dupkl19Nfz1r9F6+/YwbFjzGDP1cyQ/Md7QEBLPjTe2Pu5al2l+k3xsumloElRneekpWUhFOOOM8J78JV4K7drBwIGpX+a5OrUTgyMmZgRMl6tzff36cN7EyLD5SIww25Zku74tSYy8u2hRcWOR5pQspCJcc02YIvXgg8t/7lxf9jNnhgcAf//7zPs3tBkkuS8lQbfmSqVRspCKYBbmvCinMWNg551zP23eq1fo6+jePfP+Aw8M78nTxiY/Rd5SzeWOO5rfpTVzZmqneVty2mmta47SQJGlp2QhbdZ998Hcudn7LPJ15JFhfojEPBMAm2wSLSfXXI4/Pty5tWABPPMM/PCHYXti/vKf/jSe2lWluOGG0Bx1+eVhYiv38AT/G2+0fJySRem10b9fRIJifMm0axdG2s3nHFOmRMvJd249/ngY9vsHP9jweGrBBReEoendwzDzEJa//BI+/BBuvx1OOCEqn3yNly+HZcvCzQhSPEoWIiUyYwaccw7cdlvusj17wlFHpW4bPz7MHVEsc+aEmfKqxXXXpa5/8UW4Ey1RU3vssWhfu6Q2kt69o/elS1XrKBZNfiRSoZYtC01Sc+c231dXV/i0r+61/8X5zjtRsgBYsaL5My+SKt/Jj9RnIVKh+vSBl19uvv3TT8PDaNtum/3Y/fdP7UNpK5ITBeQ/1pfkVvXJQqPOSlvTpUsYxv3ZZ6NtyTMDuoe5IjKNU/WXv4SJh9oKJYviqfpkoVFnpa3acsvQJn/jjfD3v8O4ceFW3JYMHx7KthVKFsVT9clCpC054ojU9T59wpPh3brB3XdHt+ImJI9blezVV0sTX6XZ0KHrJaJkIVIlLrqo8LujfvtbePjh5vNdDx4cJiN64IHixSe1TclCpMLtskt4P+aY1FtE8zVyZDRfR7Iddmg+x/nVVxf++ZVMNYviUbIQqXCzZ8PKlS3f/bQhrr02Wj77bDj55NT9Tz1VmvOWg/osikfJQqTCdeiQfWa+YkieswMy94tUq0y3HkvrKFmItHHpf33vt1/q+jbbQNeu2Y+v5Hk6EvN+33RTGDBy6dIwZIgUTslCRJpJHlyxQwf46KPsZSu5XyAR249+BIsXw9Zbh5/n7rvjjasaKVmItHEdOzbfll7bSJ6zHEJne0IlJ4tsfRZHHKHpWAulZCHSxg0eHIZZv/jiaFtLCaCuDm65JVpPnsej0hx9dJjWNpPNNgvPqBx8sGbay0dFDyRoZtsCPwXq3f2wlspqIEGR4kkMOGgWJY7EtqFD4cUXo/VVq6AWBlCo4K/CkirqQIJm1t3M7jWzN8xsvpnt2cqgppjZSjN7LcO+kWb2ppktNLMLAdx9sbtPaM25RKT1pk9PfW9J+ki2Dz9c/HjKITGft2SWbzPUNcAj7j4A2AWYn7zTzDY3s25p2/rT3FRgZPpGM6sDrgdGAQOBI81sYJ6xiUiRHX54mD9i3LjCjx05Eq66Cr71Lfj614sfW6l06hR3BJUtZ7Iws/8AhgO3Arj7Onf/OK3YPsD9Ztap6ZiTgGvTyuDus4BM91UMAxY21STWAdOBQwv5QUSkuPKdB9wsDCsC8PTT4f2cc8K0sa+/XpLQJAb51Cy2Bd4HbjOzl83sFjPbOLmAu98DPAJMN7PxwAlAIX+T9AaWJq0vA3qbWQ8zmwwMMbNJmQ7UEOUi8TILAxZ+9RXss0/c0RTXwoWh83/+/Nxla10+yaI9sCtwo7sPAT4FLkwv5O6XA2uAG4FD3H11AXFkmr/L3f1Ddz/F3bdz90szHaghykXKK9FHce65cOqpsHHTn47Zxq068MDyxFUKo0eHfpuBA2HBgrijiVc+yWIZsMzdX2hav5eQPFKY2d7AIOBPwEUFxrEM2CppvQ+wvMDPEJEyuvJKuOGG3OWKOY94uSXfUjtmTHxxVIKcycLdVwBLzWzHpk37AyktkWY2BLiZ0M9wPLCpmV1SQByzge3NrJ+ZdQSOAGYWcLyIlNgVV4T3yy8v7LiePYsfS7kkDw3yzjvxxVEJ8uzC4gzgrqYv8sWEhJCsCzDW3RcBmNmxwHHpH2Jm04B9gZ5mtgy4yN1vdfcvzex04FGgDpji7vNa8fOISIlMnAhnndX8ae5CtGtX2U98Q5hJ8L77mt8S3Fafw0io6IfyCqGH8kQq14oV8K9/hS/hn/882j54cGVN8+rePEkk1NfDx+n3gdaAoj6UJyKyIXr1av7MxbRp8OST8cQjhcu3GUpEZIMl/9WePm9GpWvrd+erZiEiZTNgQPNt2Zp9pLIoWYhI2YwZA5Mnp/ZTLFkS+jIqQa7EtWhRuENq7drQf3HcceFJ9bZAHdwiUhGqoYZx1llwzTVh+ZRTQuKD6r5TSh3cIiJF9sQT0XIiUbQVShYiUnEuvBBmz4Y+fcJ6pTwF/lqzyRXaDjVDiUhFSG6GSnwtff45rFwJ22xT2c1U1fw1qmYoEal6nTuHRCHxU7IQkYowenTL++fNg/6ZplSrANVcs8iXkoWIVIRczUwDB4Yh0SvRc8/FHUHpVX2y0ORHIrUhn7/Ohw6Nlivl2QyASRmnZqstVZ8sNPmRSG0YNSq8tzRv9+67R8vf/35p4ynErFlxR1B6VZ8sRKQ2/OhH8NBD8Oyz2ct07BienF69OjRbVVqz1Nq1sN9+0dwftUS3zopI1Vq7Fjp1St22337w1FPlj2X5cthhh5DIoHo6vXXrrIjUvI02Cs9iJM+Rcffd8cRy4olRogB49NF44igVJQsRqWqdOkH7pMkWNtssnjgWLkxdHzkynjhKRclCRKpeJTT5/OMfcUdQWkoWIlL1KiFZ1DolCxERyUnJQkSqXiXXLD75BC69FN56K+5INoyShYhUvfRkMX58PHFkcv758JOfwLBhcUeyYZQsRKTqpT9rceWV8cSRSWJYkg8+qOwaUC5KFiJS9c44A/bdF6ZMCetbbAE77RRrSEBIDu+/H63fe298sWyo9rmLiIhUtm7dUp/aNguz2u26K7z8cnxx9e6duj5uXEggn34a+jJ69YonrtZQzUJEatavf527zIknlu78777bfNvq1dC1K2y5ZWiaqhZKFiJSs7baKvP2b30rWt511/LEkpA889+8eWHa2EWLyhtDa6gZSkRqVrYJlZ55JtrXrsx/Mn/0UbS8Zk3oX0ls32ST8sZSCNUsRKRm9egRLV98cXjff//UMrlm6CulG26Ilt95J7448lHRNQsz2xb4KVDv7ofFHY+IVJcttoAHHoCePWGPPeC888KcGAA77ghvvgkjRsQX3zPPRMtxJq185F2zMLM6M3vZzB5s7cnMbIqZrTSz1zLsG2lmb5rZQjO7EMDdF7v7hNaeT0Rk9OiQKCAMaZ74Un71VXjvPejXL77Y/vWvaPnQQ+OLIx+FNEOdBczPtMPMNjezbmnb+mcoOhVoNnCvmdUB1wOjgIHAkWY2sIDYREQK0rEjbL451NXFHUlQ6Z3ceSULM+sDHATckqXIPsD9ZtapqfxJwLXphdx9FvBR+nZgGLCwqSaxDpgOVHieFRFpO/KtWfw/4Hxgfaad7n4P8Agw3czGAycA4wqIozewNGl9GdDbzHqY2WRgiJlNynSgmR1sZjetWrWqgNOJiEghciYLMxsNrHT3OS2Vc/fLgTXAjcAh7r66pfLpp8n8kf6hu5/i7tu5+6VZzvuAu59cX19fwOlERCqPOzz2WHhY75ln4NvfrpwH9/KpWewFHGJmbxOah0aY2Z3phcxsb2AQ8CfgogLjWAYkPz7TB1he4GeIiFS16dPhO9+Bb3wDhg+HP/85JIxKkDNZuPskd+/j7n2BI4An3f3o5DJmNgS4mdDPcDywqZldUkAcs4HtzayfmXVsOs/MAo4XEal6Z54Z3pOfuZg7N55Y0hXrobwuwFh3X+Tu64FjgSXphcxsGvAcsKOZLTOzCQDu/iVwOvAo4Y6rGe4+r0ixiYhUhUppcsrEvJoHWE/S0NDgjY2NcYchIlUo2wNxw4bBiy+WN5ZMSvk1bWZz3L0hVzkN9yEibd5ee2Xefv755Y2jkilZiEib9/TTmbf/4AfhKe+4XX11/HN4K1mISJvXvoVR8jbfvHxxZHPOOeEOqTgpWYiIAEcfnbp+1lnxxJHNv/8d7/kretRZEZFySe7kfvFFGDIkvliymTsX3ngD9t47zH8xeHD5zq1kISICjB0Lf/gD7L47DB0adzSZpSewFSuiyZNKTclCRAQ4+GB47TXYbru4I8nf22+XL1moz0JEpMlOO0GnTnFHkb+PP4Ynn4T1GYd4LS4lCxGRKjVyZJgm9s5mo/UVn5KFiEiVe/jh0p9DyUJERHJSshARqXLPPlv6cyhZiIi0wpVXxh1BZNmy0p9DyUJEpBXGjo07gvJSshARyeH006Pl2bNh5kzYemvo3z++mMpNyUJEJIfdd4+WGxrCA3yQfR6MWqQnuEVEcjjqqJAY0ue9ULIQEZH/064djB+feXtbUdE/qplta2a3mtm9ccciIpKuLdUsciYLM+tkZi+a2StmNs/M/qu1JzOzKWa20sxey7BvpJm9aWYLzexCAHdf7O4TWns+EZFS2nffuCMon3xqFmuBEe6+C/ANYKSZ7ZFcwMw2N7Nuadsy3ScwFRiZvtHM6oDrgVHAQOBIMxuY108gIhKTbHN316KcycKD1U2rHZpenlZsH+B+M+sEYGYnAddm+KxZwEcZTjMMWNhUk1gHTAcOzfunEBGRksqrz8LM6sxsLrASeNzdX0je7+73AI8A081sPHACMK6AOHoDS5PWlwG9zayHmU0GhpjZpCyxHWxmN61ataqA04mISCHyShbu/pW7fwPoAwwzs0EZylwOrAFuBA5Jqo3kI1M3kbv7h+5+irtv5+6XZontAXc/ub6+voDTiYhsuJFNjerf/W607dAabRMp6G4od/8YeJrM/Q57A4OAPwEXFRjHMmCrpPU+wPICP0NEpKx69IA1a+DBB6Nte+4ZXzyllM/dUJuZWfem5c7AAcAbaWWGADcT+hmOBzY1s0sKiGM2sL2Z9TOzjsARwMwCjhcRicVGG4VbaK+4AnbbDU4+Oe6ISiOfmsWWwFNm9irhS/1xd38wrUwXYKy7L3L39cCxwJL0DzKzacBzwI5mtszMJgC4+5fA6cCjwHxghrvPa+0PJSJSbhMnQmMjbLJJ3JGUhrmn39hUnRoaGryxsTHuMEREYnlYr7Vf5WY2x90bcpWr6Ce4RUSkMihZiIgU2RlnRMv33RdfHMWkZCEiUmTDh0fLnTrFF0cxKVmIiBTZbrtFy7Uy2KCShYhIkfXrFy0rWYiISE5KFiIi0mYoWYiIlJBqFiIikpOShYiI5GQGK1bEHcWGU7IQESmxLbaIO4INp2QhIlICA5smhm7IOepSdWgfdwAiIrXo1Vfh88+ha9e4IykO1SxEREqgri5zovjOd8ofSzEoWYiIlNFhh8UdQesoWYiIxKRLFzjggLijyI+ShYhIGXXpEi3X18Pjj8cXSyGULEREymDqVBg7NrwSqumBPSULEZEyOPZYmDEDOnSItilZiIhITkoWIiJSU5QsRERikl6zaO2VVu9wAAAGUUlEQVSdUQMGbHgsuShZiIjEZOONw/v550PfvnDvvfCrX8Hf/lbY5zz9dLEja87cvfRnKYOGhgZvbGyMOwwRkZyefBImToTbb4fBg8M299SaRiH9GRvyNW5mc9w95whWGhtKRKTMRoyAl15K3Vbpnd1qhhIRqUC//W3cEaRSshARqUA//nHcEaRSshARkZyULEREJCclCxGRCvXWW/DYY3FHEVT03VBmti3wU6De3at0FHgRkdbp2ze8WvLRR+WIJI+ahZltZWZPmdl8M5tnZme19mRmNsXMVprZaxn2jTSzN81soZldCODui919QmvPJyJSC155BS67LPO+TTYpTwz5NEN9CZzr7l8H9gBOM7OByQXMbHMz65a2rX+Gz5oKjEzfaGZ1wPXAKGAgcGT6OURE2qqdd4YLLog3hpzJwt3fdfeXmpY/AeYDvdOK7QPcb2adAMzsJODaDJ81C8hUaRoGLGyqSawDpgOHFvKDiIhI6RTUwW1mfYEhwAvJ2939HuARYLqZjQdOAMYV8NG9gaVJ68uA3mbWw8wmA0PMbFKWmA42s5tWrVpVwOlERKQQeScLM+sK3Aec7e7/Tt/v7pcDa4AbgUPcfXUBcWR60N3d/UN3P8Xdt3P3SzMd6O4PuPvJ9fX1BZxOREQKkVeyMLMOhERxl7v/d5YyewODgD8BFxUYxzJgq6T1PsDyAj9DRERKJJ+7oQy4FZjv7hlHKzGzIcDNhH6G44FNzeySAuKYDWxvZv3MrCNwBDCzgONFRKSE8qlZ7AX8EBhhZnObXt9NK9MFGOvui9x9PXAssCT9g8xsGvAcsKOZLTOzCQDu/iVwOvAooQN9hrvPa/VPJSLSBlx5ZfnOpfksRESqRGIY886d4Z13ivOMRb7zWWi4DxGRKnHzzeH9ppvK9zBeQkUP9yEiIpETT4Tx40PNotxUsxARqSJxJApQshARkTwoWYiISE5KFiIikpOShYiI5KRkISIiOSlZiIhITkoWIiKSU80M92Fmq4AFaZvrgVVZ1pOXewIfFDGc9PNuaPmW9mfa19LPnWs9zmuRT9lsZfK5Dpm26Xci97p+J4Ja/Z3Yxt03y3l2d6+JF3BTrm3J62nLjaWOZUPKt7S/0J87j+sS27XIp2y2MvlcB/1O6HdCvxOtP38tNUM9kMe2B1rYV+pYNqR8S/sL/bnzWS+mQj47n7LZyuRzHTJt0+9EfuvFpN+J1n12qX8nWlQzzVAbwswaPY9RF9sCXYtA1yGiaxG09etQSzWLDXFT3AFUEF2LQNchomsRtOnroJqFiIjkpJqFiIjkpGQhIiI5KVmIiEhOShYZmNnGZna7md1sZuPjjicuZratmd1qZvfGHUvczOx7Tb8P95vZgXHHExcz+7qZTTaze83s1LjjiVvTd8UcMxsddyyl1maShZlNMbOVZvZa2vaRZvammS00swubNo8B7nX3k4BDyh5sCRVyHdx9sbtPiCfS0ivwWvz/pt+H44DDYwi3ZAq8DvPd/RRgHFBzt5EW+D0BcAEwo7xRxqPNJAtgKjAyeYOZ1QHXA6OAgcCRZjYQ6AMsbSr2VRljLIep5H8dat1UCr8WP2vaX0umUsB1MLNDgGeBJ8obZllMJc9rYWYHAK8D75U7yDi0mWTh7rOAj9I2DwMWNv0FvQ6YDhwKLCMkDKixa1TgdahphVwLC34DPOzuL5U71lIq9HfC3We6+zeBmmuiLfBa7AfsARwFnGRmNfVdka593AHErDdRDQJCktgduBa4zswOorSP+1eKjNfBzHoAvwKGmNkkd780lujKK9vvxBnAAUC9mfV398lxBFdG2X4n9iU0024EPBRDXHHIeC3c/XQAMzsO+MDd18cQW9m09WRhGba5u38KHF/uYGKU7Tp8CJxS7mBilu1aXEv4I6KtyHYdngaeLm8osct4Lf5vwX1q+UKJT01Xm/KwDNgqab0PsDymWOKk6xDRtQh0HSK6FihZzAa2N7N+ZtYROAKYGXNMcdB1iOhaBLoOEV0L2lCyMLNpwHPAjma2zMwmuPuXwOnAo8B8YIa7z4szzlLTdYjoWgS6DhFdi+w0kKCIiOTUZmoWIiLSekoWIiKSk5KFiIjkpGQhIiI5KVmIiEhOShYiIpKTkoWIiOSkZCEiIjkpWYiISE7/CxfmtjbVXvMKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "line, = ax.plot(loss, color='blue', lw=2)\n",
    "ax.set_yscale('log')\n",
    "ax.set_xscale('log')\n",
    "\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cgGWJJf-b55f"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "CNN_LSTM_PyTorch-BeamSearch.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}