text_summariser.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "text summariser",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zyKj5MW-TuE2",
        "outputId": "a7e4af44-175c-4048-9f67-00593e871b4e"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Szca57v8VJqB"
      },
      "source": [
        "import numpy as np  \n",
        "import pandas as pd \n",
        "import re           \n",
        "from keras.preprocessing.text import Tokenizer \n",
        "from bs4 import BeautifulSoup \n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from nltk.corpus import stopwords   \n",
        "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed, Bidirectional, Attention, GlobalAveragePooling1D\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "import warnings\n",
        "import nltk\n",
        "pd.set_option(\"display.max_colwidth\", 200)\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "SdrQgcVZWXWA",
        "outputId": "154a3d06-dc97-4121-b646-9b706a6e9649"
      },
      "source": [
        "data=pd.read_csv(\"/content/drive/MyDrive/Reviews.csv\")\n",
        "data.head()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>ProductId</th>\n",
              "      <th>UserId</th>\n",
              "      <th>ProfileName</th>\n",
              "      <th>HelpfulnessNumerator</th>\n",
              "      <th>HelpfulnessDenominator</th>\n",
              "      <th>Score</th>\n",
              "      <th>Time</th>\n",
              "      <th>Summary</th>\n",
              "      <th>Text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>B001E4KFG0</td>\n",
              "      <td>A3SGXH7AUHU8GW</td>\n",
              "      <td>delmartian</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>1303862400</td>\n",
              "      <td>Good Quality Dog Food</td>\n",
              "      <td>I have bought several of the Vitality canned dog food products and have found them all to be of good quality. The product looks more like a stew than a processed meat and it smells better. My Labr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>B00813GRG4</td>\n",
              "      <td>A1D87F6ZCVE5NK</td>\n",
              "      <td>dll pa</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1346976000</td>\n",
              "      <td>Not as Advertised</td>\n",
              "      <td>Product arrived labeled as Jumbo Salted Peanuts...the peanuts were actually small sized unsalted. Not sure if this was an error or if the vendor intended to represent the product as \"Jumbo\".</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>B000LQOCH0</td>\n",
              "      <td>ABXLMWJIXXAIN</td>\n",
              "      <td>Natalia Corres \"Natalia Corres\"</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>1219017600</td>\n",
              "      <td>\"Delight\" says it all</td>\n",
              "      <td>This is a confection that has been around a few centuries.  It is a light, pillowy citrus gelatin with nuts - in this case Filberts. And it is cut into tiny squares and then liberally coated with ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>B000UA0QIQ</td>\n",
              "      <td>A395BORC6FGVXV</td>\n",
              "      <td>Karl</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>1307923200</td>\n",
              "      <td>Cough Medicine</td>\n",
              "      <td>If you are looking for the secret ingredient in Robitussin I believe I have found it.  I got this in addition to the Root Beer Extract I ordered (which was good) and made some cherry soda.  The fl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>B006K2ZZ7K</td>\n",
              "      <td>A1UQRSCLF8GW1T</td>\n",
              "      <td>Michael D. Bigham \"M. Wassir\"</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>1350777600</td>\n",
              "      <td>Great taffy</td>\n",
              "      <td>Great taffy at a great price.  There was a wide assortment of yummy taffy.  Delivery was very quick.  If your a taffy lover, this is a deal.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Id  ...                                                                                                                                                                                                     Text\n",
              "0   1  ...  I have bought several of the Vitality canned dog food products and have found them all to be of good quality. The product looks more like a stew than a processed meat and it smells better. My Labr...\n",
              "1   2  ...           Product arrived labeled as Jumbo Salted Peanuts...the peanuts were actually small sized unsalted. Not sure if this was an error or if the vendor intended to represent the product as \"Jumbo\".\n",
              "2   3  ...  This is a confection that has been around a few centuries.  It is a light, pillowy citrus gelatin with nuts - in this case Filberts. And it is cut into tiny squares and then liberally coated with ...\n",
              "3   4  ...  If you are looking for the secret ingredient in Robitussin I believe I have found it.  I got this in addition to the Root Beer Extract I ordered (which was good) and made some cherry soda.  The fl...\n",
              "4   5  ...                                                             Great taffy at a great price.  There was a wide assortment of yummy taffy.  Delivery was very quick.  If your a taffy lover, this is a deal.\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V6VeFQD3boPP"
      },
      "source": [
        "data.drop_duplicates(subset=['Text'],inplace=True)  \n",
        "data.dropna(axis=0,inplace=True) "
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SeN_gO3LbyA2"
      },
      "source": [
        "contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\",\n",
        "\n",
        "                           \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n",
        "\n",
        "                           \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",\n",
        "\n",
        "                           \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\",\n",
        "\n",
        "                           \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n",
        "\n",
        "                           \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\",\n",
        "\n",
        "                           \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\",\n",
        "\n",
        "                           \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\n",
        "\n",
        "                           \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\",\n",
        "\n",
        "                           \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n",
        "\n",
        "                           \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\",\n",
        "\n",
        "                           \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n",
        "\n",
        "                           \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\",\n",
        "\n",
        "                           \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\",\n",
        "\n",
        "                           \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\",\n",
        "\n",
        "                           \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n",
        "\n",
        "                           \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\",\n",
        "\n",
        "                           \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n",
        "\n",
        "                           \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n",
        "\n",
        "                           \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\",\n",
        "\n",
        "                           \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\n",
        "\n",
        "                           \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n",
        "\n",
        "                           \"you're\": \"you are\", \"you've\": \"you have\"}"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nj9Tz79xcH6V"
      },
      "source": [
        "# Preprocessing\n",
        "\n",
        "- converting to lower case\n",
        "- removing html tags\n",
        "- applying contraction mapping\n",
        "- remove spl characters and stopwords\n",
        "- removing short words"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-RA384Iac0wi",
        "outputId": "af15dcfa-e723-4f88-8a3b-3198cf707002"
      },
      "source": [
        "nltk.download('stopwords')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mDtrv7kib3nm"
      },
      "source": [
        "stop_words = set(stopwords.words('english')) \n",
        "def text_cleaner(text):\n",
        "    newString = text.lower()\n",
        "    newString = BeautifulSoup(newString, \"lxml\").text\n",
        "    newString = re.sub(r'\\([^)]*\\)', '', newString)\n",
        "    newString = re.sub('\"','', newString)\n",
        "    newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(\" \")])    \n",
        "    newString = re.sub(r\"'s\\b\",\"\",newString)\n",
        "    newString = re.sub(\"[^a-zA-Z]\", \" \", newString) \n",
        "    tokens = [w for w in newString.split() if not w in stop_words]\n",
        "    long_words=[]\n",
        "    for i in tokens:\n",
        "        if len(i)>=3:                  #removing short word\n",
        "            long_words.append(i)   \n",
        "    return (\" \".join(long_words)).strip()\n",
        "\n",
        "cleaned_text = []\n",
        "for t in data['Text']:\n",
        "    cleaned_text.append(text_cleaner(t))"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gA91A36Kb7PV"
      },
      "source": [
        "def summary_cleaner(text):\n",
        "    newString = re.sub('\"','', text)\n",
        "    newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(\" \")])    \n",
        "    newString = re.sub(r\"'s\\b\",\"\",newString)\n",
        "    newString = re.sub(\"[^a-zA-Z]\", \" \", newString)\n",
        "    newString = newString.lower()\n",
        "    tokens=newString.split()\n",
        "    newString=''\n",
        "    for i in tokens:\n",
        "        if len(i)>1:                                 \n",
        "            newString=newString+i+' '  \n",
        "    return newString\n",
        "\n",
        "#Call the above function\n",
        "cleaned_summary = []\n",
        "for t in data['Summary']:\n",
        "    cleaned_summary.append(summary_cleaner(t))\n",
        "\n",
        "data['cleaned_text']=cleaned_text\n",
        "data['cleaned_summary']=cleaned_summary\n",
        "data['cleaned_summary'].replace('', np.nan, inplace=True)\n",
        "data.dropna(axis=0,inplace=True)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "whw6eTl8dcE8"
      },
      "source": [
        "# adding start and end tokens\n",
        "data['cleaned_summary'] = data['cleaned_summary'].apply(lambda x : '_START_ '+ x + ' _END_')"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VZ38bqXld6Yn",
        "outputId": "f8c41370-56e5-4370-f88e-9b8e1d8d1263"
      },
      "source": [
        "for i in range(5):\n",
        "    print(\"Review:\",data['cleaned_text'][i])\n",
        "    print(\"Summary:\",data['cleaned_summary'][i])\n",
        "    print(\"\\n\")"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Review: bought several vitality canned dog food products found good quality product looks like stew processed meat smells better labrador finicky appreciates product better\n",
            "Summary: _START_ good quality dog food  _END_\n",
            "\n",
            "\n",
            "Review: product arrived labeled jumbo salted peanuts peanuts actually small sized unsalted sure error vendor intended represent product jumbo\n",
            "Summary: _START_ not as advertised  _END_\n",
            "\n",
            "\n",
            "Review: confection around centuries light pillowy citrus gelatin nuts case filberts cut tiny squares liberally coated powdered sugar tiny mouthful heaven chewy flavorful highly recommend yummy treat familiar story lewis lion witch wardrobe treat seduces edmund selling brother sisters witch\n",
            "Summary: _START_ delight says it all  _END_\n",
            "\n",
            "\n",
            "Review: looking secret ingredient robitussin believe found got addition root beer extract ordered made cherry soda flavor medicinal\n",
            "Summary: _START_ cough medicine  _END_\n",
            "\n",
            "\n",
            "Review: great taffy great price wide assortment yummy taffy delivery quick taffy lover deal\n",
            "Summary: _START_ great taffy  _END_\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "2HQJZCi-d8TG",
        "outputId": "8ffad618-4ace-4ea7-d53d-975d3c2980ad"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "text_word_count = []\n",
        "summary_word_count = []\n",
        "\n",
        "# populate the lists with sentence lengths\n",
        "for i in data['cleaned_text']:\n",
        "      text_word_count.append(len(i.split()))\n",
        "\n",
        "for i in data['cleaned_summary']:\n",
        "      summary_word_count.append(len(i.split()))\n",
        "\n",
        "length_df = pd.DataFrame({'text':text_word_count, 'summary':summary_word_count})\n",
        "length_df.hist(bins = 30)\n",
        "plt.show()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEICAYAAABI7RO5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAdrElEQVR4nO3dfZRdVZnn8e/P8BZBTXjpEpN0J91kdEVoQDKYHp3pGlCI0DOh10LFZiRglukeocWRtgmO01GBnjCrgQak6UmbNIGOBETsZCQaI6aW45ohvEZCiAxljCZZkUjesFDA0M/8cXaZk8u9VXdX1X2pW7/PWnfVOc/Z5+3m3Dx377PvPooIzMzM6vWGVh+AmZmNLk4cZmaWxYnDzMyyOHGYmVkWJw4zM8vixGFmZlmcOMzMLIsTR4eQtFXS+9plO2bWuZw4zMwySTqs1cfQSk4cHUDS3cBvA/9LUp+kv5Q0S9L/kbRP0g8kdaey/0bSC5KmpPlTJe2V9I5q22nZSVnHk3S1pB2SfiHpWUlnS7pT0nWlMt2Stpfmt0r6jKSnJL0kaYmkLknfTNv5jqSJqexUSSHpMknb0nX+Z5L+dVp/n6Qvlbb9e5K+K2l3+owslzShYt9XS3oKeCkdx9cqzulWSbc09I1rBxHhVwe8gK3A+9L0JGA3cB7Fl4P3p/kT0vLrge8C44GNwBXVtuOXX416AW8HtgFvS/NTgd8D7gSuK5XrBraX5rcCDwNd6TrfBTwBnA4cla7rhaVtBvD3adk5wMvAPwO/VVr/D1P5k9Jn5UjgBOB7wN9W7HsDMCV9dk4EXgImpOWHpe2d0er3t9Ev1zg6038CVkfE6oj4l4hYCzxGkUgAPg+8BXgE2AHc3pKjtLHsNYr/oGdIOjwitkbEj+pc97aIeD4idgD/G1gfEU9GxMvA1ymSSNm1EfFyRHyb4j/6eyJiV2n90wEiojci1kbEKxHxc+Am4A8rtnVrRGyLiF9FxE6K5PLBtGw28EJEPJ71ToxCThyd6XeAD6aq+D5J+4D3UnxDIiJ+TfHN7mTgxkhfl8yaJSJ6gU9RfInZJWmFpLfVufrzpelfVZk/ZijlU5PXitR89iLwT8DxFdvaVjG/jOKLGunv3XWew6jmxNE5yv/5bwPujogJpdfREbEIQNIkYCHwj8CNko6ssR2zhomIr0TEeym+6ARwA0WN4I2lYm9t4iH9dTqOUyLizRSJQBVlKj8f/wz8vqSTgT8Cljf8KNuAE0fneB743TT9T8B/kHSupHGSjko3GSdLEkVtYwkwD9gJXFtjO2YNIentks5KX1pepvjm/y8U9xDOk3SspLdS1Eqa5U1AH7A/fbn6zGArpOax+4GvAI9ExE8be4jtwYmjc/x34HOpWerDwBzgs8DPKWogn6H49/4kxY3B/5aaqC4DLpP0byu3I+kvmnwONnYcCSwCXgB+RnFNXkPR1PMDihvR3wbubeIxfQF4F7AfeBB4oM71lgGnMEaaqQDk5m0zs6GT9NvAD4G3RsSLrT6eZnCNw8xsiCS9Afg0sGKsJA0o+h2bmVkmSUdT3BP8CUVX3DHDTVVmZpbFTVVmZpZl0KYqSUdR/DryyFT+/ohYKOlOil9V7k9FL42IDam75y0Uv1L+ZYo/kbY1F/hcKn9dRCxL8TMouoiOB1YDV0ZESDqWolfFVIpeFh+KiL0DHe/xxx8fU6dOrbrspZde4uijjx7slMccvy+v9/jjj78QESe0+jjqMdA130466TrrxHPJuubrGFNGwDFp+nBgPTCL4j/6C6uUPw/4ZlpvFsVwAADHAlvS34lpemJa9kgqq7TuB1L8fwAL0vQC4IbBjveMM86IWtatW1dz2Vjm9+X1gMeiDcYEquc10DXfTjrpOuvEc8m55gdtqkrb7kuzh6fXQDdG5gB3pfUeBiZIOhE4F1gbEXuiqDWsBWanZW+OiIfTwd8FXFDa1rI0vawUNzOzFqmrV5WkccDjFKNH3h4R6yX9Z+B6SX8FPERRM3iFYsTJ8ngu21NsoPj2KnGArigGEoPiR0JdNY5vPjAfoKuri56enqrn0dfXV3PZWOb3xcxy1JU4IuI14LQ0Nv3X07gs11D8Z34EsBi4Gvhiow40IkJS1ZpORCxOx8DMmTOju7u76jZ6enqotWws8/tiZjmyelVFxD5gHTA7Inam5qhXKAbLOzMV20ExXn2/ySk2UHxylTjA86kpi/R3V87xmpnZyBs0cUg6of8pWJLGUzzo5Iel/9BFce/h6bTKKuASFWYB+1Nz0xrgHEkT0xO6zgHWpGUvpifWCbgEWFna1tw0PbcUNzOzFqmnqepEYFm6z/EG4L6I+EZ6xOIJFD2hNgB/lsqvpuhZ1UvRHfcygIjYI+la4NFU7osRsSdNf4KD3XG/mV5QDIJ2n6R5FL/O/NBQT9TMzEbGoIkjIp7i9U/UIiLOqlE+gMtrLFsKLK0Sf4zioUKV8d3A2YMdo5mZNY9/OW5mZlmcOMzMLMuYGh134479XLrgwd/Mb110fguPxqz5ppau/37+HFgu1zjMzCyLE4eZmWVx4jAzsyxOHGZmlsWJw8zMsjhxmJlZFicOMzPL4sRhZmZZnDjMzCyLE4eZmWVx4jAzsyxOHGZmlsWJw8zMsjhxmGWS9F8kbZL0tKR7JB0laZqk9ZJ6Jd0r6YhU9sg035uWTy1t55oUf1bSuaX47BTrlbSg+WdoNjAnDrMMkiYBnwRmRsTJwDjgIuAG4OaIOAnYC8xLq8wD9qb4zakckmak9d4JzAb+TtK49Ijm24EPADOAj6SyZm3DicMs32HAeEmHAW8EdgJnAfen5cuAC9L0nDRPWn62JKX4ioh4JSJ+DPQCZ6ZXb0RsiYhXgRWprFnbGFMPcjIbrojYIelvgJ8CvwK+DTwO7IuIA6nYdmBSmp4EbEvrHpC0HzguxR8ubbq8zraK+Lsrj0PSfGA+QFdXFz09PXUd/1WnHHhdrN51h6uvr69p+2q0sX4uThxmGSRNpKgBTAP2AV+laGpqqohYDCwGmDlzZnR3d9e13qXVngB4cX3rDldPTw/1Hme7G+vn4qYqszzvA34cET+PiF8DDwDvASakpiuAycCONL0DmAKQlr8F2F2OV6xTK27WNgZNHKnHyCOSfpB6knwhxRvei6TWPsxa6KfALElvTPcqzgaeAdYBF6Yyc4GVaXpVmict/25ERIpflD4v04DpwCPAo8D0dO0fQXEDfVUTzsusbvXUOF4BzoqIU4HTgNmSZtGcXiS19mHWEhGxnuIm9xPARorP0GLgauDTknop7mEsSassAY5L8U8DC9J2NgH3USSdbwGXR8Rr6T7JFcAaYDNwXypr1jYGvceRvh31pdnD0ysoepH8SYovAz4P3EHR/vv5FL8f+FJlLxLgx+mDdGYq1xsRWwAkrQDmSNo8wD7MWiYiFgILK8JbOHg9l8u+DHywxnauB66vEl8NrB7+kZo1Rl03x1Ot4HHgJIrawY9ofC+S4wbYR+Xx1dXDpGv8ob1KOqVXxHB1Ug8RM2u8uhJHRLwGnCZpAvB14B0NPapM9fYwuW35Sm7cePCUm9WbpN11Ug8RM2u8rF5VEbGP4ibgH9D4XiS7B9iHmZm1SD29qk5INQ0kjQfeT3HTrqG9SNI6tfZhZmYtUk9T1YnAsnSf4w0UvTy+IekZYIWk64AnObQXyd3p5vceikRARGyS1N+L5ACpFwmApP5eJOOApaVeJFfX2IeZmbVIPb2qngJOrxJveC+SWvswM7PW8ZAjZmPc1IphSLYuOr9FR2KjhYccMTOzLE4cZmaWxYnDzMyyOHGYmVkWJw4zM8vixGFmZlmcOMzMLIsTh5mZZXHiMDOzLE4cZmaWxYnDzMyyOHGYmVkWJw4zM8vixGFmZlmcOMzMLIsTh5mZZXHiMDOzLE4cZmaWxYnDzMyyOHGYmVmWQROHpCmS1kl6RtImSVem+Ocl7ZC0Ib3OK61zjaReSc9KOrcUn51ivZIWlOLTJK1P8XslHZHiR6b53rR86kievJmZ5aunxnEAuCoiZgCzgMslzUjLbo6I09JrNUBadhHwTmA28HeSxkkaB9wOfACYAXyktJ0b0rZOAvYC81J8HrA3xW9O5czMrIUGTRwRsTMinkjTvwA2A5MGWGUOsCIiXomIHwO9wJnp1RsRWyLiVWAFMEeSgLOA+9P6y4ALSttalqbvB85O5c3MrEWy7nGkpqLTgfUpdIWkpyQtlTQxxSYB20qrbU+xWvHjgH0RcaAifsi20vL9qbyZmbXIYfUWlHQM8DXgUxHxoqQ7gGuBSH9vBD7WkKMc/NjmA/MBurq66OnpqVquazxcdcqB38zXKjfW9PX1+b0ws7rVlTgkHU6RNJZHxAMAEfF8afk/AN9IszuAKaXVJ6cYNeK7gQmSDku1inL5/m1tl3QY8JZU/hARsRhYDDBz5szo7u6ueh63LV/JjRsPnvLWi6uXG2t6enqo9Z6ZmVWqp1eVgCXA5oi4qRQ/sVTsj4Gn0/Qq4KLUI2oaMB14BHgUmJ56UB1BcQN9VUQEsA64MK0/F1hZ2tbcNH0h8N1U3szMWqSeGsd7gI8CGyVtSLHPUvSKOo2iqWor8KcAEbFJ0n3AMxQ9si6PiNcAJF0BrAHGAUsjYlPa3tXACknXAU9SJCrS37sl9QJ7KJKNmZm10KCJIyK+D1TrybR6gHWuB66vEl9dbb2I2ELR66oy/jLwwcGO0czMmse/HDczsyxOHGZmlsWJw8zMstT9Ow4zK0iaAHwZOJmic8jHgGeBe4GpFJ1FPhQRe1OvxFuA84BfApf2j8QgaS7wubTZ6yJiWYqfAdwJjKe4J3hlM3sTTl3w4CHzWxed36xd2yjhGodZvluAb0XEO4BTKYbhWQA8FBHTgYfSPBRjs01Pr/nAHQCSjgUWAu+m6BiysDT6wh3Ax0vrzW7COZnVzYnDLIOktwD/jtRlPCJejYh9HDquWuV4a3dF4WGKH7ueCJwLrI2IPRGxF1gLzE7L3hwRD6daxl2lbZm1BTdVmeWZBvwc+EdJpwKPA1cCXRGxM5X5GdCVpnPHbpuUpivjh6h3mJ1K5SF36jVSw9F00tA2Y/1cnDjM8hwGvAv484hYL+kWDjZLARARIamh9yTqHWan0qUV9y/qMVJD83TS0DZj/VzcVGWWZzuwPSL6R4i+nyKRPN8/DE/6uystrzV220DxyVXiZm3DicMsQ0T8DNgm6e0pdDbF8DrlcdUqx1u7RIVZwP7UpLUGOEfSxHRT/BxgTVr2oqRZqUfWJaVtmbUFN1WZ5ftzYHkarHMLcBnFl7D7JM0DfgJ8KJVdTdEVt5eiO+5lABGxR9K1FIN/AnwxIvak6U9wsDvuN9PLrG04cZhliogNwMwqi86uUjaAy2tsZymwtEr8MYrfiJi1JTdVmZlZFicOMzPL4sRhZmZZnDjMzCyLE4eZmWVx4jAzsyxOHGZmlsWJw8zMsjhxmJlZFicOMzPLMmjikDRF0jpJz0jaJOnKFD9W0lpJz6W/E1Nckm6V1CvpKUnvKm1rbir/XHpsZn/8DEkb0zq3psHdau7DzMxap54axwHgqoiYAcwCLpc0g+Y8KrPWPszMrEUGTRwRsTMinkjTv6B4vvIkmvOozFr7MDOzFskaHVfSVOB0YD3NeVRmrX1UHlddj9HsGn/oozM75dGPw9VJj8E0s8arO3FIOgb4GvCpiHgx3YYAmvaozJr7qPcxmrctX8mNGw+e8kg9EnO066THYJpZ49XVq0rS4RRJY3lEPJDCzXhUZq19mJlZi9TTq0rAEmBzRNxUWtSMR2XW2oeZmbVIPU1V7wE+CmyUtCHFPgssovGPyqy1DzMza5FBE0dEfB9QjcUNfVRmROyutg8zM2sd/3LczMyyOHGYmVkWJw4zM8vixGFmZlmcOMzMLIsTh5mZZXHiMDOzLE4cZmaWxYnDzMyyOHGYmVkWJw4zM8vixGFmZlmyngBoZmPP1AUPvi62ddH5LTgSaxeucZiZWRYnDjMzy+LEYWZmWZw4zMwsixOHmZllceIwM7MsThxmmSSNk/SkpG+k+WmS1kvqlXSvpCNS/Mg035uWTy1t45oUf1bSuaX47BTrlbSg2edmVg//jsMs35XAZuDNaf4G4OaIWCHp74F5wB3p796IOEnSRanchyXNAC4C3gm8DfiOpH+VtnU78H5gO/CopFUR8cxQD7TabzDMhss1DrMMkiYD5wNfTvMCzgLuT0WWARek6TlpnrT87FR+DrAiIl6JiB8DvcCZ6dUbEVsi4lVgRSpr1lYGrXFIWgr8EbArIk5Osc8DHwd+nop9NiJWp2XXUHzTeg34ZESsSfHZwC3AOODLEbEoxadRfECOAx4HPhoRr0o6ErgLOAPYDXw4IraOwDmbDcffAn8JvCnNHwfsi4gDaX47MClNTwK2AUTEAUn7U/lJwMOlbZbX2VYRf3e1g5A0H5gP0NXVRU9PT9WDveqUA1Xjw1VrfwPp6+sb0nrtaKyfSz1NVXcCX6L4T7zs5oj4m3JgiFXwrGp+1tmZjSBJ/V+gHpfU3cpjiYjFwGKAmTNnRnd39cO5tEFNVVsvrr6/gfT09FDrOEebsX4ugzZVRcT3gD11bi+rCj7Ear5Zq7wH+I+StlJcw2dR1KInSOr/EjYZ2JGmdwBTANLyt1DUnn8Tr1inVtysrQzn5vgVki4BHgOuioi95FfBh1LNf6HyQOqttneNP7Tq3ilVzeHqpGp3I0XENcA1AKnG8RcRcbGkrwIXUiSTucDKtMqqNP9/0/LvRkRIWgV8RdJNFDXz6cAjgIDpqfl2B0Xt/U+adHpmdRtq4rgDuBaI9PdG4GMjdVC56q2237Z8JTduPHjKQ6lud6JOqna3yNXACknXAU8CS1J8CXC3pF6KWvtFABGxSdJ9wDPAAeDyiHgNQNIVwBqKe4FLI2JTU8/ErA5DShwR8Xz/tKR/AL6RZgeqaleL7yZV81Oto1o1f3tFNd+s5SKiB+hJ01sommMry7wMfLDG+tcD11eJrwZWj+Chmo24IXXHlXRiafaPgafT9CrgovTDp2kcrII/SqqCpx9HXQSsiogA1lFU46F6NR9K1fyhHK+ZmY2cerrj3gN0A8dL2g4sBLolnUbRVLUV+FMYchU8q5pvZmatNWjiiIiPVAkvqRLrL59VBR9KNd/MzFrHvxw3M7MsThxmZpbFicPMzLI4cZiZWRYnDjMzy+LEYWZmWZw4zMwsixOHmZllceIwM7MsThxmZpbFicPMzLI4cZiZWRYnDjMzy+LEYWZmWYbzzHEzG6OmLnjwkPmti85v0ZFYK7jGYWZmWZw4zMwsixOHmZllceIwM7MsThxmZpbFicPMzLIMmjgkLZW0S9LTpdixktZKei79nZjiknSrpF5JT0l6V2mduan8c5LmluJnSNqY1rlVkgbah5mZtVY9NY47gdkVsQXAQxExHXgozQN8AJieXvOBO6BIAsBC4N3AmcDCUiK4A/h4ab3Zg+zDzMxaaNDEERHfA/ZUhOcAy9L0MuCCUvyuKDwMTJB0InAusDYi9kTEXmAtMDste3NEPBwRAdxVsa1q+zAzsxYa6j2OrojYmaZ/BnSl6UnAtlK57Sk2UHx7lfhA+zAzsxYa9pAjERGSYiQOZqj7kDSfommMrq4uenp6qpbrGg9XnXLgN/O1yo01fX19fi/MrG5DTRzPSzoxInam5qZdKb4DmFIqNznFdgDdFfGeFJ9cpfxA+3idiFgMLAaYOXNmdHd3Vy132/KV3Ljx4Clvvbh6ubGmp6eHWu+ZmVmloTZVrQL6e0bNBVaW4pek3lWzgP2puWkNcI6kiemm+DnAmrTsRUmzUm+qSyq2VW0fZmbWQoPWOCTdQ1FbOF7SdoreUYuA+yTNA34CfCgVXw2cB/QCvwQuA4iIPZKuBR5N5b4YEf033D9B0XNrPPDN9GKAfZiZWQsNmjgi4iM1Fp1dpWwAl9fYzlJgaZX4Y8DJVeK7q+3DzMxay78cNzOzLE4cZmaWxYnDzMyyOHGYmVkWJw4zM8vixGGWQdIUSeskPSNpk6QrU7zhI0abtQsnDrM8B4CrImIGMAu4XNIMmjNitFlbcOIwyxAROyPiiTT9C2AzxcCczRgx2qwtDHuQQ7OxStJU4HRgPc0ZMbq877oG9iwP6tlI9QyS2UmDaY71c3HiMBsCSccAXwM+FREvlm9DNGnE6LoG9rx0wYONPIyDNr70utDWRecfMt9Jg2mO9XNxU5VZJkmHUySN5RHxQAo/n5qZyBgxula81ojRZm3BicMsQ+rhtATYHBE3lRY1Y8Ros7bgpiqzPO8BPgpslLQhxT5Lc0aMNmsLThxmGSLi+0Ct31U0dMRos3bhpiozM8vixGFmZlmcOMzMLIsTh5mZZXHiMDOzLE4cZmaWxYnDzMyyOHGYmVmWYSUOSVvTA2c2SHosxfxAGzOzDjYSNY5/HxGnRcTMNO8H2piZdbBGNFX5gTZmZh1suGNVBfDt9OyB/5meEdDUB9pA/Q+16Rp/6INtOuVBLMPVSQ+lsfa1ccf+1z0fpPKZHTY6DDdxvDcidkj6LWCtpB+WFzbjgTZpP3U91Oa25Su5cePBU956cfVyY00nPZTGzBpvWE1VEbEj/d0FfJ3iHoUfaGNm1sGGnDgkHS3pTf3TFA+ieRo/0MbMrKMNp6mqC/h66iF7GPCViPiWpEcZJQ+0mer2VjOzbENOHBGxBTi1Snw3fqCNmVnH8i/HzcwsixOHmZllceIwM7MsThxmZpbFicPMzLIM95fjZmZD5i7xo5NrHGZmlsWJw8zMsjhxmJlZFicOMzPL4sRhZmZZnDjMzCyLu+OaWdtw99zRwTUOMzPL4sRhZmZZnDjMzCyLE4eZmWXxzXEza1uVN8vBN8zbgWscZmaWxYnDzMyyuKmqxNVis/bn33q0nmscZmaWpe1rHJJmA7cA44AvR8SiFh+SWcP5uq+fWwqar60Th6RxwO3A+4HtwKOSVkXEM609MrPG8XU/8ty8NbLaOnEAZwK9EbEFQNIKYA7QtA9QtW8zlXwR2ghr+XXf6er5XFfjz3qh3RPHJGBbaX478O7KQpLmA/PTbJ+kZ2ts73jghRE9QkA3jPQWm64h78so9zst3Peg133GNd8yVT4XTbvOGvWZLG23kz4z/edS9zXf7omjLhGxGFg8WDlJj0XEzCYc0qji92X0qfeabyeddJ2N9XNp915VO4AppfnJKWbWyXzdW1tr98TxKDBd0jRJRwAXAatafExmjebr3tpaWzdVRcQBSVcAayi6JS6NiE3D2OSoqto3kd+XNtKA675ddNJ1NqbPRRHRiAMxM7MO1e5NVWZm1macOMzMLMuYSBySZkt6VlKvpAWtPp5Gk7RU0i5JT5dix0paK+m59HdiikvSrem9eUrSu0rrzE3ln5M0txXnYqNTzjXY7iRNkbRO0jOSNkm6MsVH3flIOkrSI5J+kM7lCyk+TdL69P/AvalTRk0dnzhKwzd8AJgBfETSjNYeVcPdCcyuiC0AHoqI6cBDaR6K92V6es0H7oDiQwEspPjh2ZnAwtHwwbC2cSf1X4Pt7gBwVUTMAGYBl6f/Q0bj+bwCnBURpwKnAbMlzQJuAG6OiJOAvcC8gTbS8YmD0vANEfEq0D98Q8eKiO8BeyrCc4BlaXoZcEEpflcUHgYmSDoROBdYGxF7ImIvsJbX/0dgVlXmNdjWImJnRDyRpn8BbKb4df+oO5/0Oe9Ls4enVwBnAfen+KDnMhYSR7XhGya16FhaqSsidqbpnwFdabrW++P3zUZarWtw1JA0FTgdWM8oPR9J4yRtAHZRfCH8EbAvIg6kIoN+1sdC4rAKUfTBdj9sa5nReA1KOgb4GvCpiHixvGw0nU9EvBYRp1GMSHAm8I7cbYyFxOHhGwrPpyYo0t9dKV7r/fH7ZiOt1jXY9iQdTpE0lkfEAyk8as8HICL2AeuAP6Boou7/Qfign/WxkDg8fENhFdDfM2ousLIUvyT1rpoF7E/V7zXAOZImppvi56SY2VDVugbbmiQBS4DNEXFTadGoOx9JJ0iakKbHUzzzZTNFArkwFRv8XCKi41/AecD/o2jL+6+tPp4mnO89wE7g1xTtlfOA4yh6fjwHfAc4NpUVRa+zHwEbgZml7XwM6E2vy1p9Xn6NnlfONdjuL+C9FM1QTwEb0uu80Xg+wO8DT6ZzeRr4qxT/XeCR9Fn/KnDkQNvxkCNmZpZlLDRVmZnZCHLiMDOzLE4cZmaWxYnDzMyyOHGYmVkWJw4zM8vixGFmZln+P1UHqmHdDy1LAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SGiDJeopeSvF"
      },
      "source": [
        "As we need to tokenize and pad the text therefore we need a max length for text and summary "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IxnChTxOeLJc"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "max_text=80 \n",
        "max_summary=10\n",
        "x_train, x_val, y_train, y_val=train_test_split(data['cleaned_text'],data['cleaned_summary'],test_size=0.15,random_state=0,shuffle=True) "
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I8rUf72Ceews"
      },
      "source": [
        "x_tokenizer = Tokenizer()\n",
        "x_tokenizer.fit_on_texts(list(x_train))\n",
        "\n",
        "x_train    =   x_tokenizer.texts_to_sequences(x_train) \n",
        "x_val   =   x_tokenizer.texts_to_sequences(x_val)\n",
        "\n",
        "x_train    =   pad_sequences(x_train,  maxlen=max_text, padding='post') \n",
        "x_val   =   pad_sequences(x_val, maxlen=max_text, padding='post')\n",
        "\n",
        "x_voc_size   =  len(x_tokenizer.word_index) +1"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B-SM7pSXfKz9"
      },
      "source": [
        "y_tokenizer = Tokenizer()\n",
        "y_tokenizer.fit_on_texts(list(y_train))\n",
        "\n",
        "y_train    =   y_tokenizer.texts_to_sequences(y_train) \n",
        "y_val   =   y_tokenizer.texts_to_sequences(y_val) \n",
        "\n",
        "y_train    =   pad_sequences(y_train, maxlen=max_summary, padding='post')\n",
        "y_val   =   pad_sequences(y_val, maxlen=max_summary, padding='post')\n",
        "\n",
        "y_voc_size  =   len(y_tokenizer.word_index) +1"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ujEGJZm1fiIY",
        "outputId": "bffb4b9f-c71f-4fa2-f557-d97255197f61"
      },
      "source": [
        "from keras import backend as K \n",
        "K.clear_session() \n",
        "latent_dim = 500 \n",
        "\n",
        "# Encoder \n",
        "encoder_inputs = Input(shape=(max_text,)) \n",
        "enc_emb = Embedding(x_voc_size, latent_dim,trainable=True)(encoder_inputs) \n",
        "\n",
        "#LSTM 1 \n",
        "encoder_lstm1 = LSTM(latent_dim,return_sequences=True,return_state=True) \n",
        "encoder_output1, state_h1, state_c1 = encoder_lstm1(enc_emb) \n",
        "\n",
        "#LSTM 2 \n",
        "encoder_lstm2 = LSTM(latent_dim,return_sequences=True,return_state=True) \n",
        "encoder_output2, state_h2, state_c2 = encoder_lstm2(encoder_output1) \n",
        "\n",
        "#LSTM 3 \n",
        "encoder_lstm3=LSTM(latent_dim, return_state=True, return_sequences=True) \n",
        "encoder_outputs, state_h, state_c= encoder_lstm3(encoder_output2) \n",
        "\n",
        "# Set up the decoder. \n",
        "decoder_inputs = Input(shape=(None,)) \n",
        "dec_emb_layer = Embedding(y_voc_size, latent_dim,trainable=True) \n",
        "dec_emb = dec_emb_layer(decoder_inputs) \n",
        "\n",
        "#LSTM using encoder_states as initial state\n",
        "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True) \n",
        "decoder_outputs,decoder_fwd_state, decoder_back_state = decoder_lstm(dec_emb,initial_state=[state_h, state_c]) \n",
        "\n",
        "#Attention Layer\n",
        "attn_out = Attention()([decoder_outputs,encoder_outputs]) \n",
        "\n",
        "# decoder_outputs = GlobalAveragePooling1D()(decoder_outputs)\n",
        "# attn_out = GlobalAveragePooling1D()(attn_out)\n",
        "\n",
        "# Concat attention output and decoder LSTM output \n",
        "decoder_concat_input = Concatenate(name='concat_layer')([decoder_outputs, attn_out])\n",
        "\n",
        "#Dense layer\n",
        "decoder_dense = TimeDistributed(Dense(y_voc_size, activation='softmax')) \n",
        "decoder_outputs = decoder_dense(decoder_concat_input) \n",
        "\n",
        "# Define the model\n",
        "model = Model([encoder_inputs, decoder_inputs], decoder_outputs) \n",
        "model.summary()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 80)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding (Embedding)           (None, 80, 500)      55601500    input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm (LSTM)                     [(None, 80, 500), (N 2002000     embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   [(None, 80, 500), (N 2002000     lstm[0][0]                       \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, None, 500)    15092500    input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm_2 (LSTM)                   [(None, 80, 500), (N 2002000     lstm_1[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "lstm_3 (LSTM)                   [(None, None, 500),  2002000     embedding_1[0][0]                \n",
            "                                                                 lstm_2[0][1]                     \n",
            "                                                                 lstm_2[0][2]                     \n",
            "__________________________________________________________________________________________________\n",
            "attention (Attention)           (None, None, 500)    0           lstm_3[0][0]                     \n",
            "                                                                 lstm_2[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "concat_layer (Concatenate)      (None, None, 1000)   0           lstm_3[0][0]                     \n",
            "                                                                 attention[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "time_distributed (TimeDistribut (None, None, 30185)  30215185    concat_layer[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 108,917,185\n",
            "Trainable params: 108,917,185\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0NR9LRojf7YB",
        "outputId": "d204aa5a-a60a-419a-faf1-abdf40bfc30b"
      },
      "source": [
        "model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')\n",
        "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1)\n",
        "history=model.fit([x_train,y_train[:,:-1]], y_train.reshape(y_train.shape[0],y_train.shape[1], 1)[:,1:] ,epochs=10,callbacks=[es],batch_size=512, validation_data=([x_val,y_val[:,:-1]], y_val.reshape(y_val.shape[0],y_val.shape[1], 1)[:,1:]))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "653/653 [==============================] - 1144s 2s/step - loss: 2.7698 - val_loss: 2.3043\n",
            "Epoch 2/10\n",
            "653/653 [==============================] - 1130s 2s/step - loss: 2.2748 - val_loss: 2.1251\n",
            "Epoch 3/10\n",
            "653/653 [==============================] - 1131s 2s/step - loss: 2.1002 - val_loss: 2.0561\n",
            "Epoch 4/10\n",
            "653/653 [==============================] - 1130s 2s/step - loss: 1.9978 - val_loss: 2.0147\n",
            "Epoch 5/10\n",
            "653/653 [==============================] - 1145s 2s/step - loss: 1.9183 - val_loss: 1.9871\n",
            "Epoch 6/10\n",
            "653/653 [==============================] - 1148s 2s/step - loss: 1.8411 - val_loss: 1.9726\n",
            "Epoch 7/10\n",
            "653/653 [==============================] - 1148s 2s/step - loss: 1.7781 - val_loss: 1.9709\n",
            "Epoch 8/10\n",
            "653/653 [==============================] - 1147s 2s/step - loss: 1.7199 - val_loss: 1.9752\n",
            "Epoch 00008: early stopping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9LAOZy4Xs6c9"
      },
      "source": [
        "reverse_target_word_index=y_tokenizer.index_word \n",
        "reverse_source_word_index=x_tokenizer.index_word \n",
        "target_word_index=y_tokenizer.word_index"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yEQiwdtQiV3o"
      },
      "source": [
        "encoder_model = Model(inputs=encoder_inputs,outputs=[encoder_outputs, state_h, state_c])\n",
        "\n",
        "# decoder inference\n",
        "# Below tensors will hold the states of the previous time step\n",
        "decoder_state_input_h = Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = Input(shape=(latent_dim,))\n",
        "decoder_hidden_state_input = Input(shape=(max_text,latent_dim))\n",
        "\n",
        "# Get the embeddings of the decoder sequence\n",
        "dec_emb2= dec_emb_layer(decoder_inputs)\n",
        "\n",
        "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
        "decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=[decoder_state_input_h, decoder_state_input_c])\n",
        "\n",
        "#attention inference\n",
        "attn_out_inf = Attention()([decoder_outputs2,decoder_hidden_state_input])\n",
        "decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_outputs2, attn_out_inf])\n",
        "\n",
        "# A dense softmax layer to generate prob dist. over the target vocabulary\n",
        "decoder_outputs2 = decoder_dense(decoder_inf_concat)\n",
        "\n",
        "# Final decoder model\n",
        "decoder_model = Model([decoder_inputs] + [decoder_hidden_state_input,decoder_state_input_h, decoder_state_input_c],[decoder_outputs2] + [state_h2, state_c2])"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-gyBUMprrjsd"
      },
      "source": [
        "def decode_sequence(input_seq):\n",
        "    # Encode the input as state vectors.\n",
        "    e_out, e_h, e_c = encoder_model.predict(input_seq)\n",
        "\n",
        "    # Generate empty target sequence of length 1.\n",
        "    target_seq = np.zeros((1,1))\n",
        "\n",
        "    # Chose the 'start' word as the first word of the target sequence\n",
        "    target_seq[0, 0] = target_word_index['start']\n",
        "\n",
        "    stop_condition = False\n",
        "    decoded_sentence = ''\n",
        "    while not stop_condition:\n",
        "        output_tokens, h, c = decoder_model.predict([target_seq] + [e_out, e_h, e_c])\n",
        "\n",
        "        # Sample a token\n",
        "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
        "        # print(sampled_token_index)\n",
        "        if sampled_token_index==2 : break\n",
        "        else: sampled_token = reverse_target_word_index[sampled_token_index]\n",
        "\n",
        "        if(sampled_token!='end'):\n",
        "            decoded_sentence += ' '+sampled_token\n",
        "\n",
        "            # Exit condition: either hit max length or find stop word.\n",
        "            if (sampled_token == 'end' or len(decoded_sentence.split()) >= (max_summary-1)):\n",
        "                stop_condition = True\n",
        "\n",
        "        # Update the target sequence (of length 1).\n",
        "        target_seq = np.zeros((1,1))\n",
        "        target_seq[0, 0] = sampled_token_index\n",
        "\n",
        "        # Update internal states\n",
        "        e_h, e_c = h, c\n",
        "\n",
        "    return decoded_sentence"
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Gj0xNjgqkhd"
      },
      "source": [
        "def seq2summary(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "      if((i!=0 and i!=target_word_index['start']) and i!=target_word_index['end']):\n",
        "        newString=newString+reverse_target_word_index[i]+' '\n",
        "    return newString\n",
        "\n",
        "def seq2text(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "      if(i!=0):\n",
        "        newString=newString+reverse_source_word_index[i]+' '\n",
        "    return newString"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2idO_03BJtRD",
        "outputId": "4c294d3b-f20e-4693-935c-d5e5f4e0bb24"
      },
      "source": [
        "i=1\n",
        "print(\"Review:\",seq2text(x_val[i]))\n",
        "print(\"Original summary:\",seq2summary(y_val[i]))\n",
        "print(\"Predicted summary:\",decode_sequence(x_val[i].reshape(1,max_text)))\n",
        "print(\"\\n\")"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Review: whenever want add bit cream soup sauce never seem around use cream often enough even buy smallest container stuff kept long time really add creamy flavor glad found \n",
            "Original summary: this stuff is great \n",
            "Predicted summary:  love this stuff\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ahqT4mCDqsix",
        "outputId": "8eac0e69-b10c-4261-a835-202052b88f3a"
      },
      "source": [
        "for i in range(10):\n",
        "  print(\"Review:\",seq2text(x_val[i]))\n",
        "  print(\"Original summary:\",seq2summary(y_val[i]))\n",
        "  print(\"Predicted summary:\",decode_sequence(x_val[i].reshape(1,max_text)))\n",
        "  print(\"\\n\")"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Review: love hot sauces rediculous thought like good hot sauce would good tacos nachos etc nice heat stuff stupid hot bought three pack regreted moment tried standard insanity sauce cannot even tolerate dashes tacos good \n",
            "Original summary: seriously \n",
            "Predicted summary:  good hot sauce\n",
            "\n",
            "\n",
            "Review: whenever want add bit cream soup sauce never seem around use cream often enough even buy smallest container stuff kept long time really add creamy flavor glad found \n",
            "Original summary: this stuff is great \n",
            "Predicted summary:  love this stuff\n",
            "\n",
            "\n",
            "Review: glad opportunity purchase last boxes amazing tea local herb stores tell sold anymore cannot sold unless ground powder form hope buy overseas batch runs \n",
            "Original summary: last of the best \n",
            "Predicted summary:  love this tea\n",
            "\n",
            "\n",
            "Review: tried peanut butter creme version first impression fudge coating looker rather soft would messy left warm day tasting overwhelming sweetness predominant aside sweetness flavor butterfinger reeses slight crunchiness cookie portion also made texturally similar butterfinger chocolate flavor present drowned sugar fudge coating seems chocolate flavored actual chocolate excess sweetness cookies makes hard eat one good thing considering nutritional information overall cookies much sweet taste like candy bar \n",
            "Original summary: too sweet \n",
            "Predicted summary:  not bad not great\n",
            "\n",
            "\n",
            "Review: love product kids drink instead sports drinks play soccer much healthier twice received amazon times product poorly packaged times padding cartons got crushed shipping cartons leak amazon responded well shipping replacement cartons still padding leaks would like continue buy amazon figure package cartons get crushed \n",
            "Original summary: poorly packaged \n",
            "Predicted summary:  great product poor packaging\n",
            "\n",
            "\n",
            "Review: delicious coffee ever drink hot cold drink cold litle cream needs booze \n",
            "Original summary: as good as it gets \n",
            "Predicted summary:  delicious\n",
            "\n",
            "\n",
            "Review: things work well trouble leave one machine overnight sticks hard get otherwise great product \n",
            "Original summary: pretty cool \n",
            "Predicted summary:  great idea\n",
            "\n",
            "\n",
            "Review: received product described minus one boxes received three four boxes cereal beware inept service \n",
            "Original summary: pack of four \n",
            "Predicted summary:  not what ordered\n",
            "\n",
            "\n",
            "Review: love chili mix like another reviewer said earlier use little half chili powder packet provided also use rotel instead tomato sauce water comes great give try \n",
            "Original summary: great chili mix \n",
            "Predicted summary:  great chili\n",
            "\n",
            "\n",
            "Review: tablet like teaspoon sugar sweetness need sweeten gallon gallon something adding pellets way easier opening bunch little packets even spooning sweetener pour pellets bottle cap count need whatever reason well kool aid brand soft drink mix mix well wyler flavor aid tastes find grain pellets sweetens quart soft drink packet well really feel like still getting aftertaste try pellets two packets splenda always add either nectasweet splenda prefer sweeter adding directly already chilled beverages give pellets minute dissolve stir well \n",
            "Original summary: nectasweet grain sugar substitute pellets \n",
            "Predicted summary:  it is not just for me\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yBT-lmo7u03N"
      },
      "source": [
        "# TESTING"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sjA6iLn0u0Z_"
      },
      "source": [
        "inptext= \"\"\n",
        "cleaned_inp = []\n",
        "cleaned_inp.append(text_cleaner(inptext))\n",
        "inp = x_tokenizer.texts_to_sequences(cleaned_inp) \n",
        "inp = pad_sequences(inp,  maxlen=max_text, padding='post') \n",
        "print(decode_sequence(inp.reshape(1,max_text)))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uWgmz3AbLxP8"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}