Abstractive_Text_Summarization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pgen_Pointer_Generator.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "QYoIaaQHTCaT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import random\n",
        "import torch.nn.functional as F\n",
        "from gensim.models import FastText\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kjYbmDGDg-Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WbowAMISDjHJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = pd.read_csv('/content/drive/My Drive/Amazon_Food/Reviews.csv')\n",
        "len(data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S7lqLjyLLDXa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = data['Text']\n",
        "y = data['Summary']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9BlfwPHtNXI9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def clean(text):\n",
        "  text = str(text)\n",
        "  text = text.lower()\n",
        "  text = re.sub(r'\\'s',r'\\tis',text)\n",
        "  text = re.sub(r'\\'ll',r'\\twill',text)\n",
        "  text = re.sub(r'\\'m',r'\\tam',text)\n",
        "  text = re.sub(r'\\'re',r'\\tare',text)\n",
        "  text = re.sub(r'\\'d',r'\\twould',text)\n",
        "  text = re.sub(r'n\\'t',r'\\tnot',text)\n",
        "  text = re.sub('[^a-zA-Z0-9]',' ',text) \n",
        "  text = re.sub(r\"[-()\\\"#/@;:<>{}`+=~|.!?,]\", \"\", text)\n",
        "  return text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "969v-EnfDlun",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cleaned_source = list(map(clean,x))\n",
        "cleaned_summary = list(map(clean,y))\n",
        "\n",
        "for i in range(len(cleaned_summary)):\n",
        "    cleaned_summary[i] = \"<START> \" + cleaned_summary[i] + \" <END>\"\n",
        "    \n",
        "print(cleaned_source[11])\n",
        "print(cleaned_summary[11])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0RUaNE15DpNK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "min_source_length = 1999999\n",
        "max_source_length = 0\n",
        "min_target_length = 199999\n",
        "max_target_length = 0\n",
        "\n",
        "for i in range(len(data)):\n",
        "  min_source_length = min(min_source_length,len(cleaned_source[i].split()))\n",
        "  min_target_length = min(min_target_length,len(cleaned_summary[i].split()))\n",
        "  max_source_length = max(max_source_length,len(cleaned_source[i].split()))\n",
        "  max_target_length = max(max_target_length,len(cleaned_summary[i].split()))\n",
        "\n",
        "print(\"Minimum source length is:  \",min_source_length)\n",
        "print(\"Minimum target length is:  \",min_target_length)\n",
        "print(\"Maximum source length is:  \",max_source_length)\n",
        "print(\"Maximum target length is:  \",max_target_length)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2bgI2vw5DsYw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "new_source = []\n",
        "new_summary = []\n",
        "\n",
        "for i in range(len(cleaned_source)):\n",
        "  if len(cleaned_source[i].split()) <= 50 and len(cleaned_summary[i].split()) <= 15 :\n",
        "    new_source.append(cleaned_source[i])\n",
        "    new_summary.append(cleaned_summary[i])\n",
        "\n",
        "max_source_length = 50\n",
        "max_summary_length = 15\n",
        "\n",
        "print(len(new_source))\n",
        "print(len(new_summary))\n",
        "\n",
        "new_source = new_source[:30000]\n",
        "new_summary = new_summary[:30000]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WkhEAcafDbFW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sentences = new_source + new_summary\n",
        "sent_ted = []\n",
        "for sent in sentences:\n",
        "  sent_ted_child = sent.split()\n",
        "  sent_ted.append(sent_ted_child)\n",
        "\n",
        "print(sent_ted[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ir5TQ3vGZxNe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from gensim.models import FastText\n",
        "# model_ted = FastText(sent_ted, size=128, window=3, min_count=1, workers=4,sg=1, iter=1500)\n",
        "\n",
        "# import pickle\n",
        "# pickle.dump(model_ted, open('128_emb_1lakhdata.pkl', 'wb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYIUn8RXDX15",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "model_ted = pickle.load(open('/content/drive/My Drive/128_emb.pkl', 'rb'))\n",
        "weights = model_ted.wv\n",
        "print(model_ted.wv.most_similar(\"milk\"))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "otf9wqD83OxL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import OrderedDict \n",
        "\n",
        "word2Index_enc = {}\n",
        "word2Index_dec = {}\n",
        "word2Index_dec_big = {}\n",
        "\n",
        "ind2Word_enc = {}\n",
        "ind2Word_dec = {}\n",
        "ind2Word_dec_big = {}\n",
        "\n",
        "word2PsuInd_dec = {}\n",
        "psuInd2Word_dec = {}\n",
        "\n",
        "encoder_paragraph = list(set((' '.join(new_source)).split()))\n",
        "\n",
        "decoder_paragraph_list = list((' '.join(new_summary)).split())\n",
        "decoder_dict = OrderedDict()\n",
        "for word in decoder_paragraph_list:\n",
        "  try:\n",
        "    decoder_dict[word] = decoder_dict[word] + 1\n",
        "  except:\n",
        "    decoder_dict[word] = 1\n",
        "\n",
        "ind2Word_enc[0] = '<UNK>'\n",
        "ind2Word_dec[0] = '<UNK>'\n",
        "word2Index_enc['<UNK>'] = 0\n",
        "word2Index_dec['<UNK>'] = 0\n",
        "ind2Word_dec_big[0] = '<UNK>'\n",
        "word2Index_dec_big['<UNK>'] = 0\n",
        "word2PsuInd_dec['<UNK>'] = 0\n",
        "psuInd2Word_dec[0] = '<UNK>'\n",
        "\n",
        "dec_index = 1\n",
        "for (decoder_dict_word, decoder_dict_number) in decoder_dict.items():\n",
        "  word2Index_dec_big[decoder_dict_word] = dec_index\n",
        "  ind2Word_dec_big[dec_index] = decoder_dict_word\n",
        "  if decoder_dict_number >= 3 :\n",
        "    word2Index_dec[decoder_dict_word] = dec_index\n",
        "    ind2Word_dec[dec_index] = decoder_dict_word\n",
        "    psuedo_index = len(word2PsuInd_dec.keys())\n",
        "    word2PsuInd_dec[decoder_dict_word] = psuedo_index\n",
        "    psuInd2Word_dec[psuedo_index] = decoder_dict_word\n",
        "  dec_index+=1\n",
        "\n",
        "enc_index = 1\n",
        "for index,word in enumerate(encoder_paragraph):\n",
        "  if word != ' ':\n",
        "    word2Index_enc[word] = enc_index\n",
        "    ind2Word_enc[enc_index] = word \n",
        "    enc_index+=1\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r9BR3MZbR5D5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoder_input = [[word2Index_enc[word] for word in sentence.split() if word in word2Index_enc.keys()] for sentence in new_source ]\n",
        "decoder_input = [[word2Index_dec_big[word] for word in sentence.split() if word in word2Index_dec_big.keys()] for sentence in new_summary ]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m1Xrg0kzCW5D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoder_tensor = [torch.tensor(li,dtype=torch.long,device=device).view(-1, 1) for li in encoder_input]\n",
        "decoder_tensor = [torch.tensor(li,dtype=torch.long,device=device).view(-1, 1) for li in decoder_input]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ErsV--5vShLH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "  def __init__(self,input_vocab_size,hidden_size,num_layers=1,bidirectional=False):\n",
        "    super(Encoder,self).__init__()\n",
        "    self.bidirectional = bidirectional\n",
        "    self.num_layers = num_layers\n",
        "    self.hidden_size = hidden_size\n",
        "    self.input_vocab_size = input_vocab_size\n",
        "    self.gru_layer = nn.GRU(input_size = self.hidden_size,hidden_size = self.hidden_size,num_layers = self.num_layers)\n",
        "\n",
        "  def forward(self,input_,prev_hidden_state):\n",
        "    input_word = ind2Word_enc[input_.data.tolist()[0]]\n",
        "    embedded_outputs = torch.tensor(weights[input_word], device = device).view(1,1,-1)\n",
        "    output,prev_hidden_state = self.gru_layer(embedded_outputs,prev_hidden_state)  #output is batch_size times hidden_size\n",
        "    return output,prev_hidden_state\n",
        "\n",
        "  def init_hidden(self):\n",
        "    return torch.zeros(1,1,self.hidden_size,device=device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I29Rw8CA7esY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class AttentionDecoder(nn.Module):\n",
        "  def __init__(self,output_vocab_size,hidden_size,max_length_encoder,dropout_value,num_layers=1):\n",
        "      super(AttentionDecoder,self).__init__()\n",
        "      self.hidden_size = hidden_size\n",
        "      self.num_layers = num_layers\n",
        "      self.output_vocab_size = output_vocab_size\n",
        "      self.dropout_p = dropout_value\n",
        "      self.max_length_encoder = max_length_encoder\n",
        "      self.embedding_layer = nn.Embedding(self.output_vocab_size,self.hidden_size)\n",
        "      self.attention_layer = nn.Linear(self.hidden_size*2,self.max_length_encoder)\n",
        "      self.attention_combine = nn.Linear(self.hidden_size*2,self.hidden_size)\n",
        "\n",
        "      self.s_layer = nn.Linear(self.hidden_size, 1)\n",
        "      self.x_layer = nn.Linear(self.hidden_size, 1)\n",
        "      self.context_layer = nn.Linear(self.hidden_size, 1)\n",
        "      self.linear_pgen = nn.Linear(3, 1)\n",
        "\n",
        "      self.gru_layer = nn.GRU(self.hidden_size,self.hidden_size)\n",
        "      self.output_layer = nn.Linear(self.hidden_size,self.output_vocab_size)\n",
        "      self.dropout_layer = nn.Dropout(self.dropout_p)    \n",
        "\n",
        "  def forward(self,input_,prev_hidden_state,encoder_output,prev_unk_word):\n",
        "      input_word = ind2Word_dec_big[input_.data.tolist()[0]]\n",
        "      if input_word == '<UNK>':\n",
        "        embedded_outputs = torch.tensor(weights[prev_unk_word], device = device).view(1,1,-1)\n",
        "      else:\n",
        "        embedded_outputs = torch.tensor(weights[input_word], device = device).view(1,1,-1)\n",
        "        \n",
        "      embeddings_dropout = self.dropout_layer(embedded_outputs)\n",
        "      attention_layer_output = self.attention_layer(torch.cat((embeddings_dropout[0],prev_hidden_state[0]),1))\n",
        "      attention_weights = nn.functional.softmax(attention_layer_output,dim=1)\n",
        "      attention_applied = torch.bmm(attention_weights.unsqueeze(0),encoder_output.unsqueeze(0))\n",
        "      attention_combine_logits = self.attention_combine(torch.cat((embeddings_dropout[0],attention_applied[0]),1)).unsqueeze(0)  #since gru requires a batch dimension\n",
        "      attention_combine_relu = nn.functional.relu(attention_combine_logits)\n",
        "\n",
        "      s_output = self.s_layer(prev_hidden_state[0])\n",
        "      x_output = self.x_layer(embeddings_dropout[0])\n",
        "      context = torch.flatten(attention_applied)\n",
        "      context_weights = self.context_layer(attention_applied)\n",
        "      sx = torch.cat((s_output[0],x_output[0]),0)\n",
        "      sxc = torch.cat((sx,context_weights[0][0]),0)\n",
        "      linear_pgen = self.linear_pgen(sxc)\n",
        "      m = nn.Sigmoid()\n",
        "      pgen = m(linear_pgen)\n",
        "\n",
        "      output,hidden = self.gru_layer(attention_combine_relu,prev_hidden_state)\n",
        "      output_logits = self.output_layer(output)\n",
        "      output_softmax = nn.functional.log_softmax(output_logits[0],dim=1)\n",
        "      return output_softmax,hidden,attention_weights,pgen\n",
        "\n",
        "  def init_hidden(self):\n",
        "    return torch.zeros(1,1,self.hidden_size,device=device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y40789vQM8gc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "teacher_forcing_ratio = 0.5\n",
        "\n",
        "def train(encoder, decoder, input_tensor, target_tensor, encoder_optimizer, decoder_optimizer, criterion, max_length, iters):\n",
        "\n",
        "  encoder_optimizer.zero_grad()\n",
        "  decoder_optimizer.zero_grad()\n",
        "\n",
        "  prev_unk_word = ''\n",
        "\n",
        "  encoder_hidden = encoder.init_hidden()\n",
        "\n",
        "  encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device = device)\n",
        "\n",
        "  input_length = input_tensor.size(0)\n",
        "  output_length = target_tensor.size(0)\n",
        "\n",
        "  loss = 0\n",
        "\n",
        "  for encoder_index in range(0, input_length):\n",
        "    encoder_output,encoder_hidden = encoder(input_tensor[encoder_index], encoder_hidden)\n",
        "    encoder_outputs[encoder_index] = encoder_output[0,0]\n",
        "\n",
        "  decoder_input = torch.tensor([word2Index_dec['<START>']],device=device)   \n",
        "  decoder_hidden = encoder_hidden\n",
        "  use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
        "\n",
        "  extended_vocab = psuInd2Word_dec.copy()\n",
        "  reverse_extended_vocab = word2PsuInd_dec.copy()\n",
        "  duplicate_words = {}\n",
        "  extend_key = len(word2Index_dec.keys())\n",
        "  input_list = input_tensor.tolist()\n",
        "  i =0\n",
        "  for input_word in input_list:\n",
        "    if ind2Word_enc[input_word[0]] in word2Index_dec.keys():\n",
        "      duplicate_words[i] = word2PsuInd_dec[ind2Word_enc[input_word[0]]]\n",
        "    else:\n",
        "      extended_vocab[extend_key] = ind2Word_enc[input_word[0]]\n",
        "      reverse_extended_vocab[ind2Word_enc[input_word[0]]] = extend_key\n",
        "      extend_key += 1\n",
        "    i = i+1\n",
        "\n",
        "  if use_teacher_forcing:\n",
        "    for decoder_index in range(output_length):\n",
        "      decoder_output,decoder_hidden,decoder_attention,pgen = decoder(decoder_input,decoder_hidden,encoder_outputs,prev_unk_word)\n",
        "      P_over_extended_vocab = torch.exp(decoder_output)*pgen.expand_as(torch.exp(decoder_output))\n",
        "\n",
        "      decoder_attention = decoder_attention.squeeze(0)[0:input_length].unsqueeze(0)\n",
        "      p_duplicate_list = torch.zeros([input_length, P_over_extended_vocab.size(1)], device=device)\n",
        "      p_duplicate_list = p_duplicate_list.tolist()\n",
        "      for (duplicate_word_key,duplicate_word_value) in duplicate_words.items():\n",
        "        p_duplicate_list[duplicate_word_key][duplicate_word_value] = 1\n",
        "      p_duplicate = torch.tensor(p_duplicate_list, dtype=torch.float, device=device)\n",
        "      p_diag = torch.mm(decoder_attention, p_duplicate)\n",
        "      p_diag = p_diag*(torch.tensor([1], device=device).sub(pgen)).expand_as(p_diag)\n",
        "      p_add_diag = torch.diag(p_diag.squeeze(0),diagonal=0)\n",
        "      P_over_extended_vocab = torch.mm(P_over_extended_vocab,p_add_diag).add(P_over_extended_vocab)\n",
        "\n",
        "      for i in range(input_length):\n",
        "        if not (1 in p_duplicate_list[i]):\n",
        "          P_over_extended_vocab = torch.cat((P_over_extended_vocab[0], torch.mm(decoder_attention.squeeze(0)[i].unsqueeze(0).unsqueeze(0), torch.tensor([1], device=device).sub(pgen).unsqueeze(0)).squeeze(0)),0).unsqueeze(0)\n",
        "\n",
        "      try:\n",
        "        loss += -torch.log(P_over_extended_vocab[0][ reverse_extended_vocab[ ind2Word_dec_big[ target_tensor[decoder_index].item() ] ] ] + 1e-12)\n",
        "        loss.backward(retain_graph=True)\n",
        "      except KeyError:\n",
        "        loss += torch.tensor(0,dtype=torch.float,device=device)\n",
        "      decoder_input = target_tensor[decoder_index]\n",
        "  else:\n",
        "\n",
        "    for decoder_index in range(output_length):\n",
        "      decoder_output,decoder_hidden,decoder_attention,pgen = decoder(decoder_input,decoder_hidden,encoder_outputs,prev_unk_word) \n",
        "      P_over_extended_vocab = torch.exp(decoder_output)*pgen.expand_as(torch.exp(decoder_output))\n",
        "\n",
        "      decoder_attention = decoder_attention.squeeze(0)[0:input_length].unsqueeze(0)\n",
        "      p_duplicate_list = torch.zeros([input_length, P_over_extended_vocab.size(1)], device=device)\n",
        "      p_duplicate_list = p_duplicate_list.tolist()\n",
        "      for (duplicate_word_key,duplicate_word_value) in duplicate_words.items():\n",
        "        p_duplicate_list[duplicate_word_key][duplicate_word_value] = 1\n",
        "      p_duplicate = torch.tensor(p_duplicate_list, dtype=torch.float, device=device)\n",
        "      p_diag = torch.mm(decoder_attention, p_duplicate)\n",
        "      p_diag = p_diag*(torch.tensor([1], device=device).sub(pgen)).expand_as(p_diag)\n",
        "      p_add_diag = torch.diag(p_diag.squeeze(0),diagonal=0)\n",
        "      P_over_extended_vocab = torch.mm(P_over_extended_vocab,p_add_diag).add(P_over_extended_vocab)\n",
        "\n",
        "      for i in range(input_length):\n",
        "        if not (1 in p_duplicate_list[i]):\n",
        "          P_over_extended_vocab = torch.cat((P_over_extended_vocab[0], torch.mm(decoder_attention.squeeze(0)[i].unsqueeze(0).unsqueeze(0), torch.tensor([1], device=device).sub(pgen).unsqueeze(0)).squeeze(0)),0).unsqueeze(0)\n",
        "\n",
        "      try:\n",
        "        loss += -torch.log(P_over_extended_vocab[0][ reverse_extended_vocab[ ind2Word_dec_big[ target_tensor[decoder_index].item() ] ] ] + 1e-12)\n",
        "        loss.backward(retain_graph=True)\n",
        "      except KeyError:\n",
        "        loss += torch.tensor(0,dtype=torch.float,device=device)\n",
        "      idx = torch.topk(P_over_extended_vocab, k=1, dim=1)[1]\n",
        "      if idx.item() < len(word2Index_dec.keys()):   \n",
        "        decoder_input = torch.tensor([idx.item()],dtype=torch.long,device=device)\n",
        "      elif idx.item() >= len(word2Index_dec.keys()):\n",
        "        prev_unk_word = extended_vocab[idx.item()]\n",
        "        decoder_input = torch.tensor([0],dtype=torch.long,device=device)\n",
        "      elif (decoder_input.item() == word2Index_dec['<END>']):\n",
        "        break\n",
        "\n",
        "  if iters > 20000:\n",
        "    torch.nn.utils.clip_grad_norm_(rnn_encoder.parameters(),0.4)\n",
        "    torch.nn.utils.clip_grad_norm_(rnn_decoder.parameters(),0.4)\n",
        "\n",
        "  encoder_optimizer.step()\n",
        "  decoder_optimizer.step()\n",
        "\n",
        "  return loss.item()/output_length"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ocM_vNF6KTWW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import math\n",
        "\n",
        "\n",
        "def asMinutes(s):\n",
        "    m = math.floor(s / 60)\n",
        "    s -= m * 60\n",
        "    return '%dm %ds' % (m, s)\n",
        "\n",
        "\n",
        "def timeSince(since, percent):\n",
        "    now = time.time()\n",
        "    s = now - since\n",
        "    if percent != 0:\n",
        "      es = s / (percent)\n",
        "      rs = es - s\n",
        "      return '%s (- %s)' % (asMinutes(s), asMinutes(rs))\n",
        "    else:\n",
        "      return 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQnTrJ0SLSYq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "arr = np.arange(len(encoder_tensor))\n",
        "np.random.shuffle(arr)\n",
        "len(arr)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pBToTQxGLXgI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Dictionary for creating loss graph\n",
        "loss_graph = {}\n",
        "\n",
        "def train_Iters(encoder,decoder,n_iters,print_every=50, plot_every=100,learning_rate = 0.03):\n",
        "  # start = time.time()\n",
        "  plot_losses = []\n",
        "  print_loss_total = 0  # Reset every print_every\n",
        "  plot_loss_total = 0\n",
        "\n",
        "  encoder_optimizer = torch.optim.SGD(encoder.parameters(), lr=learning_rate)\n",
        "  decoder_optimizer = torch.optim.SGD(decoder.parameters(), lr=learning_rate)\n",
        "  training_pairs = [random.choice(pairs) for i in range(n_iters)]\n",
        "  \n",
        "  criterion = nn.NLLLoss()\n",
        "  for iters in range(n_iters):\n",
        "    training_pair = training_pairs[iters - 1]\n",
        "    input_tensor = training_pair[0]\n",
        "    target_tensor = training_pair[1]\n",
        "\n",
        "    input_tensor = torch.tensor(input_tensor, dtype=torch.long, device = device).view(-1, 1)\n",
        "    target_tensor = torch.tensor(target_tensor, dtype=torch.long, device = device).view(-1, 1)\n",
        "\n",
        "    loss = train(encoder,decoder,input_tensor,target_tensor,encoder_optimizer,decoder_optimizer,criterion,max_source_length, iters)\n",
        "    print_loss_total += loss\n",
        "    plot_loss_total += loss\n",
        "\n",
        "    if iters % print_every == 0:\n",
        "        print_loss_avg = print_loss_total / print_every\n",
        "        print_loss_total = 0\n",
        "        print('%s %d%%) %.4f' % (iters, iters / len(arr) * 100, print_loss_avg))\n",
        "\n",
        "        if iters > 0:\n",
        "          loss_graph[iters] = print_loss_avg\n",
        "\n",
        "    if iters % plot_every == 0:\n",
        "        plot_loss_avg = plot_loss_total / plot_every\n",
        "        plot_losses.append(plot_loss_avg)\n",
        "        plot_loss_total = 0\n",
        "\n",
        "  # showPlot(plot_losses)\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mDEs3lSzLcrt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.switch_backend('agg')\n",
        "import matplotlib.ticker as ticker\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "def showPlot(points):\n",
        "    plt.figure()\n",
        "    fig, ax = plt.subplots()\n",
        "    # this locator puts ticks at regular intervals\n",
        "    loc = ticker.MultipleLocator(base=0.2)\n",
        "    ax.yaxis.set_major_locator(loc)\n",
        "    plt.plot(points)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jgo_fbnFLgxE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pairs = []\n",
        "for enc,dec in zip(encoder_input,decoder_input):\n",
        "    pairs.append([enc,dec])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uAJAoFoJDLrm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "hidden_size = 128\n",
        "rnn_encoder = Encoder(len(word2Index_enc.keys()),hidden_size).to(device=device)\n",
        "rnn_decoder = AttentionDecoder(len(word2Index_dec.keys()),hidden_size,max_source_length,0.2).to(device=device)\n",
        "\n",
        "train_Iters(rnn_encoder,rnn_decoder,100000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5IbiBvMaDO3k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "iters = list(loss_graph.keys())\n",
        "loss_val = list(loss_graph.values())\n",
        "plt.plot(iters, loss_val)\n",
        "plt.ylim(0,8) \n",
        "plt.xlim(0,77350)\n",
        "plt.xlabel('Iterations') \n",
        "plt.ylabel('Loss')  \n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHA9RnrZgk6j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluateRandomly(encoder, decoder, n=10):\n",
        "    for i in range(n):\n",
        "        pair = random.choice(pairs)\n",
        "        pair1 = torch.tensor(pair[0],dtype=torch.long,device=device)\n",
        "        pair2 = pair[1]\n",
        "        output_words, attentions = evaluate(encoder, decoder, pair1)\n",
        "        output_sentence = ' '.join(output_words)\n",
        "        output_list = [ind2Word_dec_big[word] for word in pair2]\n",
        "        output_list = ' '.join(output_list)\n",
        "        input_sentence = [ind2Word_enc[element.item()] for element in pair1.flatten()]\n",
        "        input_sentence = ' '.join(input_sentence)\n",
        "        print(\"Sentence is  \",input_sentence)\n",
        "        print('<',output_sentence)\n",
        "        print('=',output_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ObARxnAyoUTA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(encoder, decoder, encoder_tensor, max_length=max_source_length):\n",
        "    with torch.no_grad():\n",
        "        input_tensor = encoder_tensor\n",
        "        input_length = input_tensor.size(0)\n",
        "        encoder_hidden = encoder.init_hidden()\n",
        "\n",
        "        prev_unk_word = ''\n",
        "\n",
        "        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
        "\n",
        "        for ei in range(input_length):\n",
        "            encoder_output, encoder_hidden = encoder(input_tensor[ei].unsqueeze(0),\n",
        "                                                     encoder_hidden)\n",
        "            encoder_outputs[ei] += encoder_output[0, 0]\n",
        "\n",
        "        extended_vocab = psuInd2Word_dec.copy()\n",
        "        duplicate_words = {}\n",
        "        extend_key = len(word2Index_dec.keys())\n",
        "        input_list = input_tensor.tolist()\n",
        "        i =0\n",
        "        for input_word in input_list:\n",
        "          if ind2Word_enc[input_word] in word2Index_dec.keys():\n",
        "            duplicate_words[i] = word2PsuInd_dec[ind2Word_enc[input_word]]\n",
        "          else:\n",
        "            extended_vocab[extend_key] = ind2Word_enc[input_word]\n",
        "            extend_key += 1\n",
        "          i = i+1\n",
        "\n",
        "        decoder_input = torch.tensor([word2Index_dec['<START>']], device=device)  # SOS\n",
        "\n",
        "        decoder_hidden = encoder_hidden\n",
        "\n",
        "        decoded_words = []\n",
        "        decoder_attentions = torch.zeros(max_length, max_length)\n",
        "\n",
        "        for di in range(max_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention,pgen = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs, prev_unk_word)\n",
        "            decoder_attentions[di] = decoder_attention.data\n",
        "\n",
        "            P_over_extended_vocab = torch.exp(decoder_output)*pgen.expand_as(torch.exp(decoder_output))\n",
        "\n",
        "            decoder_attention = decoder_attention.squeeze(0)[0:input_length].unsqueeze(0)\n",
        "            p_duplicate_list = torch.zeros([input_length, P_over_extended_vocab.size(1)], device=device)\n",
        "            p_duplicate_list = p_duplicate_list.tolist()\n",
        "            for (duplicate_word_key,duplicate_word_value) in duplicate_words.items():\n",
        "              p_duplicate_list[duplicate_word_key][duplicate_word_value] = 1\n",
        "            p_duplicate = torch.tensor(p_duplicate_list, dtype=torch.float, device=device)\n",
        "            p_diag = torch.mm(decoder_attention, p_duplicate)\n",
        "            p_diag = p_diag*(torch.tensor([1], device=device).sub(pgen)).expand_as(p_diag)\n",
        "            p_add_diag = torch.diag(p_diag.squeeze(0),diagonal=0)\n",
        "            P_over_extended_vocab = torch.mm(P_over_extended_vocab,p_add_diag).add(P_over_extended_vocab)\n",
        "\n",
        "            for i in range(input_length):\n",
        "              if not (1 in p_duplicate_list[i]):\n",
        "                P_over_extended_vocab = torch.cat((P_over_extended_vocab[0], torch.mm(decoder_attention.squeeze(0)[i].unsqueeze(0).unsqueeze(0), torch.tensor([1], device=device).sub(pgen).unsqueeze(0)).squeeze(0)),0).unsqueeze(0)\n",
        "\n",
        "            idx = torch.topk(P_over_extended_vocab, k=1, dim=1)[1]\n",
        "            if idx.item() < len(word2Index_dec.keys()):   \n",
        "              decoder_input = torch.tensor([idx.item()],dtype=torch.long,device=device)\n",
        "              decoded_words.append(extended_vocab[idx.item()])\n",
        "            elif idx.item() >= len(word2Index_dec.keys()):\n",
        "              decoder_input = torch.tensor([0],dtype=torch.long,device=device)\n",
        "              prev_unk_word = extended_vocab[idx.item()]\n",
        "              decoded_words.append(extended_vocab[idx.item()])\n",
        "            if idx.item() == word2Index_dec['<END>']:\n",
        "              decoded_words.append('<END>')\n",
        "              break\n",
        "\n",
        "        return decoded_words, decoder_attentions[:di + 1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oYYHCcO04vV5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "evaluateRandomly(rnn_encoder, rnn_decoder)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}