Capstone_project_merged.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Capstone_project_merged (1) (10).ipynb",
      "provenance": [],
      "collapsed_sections": [
        "qXYT8-xs7V_0",
        "UHy8slsutBCc",
        "6ZZ6G3pmsq-P",
        "s_sWrANJ15co",
        "GapK4sFicyYq",
        "qfME8DsPslXa",
        "K8yleaDI8MJf",
        "cbe5SYkGiSl5",
        "R_LIVwsfcXVA"
      ],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "AoIQFdjR-Lid"
      },
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "oDXUu5iW-Lij"
      },
      "source": [
        "Load Python modules."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L5D9Go4puFrY",
        "colab_type": "code",
        "outputId": "c3f9d78a-7cd3-4f4e-f7c5-db0d038c1e5f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip3 install symspellpy\n",
        "!pip3 install keras_bert\n",
        "!pip3 install wordcloud\n",
        "!wget -nc https://github.com/moonman239/Capstone-project/raw/master/data.zip -O data.zip\n",
        "!unzip data.zip\n",
        "!pip3 install -q keras-bert\n",
        "!wget -q https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip\n",
        "!unzip -o uncased_L-12_H-768_A-12.zip"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting symspellpy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6d/0b/2daa14bf1ed649fff0d072b2e51ae98d8b45cae6cf8fdda41be01ce6c289/symspellpy-6.5.2-py3-none-any.whl (2.6MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6MB 2.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.13.1 in /usr/local/lib/python3.6/dist-packages (from symspellpy) (1.17.5)\n",
            "Installing collected packages: symspellpy\n",
            "Successfully installed symspellpy-6.5.2\n",
            "Collecting keras_bert\n",
            "  Downloading https://files.pythonhosted.org/packages/df/fe/bf46de1ef9d1395cd735d8df5402f5d837ef82cfd348a252ad8f32feeaef/keras-bert-0.80.0.tar.gz\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from keras_bert) (1.17.5)\n",
            "Requirement already satisfied: Keras in /usr/local/lib/python3.6/dist-packages (from keras_bert) (2.2.5)\n",
            "Collecting keras-transformer>=0.30.0\n",
            "  Downloading https://files.pythonhosted.org/packages/0a/57/496b1eab888171b0801a0a44d3245a7874b8d1cc04c1fbfdbb5e3327fc7a/keras-transformer-0.31.0.tar.gz\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from Keras->keras_bert) (1.12.0)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from Keras->keras_bert) (1.4.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from Keras->keras_bert) (2.8.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from Keras->keras_bert) (1.0.8)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from Keras->keras_bert) (3.13)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from Keras->keras_bert) (1.1.0)\n",
            "Collecting keras-pos-embd>=0.10.0\n",
            "  Downloading https://files.pythonhosted.org/packages/09/70/b63ed8fc660da2bb6ae29b9895401c628da5740c048c190b5d7107cadd02/keras-pos-embd-0.11.0.tar.gz\n",
            "Collecting keras-multi-head>=0.22.0\n",
            "  Downloading https://files.pythonhosted.org/packages/40/3e/d0a64bb2ac5217928effe4507c26bbd19b86145d16a1948bc2d4f4c6338a/keras-multi-head-0.22.0.tar.gz\n",
            "Collecting keras-layer-normalization>=0.12.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a4/0e/d1078df0494bac9ce1a67954e5380b6e7569668f0f3b50a9531c62c1fc4a/keras-layer-normalization-0.14.0.tar.gz\n",
            "Collecting keras-position-wise-feed-forward>=0.5.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e3/59/f0faa1037c033059e7e9e7758e6c23b4d1c0772cd48de14c4b6fd4033ad5/keras-position-wise-feed-forward-0.6.0.tar.gz\n",
            "Collecting keras-embed-sim>=0.7.0\n",
            "  Downloading https://files.pythonhosted.org/packages/bc/20/735fd53f6896e2af63af47e212601c1b8a7a80d00b6126c388c9d1233892/keras-embed-sim-0.7.0.tar.gz\n",
            "Collecting keras-self-attention==0.41.0\n",
            "  Downloading https://files.pythonhosted.org/packages/1b/1c/01599219bef7266fa43b3316e4f55bcb487734d3bafdc60ffd564f3cfe29/keras-self-attention-0.41.0.tar.gz\n",
            "Building wheels for collected packages: keras-bert, keras-transformer, keras-pos-embd, keras-multi-head, keras-layer-normalization, keras-position-wise-feed-forward, keras-embed-sim, keras-self-attention\n",
            "  Building wheel for keras-bert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-bert: filename=keras_bert-0.80.0-cp36-none-any.whl size=37923 sha256=db60d302622fbb447753ed635124054caf294e08141fe9008ca400098524033a\n",
            "  Stored in directory: /root/.cache/pip/wheels/63/dc/87/3260cb91f3aa32c0f85c5375429a30c8fd988bbb48f5ee21b0\n",
            "  Building wheel for keras-transformer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-transformer: filename=keras_transformer-0.31.0-cp36-none-any.whl size=13385 sha256=21c68aa9a70b269afdeebe2a9ab8692f40a8630bc948f6866ba6856b302d237f\n",
            "  Stored in directory: /root/.cache/pip/wheels/a3/c5/9a/5a5130240be614a7a6fa786765d7692ae97f82601e2161bb56\n",
            "  Building wheel for keras-pos-embd (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-pos-embd: filename=keras_pos_embd-0.11.0-cp36-none-any.whl size=7553 sha256=91abf568e22260ea57bea0d995466e40a79a4657811eef1a4140649039c750a6\n",
            "  Stored in directory: /root/.cache/pip/wheels/5b/a1/a0/ce6b1d49ba1a9a76f592e70cf297b05c96bc9f418146761032\n",
            "  Building wheel for keras-multi-head (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-multi-head: filename=keras_multi_head-0.22.0-cp36-none-any.whl size=15371 sha256=90c504f0b417bac05768b1bfa405de89c6642758216e649c1dc180cce5bb8516\n",
            "  Stored in directory: /root/.cache/pip/wheels/bb/df/3f/81b36f41b66e6a9cd69224c70a737de2bb6b2f7feb3272c25e\n",
            "  Building wheel for keras-layer-normalization (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-layer-normalization: filename=keras_layer_normalization-0.14.0-cp36-none-any.whl size=5268 sha256=a76117400629c8271b5e913f25171ebd22c1f81070d7dedb127c80964b593a96\n",
            "  Stored in directory: /root/.cache/pip/wheels/54/80/22/a638a7d406fd155e507aa33d703e3fa2612b9eb7bb4f4fe667\n",
            "  Building wheel for keras-position-wise-feed-forward (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-position-wise-feed-forward: filename=keras_position_wise_feed_forward-0.6.0-cp36-none-any.whl size=5624 sha256=d66c84f228f27c4d639a1c30b7281963c13574b69caa51bb73c1007548d3f2ab\n",
            "  Stored in directory: /root/.cache/pip/wheels/39/e2/e2/3514fef126a00574b13bc0b9e23891800158df3a3c19c96e3b\n",
            "  Building wheel for keras-embed-sim (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-embed-sim: filename=keras_embed_sim-0.7.0-cp36-none-any.whl size=4676 sha256=25828509b8ec72c6622dc5e7498aba426851656c8d4acc2fc417d4b0de856e85\n",
            "  Stored in directory: /root/.cache/pip/wheels/d1/bc/b1/b0c45cee4ca2e6c86586b0218ffafe7f0703c6d07fdf049866\n",
            "  Building wheel for keras-self-attention (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-self-attention: filename=keras_self_attention-0.41.0-cp36-none-any.whl size=17290 sha256=ba94546670589ec4168b107d4f1428be79d4bb3e68b547105878b873c82f1aa3\n",
            "  Stored in directory: /root/.cache/pip/wheels/cc/dc/17/84258b27a04cd38ac91998abe148203720ca696186635db694\n",
            "Successfully built keras-bert keras-transformer keras-pos-embd keras-multi-head keras-layer-normalization keras-position-wise-feed-forward keras-embed-sim keras-self-attention\n",
            "Installing collected packages: keras-pos-embd, keras-self-attention, keras-multi-head, keras-layer-normalization, keras-position-wise-feed-forward, keras-embed-sim, keras-transformer, keras-bert\n",
            "Successfully installed keras-bert-0.80.0 keras-embed-sim-0.7.0 keras-layer-normalization-0.14.0 keras-multi-head-0.22.0 keras-pos-embd-0.11.0 keras-position-wise-feed-forward-0.6.0 keras-self-attention-0.41.0 keras-transformer-0.31.0\n",
            "Requirement already satisfied: wordcloud in /usr/local/lib/python3.6/dist-packages (1.5.0)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.6/dist-packages (from wordcloud) (6.2.2)\n",
            "Requirement already satisfied: numpy>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from wordcloud) (1.17.5)\n",
            "--2020-01-10 04:03:08--  https://github.com/moonman239/Capstone-project/raw/master/data.zip\n",
            "Resolving github.com (github.com)... 140.82.113.3\n",
            "Connecting to github.com (github.com)|140.82.113.3|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/moonman239/Capstone-project/master/data.zip [following]\n",
            "--2020-01-10 04:03:09--  https://raw.githubusercontent.com/moonman239/Capstone-project/master/data.zip\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 9048060 (8.6M) [application/zip]\n",
            "Saving to: ‘data.zip’\n",
            "\n",
            "data.zip            100%[===================>]   8.63M  24.6MB/s    in 0.4s    \n",
            "\n",
            "2020-01-10 04:03:11 (24.6 MB/s) - ‘data.zip’ saved [9048060/9048060]\n",
            "\n",
            "Archive:  data.zip\n",
            "  inflating: dev-v1.1.json           \n",
            "   creating: __MACOSX/\n",
            "  inflating: __MACOSX/._dev-v1.1.json  \n",
            "  inflating: train-v1.1.json         \n",
            "  inflating: __MACOSX/._train-v1.1.json  \n",
            "Archive:  uncased_L-12_H-768_A-12.zip\n",
            "   creating: uncased_L-12_H-768_A-12/\n",
            "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.meta  \n",
            "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.data-00000-of-00001  \n",
            "  inflating: uncased_L-12_H-768_A-12/vocab.txt  \n",
            "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.index  \n",
            "  inflating: uncased_L-12_H-768_A-12/bert_config.json  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "TGcmXwG--Lio",
        "outputId": "5a455b2d-962d-44e7-eb6d-6a09e8a509bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 131
        }
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os\n",
        "# TF_KERAS must be added to environment variables in order to use TPU\n",
        "os.environ['TF_KERAS'] = '1'\n",
        "from tensorflow.python.client import device_lib\n",
        "import tensorflow.compat.v1 as tf\n",
        "#import keras\n",
        "from tensorflow.keras.layers import Embedding,Dropout,Lambda,Dense,Input,InputLayer,LSTM,Concatenate,Flatten,Add,Reshape,GlobalAveragePooling1D,GlobalAveragePooling2D\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras import Model,Sequential\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "import sys\n",
        "assert sys.version_info[0] >= 3\n",
        "from keras.backend import slice\n",
        "print(tf.VERSION)\n",
        "print(tf.keras.__version__)\n",
        "print(\"Modules loaded!\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "1.15.0\n",
            "2.2.4-tf\n",
            "Modules loaded!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sHQNCxJRYUil",
        "outputId": "7154eb33-8b6a-4705-c983-004cde7a9105",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "import sys\n",
        "print(sys.version)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3.6.9 (default, Nov  7 2019, 10:44:02) \n",
            "[GCC 8.3.0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "vXFuFons-LjA"
      },
      "source": [
        "# Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "yqjrxOes-LjD"
      },
      "source": [
        "For the preprocessing step, we will create two Pandas DataFrames - one for the training data, and another for the test (dev) data."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Vi34A02gtoNj"
      },
      "source": [
        "## Data preprocessing techniques"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "b0Rs8tXfuAit"
      },
      "source": [
        "In order to ensure the invariance of the text, I will need to train my deep & LSTM neural networks on lower-cased data, with punctuation removed if feasibly possible."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "KRh4IJGatrO6"
      },
      "source": [
        "Natural language processing suffers from a dearth of data. I have heard that a useful technique for overcoming this problem is to create inverted data. For example, given sentence \"I used to live all over Europe. In fact, I spent a significant amount of time in France, and now I speak fluent French.\", we might also add the sentence \"French fluent speak I now and France in time of amount significant a spent I fact in Europe over all live to used I\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "B8UbZDe--LjG"
      },
      "source": [
        "### Loading JSON datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "8fgU382EF26d",
        "colab": {}
      },
      "source": [
        "import json\n",
        "import re\n",
        "regex = re.compile(r'\\W+')\n",
        "def readFile(filename):\n",
        "  with open(filename) as file:\n",
        "    fields = []\n",
        "    JSON = json.loads(file.read())\n",
        "    articles = []\n",
        "    for article in JSON[\"data\"]:\n",
        "      articleTitle = article[\"title\"]\n",
        "      article_body = []\n",
        "      for paragraph in article[\"paragraphs\"]:\n",
        "        paragraphContext = paragraph[\"context\"]\n",
        "        article_body.append(paragraphContext)\n",
        "        for qas in paragraph[\"qas\"]:\n",
        "          question = qas[\"question\"]\n",
        "          answer = qas[\"answers\"][0]\n",
        "          fields.append({\"question\":question,\"answer_text\":answer[\"text\"],\"answer_start\":answer[\"answer_start\"],\"paragraph_context\":paragraphContext,\"article_title\":articleTitle})\n",
        "      article_body = \"\\\\n\".join(article_body)\n",
        "      article = {\"title\":articleTitle,\"body\":article_body}\n",
        "      articles.append(article)\n",
        "  fields = pd.DataFrame(fields)\n",
        "  #Remove punctuation.\n",
        "  fields[\"question\"] = fields[\"question\"].str.replace(regex,\" \")\n",
        "  assert not (fields[\"question\"].str.contains(\"catalanswhat\").any())\n",
        "  fields[\"paragraph_context\"] = fields[\"paragraph_context\"].str.replace(regex,\" \")\n",
        "  fields[\"answer_text\"] = fields[\"answer_text\"].str.replace(regex,\" \")\n",
        "  assert not (fields[\"paragraph_context\"].str.contains(\"catalanswhat\").any())\n",
        "  fields[\"article_title\"] = fields[\"article_title\"].str.replace(\"_\",\" \")\n",
        "  assert not (fields[\"article_title\"].str.contains(\"catalanswhat\").any())\n",
        "  return fields\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "H1Ql0r8fOpCB",
        "colab": {}
      },
      "source": [
        "trainingData = readFile(\"train-v1.1.json\")\n",
        "devData = readFile(\"dev-v1.1.json\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jmJKM9UJ78Je",
        "colab_type": "text"
      },
      "source": [
        "Summary statistics (credit to thushv89):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B1InLWrx7_ah",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "5033b317-d143-4382-b624-3530807fab22"
      },
      "source": [
        "pd.Series(trainingData[\"question\"]).str.split(' ').str.len().describe()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    87599.000000\n",
              "mean        11.217582\n",
              "std          3.597356\n",
              "min          1.000000\n",
              "25%          9.000000\n",
              "50%         11.000000\n",
              "75%         13.000000\n",
              "max         41.000000\n",
              "Name: question, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OMm42Yh28pkU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "ecaafe53-0672-4c6a-8047-10e25d8c2e38"
      },
      "source": [
        "pd.Series(trainingData[\"paragraph_context\"]).str.split(' ').str.len().describe()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    87599.000000\n",
              "mean       123.791653\n",
              "std         50.541385\n",
              "min         21.000000\n",
              "25%         92.000000\n",
              "50%        114.000000\n",
              "75%        147.000000\n",
              "max        678.000000\n",
              "Name: paragraph_context, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BVrdDCwit_sn",
        "colab_type": "text"
      },
      "source": [
        "## Punctuation check/removal"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NSnUkyYat_so",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from string import punctuation\n",
        "def punctuationCheck(series):\n",
        "    for string in series:\n",
        "        if any(p in punctuation for p in string):\n",
        "            print(string + \"\\n\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gHlUGDVvt_sr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "punctuationCheck(trainingData[\"question\"])\n",
        "punctuationCheck(devData[\"question\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TsH5TJTst_st",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "punctuationCheck(trainingData[\"paragraph_context\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rZvNQyLOt_sw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "punctuationCheck(trainingData[\"article_title\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Y4qc0cW-J0tY"
      },
      "source": [
        "### Convert strings to lowercase."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "iaaUpdrs_S19",
        "colab": {}
      },
      "source": [
        "#@title\n",
        "def lowercase(data):\n",
        "  data[\"question\"] = data[\"question\"].str.lower()\n",
        "  data[\"article_title\"] = data[\"article_title\"].str.lower()\n",
        "  data[\"paragraph_context\"] = data[\"paragraph_context\"].str.lower()\n",
        "  data[\"answer_text\"] = data[\"answer_text\"].str.lower()\n",
        "  return data\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "nhKonZ1qvY8U",
        "colab": {}
      },
      "source": [
        "trainingData = lowercase(trainingData)\n",
        "devData = lowercase(devData)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "S0ExErvoyIxr"
      },
      "source": [
        "## Create more data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-OEfpBzzyKrT",
        "colab": {}
      },
      "source": [
        "def invert_string(string):\n",
        "\tstring_array = string.split(\" \")\n",
        "\tstring_array.reverse()\n",
        "\treturn \" \".join(string_array)\n",
        "invert_string = np.vectorize(invert_string)\n",
        "def invert_series_of_strings(series):\n",
        "  return pd.Series(invert_string(series.values))\n",
        "def createMoreData(dataFrame):\n",
        "\tnewDataFrameQuestions = dataFrame[\"question\"].append(invert_series_of_strings(dataFrame[\"question\"]),ignore_index=True)\n",
        "\tnewDataFrameArticleTitles = dataFrame[\"article_title\"].append(invert_series_of_strings(dataFrame[\"article_title\"]),ignore_index=True)\n",
        "\tnewDataFrameParagraphContexts = dataFrame[\"paragraph_context\"].append(dataFrame[\"paragraph_context\"],ignore_index=True)\n",
        "\tnewDataFrameAnswerStarts = dataFrame[\"answer_start\"].append(dataFrame[\"answer_start\"],ignore_index=True)\n",
        "\treturn pd.DataFrame(data={\"question\":newDataFrameQuestions,\"paragraph_context\":newDataFrameParagraphContexts,\"article_title\":newDataFrameArticleTitles,\"answer_start\":newDataFrameAnswerStarts})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UtuXWXT-yMFM",
        "colab": {}
      },
      "source": [
        "trainingData = createMoreData(trainingData)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lxzQt_7UsFGy",
        "colab": {}
      },
      "source": [
        "devData = createMoreData(devData)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "p21XVUncj4so",
        "outputId": "850803d5-607d-4cba-d385-43d39a46f4f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(trainingData.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(175198, 4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "fxvoIc_jvMIN"
      },
      "source": [
        "## Convert answer_starts to numeric."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "oINap6sjvezH",
        "colab": {}
      },
      "source": [
        "trainingData[\"answer_start\"] = pd.to_numeric(trainingData[\"answer_start\"])\n",
        "devData[\"answer_start\"] = pd.to_numeric(devData[\"answer_start\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "8YCm87G4ux4M"
      },
      "source": [
        "## Dataset output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "XufEvRe0ap_R",
        "colab": {}
      },
      "source": [
        "trainingData[[\"question\",\"paragraph_context\",\"article_title\",\"answer_start\"]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "h4Gn1pjgV30v",
        "colab": {}
      },
      "source": [
        "devData"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "r2OI4bXYmGlT",
        "colab": {}
      },
      "source": [
        "print(trainingData.dtypes)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "-l8jQ_mb-Lje",
        "scrolled": true,
        "colab": {}
      },
      "source": [
        "#@title\n",
        "devData[\"question\"] = devData[\"question\"].str.lower()\n",
        "devData[\"article_title\"] = devData[\"article_title\"].str.lower()\n",
        "devData[\"paragraph_context\"] = devData[\"paragraph_context\"].str.lower()\n",
        "#devData[\"answer_text\"] = devData[\"answer_text\"].str.lower()\n",
        "devData[\"answer_start\"] = pd.to_numeric(devData[\"answer_start\"])\n",
        "print(\"Finished loading dev data and lowering appropriate columns.\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "HUZ5dzDJzPJD"
      },
      "source": [
        "## Getting training data?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "3o8XNwjRt4Ad",
        "colab": {}
      },
      "source": [
        "X_2_train = trainingData[[\"question\",\"paragraph_context\"]]\n",
        "Y_2_train = trainingData[\"answer_start\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ecPo0Jetse39"
      },
      "source": [
        "### Integer encode text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "GCN8otUGse38"
      },
      "source": [
        "Used for manual encoding of text into integers."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Z19nkJI8iFmV",
        "colab": {}
      },
      "source": [
        "strings = pd.concat((trainingData,devData)).drop(\"answer_start\",axis=1)\n",
        "strings = strings.values.flatten()\n",
        "textTokenizer = Tokenizer(num_words=50000, oov_token='unk') # 37000 words appear less than 10 times, so exclude them. Credit thushv89 on StackOverflow for this suggestion and code.\n",
        "textTokenizer.fit_on_texts(strings)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0e-voi_OBAkI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iymfrGtadU0y",
        "colab": {}
      },
      "source": [
        "# Get length of vocabulary.\n",
        "vocabulary_length = len(textTokenizer.word_index) + 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hV7h7WV3t_tn",
        "colab_type": "code",
        "outputId": "472a1fe3-7d0d-4634-b720-a542bed3340e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(vocabulary_length)"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "87490\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Sqbzwlpakw8T",
        "colab": {}
      },
      "source": [
        "questionsTokenized_train = pad_sequences(textTokenizer.texts_to_sequences(trainingData[\"question\"]))\n",
        "contextTokenized_train = pad_sequences(textTokenizer.texts_to_sequences(trainingData[\"paragraph_context\"]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "eXVcBRqidB6U",
        "colab": {}
      },
      "source": [
        "questionsTokenized_dev = pad_sequences(textTokenizer.texts_to_sequences(devData[\"question\"]))\n",
        "contextTokenized_dev = pad_sequences(textTokenizer.texts_to_sequences(devData[\"paragraph_context\"]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vTXNR10rdAn-",
        "colab": {}
      },
      "source": [
        "articleTitles_train = pad_sequences(textTokenizer.texts_to_sequences(trainingData[\"article_title\"]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qvuPvl5idI5X",
        "colab": {}
      },
      "source": [
        "articleTitles_dev = pad_sequences(textTokenizer.texts_to_sequences(devData[\"article_title\"]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "JhUBOKBJVo0u"
      },
      "source": [
        "**One-hot encoding answer_start**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Pz9tVMPWVsA0",
        "colab": {}
      },
      "source": [
        "answer_start_one_hot = pd.get_dummies(pd.concat((trainingData[\"answer_start\"],devData[\"answer_start\"])))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9112GPHC9yJO",
        "colab": {}
      },
      "source": [
        "answer_start_train_one_hot = pd.get_dummies(trainingData[\"answer_start\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-vXoC9gsMk8i",
        "outputId": "fae721f0-d72f-4193-993f-5f87f51902d5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#@title Shape of answer_start\n",
        "print(trainingData[\"answer_start\"].values.shape)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(87599,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "_qsUAbWRXLXw"
      },
      "source": [
        "# Code to strip punctuation from strings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SpfetyX8XUGM",
        "colab": {}
      },
      "source": [
        "import string\n",
        "def removePunctuation(s):\n",
        "  return s.translate(str.maketrans('', '', string.punctuation))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "N1zAetLvnmHJ"
      },
      "source": [
        "#Exploratory Visualizations & Statistics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OHUPvoyKpahE",
        "colab": {}
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "data_frame = trainingData + devData\n",
        "data_frame = data_frame.astype(\"str\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0MWy2lKufqER",
        "colab": {}
      },
      "source": [
        "question_vectorizer = CountVectorizer().fit(data_frame[\"question\"])\n",
        "context_vectorizer = CountVectorizer().fit(data_frame[\"paragraph_context\"])\n",
        "title_vectorizer = CountVectorizer().fit(data_frame[\"article_title\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "cK2MPNIdqQ25",
        "colab": {}
      },
      "source": [
        "def removeNonAlphanumericCharacters(string):\n",
        "  import re\n",
        "  regex = re.compile('[^a-zA-Z]')\n",
        "  #First parameter is the replacement, second parameter is your input string\n",
        "  return regex.sub('', string)\n",
        "  #Out: 'abdE'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UG6cAFgT1p3M",
        "colab": {}
      },
      "source": [
        "def vocabulary_array(vocabulary_dictionary,strip_chars=False): # Returns an np.array with the word in the first column and the frequency in the second column.\n",
        "  vocabulary = []\n",
        "  for word,frequency in vocabulary_dictionary.items():\n",
        "    if (strip_chars):\n",
        "      word = removeNonAlphanumericCharacters(word)\n",
        "    vocabulary.append([word,str(frequency)])\n",
        "  vocabulary = np.array(vocabulary)\n",
        "  return vocabulary[np.argsort(vocabulary[:,1])[::-1]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-yTG5drP0ZLD",
        "colab": {}
      },
      "source": [
        "question_words = vocabulary_array(question_vectorizer.vocabulary_,strip_chars=True)\n",
        "print(question_words[:,1][0:10].astype(np.uint64))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "I-RYYhLfph2r",
        "colab": {}
      },
      "source": [
        "print(question_words[:,0][0:10])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "f8JUSEq_IkU8"
      },
      "source": [
        "## Bar charts"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OUKJtCmqbjZn",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "fig = plt.figure()\n",
        "fig.subplots_adjust(top=0.8)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tIW3WtjCm5rW",
        "colab": {}
      },
      "source": [
        "#@title Bar plot of most frequent words.\n",
        "from wordcloud import WordCloud,STOPWORDS\n",
        "wordcloud = WordCloud(\n",
        "    width=800,height=800,\n",
        "    stopwords = set(STOPWORDS),\n",
        "    min_font_size = 10,\n",
        "    background_color='white'\n",
        ").generate(\" \".join(pd.concat((trainingData[\"answer_text\"],devData[\"answer_text\"])).values.tolist())) \n",
        "# plot the WordCloud image                        \n",
        "plt.figure(figsize = (8, 8), facecolor = None) \n",
        "plt.imshow(wordcloud,interpolation=\"bilinear\") \n",
        "plt.axis(\"off\") \n",
        "plt.tight_layout(pad = 0) \n",
        "  \n",
        "plt.show() "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Qyj27dzTs8mK",
        "colab": {}
      },
      "source": [
        "#@title Bar plots of lengths.\n",
        "plt.xlabel(\"Length\")\n",
        "plt.ylabel(\"Frequency\")\n",
        "plt.title(\"Histogram of question lengths\")\n",
        "plt.hist(trainingData[\"question\"].str.len().values[0:10000])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "FPlwVUBj0g5Q",
        "colab": {}
      },
      "source": [
        "wordcloud = WordCloud(\n",
        "    width=800,height=800,\n",
        "    stopwords = set(STOPWORDS),\n",
        "    min_font_size = 10,\n",
        "    background_color='white'\n",
        ").generate(\" \".join(pd.concat((trainingData[\"answer_text\"],devData[\"answer_text\"])).values.tolist())) \n",
        "# plot the WordCloud image                        \n",
        "plt.figure(figsize = (8, 8), facecolor = None) \n",
        "plt.imshow(wordcloud,interpolation=\"bilinear\") \n",
        "plt.axis(\"off\") \n",
        "plt.tight_layout(pad = 0) \n",
        "  \n",
        "plt.show() "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0n1EY4Y6iYEv",
        "colab": {}
      },
      "source": [
        "plt.xlabel(\"Length\")\n",
        "plt.ylabel(\"Frequency\")\n",
        "plt.title(\"Histogram of answer lengths\")\n",
        "plt.hist(trainingData[\"answer_text\"].str.len().values[0:10000])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "JjLYDGAUujy4"
      },
      "source": [
        "### Other statistics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "pkJ61xyU-LjU",
        "scrolled": true,
        "colab": {}
      },
      "source": [
        "from sys import getsizeof\n",
        "def vocabularySize():\n",
        "  return len(vocabulary())\n",
        "def summaryStatistics(series):\n",
        "    numberOfWords = series.apply(lambda x: len(str(x).split(\" \")))\n",
        "    averageNumberOfWords = sum(numberOfWords) / len(numberOfWords)\n",
        "    return \"average: \" + str(averageNumberOfWords) + \"maximum: \" + str(max(numberOfWords)) + \" minimum: \" +str(min(numberOfWords))\n",
        "print(\"Size of vocabulary: \", vocabularySize())\n",
        "print(\"Words in each question: \",summaryStatistics(trainingData[\"question\"]))\n",
        "print(\"Words in each article title: \",summaryStatistics(trainingData[\"article_title\"]))\n",
        "print(\"Words in each context: \",summaryStatistics(trainingData[\"paragraph_context\"]))\n",
        "print(\"Words in each answer: \",summaryStatistics(trainingData[\"answer_text\"]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6C7BSPpthhUD",
        "colab": {}
      },
      "source": [
        "#@title Top ten most frequently occuring words:\n",
        "def wordFrequencies(series):\n",
        "  split_strings = series.str.split(\" \")\n",
        "  frequencies = {}\n",
        "  for split_string in split_strings:\n",
        "    for word in split_string:\n",
        "      word = word.lower()\n",
        "      word = removePunctuation(word)\n",
        "      try:\n",
        "        frequencies[word] = frequencies[word] + 1\n",
        "      except KeyError as k:\n",
        "        frequencies[word] = 1\n",
        "  return frequencies\n",
        "pd.DataFrame.from_dict(wordFrequencies(trainingData[\"question\"]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "vXrbrVdtoXxO"
      },
      "source": [
        "#Building neural networks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "mkRNjwNMqNDn"
      },
      "source": [
        "This kind of task requires a neural network to find the answer from the relevant article. In this section, we look at two kinds of neural networks."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "qXYT8-xs7V_0"
      },
      "source": [
        "## Dense Neural Networks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "lE3HkJLg7X8e"
      },
      "source": [
        "This is the most basic, unremarkable kind of network. Put simply, each layer of this network takes inputs, applies weights and biases, then outputs a value between 0 and 1 - usually either a sigmoid value or a linear function."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UHy8slsutBCc"
      },
      "source": [
        "## Long Short-Term Memory Networks\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "jm8eUb1h4PaK"
      },
      "source": [
        "Long short-term memory networks are a type of recurrent neural networs that work by recalling words it previously read to get a good feel for the context of a word or phrase."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "KPf_9uZL4Qv0"
      },
      "source": [
        "The most fundamental unit of an LSTM is the cell state - a little unit that holds information the LSTM deems important to remember. For example, consider the following piece of text:\n",
        "\n",
        "\"I used to live all over Europe. In fact, I spent a significant amount of time in France, and now I speak fluent _____.\" To fill in this blank, we only need to remember two things:\n",
        "\n",
        "1) The preceding phrase \"I speak fluent\" - this indicates that the word that follows is the name of a language.\n",
        "2) That the speaker lived in France; the fact that he lived \"all over Europe\" is irrelevant to deducing the name of the language he is about to mention he is fluent in.\n",
        "\n",
        "We can thus deduce that the word that fits in the blank is \"French\". "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "SgQiq1xp-2dp"
      },
      "source": [
        "Part of the training process of an LSTM that would fill in the blank would involve learning which of these words would be important to deducing the missing word. For example, on the first training iteration, the LSTM might decide that all of the words are important. However, we might hope that as the LSTM progresses, it narrows down the list of words to remember to \"France\",\"I\",\"speak\", and \"fluent\". (https://medium.com/@ageitgey/natural-language-processing-is-fun-9a0bff37854e)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "yZwiQn4asngG"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "6ZZ6G3pmsq-P"
      },
      "source": [
        "## Google BERT\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "fqJPLWHNsybK"
      },
      "source": [
        "As we will see, my LSTM model performed very poorly. One of the best models for learning relationships between words is Google's \"Bidirectional Encoder Representations from Transformers\" model.\n",
        "\n",
        "One of the problems that BERT attempts to solve is the dearth of sufficient reliable training data for natural language processing tasks. It does this by manipulating existing data to create more data. For example, if our training data includes the sentence \"I used to live all over Europe. In fact, I spent a significant amount of time in France, and now I speak fluent French,\" BERT will generate another sentence, such as: French fluent speak I now and France in time of amount significant a spent I fact in Europe over all live to used I\". (https://ai.googleblog.com/2018/11/open-sourcing-bert-state-of-art-pre.html)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "s_sWrANJ15co"
      },
      "source": [
        "## Benchmark\n",
        "  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "0dm2GJua1-2K"
      },
      "source": [
        "Since my simplest model is a Dense model, I will use that as a benchmark."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "_uWEIzDmTneR"
      },
      "source": [
        "## Defining metrics\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "hrTUaNEfA0rv"
      },
      "source": [
        "First, we need to define a metric we can use to score our neural network.\n",
        "\n",
        "We'll use the F1 score. This score emphasizes false positives and false negatives, and therefore should be used when they are most important. (https://medium.com/analytics-vidhya/accuracy-vs-f1-score-6258237beca2#:~:targetText=Accuracy%20is%20used%20when%20the,and%20False%20Positives%20are%20crucial&targetText=In%20most%20real%2Dlife%20classification,to%20evaluate%20our%20model%20on.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "U5YaqJosT560",
        "colab": {}
      },
      "source": [
        "#@title F1\n",
        "from tensorflow.keras import backend as K\n",
        "def f1(y_true, y_pred):\n",
        "    def recall(y_true, y_pred):\n",
        "        \"\"\"Recall metric.\n",
        "\n",
        "        Only computes a batch-wise average of recall.\n",
        "\n",
        "        Computes the recall, a metric for multi-label classification of\n",
        "        how many relevant items are selected.\n",
        "        \"\"\"\n",
        "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "        recall = true_positives / (possible_positives + K.epsilon())\n",
        "        return recall\n",
        "\n",
        "    def precision(y_true, y_pred):\n",
        "        \"\"\"Precision metric.\n",
        "\n",
        "        Only computes a batch-wise average of precision.\n",
        "\n",
        "        Computes the precision, a metric for multi-label classification of\n",
        "        how many selected items are relevant.\n",
        "        \"\"\"\n",
        "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "        precision = true_positives / (predicted_positives + K.epsilon())\n",
        "        return precision\n",
        "    precision = precision(y_true, y_pred)\n",
        "    recall = recall(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "GapK4sFicyYq"
      },
      "source": [
        "##Mapping Questions to Article Titles\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "6E-D2b4migEf"
      },
      "source": [
        "While this step is likely important in a regular search, I do not expect that Stanford evaluates a program's ability to search for article titles; therefore, I am omitting this step for now."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "oq_sLBsSse3_"
      },
      "source": [
        "## Mapping Questions and Paragraph Contexts to Answers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "IniJzW3lse3-"
      },
      "source": [
        "This neural network is used to generate answers from the questions and articles. It works by first reading the relevant article and using the question to find the answer."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "qfME8DsPslXa"
      },
      "source": [
        "## Word2Vec"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "AjrlnjbmsfS5",
        "colab": {}
      },
      "source": [
        "\n",
        "from tensorflow.keras.preprocessing.text import *\n",
        "from tensorflow.keras.preprocessing.sequence import skipgrams,make_sampling_table\n",
        "def skipgrams_labels(sequence,vocabulary_length,window_size=3):\n",
        "    couples,labels = skipgrams(sequence,vocabulary_length,window_size=window_size)\n",
        "    assert len(couples) > 0\n",
        "    target_words,contexts = zip(*couples)\n",
        "    target_words = np.array(target_words).astype(\"int32\")\n",
        "    contexts = np.array(contexts).astype(\"int32\")\n",
        "    return target_words,contexts,labels\n",
        "# This is to train word2vec.\n",
        "def word2vec_selection(sequences,vocabulary_length,window_size=3,batch_index=-1):\n",
        "    assert sequences.ndim == 2\n",
        "    if (batch_index == -1):\n",
        "        batch_index = np.random.choice(sequences.shape[0], 1, replace=False)\n",
        "    sequence = sequences[batch_index,:].flatten()\n",
        "    target_words,contexts,labels = skipgrams_labels(sequence,vocabulary_length,window_size=window_size)\n",
        "    return target_words,contexts,labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ozGCkKzUjZFl",
        "colab": {}
      },
      "source": [
        "#@title Training word2vec\n",
        "def word2vec_model(vector_dim=256):\n",
        "    input_target = Input((1,))\n",
        "    input_context = Input((1,))\n",
        "    embedding = Embedding(vocabulary_length, vector_dim, input_length=1)\n",
        "    target = embedding(input_target)\n",
        "    context = embedding(input_context)\n",
        "    from tensorflow.keras.layers import Dot,dot\n",
        "    dot_product = dot([target,context],1)\n",
        "    flat = Flatten()(dot_product)\n",
        "    output = Dense(1)(flat)\n",
        "    model = Model(inputs=[input_target,input_context],outputs=[output])\n",
        "    return model\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hKEYSlrFt_vM",
        "colab_type": "code",
        "outputId": "c1cb9b5d-f645-4717-c5e6-b3be382d28a1",
        "colab": {}
      },
      "source": [
        "word2vec = word2vec_model()\n",
        "word2vec.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 1)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            [(None, 1)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding (Embedding)           (None, 1, 256)       22394368    input_1[0][0]                    \n",
            "                                                                 input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dot (Dot)                       (None, 256, 256)     0           embedding[0][0]                  \n",
            "                                                                 embedding[1][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "flatten (Flatten)               (None, 65536)        0           dot[0][0]                        \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 1)            65537       flatten[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 22,459,905\n",
            "Trainable params: 22,459,905\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jTXh3zibt_vO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def word2vec_batch_generator(sequences,vocabulary_length,batch_size=1):\n",
        "    while True:\n",
        "        target_words,contexts,labels = word2vec_selection(sequences,vocabulary_length)\n",
        "        x = np.hstack([target_words,contexts])\n",
        "        yield [target_words,contexts],np.array(labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7G0KX9Evt_vb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word2vec.compile(\"rmsprop\",\"binary_crossentropy\",metrics=[f1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tshxQx0ot_vd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word2vec.fit_generator(\n",
        "    word2vec_batch_generator(\n",
        "        contextTokenized_train,\n",
        "        vocabulary_length),\n",
        "    validation_data=word2vec_batch_generator(\n",
        "        contextTokenized_train,\n",
        "        vocabulary_length),\n",
        "    validation_steps=2,\n",
        "    steps_per_epoch=100,\n",
        "    epochs=20,callbacks=[contexts_word2vec_checkpoint])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nXSjuYMqt_vf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test = word2vec_selection(questionsTokenized_dev,vocabulary_length)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5MFWImj_t_vh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word2vec.load_weights(\"questions_word2vec.h5\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GdqfGmmWt_vk",
        "colab_type": "code",
        "outputId": "17fb93a8-124c-4385-f501-1f85a6680f79",
        "colab": {}
      },
      "source": [
        "word2vec.evaluate(x=[test[0],test[1]],y=[test[2]])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "48/48 [==============================] - 0s 7ms/sample - loss: 0.7389 - f1: 0.7960\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.738932599623998, 0.795977]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HZnvZBJXt_vn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word2vec.evaluate(x=[context_tests[0],context_tests[1]],y=[context_tests[2]])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O3UTZZfxt_vt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Freeze models.\n",
        "word2vec.trainable = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lw0gbgDrt_vv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def positive_skipgrams(sequence,vocabulary_length):\n",
        "    targets = []\n",
        "    contexts = []\n",
        "    for i in range(len(sequence)):\n",
        "        lower_bound = 0\n",
        "        if (i > 0):\n",
        "            lower_bound = i - 1\n",
        "        upper_bound = i + 1\n",
        "        if (upper_bound >= len(sequence)):\n",
        "            upper_bound = i\n",
        "        word = sequence[i]\n",
        "        skipgram_1_word = sequence[lower_bound]\n",
        "        targets.append(word)\n",
        "        contexts.append(skipgram_1_word)\n",
        "        skipgram_2_word = sequence[upper_bound]\n",
        "        targets.append(word)\n",
        "        contexts.append(skipgram_2_word)\n",
        "    return targets,contexts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TEbn-VDyt_vw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "question_skipgrams_train = [positive_skipgrams(sequence,vocabulary_length) for sequence in questionsTokenized_train]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JG2n4fPtt_vy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "contexts_skipgrams_train = [positive_skipgrams(sequence,vocabulary_length) for sequence in contextTokenized_train]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-KMqKpSt_v0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word2vec_embeddings = Model(inputs=[word2vec.inputs],outputs=[word2vec.get_layer(\"embedding_2\").output])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hXr4rByPt_v2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "question_embeddings = word2vec_embeddings.predict(question_skipgrams_train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "NbToNYmR-ax8"
      },
      "source": [
        "## Non-word2vec stuff."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IekQsi2Yse3J",
        "colab": {}
      },
      "source": [
        "answers_network_checkpoint = ModelCheckpoint('answers_network-best.h5', verbose=1, monitor='val_loss',save_best_only=True, mode='auto') "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "khR26T3hwbVG",
        "colab_type": "code",
        "outputId": "2062f6a2-b5e1-4c9b-d7e6-ee444dbba006",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(answer_start_train_one_hot.shape)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(87599, 1604)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "K8yleaDI8MJf"
      },
      "source": [
        "## Dense Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "cRRjCs3H8KFr",
        "colab": {}
      },
      "source": [
        "questions_embedding = questions_word2vec.layers[-2].output\n",
        "contexts_embedding = contexts_word2vec.layers[-2].output\n",
        "combined_layers = Concatenate()([questions_embedding,contexts_embedding])\n",
        "answers_network_2_dense_5 = Dense(answer_start_train_one_hot.values.shape[1],activation=\"sigmoid\")(combined_layers)\n",
        "#answers_network_2.add(Dense(32)) Removing this line gives us more trainable parameters.\n",
        "answers_network_2 = Model(inputs=[questions_word2vec.input,contexts_word2vec.input],outputs=[answers_network_2_dense_5])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0llj4vYp-Fzc",
        "colab": {}
      },
      "source": [
        "answers_network_2.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "eMIuiPQz-Ld1",
        "colab": {}
      },
      "source": [
        "answers_network_2.compile(\"adam\",\"binary_crossentropy\",metrics=[f1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qfXhe0Cet_wC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def answers_network_batch_generator(question_skipgrams,context_skipgrams,vocabulary_length,answer_starts):\n",
        "    while True:\n",
        "        index = np.random.choice(question_skipgrams.shape[0], 1, replace=False)\n",
        "        question_targets,question_contexts = zip(question_skipgrams[index,:])\n",
        "        context_targets,context_contexts = zip(context_skipgrams[index,:])\n",
        "        question\n",
        "        yield [question_targets,question_contexts,context_targets,context_contexts],answer_starts[index,:]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JIfV6WSrz58D",
        "colab": {}
      },
      "source": [
        "answers_network_2.fit_generator(answers_network_batch_generator(\n",
        "    np.array(question_skipgrams_train),\n",
        "    np.array(contexts_skipgrams_train),\n",
        "    vocabulary_length,\n",
        "    answer_start_train_one_hot.values),\n",
        "                                steps_per_epoch=20)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "cbe5SYkGiSl5"
      },
      "source": [
        "## Long Short-Term Memory Network (LSTM)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qhaK14XAM5zc",
        "colab": {}
      },
      "source": [
        "#@title One-hot encode answer starts.\n",
        "outputTrain = pd.get_dummies(trainingData[\"answer_start\"]).values\n",
        "print(outputTrain.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "pZc_zpxv041O",
        "colab": {}
      },
      "source": [
        "def goodNumber(array):\n",
        "  return array.shape[1]\n",
        "def actualNumber(features,time_steps):\n",
        "  return features * time_steps\n",
        "def isGoodNumber(array,features,time_steps):\n",
        "  return goodNumber(array) == actualNumber(features,time_steps)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "35q5Gar__vqm",
        "colab": {}
      },
      "source": [
        "answers_questions_features = 8\n",
        "answers_questions_time_steps = int(questionsTokenized_train.shape[1] / answers_questions_features)\n",
        "assert isGoodNumber(questionsTokenized_train,answers_questions_features,answers_questions_time_steps)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QyKUtypntqm7",
        "colab": {}
      },
      "source": [
        "print(contextTokenized_train.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qDUpLcY-YVgv",
        "colab": {}
      },
      "source": [
        "answers_contexts_features = 677\n",
        "answers_contexts_time_steps = int(contextTokenized_train.shape[1] / answers_contexts_features)\n",
        "assertion_error_message = \"Bad number: \" + str(actualNumber(answers_contexts_features,answers_contexts_time_steps)) + \" Good number: \" + str(goodNumber(contextTokenized_train))\n",
        "assert isGoodNumber(contextTokenized_train,answers_contexts_features,answers_contexts_time_steps),assertion_error_message"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Iv25QYbbse3Q",
        "colab": {}
      },
      "source": [
        "answers_questions_input = Input(shape=(answers_questions_features,answers_questions_time_steps,))\n",
        "answers_contexts_input = Input(shape=(answers_contexts_features,answers_contexts_time_steps,))\n",
        "answers_questions_lstm = LSTM(256)(answers_questions_input)\n",
        "answers_contexts_lstm = LSTM(256)(answers_contexts_input)\n",
        "#answers_network_1.add(Embedding(715,128,input_length=715))\n",
        "answers_combined_lstm = Add()([answers_questions_lstm,answers_contexts_lstm])\n",
        "answers_combined_flattened=Flatten()(answers_combined_lstm)\n",
        "answers_hidden_1 = Dense(124)(answers_combined_flattened)\n",
        "answers_output = Dense(outputTrain.shape[1])(answers_hidden_1)\n",
        "answers_network_1 = Model(inputs=[answers_questions_input,answers_contexts_input],outputs=[answers_output])\n",
        "answers_network_1.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "eTLaKjwyse3N",
        "colab": {}
      },
      "source": [
        "answers_network_1.compile(\"adam\",\"binary_crossentropy\",metrics=[f1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fabHPM2p_DTM",
        "colab": {}
      },
      "source": [
        "#@title Reshape the inputs for LSTM.\n",
        "def reshape_for_lstm(inputs,features,time_steps):\n",
        "  assert features * time_steps == inputs.shape[1], \"Bad shape.\"\n",
        "  return np.reshape(inputs,(inputs.shape[0],features,time_steps))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "h1Z0gJC7zuGR",
        "colab": {}
      },
      "source": [
        "answers_network_1.fit(x=[reshape_for_lstm(questionsTokenized_train,answers_questions_features,answers_questions_time_steps),reshape_for_lstm(contextTokenized_train,answers_contexts_features,answers_contexts_time_steps)],y=[outputTrain],validation_split=0.1,callbacks=[answers_network_checkpoint],verbose=True,epochs=9)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "OSJyJ3qxse3L"
      },
      "source": [
        "#### Train the neural network."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_tWTvWxSse3F",
        "colab": {}
      },
      "source": [
        "print(answers_network_1.metrics_names)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OM2GYlOxApiF",
        "colab": {}
      },
      "source": [
        "def shape_for_lstm(array,features,time_steps):\n",
        "  try:\n",
        "    new_shape = (array.shape[0],features,time_steps)\n",
        "  except:\n",
        "    raise ValueError(\"Bad inputs.\")\n",
        "    return\n",
        "  try:\n",
        "    array_reshaped = np.reshape(array,new_shape)\n",
        "  except TypeError as t:\n",
        "    from traceback import print_stack\n",
        "    raise TypeError(\"Traceback: \" + str(print_stack()) + \" bad types: \" + str(features) + \" \" + str(time_steps))\n",
        "  except AttributeError as a:\n",
        "    raise AttributeError(\"What?\")\n",
        "  return array_reshaped"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "omQeoKx6EXjK",
        "colab": {}
      },
      "source": [
        "questionsTokenized_train_lstm = shape_for_lstm(questionsTokenized_train,answers_questions_features,answers_questions_time_steps)\n",
        "contextTokenized_train_lstm = shape_for_lstm(contextTokenized_train,answers_contexts_features,answers_contexts_time_steps)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9hg2J0qBse3C",
        "colab": {}
      },
      "source": [
        "\n",
        "answers_network_1.fit(x=[questionsTokenized_train_lstm,contextTokenized_train_lstm],y=[outputTrain],validation_split=0.2,callbacks=[answers_network_checkpoint],verbose=True,epochs=9)\n",
        "#print(\"Weights: \",questions_article_model.get_weights())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "R_LIVwsfcXVA"
      },
      "source": [
        "##Results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zljgLd_JTxC6",
        "colab": {}
      },
      "source": [
        "inputDev = np.pad(pd.get_dummies(devData[[\"question\",\"paragraph_context\"]].values),((0,0),(0,trainingData[[\"question\",\"paragraph_context\"]].values.shape[1])))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GrcTLoMmTThK",
        "colab": {}
      },
      "source": [
        "outputDev = np.pad(pd.get_dummies(devData[\"answer_start\"]).values,((0,0),(0,431)))\n",
        "print(outputDev.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "c-U8EwYFeRUI"
      },
      "source": [
        "The F1 score was very poor (0.0e+00). Strangely, a network of just Dense layers may have performed better."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "gT2h3iAU-LlT"
      },
      "source": [
        "## Transfer learning using Google's BERT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "dGUwjE61-LlU"
      },
      "source": [
        "The previous model has a very poor F1 score. Let's see if we can't build a better model. We'll use Google's BERT deep learning network, which is so good that 5 neural networks that use it made it to the top 10 winning neural networks for the SQuAD v2.0 dataset. These five neural networks had an average of 86.7% exact matches. For comparison, the humans that Stanford tested scored an average of 86.8% - a difference of 0.1%.\n",
        "The basic idea behind BERT is this: Neural networks rely on numerical vectors. Similar sentences and phrases should produce similar vectors.\n",
        "\n",
        "\"It is not possible to train bidirectional models by simply conditioning each word on words before and after it. Doing this would allow the word that’s being predicted to indirectly see itself in a multi-layer model. To solve this, Google researchers used a straightforward technique of masking out some words in the input and condition each word bidirectionally in order to predict the masked words. This idea is not new, but BERT is the first technique where it was successfully used to pre-train a deep neural network.\" (packtpub.com)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sdqtUB98-FZO",
        "colab": {}
      },
      "source": [
        "#@title Clear session.\n",
        "from tensorflow.keras import backend as K\n",
        "K.clear_session()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WxPPaY7jpbnB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Memory Management\n",
        "tf.keras.backend.set_floatx(\"float32\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "KUQ8UtquieFj",
        "colab": {}
      },
      "source": [
        "# @title Environment\n",
        "import os\n",
        "pretrained_path = 'uncased_L-12_H-768_A-12'\n",
        "config_path = os.path.join(pretrained_path, 'bert_config.json')\n",
        "checkpoint_path = os.path.join(pretrained_path, 'bert_model.ckpt')\n",
        "vocab_path = os.path.join(pretrained_path, 'vocab.txt')\n",
        "# Use TF_Keras\n",
        "os.environ[\"TF_KERAS\"] = \"1\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sVTPNxOyj4HJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "9c882bf6-b7bb-4b59-8bbd-20d504f2684c"
      },
      "source": [
        "# @title Load Basic Model\n",
        "import codecs\n",
        "from keras_bert import load_trained_model_from_checkpoint\n",
        "token_dict = {}\n",
        "with codecs.open(vocab_path, 'r', 'utf8') as reader:\n",
        "    for line in reader:\n",
        "        token = line.strip()\n",
        "        token_dict[token] = len(token_dict)\n",
        "\n",
        "model = load_trained_model_from_checkpoint(config_path, checkpoint_path)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ZRNnqt-8eXyJ",
        "outputId": "64536a31-e612-4cc1-c592-af3ba75f37b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "\n",
        "#@title Model Summary\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "Input-Token (InputLayer)        [(None, 512)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Input-Segment (InputLayer)      [(None, 512)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Embedding-Token (TokenEmbedding [(None, 512, 768), ( 23440896    Input-Token[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "Embedding-Segment (Embedding)   (None, 512, 768)     1536        Input-Segment[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "Embedding-Token-Segment (Add)   (None, 512, 768)     0           Embedding-Token[0][0]            \n",
            "                                                                 Embedding-Segment[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "Embedding-Position (PositionEmb (None, 512, 768)     393216      Embedding-Token-Segment[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "Embedding-Dropout (Dropout)     (None, 512, 768)     0           Embedding-Position[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "Embedding-Norm (LayerNormalizat (None, 512, 768)     1536        Embedding-Dropout[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-1-MultiHeadSelfAttentio (None, 512, 768)     2362368     Embedding-Norm[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-1-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-1-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-1-MultiHeadSelfAttentio (None, 512, 768)     0           Embedding-Norm[0][0]             \n",
            "                                                                 Encoder-1-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-1-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-1-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-1-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-1-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-1-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-1-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-1-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-1-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-1-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-1-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-1-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-2-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-1-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-2-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-2-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-2-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-1-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-2-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-2-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-2-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-2-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-2-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-2-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-2-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-2-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-2-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-2-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-2-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-2-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-3-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-2-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-3-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-3-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-3-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-2-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-3-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-3-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-3-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-3-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-3-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-3-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-3-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-3-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-3-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-3-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-3-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-3-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-4-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-3-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-4-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-4-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-4-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-3-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-4-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-4-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-4-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-4-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-4-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-4-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-4-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-4-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-4-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-4-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-4-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-4-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-5-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-4-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-5-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-5-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-5-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-4-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-5-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-5-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-5-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-5-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-5-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-5-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-5-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-5-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-5-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-5-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-5-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-5-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-6-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-5-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-6-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-6-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-6-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-5-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-6-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-6-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-6-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-6-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-6-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-6-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-6-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-6-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-6-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-6-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-6-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-6-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-7-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-6-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-7-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-7-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-7-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-6-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-7-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-7-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-7-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-7-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-7-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-7-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-7-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-7-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-7-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-7-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-7-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-7-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-8-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-7-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-8-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-8-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-8-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-7-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-8-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-8-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-8-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-8-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-8-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-8-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-8-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-8-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-8-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-8-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-8-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-8-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-9-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-8-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-9-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-9-MultiHeadSelfAttention[\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-9-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-8-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-9-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-9-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-9-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-9-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-9-MultiHeadSelfAttention-\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-9-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-9-FeedForward[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-9-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-9-MultiHeadSelfAttention-\n",
            "                                                                 Encoder-9-FeedForward-Dropout[0][\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-9-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-9-FeedForward-Add[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-10-MultiHeadSelfAttenti (None, 512, 768)     2362368     Encoder-9-FeedForward-Norm[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-10-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-10-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-10-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-9-FeedForward-Norm[0][0] \n",
            "                                                                 Encoder-10-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-10-MultiHeadSelfAttenti (None, 512, 768)     1536        Encoder-10-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-10-FeedForward (FeedFor (None, 512, 768)     4722432     Encoder-10-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-10-FeedForward-Dropout  (None, 512, 768)     0           Encoder-10-FeedForward[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-10-FeedForward-Add (Add (None, 512, 768)     0           Encoder-10-MultiHeadSelfAttention\n",
            "                                                                 Encoder-10-FeedForward-Dropout[0]\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-10-FeedForward-Norm (La (None, 512, 768)     1536        Encoder-10-FeedForward-Add[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-11-MultiHeadSelfAttenti (None, 512, 768)     2362368     Encoder-10-FeedForward-Norm[0][0]\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-11-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-11-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-11-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-10-FeedForward-Norm[0][0]\n",
            "                                                                 Encoder-11-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-11-MultiHeadSelfAttenti (None, 512, 768)     1536        Encoder-11-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-11-FeedForward (FeedFor (None, 512, 768)     4722432     Encoder-11-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-11-FeedForward-Dropout  (None, 512, 768)     0           Encoder-11-FeedForward[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-11-FeedForward-Add (Add (None, 512, 768)     0           Encoder-11-MultiHeadSelfAttention\n",
            "                                                                 Encoder-11-FeedForward-Dropout[0]\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-11-FeedForward-Norm (La (None, 512, 768)     1536        Encoder-11-FeedForward-Add[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-12-MultiHeadSelfAttenti (None, 512, 768)     2362368     Encoder-11-FeedForward-Norm[0][0]\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-12-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-12-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-12-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-11-FeedForward-Norm[0][0]\n",
            "                                                                 Encoder-12-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-12-MultiHeadSelfAttenti (None, 512, 768)     1536        Encoder-12-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-12-FeedForward (FeedFor (None, 512, 768)     4722432     Encoder-12-MultiHeadSelfAttention\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-12-FeedForward-Dropout  (None, 512, 768)     0           Encoder-12-FeedForward[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-12-FeedForward-Add (Add (None, 512, 768)     0           Encoder-12-MultiHeadSelfAttention\n",
            "                                                                 Encoder-12-FeedForward-Dropout[0]\n",
            "__________________________________________________________________________________________________\n",
            "Encoder-12-FeedForward-Norm (La (None, 512, 768)     1536        Encoder-12-FeedForward-Add[0][0] \n",
            "==================================================================================================\n",
            "Total params: 108,891,648\n",
            "Trainable params: 0\n",
            "Non-trainable params: 108,891,648\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "l2EBys0pvTCL",
        "colab": {}
      },
      "source": [
        "#@title Create tokenization stuff.\n",
        "import keras_bert\n",
        "tokenizer = keras_bert.Tokenizer(token_dict)\n",
        "def tokenize(text,max_len):\n",
        "  tokenizer.tokenize(text)\n",
        "  return tokenizer.encode(first=text,max_len=max_len)\n",
        "def tokenize_array(texts,max_len=512):\n",
        "  indices = np.zeros((texts.shape[0],max_len))\n",
        "  segments = np.zeros((texts.shape[0],max_len))\n",
        "  for i in range(texts.shape[0]):\n",
        "    tokens = tokenize(texts[i],max_len)\n",
        "    indices[i] = tokens[0]\n",
        "    segments[i] = tokens[1]\n",
        "  #print(indices.shape)\n",
        "  #print(segments.shape)\n",
        "  return segments,indices"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "uE3U3j81v79t",
        "colab": {}
      },
      "source": [
        "#@ Tokenize inputs.\n",
        "def X_Y(dataset,answer_start_one_hot,batch_size=10,max_len=512):\n",
        "    batch_indices = np.random.choice(np.arange(0,dataset.shape[0]),size=batch_size)\n",
        "    dataset = dataset.iloc[batch_indices]\n",
        "    questions = dataset[\"question\"]\n",
        "    contexts = dataset[\"paragraph_context\"]\n",
        "    question_indices,question_segments = tokenize_array(questions.values,max_len=max_len)\n",
        "    context_indices,context_segments = tokenize_array(contexts.values,max_len=max_len)\n",
        "    X = [question_indices,question_segments,context_indices,context_segments]\n",
        "    Y = answer_start_one_hot.iloc[batch_indices]\n",
        "    return X,Y\n",
        "def X_Y_generator(dataset,answer_start_one_hot,batch_size=10,max_len=512):\n",
        "    while True:\n",
        "        try:\n",
        "            X,Y = X_Y(dataset,answer_start_one_hot,batch_size=batch_size,max_len=max_len)\n",
        "            #max_int = pd.concat((trainingData[\"answer_start\"],devData[\"answer_start\"])).max()\n",
        "            yield X,Y\n",
        "        except Exception as e:\n",
        "            print(\"Unhandled exception in X_Y_generator: \",e)\n",
        "            raise"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8MVfPkFxk65a",
        "colab_type": "code",
        "outputId": "6a0b38e3-69a8-4fd0-9166-e898fe7923ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "X,Y = X_Y(trainingData,answer_start_train_one_hot,max_len=150) # Credit to the aforementioned StackOverflow user for his suggestion to use 150 words for brevity.\n",
        "print(X[0].shape)\n",
        "print(X[1].shape)\n",
        "print(X[2].shape)\n",
        "print(X[3].shape)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(10, 150)\n",
            "(10, 150)\n",
            "(10, 150)\n",
            "(10, 150)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IWyz0fwLt_w9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.trainable = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YgkFVU8r1cGN",
        "outputId": "13bd4c17-2576-45c0-fb8c-79dfe6b88675",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        }
      },
      "source": [
        "\n",
        "question_indices_layer = Input(shape=(150,))\n",
        "question_segments_layer = Input(shape=(150,))\n",
        "context_indices_layer = Input(shape=(150,))\n",
        "context_segments_layer = Input(shape=(150,))\n",
        "questions_bert_layer = model([question_indices_layer,question_segments_layer])\n",
        "print(\"Questions bert layer loaded.\")\n",
        "context_bert_layer = model([context_indices_layer,context_segments_layer])\n",
        "print(\"Context bert layer loaded.\")\n",
        "\n",
        "#Credit to the following block goes to thushv89 @ StackOverflow.com\n",
        "questions_flattened = Flatten(dtype=tf.float16)(questions_bert_layer)\n",
        "questions_flattened = Dense(128, activation='relu',dtype=tf.float16)(questions_flattened)\n",
        "questions_flattened = Dense(128, activation='relu',dtype=tf.float16)(questions_flattened)\n",
        "contexts_flattened = Flatten(dtype=tf.float16)(context_bert_layer)\n",
        "contexts_flattened = Dense(128,activation=\"relu\",dtype=tf.float16)(contexts_flattened)\n",
        "contexts_flattened = Dense(128,activation=\"relu\",dtype=tf.float16)(contexts_flattened)\n",
        "combined = Concatenate(dtype=tf.float16)([questions_flattened,contexts_flattened])\n",
        "\n",
        "\n",
        "#bert_dense_questions = Dense(256,activation=\"sigmoid\")(questions_flattened)\n",
        "#bert_dense_context = Dense(256,activation=\"sigmoid\")(context_flattened)\n",
        "answers_network_output = Dense(1604,activation=\"softmax\",dtype=tf.float16)(combined)\n",
        "#answers_network = Model(inputs=[input_layer],outputs=[questions_bert_layer,context_bert_layer])\n",
        "answers_network = Model(inputs=[question_indices_layer,question_segments_layer,context_indices_layer,context_segments_layer],outputs=[answers_network_output])\n",
        "answers_network.summary()\n"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Questions bert layer loaded.\n",
            "Context bert layer loaded.\n",
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 150)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            [(None, 150)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_3 (InputLayer)            [(None, 150)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_4 (InputLayer)            [(None, 150)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "model_1 (Model)                 multiple             108891648   input_1[0][0]                    \n",
            "                                                                 input_2[0][0]                    \n",
            "                                                                 input_3[0][0]                    \n",
            "                                                                 input_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "flatten (Flatten)               (None, 115200)       0           model_1[1][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "flatten_1 (Flatten)             (None, 115200)       0           model_1[2][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 128)          14745728    flatten[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 128)          14745728    flatten_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 128)          16512       dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 128)          16512       dense_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 256)          0           dense_1[0][0]                    \n",
            "                                                                 dense_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 1604)         412228      concatenate[0][0]                \n",
            "==================================================================================================\n",
            "Total params: 138,828,356\n",
            "Trainable params: 29,936,708\n",
            "Non-trainable params: 108,891,648\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ABSXiH5at_xC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.optimizers import SGD\n",
        "opt = SGD(lr=0.01)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YLbmIFeJ2pnn",
        "colab": {}
      },
      "source": [
        "answers_network.compile(\"adam\",\"categorical_crossentropy\",metrics=[f1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "pHEUY67q4cbc",
        "outputId": "aa731fd9-7358-423a-f81d-6be8b1ee9dc8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(answers_network.metrics_names)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['loss', 'acc']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gvavQkHW4cbe",
        "outputId": "6debcc17-bc27-483e-e17f-9836d667757d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 497
        }
      },
      "source": [
        "# Credit goes to thushv89 on StackOverflow\n",
        "# for suggesting a maximum length of 150\n",
        "answers_network.fit_generator(\n",
        "    X_Y_generator(\n",
        "        trainingData,\n",
        "        answer_start_train_one_hot,\n",
        "        batch_size=32,max_len=150),\n",
        "    steps_per_epoch=100,\n",
        "    epochs=10,\n",
        "    callbacks=[answers_network_checkpoint])"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            " 99/100 [============================>.] - ETA: 0s - loss: nan - f1: 1.9988WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
            "100/100 [==============================] - 44s 436ms/step - loss: nan - f1: 1.9988\n",
            "Epoch 2/10\n",
            " 99/100 [============================>.] - ETA: 0s - loss: nan - f1: 1.9988WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
            "100/100 [==============================] - 40s 396ms/step - loss: nan - f1: 1.9988\n",
            "Epoch 3/10\n",
            " 53/100 [==============>...............] - ETA: 18s - loss: nan - f1: 1.9988"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-31-bc00bb757f3c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     callbacks=[answers_network_checkpoint])\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1294\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1295\u001b[0m         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1296\u001b[0;31m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m   1297\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1298\u001b[0m   def evaluate_generator(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m       \u001b[0mis_deferred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_compiled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 265\u001b[0;31m       \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    266\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight, reset_metrics)\u001b[0m\n\u001b[1;32m   1015\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_sample_weight_modes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1017\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1018\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3474\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3475\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3476\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3477\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3478\u001b[0m     output_structure = nest.pack_sequence_as(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1470\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[1;32m   1471\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1472\u001b[0;31m                                                run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1473\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1474\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZ23zKl7qRH8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "d7c3721f-0fc3-431c-f68a-1054aa9ec75b"
      },
      "source": [
        "evaluate_generator = answers_network.evaluate_generator(X_Y_generator(devData,answer_start_train_one_hot,batch_size=32,max_len=150),steps = 10)\n",
        "\n",
        "j=0\n",
        "for i in evaluate_generator:\n",
        "  print(i)\n",
        "  j += 1\n",
        "  if (j==10):\n",
        "    break"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nan\n",
            "1.9987539\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVyF78DEt_xP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 130
        },
        "outputId": "c52ecada-17f7-4b0c-9bfb-0b810c90a14a"
      },
      "source": [
        "answers_network.evaluate_generator\n",
        "(\n",
        "    X_Y_generator(\n",
        "        devData,\n",
        "        answer_start_train_one_hot,\n",
        "        batch_size=32,max_len=150),\n",
        " steps=10\n",
        ")\n"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-44-84b43901dcb9>\"\u001b[0;36m, line \u001b[0;32m7\u001b[0m\n\u001b[0;31m    steps=10\u001b[0m\n\u001b[0m         ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
          ]
        }
      ]
    }
  ]
}