Detecting_depression_using_transcripts (2).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Detecting depression using transcripts.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "AfhUcum8yWjo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "36c0c163-b140-4c15-f8be-a2262128c693"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J6n4XieJCV-u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!pip install tensorflow==1.15\n",
        "#!pip install \"tensorflow_hub>=0.6.0\" "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GmPKuaUqeQ54",
        "colab_type": "text"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y1Hq-nddeM5Z",
        "colab_type": "text"
      },
      "source": [
        "### Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WsyWiBjouAS_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "8b0b819b-0d01-4227-e29b-f71444c46fb9"
      },
      "source": [
        "!pip install chart_studio"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: chart_studio in /usr/local/lib/python3.6/dist-packages (1.1.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from chart_studio) (2.23.0)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from chart_studio) (1.3.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from chart_studio) (1.12.0)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.6/dist-packages (from chart_studio) (4.4.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->chart_studio) (2.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->chart_studio) (2020.4.5.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->chart_studio) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->chart_studio) (1.24.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jU46_D3RpkKv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d97a103f-d9c3-439a-8000-cae64d9d0a8d"
      },
      "source": [
        "#importing all libraries\n",
        "import matplotlib.pyplot as plt\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "import os\n",
        "import re\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import chart_studio.plotly as py\n",
        "import plotly.graph_objs as go\n",
        "import itertools\n",
        "from scipy import stats\n",
        "from ast import literal_eval\n",
        "from sklearn.metrics import accuracy_score, classification_report\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hAvExihmkly8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.models import Sequential, Model\n",
        "from keras.layers import Dense, Dropout, Embedding, LSTM, Input, Activation, GlobalAveragePooling1D, Flatten,Permute,concatenate, multiply, Lambda, Concatenate,RepeatVector, Conv1D, MaxPooling1D,Bidirectional,TimeDistributed,CuDNNLSTM,Reshape,Conv2D,MaxPool2D\n",
        "from keras.layers.normalization import BatchNormalization\n",
        "from keras.layers.merge import concatenate\n",
        "from keras.optimizers import SGD, RMSprop, Adagrad, Adam\n",
        "from keras.preprocessing.text import one_hot, text_to_word_sequence, Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
        "from keras.utils.vis_utils import plot_model\n",
        "\n",
        "import fnmatch\n",
        "\n",
        "import warnings\n",
        "\n",
        "import string\n",
        "from pathlib import Path\n",
        "from random import shuffle\n",
        "from ast import literal_eval\n",
        "\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QAESJfKM54j8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "8d9d464f-5bc8-4afb-d368-101cc5f9a83f"
      },
      "source": [
        "!pip install nltk\n",
        "import nltk"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W5OsnVhXcUji",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "b204d21c-8685-4daf-a1cd-494f5f5f08d8"
      },
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('punkt')\n",
        "\n",
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "wordnet_lemmatizer = WordNetLemmatizer()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mAMgk2YztzS6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gensim\n",
        "import json\n",
        "\n",
        "\n",
        "import nltk\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import PorterStemmer\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.utils.vis_utils import plot_model\n",
        "import keras.utils\n",
        "from keras import utils as np_utils\n",
        "\n",
        "#Keras Tokenizer just replaces certain punctuation characters and splits on the remaining space character.\n",
        "#NLTK Tokenizer uses the Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank. "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xh2ZvAITIO2F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "WINDOWS_SIZE = 10\n",
        "labels=['none','mild','moderate','moderately severe', 'severe']\n",
        "num_classes = len(labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jKST8w7Di96R",
        "colab_type": "text"
      },
      "source": [
        "### Creating a dataframe from the transcript file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wtNdhS37uO9-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def transcripts_to_dataframe(directory):\n",
        "    rows_list = []\n",
        "        \n",
        "    filenames = os.listdir(directory)\n",
        "    \n",
        "    if \".DS_Store\" in filenames:\n",
        "        filenames.remove(\".DS_Store\")\n",
        "        \n",
        "    for filename in filenames:\n",
        "        transcript_path = os.path.join(directory, filename)\n",
        "        transcript = pd.read_csv(transcript_path, sep='\\t')\n",
        "        m = re.search(\"(\\d{3})_TRANSCRIPT.csv\", filename)\n",
        "        if m:\n",
        "            person_id = m.group(1)\n",
        "            p = {}\n",
        "            question = \"\"\n",
        "            answer = \"\"\n",
        "            lines = len(transcript)\n",
        "            for i in range(0, lines):\n",
        "                row = transcript.iloc[i]\n",
        "                if (row[\"speaker\"] == \"Ellie\") or (i == lines - 1):\n",
        "                    p[\"personId\"] = person_id\n",
        "                    if \"(\" in str(question):\n",
        "                        question = question[question.index(\"(\") + 1:question.index(\")\")]\n",
        "                    p[\"question\"] = question\n",
        "                    p[\"answer\"] = answer\n",
        "                    if question != \"\":\n",
        "                        rows_list.append(p)\n",
        "                    p = {}\n",
        "                    answer = \"\"\n",
        "                    question = row[\"value\"]\n",
        "                else:\n",
        "                    answer = str(answer) + \" \" + str(row[\"value\"])\n",
        "\n",
        "    all_participants = pd.DataFrame(rows_list, columns=['personId', 'question', 'answer'])\n",
        "    all_participants.to_csv(directory + 'all.csv', sep=',')\n",
        "    print(\"File was created\")\n",
        "    return all_participants"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_2ZvtFYb0Ysv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0e9001eb-a33e-4258-e200-8bd695d97104"
      },
      "source": [
        "#loading the data\n",
        "data_path = \"/content/drive/My Drive/transcripts/\"\n",
        "all_participants = transcripts_to_dataframe(data_path)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "File was created\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zMO4n5vR0nEX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "outputId": "22a0b38b-4c5c-4418-f1fa-ff537109e89d"
      },
      "source": [
        "all_participants.head(20)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>300</td>\n",
              "      <td>hi i'm ellie thanks for coming in today</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>300</td>\n",
              "      <td>i was created to talk to people in a safe and ...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>300</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>300</td>\n",
              "      <td>i'm here to learn about people and would love ...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>300</td>\n",
              "      <td>i'll ask a few questions to get us started and...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>300</td>\n",
              "      <td>how are you doing today</td>\n",
              "      <td>good</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>300</td>\n",
              "      <td>that's good</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>300</td>\n",
              "      <td>where are you from originally</td>\n",
              "      <td>atlanta georgia</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>300</td>\n",
              "      <td>really</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>300</td>\n",
              "      <td>why'd you move to l_a</td>\n",
              "      <td>um my parents are from here um</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>300</td>\n",
              "      <td>how do you like l_a</td>\n",
              "      <td>i love it</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>300</td>\n",
              "      <td>what are some things you really like about l_a</td>\n",
              "      <td>i like the weather i like the opportunities u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>300</td>\n",
              "      <td>how easy was it for you to get used to living ...</td>\n",
              "      <td>um it took a minute somewhat easy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>300</td>\n",
              "      <td>what are some things you don't really like abo...</td>\n",
              "      <td>congestion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>300</td>\n",
              "      <td>mhm</td>\n",
              "      <td>that's it</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>300</td>\n",
              "      <td>okay</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>300</td>\n",
              "      <td>what'd you study at school</td>\n",
              "      <td>um i took up business and administration</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>300</td>\n",
              "      <td>cool</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>300</td>\n",
              "      <td>are you still doing that</td>\n",
              "      <td>uh yeah i am here and there i'm on a break ri...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>300</td>\n",
              "      <td>what's your dream job</td>\n",
              "      <td>uh probably to open up my own business</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   personId  ...                                             answer\n",
              "0       300  ...                                                   \n",
              "1       300  ...                                                   \n",
              "2       300  ...                                                   \n",
              "3       300  ...                                                   \n",
              "4       300  ...                                                   \n",
              "5       300  ...                                               good\n",
              "6       300  ...                                                   \n",
              "7       300  ...                                    atlanta georgia\n",
              "8       300  ...                                                   \n",
              "9       300  ...                     um my parents are from here um\n",
              "10      300  ...                                          i love it\n",
              "11      300  ...   i like the weather i like the opportunities u...\n",
              "12      300  ...                  um it took a minute somewhat easy\n",
              "13      300  ...                                         congestion\n",
              "14      300  ...                                          that's it\n",
              "15      300  ...                                                   \n",
              "16      300  ...           um i took up business and administration\n",
              "17      300  ...                                                   \n",
              "18      300  ...   uh yeah i am here and there i'm on a break ri...\n",
              "19      300  ...             uh probably to open up my own business\n",
              "\n",
              "[20 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m1YNh1jN5fFs",
        "colab_type": "text"
      },
      "source": [
        "### Removing the stopwords and cleaning the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UNh5CHOz3O6J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#https://www.kaggle.com/currie32/the-importance-of-cleaning-text\n",
        "\n",
        "def text_to_wordlist(text, remove_stopwords=True, stem_words=False):    \n",
        "    # Clean the text, with the option to remove stopwords and to stem words.\n",
        "    \n",
        "    # Convert words to lower case and split them\n",
        "    text = text.lower().split()\n",
        "\n",
        "    # Optionally, remove stop words\n",
        "    if remove_stopwords:\n",
        "        stops = stopwords.words(\"english\")\n",
        "        text = [wordnet_lemmatizer.lemmatize(w) for w in text if not w in stops ]\n",
        "        text = [w for w in text if w != \"nan\" ]\n",
        "    else:\n",
        "        text = [wordnet_lemmatizer.lemmatize(w) for w in text]\n",
        "        text = [w for w in text if w != \"nan\" ]\n",
        "    \n",
        "    text = \" \".join(text)\n",
        "\n",
        "    # Clean the text\n",
        "    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n",
        "    text = re.sub(r\"what's\", \"what is \", text)\n",
        "    text = re.sub(r\"\\'s\", \" \", text)\n",
        "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
        "    text = re.sub(r\"can't\", \"cannot \", text)\n",
        "    text = re.sub(r\"n't\", \" not \", text)\n",
        "    text = re.sub(r\"i'm\", \"i am \", text)\n",
        "    text = re.sub(r\"\\'re\", \" are \", text)\n",
        "    text = re.sub(r\"\\'d\", \" would \", text)\n",
        "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
        "    text = re.sub(r\",\", \" \", text)\n",
        "    text = re.sub(r\"\\.\", \" \", text)\n",
        "    text = re.sub(r\"!\", \" ! \", text)\n",
        "    text = re.sub(r\"\\/\", \" \", text)\n",
        "    text = re.sub(r\"\\^\", \" ^ \", text)\n",
        "    text = re.sub(r\"\\+\", \" + \", text)\n",
        "    text = re.sub(r\"\\-\", \" - \", text)\n",
        "    text = re.sub(r\"\\=\", \" = \", text)\n",
        "    \n",
        "    text = re.sub(r\"\\<\", \" \", text)\n",
        "    text = re.sub(r\"\\>\", \" \", text)\n",
        "    \n",
        "    text = re.sub(r\"'\", \" \", text)\n",
        "    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n",
        "    text = re.sub(r\":\", \" : \", text)\n",
        "    text = re.sub(r\" e g \", \" eg \", text)\n",
        "    text = re.sub(r\" b g \", \" bg \", text)\n",
        "    text = re.sub(r\" u s \", \" american \", text)\n",
        "    text = re.sub(r\"\\0s\", \"0\", text)\n",
        "    text = re.sub(r\" 9 11 \", \"911\", text)\n",
        "    text = re.sub(r\"e - mail\", \"email\", text)\n",
        "    text = re.sub(r\"j k\", \"jk\", text)\n",
        "    text = re.sub(r\"\\s{2,}\", \" \", text)\n",
        "    \n",
        "    # Optionally, shorten words to their stems\n",
        "    if stem_words:\n",
        "        text = text.split()\n",
        "        stemmer = SnowballStemmer('english')\n",
        "        stemmed_words = [stemmer.stem(word) for word in text]\n",
        "        text = \" \".join(stemmed_words)\n",
        "    \n",
        "    # Return a list of words\n",
        "    return(text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fY-vwQ-a5oRX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#creating a corpus with the words from the answers without stopwords given by the patients\n",
        "all_participants_mix = all_participants.copy() # However, if you need the original list unchanged when the new list is modified, you can use copy() method. This is called shallow copy.\n",
        "all_participants_mix['answer'] = all_participants_mix.apply(lambda row: text_to_wordlist(row.answer).split(), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eFthKFlxG_5e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##creating a corpus with the words from the answers withstopwords given by the patients\n",
        "# stopwords are ={what is , have,}\n",
        "all_participants_mix_stopwords = all_participants.copy()\n",
        "all_participants_mix_stopwords['answer'] = all_participants_mix_stopwords.apply(lambda row: text_to_wordlist(row.answer, remove_stopwords=False).split(), axis=1)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_xNLUgQj5xZK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "words = [w for w in all_participants_mix['answer'].tolist()]\n",
        "words = set(itertools.chain(*words)) #chain('ABC', 'DEF') --> A B C D E F\n",
        "vocab_size = len(words)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wvJU9CR8jPl0",
        "colab_type": "text"
      },
      "source": [
        "### Getting the top common words used by the patients"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YT52O9-p-Fel",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "a6f1009b-3162-4a4f-dbff-0261e72921eb"
      },
      "source": [
        "words"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'grant',\n",
              " 'bumpy',\n",
              " 'kn',\n",
              " 'trophy',\n",
              " 'karenina',\n",
              " 'pretend',\n",
              " 'stage',\n",
              " 'ed',\n",
              " 'merit',\n",
              " 'spot',\n",
              " 'acting',\n",
              " 'diversion',\n",
              " 'anaheim',\n",
              " 'curtails',\n",
              " 'mode',\n",
              " 'town',\n",
              " 'custody',\n",
              " 'irritate',\n",
              " 'barber',\n",
              " 'energetic',\n",
              " 'sweet',\n",
              " 'blissed',\n",
              " 'entertai',\n",
              " 'ok',\n",
              " 'producer',\n",
              " 'brazil',\n",
              " 'exploded',\n",
              " 'shooting',\n",
              " 'grind',\n",
              " 'gnarly',\n",
              " 'grinding',\n",
              " 'cooking',\n",
              " 'crack',\n",
              " 'expert',\n",
              " 'directorship',\n",
              " 'res',\n",
              " 'week',\n",
              " 'since',\n",
              " 'playful',\n",
              " 'edge',\n",
              " 'in',\n",
              " 'good',\n",
              " 'spay',\n",
              " 'spill',\n",
              " 'neurologist',\n",
              " 'argu',\n",
              " 'heritage',\n",
              " 'l',\n",
              " 'heaven',\n",
              " 'foggy',\n",
              " 'rockstar',\n",
              " 'breakdown',\n",
              " 'abbey',\n",
              " 'pareos',\n",
              " 'deceitful',\n",
              " 'parent',\n",
              " 'disappointment',\n",
              " 'hippie',\n",
              " 'physician',\n",
              " 'achieve',\n",
              " 'extraneous',\n",
              " 'blue',\n",
              " 'trainer',\n",
              " 'hungary',\n",
              " 'invol',\n",
              " 'rein',\n",
              " 'investing',\n",
              " 'worsened',\n",
              " 'accepts',\n",
              " 'insightful',\n",
              " 'master',\n",
              " 'accounting',\n",
              " 'severe',\n",
              " 'got',\n",
              " 'subject',\n",
              " 'bitter',\n",
              " 'next',\n",
              " 'instruction',\n",
              " 'canceling',\n",
              " 'detail',\n",
              " 'sigh',\n",
              " 'breed',\n",
              " 'huh',\n",
              " 'discovered',\n",
              " 'lackadaisical',\n",
              " 'anyway',\n",
              " 'pregnancy',\n",
              " 'come',\n",
              " 'cost',\n",
              " 'casino',\n",
              " 'shh',\n",
              " 'obtain',\n",
              " 'overcoming',\n",
              " 'slip',\n",
              " 'abrupt',\n",
              " 'capture',\n",
              " 'fact',\n",
              " 'visual',\n",
              " 'mescaline',\n",
              " 'murphy',\n",
              " 'competition',\n",
              " 'nap',\n",
              " 'avoidance',\n",
              " 'deserved',\n",
              " 'thirteen',\n",
              " 'unit',\n",
              " 'girl',\n",
              " 'planned',\n",
              " 'dad',\n",
              " 'become',\n",
              " 'adjust',\n",
              " 'matured',\n",
              " 'pauper',\n",
              " 'rattle',\n",
              " 'cope',\n",
              " 'crosswalk',\n",
              " 'however',\n",
              " 'deserve',\n",
              " 'independence',\n",
              " 'wyoming',\n",
              " 'bike',\n",
              " 'experienced',\n",
              " 'starting',\n",
              " 'fashion',\n",
              " 'worldwide',\n",
              " 'cracking',\n",
              " 'groove',\n",
              " 'tattoo',\n",
              " 'invisible',\n",
              " 'verbally',\n",
              " 'certain',\n",
              " 'boundary',\n",
              " 'viewing',\n",
              " 'permanent',\n",
              " 'monetize',\n",
              " 'offensive',\n",
              " 'cuisine',\n",
              " 'chili',\n",
              " 'reser',\n",
              " 'london',\n",
              " 'pleasant',\n",
              " 'coincidentally',\n",
              " 'collard',\n",
              " 'default',\n",
              " 'pleaser',\n",
              " 'becoming',\n",
              " 'effect',\n",
              " 'rachel',\n",
              " 'faithful',\n",
              " 'saying',\n",
              " 'abandoning',\n",
              " 'enlisted',\n",
              " 'neutral',\n",
              " 'phys',\n",
              " 'picking',\n",
              " 'year',\n",
              " 'underst',\n",
              " 'prospect',\n",
              " 'black',\n",
              " 'unhappy',\n",
              " 'documentary',\n",
              " 'abundance',\n",
              " 'ea',\n",
              " 'bewildered',\n",
              " 'noisy',\n",
              " 'commonality',\n",
              " 'inquisitive',\n",
              " 'treat',\n",
              " 'knaan',\n",
              " 'to',\n",
              " 'stretching',\n",
              " 'november',\n",
              " 'dryer',\n",
              " 'shirt',\n",
              " 'wou',\n",
              " 'boy',\n",
              " 'annoy',\n",
              " 'incredibly',\n",
              " 'belonged',\n",
              " 'presence',\n",
              " 'grasp',\n",
              " 'resulting',\n",
              " 'design',\n",
              " 'scuba',\n",
              " 'premature',\n",
              " 'prestige',\n",
              " 'ayn',\n",
              " 'post',\n",
              " 'inqui',\n",
              " 'continual',\n",
              " 'shame',\n",
              " 'tax',\n",
              " 'regretful',\n",
              " 'altercation',\n",
              " 'specifically',\n",
              " 'spree',\n",
              " 'dy',\n",
              " 'benefited',\n",
              " 'knowledge',\n",
              " 'estranged',\n",
              " 'suck',\n",
              " 'maritime',\n",
              " 'farmer',\n",
              " 'query',\n",
              " 'idiot',\n",
              " 'weekly',\n",
              " 'thought',\n",
              " 'compatible',\n",
              " 'somthing',\n",
              " 'rich',\n",
              " 'flyer',\n",
              " 'haunt',\n",
              " 'toolbox',\n",
              " 'right',\n",
              " 'ship',\n",
              " 'acclimate',\n",
              " 'overwhelming',\n",
              " 'screw',\n",
              " 'devil',\n",
              " 'side',\n",
              " 'sometime',\n",
              " 'ethiopia',\n",
              " 'identify',\n",
              " 'making',\n",
              " 'assistance',\n",
              " 'wage',\n",
              " 'certainly',\n",
              " 'cancun',\n",
              " 'mediocre',\n",
              " 'meaningless',\n",
              " 'ticket',\n",
              " 'want',\n",
              " 'seattle',\n",
              " 'al',\n",
              " 'literally',\n",
              " 'ynez',\n",
              " 'bernardino',\n",
              " 'legitimate',\n",
              " 'portugal',\n",
              " 'sweat',\n",
              " 'rode',\n",
              " 'nobody',\n",
              " 'quitter',\n",
              " 'gasp',\n",
              " 'flexible',\n",
              " 'welcome',\n",
              " 'unfulfilling',\n",
              " 'rebel',\n",
              " 'bud',\n",
              " 'disappoint',\n",
              " 'paced',\n",
              " 'flowing',\n",
              " 'wagon',\n",
              " 'acce',\n",
              " 'dwindling',\n",
              " 'overcome',\n",
              " 'sensor',\n",
              " 'costly',\n",
              " 'killed',\n",
              " 'twist',\n",
              " 'outlive',\n",
              " 'cleaned',\n",
              " 'finding',\n",
              " 'response',\n",
              " 'fifty',\n",
              " 'through',\n",
              " 'rejecting',\n",
              " 'kids',\n",
              " 'karma',\n",
              " 'trans',\n",
              " 'military',\n",
              " 'registered',\n",
              " 'introvert',\n",
              " 'trazadone',\n",
              " 'illegal',\n",
              " 'veers',\n",
              " 'auth',\n",
              " 'accidentally',\n",
              " 'philosphy',\n",
              " 'paid',\n",
              " 'homie',\n",
              " 'exhilirating',\n",
              " 'pomona',\n",
              " 'positiveness',\n",
              " 'mistake',\n",
              " 'betterment',\n",
              " 'freefalling',\n",
              " 'scientologist',\n",
              " 'se',\n",
              " 'furniture',\n",
              " 'spacey',\n",
              " 'sin',\n",
              " 'picture',\n",
              " 'gunpoint',\n",
              " 'imposing',\n",
              " 'ethereal',\n",
              " 'napa',\n",
              " 'calmly',\n",
              " 'inmate',\n",
              " 'autism',\n",
              " 'arrested',\n",
              " 'crave',\n",
              " 'price',\n",
              " 'smooth',\n",
              " 'retainer',\n",
              " 'crime',\n",
              " 'mark',\n",
              " 'lacked',\n",
              " 'unconditionally',\n",
              " 'forbid',\n",
              " 'fused',\n",
              " 'dreaming',\n",
              " 'arthritis',\n",
              " 'n',\n",
              " 'vicious',\n",
              " 'ordained',\n",
              " 'concert',\n",
              " 'photographing',\n",
              " 'swim',\n",
              " 'dressed',\n",
              " 'loving',\n",
              " 'rectify',\n",
              " 'roadtrip',\n",
              " 'person',\n",
              " 'fulfilling',\n",
              " 'austria',\n",
              " 'lashing',\n",
              " 'formally',\n",
              " 'enjoy',\n",
              " 'ran',\n",
              " 'progressed',\n",
              " 'dunking',\n",
              " 'gossip',\n",
              " 'politics',\n",
              " 'two',\n",
              " 'alley',\n",
              " 'problem',\n",
              " 'mult',\n",
              " 'reso',\n",
              " 'pay',\n",
              " 'programming',\n",
              " 'chameleon',\n",
              " 'prissy',\n",
              " 'juicing',\n",
              " 'hurry',\n",
              " 'briggs',\n",
              " 'humorous',\n",
              " 'remove',\n",
              " 'giza',\n",
              " 'collision',\n",
              " 'react',\n",
              " 'milwaukee',\n",
              " 'monument',\n",
              " 'variation',\n",
              " 'fimmaker',\n",
              " 'hypnotherapist',\n",
              " 'true',\n",
              " 'harm',\n",
              " 'terribly',\n",
              " 'confederate',\n",
              " 'outer',\n",
              " 'fearless',\n",
              " 'recreational',\n",
              " 'bay',\n",
              " 'fog',\n",
              " 'sprayed',\n",
              " 'angry',\n",
              " 'detox',\n",
              " 'contemplative',\n",
              " 'carlos',\n",
              " 'survival',\n",
              " 'wore',\n",
              " 'prominent',\n",
              " 'flick',\n",
              " 'agree',\n",
              " 'husband',\n",
              " 'flat',\n",
              " 'rash',\n",
              " 'idolize',\n",
              " 'month',\n",
              " 'software',\n",
              " 'kno',\n",
              " 'ss',\n",
              " 'tolerated',\n",
              " 'ordinary',\n",
              " 'playing',\n",
              " 'frien',\n",
              " 'sticking',\n",
              " 'fish',\n",
              " 'public',\n",
              " 'fallback',\n",
              " 'green',\n",
              " 'exception',\n",
              " 'owed',\n",
              " 'could',\n",
              " 'music',\n",
              " 'contraction',\n",
              " 'categorized',\n",
              " 'looping',\n",
              " 'continuing',\n",
              " 'apnea',\n",
              " 'man',\n",
              " 'fate',\n",
              " 'least',\n",
              " 'alluded',\n",
              " 'procrastinate',\n",
              " 'o',\n",
              " 'roa',\n",
              " 'chill',\n",
              " 'aneenia',\n",
              " 'smaller',\n",
              " 'dramatic',\n",
              " 'smoother',\n",
              " 'explore',\n",
              " 'naitivity',\n",
              " 'discovery',\n",
              " 'clue',\n",
              " 'purpuse',\n",
              " 'referred',\n",
              " 'scattered',\n",
              " 'encounter',\n",
              " 'memorable',\n",
              " 'journalism',\n",
              " 'loiusiana',\n",
              " 'unethical',\n",
              " 'tight',\n",
              " 'german',\n",
              " 'devastated',\n",
              " 'harsh',\n",
              " 'diffing',\n",
              " 'desk',\n",
              " 'disagreeing',\n",
              " 'astrological',\n",
              " 'stigma',\n",
              " 'trial',\n",
              " 'circus',\n",
              " 'possible',\n",
              " 'interactive',\n",
              " 'graphic',\n",
              " 'picker',\n",
              " 'national',\n",
              " 'candle',\n",
              " 'daytime',\n",
              " 'wishing',\n",
              " 'thomas',\n",
              " 'language',\n",
              " 'specified',\n",
              " 'domestic',\n",
              " 'challenge',\n",
              " 'severely',\n",
              " 'said',\n",
              " 'j',\n",
              " 'enforce',\n",
              " 'parents',\n",
              " 'freaking',\n",
              " 'columnist',\n",
              " 'does',\n",
              " 'competitive',\n",
              " 'invincible',\n",
              " 'therapist',\n",
              " 'intimate',\n",
              " 'twel',\n",
              " 'everyday',\n",
              " 'finger',\n",
              " 'motel',\n",
              " 'sleeper',\n",
              " 'hybrid',\n",
              " 'deny',\n",
              " 'market',\n",
              " 'subconscious',\n",
              " 'essay',\n",
              " 'bungee',\n",
              " 'ah',\n",
              " 'worm',\n",
              " 'sunglass',\n",
              " 'besides',\n",
              " 'exceedingly',\n",
              " 'definition',\n",
              " 'participating',\n",
              " 'pace',\n",
              " 'agitated',\n",
              " 'dutch',\n",
              " 'responded',\n",
              " 'great',\n",
              " 'eats',\n",
              " 'upscale',\n",
              " 'answer',\n",
              " 'miniature',\n",
              " 'sightsee',\n",
              " 'vi',\n",
              " 'nervous',\n",
              " 'periphery',\n",
              " 'tapicitate',\n",
              " 'currently',\n",
              " 'fulfillment',\n",
              " 'rejuvenated',\n",
              " 'answe',\n",
              " 'mention',\n",
              " 'number',\n",
              " 'gambling',\n",
              " 'yawning',\n",
              " 'realm',\n",
              " 'korean',\n",
              " 'thy',\n",
              " 'point',\n",
              " 'ho',\n",
              " 'grip',\n",
              " 'insurance',\n",
              " 'hmm',\n",
              " 'octopus',\n",
              " 'equipped',\n",
              " 'older',\n",
              " 'stri',\n",
              " 'license',\n",
              " 'choice',\n",
              " 'cultural',\n",
              " 'fourteen',\n",
              " 'articulator',\n",
              " 'snap',\n",
              " 'giving',\n",
              " 'ladder',\n",
              " 'powerful',\n",
              " 'steam',\n",
              " 'potholed',\n",
              " 'optometrist',\n",
              " 'unimpeachably',\n",
              " 'overly',\n",
              " 'commonly',\n",
              " 'hot',\n",
              " 'film',\n",
              " 'lasting',\n",
              " 'had',\n",
              " 'entirely',\n",
              " 'skittish',\n",
              " 'volume',\n",
              " 'cayce',\n",
              " 'account',\n",
              " 'adjusting',\n",
              " 'foreign',\n",
              " 'strong',\n",
              " 'employment',\n",
              " 'extension',\n",
              " 'offered',\n",
              " 'remem',\n",
              " 'flashy',\n",
              " 'celebrating',\n",
              " 'antithesis',\n",
              " 'fame',\n",
              " 'aztec',\n",
              " 'forth',\n",
              " 'generation',\n",
              " 'rethink',\n",
              " 'stalactite',\n",
              " 'ha',\n",
              " 'january',\n",
              " 'pen',\n",
              " 'br',\n",
              " 'address',\n",
              " 'overlooked',\n",
              " 'majestic',\n",
              " 'sporadically',\n",
              " 'matrix',\n",
              " 'romania',\n",
              " 'around',\n",
              " 'standup',\n",
              " 'uhon',\n",
              " 'jet',\n",
              " 'attempted',\n",
              " 'cord',\n",
              " 'patriotic',\n",
              " 'city',\n",
              " 'compliment',\n",
              " 'theme',\n",
              " 'relocating',\n",
              " 'deprived',\n",
              " 'found',\n",
              " 'giraffe',\n",
              " 'recycle',\n",
              " 'traveling',\n",
              " 'leaf',\n",
              " 'encompass',\n",
              " 'sending',\n",
              " 'yesterday',\n",
              " 'mainstream',\n",
              " 'switzerland',\n",
              " 'non',\n",
              " 'unemployed',\n",
              " 'ass',\n",
              " 'filmm',\n",
              " 'seven',\n",
              " 'rub',\n",
              " 'lifelong',\n",
              " 'majored',\n",
              " 'extreme',\n",
              " 'authentic',\n",
              " 'awoken',\n",
              " 'motocross',\n",
              " 'honestly',\n",
              " 'guru',\n",
              " 'much',\n",
              " 'etcetera',\n",
              " 'are',\n",
              " 'supervisory',\n",
              " 'elbow',\n",
              " 'imagination',\n",
              " 'obligated',\n",
              " 'boston',\n",
              " 'politely',\n",
              " 'helped',\n",
              " 'designated',\n",
              " 'flashiness',\n",
              " 'disingenuous',\n",
              " 'breathtakingly',\n",
              " 'mattress',\n",
              " 'edgar',\n",
              " 'ty',\n",
              " 'free',\n",
              " 'network',\n",
              " 'yin',\n",
              " 'consumer',\n",
              " 'dysthymia',\n",
              " 'depending',\n",
              " 'concentrating',\n",
              " 'coach',\n",
              " 'thankful',\n",
              " 'typically',\n",
              " 'consciously',\n",
              " 'onset',\n",
              " 'bring',\n",
              " 'surround',\n",
              " 'righteously',\n",
              " 'daylight',\n",
              " 'minor',\n",
              " 'raised',\n",
              " 'cousin',\n",
              " 'greener',\n",
              " 'concentration',\n",
              " 'portray',\n",
              " 'orient',\n",
              " 'rid',\n",
              " 'chair',\n",
              " 'unchained',\n",
              " 'einstein',\n",
              " 'date',\n",
              " 'criticize',\n",
              " 'midland',\n",
              " 'bloomington',\n",
              " 'everthing',\n",
              " 'background',\n",
              " 'mediation',\n",
              " 'egotistical',\n",
              " 'fu',\n",
              " 'determine',\n",
              " 'cubicle',\n",
              " 'desire',\n",
              " 'close',\n",
              " 'et',\n",
              " 'translate',\n",
              " 'accurate',\n",
              " 'emotion',\n",
              " 'press',\n",
              " 'remembering',\n",
              " 'figured',\n",
              " 'taco',\n",
              " 'chilling',\n",
              " 'asperger',\n",
              " 'dog',\n",
              " 'rift',\n",
              " 'location',\n",
              " 'sears',\n",
              " 'te',\n",
              " 'madagascar',\n",
              " 'residual',\n",
              " 'device',\n",
              " 'prepared',\n",
              " 'influenced',\n",
              " 'might',\n",
              " 'connection',\n",
              " 'purposeful',\n",
              " 'alls',\n",
              " 'technique',\n",
              " 'madhya',\n",
              " 'guard',\n",
              " 'nicer',\n",
              " 'strength',\n",
              " 'questionnaire',\n",
              " 'buddha',\n",
              " 'perfume',\n",
              " 'hide',\n",
              " 'ecstatic',\n",
              " 'available',\n",
              " 'goal',\n",
              " 'thanks',\n",
              " 'communit',\n",
              " 'amazes',\n",
              " 'attorney',\n",
              " 'parole',\n",
              " 'tower',\n",
              " 'calms',\n",
              " 'consuming',\n",
              " 'masturbate',\n",
              " 'blow',\n",
              " 'petitioning',\n",
              " 'tail',\n",
              " 'youtube',\n",
              " 'for',\n",
              " 'lip',\n",
              " 'autonomy',\n",
              " 'promise',\n",
              " 'graffiti',\n",
              " 'conceivably',\n",
              " 'depend',\n",
              " 'take',\n",
              " 'cat',\n",
              " 'nominated',\n",
              " 'whoever',\n",
              " 'within',\n",
              " 'declutter',\n",
              " 'another',\n",
              " 'shape',\n",
              " 'father',\n",
              " 'gam',\n",
              " 'suddenly',\n",
              " 'thrown',\n",
              " 'weakness',\n",
              " 'unselfish',\n",
              " 'adversity',\n",
              " 'requirement',\n",
              " 'hooked',\n",
              " 'barely',\n",
              " 'exit',\n",
              " 'mood',\n",
              " 'phuket',\n",
              " 'surgery',\n",
              " 'whatsoever',\n",
              " 'zz',\n",
              " 'bashful',\n",
              " 'fie',\n",
              " 'financial',\n",
              " 'faux',\n",
              " 'draconian',\n",
              " 'irrational',\n",
              " 'candy',\n",
              " 'plague',\n",
              " 'peace',\n",
              " 'effortlessly',\n",
              " 'rut',\n",
              " 'humanity',\n",
              " 'xanax',\n",
              " 'signature',\n",
              " 'spread',\n",
              " 'philosophical',\n",
              " 'think',\n",
              " 'lane',\n",
              " 'banos',\n",
              " 'avoided',\n",
              " 'merchandise',\n",
              " 'insists',\n",
              " 'throwing',\n",
              " 'molding',\n",
              " 'promotion',\n",
              " 'norwegian',\n",
              " 'occur',\n",
              " 'countryside',\n",
              " 'cosmopolitan',\n",
              " 'breathtaking',\n",
              " 'ese',\n",
              " 'forest',\n",
              " 'body',\n",
              " 'cultivate',\n",
              " 'signed',\n",
              " 'hella',\n",
              " 'adapt',\n",
              " 'acquired',\n",
              " 'building',\n",
              " 'helping',\n",
              " 'defines',\n",
              " 'scuse',\n",
              " 'heel',\n",
              " 'expressed',\n",
              " 'logicalness',\n",
              " 'breakup',\n",
              " 'removal',\n",
              " 'trace',\n",
              " 'affectionate',\n",
              " 'santa',\n",
              " 'payment',\n",
              " 'decently',\n",
              " 'disposable',\n",
              " 'nervousness',\n",
              " 'mentally',\n",
              " 'ultimate',\n",
              " 'spend',\n",
              " 'up',\n",
              " 'totality',\n",
              " 'asked',\n",
              " 'gotten',\n",
              " 'derelict',\n",
              " 'sanity',\n",
              " 'pompous',\n",
              " 'staying',\n",
              " 'uplifting',\n",
              " 'heated',\n",
              " 'im',\n",
              " 'tie',\n",
              " 'oldie',\n",
              " 'simultaneously',\n",
              " 'wear',\n",
              " 'squeamish',\n",
              " 'steer',\n",
              " 'allows',\n",
              " 'phy',\n",
              " 'slumdog',\n",
              " 'sight',\n",
              " 'litigated',\n",
              " 'return',\n",
              " 'funding',\n",
              " 'divorce',\n",
              " 'ashamed',\n",
              " 'version',\n",
              " 'drown',\n",
              " 'action',\n",
              " 'study',\n",
              " 'influence',\n",
              " 'stepfather',\n",
              " 'downsizing',\n",
              " 'fear',\n",
              " 'vehicle',\n",
              " 'sci',\n",
              " 'physical',\n",
              " 'provider',\n",
              " 'pastor',\n",
              " 'entertain',\n",
              " 'noise',\n",
              " 'rear',\n",
              " 'diligent',\n",
              " 'grandmother',\n",
              " 'almo',\n",
              " 'inspirational',\n",
              " 'pradesh',\n",
              " 'conveyance',\n",
              " 'behoove',\n",
              " 'dakota',\n",
              " 'sea',\n",
              " 'fine',\n",
              " 'immediate',\n",
              " 'excellent',\n",
              " 'christ',\n",
              " 'stockbrokage',\n",
              " 'freeze',\n",
              " 'main',\n",
              " 'willed',\n",
              " 'trapped',\n",
              " 'adventurous',\n",
              " 'deficit',\n",
              " 'washing',\n",
              " 'watch',\n",
              " 'uncomfortable',\n",
              " 'medium',\n",
              " 'qu',\n",
              " 'diabetic',\n",
              " 'defiant',\n",
              " 'piano',\n",
              " 'crosby',\n",
              " 'interesting',\n",
              " 'mix',\n",
              " 'prostate',\n",
              " 'absorbed',\n",
              " 'rushed',\n",
              " 'headache',\n",
              " 'played',\n",
              " 'woah',\n",
              " 'department',\n",
              " 'aspect',\n",
              " 'broad',\n",
              " 'tortuous',\n",
              " 'draw',\n",
              " 'oof',\n",
              " 'consultant',\n",
              " 'masai',\n",
              " 'northern',\n",
              " 'togethers',\n",
              " 'inhale',\n",
              " 'bumper',\n",
              " 'painful',\n",
              " 'lear',\n",
              " 'philosopher',\n",
              " 'sabotage',\n",
              " 'loosening',\n",
              " 'brazen',\n",
              " 'wavelength',\n",
              " 'box',\n",
              " 'additional',\n",
              " 'polar',\n",
              " 'functional',\n",
              " 'los',\n",
              " 'apply',\n",
              " 'education',\n",
              " 'honking',\n",
              " 'barstow',\n",
              " 'anemia',\n",
              " 'bolivia',\n",
              " 'temperment',\n",
              " 'clears',\n",
              " 'little',\n",
              " 'sniff',\n",
              " 'admire',\n",
              " 'patient',\n",
              " 'fiscal',\n",
              " 'facing',\n",
              " 'orange',\n",
              " 'visit',\n",
              " 'research',\n",
              " 'working',\n",
              " 'clip',\n",
              " 'hindsight',\n",
              " 'beneficial',\n",
              " 'textbook',\n",
              " 'intent',\n",
              " 'deck',\n",
              " 'gallery',\n",
              " 'isolated',\n",
              " 'phenomenal',\n",
              " 'scream',\n",
              " 'yada',\n",
              " 'injured',\n",
              " 'elective',\n",
              " 'pos',\n",
              " 'clearing',\n",
              " 'morocco',\n",
              " 'old',\n",
              " 'networking',\n",
              " 'hop',\n",
              " 'pig',\n",
              " 'oregon',\n",
              " 'groupon',\n",
              " 'regarding',\n",
              " 'suffers',\n",
              " 'wantitis',\n",
              " 'exhausted',\n",
              " 'chemistry',\n",
              " 'turf',\n",
              " 'flee',\n",
              " 'thinker',\n",
              " 'academically',\n",
              " 'ordered',\n",
              " 'mysef',\n",
              " 'gold',\n",
              " 'bear',\n",
              " 'crunch',\n",
              " 'theoretical',\n",
              " 'thousand',\n",
              " 'awhile',\n",
              " 'conducive',\n",
              " 'procrastinating',\n",
              " 'stress',\n",
              " 'realize',\n",
              " 'chicago',\n",
              " 'lamb',\n",
              " 'chemical',\n",
              " 'labor',\n",
              " 'intimidated',\n",
              " 'finance',\n",
              " 'last',\n",
              " 'falling',\n",
              " 'seaf',\n",
              " 'childhood',\n",
              " 'psychosocial',\n",
              " 'tasting',\n",
              " 'uninspired',\n",
              " 'here',\n",
              " 'kayak',\n",
              " 'lupus',\n",
              " 'went',\n",
              " 'happens',\n",
              " 'central',\n",
              " 'among',\n",
              " 'many',\n",
              " 'straighten',\n",
              " 'curiousity',\n",
              " 'trying',\n",
              " 'isolation',\n",
              " 'glaring',\n",
              " 'proph',\n",
              " 'blamed',\n",
              " 'getaway',\n",
              " 'versailles',\n",
              " 'note',\n",
              " 'meantime',\n",
              " 'seclusive',\n",
              " 'intensity',\n",
              " 'believed',\n",
              " 'insane',\n",
              " 'socia',\n",
              " 'abroad',\n",
              " 'okay',\n",
              " 'dublin',\n",
              " 'gotta',\n",
              " 'fill',\n",
              " 'installed',\n",
              " ...}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DgnZUBWYFn29",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "words_stop = [w for w in all_participants_mix_stopwords['answer'].tolist()]\n",
        "words_stop = set(itertools.chain(*words_stop))\n",
        "vocab_size_stop = len(words_stop)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vYALBAJvG388",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3c6b883c-d7d9-4f4a-b2ab-ed2041fd7d36"
      },
      "source": [
        "words_stop"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'grant',\n",
              " 'bumpy',\n",
              " 'kn',\n",
              " 'trophy',\n",
              " 'karenina',\n",
              " 'pretend',\n",
              " 'stage',\n",
              " 'ed',\n",
              " 'merit',\n",
              " 'spot',\n",
              " 'acting',\n",
              " 'diversion',\n",
              " 'anaheim',\n",
              " 'curtails',\n",
              " 'mode',\n",
              " 'town',\n",
              " 'custody',\n",
              " 'irritate',\n",
              " 'barber',\n",
              " 'energetic',\n",
              " 'sweet',\n",
              " 'blissed',\n",
              " 'entertai',\n",
              " 'ok',\n",
              " 'producer',\n",
              " 'brazil',\n",
              " 'exploded',\n",
              " 'shooting',\n",
              " 'grind',\n",
              " 'gnarly',\n",
              " 'grinding',\n",
              " 'cooking',\n",
              " 'crack',\n",
              " 'expert',\n",
              " 'directorship',\n",
              " 'res',\n",
              " 'week',\n",
              " 'since',\n",
              " 'playful',\n",
              " 'edge',\n",
              " 'in',\n",
              " 'good',\n",
              " 'spay',\n",
              " 'spill',\n",
              " 'neurologist',\n",
              " 'argu',\n",
              " 'heritage',\n",
              " 'l',\n",
              " 'heaven',\n",
              " 'foggy',\n",
              " 'rockstar',\n",
              " 'breakdown',\n",
              " 'abbey',\n",
              " 'pareos',\n",
              " 'deceitful',\n",
              " 'parent',\n",
              " 'disappointment',\n",
              " 'hippie',\n",
              " 'physician',\n",
              " 'achieve',\n",
              " 'extraneous',\n",
              " 'blue',\n",
              " 'trainer',\n",
              " 'hungary',\n",
              " 'invol',\n",
              " 'rein',\n",
              " 'investing',\n",
              " 'worsened',\n",
              " 'accepts',\n",
              " 'insightful',\n",
              " 'master',\n",
              " 'accounting',\n",
              " 'severe',\n",
              " 'got',\n",
              " 'subject',\n",
              " 'bitter',\n",
              " 'next',\n",
              " 'instruction',\n",
              " 'canceling',\n",
              " 'detail',\n",
              " 'sigh',\n",
              " 'breed',\n",
              " 'huh',\n",
              " 'discovered',\n",
              " 'lackadaisical',\n",
              " 'anyway',\n",
              " 'pregnancy',\n",
              " 'come',\n",
              " 'cost',\n",
              " 'casino',\n",
              " 'shh',\n",
              " 'obtain',\n",
              " 'overcoming',\n",
              " 'slip',\n",
              " 'abrupt',\n",
              " 'capture',\n",
              " 'fact',\n",
              " 'visual',\n",
              " 'mescaline',\n",
              " 'murphy',\n",
              " 'competition',\n",
              " 'nap',\n",
              " 'avoidance',\n",
              " 'deserved',\n",
              " 'thirteen',\n",
              " 'unit',\n",
              " 'girl',\n",
              " 'planned',\n",
              " 'dad',\n",
              " 'yourself',\n",
              " 'become',\n",
              " 'adjust',\n",
              " 'matured',\n",
              " 'pauper',\n",
              " 'rattle',\n",
              " 'cope',\n",
              " 'crosswalk',\n",
              " 'however',\n",
              " 'deserve',\n",
              " 'independence',\n",
              " 'wyoming',\n",
              " 'bike',\n",
              " 'experienced',\n",
              " 'starting',\n",
              " 'fashion',\n",
              " 'worldwide',\n",
              " 'cracking',\n",
              " 'groove',\n",
              " 'tattoo',\n",
              " 'invisible',\n",
              " 'verbally',\n",
              " 'certain',\n",
              " 'boundary',\n",
              " 'viewing',\n",
              " 'permanent',\n",
              " 'monetize',\n",
              " 'offensive',\n",
              " 'cuisine',\n",
              " 'chili',\n",
              " 'reser',\n",
              " 'london',\n",
              " 'pleasant',\n",
              " 'coincidentally',\n",
              " 'collard',\n",
              " 'default',\n",
              " 'pleaser',\n",
              " 'becoming',\n",
              " 'effect',\n",
              " 'rachel',\n",
              " 'faithful',\n",
              " 'saying',\n",
              " 'abandoning',\n",
              " 'enlisted',\n",
              " 'neutral',\n",
              " 'phys',\n",
              " 'picking',\n",
              " 'year',\n",
              " 'underst',\n",
              " 'prospect',\n",
              " 'black',\n",
              " 'unhappy',\n",
              " 'documentary',\n",
              " 'abundance',\n",
              " 'ea',\n",
              " 'bewildered',\n",
              " 'noisy',\n",
              " 'commonality',\n",
              " 'inquisitive',\n",
              " 'treat',\n",
              " 'to',\n",
              " 'knaan',\n",
              " 'stretching',\n",
              " 'november',\n",
              " 'dryer',\n",
              " 'shirt',\n",
              " 'wou',\n",
              " 'boy',\n",
              " 'annoy',\n",
              " 'incredibly',\n",
              " 'belonged',\n",
              " 'presence',\n",
              " 'grasp',\n",
              " 'resulting',\n",
              " 'design',\n",
              " 'scuba',\n",
              " 'premature',\n",
              " 'prestige',\n",
              " 'ayn',\n",
              " 'post',\n",
              " 'inqui',\n",
              " 'continual',\n",
              " 'shame',\n",
              " 'tax',\n",
              " 'regretful',\n",
              " 'altercation',\n",
              " 'specifically',\n",
              " 'spree',\n",
              " 'dy',\n",
              " 'benefited',\n",
              " 'knowledge',\n",
              " 'estranged',\n",
              " 'suck',\n",
              " 'maritime',\n",
              " 'farmer',\n",
              " 'query',\n",
              " 'idiot',\n",
              " 'weekly',\n",
              " 'thought',\n",
              " 'compatible',\n",
              " 'somthing',\n",
              " 'rich',\n",
              " 'flyer',\n",
              " 'haunt',\n",
              " 'toolbox',\n",
              " 'right',\n",
              " 'ship',\n",
              " 'acclimate',\n",
              " 'overwhelming',\n",
              " 'screw',\n",
              " 'devil',\n",
              " 'side',\n",
              " 'sometime',\n",
              " 'ethiopia',\n",
              " 'identify',\n",
              " 'making',\n",
              " 'assistance',\n",
              " 'wage',\n",
              " 'certainly',\n",
              " 'cancun',\n",
              " 'mediocre',\n",
              " 'meaningless',\n",
              " 'ticket',\n",
              " 'want',\n",
              " 'seattle',\n",
              " 'al',\n",
              " 'literally',\n",
              " 'ynez',\n",
              " 'bernardino',\n",
              " 'legitimate',\n",
              " 'portugal',\n",
              " 'sweat',\n",
              " 'rode',\n",
              " 'nobody',\n",
              " 'quitter',\n",
              " 'gasp',\n",
              " 'flexible',\n",
              " 'welcome',\n",
              " 'unfulfilling',\n",
              " 'rebel',\n",
              " 'bud',\n",
              " 'disappoint',\n",
              " 'paced',\n",
              " 'flowing',\n",
              " 'wagon',\n",
              " 'acce',\n",
              " 'dwindling',\n",
              " 'overcome',\n",
              " 'sensor',\n",
              " 'costly',\n",
              " 'killed',\n",
              " 'twist',\n",
              " 'outlive',\n",
              " 'cleaned',\n",
              " 'finding',\n",
              " 'response',\n",
              " 'fifty',\n",
              " 'through',\n",
              " 'rejecting',\n",
              " 'kids',\n",
              " 'karma',\n",
              " 'trans',\n",
              " 'military',\n",
              " 'registered',\n",
              " 'introvert',\n",
              " 'trazadone',\n",
              " 'illegal',\n",
              " 'veers',\n",
              " 'auth',\n",
              " 'accidentally',\n",
              " 'philosphy',\n",
              " 'paid',\n",
              " 'homie',\n",
              " 'exhilirating',\n",
              " 'pomona',\n",
              " 'positiveness',\n",
              " 'mistake',\n",
              " 'betterment',\n",
              " 'freefalling',\n",
              " 'scientologist',\n",
              " 'se',\n",
              " 'furniture',\n",
              " 'once',\n",
              " 'spacey',\n",
              " 'sin',\n",
              " 'picture',\n",
              " 'gunpoint',\n",
              " 'between',\n",
              " 'imposing',\n",
              " 'ethereal',\n",
              " 'napa',\n",
              " 'calmly',\n",
              " 'inmate',\n",
              " 'autism',\n",
              " 'arrested',\n",
              " 'crave',\n",
              " 'price',\n",
              " 'smooth',\n",
              " 'retainer',\n",
              " 'crime',\n",
              " 'mark',\n",
              " 'lacked',\n",
              " 'unconditionally',\n",
              " 'forbid',\n",
              " 'fused',\n",
              " 'dreaming',\n",
              " 'arthritis',\n",
              " 'n',\n",
              " 'vicious',\n",
              " 'ordained',\n",
              " 'concert',\n",
              " 'photographing',\n",
              " 'swim',\n",
              " 'dressed',\n",
              " 'loving',\n",
              " 'rectify',\n",
              " 'roadtrip',\n",
              " 'person',\n",
              " 'fulfilling',\n",
              " 'austria',\n",
              " 'lashing',\n",
              " 'formally',\n",
              " 'enjoy',\n",
              " 'ran',\n",
              " 'progressed',\n",
              " 'dunking',\n",
              " 'gossip',\n",
              " 'politics',\n",
              " 'two',\n",
              " 'alley',\n",
              " 'problem',\n",
              " 'mult',\n",
              " 'reso',\n",
              " 'pay',\n",
              " 'programming',\n",
              " 'chameleon',\n",
              " 'prissy',\n",
              " 'juicing',\n",
              " 'hurry',\n",
              " 'briggs',\n",
              " 'humorous',\n",
              " 'remove',\n",
              " 'giza',\n",
              " 'collision',\n",
              " 'react',\n",
              " 'milwaukee',\n",
              " 'monument',\n",
              " 'variation',\n",
              " 'fimmaker',\n",
              " 'hypnotherapist',\n",
              " 'true',\n",
              " 'harm',\n",
              " 'terribly',\n",
              " 'confederate',\n",
              " 'outer',\n",
              " 'fearless',\n",
              " 'recreational',\n",
              " 'bay',\n",
              " 'fog',\n",
              " 'sprayed',\n",
              " 'angry',\n",
              " 'detox',\n",
              " 'contemplative',\n",
              " 'carlos',\n",
              " 'survival',\n",
              " 'wore',\n",
              " 'prominent',\n",
              " 'flick',\n",
              " 'agree',\n",
              " 'husband',\n",
              " 'flat',\n",
              " 'rash',\n",
              " 'idolize',\n",
              " 'month',\n",
              " 'software',\n",
              " 'kno',\n",
              " 'ss',\n",
              " 'tolerated',\n",
              " 'ordinary',\n",
              " 'playing',\n",
              " 'frien',\n",
              " 'sticking',\n",
              " 'fish',\n",
              " 'public',\n",
              " 'fallback',\n",
              " 'green',\n",
              " 'exception',\n",
              " 'owed',\n",
              " 'could',\n",
              " 'music',\n",
              " 'contraction',\n",
              " 'categorized',\n",
              " 'looping',\n",
              " 'continuing',\n",
              " 'apnea',\n",
              " 'man',\n",
              " 'fate',\n",
              " 'least',\n",
              " 'alluded',\n",
              " 'procrastinate',\n",
              " 'o',\n",
              " 'roa',\n",
              " 'chill',\n",
              " 'aneenia',\n",
              " 'smaller',\n",
              " 'dramatic',\n",
              " 'smoother',\n",
              " 'explore',\n",
              " 'naitivity',\n",
              " 'discovery',\n",
              " 'clue',\n",
              " 'purpuse',\n",
              " 'referred',\n",
              " 'scattered',\n",
              " 'encounter',\n",
              " 'memorable',\n",
              " 'journalism',\n",
              " 'loiusiana',\n",
              " 'unethical',\n",
              " 'tight',\n",
              " 'german',\n",
              " 'devastated',\n",
              " 'harsh',\n",
              " 'diffing',\n",
              " 'desk',\n",
              " 'disagreeing',\n",
              " 'astrological',\n",
              " 'stigma',\n",
              " 'trial',\n",
              " 'circus',\n",
              " 'possible',\n",
              " 'interactive',\n",
              " 'graphic',\n",
              " 'picker',\n",
              " 'national',\n",
              " 'doe',\n",
              " 'candle',\n",
              " 'daytime',\n",
              " 'wishing',\n",
              " 'thomas',\n",
              " 'language',\n",
              " 'specified',\n",
              " 'domestic',\n",
              " 'challenge',\n",
              " 'severely',\n",
              " 'said',\n",
              " 'j',\n",
              " 'enforce',\n",
              " 'parents',\n",
              " 'freaking',\n",
              " 'columnist',\n",
              " 'does',\n",
              " 'competitive',\n",
              " 'invincible',\n",
              " 'therapist',\n",
              " 'intimate',\n",
              " 'twel',\n",
              " 'everyday',\n",
              " 'finger',\n",
              " 'motel',\n",
              " 'sleeper',\n",
              " 'hybrid',\n",
              " 'deny',\n",
              " 'market',\n",
              " 'subconscious',\n",
              " 'essay',\n",
              " 'bungee',\n",
              " 'ah',\n",
              " 'worm',\n",
              " 'sunglass',\n",
              " 'besides',\n",
              " 'exceedingly',\n",
              " 'definition',\n",
              " 'participating',\n",
              " 'pace',\n",
              " 'agitated',\n",
              " 'dutch',\n",
              " 'responded',\n",
              " 'great',\n",
              " 'eats',\n",
              " 'upscale',\n",
              " 'answer',\n",
              " 'miniature',\n",
              " 'sightsee',\n",
              " 'vi',\n",
              " 'nervous',\n",
              " 'periphery',\n",
              " 'tapicitate',\n",
              " 'currently',\n",
              " 'fulfillment',\n",
              " 'rejuvenated',\n",
              " 'answe',\n",
              " 'mention',\n",
              " 'number',\n",
              " 'gambling',\n",
              " 'yawning',\n",
              " 'realm',\n",
              " 'korean',\n",
              " 'thy',\n",
              " 'point',\n",
              " 'ho',\n",
              " 'grip',\n",
              " 'insurance',\n",
              " 'hmm',\n",
              " 'octopus',\n",
              " 'equipped',\n",
              " 'older',\n",
              " 'stri',\n",
              " 'license',\n",
              " 'choice',\n",
              " 'cultural',\n",
              " 'fourteen',\n",
              " 'articulator',\n",
              " 'snap',\n",
              " 'giving',\n",
              " 'ladder',\n",
              " 'powerful',\n",
              " 'steam',\n",
              " 'potholed',\n",
              " 'optometrist',\n",
              " 'unimpeachably',\n",
              " 'overly',\n",
              " 'commonly',\n",
              " 'hot',\n",
              " 'film',\n",
              " 'lasting',\n",
              " 'had',\n",
              " 'entirely',\n",
              " 'skittish',\n",
              " 'volume',\n",
              " 'cayce',\n",
              " 'account',\n",
              " 'adjusting',\n",
              " 'foreign',\n",
              " 'strong',\n",
              " 'employment',\n",
              " 'extension',\n",
              " 'offered',\n",
              " 'remem',\n",
              " 'flashy',\n",
              " 'celebrating',\n",
              " 'antithesis',\n",
              " 'fame',\n",
              " 'aztec',\n",
              " 'forth',\n",
              " 'generation',\n",
              " 'rethink',\n",
              " 'stalactite',\n",
              " 'ha',\n",
              " 'january',\n",
              " 'pen',\n",
              " 'br',\n",
              " 'address',\n",
              " 'overlooked',\n",
              " 'majestic',\n",
              " 'sporadically',\n",
              " 'matrix',\n",
              " 'romania',\n",
              " 'around',\n",
              " 'standup',\n",
              " 'uhon',\n",
              " 'jet',\n",
              " 'attempted',\n",
              " 'cord',\n",
              " 'patriotic',\n",
              " 'city',\n",
              " 'compliment',\n",
              " 'theme',\n",
              " 'relocating',\n",
              " 'deprived',\n",
              " 'found',\n",
              " 'giraffe',\n",
              " 'recycle',\n",
              " 'traveling',\n",
              " 'leaf',\n",
              " 'encompass',\n",
              " 'sending',\n",
              " 'should',\n",
              " 'yesterday',\n",
              " 'mainstream',\n",
              " 'switzerland',\n",
              " 'non',\n",
              " 'unemployed',\n",
              " 'ass',\n",
              " 'filmm',\n",
              " 'seven',\n",
              " 'rub',\n",
              " 'lifelong',\n",
              " 'majored',\n",
              " 'extreme',\n",
              " 'authentic',\n",
              " 'awoken',\n",
              " 'motocross',\n",
              " 'honestly',\n",
              " 'guru',\n",
              " 'much',\n",
              " 'etcetera',\n",
              " 'are',\n",
              " 'supervisory',\n",
              " 'elbow',\n",
              " 'imagination',\n",
              " 'obligated',\n",
              " 'boston',\n",
              " 'politely',\n",
              " 'helped',\n",
              " 'designated',\n",
              " 'flashiness',\n",
              " 'disingenuous',\n",
              " 'breathtakingly',\n",
              " 'mattress',\n",
              " 'edgar',\n",
              " 'ty',\n",
              " 'free',\n",
              " 'network',\n",
              " 'yin',\n",
              " 'consumer',\n",
              " 'dysthymia',\n",
              " 'were',\n",
              " 'depending',\n",
              " 'concentrating',\n",
              " 'coach',\n",
              " 'thankful',\n",
              " 'typically',\n",
              " 'consciously',\n",
              " 'onset',\n",
              " 'bring',\n",
              " 'themselves',\n",
              " 'surround',\n",
              " 'righteously',\n",
              " 'daylight',\n",
              " 'minor',\n",
              " 'raised',\n",
              " 'cousin',\n",
              " 'greener',\n",
              " 'concentration',\n",
              " 'portray',\n",
              " 'orient',\n",
              " 'rid',\n",
              " 'chair',\n",
              " 'unchained',\n",
              " 'einstein',\n",
              " 'date',\n",
              " 'criticize',\n",
              " 'midland',\n",
              " 'bloomington',\n",
              " 'everthing',\n",
              " 'background',\n",
              " 'mediation',\n",
              " 'egotistical',\n",
              " 'fu',\n",
              " 'determine',\n",
              " 'cubicle',\n",
              " 'desire',\n",
              " 'close',\n",
              " 'et',\n",
              " 'translate',\n",
              " 'accurate',\n",
              " 'emotion',\n",
              " 'press',\n",
              " 'remembering',\n",
              " 'figured',\n",
              " 'taco',\n",
              " 'chilling',\n",
              " 'asperger',\n",
              " 'dog',\n",
              " 'rift',\n",
              " 'location',\n",
              " 'sears',\n",
              " 'te',\n",
              " 'madagascar',\n",
              " 'residual',\n",
              " 'device',\n",
              " 'prepared',\n",
              " 'influenced',\n",
              " 'might',\n",
              " 'connection',\n",
              " 'purposeful',\n",
              " 'alls',\n",
              " 'technique',\n",
              " 'madhya',\n",
              " 'guard',\n",
              " 'nicer',\n",
              " 'strength',\n",
              " 'questionnaire',\n",
              " 'buddha',\n",
              " 'perfume',\n",
              " 'hide',\n",
              " 'ecstatic',\n",
              " 'available',\n",
              " 'goal',\n",
              " 'thanks',\n",
              " 'communit',\n",
              " 'amazes',\n",
              " 'attorney',\n",
              " 'parole',\n",
              " 'tower',\n",
              " 'calms',\n",
              " 'consuming',\n",
              " 'masturbate',\n",
              " 'blow',\n",
              " 'petitioning',\n",
              " 'tail',\n",
              " 'youtube',\n",
              " 'for',\n",
              " 'lip',\n",
              " 'autonomy',\n",
              " 'promise',\n",
              " 'graffiti',\n",
              " 'conceivably',\n",
              " 'depend',\n",
              " 'take',\n",
              " 'cat',\n",
              " 'nominated',\n",
              " 'whoever',\n",
              " 'within',\n",
              " 'declutter',\n",
              " 'another',\n",
              " 'shape',\n",
              " 'father',\n",
              " 'gam',\n",
              " 'suddenly',\n",
              " 'thrown',\n",
              " 'weakness',\n",
              " 'unselfish',\n",
              " 'adversity',\n",
              " 'requirement',\n",
              " 'hooked',\n",
              " 'barely',\n",
              " 'exit',\n",
              " 'mood',\n",
              " 'phuket',\n",
              " 'surgery',\n",
              " 'whatsoever',\n",
              " 'zz',\n",
              " 'bashful',\n",
              " 'fie',\n",
              " 'financial',\n",
              " 'faux',\n",
              " 'draconian',\n",
              " 'irrational',\n",
              " 'candy',\n",
              " 'plague',\n",
              " 'peace',\n",
              " 'effortlessly',\n",
              " 'rut',\n",
              " 'humanity',\n",
              " 'xanax',\n",
              " 'signature',\n",
              " 'spread',\n",
              " 'philosophical',\n",
              " 'think',\n",
              " 'lane',\n",
              " 'banos',\n",
              " 'avoided',\n",
              " 'merchandise',\n",
              " 'insists',\n",
              " 'throwing',\n",
              " 'molding',\n",
              " 'promotion',\n",
              " 'norwegian',\n",
              " 'occur',\n",
              " 'each',\n",
              " 'countryside',\n",
              " 'cosmopolitan',\n",
              " 'breathtaking',\n",
              " 'ese',\n",
              " 'forest',\n",
              " 'body',\n",
              " 'cultivate',\n",
              " 'signed',\n",
              " 'hella',\n",
              " 'adapt',\n",
              " 'acquired',\n",
              " 'building',\n",
              " 'helping',\n",
              " 'defines',\n",
              " 'scuse',\n",
              " 'heel',\n",
              " 'expressed',\n",
              " 'logicalness',\n",
              " 'breakup',\n",
              " 'removal',\n",
              " 'trace',\n",
              " 'affectionate',\n",
              " 'santa',\n",
              " 'payment',\n",
              " 'decently',\n",
              " 'disposable',\n",
              " 'nervousness',\n",
              " 'mentally',\n",
              " 'ultimate',\n",
              " 'spend',\n",
              " 'up',\n",
              " 'totality',\n",
              " 'asked',\n",
              " 'gotten',\n",
              " 'derelict',\n",
              " 'sanity',\n",
              " 'pompous',\n",
              " 'staying',\n",
              " 'uplifting',\n",
              " 'heated',\n",
              " 'im',\n",
              " 'tie',\n",
              " 'oldie',\n",
              " 'simultaneously',\n",
              " 'wear',\n",
              " 'squeamish',\n",
              " 'steer',\n",
              " 'allows',\n",
              " 'phy',\n",
              " 'slumdog',\n",
              " 'sight',\n",
              " 'litigated',\n",
              " 'return',\n",
              " 'funding',\n",
              " 'divorce',\n",
              " 'ashamed',\n",
              " 'version',\n",
              " 'drown',\n",
              " 'action',\n",
              " 'study',\n",
              " 'influence',\n",
              " 'stepfather',\n",
              " 'downsizing',\n",
              " 'fear',\n",
              " 'vehicle',\n",
              " 'sci',\n",
              " 'physical',\n",
              " 'provider',\n",
              " 'pastor',\n",
              " 'entertain',\n",
              " 'noise',\n",
              " 'rear',\n",
              " 'diligent',\n",
              " 'grandmother',\n",
              " 'almo',\n",
              " 'inspirational',\n",
              " 'pradesh',\n",
              " 'conveyance',\n",
              " 'behoove',\n",
              " 'dakota',\n",
              " 'sea',\n",
              " 'fine',\n",
              " 'immediate',\n",
              " 'excellent',\n",
              " 'christ',\n",
              " 'stockbrokage',\n",
              " 'freeze',\n",
              " 'main',\n",
              " 'willed',\n",
              " 'trapped',\n",
              " 'adventurous',\n",
              " 'deficit',\n",
              " 'washing',\n",
              " 'watch',\n",
              " 'uncomfortable',\n",
              " 'medium',\n",
              " 'qu',\n",
              " 'diabetic',\n",
              " 'defiant',\n",
              " 'piano',\n",
              " 'crosby',\n",
              " 'interesting',\n",
              " 'mix',\n",
              " 'prostate',\n",
              " 'absorbed',\n",
              " 'rushed',\n",
              " 'now',\n",
              " 'headache',\n",
              " 'played',\n",
              " 'woah',\n",
              " 'department',\n",
              " 'aspect',\n",
              " 'broad',\n",
              " 'tortuous',\n",
              " 'draw',\n",
              " 'oof',\n",
              " 'consultant',\n",
              " 'masai',\n",
              " 'northern',\n",
              " 'togethers',\n",
              " 'inhale',\n",
              " 'bumper',\n",
              " 'painful',\n",
              " 'lear',\n",
              " 'philosopher',\n",
              " 'sabotage',\n",
              " 'loosening',\n",
              " 'brazen',\n",
              " 'wavelength',\n",
              " 'box',\n",
              " 'additional',\n",
              " 'polar',\n",
              " 'functional',\n",
              " 'los',\n",
              " 'apply',\n",
              " 'education',\n",
              " 'honking',\n",
              " 'barstow',\n",
              " 'anemia',\n",
              " 'bolivia',\n",
              " 'temperment',\n",
              " 'clears',\n",
              " 'little',\n",
              " 'sniff',\n",
              " 'admire',\n",
              " 'patient',\n",
              " 'fiscal',\n",
              " 'facing',\n",
              " 'orange',\n",
              " 'visit',\n",
              " 'research',\n",
              " 'working',\n",
              " 'clip',\n",
              " 'hindsight',\n",
              " 'beneficial',\n",
              " 'textbook',\n",
              " 'intent',\n",
              " 'deck',\n",
              " 'gallery',\n",
              " 'isolated',\n",
              " 'phenomenal',\n",
              " 'scream',\n",
              " 'yada',\n",
              " 'injured',\n",
              " 'elective',\n",
              " 'pos',\n",
              " 'clearing',\n",
              " 'morocco',\n",
              " 'old',\n",
              " 'networking',\n",
              " 'hop',\n",
              " 'pig',\n",
              " 'oregon',\n",
              " 'ours',\n",
              " 'groupon',\n",
              " 'regarding',\n",
              " 'suffers',\n",
              " 'wantitis',\n",
              " 'exhausted',\n",
              " 'chemistry',\n",
              " 'turf',\n",
              " 'flee',\n",
              " 'thinker',\n",
              " 'academically',\n",
              " 'ordered',\n",
              " 'mysef',\n",
              " 'gold',\n",
              " 'bear',\n",
              " 'crunch',\n",
              " 'theoretical',\n",
              " 'thousand',\n",
              " 'awhile',\n",
              " 'conducive',\n",
              " 'procrastinating',\n",
              " 'stress',\n",
              " 'realize',\n",
              " 'chicago',\n",
              " 'lamb',\n",
              " 'chemical',\n",
              " 'labor',\n",
              " 'intimidated',\n",
              " 'finance',\n",
              " 'below',\n",
              " 'last',\n",
              " 'falling',\n",
              " 'seaf',\n",
              " 'childhood',\n",
              " 'psychosocial',\n",
              " 'tasting',\n",
              " 'uninspired',\n",
              " 'here',\n",
              " 'kayak',\n",
              " 'lupus',\n",
              " 'went',\n",
              " 'happens',\n",
              " 'central',\n",
              " 'among',\n",
              " 'many',\n",
              " 'straighten',\n",
              " 'only',\n",
              " 'curiousity',\n",
              " 'trying',\n",
              " 'isolation',\n",
              " 'glaring',\n",
              " 'proph',\n",
              " 'blamed',\n",
              " 'getaway',\n",
              " 'versailles',\n",
              " 'note',\n",
              " ...}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iBMsnKpiHFso",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        },
        "outputId": "9ace1951-bac8-42ad-afb2-e1b6aea324b3"
      },
      "source": [
        "windows_size = WINDOWS_SIZE\n",
        "tokenizer = Tokenizer(num_words=vocab_size)\n",
        "tokenizer.fit_on_texts(all_participants_mix['answer']) # fit_on_texts creates the vocabulary index based on word frequency.\n",
        "#The cat sat on the mat.\" It will create a dictionary s.t. word_index[\"the\"] = 1; word_index[\"cat\"] = 2 it is word -> index dictionary so every word gets a unique integer value. 0 is reserved for padding. So lower integer means more frequent word \n",
        "\n",
        "tokenizer.fit_on_sequences(all_participants_mix['answer']) #texts_to_sequences Transforms each text in texts to a sequence of integers. So it basically takes each word in the text and replaces it with its corresponding integer value from the word_index dictionary.\n",
        "\n",
        "all_participants_mix['t_answer'] = tokenizer.texts_to_sequences(all_participants_mix['answer'])\n",
        "all_participants_mix.head(15)\n",
        "\n",
        "#   why are the output as numbers when text_to_sequences is called?\n",
        "# the Tokenizer stores everything in the word_index during fit_on_texts. Then, when calling the texts_to_sequences method, only the top num_words are considered."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>t_answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>300</td>\n",
              "      <td>hi i'm ellie thanks for coming in today</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>300</td>\n",
              "      <td>i was created to talk to people in a safe and ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>300</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>300</td>\n",
              "      <td>i'm here to learn about people and would love ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>300</td>\n",
              "      <td>i'll ask a few questions to get us started and...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>300</td>\n",
              "      <td>how are you doing today</td>\n",
              "      <td>[good]</td>\n",
              "      <td>[16]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>300</td>\n",
              "      <td>that's good</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>300</td>\n",
              "      <td>where are you from originally</td>\n",
              "      <td>[atlanta, georgia]</td>\n",
              "      <td>[1634, 1997]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>300</td>\n",
              "      <td>really</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>300</td>\n",
              "      <td>why'd you move to l_a</td>\n",
              "      <td>[um, parent, um]</td>\n",
              "      <td>[1, 131, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>300</td>\n",
              "      <td>how do you like l_a</td>\n",
              "      <td>[love]</td>\n",
              "      <td>[63]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>300</td>\n",
              "      <td>what are some things you really like about l_a</td>\n",
              "      <td>[like, weather, like, opportunity, um, yes]</td>\n",
              "      <td>[5, 142, 5, 334, 1, 39]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>300</td>\n",
              "      <td>how easy was it for you to get used to living ...</td>\n",
              "      <td>[um, took, minute, somewhat, easy]</td>\n",
              "      <td>[1, 154, 527, 608, 100]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>300</td>\n",
              "      <td>what are some things you don't really like abo...</td>\n",
              "      <td>[congestion]</td>\n",
              "      <td>[1998]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>300</td>\n",
              "      <td>mhm</td>\n",
              "      <td>[that]</td>\n",
              "      <td>[20]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   personId  ...                 t_answer\n",
              "0       300  ...                       []\n",
              "1       300  ...                       []\n",
              "2       300  ...                       []\n",
              "3       300  ...                       []\n",
              "4       300  ...                       []\n",
              "5       300  ...                     [16]\n",
              "6       300  ...                       []\n",
              "7       300  ...             [1634, 1997]\n",
              "8       300  ...                       []\n",
              "9       300  ...              [1, 131, 1]\n",
              "10      300  ...                     [63]\n",
              "11      300  ...  [5, 142, 5, 334, 1, 39]\n",
              "12      300  ...  [1, 154, 527, 608, 100]\n",
              "13      300  ...                   [1998]\n",
              "14      300  ...                     [20]\n",
              "\n",
              "[15 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yZmZ-FwHH-CJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        },
        "outputId": "950ad8fd-4fc6-44c7-e3a9-da1238ad8ce8"
      },
      "source": [
        "\n",
        "windows_size = WINDOWS_SIZE\n",
        "tokenizer = Tokenizer(num_words=vocab_size_stop)\n",
        "tokenizer.fit_on_texts(all_participants_mix_stopwords['answer'])\n",
        "tokenizer.fit_on_sequences(all_participants_mix_stopwords['answer'])\n",
        "\n",
        "all_participants_mix_stopwords['t_answer'] = tokenizer.texts_to_sequences(all_participants_mix_stopwords['answer'])\n",
        "all_participants_mix_stopwords.head(15)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>t_answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>300</td>\n",
              "      <td>hi i'm ellie thanks for coming in today</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>300</td>\n",
              "      <td>i was created to talk to people in a safe and ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>300</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>300</td>\n",
              "      <td>i'm here to learn about people and would love ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>300</td>\n",
              "      <td>i'll ask a few questions to get us started and...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>300</td>\n",
              "      <td>how are you doing today</td>\n",
              "      <td>[good]</td>\n",
              "      <td>[42]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>300</td>\n",
              "      <td>that's good</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>300</td>\n",
              "      <td>where are you from originally</td>\n",
              "      <td>[atlanta, georgia]</td>\n",
              "      <td>[1727, 2089]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>300</td>\n",
              "      <td>really</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>300</td>\n",
              "      <td>why'd you move to l_a</td>\n",
              "      <td>[um, my, parent, are, from, here, um]</td>\n",
              "      <td>[5, 12, 205, 37, 69, 113, 5]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>300</td>\n",
              "      <td>how do you like l_a</td>\n",
              "      <td>[i, love, it]</td>\n",
              "      <td>[1, 119, 6]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>300</td>\n",
              "      <td>what are some things you really like about l_a</td>\n",
              "      <td>[i, like, the, weather, i, like, the, opportun...</td>\n",
              "      <td>[1, 16, 7, 218, 1, 16, 7, 423, 5, 89]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>300</td>\n",
              "      <td>how easy was it for you to get used to living ...</td>\n",
              "      <td>[um, it, took, a, minute, somewhat, easy]</td>\n",
              "      <td>[5, 6, 232, 3, 620, 700, 166]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>300</td>\n",
              "      <td>what are some things you don't really like abo...</td>\n",
              "      <td>[congestion]</td>\n",
              "      <td>[2090]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>300</td>\n",
              "      <td>mhm</td>\n",
              "      <td>[that, it]</td>\n",
              "      <td>[8, 6]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   personId  ...                               t_answer\n",
              "0       300  ...                                     []\n",
              "1       300  ...                                     []\n",
              "2       300  ...                                     []\n",
              "3       300  ...                                     []\n",
              "4       300  ...                                     []\n",
              "5       300  ...                                   [42]\n",
              "6       300  ...                                     []\n",
              "7       300  ...                           [1727, 2089]\n",
              "8       300  ...                                     []\n",
              "9       300  ...           [5, 12, 205, 37, 69, 113, 5]\n",
              "10      300  ...                            [1, 119, 6]\n",
              "11      300  ...  [1, 16, 7, 218, 1, 16, 7, 423, 5, 89]\n",
              "12      300  ...          [5, 6, 232, 3, 620, 700, 166]\n",
              "13      300  ...                                 [2090]\n",
              "14      300  ...                                 [8, 6]\n",
              "\n",
              "[15 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "81uq4fnuLebm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "387d29b4-f5f2-4455-9c31-dd6433faa2bd"
      },
      "source": [
        "word_index = tokenizer.word_index\n",
        "word_size = len(word_index)\n",
        "print(word_index[\"sad\"])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "654\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AOe7nupgj-kv",
        "colab_type": "text"
      },
      "source": [
        "# Data Augmentation of the tokenized words to improve model performance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BZGbsH_-9Mns",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#windows_size = WINDOWS_SIZE\n",
        "#cont = 0\n",
        "#word_index = tokenizer\n",
        "#phrases_lp = pd.DataFrame(columns=['personId','answer','t_answer'])\n",
        "#answers = all_participants_mix.groupby('personId').agg('sum', axis=1)\n",
        "\n",
        "#for p in answers.iterrows():      \n",
        "    #words = p[1][\"answer\"]\n",
        "    #size = len(words)\n",
        "    #word_tokens = p[1][\"t_answer\"]\n",
        " \n",
        "    #for i in range(size):\n",
        "        #sentence = words[i:min(i+windows_size,size)]  \n",
        "        #tokens = word_tokens[i:min(i+windows_size,size)]  \n",
        "        #phrases_lp.loc[cont] = [p[0], sentence, tokens]\n",
        "        #cont = cont + 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2hMFmhj1oK_5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#phrases_lp.head()\n",
        "#phrases_lp.to_csv('/content/drive/My Drive/Final-year-project-Phase2/phrases_lp.csv', sep='\\t')\n",
        "#print(\"File was created\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-XuEzXgcrKSR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#phrases_lp[\"t_answer\"] = pad_sequences(phrases_lp[\"t_answer\"], value=0, padding=\"post\", maxlen=windows_size).tolist()\n",
        "#phrases_lp.drop(phrases_lp[phrases_lp[\"t_answer\"].map(len) > 10].index, inplace = True)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FfLvcEgJryZm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#phrases_lp_stop.to_csv('/content/drive/My Drive/Final-year-project-Phase2/phrases_lp_stop.csv', sep='\\t')\n",
        "#print(\"File was created\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pFLqZ9G8NEGh",
        "colab_type": "text"
      },
      "source": [
        "### Loading the data augmented dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Lt1nYlfsscT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        },
        "outputId": "728888d2-7df8-43de-abec-3d7666b29ffd"
      },
      "source": [
        "phrases_lp = pd.read_csv('/content/drive/My Drive/Final-year-project-Phase 2/phrases_lp.csv', sep='\\t', converters={\"t_answer\": literal_eval}) \n",
        "phrases_lp.head(15)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>personId</th>\n",
              "      <th>answer</th>\n",
              "      <th>t_answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>300</td>\n",
              "      <td>['good', 'atlanta', 'georgia', 'um', 'parent',...</td>\n",
              "      <td>[16, 1634, 1997, 1, 131, 1, 63, 5, 142, 5]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>300</td>\n",
              "      <td>['atlanta', 'georgia', 'um', 'parent', 'um', '...</td>\n",
              "      <td>[1634, 1997, 1, 131, 1, 63, 5, 142, 5, 334]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>300</td>\n",
              "      <td>['georgia', 'um', 'parent', 'um', 'love', 'lik...</td>\n",
              "      <td>[1997, 1, 131, 1, 63, 5, 142, 5, 334, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>300</td>\n",
              "      <td>['um', 'parent', 'um', 'love', 'like', 'weathe...</td>\n",
              "      <td>[1, 131, 1, 63, 5, 142, 5, 334, 1, 39]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>300</td>\n",
              "      <td>['parent', 'um', 'love', 'like', 'weather', 'l...</td>\n",
              "      <td>[131, 1, 63, 5, 142, 5, 334, 1, 39, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>5</td>\n",
              "      <td>300</td>\n",
              "      <td>['um', 'love', 'like', 'weather', 'like', 'opp...</td>\n",
              "      <td>[1, 63, 5, 142, 5, 334, 1, 39, 1, 154]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>6</td>\n",
              "      <td>300</td>\n",
              "      <td>['love', 'like', 'weather', 'like', 'opportuni...</td>\n",
              "      <td>[63, 5, 142, 5, 334, 1, 39, 1, 154, 527]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>7</td>\n",
              "      <td>300</td>\n",
              "      <td>['like', 'weather', 'like', 'opportunity', 'um...</td>\n",
              "      <td>[5, 142, 5, 334, 1, 39, 1, 154, 527, 608]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>8</td>\n",
              "      <td>300</td>\n",
              "      <td>['weather', 'like', 'opportunity', 'um', 'yes'...</td>\n",
              "      <td>[142, 5, 334, 1, 39, 1, 154, 527, 608, 100]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>9</td>\n",
              "      <td>300</td>\n",
              "      <td>['like', 'opportunity', 'um', 'yes', 'um', 'to...</td>\n",
              "      <td>[5, 334, 1, 39, 1, 154, 527, 608, 100, 1998]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>10</td>\n",
              "      <td>300</td>\n",
              "      <td>['opportunity', 'um', 'yes', 'um', 'took', 'mi...</td>\n",
              "      <td>[334, 1, 39, 1, 154, 527, 608, 100, 1998, 20]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>11</td>\n",
              "      <td>300</td>\n",
              "      <td>['um', 'yes', 'um', 'took', 'minute', 'somewha...</td>\n",
              "      <td>[1, 39, 1, 154, 527, 608, 100, 1998, 20, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>12</td>\n",
              "      <td>300</td>\n",
              "      <td>['yes', 'um', 'took', 'minute', 'somewhat', 'e...</td>\n",
              "      <td>[39, 1, 154, 527, 608, 100, 1998, 20, 1, 154]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>13</td>\n",
              "      <td>300</td>\n",
              "      <td>['um', 'took', 'minute', 'somewhat', 'easy', '...</td>\n",
              "      <td>[1, 154, 527, 608, 100, 1998, 20, 1, 154, 188]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>14</td>\n",
              "      <td>300</td>\n",
              "      <td>['took', 'minute', 'somewhat', 'easy', 'conges...</td>\n",
              "      <td>[154, 527, 608, 100, 1998, 20, 1, 154, 188, 1376]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Unnamed: 0  ...                                           t_answer\n",
              "0            0  ...         [16, 1634, 1997, 1, 131, 1, 63, 5, 142, 5]\n",
              "1            1  ...        [1634, 1997, 1, 131, 1, 63, 5, 142, 5, 334]\n",
              "2            2  ...           [1997, 1, 131, 1, 63, 5, 142, 5, 334, 1]\n",
              "3            3  ...             [1, 131, 1, 63, 5, 142, 5, 334, 1, 39]\n",
              "4            4  ...             [131, 1, 63, 5, 142, 5, 334, 1, 39, 1]\n",
              "5            5  ...             [1, 63, 5, 142, 5, 334, 1, 39, 1, 154]\n",
              "6            6  ...           [63, 5, 142, 5, 334, 1, 39, 1, 154, 527]\n",
              "7            7  ...          [5, 142, 5, 334, 1, 39, 1, 154, 527, 608]\n",
              "8            8  ...        [142, 5, 334, 1, 39, 1, 154, 527, 608, 100]\n",
              "9            9  ...       [5, 334, 1, 39, 1, 154, 527, 608, 100, 1998]\n",
              "10          10  ...      [334, 1, 39, 1, 154, 527, 608, 100, 1998, 20]\n",
              "11          11  ...        [1, 39, 1, 154, 527, 608, 100, 1998, 20, 1]\n",
              "12          12  ...      [39, 1, 154, 527, 608, 100, 1998, 20, 1, 154]\n",
              "13          13  ...     [1, 154, 527, 608, 100, 1998, 20, 1, 154, 188]\n",
              "14          14  ...  [154, 527, 608, 100, 1998, 20, 1, 154, 188, 1376]\n",
              "\n",
              "[15 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YYiNu5gukIAk",
        "colab_type": "text"
      },
      "source": [
        "### Loading the train, validation and test data containing the PHQ Scores"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yXUewPnosxiQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_avec_dataset_file(path,score_column):\n",
        "    ds = pd.read_csv(path, sep=',')\n",
        "    ds['level'] = pd.cut(ds[score_column], bins=[-1,0,5,10,15,25], labels=[0,1,2,3,4])  #cut function used to segregate array into bins 5 levels - 'none','mild','moderate','moderately severe', 'severe'\n",
        "    ds['PHQ8_Score'] = ds[score_column]\n",
        "    ds['cat_level'] = keras.utils.to_categorical(ds['level'], num_classes).tolist() #categorical levels \n",
        "    ds = ds[['Participant_ID', 'level', 'cat_level', 'PHQ8_Score','Gender']] \n",
        "    ds = ds.astype({\"Participant_ID\": float, \"level\": int, 'PHQ8_Score': int})\n",
        "    return ds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UxYpM5l9voi9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "986a2253-b53a-4bd4-f60d-d5c50cbe3929"
      },
      "source": [
        "train = load_avec_dataset_file('/content/drive/My Drive/Depression_detect/train_split_Depression_AVEC2017 (1).csv','PHQ8_Score')\n",
        "dev = load_avec_dataset_file('/content/drive/My Drive/Depression_detect/dev_split_Depression_AVEC2017.csv','PHQ8_Score')\n",
        "test = load_avec_dataset_file('/content/drive/My Drive/Depression_detect/full_test_split.csv','PHQ8_Score')\n",
        "print(\"Size: train= {}, dev= {}, test={}\".format(len(train), len(dev), len(test)))\n",
        "train.head()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Size: train= 107, dev= 35, test=47\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Participant_ID</th>\n",
              "      <th>level</th>\n",
              "      <th>cat_level</th>\n",
              "      <th>PHQ8_Score</th>\n",
              "      <th>Gender</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>303.0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>304.0</td>\n",
              "      <td>2</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>305.0</td>\n",
              "      <td>2</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>310.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>312.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Participant_ID  level                  cat_level  PHQ8_Score  Gender\n",
              "0           303.0      0  [1.0, 0.0, 0.0, 0.0, 0.0]           0       0\n",
              "1           304.0      2  [0.0, 0.0, 1.0, 0.0, 0.0]           6       0\n",
              "2           305.0      2  [0.0, 0.0, 1.0, 0.0, 0.0]           7       1\n",
              "3           310.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           4       1\n",
              "4           312.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           2       1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BJtt2WSUyJf2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "37f3e584-45b7-41f6-c25e-223728533710"
      },
      "source": [
        "ds_total = pd.concat([dev,train,test])\n",
        "total_phq8 = len(ds_total)\n",
        "print(\"Total size = {}\".format(total_phq8))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total size = 189\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7dXT735x-Gn5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "c58a501d-99df-4781-d07b-0d7049e00322"
      },
      "source": [
        "\n",
        "ds_total"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Participant_ID</th>\n",
              "      <th>level</th>\n",
              "      <th>cat_level</th>\n",
              "      <th>PHQ8_Score</th>\n",
              "      <th>Gender</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>302.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>307.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>331.0</td>\n",
              "      <td>2</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>8</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>335.0</td>\n",
              "      <td>3</td>\n",
              "      <td>[0.0, 0.0, 0.0, 1.0, 0.0]</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>346.0</td>\n",
              "      <td>4</td>\n",
              "      <td>[0.0, 0.0, 0.0, 0.0, 1.0]</td>\n",
              "      <td>23</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>467.0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>469.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>470.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>480.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>481.0</td>\n",
              "      <td>2</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>189 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    Participant_ID  level                  cat_level  PHQ8_Score  Gender\n",
              "0            302.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           4       1\n",
              "1            307.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           4       0\n",
              "2            331.0      2  [0.0, 0.0, 1.0, 0.0, 0.0]           8       1\n",
              "3            335.0      3  [0.0, 0.0, 0.0, 1.0, 0.0]          12       0\n",
              "4            346.0      4  [0.0, 0.0, 0.0, 0.0, 1.0]          23       0\n",
              "..             ...    ...                        ...         ...     ...\n",
              "42           467.0      0  [1.0, 0.0, 0.0, 0.0, 0.0]           0       1\n",
              "43           469.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           3       0\n",
              "44           470.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           3       0\n",
              "45           480.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           1       1\n",
              "46           481.0      2  [0.0, 0.0, 1.0, 0.0, 0.0]           7       1\n",
              "\n",
              "[189 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M_NwXFRKw__w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "65166b97-801f-4d68-8f23-c80990560bc5"
      },
      "source": [
        "ds_total.to_csv('/content/drive/My Drive/Depression_detect/ds_total.csv', sep='\\t')\n",
        "print(\"File was created\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "File was created\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "08_JW1g9-NYG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "outputId": "5a2607b1-6009-432f-c6a3-6037b8cd5da3"
      },
      "source": [
        "bins=[-1,0,5,10,15,25]\n",
        "plt.figure()\n",
        "plt.hist(ds_total[\"PHQ8_Score\"], rwidth=0.6, bins=5)\n",
        "plt.xlabel('PHQ8 score')\n",
        "plt.ylabel('Number of participants')\n",
        "plt.show()\n",
        "plt.savefig('/content/drive/My Drive/Depression_detect/bins.png')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAUZElEQVR4nO3dfbRddX3n8fdHIguwPJMiAjGIjA61S8SMxWorSFUerDiOWl0OCyk2dA1aUNvCOFZc07EFp6D4MFo6iNGhIiIVusAHSiUjzgwlQaYQIgvKQAsGiCM2ER8Y4Dt/7B1zc8m92dxkn5Nz9/u11l1n79/Z+5xv9jr53H1/57d/O1WFJGk4njbuAiRJo2XwS9LAGPySNDAGvyQNjMEvSQOzYNwFdLHPPvvU4sWLx12GJE2UlStXfr+qFk5vn4jgX7x4MStWrBh3GZI0UZLcu7l2u3okaWAMfkkaGINfkgbG4JekgTH4JWlgDH5JGhiDX5IGxuCXpIEx+CVpYCbiyt2tsfisq8ddwqzuOef4cZcgaWA845ekgTH4JWlgDH5JGhiDX5IGxuCXpIEx+CVpYAx+SRoYg1+SBsbgl6SBMfglaWB6Df4k706yKsltSb6QZKckByW5McldSb6YZMc+a5Akbaq34E+yP/B7wJKqegGwA/AW4FzgI1X1XOBh4JS+apAkPVnfXT0LgJ2TLAB2AdYArwQub59fBry+5xokSVP0FvxVdT/wZ8A/0gT+PwMrgR9W1WPtZvcB+29u/yRLk6xIsmLt2rV9lSlJg9NnV8+ewAnAQcCzgGcAx3Tdv6ourKolVbVk4cKFPVUpScPTZ1fPbwD/p6rWVtX/A64AXgbs0Xb9ABwA3N9jDZKkafoM/n8EjkiyS5IARwO3A98E3thucxJwZY81SJKm6bOP/0aaL3FvBm5t3+tC4EzgPUnuAvYGLuqrBknSk/V668WqOhs4e1rz3cBL+nxfSdLMvHJXkgbG4JekgTH4JWlgDH5JGhiDX5IGxuCXpIEx+CVpYAx+SRoYg1+SBsbgl6SBMfglaWAMfkkaGINfkgbG4JekgTH4JWlgDH5JGhiDX5IGxuCXpIEx+CVpYAx+SRoYg1+SBsbgl6SBMfglaWC2GPxJTk+yWxoXJbk5yatHUZwkadvrcsb/21W1Dng1sCdwInBOr1VJknrTJfjTPh4HfL6qVk1pkyRNmC7BvzLJN2iC/+tJdgWe6LcsSVJfFnTY5hTgMODuqvpxkr2Bk/stS5LUly5n/NdW1c1V9UOAqvq/wEf6LUuS1JcZz/iT7ATsAuyTZE829uvvBuw/gtokST2YravnVOAM4FnASjYG/zrgEz3XJUnqyYzBX1UXABckeVdVfXyENUmSerTFL3er6uNJfhVYPHX7qvpcj3VJknqyxeBP8nngYOAW4PG2uQCDX5ImUJfhnEuAQ6uq+i5GktS/LsM5bwOe2XchkqTR6HLGvw9we5K/A362obGqXtdbVZKk3nQJ/g/2XYQkaXS6jOpZPopCJEmj0WU+/iOS3JTkR0keTfJ4knVdXjzJHkkuT/LdJKuTvDTJXkmuTXJn+7jn1v8zJElddfly9xPAW4E7gZ2BdwCf7Pj6FwBfq6rnAy8EVgNnAddV1SHAde26JGlEOt16saruAnaoqser6mLgmC3tk2R34NeBi9rXeLSd6O0EYFm72TLg9XMpXJI0N12+3P1xkh2BW5J8GFhDt18YBwFrgYuTvJBmvp/TgX2rak27zQPAvpvbOclSYCnAokWLOrydJKmLLgF+YrvdO4FHgAOBf9NhvwXA4cCnqupF7b6bdOu0F4Vt9sKwqrqwqpZU1ZKFCxd2eDtJUhddRvXc257xLwauAO6oqkc7vPZ9wH1VdWO7fjlN8D+YZL+qWpNkP+ChuZUuSZqLLqN6jgf+AfgYzRe9dyU5dkv7VdUDwD8leV7bdDRwO3AVcFLbdhJw5RzqliTNUZc+/vOAo9oveElyMHA18NUO+74LuKT9i+Fumls2Pg24LMkpwL3Am+dSuCRpbroE//oNod+6G1jf5cWr6haaSd6mO7rL/pKkba9L8K9Icg1wGc0XsW8CbkryBoCquqLH+iRJ21iX4N8JeBB4Rbu+luZCrt+k+UVg8EvSBOkyqufkURQiSRqNGYM/yR9W1YeTfJzNjLWvqt/rtTJJUi9mO+Nf3T6uGEUhkqTRmDH4q+qv28dlM20jSZo8XS7gujbJHlPW90zy9X7LkiT1pctcPQvbWTUBqKqHgV/sryRJUp+6BP/jSX4+PWaSZzPDxGqSpO1fl3H8/wG4IclyIMCv0U6XLEmaPF3G8X8tyeHAEW3TGVX1/X7LkiT1ZcauniTPbx8PBxYB32t/FrVtkqQJNNsZ/3tounTO28xzBbyyl4okSb2abRz/hn78Y6vqp1OfS7JTr1VJknrTZVTP/+jYJkmaALPN1fNMYH9g5yQvohnRA7AbsMsIapMk9WC2Pv7XAG8HDqDp598Q/OuA9/VbliSpL7P18S9L8nngrVV1yQhrkiT1aNY+/qp6Anj3iGqRJI1Aly93/ybJ7yc5MMleG356r0yS1IsuUzb8Vvt42pS2Ap6z7cuRJPWty5QNB42iEEnSaHQ54yfJC4BDaW68DkBVfa6voiRJ/dli8Cc5GziSJvivAY4FbgAMfkmaQF2+3H0jcDTwQFWdDLwQ2L3XqiRJvekS/D9ph3U+lmQ34CHgwH7LkiT1pUsf/4r2nrt/AawEfgT8z16rkiT1psuonn/XLn46ydeA3arq7/stS5LUl66jet4AvJxm/P4NgME/YRafdfW4S5jVPeccP+4SpMHYYh9/kv8C/C5wK3AbcGqST/ZdmCSpH13O+F8J/MuqKoAky4BVvVYlSepNl1E9d9Hcc3eDA9s2SdIE6nLGvyuwOsnf0fTxv4RmpM9VAFX1uh7rkyRtY12C/wO9VyFJGpkuwzmXj6IQSdJodOnjlyTNIwa/JA3MjMGf5Lr28dzRlSNJ6ttsffz7JflV4HVJLgUy9cmqurnXyiRJvZgt+D8A/BFwAHD+tOeK5sKuLUqyA7ACuL+qXpvkIOBSYG+aSd9OrKpHn2rhkqS5mbGrp6our6pjgQ9X1VHTfjqFfut0YPWU9XOBj1TVc4GHgVPmVLkkaU62+OVuVf1xktcl+bP257VdXzzJAcDxwH9t10Pzl8Ll7SbLgNc/9bIlSXPVZZK2P6U5a7+9/Tk9yZ90fP2PAn8IPNGu7w38sKoea9fvA/af4X2XJlmRZMXatWs7vp0kaUu6DOc8HnhVVX2mqj4DHANs8ay//cvgoapaOZfCqurCqlpSVUsWLlw4l5eQJG1Gp/n4gT2AH7TLXe+3+zKaEUHHATsBuwEXAHskWdCe9R8A3P8U6pUkbaUuZ/x/CnwnyWfbKZlXAh/a0k5V9e+r6oCqWgy8Bfjbqnob8E2aG7gDnARcOafKJUlz0mWuni8kuR74V23TmVX1wFa855nApUn+E/Ad4KKteC1J0lPUqaunqtYAV831TarqeuD6dvlumqmdJUlj4Fw9kjQwBr8kDcyswZ9khyTfHVUxkqT+zRr8VfU4cEeSRbNtJ0maHF2+3N0TWNXec/eRDY3ea1eSJlOX4P+j3quQJI1Mp3vuJnk2cEhV/U2SXYAd+i9NktSHLpO0/Q7NbJp/3jbtD3ylz6IkSf3pMpzzNJp5d9YBVNWdwC/2WZQkqT9dgv9nU++QlWQBzR24JEkTqEvwL0/yPmDnJK8CvgT8db9lSZL60iX4zwLWArcCpwLXAO/vsyhJUn+6jOp5op2O+UaaLp47qsquHkmaUFsM/iTHA58G/gEIcFCSU6vqq30XJ0na9rpcwHUecFRV3QWQ5GDgasDgl6QJ1KWPf/2G0G/dDazvqR5JUs9mPONP8oZ2cUWSa4DLaPr43wTcNILaJEk9mK2r5zenLD8IvKJdXgvs3FtFkqRezRj8VXXyKAuRJI1Gl1E9BwHvAhZP3d5pmSVpMnUZ1fMV4CKaq3Wf6LccSVLfugT/T6vqY71XIkkaiS7Bf0GSs4FvAD/b0FhVN/dWlSSpN12C/5eBE4FXsrGrp9p1aaIsPuvqcZcwq3vOOX7cJWgAugT/m4DnTJ2aWZI0ubpcuXsbsEffhUiSRqPLGf8ewHeT3MSmffwO55SkCdQl+M/uvQpJ0sh0mY9/+SgKkSSNRpcrd9ez8R67OwJPBx6pqt36LEyS1I8uZ/y7blhOEuAE4Ig+i5Ik9afLqJ6fq8ZXgNf0VI8kqWddunreMGX1acAS4Ke9VSRJ6lWXUT1T5+V/DLiHprtHkjSBuvTxOy+/JM0js9168QOz7FdV9cc91CNJ6tlsZ/yPbKbtGcApwN6AwS9JE2i2Wy+et2E5ya7A6cDJwKXAeTPtJ0navs3ax59kL+A9wNuAZcDhVfXwKAqTJPVjxnH8Sf4zcBOwHvjlqvrgUwn9JAcm+WaS25OsSnJ6275XkmuT3Nk+7rnV/wpJUmezXcD1XuBZwPuB7yVZ1/6sT7Kuw2s/Bry3qg6ludL3tCSHAmcB11XVIcB17bokaURm6+N/Slf1bmb/NcCadnl9ktXA/jTXABzZbrYMuB44c2veS5LU3VaFe1dJFgMvAm4E9m1/KQA8AOw7ihokSY0uV+5ulSS/AHwZOKOq1jXzvDWqqpLUDPstBZYCLFq0qO8ypcHZnu8/7L2H+9XrGX+Sp9OE/iVVdUXb/GCS/drn9wMe2ty+VXVhVS2pqiULFy7ss0xJGpTegr+dwvkiYHVVnT/lqauAk9rlk4Ar+6pBkvRkfXb1vAw4Ebg1yS1t2/uAc4DLkpwC3Au8uccaJEnT9Bb8VXUDkBmePrqv95UkzW4ko3okSdsPg1+SBsbgl6SBMfglaWAMfkkaGINfkgbG4JekgTH4JWlgDH5JGhiDX5IGxuCXpIEx+CVpYAx+SRoYg1+SBsbgl6SBMfglaWAMfkkaGINfkgbG4JekgTH4JWlgDH5JGpgF4y5AksZp8VlXj7uEGd1zzvG9vK5n/JI0MAa/JA2MwS9JA2PwS9LAGPySNDAGvyQNjMEvSQNj8EvSwBj8kjQwBr8kDYzBL0kDY/BL0sAY/JI0MAa/JA2MwS9JA2PwS9LAGPySNDAGvyQNzFiCP8kxSe5IcleSs8ZRgyQN1ciDP8kOwCeBY4FDgbcmOXTUdUjSUI3jjP8lwF1VdXdVPQpcCpwwhjokaZBSVaN9w+SNwDFV9Y52/UTgV6rqndO2WwosbVefB9wB7AN8f4TlThqPz+w8PrPz+MxsUo/Ns6tq4fTGBeOopIuquhC4cGpbkhVVtWRMJW33PD6z8/jMzuMzs/l2bMbR1XM/cOCU9QPaNknSCIwj+G8CDklyUJIdgbcAV42hDkkapJF39VTVY0neCXwd2AH4TFWt6rj7hVveZNA8PrPz+MzO4zOzeXVsRv7lriRpvLxyV5IGxuCXpIGZiOB3iofZJbknya1JbkmyYtz1jFuSzyR5KMltU9r2SnJtkjvbxz3HWeM4zXB8Ppjk/vYzdEuS48ZZ4zglOTDJN5PcnmRVktPb9nnzGdrug98pHjo7qqoOm09jjbfCZ4FjprWdBVxXVYcA17XrQ/VZnnx8AD7SfoYOq6prRlzT9uQx4L1VdShwBHBamznz5jO03Qc/TvGgp6iq/jvwg2nNJwDL2uVlwOtHWtR2ZIbjo1ZVramqm9vl9cBqYH/m0WdoEoJ/f+Cfpqzf17ZpowK+kWRlO9WFnmzfqlrTLj8A7DvOYrZT70zy921X0MR2Y2xLSRYDLwJuZB59hiYh+LVlL6+qw2m6w05L8uvjLmh7Vs0YZscxb+pTwMHAYcAa4LzxljN+SX4B+DJwRlWtm/rcpH+GJiH4neJhC6rq/vbxIeCvaLrHtKkHk+wH0D4+NOZ6titV9WBVPV5VTwB/wcA/Q0meThP6l1TVFW3zvPkMTULwO8XDLJI8I8muG5aBVwO3zb7XIF0FnNQunwRcOcZatjsbAq31rxnwZyhJgIuA1VV1/pSn5s1naCKu3G2Hln2UjVM8fGjMJW03kjyH5iwfmik4/nLoxyfJF4AjaabSfRA4G/gKcBmwCLgXeHNVDfILzhmOz5E03TwF3AOcOqU/e1CSvBz4FnAr8ETb/D6afv558RmaiOCXJG07k9DVI0nahgx+SRoYg1+SBsbgl6SBMfglaWAMfs07SR5vZ5i8LcmXkuzStv9o2nZvT/KJKetLk3y3/VmR5Mgpzx2d5Ob2dW9I8tyR/YOkbczg13z0k3aGyRcAjwK/u6UdkrwWOJVm+ovnA0uB/5Zkw7xQnwLeVlWHAX8JvL+f0jepaeS3RtUwGPya774FdDk7PxP4g6r6PkA7O+PFwGnt8wXs1i7vDnxv+gskecWU+ey/M+WK6jPb+yX87yTntG2HJflf7aRof7VhUrQk1yf5aHtfhdOTvDjJ8nYCvq9Pu8JWmhPPKDRvtWfMxwJfa5t2TnLLlE32YuP0H78ErJz2EiuAk9vldwDXJPkJsI5mnvbpfh84raq+3U7w9dMkx9JM5/srVfXjJHu1234OeFdVLU/yH2munj2jfW7HqlrSzhezHDihqtYm+S3gQ8BvP8VDIW3C4Nd8NDXgv0Uz7wq0XUAbNkrydqDrjWveDRxXVTcm+QPgfJpfBlN9Gzg/ySXAFVV1X5LfAC6uqh8DVNUPkuwO7FFVy9v9lgFfmvI6X2wfnwe8ALi2mT6GHWhmzpS2isGv+WiTgO/oduDFwN9OaXsxsCLJQuCFVXVj2/5FNv4V8XNVdU6Sq4HjgG8nec1TLx2AR9rHAKuq6qVzfB1ps+zjlxofBs5Nsjc0ffA0s1T+OfAwsHuSf9Fu+yqauzJtIsnBVXVrVZ1LM6vs84FrgZOnjCzaq6r+GXg4ya+1u55I06Uz3R3AwiQvbfd9epJf2jb/XA2ZZ/wSUFVXJXkWzZn6AuCZNGf5awGS/A7w5SRP0Pwi2Fw/+xlJjqKZ0XEV8NWq+ln7S2RFkkeBa2hmejwJ+HT7C+FuNn6XMLWmR5O8EfhY2z20gGaW2lXb9B+vwXF2TmmaNvgvpvmL+N+W/0k0zxj8kjQw9vFL0sAY/JI0MAa/JA2MwS9JA2PwS9LAGPySNDD/HwgaenMQg3WxAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qXjdhA9VkbqV",
        "colab_type": "text"
      },
      "source": [
        "### Splitting the dataset and grouping them based on the 5 different level"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZm8A3XG-Uu-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def split_by_phq_level(ds):\n",
        "    none_ds = ds[ds['level']==0]\n",
        "    mild_ds = ds[ds['level']==1]\n",
        "    moderate_ds = ds[ds['level']==2]\n",
        "    moderate_severe_ds = ds[ds['level']==3]\n",
        "    severe_ds = ds[ds['level']==4]\n",
        "    return (none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tzuAE0BO-mcy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e8ea631f-df39-4468-eef2-754de86e3848"
      },
      "source": [
        "none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds = split_by_phq_level(ds_total)\n",
        "print(\"Quantity per none_ds: {}, mild_ds: {}, moderate_ds {}, moderate_severe_ds: {}, severe_ds {}\".format(len(none_ds), len(mild_ds), len(moderate_ds), len(moderate_severe_ds), len(severe_ds)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Quantity per none_ds: 26, mild_ds: 70, moderate_ds 47, moderate_severe_ds: 24, severe_ds 22\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vFUUHymf1ril",
        "colab_type": "text"
      },
      "source": [
        "## Random Sampling to overcome Imbalance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q-Oe8kjh-q1o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "b_none_ds = ds_total[ds_total['level']==0]\n",
        "b_mild_ds = ds_total[ds_total['level']==1].sample(26)\n",
        "b_moderate_ds = ds_total[ds_total['level']==2].sample(26)\n",
        "b_moderate_severe_ds = ds_total[ds_total['level']==3]\n",
        "b_severe_ds = ds_total[ds_total['level']==4]\n",
        "\n",
        "ds_total_b = pd.concat([b_none_ds, b_mild_ds, b_moderate_ds, b_moderate_severe_ds, b_severe_ds])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zHZooSJTm1hh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ds_= ds_total_b.to_csv('ds_total_b.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tBg3OyJKk3CG",
        "colab_type": "text"
      },
      "source": [
        "### Merging the phrases_lp (consisting of the transcripts) and ds_total consisting of PHQ Score"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ccRbZsAG-29q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ds_lp = pd.merge(ds_total, phrases_lp,left_on='Participant_ID', right_on='personId')\n",
        "ds_lp.drop(ds_lp[ds_lp[\"t_answer\"].map(len) < 10].index, inplace = True)\n",
        "ds_lp_b = pd.merge(ds_total_b, phrases_lp,left_on='Participant_ID', right_on='personId')\n",
        "ds_lp_b.drop(ds_lp_b[ds_lp_b[\"t_answer\"].map(len) < 10].index, inplace = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gDCCeFTjQ8aK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#ds_lp[\"t_answer\"][142810]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZ2jOzPX5VAB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "052cd3e3-c968-4c2e-965b-1ae0d526ee98"
      },
      "source": [
        "(len(ds_lp))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "141137"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OuJ-MFwOkoyL",
        "colab_type": "text"
      },
      "source": [
        "### Splitting ds_total into train, dev and test in the ratio of 70:14:16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJUdsiWIreIW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def distribute_instances(ds, split_in = [70,14,16]):\n",
        "    ds_shuffled = ds.sample(frac=1)\n",
        "    none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds = split_by_phq_level(ds_shuffled)\n",
        "    eq_ds = dict()\n",
        "    prev_none = prev_mild = prev_moderate = prev_moderate_severe = prev_severe = 0\n",
        "    split = split_in\n",
        "    for p in split:\n",
        "        last_none = min(len(none_ds), prev_none + round(len(none_ds) * p/100))\n",
        "        last_mild = min(len(mild_ds), prev_mild + round(len(mild_ds) * p/100))\n",
        "        last_moderate = min(len(moderate_ds), prev_moderate + round(len(moderate_ds) * p/100))\n",
        "        last_moderate_severe = min(len(moderate_severe_ds), prev_moderate_severe + round(len(moderate_severe_ds) * p/100))\n",
        "        last_severe = min(len(severe_ds), prev_severe + round(len(severe_ds) * p/100))  \n",
        "        eq_ds['d'+str(p)] = pd.concat([none_ds[prev_none: last_none], mild_ds[prev_mild: last_mild], moderate_ds[prev_moderate: last_moderate], moderate_severe_ds[prev_moderate_severe: last_moderate_severe], severe_ds[prev_severe: last_severe]])\n",
        "        prev_none = last_none\n",
        "        prev_mild = last_mild\n",
        "        prev_moderate = last_moderate\n",
        "        prev_moderate_severe = last_moderate_severe\n",
        "        prev_severe = last_severe  \n",
        "    return (eq_ds['d70'], eq_ds['d14'], eq_ds['d16'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "79bjNsvCrj2z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "train_lp, dev_lp, test_lp = distribute_instances(ds_lp)\n",
        "train_lp_b, dev_lp_b, test_lp_b = distribute_instances(ds_lp_b)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C-7zqE5olMwD",
        "colab_type": "text"
      },
      "source": [
        "### Creating a confusion matrix which consists of the TPs, TNs, FPs and FNs of the predicted labels for the transcripts"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bp-JesyOrt1h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def confusion_matrix(model, x, y):\n",
        "    prediction = model.predict(x, batch_size=None, verbose=0, steps=None)\n",
        "    labels=['none','mild','moderate','moderately severe', 'severe']\n",
        "\n",
        "    max_prediction = np.argmax(prediction, axis=1)\n",
        "    \n",
        "    max_actual = np.argmax(y, axis=1)\n",
        "\n",
        "    y_pred = pd.Categorical.from_codes(max_prediction, labels)\n",
        "    y_actu = pd.Categorical.from_codes(max_actual, labels)\n",
        "\n",
        "    return pd.crosstab(y_actu, y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vfjl_n2flc2i",
        "colab_type": "text"
      },
      "source": [
        "### Loading the Glove Embeddings into a file and putting each vector into an np array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6yojHumwrwqj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embeddings_index = dict()\n",
        "f = open('/content/drive/My Drive/glove.6B.100d.txt', encoding=\"utf8\")\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "f.close()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iYBxoJu7lxRe",
        "colab_type": "text"
      },
      "source": [
        "### Creating a embedding matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8a-Q-8eVr01V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def fill_embedding_matrix(tokenizer):\n",
        "    vocab_size = len(tokenizer.word_index) # tokenizer.word_index is the list that consist of all the unique words\n",
        "    embedding_matrix = np.zeros((vocab_size+1, 100)) # creating an embedding matrix\n",
        "    for word, i in tokenizer.word_index.items():\n",
        "        embedding_vector = embeddings_index.get(word)\n",
        "        if embedding_vector is not None:        \n",
        "            embedding_matrix[i] = embedding_vector\n",
        "    return embedding_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tph-lY3TmT0T",
        "colab_type": "text"
      },
      "source": [
        "### calling the embedding matrix function for creating vectors for each unique word"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NmnVpIfBNxFt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c918e65a-713c-499e-c140-0b727679e66d"
      },
      "source": [
        "embedding_matrix_lp = fill_embedding_matrix(tokenizer)\n",
        "embedding_matrix_lp.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(7450, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e33COWoK-3s0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "4abb8aee-7eec-482f-df51-48e25663d223"
      },
      "source": [
        "train_lp['t_answer']\n",
        "train_lp.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Participant_ID</th>\n",
              "      <th>level</th>\n",
              "      <th>cat_level</th>\n",
              "      <th>PHQ8_Score</th>\n",
              "      <th>Gender</th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>personId</th>\n",
              "      <th>answer</th>\n",
              "      <th>t_answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>95368</th>\n",
              "      <td>464.0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>122008</td>\n",
              "      <td>464</td>\n",
              "      <td>['that', 'pretty', 'much', 'temper', 'try', 'm...</td>\n",
              "      <td>[20, 23, 32, 218, 62, 45, 329, 4, 5, 740]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50399</th>\n",
              "      <td>363.0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>41094</td>\n",
              "      <td>363</td>\n",
              "      <td>['am', 'child', 'like', 'around', 'good', 'lat...</td>\n",
              "      <td>[6, 112, 5, 91, 16, 453, 468, 887, 26, 593]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>115332</th>\n",
              "      <td>361.0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>39697</td>\n",
              "      <td>361</td>\n",
              "      <td>['social', 'person', 'um', 'nothing', 'much', ...</td>\n",
              "      <td>[429, 71, 1, 174, 32, 35, 2, 51, 2, 45]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>127008</th>\n",
              "      <td>411.0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>83734</td>\n",
              "      <td>411</td>\n",
              "      <td>['strong', 'decision', 'usually', 'decision', ...</td>\n",
              "      <td>[445, 105, 126, 105, 100, 5622, 1223, 3852, 12...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25358</th>\n",
              "      <td>492.0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>142367</td>\n",
              "      <td>492</td>\n",
              "      <td>['iphone', 'yeah', 'depends', 'situation', 'i'...</td>\n",
              "      <td>[5950, 11, 233, 89, 3, 6, 155, 322, 714, 30]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        Participant_ID  ...                                           t_answer\n",
              "95368            464.0  ...          [20, 23, 32, 218, 62, 45, 329, 4, 5, 740]\n",
              "50399            363.0  ...        [6, 112, 5, 91, 16, 453, 468, 887, 26, 593]\n",
              "115332           361.0  ...            [429, 71, 1, 174, 32, 35, 2, 51, 2, 45]\n",
              "127008           411.0  ...  [445, 105, 126, 105, 100, 5622, 1223, 3852, 12...\n",
              "25358            492.0  ...       [5950, 11, 233, 89, 3, 6, 155, 322, 714, 30]\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0JbU07MoOOEC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_a =np.stack(train_lp['t_answer'])\n",
        "dev_a = np.stack(dev_lp['t_answer'])\n",
        "train_y = np.stack(train_lp['cat_level'], axis=0) \n",
        "dev_y = np.stack(dev_lp['cat_level'], axis=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-xy1isHjlaxU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a2aeae2f-ec2c-4502-f13e-4fcc421ac0e2"
      },
      "source": [
        "dev_a.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(19760, 10)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gHcPxsdQmiDd",
        "colab_type": "text"
      },
      "source": [
        "### stacking all the arrays together to form a single array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M49JkNY4Ruwu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_a_b = np.stack(train_lp_b['t_answer'], axis=0)\n",
        "dev_a_b = np.stack(dev_lp_b['t_answer'], axis=0)\n",
        "train_y_b = np.stack(train_lp_b['cat_level'], axis=0)\n",
        "dev_y_b = np.stack(dev_lp_b['cat_level'], axis=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e2njzWc8d7Wb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "early_stopping = EarlyStopping(monitor='val_loss', patience=3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wN-ygkjymr-G",
        "colab_type": "text"
      },
      "source": [
        "### creating function for plotting the loss and accuracies of the models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JGaymo_kd9oY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_acc(history, title=\"Model Accuracy\"):\n",
        "    plt.plot(history.history['accuracy'])\n",
        "    plt.plot(history.history['val_accuracy'])\n",
        "    plt.title(title)\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(['Train', 'Val'], loc='upper left')\n",
        "    plt.show()\n",
        "    \n",
        "def plot_loss(history, title=\"Model Loss\"):\n",
        "    plt.plot(history.history['loss'])\n",
        "    plt.plot(history.history['val_loss'])\n",
        "    plt.title(title)\n",
        "    plt.ylabel('Loss')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(['Train', 'Val'], loc='upper right')\n",
        "    plt.show()\n",
        "    \n",
        "def plot_compare_losses(history1, history2, name1=\"Red 1\", name2=\"Red 2\", title=\"Graph title\"):\n",
        "    plt.plot(history1.history['loss'], color=\"green\")\n",
        "    plt.plot(history1.history['val_loss'], 'r--', color=\"green\")\n",
        "    plt.plot(history2.history['loss'], color=\"blue\")\n",
        "    plt.plot(history2.history['val_loss'], 'r--', color=\"blue\")\n",
        "    plt.title(title)\n",
        "    plt.ylabel('Loss')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(['Train ' + name1, 'Val ' + name1, \n",
        "                'Train ' + name2, 'Val ' + name2],\n",
        "               loc='upper right')\n",
        "    plt.show()\n",
        "    \n",
        "def plot_compare_accs(history1, history2, name1=\"Red 1\",\n",
        "                      name2=\"Red 2\", title=\"Graph title\"):\n",
        "    \"\"\"Compara accuracies de dos entrenamientos con nombres name1 y name2\"\"\"\n",
        "    plt.plot(history1.history['acc'], color=\"green\")\n",
        "    plt.plot(history1.history['val_acc'], 'r--', color=\"green\")\n",
        "    plt.plot(history2.history['acc'], color=\"blue\")\n",
        "    plt.plot(history2.history['val_acc'], 'r--', color=\"blue\")\n",
        "    plt.title(title)\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(['Train ' + name1, 'Val ' + name1, \n",
        "                'Train ' + name2, 'Val ' + name2], \n",
        "               loc='lower right')\n",
        "    plt.show()\n",
        "\n",
        "def plot_compare_multiple_metrics(history_array, names, colors, title=\"Graph title\", metric='acc'):  \n",
        "    legend = []\n",
        "    for i in range(0, len(history_array)):\n",
        "        plt.plot(history_array[i].history[metric], color=colors[i])\n",
        "        plt.plot(history_array[i].history['val_' + metric], 'r--', color=colors[i])\n",
        "        legend.append('Train ' + names[i])\n",
        "        legend.append('Val ' + names[i])\n",
        "    \n",
        "    plt.title(title)\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.xlabel('Epoch')   \n",
        "    plt.axis\n",
        "    plt.legend(legend, \n",
        "               loc='lower right')\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MRJf5uFJFMkv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "early_stopping = EarlyStopping(monitor='val_loss', patience=3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xp-WC8utm8YO",
        "colab_type": "text"
      },
      "source": [
        "### answer_emb1 is the input consisting of the vectors having a window size of 10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "trHt670PeBZ0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "answer_inp = Input(shape=(windows_size, ))\n",
        "embedding_size_glove = 100\n",
        "answer_emb1 = Embedding(vocab_size_stop+1, embedding_size_glove, weights=[embedding_matrix_lp], input_length=windows_size, trainable=False)(answer_inp)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KVM0aldzOP3u",
        "colab_type": "text"
      },
      "source": [
        "### GLoVE + LSTM (imbalanced) MODEL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7UszmHs-FmDG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        },
        "outputId": "025ff1b5-4542-469b-a8e1-e7c531bf1639"
      },
      "source": [
        "bt = BatchNormalization()(answer_emb1)\n",
        "lstm = LSTM(embedding_size_glove, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(bt)\n",
        "\n",
        "dense1 = Dense(units=256, activation=\"relu\")(lstm)\n",
        "dense2 = Dense(units=256, activation=\"relu\")(dense1)\n",
        "\n",
        "flatten = Flatten()(dense2)\n",
        "\n",
        "out = Dense(5,  activation='softmax')(flatten)\n",
        "\n",
        "model = Model(inputs=[answer_inp], outputs=[out])\n",
        "model.compile(optimizer=\"adam\", loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         (None, 10)                0         \n",
            "_________________________________________________________________\n",
            "embedding_1 (Embedding)      (None, 10, 100)           745000    \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 10, 100)           400       \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 10, 100)           80400     \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 10, 256)           25856     \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 10, 256)           65792     \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 2560)              0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 5)                 12805     \n",
            "=================================================================\n",
            "Total params: 930,253\n",
            "Trainable params: 185,053\n",
            "Non-trainable params: 745,200\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z7PHs9WrVYmW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "61dea500-a4a5-431e-8edd-a0321312fa52"
      },
      "source": [
        "model_glove_lstm_hist = model.fit(train_a, train_y, validation_data=(dev_a, dev_y), epochs=30, batch_size=64, shuffle=True, callbacks=[early_stopping])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 98795 samples, validate on 19760 samples\n",
            "Epoch 1/30\n",
            "98795/98795 [==============================] - 50s 503us/step - loss: 1.4525 - accuracy: 0.3747 - val_loss: 1.3414 - val_accuracy: 0.4367\n",
            "Epoch 2/30\n",
            "98795/98795 [==============================] - 49s 493us/step - loss: 1.2671 - accuracy: 0.4734 - val_loss: 1.1069 - val_accuracy: 0.5646\n",
            "Epoch 3/30\n",
            "98795/98795 [==============================] - 49s 498us/step - loss: 1.1056 - accuracy: 0.5524 - val_loss: 0.9235 - val_accuracy: 0.6487\n",
            "Epoch 4/30\n",
            "98795/98795 [==============================] - 54s 547us/step - loss: 0.9935 - accuracy: 0.6020 - val_loss: 0.8078 - val_accuracy: 0.6917\n",
            "Epoch 5/30\n",
            "98795/98795 [==============================] - 49s 498us/step - loss: 0.9198 - accuracy: 0.6371 - val_loss: 0.6993 - val_accuracy: 0.7409\n",
            "Epoch 6/30\n",
            "98795/98795 [==============================] - 49s 501us/step - loss: 0.8533 - accuracy: 0.6666 - val_loss: 0.6363 - val_accuracy: 0.7638\n",
            "Epoch 7/30\n",
            "98795/98795 [==============================] - 49s 501us/step - loss: 0.8041 - accuracy: 0.6882 - val_loss: 0.5682 - val_accuracy: 0.7966\n",
            "Epoch 8/30\n",
            "98795/98795 [==============================] - 49s 499us/step - loss: 0.7605 - accuracy: 0.7064 - val_loss: 0.5284 - val_accuracy: 0.8126\n",
            "Epoch 9/30\n",
            "98795/98795 [==============================] - 49s 497us/step - loss: 0.7225 - accuracy: 0.7205 - val_loss: 0.4856 - val_accuracy: 0.8324\n",
            "Epoch 10/30\n",
            "98795/98795 [==============================] - 49s 495us/step - loss: 0.6886 - accuracy: 0.7360 - val_loss: 0.4503 - val_accuracy: 0.8367\n",
            "Epoch 11/30\n",
            "98795/98795 [==============================] - 49s 496us/step - loss: 0.6604 - accuracy: 0.7487 - val_loss: 0.4216 - val_accuracy: 0.8526\n",
            "Epoch 12/30\n",
            "98795/98795 [==============================] - 49s 494us/step - loss: 0.6336 - accuracy: 0.7606 - val_loss: 0.3991 - val_accuracy: 0.8576\n",
            "Epoch 13/30\n",
            "98795/98795 [==============================] - 48s 490us/step - loss: 0.6125 - accuracy: 0.7678 - val_loss: 0.3605 - val_accuracy: 0.8772\n",
            "Epoch 14/30\n",
            "98795/98795 [==============================] - 49s 493us/step - loss: 0.5854 - accuracy: 0.7789 - val_loss: 0.3448 - val_accuracy: 0.8830\n",
            "Epoch 15/30\n",
            "98795/98795 [==============================] - 49s 493us/step - loss: 0.5669 - accuracy: 0.7860 - val_loss: 0.3185 - val_accuracy: 0.8908\n",
            "Epoch 16/30\n",
            "98795/98795 [==============================] - 53s 538us/step - loss: 0.5485 - accuracy: 0.7954 - val_loss: 0.3065 - val_accuracy: 0.8962\n",
            "Epoch 17/30\n",
            "98795/98795 [==============================] - 49s 497us/step - loss: 0.5297 - accuracy: 0.8016 - val_loss: 0.2931 - val_accuracy: 0.9029\n",
            "Epoch 18/30\n",
            "98795/98795 [==============================] - 49s 496us/step - loss: 0.5123 - accuracy: 0.8079 - val_loss: 0.2737 - val_accuracy: 0.9089\n",
            "Epoch 19/30\n",
            "98795/98795 [==============================] - 49s 496us/step - loss: 0.4972 - accuracy: 0.8149 - val_loss: 0.2570 - val_accuracy: 0.9169\n",
            "Epoch 20/30\n",
            "98795/98795 [==============================] - 49s 497us/step - loss: 0.4809 - accuracy: 0.8214 - val_loss: 0.2556 - val_accuracy: 0.9154\n",
            "Epoch 21/30\n",
            "98795/98795 [==============================] - 49s 499us/step - loss: 0.4670 - accuracy: 0.8275 - val_loss: 0.2422 - val_accuracy: 0.9218\n",
            "Epoch 22/30\n",
            "98795/98795 [==============================] - 50s 503us/step - loss: 0.4583 - accuracy: 0.8302 - val_loss: 0.2347 - val_accuracy: 0.9234\n",
            "Epoch 23/30\n",
            "98795/98795 [==============================] - 50s 505us/step - loss: 0.4464 - accuracy: 0.8353 - val_loss: 0.2073 - val_accuracy: 0.9323\n",
            "Epoch 24/30\n",
            "98795/98795 [==============================] - 49s 497us/step - loss: 0.4357 - accuracy: 0.8395 - val_loss: 0.2063 - val_accuracy: 0.9330\n",
            "Epoch 25/30\n",
            "98795/98795 [==============================] - 49s 498us/step - loss: 0.4240 - accuracy: 0.8419 - val_loss: 0.1934 - val_accuracy: 0.9352\n",
            "Epoch 26/30\n",
            "98795/98795 [==============================] - 49s 492us/step - loss: 0.4151 - accuracy: 0.8465 - val_loss: 0.1839 - val_accuracy: 0.9397\n",
            "Epoch 27/30\n",
            "98795/98795 [==============================] - 48s 489us/step - loss: 0.4045 - accuracy: 0.8507 - val_loss: 0.1966 - val_accuracy: 0.9337\n",
            "Epoch 28/30\n",
            "98795/98795 [==============================] - 49s 498us/step - loss: 0.3960 - accuracy: 0.8550 - val_loss: 0.1794 - val_accuracy: 0.9390\n",
            "Epoch 29/30\n",
            "98795/98795 [==============================] - 52s 527us/step - loss: 0.3899 - accuracy: 0.8562 - val_loss: 0.1664 - val_accuracy: 0.9458\n",
            "Epoch 30/30\n",
            "98795/98795 [==============================] - 48s 490us/step - loss: 0.3785 - accuracy: 0.8614 - val_loss: 0.1586 - val_accuracy: 0.9474\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XdK30MF38EBu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_a = np.stack(test_lp['t_answer'], axis=0)\n",
        "test_y = np.stack(test_lp['cat_level'], axis=0)\n",
        "test_a_b = np.stack(test_lp_b['t_answer'], axis=0)\n",
        "test_y_b = np.stack(test_lp_b['cat_level'], axis=0)\n",
        "#df_confusion1 = confusion_matrix(model, test_a, test_y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UAxLDsIZ8OAG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "f7508adb-44d7-4e34-f358-0371c7494e24"
      },
      "source": [
        "df_confusion1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>col_0</th>\n",
              "      <th>none</th>\n",
              "      <th>mild</th>\n",
              "      <th>moderate</th>\n",
              "      <th>moderately severe</th>\n",
              "      <th>severe</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>row_0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>none</th>\n",
              "      <td>3359</td>\n",
              "      <td>106</td>\n",
              "      <td>47</td>\n",
              "      <td>18</td>\n",
              "      <td>26</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mild</th>\n",
              "      <td>62</td>\n",
              "      <td>7437</td>\n",
              "      <td>143</td>\n",
              "      <td>48</td>\n",
              "      <td>46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>moderate</th>\n",
              "      <td>64</td>\n",
              "      <td>199</td>\n",
              "      <td>5731</td>\n",
              "      <td>43</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>moderately severe</th>\n",
              "      <td>28</td>\n",
              "      <td>83</td>\n",
              "      <td>66</td>\n",
              "      <td>2608</td>\n",
              "      <td>24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>severe</th>\n",
              "      <td>25</td>\n",
              "      <td>37</td>\n",
              "      <td>37</td>\n",
              "      <td>24</td>\n",
              "      <td>2266</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "col_0              none  mild  moderate  moderately severe  severe\n",
              "row_0                                                             \n",
              "none               3359   106        47                 18      26\n",
              "mild                 62  7437       143                 48      46\n",
              "moderate             64   199      5731                 43      55\n",
              "moderately severe    28    83        66               2608      24\n",
              "severe               25    37        37                 24    2266"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3hdoi3h71HV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "db100979-923d-4167-c2eb-d425780b20ac"
      },
      "source": [
        "score = model.evaluate(test_a, test_y, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.16040187424668892\n",
            "Test accuracy: 0.9477016925811768\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JVZMqv26v3dD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0958623d-054c-40c2-9ef2-26d298490596"
      },
      "source": [
        "from sklearn.externals import joblib\n",
        "joblib.dump(model,\"model.pkl\"),"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['model.pkl'],)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WLgl65q0DTLW",
        "colab_type": "text"
      },
      "source": [
        "## **LSTM Model for the balanced dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U3X7w3useV3o",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        },
        "outputId": "dd554fa2-13c0-4660-aa9b-7a6182f2e42e"
      },
      "source": [
        "model_glove_lstm_hist_b = model.fit(train_a_b, train_y_b, validation_data=(dev_a_b, dev_y_b), epochs=30, batch_size=64, shuffle=True, callbacks=[early_stopping])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 64179 samples, validate on 12836 samples\n",
            "Epoch 1/30\n",
            "64179/64179 [==============================] - 31s 484us/step - loss: 0.3943 - accuracy: 0.8577 - val_loss: 0.0728 - val_accuracy: 0.9825\n",
            "Epoch 2/30\n",
            "64179/64179 [==============================] - 31s 486us/step - loss: 0.3426 - accuracy: 0.8763 - val_loss: 0.0662 - val_accuracy: 0.9829\n",
            "Epoch 3/30\n",
            "64179/64179 [==============================] - 31s 489us/step - loss: 0.3222 - accuracy: 0.8842 - val_loss: 0.0651 - val_accuracy: 0.9847\n",
            "Epoch 4/30\n",
            "64179/64179 [==============================] - 32s 501us/step - loss: 0.2973 - accuracy: 0.8929 - val_loss: 0.0545 - val_accuracy: 0.9863\n",
            "Epoch 5/30\n",
            "64179/64179 [==============================] - 32s 493us/step - loss: 0.2929 - accuracy: 0.8952 - val_loss: 0.0523 - val_accuracy: 0.9874\n",
            "Epoch 6/30\n",
            "64179/64179 [==============================] - 32s 494us/step - loss: 0.2762 - accuracy: 0.9002 - val_loss: 0.0541 - val_accuracy: 0.9861\n",
            "Epoch 7/30\n",
            "64179/64179 [==============================] - 32s 493us/step - loss: 0.2625 - accuracy: 0.9060 - val_loss: 0.0544 - val_accuracy: 0.9843\n",
            "Epoch 8/30\n",
            "64179/64179 [==============================] - 32s 493us/step - loss: 0.2572 - accuracy: 0.9094 - val_loss: 0.0454 - val_accuracy: 0.9868\n",
            "Epoch 9/30\n",
            "64179/64179 [==============================] - 32s 491us/step - loss: 0.2471 - accuracy: 0.9123 - val_loss: 0.0481 - val_accuracy: 0.9869\n",
            "Epoch 10/30\n",
            "64179/64179 [==============================] - 31s 489us/step - loss: 0.2399 - accuracy: 0.9145 - val_loss: 0.0463 - val_accuracy: 0.9875\n",
            "Epoch 11/30\n",
            "64179/64179 [==============================] - 31s 488us/step - loss: 0.2358 - accuracy: 0.9166 - val_loss: 0.0441 - val_accuracy: 0.9881\n",
            "Epoch 12/30\n",
            "64179/64179 [==============================] - 31s 488us/step - loss: 0.2295 - accuracy: 0.9188 - val_loss: 0.0449 - val_accuracy: 0.9883\n",
            "Epoch 13/30\n",
            "64179/64179 [==============================] - 31s 490us/step - loss: 0.2234 - accuracy: 0.9193 - val_loss: 0.0427 - val_accuracy: 0.9871\n",
            "Epoch 14/30\n",
            "64179/64179 [==============================] - 31s 491us/step - loss: 0.2162 - accuracy: 0.9244 - val_loss: 0.0384 - val_accuracy: 0.9895\n",
            "Epoch 15/30\n",
            "64179/64179 [==============================] - 32s 504us/step - loss: 0.2097 - accuracy: 0.9252 - val_loss: 0.0423 - val_accuracy: 0.9874\n",
            "Epoch 16/30\n",
            "64179/64179 [==============================] - 35s 545us/step - loss: 0.2156 - accuracy: 0.9241 - val_loss: 0.0381 - val_accuracy: 0.9893\n",
            "Epoch 17/30\n",
            "64179/64179 [==============================] - 32s 496us/step - loss: 0.2046 - accuracy: 0.9277 - val_loss: 0.0365 - val_accuracy: 0.9898\n",
            "Epoch 18/30\n",
            "64179/64179 [==============================] - 32s 495us/step - loss: 0.1999 - accuracy: 0.9292 - val_loss: 0.0400 - val_accuracy: 0.9890\n",
            "Epoch 19/30\n",
            "64179/64179 [==============================] - 32s 499us/step - loss: 0.1967 - accuracy: 0.9305 - val_loss: 0.0373 - val_accuracy: 0.9894\n",
            "Epoch 20/30\n",
            "64179/64179 [==============================] - 32s 496us/step - loss: 0.1958 - accuracy: 0.9301 - val_loss: 0.0374 - val_accuracy: 0.9892\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kxc08Yjxeafy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "079e642e-61de-47b6-ff1a-031199222b14"
      },
      "source": [
        "plot_loss(model_glove_lstm_hist)\n",
        "plot_acc(model_glove_lstm_hist)\n",
        "plot_loss(model_glove_lstm_hist_b)\n",
        "plot_acc(model_glove_lstm_hist_b)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3zU9f3A8dc7l703I5NAkD0DyFDBBYoVd0Wl8tPqD1trl1q1Q61aO6y1tXa46qjCzy2KigsFBIGwJawQAkkIEJKQQQhZn98f328gQibJ5XJ37+fjcY+7+657fz25dz5bjDEopZTybj6uDkAppZTraTJQSimlyUAppZQmA6WUUmgyUEophSYDpZRSaDJQqk0ikioiRkR823HsXBFZ3h1xKdWVNBkojyIiuSJSIyKxJ21fb/+gp7omso4lFaW6myYD5Yl2A7Mb34jIcCDYdeEo1fNpMlCe6GXge03e3wi81PQAEYkQkZdEpEhE9ojIr0TEx97nEJHHROSQiOQAM5s59zkRKRSRAhF5WEQcnQlYRPqKyEIRKRGRbBG5pcm+8SKSKSLlInJARB63tweKyH9FpFhEDovIGhHp1Zk4lPfSZKA80ddAuIgMtn+krwX+e9IxTwIRQBpwDlby+B973y3AJcBoIAO46qRzXwDqgAH2MRcC3+9kzAuAfKCv/Xm/E5Fz7X1/Bf5qjAkH+gOv2dtvtO8hCYgB5gFHOxmH8lKaDJSnaiwdXABsBQoadzRJEPcaYyqMMbnAn4E59iHXAE8YY/KMMSXAo03O7QVcDPzEGHPEGHMQ+It9vdMiIknAZOAXxphqY8wG4FlOlG5qgQEiEmuMqTTGfN1kewwwwBhTb4xZa4wpP904lHfTZKA81cvAdcBcTqoiAmIBP2BPk217gAT7dV8g76R9jVLscwvtqpnDwL+B+E7E2hcoMcZUtBDPzcBAYJtdFXSJvf1lYDGwQET2icgfRcSvE3EoL6bJQHkkY8werIbki4G3Ttp9COuv6pQm25I5UXooxKp6abqvUR5wDIg1xkTaj3BjzNBOhLsPiBaRsObiMcbsNMbMxko4fwDeEJEQY0ytMeZBY8wQYBJW1db3UOo0aDJQnuxm4FxjzJGmG40x9Vj17o+ISJiIpAA/40S7wmvAHSKSKCJRwD1Nzi0EPgb+LCLhIuIjIv1F5JwOxBVgN/4Gikgg1o/+CuBRe9sIO/b/AojIDSISZ4xpAA7b12gQkWkiMtyu9irHSnANHYhDqeM0GSiPZYzZZYzJbGH3j4AjQA6wHHgVeN7e9wxW9ctGYB2nliy+B/gDWUAp8AbQpwOhVWI19DY+zsXqCpuKVUp4G7jfGPOpffwMYIuIVGI1Jl9rjDkK9LY/uxyrXeRLrKojpTpMdHEbpZRSWjJQSimlyUAppZQmA6WUUmgyUEopBbjd7ImxsbEmNTXV1WEopZRbWbt27SFjTFxL+90uGaSmppKZ2VJvQaWUUs0RkT2t7ddqIqWUUpoMlFJKaTJQSimFG7YZKKVUR9XW1pKfn091dbWrQ3G6wMBAEhMT8fPr2AS2mgyUUh4vPz+fsLAwUlNTERFXh+M0xhiKi4vJz8+nX79+HTpXq4mUUh6vurqamJgYj04EACJCTEzMaZWANBkopbyCpyeCRqd7n16TDLIPVvDb97KoqdPp3pVS6mRekwz2llTx/Fe7WbL9oKtDUUp5meLiYkaNGsWoUaPo3bs3CQkJx9/X1NS0em5mZiZ33HGH02P0mgbks9PjiA0N4K11+Uwf2tvV4SilvEhMTAwbNmwA4IEHHiA0NJQ777zz+P66ujp8fZv/Oc7IyCAjI8PpMXpNycDX4cNlo/ry+baDlB5pPRMrpZSzzZ07l3nz5jFhwgTuvvtuVq9ezcSJExk9ejSTJk1i+/btAHzxxRdccsklgJVIbrrpJqZOnUpaWhp/+9vfuiwerykZAFw5NpFnl+9m4cZ93Dgp1dXhKKVc4MH3tpC1r7xLrzmkbzj3f2doh8/Lz89nxYoVOBwOysvLWbZsGb6+vnz66afcd999vPnmm6ecs23bNpYsWUJFRQVnnHEGt912W4fHFDTHq5LB4D7hDOkTzpvr8jUZKKVc7uqrr8bhcABQVlbGjTfeyM6dOxERamtrmz1n5syZBAQEEBAQQHx8PAcOHCAxMbHTsTgtGYjI88AlwEFjzLBWjhsHrMRa5PsNZ8XT6MqxiTz0fhY7D1SQ3ivM2R+nlOphTucveGcJCQk5/vrXv/4106ZN4+233yY3N5epU6c2e05AQMDx1w6Hg7q6ui6JxZltBi8AM1o7QEQcwB+Aj50Yx7fMGtUXh4/wxrr87vpIpZRqU1lZGQkJCQC88MIL3f75TksGxpilQEkbh/0IeBPotv6esaEBTB0YxzvrC6hvMN31sUop1aq7776be++9l9GjR3fZX/sdIcY47wdRRFKB95urJhKRBOBVYBrwvH1cs9VEInIrcCtAcnLy2D17Wl2joU0fbC7kB6+s48WbxnPOwBYX/lFKeYitW7cyePBgV4fRbZq7XxFZa4xpsY+qK7uWPgH8whjT5pBgY8zTxpgMY0xGXFznf7zPGxxPRJAfb2lVkVJKAa7tTZQBLLDn0YgFLhaROmPMO87+4ABfB98Z2Yc31uZTUV1LWGDnu2UppZQ7c1nJwBjTzxiTaoxJBd4AftAdiaDRlWMSqa5t4IPNhd31kUop1WM5LRmIyHysLqNniEi+iNwsIvNEZJ6zPrMjRiVFkhYXwptrC1wdilJKuZzTqomMMbM7cOxcZ8XREhHhyjGJ/GnxdvYWV5EcE9zdISilVI/hNXMTNefy0QmIwJvakKyU8nJenQz6RgYxqX8Mb63Pp0HHHCilnGTatGksXrz4W9ueeOIJbrvttmaPnzp1KpmZmd0R2nHekwwObIFFd0Ltt5eDu3JMInklR8ncU+qiwJRSnm727NksWLDgW9sWLFjA7Nntrk13Ou9JBuX7YM0zkLv8W5tnDOtNiL+DN9dqVZFSyjmuuuoqFi1adHwhm9zcXPbt28f8+fPJyMhg6NCh3H///S6N0XtmLU2dAr5BsHMxpJ9/fHOwvy8XDe/Dos2FPHDpUIL8HS4MUinldB/eA/s3d+01ew+Hi37f4u7o6GjGjx/Phx9+yKxZs1iwYAHXXHMN9913H9HR0dTX13PeeeexadMmRowY0bWxtZP3lAz8giDtHNixGE6aguPKMYlUHqvj46z9LgpOKeXpmlYVNVYRvfbaa4wZM4bRo0ezZcsWsrKyXBaf95QMANIvhB0fwaEdEHfG8c0T+kWTEBnEG2vzmTUqwYUBKqWcrpW/4J1p1qxZ/PSnP2XdunVUVVURHR3NY489xpo1a4iKimLu3LlUV1e3fSEn8Z6SAVjJAKzSQRM+PsIVYxL4KvsQ+8tc92UopTxXaGgo06ZN46abbmL27NmUl5cTEhJCREQEBw4c4MMPP3RpfN6VDCKTIH4o7Dx1+YQrxiTSYODt9ToiWSnlHLNnz2bjxo3Mnj2bkSNHMnr0aAYNGsR1113H5MmTXRqbd1UTAQy8EFY8CdVlEBhxfHO/2BDGpkTx5rp85p2Thj2BnlJKdZnLLruMpssGtLSIzRdffNE9ATXhXSUDgPTp0FAHuz4/ZdeVYxLJPljJ5oIyFwSmlFKu433JIHEcBEbCjlOrimaO6IO/r4+OOVBKeR3vSwYOXxhwPmR/Ag3fXlcnIsiPC4f0YuHGfdTUtbnmjlLKjThzVcee5HTv0/uSAcDA6XCkCPatP2XXlWMTKa2q5fNt3bYss1LKyQIDAykuLvb4hGCMobi4mMDAwA6f630NyGCVDMTHGo2cOPZbu84aEEtcWACvZeYxY1hvFwWolOpKiYmJ5OfnU1RU5OpQnC4wMJDExMQOn+edySA42mo72LEYpt33rV2+Dh9mj0viySXZus6BUh7Cz8+Pfv36uTqMHs07q4nAGoBWuAEqTp2C4roJKThEeGllbreHpZRSruC9yWDgdOt55yen7OodEcj0Yb15LTOPqpq6bg5MKaW6n/cmg17DIDzBajdoxtxJqZRX1/HO+n3dHJhSSnU/700GIpB+Aez6AupqTtmdkRLFkD7hvLgi1+N7ICillPcmA7BGI9dUwN4Vp+wSEeZOSmX7gQq+zilxQXBKKdV9vDsZpJ0DjoBmRyMDXDqqL1HBfry4Ird741JKqW7mtGQgIs+LyEER+aaF/deLyCYR2SwiK0RkpLNiaZF/iLUCWgvtBoF+Dr47LpmPs/ZTcPhoNwenlFLdx5klgxeAGa3s3w2cY4wZDjwEPO3EWFo2cDoUZ0PxrmZ333BmMgD//XpPd0allFLdymnJwBizFGixst0Ys8IYU2q//Rro+JC5rtC44E0zaxwAJEYFc/7gXixYvZfq2vpuDEwppbpPT2kzuBlocZkfEblVRDJFJLPLh5NH94PYgaesftbU3EmplFbVsnCjdjNVSnkmlycDEZmGlQx+0dIxxpinjTEZxpiMuLi4rg8i/ULY8xUcq2x298T+MQzsFardTJVSHsulyUBERgDPArOMMcUuC2TgdKivgZwvmt0tInxvYipb9pWzbm9ps8copZQ7c1kyEJFk4C1gjjFmh6viACB5IgSEt9irCODy0QmEBfrywgptSFZKeR5ndi2dD6wEzhCRfBG5WUTmicg8+5DfADHAP0Rkg4hkOiuWNjn8oP80a56iFqqBQgJ8uSYjiQ83F3KgvLqbA1RKKedyZm+i2caYPsYYP2NMojHmOWPMv4wx/7L3f98YE2WMGWU/MpwVS7ukT4eKQti/qcVD5pyZQr0xvLJqbzcGppRSzufyBuQeI/0C67mF0cgAqbEhTB0Yx6ur9uqymEopj6LJoFFoPPQd02q7AcCNk1I5VHmMDzYXdlNgSinlfJoMmho4HfIz4cihFg85Oz2OfrEhvKDzFSmlPIgmg6bSLwQMZH/a4iE+PsL3JqawIe8wG/MOd19sSinlRJoMmuozCkLiWx2NDHDV2ERC/B28uDK3W8JSSiln02TQlI+PVTrY9RnUt7zcZVigH1eOTeT9jYUcqjzWjQEqpZRzaDI42cALoboM8la1etj3JqZSU9/AgtXazVQp5f40GZwsbRr4+MGOj1o9bEB8KGelx/Lfr/dSW6/dTJVS7k2TwckCw60V0La8DQ2t/8jfNKUf+8urdSU0pZTb02TQnJGzoSwPcpe1etjUgXFMOyOOv3yyg8IyXQlNKeW+NBk0Z9BMCIiADa+2epiI8OClw6hrMPz2vaxuCk4ppbqeJoPm+AXBsCsg612oLm/10OSYYH507gA+/GY/S7Yf7KYAlVKqa2kyaMmo66HuqJUQ2nDL2WmkxYVw/7tbdGlMpZRb0mTQksQMiElvs6oIIMDXwcOzhrG3pIqnlmR3Q3BKKdW1NBm0RARGXQd7V0BJTpuHTxoQy2Wj+vKvL3exq6j55TOVUqqn0mTQmpHXgvjAxgXtOvyXM4cQ6Ofg1+98o2slK6XciiaD1oT3hbSpsGF+m2MOAOLCArh7+hms2FXMwo37nB6eUkp1FU0GbRl1PZTthT3L23X4dRNSGJEYwUPvb6W8utbJwSmlVNfQZNCWQTMhILxdDckADh/hkcuGU3LkGH9evN3JwSmlVNfQZNCWpmMOjlW065ThiRHMOTOFl7/ew+b8MicHqJRSnafJoD1GXQ+1Ve0ac9Do59PPICY0gF++s5n6Bm1MVkr1bE5LBiLyvIgcFJFvWtgvIvI3EckWkU0iMsZZsXRa4jiIGWA1JLdTeKAfv5o5mE35Zby6ao8Tg1NKqc5zZsngBWBGK/svAtLtx63AP50YS+c0jjnYsxxKdrf7tEtH9mXygBj+uHg7ByuqnRigUkp1jtOSgTFmKVDSyiGzgJeM5WsgUkT6OCueThtxLSDtHnMA1kR2D80axrHaBn63aKvzYlNKqU5yZZtBApDX5H2+ve0UInKriGSKSGZRUVG3BHeKiARrzMHGV9s15qBRWlwo885J450N+/gq+5DTwlNKqc5wiwZkY8zTxpgMY0xGXFyc6wIZdT0c3gt7vurQaT+YNoDUmGDumL+eHJ2qQinVA7kyGRQASU3eJ9rbeq4OjjloFOjn4Lm54wCY89xq9h3WhXCUUj2LK5PBQuB7dq+iM4EyY0yhC+Npm38wDL3cHnPQsb/w+8eF8uJN4yk/WssNz62iuPKYk4JUSqmOc2bX0vnASuAMEckXkZtFZJ6IzLMP+QDIAbKBZ4AfOCuWLjXqeqg9AlsXdvjUYQkRPDd3HAWlR7nxP6up0OkqlFI9hLjb7JoZGRkmMzPTdQEYA0+OtSaxm/v+aV1iybaD3PJSJmNSonjppvEE+jm6OEillPo2EVlrjMloab9bNCD3KI1jDnKXQWnuaV1i2qB4/nzNSNbklvDDV9ZRW9/+3klKKeUMmgxOx8iOjzk42axRCTw0axifbTvIna9vpEGnrFBKuZAmg9MRkQhp51i9ijow5uBkN5yZwl3Tz+DdDft44L0tuiCOUsplNBmcrlHXw+E91rKYnfCDqf3537PTeGnlHh7/ZEcXBaeUUh3j6+oA3NagS8A/zJq8LnXKaV9GRLjnokGUHa3lyc+ziQjy4/tnpXVhoEop1TYtGZwu/2AYdjlseRuqyzt1KRHhkcuHM3N4Hx5etJXX1uS1fZJSSnUhTQadkXGzNeZg2WOdvpTDR3j8uyM5Kz2We97axFvr8rsgQKWUah9NBp3RdxSMvgFW/gOKOl/fH+Dr4N9zxjKxfww/f30jC1bv7YIglVKqbZoMOuu8B8AvGD682xqQ1knB/r48d+M4zhkYxz1vbeallbmdvqZSSrVFk0FnhcbBub+EnCWw7fRGJJ8s0M8qIVwwpBe/eXcLzyzN6ZLrKqVUSzQZdIWMmyF+KHx0H9RUdcklA3wd/OP6Mcwc3odHPtjK3z/f2SXXVUqp5mgy6AoOX7j4T1C2F756ossu6+fw4a/XjuKK0Qk89vEO/vzxdh2YppRyCh1n0FVSJ8Pwq2H5EzByNkT365LL+jp8+NPVI/H39eHJz7M5VtfAvRcNQkS65PpKKQVaMuhaFzwEDj9YfF+XXtbhI/zu8uF8b2IKTy/N4YGFW3QuI6VUl9Jk0JXC+8A5v4DtH8COj7v00j4+woOXDuX7U/rx4so9/PKdzZoQlFJdRpNBV5swD2IHWl1Na6u79NIiwi9nDub2aQOYvzqPO9/YSJ1Of62U6gLtSgYiEiIiPvbrgSJyqYj4OTc0N+XrDxf9AUp3w8q/d/nlRYQ7p5/Bzy8YyFvrCrjlpUxdQlMp1WntLRksBQJFJAH4GJgDvOCsoNxe/3Nh8KWw9DE47Jx5hn50XjoPXTaMr3YVM/2JZXy5o8gpn6OU8g7tTQZijKkCrgD+YYy5GhjqvLA8wPRHrOePf+W0j5hzZgoLb59MdIgfNz6/mofez+JYXb3TPk8p5bnanQxEZCJwPbDI3qYL97YmMhnO+jlkvQM5XzjtYwb1Dmfh7VO4cWIKzy3fzWVPrWDngQqnfZ5SyjO1Nxn8BLgXeNsYs0VE0oAlzgvLQ0z6EUSlwgd3Q32t0z4m0M/Bg7OG8dyNGRwor+aSJ5fz8td7dICaUqrd2pUMjDFfGmMuNcb8wW5IPmSMucPJsbk/v0CY8Qc4tB1W/dvpH3fe4F589JOzmJAWw6/f+YZbXlpLyZEap3+uUsr9tbc30asiEi4iIcA3QJaI3NWO82aIyHYRyRaRe5rZnywiS0RkvYhsEpGLO34LPdwZMyB9Onzxe6jY7/SPiw8L5IW54/j1JUNYuqOI6U8sZdlObVxWSrWuvdVEQ4wx5cBlwIdAP6weRS0SEQfwFHARMASYLSJDTjrsV8BrxpjRwLXAPzoQu/uY8SjUH3NqY3JTPj7CzVP68c4PJxMZ5Mec51bzyKIsqmu1cVkp1bz2JgM/e1zBZcBCY0wt0FaF9Hgg2xiTY4ypARYAs046xgDh9usIYF8743EvMf1hys9g8+uQ/Wm3feyQvlbj8pwzU3hm2W5mPLGUFbsOddvnK6XcR3uTwb+BXCAEWCoiKUBbC/8mAE072efb25p6ALhBRPKBD4AfNXchEblVRDJFJLOoyE2rPM76GcSkw/s/hZoj3faxQf4OHrpsGK98fwIGuO6ZVdz1+kYOV2lbglLqhPY2IP/NGJNgjLnYWPYA07rg82cDLxhjEoGLgZcbRzqf9PlPG2MyjDEZcXFxXfCxLuAbAN/5Kxzea7UfdLPJA2JZ/JOzuW1qf95aX8D5j3/Jwo37tMeRUgpofwNyhIg83vjXuYj8GauU0JoCIKnJ+0R7W1M3A68BGGNWAoFAbLsid0epk2HMjbDyKSjc2O0fH+jn4BczBvHe7VNIiAzijvnr+Z8X1pBf2jUL8iil3Fd7q4meByqAa+xHOfCfNs5ZA6SLSD8R8cdqIF540jF7gfMARGQwVjJw03qgdrrgQQiOgYV3QINrGnSH9A3nrR9M5jeXDGH17hIueHwpzy7LoV5nQVXKa7U3GfQ3xtxvNwbnGGMeBNJaO8EYUwfcDiwGtmL1GtoiIr8VkUvtw34O3CIiG4H5wFzj6fUWQVHWRHaFG7pl7EFLHD7CTVP68fFPz+bMtGgeXrSVy//xFVv2lbksJqWU60h7fntFZCVwlzFmuf1+MvCYMWaik+M7RUZGhsnMzOzuj+1axsCr34Xc5fDDr62pK1wajuH9TYU8+N4WSqtquXFiKnecN4DIYH+XxqWU6joistYYk9HS/vaWDOYBT4lIrojkAn8H/rcL4vNOIjDzMev1ojut5ODScITvjOzLpz87h2syEvnPit2c/cclPLM0Rye+U8pLtLc30UZjzEhgBDDCHiR2rlMj83SRyXDur2DnYtjytqujASAy2J9HrxjBhz8+i9HJUTzywVbOf/xL3tNeR0p5vHZVEzV7osheY0y31294RDVRo4Z6ePY8KCuA21db7Qk9yLKdRTyyaCvb9lcwKimSX84czLjUaFeHpZQ6DV1VTdTstTtxrgLwcVhjD6qK4ZP7XR3NKc5Kj2PRHWfxp6tGsL+smqv/tZL/fTmTnKJKV4emlOpinUkGWm/QFfqMhIk/hHUvWg3KPYzDR7g6I4kld07lzgsHsnznIS78y1Luf/cbXW5TKQ/SajWRiFTQ/I++AEHGGF9nBdYSj6omalRzBP4xERz+MG+5NfV1D1VUcYy/fraD+avzCPJzMHdSKjdP6UdUiPY8Uqon61Q1kTEmzBgT3swjzBWJwGP5h8Alj0PxTlj+uKujaVVcWAAPXzacxT85m7MHxvLUF9lM+cPn/OGjbVpSUMqNnXYDsqt4ZMmg0Zu3WD2L5i2H+EGujqZddhyo4MnPs3l/0z6C/BzMOTOFW85OIzY0wNWhKaWaaKtkoMmgJ6ksgqfGQewZMHcRONyn8JV9sIK/f57Nwo378Pf14YYJKdx6ThrxYT23ykspb6LJwN1smA/vzINBl8CVz/Xo9oPm7Cqq5Kkl2byzvgA/hw/XTUhm3jn96RXuXvehlKfRZOCOvv4XfPQLSD0Lrn0VAsPbPqeHyT10hKeWZPPW+gIcPsIVoxO44cwUhiVEuDo0pbySJgN3tel1q4QQPwRueBNC410d0WnZW1zFP7/M5u31BVTXNjAyKZIbJiTznZF9CfRzuDo8pbyGJgN3tvMT+L85EN4H5rwNUamujui0lVXV8ua6fF5ZtYddRUeICPLjqrGJXD8hmbS4UFeHp5TH02Tg7vJWwytXg28gzHkLeg11dUSdYoxhZU4xr3y9l8Vb9lPXYJg8IIYbJqRw/pBe+Dk6Mw5SKdUSTQae4OBWePkKqD0C170GyWe6OqIucbCimtfW5DF/dR4Fh48SHxbAteOSuGZcEolRwa4OTymPosnAUxzeCy9fDmX5cM1LMHC6qyPqMvUNhiXbDvLfVXv4coe10N2UAbFcOy6Z84fEE+CrbQtKdZYmA09y5BD890rYvxku+weMvNbVEXW5/NIqXs/M5/XMPPaVVRMd4s8VoxP47rgk0nuFuTo8pdyWJgNPc6wCFlwHu5fC9N9Zk9x5oPoGw7KdRbyWmccnWQeorTeMSY7k2nHJzBzRh5AA9xmQp1RPoMnAE9Udgze/D1sXwtl3wbRfWquneahDlcd4e10B/5eZR/bBSkL8HXxnZF9mjUpgXGoUvtrorFSbNBl4qoZ6eP8nsO4lmPIzOO83Hp0QwOqJtG5vKQtW5/H+pkKO1tYTGezHuWfEc8GQXpw9ME5LDEq1oK1koP9y3JWPAy75K4jDmunU1MP5D3p0QhARxqZEMzYlmgcuHcrSHUV8knWAz7Yd5K31Bfj7+jC5fwwXDOnN+YPjidcpMJRqN6eWDERkBvBXwAE8a4z5fTPHXAM8gLVuwkZjzHWtXVNLBidpaIAP7oTM52DSj+CChzw6ITSnrr6BNbmlfJJ1gE+27iev5CgAI5MiuXBILy4c0ksbn5XXc1k1kYg4gB3ABUA+sAaYbYzJanJMOvAacK4xplRE4o0xB1u7riaDZhgDH9wFa56BibfDhQ97XUJoZIxhx4FKPsnazydZB9iYXwZYiWH2uCS+M7KvViUpr+TKZDAReMAYM91+fy+AMebRJsf8EdhhjHm2vdfVZNACY+Cje2DVv2DCbTDjUa9NCE3tL6tm0eZCFqzey0678fnSUQnMHp/E8IQIRP8bKS/hyjaDBCCvyft8YMJJxwwEEJGvsKqSHjDGfHTyhUTkVuBWgOTkZKcE6/ZEYMbvAYFV/wTTABf9wesTQu+IQG6e0o+bJqeybm8p81fn8fb6fOav3suQPuHMHp/ErNEJhAf6uTpUpVzKmSWDq4AZxpjv2+/nABOMMbc3OeZ9oBa4BkgElgLDjTGHW7qulgzaYAx8/CtY+XcYdwtc/CevTwgnK6+u5d0N+5i/ai9ZheUE+vkwc3hfZo9PYmxKlJYWlEdyZcmgAEhq8j7R3tZUPrDKGFML7BaRHUA6VvuCOh0iJ9oMVjxplRAufgx8tC9+o/BAP+acmcINE5LZXFDG/NV5LNxQwJvr8ukTEciUAbFMSY9l8oBYXb5TeQ1nJoM1QLqI9MNKAiFUnKgAABbmSURBVNcCJ/cUegeYDfxHRGKxqo1ynBiTdxCxexX5wFd/tRLCzMc1IZxERBiRGMmIxEh+NXMwizYX8sX2g3ycdYDX1+YDMLhPOGfZiWF8ajRB/jpPkvJMTksGxpg6EbkdWIzVHvC8MWaLiPwWyDTGLLT3XSgiWUA9cJcxpthZMXkVEXvcgQ8s/ws01MHMP4Ov/qXbnJAAX67JSOKajCTqGwzfFJSxPPsQy3ce4oWvcnl6aQ7+Dh8yUqOYPCCWs9JjGdo3AoePVikpz6AjkD2dMfD5w7DsMYjqZ81ndMZF2o7QAVU1dazeXcJX2YdYtvMQ2/ZXABAZ7Mek/jFMGRDHlAGxJMfotNuq59LpKJQl+zP46F44tB36nwvTH4X4Qa6Oyi0VVRzjq+xDx0sO+8urAUiODmZKeixTBsQyqX8MkcH+Lo5UqRM0GagT6mthzXPwxe/gWCWMvwWm3gNBUa6OzG0ZY9hVdITlO4tYnn2Ir3NKqDxWhwiMSIhgst0YnZESjb+vttko19FkoE515BAseQTWvgCBkXDur2DsXGu+I9UptfUNbMw7fLzUsD7vMPUNhmB/B2emxXBWeixnpcfRPy5Eu7CqbqXJQLWscJNVdbRnOfQaZg1SS53i6qg8SkV1LV/nlLBsZxHLdh5i96EjAPSNCOSs9DjOGhjL5P6xRIVolZJyLk0GqnXGQNY78PGvoSwPhlwGFz4EkTrS2xnySqpYtvMQS3cU8dWuQ1RUn6hSmpIey6T+sYxNiSLQT0tpqmtpMlDtU3sUvvqb1Q1VBKbeC2feBg6dpsFZ6uob2JhfdrzUsMGuUvJ3+DAqOZKJaTGcmRbD6ORITQ6q0zQZqI45nGfNgLrjQ6vq6JInIGmcq6PyChXVtWTmlrIyp5iVu4r5Zl8ZxkCArw9jkqOY2D+Gif1jGJkYqY3RqsM0GaiOMwa2vQ8f3A0VhTDuZmsltcAIV0fmVcqO1rJ6dwkrdxWzMqeYrYXlAAT5ORiVFMno5EhGJ0cxOjlSp81QbdJkoE7fsQr4/BFY/W8IibNmRR16uQ5Yc5HSIzWs2l3C1znFrN1TytbCcuoarH+/ydHBVnJIimRMShSDeodr6UF9iyYD1Xn71sN7P4HCDTDgfGtai6hUV0fl9Y7W1PPNvjLW7y1l/d7DrNtbyoHyY4BVtTQ8IYIxKVFkpEQxLjVaeyx5OU0Gqms01MPqZ+Dzh6zX59xtLbOpDcw9hjGGwrJq1u89bCWIvMNszi+jpr4BgAHxoYxLjWZ8vygyUqJJjArSsQ5eRJOB6lplBfDRL2DrexA32GpL0LmOeqzq2no2F5SxJreENbtLyNxTSkV1HQB9IgLJSI1mfGoU4/pFkx4fphPveTBNBso5tn8Ii++DkhzoOwbO/SX0P0+TQg9X32DYcaCCNbklrN5dwprckuNVSyH+DoYmRDAiIYLhiRGMSIwkJToYH00QHkGTgXKe+jrYOB++/COU7YWkM62pLfqd5erIVDsZY8gvPcrq3SVsyj/MpoIysvaVc6zOqloKC/RleGNySIhkRGKEVi+5KU0GyvnqamD9S7D0Masrar9zrKSQNN7VkanTUFvfwM4DlWwuOMym/DI2F5SxtbCc2nrrtyIs0Jfk6GASo4JIirKeE6OCSYy2nkMDnLlmljpdmgxU96k9Cpn/geWPw5EiSL8Qpt0HfUe7OjLVScfq6tmxv5JNBYfZVlhBfmkVeaVHyS+torq24VvHRgb7HU8SaXEhDOwVxqDe4aTFheDn0O6urqLJQHW/miOw+mlryc2jpTDoEjj7Tk0KHsgYQ/GRGvLtxJBfepS8Evu5tIo9xVXU22Mh/BxC/7hQBvYK44zeYQzqbT0nRGq1U3fQZKBcp7ocvv4nrPw7HCuHhAxrDYUhl4FfoKujU93gWF09OUVH2L6/gm37K9hxoILt+ysoOHz0+DGhAb6k9wplQFwo/eNDSYsNoX98KMnRwVqS6EKaDJTrVZfBhvmw5lko3gnBMTDme5Bxk86O6qXKq2vZsb+C7XZy2L6/gpxDRyiqOHb8GF8fISUmmP4nJYkB8aGEB+r4lo7SZKB6DmMg5wsrKWz/wNo2cAaM+z6kTQMf/SvQ25VX15JTdIRdByvZVdT4OMKe4iPHG7DBGiMxsFcYA3uFkt4rjIG9wkiPDyVEG69bpMlA9UyH82Dtf2Dti1B1CKL7W0lh1HUQFOnq6FQPU1ffQF7pUbIPVrLzYAU7D1SyfX8F2UWV1NSdaMBOjAqyEkOvUNLjw0iODiY5Opj4sACvHy+hyUD1bHXHIOtda6qL/NXgG2jNfzTkMhg4HQLDXR2h6sHqGwx7S6rYvr+CnQcq2HGwkp0HKthVVPmtkoS/rw+JkUEkRQeTFB1EcnQwSVHB9vtgIoI8v9rJpclARGYAfwUcwLPGmN+3cNyVwBvAOGNMq7/0mgw8WOFGWP9fa6qLikJw+EP/c2HILGvKi6AoV0eo3ERtfQN5JVb3170lVeSXVJFXWsXekirySo5SdrT2W8dHh/hbbRJxoaTFnXhOjg7G10MasV2WDETEAewALgDygTXAbGNM1knHhQGLAH/gdk0GioYGyF9jlRi2LrSW4/TxtQazDZlldVUNiXF1lMqNlR2ttbvAWgli96Ej7Co6Qk7REQ5VnmjE9nMIydHBpMWFWgkiNuT4ILs+kYFu1dvJlclgIvCAMWa6/f5eAGPMoycd9wTwCXAXcKcmA/UtxsC+dVZiyHoXSnNBHJA6GTJutpKD9lFXXajsaC05dsN1jt2InVN0hNyTGrF9BHqHB5IYFUxCVJCdJKxEkRAZRK/wQIL8e85ypW0lA2c2vScAeU3e5wMTmh4gImOAJGPMIhG5q6ULicitwK0AycnaFdGriEDCWOtx/oOwf7OVFLa8Ba/fCEkTYPqjkDjW1ZEqDxER5GevIPftasm6+gb2Ha4m/3CVPcjuxEC71btLeHfDURpO+ts6xN9BTGgAsaH+9rP1OjY0gJhQf2JCAugdEUhSVJDLq6Nc1g9LRHyAx4G5bR1rjHkaeBqskoFzI1M9lgj0GWE9pt1ntS98/jA8ey4MvxrOux8ik1wdpfJQvg4fkmOCSY4JbnZ/bX0D+8uqjyeJgxXHKK6s4VDlMYqPHGNvcRXr95ZScqTmlKTh7/AhLS7E6iYbb3WXTe8VSko3tlk4MxkUAE3/ZSba2xqFAcOAL+yh6L2BhSJyaVtVRUrh44CxN8KwK2D5E9Yo563vwcQfwpSfQkCYqyNUXsbP4XO8dxK03KZV32AorbKTRGUNBYftLrMHKli3p5T3Nu47fqy/rw9psSHHx1FMSY89pcTSVZyZDNYA6SLSDysJXAtc17jTGFMGxDa+F5EvaEebgVLfEhAG5/0axs6Fz34Ly/4M616yZk0dPcdKGkr1IA4fsauLAprdX3msjl0HK9lxoIKddpJYu6eUhRv3UVPf4H7JwBhTJyK3A4uxupY+b4zZIiK/BTKNMQud9dnKC0UmwZXPwIR51qI77/0YVj0N0x+2uqcq5SZCA3wZmRTJyKRvD76sPFZHfb0ThwLooDPlcYyxGpk/+Q0c3gNpUyF9OiRPgN4jdN1m5ZVc2ZtIKdcQgaGXWQPVVv0LVj8LOfda+3yDrJ5JSeMh+UxIHAfB0a6NV6keQEsGyjuU74O8VZC3GvZ+Dfs3QYO1MDyxZ5xIDgPOh7Dero1VKSfQuYmUak5NlTWYLW8V7F1lPVcfBvGBARfA6BusGVV9/V0dqVJdQquJlGqOfzCkTrEeYE2BUbQNvnkDNrwKr82x1l0Yca2VGHoNcW28SjmZlgyUOll9HeQsgfUvw7YPoKEW+o6B0dfDsKt0im3llrSaSKnOOFIMm1+DdS/DwS3WFNuDvwMjvmslCJ0wT7kJTQZKdQVjoHCDNQXGptfhWJm1PbQXxA+BXkOt5/jBEDfIqoZSqgfRZKBUV6s9CntWwMEsOJBllRiKtkNdtX2AQHSa1c4QPwR6D4c+IyEiSWdYVS6jDchKdTW/IBhwnvVo1FAPJbutxNCYIA5kwdb3AfsPrqAoKyn0GWU/j4Sofrr2s+oRNBko1RV8HBA7wHoMmXVie02VVYIo3GCt5Fa4EVY+ZTVKAwSEW6Oi+4yEvqMh7RwIjXfNPSivpslAKWfyD4bEDOvRqK4GiraeSA6FGyHzeag7au3vM8oa/DbgfGuEtEP/mSrn0//LlOpuvv4nqoka1dfBgc2Q/SlkfwbL/wLLHoOACKu00JgcIhJcF7fyaJoMlOoJHL5WNVHf0XD2XXD0MOz+8kRy2GpP8hs/xGqr6DfVmkIjMNylYSvPob2JlOrpjLFGR2d/aj32rID6GmvqjD4jIWWy9Ug+UyfdUy3SrqVKeZqaKshfbSWF3K8gfw3UH7P2xQ+F1MmQMslKENoYrWzatVQpT+MfbK3RkDbVel93DArWwp6vrASx/hVY/bS1LzzR6grr8Leqonz8rPUcfHztZ/u9ww8ik090fdUur15Hk4FS7s43wC4JTLLe19dC4SbYsxwObLGqlOprrSm7m76uPWp1ca2vs0oW2xZZ+wH8w6DPiBMN3X1GQky69mzyYPrNKuVpHH6QONZ6dERdjdU2cbzL6wbI/M+JLq++QdZo6l5DISIRwhMgvO+Jh39I19+L6jaaDJRSFl9/uzQwAphjbauvg+Kd3x4TkfUOHC099fzAiBMJIqyP9brf2VaJRafh6PG0AVkp1XE1VVBRaK0gV74Pygus54rCE68rDwLGql4aOxdGztZZXl1IG5CVUl3PPxhi+luPltQcgax3Ye0L8PEv4bMHYfClVmJInaKlhR7Gqd0FRGSGiGwXkWwRuaeZ/T8TkSwR2SQin4lIijPjUUp1I/8QGHUd3Pwx3LYSMm6C7E/gxUvg7xnw1d/gyCFXR6lsTqsmEhEHsAO4AMgH1gCzjTFZTY6ZBqwyxlSJyG3AVGPMd1u7rlYTKeXGaqpOlBbyvra6tg7+Dgy7EgLCrC6vPo4Tz9L4unG7w2qP8A1w9Z24HVdWE40Hso0xOXYgC4BZwPFkYIxZ0uT4r4EbnBiPUsrV/INh1GzrcXArrH0RNs6HLW+1/xo+vtYCQr2Hn3j0GqajrzvJmckgAchr8j4fmNDK8TcDHzoxHqVUTxI/GC76PZx/P+zffGL8Q0MdmIYTrxvqrPUiGuqtcRClu63jc76wEkmjiKSTEsRQiEju3NiIhgY4nGt93v5voKrYmhMqdYrVvdaD9IgGZBG5AcgAzmlh/63ArQDJycndGJlSyun8gqwf2NNRWQT7N9k/1vZjx0dWMgGrFBGZbI2oju5nrUDX+Doq1frsRjVHrNJK43UOfGMN2quptPaLD/iFQOZz1vuoflZSSD3LTg7uPaOsM9sMJgIPGGOm2+/vBTDGPHrScecDTwLnGGMOtnVdbTNQSrWqpsr6UT+YZZUiSnKsVehKdp9Yu7pRWF+ISoEjRVC8i+Or0gWEW1VPvYfZz8OtkowjwFrFbvcyyF1ujfKutq8Znfbt5BDet1tvuy0um6hORHyxGpDPAwqwGpCvM8ZsaXLMaOANYIYxZmd7rqvJQCl1WoyxBsuV2Ami1E4QpblWe0Nj20PvYRCZ0r6urw31VukhtzE5fHUiOYQn2FVWI05UXUWluqxLrUtnLRWRi4EnAAfwvDHmERH5LZBpjFkoIp8Cw4FC+5S9xphLW7umJgOlVI/VUG9VL+Uuh30brOqmQ9tPVFsFhDdp17CTRNwga/S3k+kU1kop5Uq1R60qq/2brQkEG9sjaqus/eIDob2skkREgjXTbESC/d6eAyo03upW2wk6AlkppVzJLwgSxlqPRg31VlXV/k1QtB3KCqA836py2vHxickBG/n4Wu0bE26FST9ySpiaDJRSqrv5OCA23XqcrLFtoyzfmufp+HMBhPZ2WkiaDJRSqicRsRq0g6PtGWS7hy5lpJRSSpOBUkopTQZKKaXQZKCUUgpNBkoppdBkoJRSCk0GSiml0GSglFIKN5ybSESKgD2neXos4GmLrnraPXna/YDn3ZOn3Q943j01dz8pxpi4lk5wu2TQGSKS2dpETe7I0+7J0+4HPO+ePO1+wPPu6XTuR6uJlFJKaTJQSinlfcngaVcH4ASedk+edj/geffkafcDnndPHb4fr2ozUEop1TxvKxkopZRqhiYDpZRS3pMMRGSGiGwXkWwRucfV8XQFEckVkc0iskFE3G5haBF5XkQOisg3TbZFi8gnIrLTfo5yZYwd1cI9PSAiBfb3tEFELnZljB0hIkkiskREskRki4j82N7ult9TK/fjzt9RoIisFpGN9j09aG/vJyKr7N+8/xMR/1av4w1tBiLiAHYAFwD5wBpgtjEmy6WBdZKI5AIZxhi3HCwjImcDlcBLxphh9rY/AiXGmN/bSTvKGPMLV8bZES3c0wNApTHmMVfGdjpEpA/QxxizTkTCgLXAZcBc3PB7auV+rsF9vyMBQowxlSLiBywHfgz8DHjLGLNARP4FbDTG/LOl63hLyWA8kG2MyTHG1AALgFkujsnrGWOWAiUnbZ4FvGi/fhHrH6rbaOGe3JYxptAYs85+XQFsBRJw0++plftxW8ZSab/1sx8GOBd4w97e5nfkLckgAchr8j4fN/8fwGaAj0VkrYjc6upgukgvY0yh/Xo/0MuVwXSh20Vkk12N5BZVKicTkVRgNLAKD/ieTrofcOPvSEQcIrIBOAh8AuwCDhtj6uxD2vzN85Zk4KmmGGPGABcBP7SrKDyGseowPaEe859Af2AUUAj82bXhdJyIhAJvAj8xxpQ33eeO31Mz9+PW35Expt4YMwpIxKoJGdTRa3hLMigAkpq8T7S3uTVjTIH9fBB4G+t/And3wK7XbazfPejieDrNGHPA/sfaADyDm31Pdj30m8Arxpi37M1u+z01dz/u/h01MsYcBpYAE4FIEfG1d7X5m+ctyWANkG63rvsD1wILXRxTp4hIiN0AhoiEABcC37R+lltYCNxov74ReNeFsXSJxh9N2+W40fdkN04+B2w1xjzeZJdbfk8t3Y+bf0dxIhJpvw7C6iizFSspXGUf1uZ35BW9iQDsrmJPAA7geWPMIy4OqVNEJA2rNADgC7zqbvckIvOBqVjT7R4A7gfeAV4DkrGmKr/GGOM2DbIt3NNUrOoHA+QC/9ukvr1HE5EpwDJgM9Bgb74Pq57d7b6nVu5nNu77HY3AaiB2YP2B/5ox5rf2b8QCIBpYD9xgjDnW4nW8JRkopZRqmbdUEymllGqFJgOllFKaDJRSSmkyUEophSYDpZRSaDJQ6hQiUt9k9soNXTnLrYikNp3RVKmewrftQ5TyOkftof1KeQ0tGSjVTvb6EX+015BYLSID7O2pIvK5PcnZZyKSbG/vJSJv2/PMbxSRSfalHCLyjD33/Mf2qFGlXEqTgVKnCjqpmui7TfaVGWOGA3/HGtEO8CTwojFmBPAK8Dd7+9+AL40xI4ExwBZ7ezrwlDFmKHAYuNLJ96NUm3QEslInEZFKY0xoM9tzgXONMTn2ZGf7jTExInIIa8GUWnt7oTEmVkSKgMSmUwDY0yZ/YoxJt9//AvAzxjzs/DtTqmVaMlCqY0wLrzui6fww9WjbneoBNBko1THfbfK80n69AmsmXIDrsSZCA/gMuA2OLz4S0V1BKtVR+heJUqcKsleNavSRMaaxe2mUiGzC+ut+tr3tR8B/ROQuoAj4H3v7j4GnReRmrBLAbVgLpyjV42ibgVLtZLcZZBhjDrk6FqW6mlYTKaWU0pKBUkopLRkopZRCk4FSSik0GSillEKTgVJKKTQZKKWUAv4fOin2Jj0OOh8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU9bn48c+TfYOEJBAgCwFkl00DrlVccK1iXaFWxfZXu9nW2lqXa3vVem97e22vXbztVevWRcSlFq2oaMUN1IRVCIsQQxZICAkJ2dfn98c5gSEkMEAmsz3v12tec+acM2eew4TzzPmuoqoYY4wJbxH+DsAYY4z/WTIwxhhjycAYY4wlA2OMMVgyMMYYgyUDY4wxWDIwYUJEckVERSTKi30XisgHAxGXMYHCkoEJOCJSLCJtIpLeY/0a94Ke65/IDoolSUQaRGSpv2Mxpj9YMjCB6nNgQfcLEZkKJPgvnENcBbQCc0Vk+EB+sDd3N8YcLUsGJlD9GbjR4/VNwDOeO4hIsog8IyJVIrJDRO4VkQh3W6SIPCQie0SkCLi0l/f+SUR2iUi5iDwoIpFHEd9NwB+B9cBXehz7TBFZISK1IlIqIgvd9fEi8is31joR+cBdN0dEynoco1hEzneX7xORF0TkLyKyD1goIrNFZKX7GbtE5PciEuPx/ikiskxEakSkUkTuEZHhItIkImke+53k/vtFH8W5mxBkycAEqo+AwSIyyb1Izwf+0mOf3wHJwBjgbJzkcbO77evAF4GZQB5wdY/3PgV0ACe4+1wA/D9vAhORUcAc4K/u48Ye25a6sQ0FZgBr3c0PAScDpwOpwI+BLm8+E5gHvACkuJ/ZCfwASAdOA84Dvu3GMAh4C3gdGOme49uqWgEsB671OO4NwCJVbfcyDhOqVNUe9gioB1AMnA/cC/wcuAhYBkQBCuQCkUAbMNnjfd8AlrvL/wK+6bHtAve9UUAGThFPvMf2BcA77vJC4IPDxHcvsNZdzsS5MM90X98N/L2X90QAzcD0XrbNAcp6+zdwl+8D3jvCv9lt3Z/rnsuaPva7DvjQXY4EKoDZ/v7O7eH/h5U9mkD2Z+A9YDQ9iohwfhFHAzs81u3AuTiD84u4tMe2bqPc9+4Ske51ET32P5wbgccAVLVcRN7FKTZaA2QD23t5TzoQ18c2bxwUm4iMB36Nc9eTgJPkVrmb+4oB4B/AH0VkNDABqFPVT44xJhNCrJjIBCxV3YFTkXwJ8FKPzXuAdpwLe7ccoNxd3oVzUfTc1q0U584gXVVT3MdgVZ1ypJhE5HRgHHC3iFSISAVwCvBlt2K3FBjby1v3AC19bGvEo3LcLRYb2mOfnsML/wHYDIxT1cHAPUB3ZivFKTo7hKq2AItx6jluwEm4xlgyMAHva8C5qtrouVJVO3Euav8hIoPcsvrbOVCvsBj4nohkicgQ4C6P9+4C3gR+JSKDRSRCRMaKyNlexHMTTpHVZJz6gBnAiUA8cDFOef75InKtiESJSJqIzFDVLuAJ4NciMtKt4D5NRGKBrUCciFzqVuTeC8QeIY5BwD6gQUQmAt/y2PYqMEJEbhORWPff5xSP7c/gFIVdjiUD47JkYAKaqm5X1YI+Nn8X51d1EfAB8DecCy44xThvAOuA1Rx6Z3EjEAMUAntxKmdHHC4WEYnDqXz9napWeDw+x7mo3qSqJTh3Mj8EanAqj6e7h/gR8CmQ7277LyBCVetwKn8fx7mzaQQOal3Uix8BXwbq3XN9rnuDqtYDc4HLcOoEPgPO8dj+IU7F9Wr37ssYRNUmtzEm3IjIv4C/qerj/o7FBAZLBsaEGRGZhVPUle3eRRhjxUTGhBMReRqnD8JtlgiMJ7szMMYYY3cGxhhjCL5OZ+np6Zqbm+vvMIwxJqisWrVqj6r27L+yX9Alg9zcXAoK+mppaIwxpjcicthmxFZMZIwxxpKBMcYYSwbGGGMIwjqD3rS3t1NWVkZLS4u/Q/G5uLg4srKyiI62uUiMMf0nJJJBWVkZgwYNIjc3F48hiUOOqlJdXU1ZWRmjR4/2dzjGmBASEsVELS0tpKWlhXQiABAR0tLSwuIOyBgzsEIiGQAhnwi6hct5GmMGVkgUExljTNBShfZmaGuA1nrn0dYArQ0H1nU/j78QMk/2SRiWDPpBdXU15513HgAVFRVERkYydKjT0e+TTz4hJiamz/cWFBTwzDPP8Nvf/nZAYjXG+FFXF+zZCmWfQOnHUJoP1dtAO717f1KGJYNAlpaWxtq1awG47777SEpK4kc/+tH+7R0dHURF9f5PnZeXR15e3oDEaYwZYC37oHwVlH7iJICyfGipc7bFD4Gs2TDxEogdDLGDnEdMEsQmucuDnOUY9xHhu5J9SwY+snDhQuLi4lizZg1nnHEG8+fP5/vf/z4tLS3Ex8fz5JNPMmHCBJYvX85DDz3Eq6++yn333UdJSQlFRUWUlJRw22238b3vfc/fp2JMaFCFhkqo3u78Gq/e5izX7oDBmTBiOoycASNmwOCRcLT1c001sHsT7C6Eyg1QVgCVG3GmrxYYNgkmXwHZp0D2bEg74eg/w4dCLhnc/8pGCnfu69djTh45mH+/7IhzpR+irKyMFStWEBkZyb59+3j//feJiorirbfe4p577uHFF1885D2bN2/mnXfeob6+ngkTJvCtb33L+hQYcySq0N4EzXsPPOorPC767oW/reHAeyJjIXUMpORAbQlsWwba5WxLSD+QGLqTRHK2c/Fub4aqLc5Ff3chVLrP9bsOHDsu2SnOmfhF58KfleesC2AhlwwCyTXXXENkZCQAdXV13HTTTXz22WeICO3t7b2+59JLLyU2NpbY2FiGDRtGZWUlWVlZAxm2MQOr5nPnYtrRCp1tBx4dbQe/7l7XUnfggt9Se2C5s62Xg4tzsU87AbJPdZ7TxjrPyVkQEXlg17ZG55f8zrWway3sWgfb/+dAeX58qlO0s/fzA0kjMhaGToDRZ0PGZBg2xXkeNCKgfvV7I+SSwbH8gveVxMTE/cs/+clPOOecc/j73/9OcXExc+bM6fU9sbGx+5cjIyPp6OjwdZjGDKyONihZAZ8tg61vQPVnXrxJICoWImOc8vX4IRCfAunj3eVeHknDYEiu8z5vxCQ6v+KzZx9Y197sJog1TnJoqYOpV8Owyc4jdQxEhsZlNDTOIgjU1dWRmZkJwFNPPeXfYIwZaPt2OcUwW9+AouVOcU1kDOR+AWb9P+cCHJ0AkdEHLvqR0c4v78gY5xe8P35pR8c7RTxZod/Iw5LBAPnxj3/MTTfdxIMPPsill17q73CM6X8dbW6beLetfFM1FH/gJICK9c4+gzNh6jUw7gIYc7bza9wEhKCbAzkvL097Tm6zadMmJk2a5KeIBl64na/xM1WnnLx8tfOo3ubROar+wHJvZfYS4bSeGXeB02Fq2OSgK0sPFSKySlX7vMWxOwNjzMEadjtt48tXO887VzsVtABRcZA+DuJSICXbbRPvto+PTTrQXr57/cgZTvm9CXiWDIwJJa0NsGOFUy7/+btQU+SUe8ckuh2XEntZTnTK5vdsdSpK60qdY0mE80t+0mUw8iSnqeSwSU5Zvgk5lgyMCWYdbVBeAEXvOgmgvAC6OpyLe84pTpPHjhan2WRbg/vcCA1VB7/uaHZa3mTNglO+6Vz4R0yzMv0wYsnAmGDS0ea0yS9+30kAO1ZAeyMgMHImnP5dGDPHKaePjvf+uF1dPh3qwAQ+nyYDEbkI+A0QCTyuqr/osX0U8AQwFKgBvqKqZb6MyZig0d3GvbsD1K71TiLorqhNGwczvuy0ysk98/jK5i0RhD2fJQMRiQQeAeYCZUC+iCxR1UKP3R4CnlHVp0XkXODnwA2+ismYgNXa4DS/3LXuwKNqi0fv1yHOsAinfguGT4Oc0yA5078xm5DiyzuD2cA2VS0CEJFFwDzAMxlMBm53l98BXvZhPD5zzjnncNddd3HhhRfuX/fwww+zZcsW/vCHPxyy/5w5c3jooYdstNJwpepU7HqOZFm58cAQB0nDnQv/xEud5xHTD4yLY4yP+DIZZAKlHq/LgFN67LMOuBKnKOlLwCARSVPVah/G1e8WLFjAokWLDkoGixYt4pe//KUfozIBo7XeaaJZlu+MX1+WD801zraYQZB1MnzhR04v1xHTYdBw/8ZrwpK/K5B/BPxeRBYC7wHlwCGzPIjILcAtADk5OQMZn1euvvpq7r33Xtra2oiJiaG4uJidO3fy7LPPcvvtt9Pc3MzVV1/N/fff7+9QjS+pwr6dzq/83Rud58qNzrDGuJ070yc449dnzXLGsh864eDB0ozxE18mg3Ig2+N1lrtuP1XdiXNngIgkAVepam3PA6nqo8Cj4PRAPuynLr0LKj49rsAPMXwqXPyLPjenpqYye/Zsli5dyrx581i0aBHXXnst99xzD6mpqXR2dnLeeeexfv16pk2b1r+xGf9obXAu8pUb3GGMNzrL3ROXAAzOckawnHQ5ZM9ymmtaBywToHyZDPKBcSIyGicJzAe+7LmDiKQDNaraBdyN07IoKHUXFXUngz/96U8sXryYRx99lI6ODnbt2kVhYaElg2DVXOs04/z8Peexe+OBbTFJTuesKVdCxhTnMWySXfhNUPFZMlDVDhG5FXgDp2npE6q6UUQeAApUdQkwB/i5iChOMdF3jvuDD/ML3pfmzZvHD37wA1avXk1TUxOpqak89NBD5OfnM2TIEBYuXEhLS4tfYjPHoLUBSj5yevEWv++07tEuZziG7FNgzt3OHWPGFEjOsaaZJuj5tM5AVV8DXuux7qceyy8AL/gyhoGSlJTEOeecw1e/+lUWLFjAvn37SExMJDk5mcrKSpYuXdrnHAYmADTVOBf8HR86v/zLVzk9eSOinfL9s+6A0Wc5y96Oj29MEPF3BXJIWbBgAV/60pdYtGgREydOZObMmUycOJHs7GzOOOMMf4dnwK3kLXc6cFWsP/DsOR7PyJOcnryjz3LuAmxIBhMGLBn0oyuuuALPIcH7msRm+fLlAxOQgZZ98Nmbzq/+ivVO44Km7pbL4k6HONuZYGXENKeSN8DnqjXGFywZmNC1+Z/wzx86E5VHRDuVuhMuhuHTnQt/xonOsMvGGEsGJgTVV8LSO6DwH84E5Vc+5hT3RMX4OzJjAlbIJANVRcKgu36wzUw3oFRhzV/gzX9zBnk791444zYbf98YL4REMoiLi6O6upq0tLSQTgiqSnV1NXFxcf4OJfDUFMEr33daAuWcDpf9BoaO93dUxgSNkEgGWVlZlJWVUVVV5e9QfC4uLo6srCx/hxE4Ojvgo/+Fd/4TIqLg0l/DyTdbu39jjlJIJIPo6GhGjx7t7zDMQNu1HpZ81xnvf/zFcOmvbFhnExK6upTy2ma2VtazpbKerRX1bKls4HvnnsDFU0f45DNDIhmYMNHeAtXboGqz0zu44AlISIWrn4QpX7Ihnk3Q6epS9jS2srWiweOiX89nlfU0th0Ys3NkchzjMgYRH+O7QQ0tGZjA09YE1Z85k7tUbXaed2+CvZ8fGPNfImD6ArjgQSchGONHDa0d7KhuZEd1E8XVjZTWNFHf0kFLeyfN7Z20tHfR3Nbp8frAek+piTFMyBjENXnZjM8YxIThSZwwbBDJ8b5vBGHJwASG5lpY+Xv49AXYW8z+IZ8joiB1rDMG0IlXOUM+D53odBaLtop0M3DqmtrZUdNIcXUTO/a4z9XO856G1oP2TUuMITkhmvjoSOKiI4mPjmRIQgzxMZHERUUQHxO5f1tKQjQTMgYxfvgg0pP8N9SJJQPjX21N8Mn/wQcPQ0stjLsAps8/cNFPHWv9A8yAUFX2NLTt/4XffaHfUeMs1za1H7T/iOQ4clITOG/iMEalJ5CblsiotARGpSWSFBt8l9bgi9iEho42WP00vPff0FAJ4y50+gWMsCG+jW/VNbdTVNVAUVUjRXuc5+LqJkqqGw8qp48QyBwSz6jURC6dOoLctERy0hIYnZ5ITmoCcdGhNSmRJQMzsLo6Yf1iWP6fUFsCo86Aa5+BnFP9HZkJIc1tneyqaz7ogt+9vKehbf9+kRHCqNQEctMTOXVMKqNSExiVnkhuWiKZKfHERIVPE2VLBmZgqMKmV+Cd/3AqhUdMhy/+D4w9z1oBmSPq6OxiV10LO6qbqG5spaaxjb1N7extbGNvk/OoaWyntqmNmsY2WjsOrphNS4xhzNBEzpuYwZihiYwZmsSYoc4v/OjI8LngH44lA+M7nR1QVwIVG+CDX8PONZA+Hq55GibPsyRgDtLa0UlpTfMhZfYlNU2U1jTR0XXwUCwikBwfTWpCDCkJ0WSmxDFl5GBSE53XQ5NiGTM0ibFDE0lJsHqnI7FkYI5PewvU7oCaz50hITwfdaXOBDEAydkw739h2nUQaX924ayhtYPPKuvZWlnP1soGtlbWU1TVyM66ZjyH3kqKjWJUWgKTRwzm4hOHMyotgZzURIYNjmVIQgzJ8dFERtgPiv5i/yvN0etogw8fhtXPQF0Z+5uBAsQmQ+poGDkTTrwSUsc4j8yTbYawMNPY2sH2qga2VjbwWWV3Z6oGymub9+8TGxXBCcOSyMsdQm5aFrnpzgU/Ny2B1MSYkB5rLNBYMjBHp3w1/ONWZ0L4cRfAzK8cuOAPGe10ALP/wCGrua2THTWNVDe0saehleqGNqobW9lT7z67r6sb2mjyaJkTExnBmKGJ5OUO4csZOYwblsT4jEFkpybYr/sAYcnAeKe9GZb/HFb8DpIyYMEiZ6IYE7Ka2zop3LWPDeV1fFpex4byOj7b3UBnj7L7qAghNTGG9KRY0pJiGJ2eSFpiDGlJsYxOT2BcxiBGpSYQZRW1Ac2SgTmyHSucAeGqt8FJN8Lcn0F8ir+jMv2opb2TjTv7vvCnJ8UwNTOZCyZnMH74IIYmxZKWFEt6UgyD46KJsF/3Qc+Sgelbaz28dT/kPwYpo+DGf8CYOf6Oyhynri6laE8Da0pqWVvqPDZX1B904T/RvfCfmJnM1Kxkhg+Os/L7EGfJwPRu29vOZDF1ZXDKt+C8n0BMor+jMseguqF1/0V/TUkt68pqqW9xWnkNio1iWnYy3zx7DNOyUphmF/6wZcnAHKx5L7zxb7D2r5A2Dr76BuSc4u+ojJc6u5StlfUU7NjLquIaVpXspbTGab0TGSFMyBjEZdNHMiM7hZnZKYwdmmRFPAawZGC6dbTCqqfhvV9CUw2ceTucfaeNDBrgmto6WFtay6rivRTs2MvqHXupb3V+9Q8bFMvJo4bwlVNGMSM7halZySTE2H950zv7ywh3PccKyv2CM0fAyBn+jsz0oOrMfrW+rI6C4r2s2lHDxp379vfMnZAxiMtmjCRv1BDyRqWSnRpvxT3Gaz5NBiJyEfAbIBJ4XFV/0WN7DvA0kOLuc5eqvubLmIxLFTb/E/71IFRtghEz4IsPw9hzrZ9AgNi9r4X1ZXWsL6tlfXkd68vqqGl0BlmLjYpgRnYK3zh7DHmjUjkpZwjJCb6fAMWELp8lAxGJBB4B5gJlQL6ILFHVQo/d7gUWq+ofRGQy8BqQ66uYjKvoXXj7fihf5dQL2FhBfre3sY315XV8WlbLurI6Pi2ro2JfC+AMpTw+YxDnTxrG1KwUpmUmM2nE4LAaUdP4ni/vDGYD21S1CEBEFgHzAM9koMBgdzkZ2OnDeEz5Knj7AShaDoOz4PLfO1NH2lhBA6q+pZ1Py50LvvOLv3Z/JS/AGHc45alZKUzPSmbyyMFW1m98zpd/YZlAqcfrMqBns5T7gDdF5LtAInB+bwcSkVuAWwBycnL6PdCQV1cGr9/lDCGdkAYX/hzyvmqVwwPA6cVbx7pSpzPXurJaiqoa92/PGhLPtKxkrj9lFNMykzkxK5nBcVbcYwaev39uLACeUtVfichpwJ9F5ERVPWgwclV9FHgUIC8vT3s5julLfSU89UVorII598Bp34bYQf6OKqTt3tfCsk2VvLmxkpXbq2nrdP6cMwbHMi0rhS/NyGRqVjLTslJITbShlU1g8GUyKAeyPV5nues8fQ24CEBVV4pIHJAO7PZhXOGjuRb+chU07HZ6D2fP8ndEIUlV2ba7gTcLK3mzsJJ1pbUA5KQmcMNpozh1TBrTspLJGGx3YiZw+TIZ5APjRGQ0ThKYD3y5xz4lwHnAUyIyCYgDqnwYU/hoa4K/XefMKnb9YksE/ayzS1ldspc3N1awrLCS4uomAKZnJfOjC8Yzd/JwxmckWdNOEzR8lgxUtUNEbgXewGk2+oSqbhSRB4ACVV0C/BB4TER+gFOZvFBVrRjoeHW0weIboewTuPpJp7moOS6dXcqWinryi2vIL65h5fZqqhvbiI4UThubzte+MIa5kzIYnmy//k1w8mmdgdtn4LUe637qsVwInOHLGMJOVxe8/C3Ytgwu+w1MucLfEQWllvZO1pbWUlBcQ37xwT17hw+O48xx6Zw/KYM5E4YyyCp8TQjwdwWy6U+qsPQO2PACnH8fnLzQzwEFj7aOLj7ctoeVRdXkF9ewobyO9k7nJnV8RhKXzRjJ7NxU8nKHkJliPXtN6LFkEEre+U/IfxxO/x6c+QN/RxPwVJV1ZXX8fXUZS9btZG9TOzGREUzLSuZrZ45hVu4QTh41xCZTN2HBkkGoWPm/ziBzM2+AuQ/4O5qAVra3iZfXlPPS6nKK9jQSExXB3MkZXDkzkzNOSCcuOtLfIRoz4CwZhIK1z8Ibd8Oky516AivCOMS+lnaWfrqLF1eX88nnNQDMHp3KN84ew8VTR1hHLxP2LBkEu82vwT++48xAdtXjEGG/asEpAirb20zBjhre3rSbZYWVtHZ0MSY9kR9dMJ55MzLJTk3wd5jGBAxLBsHs8/fh+YUwciZc91eIivV3RH7T2aVsrthHQfFe8otrKCjeu3+gtyEJ0Vw3K5srT8pielayVf4a0wtLBsFq7bPwz9shdTRc/zzEJvk7ogHV3ObR9NOd1KXBbfo5IjmOWaNTmZXrjOs/YfggIm02L2MOy5JBsGlrcpqPrvkLjDoTrn4CElL9HdWAUHV6/S76pJRX1++iub0TEWdSlytmjmRWbip5ualkpsT7O1Rjgo4lg2BStRWevwl2b4Kz7oCz7wqL4adrGtt4aXUZz+WX8tnuBhJjIrli5kgumDzcJnUxpp+E/pUkVKxfDK/c5gw7/ZUX4YTz/B2RT3V1KSu2V7Mov4Q3N1bS1tnFzJwUfnnVNC6dNoLEWPvTNaY/2f+oQNfeDEvvhNVPQ87pcPWfYPBIf0flMxV1LbywqpTnCkoprWkmJSGa60/NYf6sHCYMt6G3jfEVSwaBbM9nsPgm2L0RvvBDZz6CECwWUlU+3FbNMyuLeWtTJV0Kp49N40cXTODCKcOtE5gxAyD0riyh4tMX4JXvQ2QMXP8ijOt1Erigtq+lnZdWlfHMRzsoqmokNTGGW84ay/xZ2eSmJ/o7PGPCiiWDQNPe4kxRuepJyD7VaS2UnOnvqPrVlop6nllZzN/XlNPU1smM7BR+fe10Lpk6wu4CjPETSwaB5vU7YdVTcMZtcO69EBkaLWXaO7t4c2Mlz6ws5uPPa4iNiuDy6SO58bRcpmYl+zs8Y8KeJYNAUvKxkwhOuxXm3u/vaPpFXXM7z6wo5i8f76ByXyvZqfHcffFErs3LZojN/2tMwLBkECg62+HVH8DgTJhzt7+jOW71Le089WExj71fxL6WDs4eP5SfXzmKs8cPs97AxgQgSwaB4qM/OK2GrvtrUA8t0djawdMri3n0vSJqm9qZOzmD284fx5SRVhRkTCCzZBAIaktg+c9h/MUw8VJ/R3NMmts6+fNHxfzx3SJqGts4d+Iwbjt/HNOyUvwdmjHGC5YMAsHSO53nS34ZdHMRtLR38tePS/jD8u3saWjlrPFD+cH545iZM8TfoRljjoIlA3/b/E/Y8pozO1lKjr+j8VpLeyfP5ZfyyDvb2F3fyhknpPHH808iLzc8Bs0zJtRYMvCn1gZ47ccwbDKc+m1/R+OVfS3t/OWjHTzxQTF7GlqZnZvKbxfM5NQxaf4OzRhzHI6YDETkMuCfqto1APGEl+U/h31lcPWbAd+foHJfC0988Dl//biEhtYOvjAunW+dPYPTxqbZZDHGhABv7gyuAx4WkReBJ1R1s49jCg8VnzotiE66CXJO8Xc0fSqqauDR94p4aXU5HV1dXDJ1BN88eywnZlrrIGNCyRGTgap+RUQGAwuAp0REgSeBZ1W13tcBhqSuLqdPQXwKnH+fv6Pp1drSWv64fDtvFFYQExnBtbOy+PoXxjAqzcYMMiYUeVVnoKr7ROQFIB64DfgScIeI/FZVf9fX+0TkIuA3QCTwuKr+osf2/wHOcV8mAMNUNfTbIq5+Gsry4Yo/BtwsZe9/VsUj72zjo6IaBsdF8Z05J7DwjFzSk8J3fmVjwoE3dQaXAzcDJwDPALNVdbeIJACFQK/JQEQigUeAuUAZkC8iS1S1sHsfVf2Bx/7fBWYex7kEh4bd8Na/O1NWTp/v72j2K6lu4v5XNvL25t0MHxzHvZdOYv7sHJJsEhljwoI3/9OvAv5HVd/zXKmqTSLytcO8bzawTVWLAERkETAPJ4H0ZgHw717EE9zevNeZx/iL/xMQfQpa2jv53+Xb+eO724mKEO6+eCILz8glNspGDzUmnHiTDO4DdnW/EJF4IENVi1X17cO8LxMo9XhdBvRaUyoio4DRwL+8iCd4Fb0L659z5i8eOt6voagqbxZW8rNXCynb28zl00dyzyWTGJ4c59e4jDH+4U0yeB443eN1p7tuVj/GMR94QVU7e9soIrcAtwDk5ARPx6yDdLTCP2+HIbnOrGV+VFTVwP2vFPLu1irGZyTx7NdP5bSx1k/AmHDmTTKIUtW27heq2iYi3ow9XA5ke7zOctf1Zj7wnb4OpKqPAo8C5OXlqRefHXg+eBiqtzmzlkXH+yWEprYOfv+vbTz+/ufERkXwky9O5sbTRhEdGeGXeIwxgcObZFAlIper6hIAEZkH7PHiffnAOBEZjZME5gNf7rmTiEwEhgArvY462FRsgPcfgilf8sv0larK0ncHtAsAABdDSURBVA0VPPhqITvrWrhyZiZ3XTKRYYOsSMgY4/AmGXwT+KuI/B4QnHqAG4/0JlXtEJFbgTdwmpY+oaobReQBoKA7ueAkiUWqGpy/+I+kvQVe+jrEpcAlDw34x9c0tnHni+tZVljJpBGD+c2Cmcyy8YOMMT140+lsO3CqiCS5rxu8Pbiqvga81mPdT3u8vs/b4wWlt++H3YVw/QuQmD6gH/3+Z1XcvngddU3t/Nslk7j5jFyirEjIGNMLrxqRi8ilwBQgrnscGlV9wIdxhYZtb8NH/wuzb4FxcwfsY1s7Ovnv17fw+AefM25YEk/fPJvJIwcP2OcbY4KPN53O/ojTO/gc4HHgauATH8cV/Bqr4eVvw9CJzvDUA+Szynq+t2gtm3bt48bTRnHPJZOIi7Y+A8aYw/PmzuB0VZ0mIutV9X4R+RWw1NeBBTVVeOV70FQN1z8/IK2HVJW/fFzCg68WkhgbxZ9uyuO8SRk+/1xjTGjwJhm0uM9NIjISqAZG+C6kELDmz7D5VZj7MxgxzecfV93Qyp0vruetTbs5a/xQHrpmmrUUMsYcFW+SwSsikgL8N7AaUOAxn0YVzKq3w9K7YPRZcNqtPv+497ZW8cPnnUrin35xMgtPzyUiwv/DXBhjgsthk4GIRABvq2ot8KKIvArEqWrdgEQXbDrbnWakkdHOiKQRvmu509ml/Nfrm3n0vSLGZyTxzFdnM2mEVRIbY47NYZOBqnaJyCO4o4mqaivQOhCBBaV3fwnlq+CapyA502cf09HZxY9fWM9La8q54dRR/NulVklsjDk+3vx0fVtErhKb2/DwSj5yehlP/7LT09hH2ju7+P5za3lpTTk/nDuen11xoiUCY8xx86bO4BvA7UCHiLTg9EJWVbUyiW4t+5zioeRsuPi/fPYxrR2d3Pq3NSwrrOSeSyZyy1ljffZZxpjw4k0P5EEDEUhQW/pjqCuDm1+HON/kyJb2Tr75l1Us31LF/ZdP4abTc33yOcaY8ORNp7Ozelvfc7KbsLXhJVj3LJx9p88mtm9q6+CWZ1bx4fY9/PzKqSyYHaTDeBtjApY3xUR3eCzH4cxgtgo41ycRBZO6cnj1NsjMg7N+7JOPaGjt4KtP5VNQXMNDV0/nqpOzfPI5xpjw5k0x0WWer0UkG3jYZxEFkw8fdkYlvfJRiOz/uYLrmttZ+OQnrC+r4zfzZ3LZ9JH9/hnGGANeDlTXQxkwqb8DCTqd7bDhRZh4CaT1f0VubVMbN/zpEzZX7OORL5/ERScO7/fPMMaYbt7UGfwOp9cxOE1RZ+D0RA5v2952xh6adl2/H7q6oZXrH/+Yoj2N/N8NJ3PuRBtjyBjjW97cGRR4LHcAz6rqhz6KJ3isXwTxqXBC/85ctru+hesf+5jSvU386aY8vjBuaL8e3xhjeuNNMngBaOmerF5EIkUkQVWbfBtaAGupgy1LYeYNztAT/aSxtYObn8ynvLaZp26ezaljbJJ6Y8zA8KoHMuA5BnM88JZvwgkShUugowWmz++3Q3Z0dvHdZ9ewuaKeR64/yRKBMWZAeZMM4jynunSXE3wXUhBY/xykjoXMk/vlcKrKfa9s5F+bd/PAvCmcM2FYvxzXGGO85U0yaBSRk7pfiMjJQLPvQgpwdWVQ/IFTcdxPwzU9/v7n/OWjEr5x1hiuP2VUvxzTGGOOhjd1BrcBz4vITpxxiYYD/d+EJlisXwwoTLu2Xw639NNd/OfSTVwydTh3XjSxX45pjDFHy5tOZ/kiMhGY4K7aoqrtvg0rQKk6RUTZp0Dq6OM+3OqSvdz23FpmZqfw62tn2KQ0xhi/OWIxkYh8B0hU1Q2qugFIEpFv+z60AFSxHqo290vfgh3VjXz96QKGJ8fx2I15Ngy1McavvKkz+Lo70xkAqroX+LrvQgpg6xdDRPRxz1dQ29TGzU/m06nKkwtnkZYU208BGmPMsfEmGUR6TmwjIpFAjO9CClCdHfDp8zD+QkhIPebDtHZ0csszqyjb28yjN+QxZmhSPwZpjDHHxpsK5NeB50Tk/9zX3wCW+i6kAPX5cmioPK4ioq4u5Y7n1/NJcQ2/mT+D2aOPPakYY0x/8ubO4E7gX8A33cenHNwJrU8icpGIbBGRbSJyVx/7XCsihSKyUUT+5m3gA279YohLdu4MjtGvl21lybqd3HHhBObN8N0cycYYc7S8aU3UJSIfA2OBa4F04MUjvc8tTnoEmIsz0mm+iCxR1UKPfcYBdwNnqOpeEQnM3latDbDpFac5adSxle8vzi/l9+9sY/6sbL49x6arNMYElj6TgYiMBxa4jz3AcwCqeo6Xx54NbFPVIvd4i4B5QKHHPl8HHnErpVHV3Ud7AgNi8z+hvQmmHdvwE9urGrj35Q18YVw6P7viRKSfOqsZY0x/OVwx0Wac2cy+qKpnqurvgM6jOHYmUOrxusxd52k8MF5EPhSRj0Tkot4OJCK3iEiBiBRUVVUdRQj9ZP0iSMlx+hccJVXlJy9vIC46gl9fO4PoSG9K5owxZmAd7sp0JbALeEdEHhOR83B6IPenKGAcMAfnDuQxEUnpuZOqPqqqeaqaN3ToAA/pXF8BRcudiuOIo7+QL1m3kxXbq7njookMHWRNSI0xganPq5uqvqyq84GJwDs4w1IME5E/iMgFXhy7HMj2eJ3lrvNUBixR1XZV/RzYipMcAsenL4B2HVMrorrmdn726iamZyXzZZvE3hgTwI74U1dVG1X1b+5cyFnAGpwWRkeSD4wTkdEiEgPMB5b02OdlnLsCRCQdp9ioyPvwB8D6RTDyJEg/+hz1qze3UNPYyoNXTCXShpowxgSwoyr3UNW9bpHNeV7s2wHcCrwBbAIWq+pGEXlARC53d3sDqBaRQpy7jztUtfroTsGHKguh4tNjmrdgfVktf/5oBzecOoqpWck+CM4YY/qPN53Ojpmqvga81mPdTz2WFbjdfQSe9c+BRMKUK4/qbZ1dyr/9fQPpSbH88MIJR36DMcb4mTVt6UtXlzP8xAnnQ9LRVVr/9eMdfFpex72XTmJwXP9Ni2mMMb5iyaAvOz6AfeVHPW/B7voW/vv1LZxxQhqXTx/po+CMMaZ/WTLoy7rnIGYQTLz0qN72H//cRGtHFz+bZ53LjDHBw5JBb9qaoPAfMHkeRHs1DBMAH27bwz/W7uSbZ4+x0UiNMUHFkkFvti6FtvqjKiJq7ejkJy9vICc1gW+fc4IPgzPGmP7n09ZEQWvdczA4E3K/4PVbHn23iKI9jTx18yybtcwYE3TszqCntibY/rYzm5mXw0+UVDfx+3e2ccnU4cyZEJgDrxpjzOFYMuipLB+6OmD02V7trqr8dMkGoiKEn35xio+DM8YY37Bk0FPJSkAge7ZXu7++oYLlW6r4wdzxDE+O821sxhjjI5YMetqxAjJOhPhDBk89RENrB/e/UsikEYNZeHqu72MzxhgfsWTgqbMdygpg1Gle7f7kB59Tsa+FB684kSibp8AYE8TsCuapYj20N0LOkZNBV5fyXEEpZ5yQxsmjhgxAcMYY4zuWDDztWOk8jzr9iLuuLKqmbG8z1+ZlH3FfY4wJdJYMPJWshCGjYdDwI+76XH4pg+OiuHDKkfc1xphAZ8mgm6qTDLy4K6hrauf1jRVcMTPTOpgZY0KCJYNue7ZCUzXknHrEXf+xrpy2ji4rIjLGhAxLBt12rHCec458Z7C4oJTJIwZzYqbNYGaMCQ2WDLqVfASJQyFt7GF327izjg3l+7g2L2uAAjPGGN+zZNCtZIXTpPQIcxA8X1BGTGQEV8zMHKDAjDHG9ywZANSVQ23JESuPW9o7+fuaci6YkkFKQswABWeMMb5nyQDc8Yg4YmezZYWV1DW3c90sqzg2xoQWSwbgVB7HDHLGJDqMxQWlZKbEc8bY9AEKzBhjBoYlA3DuDLJnQWTfc/2U7W3ig217uPrkLCIibG5jY0xosWTQVAO7C4/YpPSFVWUAXGOtiIwxIciSQeknzvNhRirt6lKeLyjjjLHpZA1JGKDAjDFm4FgyKFkBEdGQeXKfu6zYXk15bTPXWsWxMSZE+TQZiMhFIrJFRLaJyF29bF8oIlUistZ9/D9fxtOrHSsh8ySIju9zl+cKSkmOj+aCyRkDGJgxxgwcnyUDEYkEHgEuBiYDC0Rkci+7PqeqM9zH476Kp1ftzbBzzWGblNY2tfHGxgqumDHSBqUzxoQsX94ZzAa2qWqRqrYBi4B5Pvy8o1dWAF3th+1s9o+1O2nr6OIaG5TOGBPCfJkMMoFSj9dl7rqerhKR9SLygoj0esUVkVtEpEBECqqqqvovwpKVgED27D53WVxQypSRNiidMSa0+bsC+RUgV1WnAcuAp3vbSVUfVdU8Vc0bOnRo/336jhUwbDLE9z5t5YbyOjbu3GdDVRtjQp4vk0E54HkVzXLX7aeq1ara6r58HOi7SU9/6+yAsvzDNil9vqCUmKgIrphhg9IZY0KbL5NBPjBOREaLSAwwH1jiuYOIjPB4eTmwyYfxHKzyU2hr6LPyuKW9k5fX7uSiKcNJTogesLCMMcYf+h5/4TipaoeI3Aq8AUQCT6jqRhF5AChQ1SXA90TkcqADqAEW+iqeQ+xwB6fro/L4TXdQOisiMsaEA58lAwBVfQ14rce6n3os3w3c7csY+lSyAlJGweCRvW5enO8MSnf62LQBDswYYwaevyuQ/UPVuTPo466gtKaJD7fv4Zo8G5TOGBMewjMZVG+Dpj191hd0D0p39ck2KJ0xJjyEZzLYscJ57uXOQFV5YVUZZ55gg9IZY8JHeCaDkpWQkA5pJxyyaUP5Psprm5lnzUmNMWEkfJNBzqkgh9YHLCusIELg3InD/BCYMcb4R/glg327YG/xYZuU5uWmkppoE94bY8JH+CWDEre+oJfK49KaJjZX1DN3kg1VbYwJL+GXDHashJgkGD7tkE3LCisBmGvzFhhjwkz4JYOSlZA1CyIP7W+3rLCSccOSyE1P9ENgxhjjP+GVDJproXJjr/UFtU1tfFJcY3cFxpiwFF7JoPRjQHutL1i+pYrOLrVkYIwJS+GVDEpWQkQ0ZB46UvaywkqGDYplelaKHwIzxhj/Cq9ksGMljJwBMQf3LG7t6GT5lt2cNynDxiIyxoSl8EkG7S2wc3WvRUQrt1fT2NbJ3MnW0cwYE57CJxmUr4LOtl4rj5cVVpIQE8npY9P9EJgxxvhf+CSD7s5m2acctLqrS3lrUyVnjRtKXHSkHwIzxhj/8+nkNgHl5Jth5ExISD1o9afldVTua7VWRMaYsBY+dwaJ6XDC+YesXlZYSWSE2MB0xpiwFj7JoA9vbaokb9QQhtjAdMaYMBbWyWD/wHRWRGSMCXNhnQzetIHpjDEGCPNksKywgvEZSYxKs4HpjDHhLWyTQW1TG/nFe+2uwBhjCONk8K/Nu92B6Yb7OxRjjPG7sE0G3QPTTctM9ncoxhjjdz5NBiJykYhsEZFtInLXYfa7SkRURPJ8GU+3lvZO3t1axfmTbWA6Y4wBHyYDEYkEHgEuBiYDC0Rkci/7DQK+D3zsq1h6WllUTVNbp9UXGGOMy5d3BrOBbapapKptwCJgXi/7/Qz4L6DFh7EcpHtgutPGpA3URxpjTEDzZTLIBEo9Xpe56/YTkZOAbFX95+EOJCK3iEiBiBRUVVUdV1BdXcpbhZWcPd4GpjPGmG5+q0AWkQjg18APj7Svqj6qqnmqmjd06NDj+tz15XXsrreB6YwxxpMvk0E5kO3xOstd120QcCKwXESKgVOBJb6uRF5WWGED0xljTA++TAb5wDgRGS0iMcB8YEn3RlWtU9V0Vc1V1VzgI+ByVS3wYUwsK6xkVu4QUhJsYDpjjOnms2Sgqh3ArcAbwCZgsapuFJEHRORyX33u4eyobmRrZYN1NDPGmB58OrmNqr4GvNZj3U/72HeOL2MB564AYO4kqy8wxhhPYdUDeVlhJRMyBpGTluDvUIwxJqCETTLY29hGfnGNtSIyxphehE0y+Nfm3XSpzV1gjDG9CZtkMDg+mrmTM5hqA9MZY8whfFqBHEjmTs6wuwJjjOlD2NwZGGOM6ZslA2OMMZYMjDHGWDIwxhiDJQNjjDFYMjDGGIMlA2OMMVgyMMYYA4iq+juGoyIiVcCOY3x7OrCnH8MJBKF2TqF2PhB65xRq5wOhd069nc8oVe1zqsigSwbHQ0QKVNWnM6kNtFA7p1A7Hwi9cwq184HQO6djOR8rJjLGGGPJwBhjTPglg0f9HYAPhNo5hdr5QOidU6idD4TeOR31+YRVnYExxpjehdudgTHGmF5YMjDGGBM+yUBELhKRLSKyTUTu8nc8x0tEikXkUxFZKyIF/o7nWIjIEyKyW0Q2eKxLFZFlIvKZ+zzEnzEejT7O5z4RKXe/p7Uicok/YzxaIpItIu+ISKGIbBSR77vrg/J7Osz5BO33JCJxIvKJiKxzz+l+d/1oEfnYveY9JyIxhz1OONQZiEgksBWYC5QB+cACVS30a2DHQUSKgTxVDdqOMiJyFtAAPKOqJ7rrfgnUqOov3KQ9RFXv9Gec3urjfO4DGlT1IX/GdqxEZAQwQlVXi8ggYBVwBbCQIPyeDnM+1xKk35OICJCoqg0iEg18AHwfuB14SVUXicgfgXWq+oe+jhMudwazgW2qWqSqbcAiYJ6fYwp7qvoeUNNj9TzgaXf5aZz/qEGhj/MJaqq6S1VXu8v1wCYgkyD9ng5zPkFLHQ3uy2j3ocC5wAvu+iN+R+GSDDKBUo/XZQT5HwDOl/2miKwSkVv8HUw/ylDVXe5yBRAKE1ffKiLr3WKkoChO6Y2I5AIzgY8Jge+px/lAEH9PIhIpImuB3cAyYDtQq6od7i5HvOaFSzIIRWeq6knAxcB33CKKkKJOGWawl2P+ARgLzAB2Ab/ybzjHRkSSgBeB21R1n+e2YPyeejmfoP6eVLVTVWcAWTglIROP9hjhkgzKgWyP11nuuqClquXu827g7zh/AKGg0i3X7S7f3e3neI6Lqla6/1G7gMcIwu/JLYd+Efirqr7krg7a76m38wmF7wlAVWuBd4DTgBQRiXI3HfGaFy7JIB8Y59auxwDzgSV+jumYiUiiW/mFiCQCFwAbDv+uoLEEuMldvgn4hx9jOW7dF0zXlwiy78mtnPwTsElVf+2xKSi/p77OJ5i/JxEZKiIp7nI8TkOZTThJ4Wp3tyN+R2HRmgjAbSr2MBAJPKGq/+HnkI6ZiIzBuRsAiAL+FoznIyLPAnNwhtutBP4deBlYDOTgDFV+raoGRaVsH+czB6foQYFi4BseZe0BT0TOBN4HPgW63NX34JSzB933dJjzWUCQfk8iMg2ngjgS5wf+YlV9wL1OLAJSgTXAV1S1tc/jhEsyMMYY07dwKSYyxhhzGJYMjDHGWDIwxhhjycAYYwyWDIwxxmDJwJhDiEinx+iVa/tzlFsRyfUc1dSYQBF15F2MCTvNbtd+Y8KG3RkY4yV3DolfuvNIfCIiJ7jrc0XkX+4gZ2+LSI67PkNE/u6OM79ORE53DxUpIo+5Y8+/6fYaNcavLBkYc6j4HsVE13lsq1PVqcDvcXq0A/wOeFpVpwF/BX7rrv8t8K6qTgdOAja668cBj6jqFKAWuMrH52PMEVkPZGN6EJEGVU3qZX0xcK6qFrmDnVWoapqI7MGZMKXdXb9LVdNFpArI8hwCwB02eZmqjnNf3wlEq+qDvj8zY/pmdwbGHB3tY/loeI4P04nV3ZkAYMnAmKNzncfzSnd5Bc5IuADX4wyEBvA28C3YP/lI8kAFaczRsl8kxhwq3p01qtvrqtrdvHSIiKzH+XW/wF33XeBJEbkDqAJudtd/H3hURL6GcwfwLZyJU4wJOFZnYIyX3DqDPFXd4+9YjOlvVkxkjDHG7gyMMcbYnYExxhgsGRhjjMGSgTHGGCwZGGOMwZKBMcYY4P8DX6Xbytua7HgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXwddb3/8dcnSZO02ZPuTdKdlha6EcqmUASkLFIXEIoLCMoF5IfoVQT1Xrmo150LKFxFLxfRqxVEtIoVAdm0FJqWLrQU2qZpm65p0ibdkmb5/P6YSXOanqRJk5OTJu/n43EeZ+Y7M+d8cpqcd2e+M98xd0dERKSlhHgXICIiPZMCQkREolJAiIhIVAoIERGJSgEhIiJRKSBERCQqBYTIcTKzUWbmZpbUjnWvN7N/dEddIl1FASF9gpmVmtkhMxvYov3N8Et+VHwq61jQiHQnBYT0JRuAuU0zZnYqMCB+5Yj0bAoI6Ut+CXwyYv464PHIFcwsy8weN7NyM9toZl8zs4RwWaKZ/cDMdplZCXBZlG3/x8y2mdkWM/ummSV2pmAzG25m882s0szWmdlnIpbNNLNiM6s2sx1mdl/YnmpmvzKzCjPbY2aLzWxIZ+qQvkkBIX3JIiDTzE4Ov7ivAX7VYp0fAVnAGOA8gkD5VLjsM8DlwHSgCLiyxbaPAfXAuHCd9wOf7mTN84AyYHj4fv9pZu8Llz0APODumcBY4Imw/brwZygA8oCbgYOdrEP6IAWE9DVNexEXAW8DW5oWRITG3e6+191LgR8CnwhX+Shwv7tvdvdK4NsR2w4BLgXucPf97r4T+K/w9Y6LmRUA5wBfdvcad18G/JzmvaA6YJyZDXT3fe6+KKI9Dxjn7g3uvsTdq4+3Dum7FBDS1/wSuBa4nhaHl4CBQD9gY0TbRmBEOD0c2NxiWZOR4bbbwsM6e4CfAoM7UetwoNLd97ZSz43AScCa8DDS5WH7L4FngXlmttXMvmdm/TpRh/RRCgjpU9x9I0Fn9aXA71ss3kXwv++REW2FNO9lbCM4bBO5rMlmoBYY6O7Z4SPT3Sd3otytQK6ZZUSrx93XuvtcghD6LvA7M0tz9zp3/w93nwScTXBY7JOIdJACQvqiG4H3ufv+yEZ3byA4jv8tM8sws5HAF2jup3gCuN3M8s0sB7grYtttwN+AH5pZppklmNlYMzuvA3WlhB3MqWaWShAEC4Fvh21Twtp/BWBmHzezQe7eCOwJX6PRzM43s1PDQ2bVBKHX2IE6RAAFhPRB7r7e3YtbWfz/gP1ACfAP4NfAo+GynxEculkOLOXoPZBPAsnAamA38DtgWAdK20fQmdz0eB/BabmjCPYmnga+7u7Ph+vPBlaZ2T6CDutr3P0gMDR872qCfpaXCQ47iXSI6YZBIiISjfYgREQkKgWEiIhEpYAQEZGoFBAiIhJVrxk9cuDAgT5q1Kh4lyEickJZsmTJLncfFG1ZrwmIUaNGUVzc2pmLIiISjZltbG2ZDjGJiEhUCggREYkqpgFhZrPN7J1wHPu72ljvI+EdtYoi2u4Ot3vHzC6OZZ0iInK0mPVBhOPAPEQwrHIZsNjM5rv76hbrZQCfA16PaJtEMEzyZIIRLZ83s5PCsXJERLpEXV0dZWVl1NTUxLuUmEtNTSU/P59+/do/sG8sO6lnAuvcvQTAzOYBcwjGqYn0DYKRKL8U0TYHmOfutcAGM1sXvt5rMaxXRPqYsrIyMjIyGDVqFGYW73Jixt2pqKigrKyM0aNHt3u7WB5iGsGRY+eX0TyOPQBmNgMocPdnOrptuP1N4S0Xi8vLy7umahHpM2pqasjLy+vV4QBgZuTl5XV4TylundThfX7vA/71eF/D3R9x9yJ3Lxo0KOppvCIibert4dDkeH7OWAbEFo68uUo+Ebd3BDKAU4CXzKwUOBOYH3ZUH2vbLrOvtp7v/XUNpbv2H3tlEZE+JJYBsRgYb2ajzSyZoNN5ftNCd69y94HuPsrdRxHcUP6KcJz++cA1ZpZiZqOB8cAbsSjyQG09jy0s5bt/XROLlxcRaVVFRQXTpk1j2rRpDB06lBEjRhyeP3ToUJvbFhcXc/vtt8e0vph1Urt7vZndRnCDlUTgUXdfZWb3AsXuPr+NbVeZ2RMEHdr1wGdjdQbT4MxUbj5vLPc99y5vbKhk5ujcWLyNiMhR8vLyWLZsGQD33HMP6enpfPGLXzy8vL6+nqSk6F/TRUVFFBUVRV3WVWLaB+Huf3H3k9x9rLt/K2z792jh4O6zIu/y5e7fCreb4O4LYlnnZ947hqGZqXzrmdU0NuoGSiISP9dffz0333wzZ5xxBnfeeSdvvPEGZ511FtOnT+fss8/mnXfeAeCll17i8ssvB4JwueGGG5g1axZjxozhwQcf7JJaes1YTJ3RPzmRL108gX99cjnzl2/lg9OPOmFKRHq5//jTKlZvre7S15w0PJOvf2Byh7crKytj4cKFJCYmUl1dzauvvkpSUhLPP/88X/nKV3jqqaeO2mbNmjW8+OKL7N27lwkTJnDLLbd06JqHaBQQoQ9NH8H/LtzA9/66htmnDCW1X2K8SxKRPuqqq64iMTH4DqqqquK6665j7dq1mBl1dXVRt7nssstISUkhJSWFwYMHs2PHDvLz8ztVhwIilJBgfPXSScz92SL+5x8b+Oz54+Jdkoh0o+P5n36spKWlHZ7+t3/7N84//3yefvppSktLmTVrVtRtUlJSDk8nJiZSX1/f6To0WF+Es8bmcdGkITz84jrK99bGuxwREaqqqhgxIjjs/dhjj3XreysgWrj7konU1jfyX8+/G+9SRES48847ufvuu5k+fXqX7BV0hLn3jrN2ioqKvKtuGHTP/FU8/lopCz53LhOGZnTJa4pIz/P2229z8sknx7uMbhPt5zWzJe4e9XxZ7UFE8bkLxpOeksR//uXteJciIhI3CogoctKSuf2C8bz8bjkvv6tBAEWkb1JAtOITZ42kMHcA33pmNfUNjfEuR0Sk2ykgWpGSlMjdl0zk3R37eKK4LN7liIh0OwVEG2afMpTTR+Vw33PvsK+2e88eEBGJNwVEG8yMr142iV37DvHfL62LdzkiIt1KAXEM0wqymTNtOD9/dQNb9hyMdzki0oucf/75PPvss0e03X///dxyyy1R1581axZddTp/eygg2uHO2RMB+L7uGSEiXWju3LnMmzfviLZ58+Yxd+7cOFV0JAVEO4zI7s+N7xnNH5ZtZfnmPfEuR0R6iSuvvJJnnnnm8M2BSktL2bp1K7/5zW8oKipi8uTJfP3rX49bfRqsr51umTWWJ4o3881nVvPEv5zVZ+5jK9JnLLgLtq/s2tcceipc8p1WF+fm5jJz5kwWLFjAnDlzmDdvHh/96Ef5yle+Qm5uLg0NDVxwwQWsWLGCKVOmdG1t7aA9iHbKSO3H5y86icWlu3l21fZ4lyMivUTkYaamw0tPPPEEM2bMYPr06axatYrVq1fHpTbtQXTA1UUF/GJhKd9esIb3TRxCcpLyVaTXaON/+rE0Z84cPv/5z7N06VIOHDhAbm4uP/jBD1i8eDE5OTlcf/311NTUxKU2fcN1QFJiAl+59GQ2Vhzg8ddK412OiPQC6enpnH/++dxwww3MnTuX6upq0tLSyMrKYseOHSxYENM7LrcppgFhZrPN7B0zW2dmd0VZfrOZrTSzZWb2DzObFLaPMrODYfsyM/tJLOvsiFkTBnPuSYN48IW17N5/KN7liEgvMHfuXJYvX87cuXOZOnUq06dPZ+LEiVx77bWcc845casrZsN9m1ki8C5wEVAGLAbmuvvqiHUy3b06nL4CuNXdZ5vZKODP7n5Ke9+vK4f7PpZ3tu/lkgde4bqzR/Wou1CJSMdouO/4Dfc9E1jn7iXufgiYB8yJXKEpHEJpwAlxc4oJQzO4+vRCfvnaRkrK98W7HBGRmIhlQIwANkfMl4VtRzCzz5rZeuB7wO0Ri0ab2Ztm9rKZvTfaG5jZTWZWbGbF5eXdOyz3Fy46iZSkBL6zQBfPiUjvFPdOand/yN3HAl8GvhY2bwMK3X068AXg12aWGWXbR9y9yN2LBg0a1H1FA4MyUrj1/HH8bfUOFpVUdOt7i0jX6S131TyW4/k5YxkQW4CCiPn8sK0184APArh7rbtXhNNLgPXASTGq87jd+J7RDM9K5ZvPrKaxsW/8kon0JqmpqVRUVPT6kHB3KioqSE1N7dB2sbwOYjEw3sxGEwTDNcC1kSuY2Xh3XxvOXgasDdsHAZXu3mBmY4DxQEkMaz0uqf0SuXP2RO747TK++czb3HXJRF0bIXICyc/Pp6ysjO4+RB0Pqamp5Ofnd2ibmAWEu9eb2W3As0Ai8Ki7rzKze4Fid58P3GZmFwJ1wG7gunDzc4F7zawOaARudvfKWNXaGVdMHc4bpZU8+s8NvL6hggeumca4wRnxLktE2qFfv36MHj063mX0WDE7zbW7dedprtE8u2o7dz21ggOHGvjqZSfziTNHarwmEenx4nWaa59y8eShPHvHuZw5Jo9//+Mqrv/fxezcG5/L40VEuoICogsNzkzlsU+dzjfmTGZRSQUX/9cr/PUtDewnIicmBUQXMzM+cdYonrn9PYzI6c/Nv1rCl3+3gv26p7WInGAUEDEybnAGv7/lHG6dNZYnlmzm0gdfZcnG3fEuS0Sk3RQQMZSclMCdsyfy25vOor7BueonC7nvuXepa2iMd2kiIsekgOgGM0fnsuCO9/LB6SN48IW1XPmT19iwa3+8yxIRaZMCoptkpvbjvo9O48fXTqd0134ufeBVfvPGpl5/BaeInLgUEN3s8inDefaOczltZA53/34ln3l8Cbv21ca7LBGRoygg4mBoViqP3zCTf7t8Eq+sLWf2/a+wcN2ueJclInIEBUScJCQYN75nNH+67T3kDEjmk4++wROLNx97QxGRbqKAiLMJQzN46tazOWtsHnc+tYLvP7tGI8OKSI+ggOgBMlP78ej1pzN3ZgEPvbie2+e9SU1dQ7zLEpE+LpbDfUsH9EtM4D8/dCqj8tL49oI1bKuq4ZFPnEZeekq8SxORPkp7ED2ImfEv543l4Y/N4K0tVXzo4YWs1z2vRSROFBA90KWnDmPeTWdy4FA9H354Ia+t1y1NRaT7KSB6qOmFOTx96zkMykjhk4++zlNLyuJdkoj0MQqIHqwgdwBP3XI2p4/K5V+fXM59z72rK69FpNsoIHq4rP79eOxTM/loUT4PvrCWO367jNp6neEkIrEX04Aws9lm9o6ZrTOzu6Isv9nMVprZMjP7h5lNilh2d7jdO2Z2cSzr7OmSkxL47kem8KWLJ/DHZVv5+M9fp3L/oXiXJSK9XMwCwswSgYeAS4BJwNzIAAj92t1PdfdpwPeA+8JtJwHXAJOB2cDD4ev1WWbGZ88fx4/mTmd5WRUffvifGhFWRGIqlnsQM4F17l7i7oeAecCcyBXcvTpiNg1oOsA+B5jn7rXuvgFYF75en/eBqcP5zWfOoLqmng89/E/e2FAZ75JEpJeKZUCMACIHFyoL245gZp81s/UEexC3d3Dbm8ys2MyKy8vLu6zwnu60kbk8fevZ5KYl8/Gfv84f3twS75JEpBeK+5XU7v4Q8JCZXQt8DbiuA9s+AjwCUFRU1KdO7xmZl8bTt5zDv/yqmDt+u4yfvlLCmEFpjB2YxphB6YwZFDynp8T9n1hETlCx/PbYAhREzOeHba2ZB/z3cW7bJ2UN6MfjN5zBT15ez9JNu1lZVsWClduIHOtvcEbK4bAYMzCNsWF45OcMIDHB4le8iPR4sQyIxcB4MxtN8OV+DXBt5ApmNt7d14azlwFN0/OBX5vZfcBwYDzwRgxrPWElJyVw+wXjD8/X1jewseIAJeX7WF++n5Ly/ZTs2sczK7ZRdbCuebvEBEbmDQj2OgalM2faCCYMzYjHjyAiPVTMAsLd683sNuBZIBF41N1Xmdm9QLG7zwduM7MLgTpgN+HhpXC9J4DVQD3wWXfXyf/tkJKUyElDMjhpyJFf9u5O5f5DbNgVhMb6XfsoKd/Pup37eOHtnfzk5fVce0YhX7hoArlpyXGqXkR6EustV+YWFRV5cXFxvMs4Ie05cIj7n1/LLxdtJC05kc9deBKfPGsk/RJ1HaVIb2dmS9y9KNoyfQMI2QOSueeKyfz1c+9lWmEO3/jzai6+/xVeXLMz3qWJSBwpIOSw8UMy+MWnTufR64vA4VOPLea6R99g3c698S5NROJAASFHMDPeN3EIf73jXL522cks3bSbi+9/lXvmr2LPAQ3vIdKXKCAkquSkBD793jG89MVZXHN6AY+/VsqsH7zELxaWUt/QGO/yRKQbKCCkTXnpKXzrQ6fyzO3vZdKwTL4+fxWXPPAqr7zbd65cF+mrFBDSLicPy+T/Pn0GP/3EadTWN/LJR9/gxscWU6Jboor0WgoIaTcz4+LJQ3nuC+dy1yUTeX1DJRff/wrf/PNqqg7UHfsFROSEousg5Ljt3FvDD599lyeWbGZAv0Q+fuZIbnzPaAZnpsa7NBFpp7aug1BASKet2V7Nwy+u588rtpKUmMBHi/L5l3PHUpA7IN6licgxKCCkW5Tu2s9PXynhqSVlNLhzxdTh3DJr7FHDfohIz6GAkG61vaqGn79awq/f2MSBQw28f9IQbj1/HNMKsuNdmoi0oICQuNi9/xCPLSzlsYWlVB2s45xxedw6axxnj83DTEONi/QECgiJq3219fzm9U387NUSdu6tZWpBNp+dNZYLTx5Cgu5JIRJXCgjpEWrqGvj90i385OX1bKo8wElD0rll1lg+MGU4SRo5ViQuFBDSo9Q3NPLMym08/OJ63tmxl4Lc/lw8aSgTh2Vy8rAMxg1OJyUpMd5livQJbQWEblgs3S4pMYE500bwgSnD+fuanfzs1RJ+uWgjtfXBGE+JCcbYQWmcPCyTiUMzmTgsg0nDMhmckaK+C5FupICQuElIMC6cNIQLJw2hodHZsGs/a7ZXs2bbXt7eVk1x6W7+uGzr4fVzBvRj4tDMIDiGZXDy0EzGD0kntZ/2NkRiQQEhPUJigjFucDrjBqdz+ZTm9qqDdazZVs2a7XtZs72a1dv28us3NlJT17y3MXpgGueMzeOKaSOYUZitvQyRLqI+CDnhNDQ6Gyv2B6GxrZq3tlbzj3W7OFTfSH5Ofz4wdThXTB3OxKEZCguRY4hbJ7WZzQYeABKBn7v7d1os/wLwaaAeKAducPeN4bIGYGW46iZ3v6Kt91JA9G3VNXX8bdUO5i/fyj/X7aKh0Rk/OJ0504ZzxdQRFOZp2A+RaOISEGaWCLwLXASUAYuBue6+OmKd84HX3f2Amd0CzHL3q8Nl+9w9vb3vp4CQJrv21bJg5TbmL9/K4tLdAEwtyOaKqcP5wJRhGkxQJEK8AuIs4B53vzicvxvA3b/dyvrTgR+7+znhvAJCOm3LnoP8eflW5i/fyqqt1ZjBWWPyuGLqcC45ZRhZA/rFu0SRuIpXQFwJzHb3T4fznwDOcPfbWln/x8B2d/9mOF8PLCM4/PQdd/9DlG1uAm4CKCwsPG3jxo0x+Vmkd1i3cx/zl2/lT8u3smHXfvolGuedNIgPTB3O+yYOJiNVYSF9T4+/DsLMPg4UAedFNI909y1mNgb4u5mtdPf1kdu5+yPAIxDsQXRbwXJCGjc4nS9cdBKfv3A8b22pZv7yLfxp+Taef3snCQanjMjijNG5nDkmj6JRuWT1V2BI3xbLgNgCFETM54dtRzCzC4GvAue5e21Tu7tvCZ9LzOwlYDqwvuX2Ih1lZpyan8Wp+VncfcnJFG/czT/X7WJRSQW/WLiRn726gQSDScMzOXN0HmeOyeP00QoM6XtieYgpiaCT+gKCYFgMXOvuqyLWmQ78juBQ1NqI9hzggLvXmtlA4DVgTmQHd0vqg5CuUFPXwJub9vD6hgoWlVSwdNMeDtU3YgaThmVyxug8zhyTy8zRuWQPSI53uSKdFpdDTO5eb2a3Ac8SnOb6qLuvMrN7gWJ3nw98H0gHngzPV286nfVk4Kdm1khw3+zvtBUOIl0ltV8iZ43N46yxeUAQGMs372FRSSWvb6jg/17fyKP/3IAZTByayZljcjljdB6njMhkWFZ/EjU6rfQiulBOpANq6xtYvrmKRSUVvL6hgiUbdx++qjs5MYGC3P6MyktjZF4aowYOCJ7zBjAiu79GrJUeqcd3UoucKFKSEpk5OjjEBOOprW9gZVkVa3fuo7RiPxt3HaC0Yj8L11dwsK7h8HZJCUZ+Tv/DgREZIPk5/TV6rfRICgiRTkhJSqRoVC5Fo3KPaHd3yvfWUloRBMbGiv2UVhxgY8V+lm7czd7a+sPrJhiMHZTO9MJsphfmML0wm/GDM3S4SuJOASESA2bG4MxUBmemhnsbzdydyv2HKK04wKbK/Wwo389bW6t5bvUOniguAyAtOZGpBdlBaBTkMK0wm4HpKfH4UaQPU0CIdDMzIy89hbz0FE4bmXO43d3ZWHGANzfv5s1Ne3hz0x5+8nIJDY1BP2Fh7oAwMII9jZOHZZKcpH4NiR0FhEgPYWaMGpjGqIFpfGh6PgAHDzWwcksVb24KQuO19RWH75GRnJTAqSOymF6QzbTCbKbmZ5Of018j2EqXaddZTGaWBhx090YzOwmYCCxw97pYF9heOotJ+gJ3Z1tVTbiHsZs3N+9h5ZYqDoV348tNS2ZKfhZT8rOZGj4PytChKWldV5zF9Arw3vACtr8RXPR2NfCxrilRRNrDzBie3Z/h2f25bMowAA7VN/LO9r0sL9vD8s17WFFWxSvvriU8MsXwrFSmFmQfDo1T8rPI1LhT0g7tDQgLh+S+EXjY3b9nZstiWZiItE9yUsLhoUM+fuZIAPbX1rNqazXLN+9heVkQGgve2n54mzGD0piWnx3sbRRkM3l4pk61laO0OyDC4bs/BtwYtum3SaSHSktJirheI1C5/xArwrBYUbaHV9bu4vdvBsOjpSQlMKMwhzPHBEOJTCvMVmBIuwPiDuBu4OlwuIwxwIuxK0tEulpuWjKzJgxm1oTBQHN/xvLNe1hcuptFJRXc/8K7+PNBYEwvzA4DI49pBdmk9lNg9DUdHmrDzBKAdHevjk1Jx0ed1CKdV3WgjjdKK1lUEgxWuHpbNe7BYawZCoxeqdM3DDKzXwM3Aw0EHdSZwAPu/v2uLLQzFBAiXa8pMF4vqWDRhgpWbW0OjOkFzYExvVCBcaLqioBY5u7TzOxjwAzgLmCJu0/p2lKPnwJCJPaqDtaxeENlOFhhJau2VtHowVhTk4dnMr0whxkjc5hRmM2I7Nhck+HubK2qYWXYn7JySxV5aclceVoBZ4/NI0FDlHRIVwTEKmAa8GuC+0a/bGbL3X1q15Z6/BQQIt2v6mAdxaWVFG/czdKNu1lRVnV4kMLBGSnMKMxhxshsZhTmcMqIrOPay9hRXRMEQdkeVmypYmVZFRX7DwFBMI0fksHWPQepOljHiOz+fGTGCK48rYDCvAFd+rP2Vl0RELcDXwaWA5cBhcCv3P29XVloZyggROKvriG4JmPppt0s2bibpZt2s7nyIAD9Eo1Jw7OYURgExmkjcxie3f+I7cv31vLWlqpwzyDYQ9i5N7jRZILBSUMyOHVEFlPyszg1P5uJQzNI7ZdITV0Dz63ewZNLynh1bTnucOaYXK46rYBLTh3KgGQNGtGaTgdEKy+a5O71x16zeyggRHqm8r21LN0UhMWbG4PrMmrDK7+HZqYyY2Q2DY3OyrIqtlbVAGDhCLdTRgTXd0zJz2LSsCz6Jx97D2TrnoP8fmkZTy4pY2PFAdJTkrh8yjCuKipgRmG2hiJpoSv2ILKArwPnhk0vA/e6e1WXVdlJCgiRE0NdQyNvb6tm6cbdLN20hzc37ybRjCnhhXunjshi8ogs0lM6979+d2dx6W6eKN7MX1Zu48ChBsYMSuOq0wr48IwRDMlM7aKf6MTWFQHxFPAW8Iuw6RPAVHf/cJdV2UkKCBFpzb7aev6yYhtPLtnM4tLdJBjMmjCYq07L54KTh/TpUXG77CymY7VF2W428ADBVdc/d/fvtFj+BeDTQD1QDtzg7hvDZdcBXwtX/aa7/4I2KCBEpD1KyvfxuyVlPLW0jB3VteQM6Md5Jw1i/JAMxg1OZ/zgdApzB/SZW8R2RUC8BnzJ3f8Rzp8D/MDdz2pjm0TgXeAioIzg+om57r46Yp3zgdfDcZ5uAWa5+9VmlgsUA0WAA0uA09x9d2vvp4AQkY5oaHReXVvOk0vKWLpxN9vC/g8I7i8+emAa44akM25QOuOHpDNucDqjB6b1uiFIumI015uBx8O+CIDdwHXH2GYmsM7dS8Ii5gFzgMMB4e6Rw3UsAj4eTl8MPOfuleG2zwGzgd+0s14RkTYlJtgRQ4/sraljffl+1u3cx9qde1m3Yx9vbaniLyu30fT/6ASDkXlpjBucfnhvY/zgDEYOHEBGSlKv6wBvV0C4+3JgqpllhvPVZnYHsKKNzUYAmyPmy4Az2lj/RmBBG9uOaE+tIiLHIyO1H9MKsplWkH1Ee01dAyXl+1m7cy/rd+5jbfh4cc1O6hubj8CkJScyNCs1eGT2Z1g43fzcn5wB/U6oEOnQaQItxl/6AnB/VxRhZh8nOJx0Xge3uwm4CaCwsLArShEROUJqv0QmDc9k0vDMI9rrGhrZWLGftTv2Ubb7INuqatheHTwvXL+LHdU1NLY4gp+clMDQzBbBkZlKTloyKUkJpPRLJCUpgdQ2npMSrNtCpjPnkR2rwi1AQcR8fth25IuYXQh8FTjP3Wsjtp3VYtuXWm7r7o8Aj0DQB9HOukVEOq1fYgLjBmcwbnBG1OX1DY3s2neIbVUH2V5Vw/bqGrZX1QRBUlXD0k272VFVy6GGxg69b4JxVHCcmp/Nj+ZO74of6widCYhjfSEvBsab2WiCL/xrgGsjVzCz6cBPgdnuvjNi0bPAf4Z3sAN4P8Fw4yIiJ4SkxITDh5xa09joVOw/RHVNHTV1DdTWN1Jb10hNfQO1dY3URjzXtHiurW88vE1+Tv9W36NTP0NbC81sL9GDwIA2K3L3ejO7jeDLPksw+LQAABKKSURBVBF4NLyXxL1AsbvPB74PpANPhrtMm9z9CnevNLNvEIQMBBflVXbkBxMR6ekSEoxBGSk99r7hxz3URk+j01xFRDqurdNc+8aVICIi0mEKCBERiUoBISIiUSkgREQkKgWEiIhEpYAQEZGoFBAiIhKVAkJERKJSQIiISFQKCBERiUoBISIiUSkgREQkKgWEiIhEpYAQEZGoFBAiIhKVAkJERKJSQIiISFQKCBERiUoBISIiUcU0IMxstpm9Y2brzOyuKMvPNbOlZlZvZle2WNZgZsvCx/xY1ikiIkdLitULm1ki8BBwEVAGLDaz+e6+OmK1TcD1wBejvMRBd58Wq/pERKRtMQsIYCawzt1LAMxsHjAHOBwQ7l4aLmuMYR0iInIcYnmIaQSwOWK+LGxrr1QzKzazRWb2wWgrmNlN4TrF5eXlnalVRERa6Mmd1CPdvQi4FrjfzMa2XMHdH3H3IncvGjRoUPdXKCLSi8UyILYABRHz+WFbu7j7lvC5BHgJmN6VxYmISNtiGRCLgfFmNtrMkoFrgHadjWRmOWaWEk4PBM4hou9CRERiL2YB4e71wG3As8DbwBPuvsrM7jWzKwDM7HQzKwOuAn5qZqvCzU8Gis1sOfAi8J0WZz+JiEiMmbvHu4YuUVRU5MXFxfEuQ0TkhGJmS8L+3qP05E5qERGJIwWEiIhEpYAQEZGoFBAiIhKVAkJERKJSQIiISFQKCBERiUoBISIiUSkgREQkKgWEiIhEpYAQEZGoFBAiIhKVAkJERKJSQIiISFQKCBERiUoBISIiUSkgREQkKgWEiIhEpYAQEZGoYhoQZjbbzN4xs3VmdleU5eea2VIzqzezK1ssu87M1oaP62JZp4iIHC1mAWFmicBDwCXAJGCumU1qsdom4Hrg1y22zQW+DpwBzAS+bmY5sapVRESOFss9iJnAOncvcfdDwDxgTuQK7l7q7iuAxhbbXgw85+6V7r4beA6YHcNaRUSkhVgGxAhgc8R8WdjWZdua2U1mVmxmxeXl5cddqIiIHO2E7qR290fcvcjdiwYNGhTvckREepVYBsQWoCBiPj9si/W2IiLSBWIZEIuB8WY22sySgWuA+e3c9lng/WaWE3ZOvz9sExGRbhKzgHD3euA2gi/2t4En3H2Vmd1rZlcAmNnpZlYGXAX81MxWhdtWAt8gCJnFwL1hm4iIdBNz93jX0CWKioq8uLg43mWIiJxQzGyJuxdFW3ZCd1KLiEjsKCBERCQqBYSIiESlgBARkagUECIiEpUCQkREolJAiIhIVAoIgMaGeFcgItLjJMW7gLir3Qc/nAjDpkLBTCg4I3gekBvvykRE4koBUV8D0z8Om1+HhQ9CY33Qnje+OSwKzoCBJ0GCdrhEpO9QQKQNhEu+E0wfOgBb3wzCYvMb8M5fYNmvgmWp2WFYhIExfAakpMevbhGRGFNAREoeAKPOCR4A7lCxPgyMMDTW/i1YZokw9JQgLPJnQsYQ6DcA+vUPn8Pp5DRISIzfzyQicpw0WF9HHdwNZUuaQ6OsGOr2t71NYnIYHGnNAZI8oLkteQCMPAcmfwj6Z8f+ZxARCbU1WJ8CorMa6mHXO0FwHDoAdZGPgxFtB4MgOartAByohL1bITEFJsyGqXNh3IWQ2K/7fx4R6VPaCggdYuqsxCQYMrlzr+Ee9H2s+C2sfBJW/xEG5MEpH4Gp1wT9HWZdU6+ISDtpD6KnaaiDdS/Ainmw5i/QUBucUTX1aphyNWQXxrtCEelFdIjpRHVwT7A3sXwebFoYtI18T7BXMWkOpGbGtz4ROeEpIHqD3aWw4okgLCrXQ1IqTLwMplwDY98XHOoSEemguAWEmc0GHgASgZ+7+3daLE8BHgdOAyqAq9291MxGEdzH+p1w1UXufnNb79XrA6KJO2xZAst/A289FXSOpw2CwScH/RYD8qB/bvP0gJwj25PTOtaf0dgAtdXB3kxNFdTsOXK6pip41NcGtXkDeGOwnTdGzDe2mG9a3hhsN/QUmHApjDxbnfMi3SguAWFmicC7wEVAGbAYmOvuqyPWuRWY4u43m9k1wIfc/eowIP7s7qe09/36TEBEqj8E656Dt34PVZvhQEVwRtTB3UAr/66JKWFg5IaPPOifAw2HWoRA+MVfW936a0FwPUhqZrBHYwnBvFlw7cfh+YRw3prnD7clBCGxdVnQ35KaDSfNhomXwtgLdDGiSIzF6yymmcA6dy8Ji5gHzAFWR6wzB7gnnP4d8GMzna7TbknJwWGmiZcd2d7YEHzZH6wMQyMMjqbpg5XN89vfCuYTUyA1K7gOIzMfBk8OplOzgi/tpmVHTGdBcnrXnGF1aD+s/zuseQbeWRB00iemwNjzgz2LCZdA+uDOv4+ItFssA2IEsDlivgw4o7V13L3ezKqAvHDZaDN7E6gGvubur7Z8AzO7CbgJoLBQZ/cclpAIaXnBg/HxrqZ9ktPg5A8Ej4Z62PRaEBZrnoF3/wp/suCq9aZAzBvbufdzh9q9sG8n7NsR7MmkZjU/OnooTqQX6qk9m9uAQnevMLPTgD+Y2WR3r45cyd0fAR6B4BBTHOqUWEhMgtHvDR6zvw073grD4s/w3L8Fj0ETg6CYcBkMn948kGJ9bfilH37x79vRyvROqD/Yeg1Nh85SsyAlMyI8spvbI5f1z2k+dNc/R/0o0ivEMiC2AAUR8/lhW7R1yswsCcgCKjzoGKkFcPclZrYeOAnoY50MghkMPTV4zLoLdm8MDkGt+TP843549YeQPjT4kt63I+g/iWZAHqQPCQ5TFZ4ZPKcPCR5pA4M9iqY+l6aO95qI6dpqqCxpnj+0r+26U7KaTxA4fNJA2O8TOd80nZAYhFvDofC5NuhjamhqO3Rk2xHr1kFSSsSJCXnN/Uup2RqFWI5bLANiMTDezEYTBME1wLUt1pkPXAe8BlwJ/N3d3cwGAZXu3mBmYwiOk5TEsFY5UeSMhDNvDh4HKoPBE999Njg7asx5R37xN02nDer6/9E31Aeh0XSG18HdEX07lRF9PRWwvxzK3wnmjxUsXc0SgpA4Ijhyjz7jLWsE5I6Nz0kBjQ3hiQs6pNfTxCwgwj6F24BnCU5zfdTdV5nZvUCxu88H/gf4pZmtAyoJQgTgXOBeM6sDGoGb3b0yVrXKCWpAbnDR4NRrjr1uV0tMav6yzenAdvW14ZlmlUeePOCNwV5AYkpw8sERzylBwB2eTj76ue5glJMQWp6YUAF7NgdnjB2oCPZEWsoYBnnjjn7kjOxcyNbuDa7l2V0KlRuap3dvCGpKSILc0ZA7pvmRNzZ4zhiuvaA40YVyIn2RezhQZAXs3xWcJl2xLhjevmJd8DhQ0by+JULOqIjQGNs8nTk8eL2925q/9FuGwYFdR75/anbwejmjgvBpbAjeu7Ik2L7hUPO6SamQ0xQeo5uDI3dMcMZdd4VHY0PwM+7ZHHxeezYFz1VlwbL0IZA+6Mg92LRwL7Z/To8NOQ3WJyJHMgvO1EpOC8b3GjHj6HUOVB4ZGE0BsuGVIzv4+w0IviAj90gsAbIKggCYeFnwnDu6ORT6t7Hb1dgA1VuCsGh6VITP618I7gLZJDEleL2MocHJAylZ4XNGeAJBZsRzVtDe1Nav/5GHtepqgi/7qk3NIVBVFk5vguqtzXecbDIgL/g5ExKDz2bfjuh7ZglJYVhEPoY0HwJNyWi+aBSPeG6rjea2/rkw/sLWP9PjpIAQkeiaDqEVnH5ke2NjMDx9ZGgkJIb/yw9DIKvg+A9JJSQGoZVdCGNmRX/vw8ER7nXsL4ddO8OTDKrh0N52vE9SEBQpGcEhuv07j1xuCcHhrewCKDgTsvKD6azC8Dk/CNhITSc77NsZvN4RZ86VB897t8O2FUHN3nB8n1FLI4piEhA6xCQivU9jYxASNdXNoVFbHfSFHD5bLWJZUkoQSFkF4Zd/QXDoLJanKzc2Bn1De7cHAWUJYAAWjjqQ0Dx9rLaklCCwjoMOMYlI35IQceFjT5WQEJxinTYw3pW0qmf2moiISNwpIEREJCoFhIiIRKWAEBGRqBQQIiISlQJCRESiUkCIiEhUCggREYmq11xJbWblwMZOvMRAYNcx14of1dc5qq9zVF/n9OT6Rrr7oGgLek1AdJaZFbd2uXlPoPo6R/V1jurrnJ5eX2t0iElERKJSQIiISFQKiGaPxLuAY1B9naP6Okf1dU5Pry8q9UGIiEhU2oMQEZGoFBAiIhJVnwoIM5ttZu+Y2TozuyvK8hQz+224/HUzG9WNtRWY2YtmttrMVpnZ56KsM8vMqsxsWfj49+6qL6KGUjNbGb7/Ubfws8CD4We4wsyi3Ow4ZrVNiPhslplZtZnd0WKdbv0MzexRM9tpZm9FtOWa2XNmtjZ8jnqDZjO7LlxnrZld1431fd/M1oT/fk+bWXYr27b5uxDD+u4xsy0R/4aXtrJtm3/vMazvtxG1lZrZsla2jfnn12nu3iceQCKwHhgDJAPLgUkt1rkV+Ek4fQ3w226sbxgwI5zOAN6NUt8s4M9x/hxLgYFtLL8UWEBw88Qzgdfj+O+9neAioLh9hsC5wAzgrYi27wF3hdN3Ad+Nsl0uUBI+54TTOd1U3/uBpHD6u9Hqa8/vQgzruwf4Yjv+/dv8e49VfS2W/xD493h9fp199KU9iJnAOncvcfdDwDxgTot15gC/CKd/B1xgZtYdxbn7NndfGk7vBd4GRnTHe3exOcDjHlgEZJvZsDjUcQGw3t07c3V9p7n7K0Bli+bI37NfAB+MsunFwHPuXunuu4HngNndUZ+7/83d68PZRcDx3ey4C7Ty+bVHe/7eO62t+sLvjo8Cv+nq9+0ufSkgRgCbI+bLOPoL+PA64R9IFZDXLdVFCA9tTQdej7L4LDNbbmYLzGxytxYWcOBvZrbEzG6Ksrw9n3N3uIbW/zDj/RkOcfdt4fR2YEiUdXrK53gDwR5hNMf6XYil28JDYI+2coiuJ3x+7wV2uPvaVpbH8/Nrl74UECcEM0sHngLucPfqFouXEhwymQr8CPhDd9cHvMfdZwCXAJ81s3PjUEObzCwZuAJ4MsrinvAZHubBsYYeea65mX0VqAf+r5VV4vW78N/AWGAasI3gME5PNJe29x56/N9SXwqILUBBxHx+2BZ1HTNLArKAim6pLnjPfgTh8H/u/vuWy9292t33hdN/AfqZ2cDuqi983y3h807gaYJd+Ujt+Zxj7RJgqbvvaLmgJ3yGwI6mw27h884o68T1czSz64HLgY+FIXaUdvwuxIS773D3BndvBH7WyvvG+/NLAj4M/La1deL1+XVEXwqIxcB4Mxsd/g/zGmB+i3XmA01ni1wJ/L21P46uFh6v/B/gbXe/r5V1hjb1iZjZTIJ/v+4MsDQzy2iaJujMfKvFavOBT4ZnM50JVEUcTukurf7PLd6fYSjy9+w64I9R1nkWeL+Z5YSHUN4ftsWcmc0G7gSucPcDrazTnt+FWNUX2af1oVbetz1/77F0IbDG3cuiLYzn59ch8e4l784HwRk27xKc3fDVsO1egj8EgFSCwxLrgDeAMd1Y23sIDjWsAJaFj0uBm4Gbw3VuA1YRnJGxCDi7mz+/MeF7Lw/raPoMI2s04KHwM14JFHVzjWkEX/hZEW1x+wwJgmobUEdwHPxGgn6tF4C1wPNAbrhuEfDziG1vCH8X1wGf6sb61hEcv2/6PWw6s2848Je2fhe6qb5fhr9bKwi+9Ie1rC+cP+rvvTvqC9sfa/qdi1i32z+/zj401IaIiETVlw4xiYhIByggREQkKgWEiIhEpYAQEZGoFBAiIhKVAkKkA8ysocWIsV02SqiZjYocFVQk3pLiXYDICeagu0+LdxEi3UF7ECJdIBzb/3vh+P5vmNm4sH2Umf09HFjuBTMrDNuHhPdaWB4+zg5fKtHMfmbBPUH+Zmb94/ZDSZ+ngBDpmP4tDjFdHbGsyt1PBX4M3B+2/Qj4hbtPIRj07sGw/UHgZQ8GDZxBcDUtwHjgIXefDOwBPhLjn0ekVbqSWqQDzGyfu6dHaS8F3ufuJeGgi9vdPc/MdhEMBVEXtm9z94FmVg7ku3ttxGuMIrgHxPhw/stAP3f/Zux/MpGjaQ9CpOt4K9MdURsx3YD6CSWOFBAiXefqiOfXwumFBCOJAnwMeDWcfgG4BcDMEs0sq7uKFGkv/e9EpGP6t7gJ/V/dvelU1xwzW0GwFzA3bPt/wP+a2ZeAcuBTYfvngEfM7EaCPYVbCEYFFekx1Ach0gXCPogid98V71pEuooOMYmISFTagxARkai0ByEiIlEpIEREJCoFhIiIRKWAEBGRqBQQIiIS1f8He2qBna+b7xEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU9b3/8dcn+wYJJICQsCq7gNiIe12o1qVKrbYFWyu3rVZb22rrba3dvN72ZxfttYvXVlurdBG99rb1ahWtxVZbFyICsm8iJBAgQBIgZP/8/jgnMAwTGCSTyfJ+Ph7zOMv3e2Y+M5mcz3y/37OYuyMiIhItJdkBiIhI16QEISIiMSlBiIhITEoQIiISkxKEiIjEpAQhIiIxKUFIr2dmI8zMzSwtjrqzzezlzohLJNmUIKRbMbMNZtZoZkVR698Md/IjkhPZQbHkmdkeM3sm2bGIHAslCOmO3gZmtS2Y2SQgJ3nhHOJKoAG4wMyO68wXjqcVJBIvJQjpjn4DfCJi+VpgTmQFM8s3szlmtt3M3jGzb5hZSliWamZ3m1mVma0HLo2x7a/MbIuZVZjZd8ws9Sjiuxb4ObAE+HjUc59lZv8ys2oz22Rms8P12WZ2TxhrjZm9HK4718zKo55jg5m9L5y/w8yeMLPfmlktMNvMppnZK+FrbDGzn5lZRsT2E83seTPbaWZbzex2MzvOzOrMrDCi3snh55d+FO9dehAlCOmOXgX6mtn4cMc9E/htVJ2fAvnAKOAcgoTyb2HZdcAHgKlAKXBV1LYPA83ACWGdC4FPxxOYmQ0HzgV+Fz4+EVX2TBjbAOAkYFFYfDfwHuAMoD/wFaA1ntcEZgBPAAXha7YAtwBFwOnAdOCzYQx9gL8CzwJDwvf4grtXAi8CH4l43muAue7eFGcc0sMoQUh31daKuABYAVS0FUQkja+5+2533wDcQ7DDg2AneK+7b3L3ncBdEdsOAi4Bbnb3ve6+Dfiv8PnicQ2wxN2XA3OBiWY2NSy7Gviruz/q7k3uvsPdF4Utm08CX3T3Cndvcfd/uXtDnK/5irv/yd1b3X2fu7/h7q+6e3P43n9BkCQhSIyV7n6Pu9eHn89rYdkjhC2e8DOcRfA5Sy+l/krprn4D/AMYSVT3EsEv53TgnYh17wDF4fwQYFNUWZvh4bZbzKxtXUpU/cP5BPAggLtXmNnfCbqc3gSGAutibFMEZLVTFo+DYjOzMcCPCFpHOQT/52+Exe3FAPBn4OdmNhIYC9S4++vvMibpAdSCkG7J3d8hGKy+BPjfqOIqoIlgZ99mGAdaGVsIdpSRZW02EQwwF7l7Qfjo6+4TjxSTmZ0BjAa+ZmaVZlYJnApcHQ4ebwKOj7FpFVDfTtleIgbgw1/2A6LqRF+S+X5gJTDa3fsCtwNt2W4TQbfbIdy9HnicoBVxDWo99HpKENKdfQo43933Rq509xaCHd13zaxP2Pf/JQ6MUzwOfMHMSsysH3BbxLZbgOeAe8ysr5mlmNnxZnYOR3Yt8DwwgWB84STgRCAbuJhgfOB9ZvYRM0szs0IzO8ndW4GHgB+Z2ZBwEP10M8sEVgNZZnZpOFj8DSDzCHH0AWqBPWY2DrgxouwpYLCZ3WxmmeHnc2pE+RxgNnA5ShC9nhKEdFvuvs7dy9op/jzBr+/1wMvA7wl2whB0Ac0DFgMLObQF8gkgA1gO7CIYAB58uFjMLItgbOOn7l4Z8XibYEd7rbtvJGjxfBnYSTBAPSV8iluBt4AFYdn3gRR3ryEYYP4lQQtoL3DQUU0x3Eow3rE7fK+PtRW4+26CcZvLgEpgDXBeRPk/CQbHF4atNOnFTDcMEpFIZvY34Pfu/stkxyLJpQQhIvuZ2SkE3WRDw9aG9GLqYhIRAMzsEYJzJG5WchBQC0JERNqhFoSIiMTUY06UKyoq8hEjRiQ7DBGRbuWNN96ocvfoc2uAHpQgRowYQVlZe0c8iohILGbW7uHM6mISEZGYlCBERCQmJQgREYmpx4xBxNLU1ER5eTn19fXJDiXhsrKyKCkpIT1d93YRkY7RoxNEeXk5ffr0YcSIEURcurnHcXd27NhBeXk5I0eOTHY4ItJD9Ogupvr6egoLC3t0cgAwMwoLC3tFS0lEOk+PThBAj08ObXrL+xSRztOju5hEpBO0toJZ8OgqWlugqQ4a66Bxz4H5pr3hNFzfNp+SCtn9Ibsf5PQ/eD49O9nvJmmUIBJox44dTJ8+HYDKykpSU1MZMCA4YfH1118nIyOj3W3LysqYM2cOP/nJTzolVpG47a6E8jIoXxBMNy+ElHQYPBkGTwkfJ0Hh8cGOt6M11UPVKqhcCluXwdalsGcrNO4NHk110NyB3a1p2RGJo1/UfP9gPiX9wOs27QsezfuCWA+aj6jTHC431YO3QmoGpKYF05R0SG17ZEBKuL5tXUr6wcv9RsKZX+i499z21jv8GWW/wsJCFi1aBMAdd9xBXl4et9566/7y5uZm0tJi/wlKS0spLS3tlDilC3GHliZoaYTWpnC+bbk5mLY0QkvzwXUy+0LfwdBncLDD6ChN9VC5JEwGYUKoCW+B3ZYUpl4TxLFlMbz+ILQ0BOXpuXDcJBhy0oHEUTQ22AnG+1nsrgwSwNalBxJC1WrwlqBOWjYMHA9FYyAjN3ik50RMc4I4MnIj5mOsa22Cfbugbifs2xkxvytYrtt1YL5q9YF6rc3tx5+WHbQ+0rMhLevg+byB4Xw2pGeBpQR/x8P9jZv2tf89GDxZCaInmD17NllZWbz55puceeaZzJw5ky9+8YvU19eTnZ3Nr3/9a8aOHcuLL77I3XffzVNPPcUdd9zBxo0bWb9+PRs3buTmm2/mC1/o+C+DHKXW1qCbomF38Gibb9wDDXvaX45e17gHmhsO/PMfEwt2Pn2HQN/icBo132dIsFOK5g7V70S0DhbAliXBDgogfyiUlMJpN0LJKXDc5EOfp6Up2IFuXhQkjC2LYeGc4JcyBDvHQScGyaItcQwYH/yCPqhV8FYwrdtx4Lnzh8KgiTDu0mB63CToP6pjWimpacEOu++Q+LdxD/52dWGi2J8IciAts2t1ub1LvSZB/Mf/LWP55toOfc4JQ/ry7cuOeC/7Q5SXl/Ovf/2L1NRUamtreemll0hLS+Ovf/0rt99+O3/4wx8O2WblypXMnz+f3bt3M3bsWG688cb4znlwh4ba4JfY7i2xp3u2QmYfKBgOBcOCf8SCYQce2QVH/R47VdO+oGvBWyMeHrUctY6I+daW4DkaaqG+FhpqwmntkafEcbl8S4XMPMjoE07zgmneoOBzz8gLdrT7uw3auhOiuxYil6O6I+proLYCajcfmO5cDxteCsqi5RQeSBx9BgffgfIFsHd7UJ6eA0NOhtM/FySF4tKghXIkqenBznvQRJj6sWBdawvsWHsgYWxeBG/9D5T9KtwmI6gT3SoYe0mQBNqeL7vfkV+/M5kFf7/MPsmOJGF6TYLoSj784Q+Tmhr86qmpqeHaa69lzZo1mBlNTU0xt7n00kvJzMwkMzOTgQMHsnXrVkqKi4NfLm2/Pht2w7yvhzv/Sti9OZi2/XqLlNkX+hwXPIpLg53djnWwbn4wkBddty1ZHJQ8hgZJJbtf5/5a2l0JG18NHpteDX7ltu1cOlJKWvDes/qG03zoNyJiuW+4g+h7YEfRtvPPyIvY+Wcn99dkw57gx0B0AmmbL18Q/A1PuCBIBiWnwMAJ8XcFHUlKKgwYGzwmfyRY19oKu96GLWFLIyUtaFkMOjFxYxdy1HpNgng3v/QTJTc3d//8N7/5Tc477zz++Mc/smHDBs4999xDN3AnMy0F9lVDcz2ptNK8dRWk7jx4x7hvFyz4VbjjHwxDpgbTtuW2ad6gYCcWi3vQZK7ZCNVtj03BdNc78PZL0Bh1s7GMvKCpP2Bs0Mc8YEww7T8K0tofiI9La2vQZbHxFdj0WjDdtSEoS8sOdmhn3RK8N7OgLzfyQfQ6i5qGdTJyIDP/wM4/s0/yd+wdJTMPMkdD0ehkR3JASkqQCAqPhxOvTHY00o5ekyC6qpqaGoqLi8Gdhx/6VbCD3lsFe7YHh+BtXQZ7KsFzgl9cEHSLWErwqy8tM+j3TE2H6gz4+pZj26mZQW5h8Bgy9dByd6ivPjhxVL8TdCFsfC3oOtj/XKkRiWPMgWnRmPYTVFM9bH4zaBm0tRLqq4Oy3AEw7DQ45ToYdnowMNeRA7IichAliGRoaQr6hZvr+coN13DtZ7/Ed779dS6dflYwIFizKSj31vBIjLyge6NobJgQMqH/yKCLJ1LbL+NEMjtwqN/gKYeWN+6FqjXBr/7tq4KBx+2rYfWzBw/A9i050NIoPD54zxtfDZJDS2NQp3A0jL8sSAbDTguSTU/4RS/STfSYe1KXlpZ69A2DVqxYwfjx45MTUNvhis31B46Bbg4f3nqgXkpaeORDFqRmHdwiOMqdYVLf75G0NMHOt8OEsSoigawJxjxS0oMWy7BTg4Qw9FTILUp21CI9npm94e4xj6lXC+Ldij4qprkhvkSQUxhM2x4dNRDY1aWmBy2GAWOCVkGb1tZgALWXn7Eq0hX1kr3TYbS2BEfFxDockvYOmXTaPbxxfyLoHyaB7N6VCI5WSgrkFyc7ChGJQXuttkHhQ46ACZdT0mMcARNjOTVTiUBEepSE7s3M7CLgx0Aq8Et3/15U+XDgIWAAsBP4uLuXh2U/AC4luOLs88AXPREDJqlpMCTGYKuISC+XsMt9m1kqcB9wMTABmGVmE6Kq3Q3McffJwJ3AXeG2ZwBnApOBE4FTgHMSFauIiBwqkfeDmAasdff17t4IzAVmRNWZAPwtnJ8fUe5AFpABZALpwNYExioiIlESmSCKgU0Ry+XhukiLgQ+F81cAfcys0N1fIUgYW8LHPHdfEf0CZna9mZWZWdn27ds7/A10hPPOO4958+YdtO7ee+/lxhtvjFn/3HPPJfpwXRGRZEj2HeVuBc4xszcJupAqgBYzOwEYD5QQJJXzzezs6I3d/QF3L3X30rb7LHQ1s2bNYu7cuQetmzt3LrNmzUpSRCIi8UlkgqgAIk/1LQnX7efum939Q+4+Ffh6uK6aoDXxqrvvcfc9wDPA6QmMNWGuuuoqnn76aRobg7ODN2zYwObNm3n00UcpLS1l4sSJfPvb305ylCIih0rkUUwLgNFmNpIgMcwEro6sYGZFwE53bwW+RnBEE8BG4DozuwswgtbFvccUzTO3QeVbx/QUhzhuElz8vcNW6d+/P9OmTeOZZ55hxowZzJ07l4985CPcfvvt9O/fn5aWFqZPn86SJUuYPHlyx8YnInIMEtaCcPdm4CZgHrACeNzdl5nZnWZ2eVjtXGCVma0GBgHfDdc/AawD3iIYp1js7v+XqFgTLbKbqa176fHHH+fkk09m6tSpLFu2jOXLlyc5ShGRgyX0PAh3/wvwl6h134qYf4IgGURv1wJ8pkODOcIv/USaMWMGt9xyCwsXLqSuro7+/ftz9913s2DBAvr168fs2bOpr+/Ae+iKiHSAZA9S9wp5eXmcd955fPKTn2TWrFnU1taSm5tLfn4+W7du5Zlnnkl2iCIih9B1ITrJrFmzuOKKK5g7dy7jxo1j6tSpjBs3jqFDh3LmmWcmOzwRkUMoQXSSD37wg0ReKeThhx+OWe/FF1/snIBERI5AXUwiIhKTEoSIiMTU4xNET7lj3pH0lvcpIp2nRyeIrKwsduzY0eN3nu7Ojh07yMrKSnYoItKD9OhB6pKSEsrLy+mqF/LrSFlZWZSUlCQ7DBHpQXp0gkhPT2fkyJHJDkNEpFvq0V1MIiLy7ilBiIhITEoQIiISkxKEiIjEpAQhIiIxKUGIiEhMShAiIhKTEoSIiMSkBCEiIjEpQYiISExKECIiEpMShIiIxJTQBGFmF5nZKjNba2a3xSgfbmYvmNkSM3vRzEoiyoaZ2XNmtsLMlpvZiETGKiIiB0tYgjCzVOA+4GJgAjDLzCZEVbsbmOPuk4E7gbsiyuYAP3T38cA0YFuiYhURkUMlsgUxDVjr7uvdvRGYC8yIqjMB+Fs4P7+tPEwkae7+PIC773H3ugTGKiIiURKZIIqBTRHL5eG6SIuBD4XzVwB9zKwQGANUm9n/mtmbZvbDsEVyEDO73szKzKysN9wUSESkMyV7kPpW4BwzexM4B6gAWghuZHR2WH4KMAqYHb2xuz/g7qXuXjpgwIBOC1pEpDdIZIKoAIZGLJeE6/Zz983u/iF3nwp8PVxXTdDaWBR2TzUDfwJOTmCsIiISJZEJYgEw2sxGmlkGMBN4MrKCmRWZWVsMXwMeiti2wMzamgXnA8sTGKuIiERJWIIIf/nfBMwDVgCPu/syM7vTzC4Pq50LrDKz1cAg4Lvhti0E3UsvmNlbgAEPJipWERE5lLl7smPoEKWlpV5WVpbsMEREuhUze8PdS2OVJXuQWkREuiglCBERiUkJQkREYlKCEBGRmJQgREQkJiUIERGJSQlCRERiUoIQEZGYlCBERCQmJQgREYlJCUJERGJSghARkZiUIEREJCYlCBERiUkJQkREYlKCEBGRmJQgREQkJiUIERGJSQlCRERiUoIQEZGYlCBERCSmhCYIM7vIzFaZ2Vozuy1G+XAze8HMlpjZi2ZWElXe18zKzexniYxTREQOlbAEYWapwH3AxcAEYJaZTYiqdjcwx90nA3cCd0WV/yfwj0TFKCIi7UtkC2IasNbd17t7IzAXmBFVZwLwt3B+fmS5mb0HGAQ8l8AYRUSkHYlMEMXApojl8nBdpMXAh8L5K4A+ZlZoZinAPcCth3sBM7vezMrMrGz79u0dFLaIiEDyB6lvBc4xszeBc4AKoAX4LPAXdy8/3Mbu/oC7l7p76YABAxIfrYhIL5KWwOeuAIZGLJeE6/Zz982ELQgzywOudPdqMzsdONvMPgvkARlmtsfdDxnoFhGRxEhkglgAjDazkQSJYSZwdWQFMysCdrp7K/A14CEAd/9YRJ3ZQKmSg4hI5zpiF5OZXRaOCRwVd28GbgLmASuAx919mZndaWaXh9XOBVaZ2WqCAenvHu3riIhIYpi7H76C2W+B04E/AA+5+8rOCOxolZaWellZWbLDEBHpVszsDXcvjVV2xJaBu38cmAqsAx42s1fCo4f6dHCcIiLShcTVdeTutcATBOcyDCY4JHWhmX0+gbGJiEgSxTMGcbmZ/RF4EUgHprn7xcAU4MuJDU9ERJIlnqOYrgT+y90PuuSFu9eZ2acSE5aIiCRbPAniDmBL24KZZQOD3H2Du7+QqMBERCS54hmD+B+gNWK5JVwnIiI9WDwJIi282B4A4XxG4kISEZGuIJ4upu1mdrm7PwlgZjOAqsSGJSLSs7g7O/c2UlG9j83V+yjftY/N1fVUVNdR39RKVnoK2empZGekkpkWTLPTU/evzwofbXWy0lP2L+dlpTGwT1aHxxxPgrgB+F140x4juELrJzo8EhGRbqyppZXKmnoqqvdRsStIAhURj83V+6hvaj1om5yMVIoLssnJSKW+qZX65hb2Nbawr6mF+qYWmloOfyJzmylDC/jz587s8Pd0xATh7uuA08KL6eHuezo8ChGRbqCusZkNVXVs2LGXt6v2sqFqL+/sqGPTrjq21tbTGrU/L8rLoLggm3HH9eH8sQMp7pfNkIJsisNHQU46Ztbu6zW3tFLf3Mq+xiBh1De1JY9W9jUFyaShuYW8zMRcVi+uZzWzS4GJQFbbm3H3OxMSkYhIEtU3tbBhR7Dzf7uqjg1Ve4PlHXvZWttwUN0BfTIZUZjD6ccXUlKQTXG/bIoLchhSkMWQgmyy0lOPKZa01BTyUlMSlgCO+PpHqmBmPwdygPOAXwJXAa8nOC4RkYTbubeR55ZVsri8Zn8i2FJTf1CdwtwMRhTlctYJAxhZlMOIolxGFOYyoig3aTvuzhLPuzvD3Seb2RJ3/w8zuwd4JtGBiYgkws69jcxbVsnTS7bwyvodtLQ6BTnpjCzK5fRRhUECKMplRGGQDPpmpSc75KSJJ0G0pdM6MxsC7CC4HpOISLcQKymMKMzhhnNGccmkwUwY3PewYwG9VTwJ4v/MrAD4IbAQcODBhEYlInKMlBSO3WETRHijoBfcvRr4g5k9BWS5e02nRCci3ZK7U9/Uyq66Rqrrmqje10hNXRO7Iubb1lfXNZGemsLAvpkM7JPFoL6ZDOobTAf2yWJAn8y4B3tjJYWRRblKCu/SYROEu7ea2X0E94PA3RuAhsNtIyK9x449Dcx55R1WbKmlel8T1fsTQhONza3tbpeRlkK/nHQKsjPIz0lnb2Mzr63fy7bd9TGP/S/ISWdgnyBxtCWR/ct9M1m9dY+SQgLE08X0gpldCfyvH+n2cyLSK2yp2ccD/1jPo69vpKG5ldED8+iXk8HIolwKsjMoyA12/gU56RRkp1OQE87npNMvJ6PdFkFrq7OrrpFtuxvYWlvPttoGtu2uZ2ttuLy7gXXbqti2u4HmqJMOlBQ6XjwJ4jPAl4BmM6snOJva3b1vQiMTkS5nQ9Vefv73dfxhYTmtDh88qZgbzz2eEwbmdcjzp6QYhXmZFOZlMn5w+7uY1lZnZ10j22ob2Lq7nkF9shg/uI+SQgeL50xq3VpUpJdbWVnLf89fx1NLNpOWmsLMU4Zx/XtHMbR/TlLiSUkxivIyKcrLZAL6rZoo8Zwo995Y66NvICQiPc+iTdX87G9r+euKreRmpHLd2aP41NkjE3JhOOl64uli+veI+SxgGvAGcP6RNjSzi4AfA6nAL939e1Hlw4GHgAHATuDj7l5uZicB9wN9Ce4/8V13fyyOWEXkGLk7r6zfwX/PX8fLa6vIz07n5veNZvYZIyjI0ZX+e5N4upgui1w2s6HAvUfazsxSgfuAC4ByYIGZPenuyyOq3Q3McfdHzOx84C7gGqAO+IS7rwlPznvDzOaFh9uKSAK4O39buY375q9l4cZqBvTJ5PZLxnH1qcN7/CUlJLZ381cvB8bHUW8asNbd1wOY2VxgBhCZICYQDIADzAf+BODuq9squPtmM9tG0MpQghDpYC2tzjNLt3Df/HWs2FJLcUE2//nBE/nwe0qO+WJz0r3FMwbxU4KzpyG4A91JBGdUH0kxwb0j2pQDp0bVWQx8iKAb6gqgj5kVuvuOiNefRnAHu3UxYrseuB5g2LBhcYQk0rPsaWhmaUUN23c37L+PQF3E/QTqGpvZ19jKvqbm/eWR9eqbWtjT0Ex9UyvHD8jlng9P4fKThpCeGs/NJqWni6cFURYx3ww86u7/7KDXvxX4mZnNBv4BVBCMOQBgZoOB3wDXuvshZ924+wPAAwClpaU6R0N6tKaWVlZV7mZxeTWLN1WzaFM1a7btob2zk9ruPBY9LcjJYEjb3crCaenwflw48ThSU3SYqBwQT4J4Aqh39xYIxhbMLMfd646wXQUwNGK5JFy3n7tvJmhBEN6Q6Mq2cQYz6ws8DXzd3V+N582I9BTuTvmufSwKE8HiTdUs3Vyz/45k/XLSOWloAZdMGsyUoQUM7RfceyAnI43s9FQy01JI0c5ejlFcZ1ID7wPa7iSXDTwHnHGE7RYAo81sJEFimAlcHVnBzIqAnWHr4GsERzRhZhnAHwkGsJ+I762IdF/VdY0sLq9h0cbq/S2EHXsbAchMS+HE4nyunjacKUPzmTq0H0P7Z+ukMEm4eBJEVuRtRt19j5kd8ewYd282s5uAeQSHuT7k7svM7E6gzN2fBM4F7jIzJ+hi+ly4+UeA9wKFYfcTwGx3XxTn+xLpsnbtbWTZ5lqWbq7hrYoallXUsGFH0CA3g+MH5HHeuIFMGVrA1KEFjD2uj8YEJCnsSJdXMrN/Ap9394Xh8nuAn7n76Z0QX9xKS0u9rKzsyBVFOlHVnob9SWBpRZAUynft219eXJDNpOJ8JpXkc9LQAiaV5PfqG9RI5zOzN9y9NFZZPC2Im4H/MbPNBNdhOg74aAfGJ9LtuTtbaxtYWlHD0s01wbSilsraA7evHFGYw5ShBXzs1OFMKs5n4pC+9MvViWfSdcVzotwCMxsHjA1XrXL3psSGJdK1ba2tZ0l5DW+VV7MkTAZVe4Ir4ZvBqKJcThvVnxOL85k4JJ+JxX3VMpBuJ57zID4H/M7dl4bL/cxslrv/d8KjE+kCtu9u4K2K6jAhBOMG23YHySDF4ISBeZwzZgAnFvdlUnE+4wf3JVdnHksPEM+3+Dp3v69twd13mdl1gBKE9Dg7wjGDt8prwpZBDVtqgm6itgHks04oYlJJPpNLgmSQk6FkID1TPN/sVDOztpsFhddYUsepdHutrc6KylpeXlPFmxureauihorqAwPIo4pymTayP5OK85lcUsCEIX11TSLpVeL5tj8LPGZmvwiXPwM8k7iQRBJnW209L62p4qU123l5bRVVe4JzDYYX5jB1WAHXnjGcScUFGjMQIb4E8VWC6x3dEC4vITiSSaTLq29q4fW3d/LSmu28tKaKlZW7ASjMzeDs0UWcPXoAZ40uYlBf3d9AJFo8RzG1mtlrwPEEJ7AVAX9IdGAi74a7s7Jy9/6E8NrbO2lsbiUjNYXSEf346kXjOHt0ERMG99WlKESOoN0EYWZjgFnhowp4DMDdz+uc0ETiU7WngZfXVPGP1dt5aW0V28MjjEYPzOPjpw7n7DFFnDqyvwaTRY7S4f5jVgIvAR9w97UAZnZLp0QlcgSbdtYxb1kl85ZVUvbOLtyDC9idNXpA2HVUxOD87GSHKdKtHS5BfIjgAnvzzexZYC7BmdQinc7dWbNtD/OWVvLsskqWba4FYNxxffjC+aOZPn4gJw7JV7eRSAdqN0G4+5+AP5lZLsGd4G4GBprZ/cAf3f25TopReil3Z3F5Dc8ureS5ZZWsr9oLwMnDCrj9knG8f+JxDC/MTXKUIj1XPIPUe4HfA783s37AhwmObFKCkA7X3NLK6xt2Mm9pJc8t38qWmnrSUozTRhXyb2eN5MIJg3TEkUgnOapRO3ffRXAHtwcSE470RvVNLfxzbRXPLq3kryu2squuicy0FM4ZM4BbLxzL9PEDKcjRuZkinU2HdUjSNDS38JtX3uG++baobrkAABLtSURBVGvZVddEn8w0po8fyPsnHsc5YwfoqCORJNN/oHS6llbnz4squOe51VRU7+Ps0UV86qyRnHF8ERlpujGOSFehBCGdxt15cdV2vv/sSlZW7mZScT7fv3IyZ40uSnZoIhKDEoR0ijc37uJ7z6zktbd3Mrwwh5/OmsqlkwbrsFSRLkwJQhJq3fY93D1vFc8sraQoL4P/nDGRj54yTF1JIt2AEoQkxLbaeu59YQ2PLdhEVloKt7xvDJ8+e6RupCPSjST0v9XMLgJ+DKQCv3T370WVDwceAgYAO4GPu3t5WHYt8I2w6nfc/ZFExiodo7a+iV/8fR2/evltWlqda04bzk3nn0BRXmayQxORo5SwBBHeWOg+4AKgHFhgZk+6+/KIancDc9z9ETM7H7gLuMbM+gPfBkoBB94It92VqHjl2EQfsnr5lCF8+cIxOtNZpBtLZAtiGrDW3dcDmNlcgkt2RCaICcCXwvn5wJ/C+fcDz7v7znDb54GLgEcTGK+8C00trfx50Wb+6/kDh6x+9aJxnFicn+zQROQYJTJBFAObIpbLgVOj6iwmuCjgj4ErgD5mVtjOtsWJC1WO1r7GFh5bsJEHX3qbiup9nFjcV4esivQwyR4xvBX4mZnNBv4BVAAt8W5sZtcT3O2OYcOGJSI+iVJd18icV97h4X9tYOfeRkqH9+M/Lp/I+eMG6pBVkR4mkQmiAhgasVwSrtvP3TcTtCAwszzgSnevNrMK4NyobV+MfgF3339dqNLSUu/A2CXK5up9/Orlt3n09Y3UNbYwfdxAbjj3eE4Z0T/ZoYlIgiQyQSwARpvZSILEMBO4OrKCmRUBO929FfgawRFNAPOA/xdePRbgwrBcOtnabbv5+d/X86c3K3Dg8ilD+Mw5oxh3XN9khyYiCZawBOHuzWZ2E8HOPhV4yN2XmdmdQJm7P0nQSrjLzJygi+lz4bY7zew/CZIMwJ1tA9bSORZu3MX9L67j+eVbyUpP4eOnDedTZ41kaP+cZIcmIp3E3HtGz0xpaamXlZUlO4xuzd15cfV27n9xHa+/vZP87HSuPWME154+nEKdxyDSI5nZG+5eGqss2YPU0gU0t7Ty9FtbuP/Fdays3M3g/Cy++YEJzDxlqM58FunF9N/fC+1paOat8hoWl1ezeFM1Ze/sYvvuBo4fkMsPr5rMjJOKda0kEVGC6OmaWlpZVbmbRZuCZLC4vJo12/bQ1rM4rH8Op47sz2VThnDB+EE6VFVE9lOC6EHcnXd21LG4vHp/Qli2uZaG5lYA+udmMKUkn0smDWbK0AKmlBTQP1e38hSR2JQgurmmllZ+/c+3eXntDhZvqqZmXxMAWekpTCrO55rThjNlaAEnDS2gpF82ZmohiEh8lCC6sV17G/nc7xfyr3U7GDuoDxdNPG5/MhgzKI+0VI0jiMi7pwTRTa2q3M11c8qorKnn7g9P4ar3lCQ7JBHpYZQguqHnllVyy2OLyMlMY+5nTuPkYf2OvJGIyFFSguhG3J375q/l7udWM7kknweuKeW4/KxkhyUiPZQSRDdR19jMvz+xhKeXbOGDJw3he1dOJis9NdlhiUgPpgTRDVRU7+O6R8pYUVnLbReP4zPvHaWjkUQk4ZQgurgFG3Zyw2/eoLG5lYeuPYXzxg1Mdkgi0ksoQXRhj76+kW/9eSkl/XJ48BOlnDAwL9khiUgvogTRBTW1tPKdp5bzyCvv8N4xA/jpzKnk56QnOywR6WWUILqYXXsb+ezvFvLK+h1cd/ZIvnrROJ3wJiJJoQTRhayq3M2n5yxga00D93x4Clfq5DcRSSIliC5CJ7+JSFejBJFkra3Oz+av5UfP6+Q3EelalCCSaPvuBr70+CJeWlOlk99EpMtRgkiSl9dUcfNji9hd38R3rziRq6cN08lvItKlKEF0sqaWVv7r+dXc//d1HD8gj99+ehrjjuub7LBERA6R0OMnzewiM1tlZmvN7LYY5cPMbL6ZvWlmS8zsknB9upk9YmZvmdkKM/taIuPsLOW76vjoL17hv19cx0feM5QnbzpTyUFEuqyEtSDMLBW4D7gAKAcWmNmT7r48oto3gMfd/X4zmwD8BRgBfBjIdPdJZpYDLDezR919Q6LiTbRnl27hK08sodXhxzNPYsZJxckOSUTksBLZxTQNWOvu6wHMbC4wA4hMEA60/YTOBzZHrM81szQgG2gEahMYa8LUN7Xw3adX8JtX32FyST4/nTWV4YW5yQ5LROSIEpkgioFNEcvlwKlRde4AnjOzzwO5wPvC9U8QJJMtQA5wi7vvTGCsCbF22x5u+v1CVlbu5rqzR/Lv7x9HRprOihaR7iHZg9SzgIfd/R4zOx34jZmdSND6aAGGAP2Al8zsr22tkTZmdj1wPcCwYcM6N/LDcHeeeKOcb/15GdkZqfx6tq7CKiLdTyITRAUwNGK5JFwX6VPARQDu/oqZZQFFwNXAs+7eBGwzs38CpcBBCcLdHwAeACgtLfVEvImjtaehmW/88S3+tGgzp43qz70fnaoT30SkW0pkf8cCYLSZjTSzDGAm8GRUnY3AdAAzGw9kAdvD9eeH63OB04CVCYy1Q7xVXsMHfvISTy7ezJcuGMPvPn2akoOIdFsJa0G4e7OZ3QTMA1KBh9x9mZndCZS5+5PAl4EHzewWgoHp2e7uZnYf8GszWwYY8Gt3X5KoWI+Vu/PQPzfwvWdWUJSXydzrT2fayP7JDktE5JiYe5fomTlmpaWlXlZW1umv29zSymd/t5Dnlm/lfeMH8cOrJtMvN6PT4xAReTfM7A13L41VluxB6m7v0QWbeG75Vr560ThuOEf3ihaRnkPHXB6DmromfvTcKk4b1V/JQUR6HCWIY3DvC6up2dfEtz4wUclBRHocJYh3ae223cx55R1mThvGhCG6npKI9DxKEO+Cu3PnUyvIyUjlyxeMSXY4IiIJoQTxLsxftY1/rN7OF6ePpjAvM9nhiIgkhBLEUWpsbuU7T61gVFEunzh9RLLDERFJGCWIozTnlQ2sr9rLNz8wQRfeE5EeTXu4o7BjTwM/fmEN54wZoIvviUiPpwRxFO55fjX7Glv45gfGJzsUEZGEU4KI0/LNtcx9fSPXnD6cEwb2SXY4IiIJpwQRh+Cw1mXkZ6dz83Qd1ioivYMSRBzmLavk1fU7+dKFY8nPSU92OCIinUIJ4gjqm1r4ztMrGDuoD7NOGXrkDUREeggliCP41ctvU75rH9+6bAJpqfq4RKT30B7vMLbW1nPf/LVcOGEQZ55QlOxwREQ6lRLEYfzg2VU0tzhfv1SHtYpI76ME0Y5Fm6r5w8JyPnnWSIYX5iY7HBGRTqcEEYO7c+f/LaMoL5Obzj8h2eGIiCSFEkQMTy7ezMKN1XzlorHkZequrCLSOylBRKlrbOauv6xkUnE+V51ckuxwRESSJqEJwswuMrNVZrbWzG6LUT7MzOab2ZtmtsTMLokom2xmr5jZMjN7y8yyEhlrm5//fT2VtfV867IJpKToNqIi0nslrP/EzFKB+4ALgHJggZk96e7LI6p9A3jc3e83swnAX4ARZpYG/Ba4xt0Xm1kh0JSoWNuU76rjF39fx2VThnDKiP6JfjkRkS4tkS2IacBad1/v7o3AXGBGVB0H2m7onA9sDucvBJa4+2IAd9/h7i0JjBWA7z2zEjO47eJxiX4pEZEuL5EJohjYFLFcHq6LdAfwcTMrJ2g9fD5cPwZwM5tnZgvN7CuxXsDMrjezMjMr2759+zEF+/rbO3lqyRY+897jKS7IPqbnEhHpCZI9SD0LeNjdS4BLgN+YWQpB19dZwMfC6RVmNj16Y3d/wN1L3b10wIAB7zqI1tbgaq2D87O44Zzj3/XziIj0JIlMEBVA5NXtSsJ1kT4FPA7g7q8AWUARQWvjH+5e5e51BK2LkxMV6BNvlLO0opbbLh5HdkZqol5GRKRbSWSCWACMNrORZpYBzASejKqzEZgOYGbjCRLEdmAeMMnMcsIB63OA5STA7vomfjBvJe8Z3o/LpwxJxEuIiHRLCTuKyd2bzewmgp19KvCQuy8zszuBMnd/Evgy8KCZ3UIwYD3b3R3YZWY/IkgyDvzF3Z9ORJz7mlo4eVg/PnfeCZjpsFYRkTYW7I+7v9LSUi8rK0t2GCIi3YqZveHupbHKkj1ILSIiXZQShIiIxKQEISIiMSlBiIhITEoQIiISkxKEiIjEpAQhIiIxKUGIiEhMPeZEOTPbDrxzDE9RBFR1UDiJoPiOjeI7Norv2HTl+Ia7e8yrnfaYBHGszKysvbMJuwLFd2wU37FRfMemq8fXHnUxiYhITEoQIiISkxLEAQ8kO4AjUHzHRvEdG8V3bLp6fDFpDEJERGJSC0JERGJSghARkZh6VYIws4vMbJWZrTWz22KUZ5rZY2H5a2Y2ohNjG2pm881suZktM7MvxqhzrpnVmNmi8PGtzoovIoYNZvZW+PqH3KHJAj8JP8MlZpawe4nHeO2xEZ/NIjOrNbObo+p06mdoZg+Z2TYzWxqxrr+ZPW9ma8Jpv3a2vTass8bMru3E+H5oZivDv98fzaygnW0P+11IYHx3mFlFxN/wkna2Pez/ewLjeywitg1mtqidbRP++R0zd+8VD4Lbnq4DRgEZwGJgQlSdzwI/D+dnAo91YnyDgZPD+T7A6hjxnQs8leTPcQNQdJjyS4BnAANOA15L4t+7kuAkoKR9hsB7gZOBpRHrfgDcFs7fBnw/xnb9gfXhtF8436+T4rsQSAvnvx8rvni+CwmM7w7g1jj+/of9f09UfFHl9wDfStbnd6yP3tSCmAasdff17t4IzAVmRNWZATwSzj8BTLdOulG1u29x94Xh/G5gBVDcGa/dwWYAczzwKlBgZoOTEMd0YJ27H8vZ9cfM3f8B7IxaHfk9ewT4YIxN3w887+473X0X8DxwUWfE5+7PuXtzuPgqUNLRrxuvdj6/eMTz/37MDhdfuO/4CPBoR79uZ+lNCaIY2BSxXM6hO+D9dcJ/kBqgsFOiixB2bU0FXotRfLqZLTazZ8xsYqcGFnDgOTN7w8yuj1Eez+fcGWbS/j9msj/DQe6+JZyvBAbFqNNVPsdPErQIYznSdyGRbgq7wB5qp4uuK3x+ZwNb3X1NO+XJ/Pzi0psSRLdgZnnAH4Cb3b02qnghQZfJFOCnwJ86Oz7gLHc/GbgY+JyZvTcJMRyWmWUAlwP/E6O4K3yG+3nQ19AljzU3s68DzcDv2qmSrO/C/cDxwEnAFoJunK5oFodvPXT5/6XelCAqgKERyyXhuph1zCwNyAd2dEp0wWumEySH37n7/0aXu3utu+8J5/8CpJtZUWfFF75uRTjdBvyRoCkfKZ7POdEuBha6+9bogq7wGQJb27rdwum2GHWS+jma2WzgA8DHwiR2iDi+Cwnh7lvdvcXdW4EH23ndZH9+acCHgMfaq5Osz+9o9KYEsQAYbWYjw1+YM4Eno+o8CbQdLXIV8Lf2/jk6Wthf+Stghbv/qJ06x7WNiZjZNIK/X2cmsFwz69M2TzCYuTSq2pPAJ8KjmU4DaiK6UzpLu7/ckv0ZhiK/Z9cCf45RZx5woZn1C7tQLgzXJZyZXQR8Bbjc3evaqRPPdyFR8UWOaV3RzuvG8/+eSO8DVrp7eazCZH5+RyXZo+Sd+SA4wmY1wdENXw/X3UnwjwCQRdAtsRZ4HRjVibGdRdDVsARYFD4uAW4Abgjr3AQsIzgi41XgjE7+/EaFr704jKPtM4yM0YD7ws/4LaC0k2PMJdjh50esS9pnSJCotgBNBP3gnyIY13oBWAP8Fegf1i0Ffhmx7SfD7+Ja4N86Mb61BP33bd/DtiP7hgB/Odx3oZPi+0343VpCsNMfHB1fuHzI/3tnxBeuf7jtOxdRt9M/v2N96FIbIiISU2/qYhIRkaOgBCEiIjEpQYiISExKECIiEpMShIiIxKQEIXIUzKwl6oqxHXaVUDMbEXlVUJFkS0t2ACLdzD53PynZQYh0BrUgRDpAeG3/H4TX93/dzE4I148ws7+FF5Z7wcyGhesHhfdaWBw+zgifKtXMHrTgniDPmVl20t6U9HpKECJHJzuqi+mjEWU17j4J+Blwb7jup8Aj7j6Z4KJ3PwnX/wT4uwcXDTyZ4GxagNHAfe4+EagGrkzw+xFpl86kFjkKZrbH3fNirN8AnO/u68OLLla6e6GZVRFcCqIpXL/F3YvMbDtQ4u4NEc8xguAeEKPD5a8C6e7+ncS/M5FDqQUh0nG8nfmj0RAx34LGCSWJlCBEOs5HI6avhPP/IriSKMDHgJfC+ReAGwHMLNXM8jsrSJF46deJyNHJjroJ/bPu3naoaz8zW0LQCpgVrvs88Gsz+3dgO/Bv4fovAg+Y2acIWgo3ElwVVKTL0BiESAcIxyBK3b0q2bGIdBR1MYmISExqQYiISExqQYiISExKECIiEpMShIiIxKQEISIiMSlBiIhITP8fSVWjgoSiIQIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovYkSmQOehnu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_a = np.stack(test_lp['t_answer'], axis=0)\n",
        "test_y = np.stack(test_lp['cat_level'], axis=0)\n",
        "test_a_b = np.stack(test_lp_b['t_answer'], axis=0)\n",
        "test_y_b = np.stack(test_lp_b['cat_level'], axis=0)\n",
        "df_confusion = confusion_matrix(model, test_a_b, test_y_b)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qirbx_OieiVc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "771af7f3-4834-4a10-8d53-2a8fc58375ce"
      },
      "source": [
        "df_confusion"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>col_0</th>\n",
              "      <th>none</th>\n",
              "      <th>mild</th>\n",
              "      <th>moderate</th>\n",
              "      <th>moderately severe</th>\n",
              "      <th>severe</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>row_0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>none</th>\n",
              "      <td>3524</td>\n",
              "      <td>5</td>\n",
              "      <td>11</td>\n",
              "      <td>11</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mild</th>\n",
              "      <td>24</td>\n",
              "      <td>2903</td>\n",
              "      <td>8</td>\n",
              "      <td>15</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>moderate</th>\n",
              "      <td>13</td>\n",
              "      <td>5</td>\n",
              "      <td>2916</td>\n",
              "      <td>15</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>moderately severe</th>\n",
              "      <td>12</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>2782</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>severe</th>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>8</td>\n",
              "      <td>2367</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "col_0              none  mild  moderate  moderately severe  severe\n",
              "row_0                                                             \n",
              "none               3524     5        11                 11       5\n",
              "mild                 24  2903         8                 15       9\n",
              "moderate             13     5      2916                 15       8\n",
              "moderately severe    12     3         3               2782       9\n",
              "severe                7     2         5                  8    2367"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9xHC6drfempp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "b6726358-e08f-472b-f49e-9af4983459fc"
      },
      "source": [
        "score = model.evaluate(test_a_b, test_y_b, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.03850240387929059\n",
            "Test accuracy: 0.9878664016723633\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X3RRX6MCep5k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.save('/content/drive/My Drive/Depression_detect/model_glove_lstm_b.h5')\n",
        "\n",
        "with open('/content/drive/My Drive/Depression_detect/model_glove_lstm_b_hist.json', 'w') as f:\n",
        "    json.dump(str(model_glove_lstm_hist.history), f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ANXUOoswbnvH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def test_model(text,model):\n",
        "    print(text)\n",
        "    word_list = text_to_wordlist(text)\n",
        "    sequences = tokenizer.texts_to_sequences([word_list])\n",
        "    sequences_input = list(itertools.chain(*sequences))\n",
        "    sequences_input =  pad_sequences([sequences_input], value=0, padding=\"post\", maxlen=windows_size).tolist()\n",
        "    print(sequences_input)\n",
        "    input_a = np.asarray(sequences_input)\n",
        "    pred = model.predict(input_a, batch_size=None, verbose=0, steps=None)\n",
        "    predicted_class = np.argmax(pred)\n",
        "    print(labels[predicted_class])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AIJ0YybjZS2U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        },
        "outputId": "864ebf4d-f77b-4859-e7a5-d46daa0b7316"
      },
      "source": [
        "text= \"I want an ice cream and have some fries for lunch\"\n",
        "test_model(text, model)\n",
        "sen = \"I'm afraid of losing my work, I don't have any money\"\n",
        "test_model(sen, model)\n",
        "sen = \"I'm worried about my future, I'm afraid of it\"\n",
        "test_model(sen, model)\n",
        "sen = \"I am a graduate student\"\n",
        "test_model(sen, model)\n",
        "sen = \"I am getting married\"\n",
        "test_model(sen, model)\n",
        "sen = \"This party is great, I know lots of people\"\n",
        "test_model(sen, model)\n",
        "sen = \"I miss my parents, brothers and sisters\"\n",
        "test_model(sen, model)\n",
        "sen = \"I detest my horrible job\"\n",
        "test_model(sen, model)\n",
        "sen = \"I cannot handle this anymore\"\n",
        "test_model(sen, model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I want an ice cream and have some fries for lunch\n",
            "[[193, 2438, 2825, 6594, 4186, 0, 0, 0, 0, 0]]\n",
            "severe\n",
            "I'm afraid of losing my work, I don't have any money\n",
            "[[1, 18, 780, 908, 108, 211, 0, 0, 0, 0]]\n",
            "none\n",
            "I'm worried about my future, I'm afraid of it\n",
            "[[1, 18, 1070, 566, 1, 18, 780, 0, 0, 0]]\n",
            "mild\n",
            "I am a graduate student\n",
            "[[833, 967, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
            "none\n",
            "I am getting married\n",
            "[[147, 440, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
            "none\n",
            "This party is great, I know lots of people\n",
            "[[563, 150, 14, 51, 39, 0, 0, 0, 0, 0]]\n",
            "none\n",
            "I miss my parents, brothers and sisters\n",
            "[[602, 3832, 294, 291, 0, 0, 0, 0, 0, 0]]\n",
            "none\n",
            "I detest my horrible job\n",
            "[[860, 123, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
            "none\n",
            "I cannot handle this anymore\n",
            "[[127, 537, 369, 0, 0, 0, 0, 0, 0, 0]]\n",
            "none\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4r0xI6S-gtjM",
        "colab_type": "text"
      },
      "source": [
        "## MODEL 2  word2vec + 2LSTMs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "agmqozMZbG7e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        },
        "outputId": "e3c65a86-2cd7-47bf-d098-91080e42bbd3"
      },
      "source": [
        "answer_inp = Input(shape=(windows_size, ))\n",
        "embedding_size_glove = 100\n",
        "answer_emb1 = Embedding(vocab_size_stop+1, embedding_size_glove, weights=[embedding_matrix_lp], input_length=windows_size, trainable=False)(answer_inp)\n",
        "\n",
        "\n",
        "lstm1 = LSTM(embedding_size_glove, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(answer_emb1)\n",
        "lstm2 = LSTM(embedding_size_glove, dropout=0.2, recurrent_dropout=0.2)(lstm1)\n",
        "\n",
        "X = Dropout(0.2)(lstm2)\n",
        "bt = BatchNormalization()(X)\n",
        "dense1 = Dense(units=256, activation=\"relu\")(bt)\n",
        "\n",
        "out = Dense(5,  activation='softmax')(dense1)\n",
        "\n",
        "model_2lstm = Model(inputs=[answer_inp], outputs=[out])\n",
        "model_2lstm.compile(optimizer=\"adam\", loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "model_2lstm.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_2 (InputLayer)         (None, 10)                0         \n",
            "_________________________________________________________________\n",
            "embedding_2 (Embedding)      (None, 10, 100)           745000    \n",
            "_________________________________________________________________\n",
            "lstm_2 (LSTM)                (None, 10, 100)           80400     \n",
            "_________________________________________________________________\n",
            "lstm_3 (LSTM)                (None, 100)               80400     \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 100)               0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_2 (Batch (None, 100)               400       \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 256)               25856     \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 5)                 1285      \n",
            "=================================================================\n",
            "Total params: 933,341\n",
            "Trainable params: 188,141\n",
            "Non-trainable params: 745,200\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xBE7NnA9hS2D",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "78b10d51-f928-4ef9-9235-e5c3f9b5d9a8"
      },
      "source": [
        "model_glove_2lstm_b_hist = model_2lstm.fit(train_a_b, train_y_b, validation_data=(dev_a_b, dev_y_b), epochs=30, batch_size=64, shuffle=True, callbacks=[early_stopping])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 64179 samples, validate on 12836 samples\n",
            "Epoch 1/30\n",
            "64179/64179 [==============================] - 36s 558us/step - loss: 1.6247 - accuracy: 0.2333 - val_loss: 1.5864 - val_accuracy: 0.2562\n",
            "Epoch 2/30\n",
            "64179/64179 [==============================] - 35s 546us/step - loss: 1.5833 - accuracy: 0.2651 - val_loss: 1.5531 - val_accuracy: 0.2901\n",
            "Epoch 3/30\n",
            "64179/64179 [==============================] - 35s 551us/step - loss: 1.5498 - accuracy: 0.2982 - val_loss: 1.5092 - val_accuracy: 0.3378\n",
            "Epoch 4/30\n",
            "64179/64179 [==============================] - 39s 605us/step - loss: 1.4980 - accuracy: 0.3374 - val_loss: 1.4387 - val_accuracy: 0.3942\n",
            "Epoch 5/30\n",
            "64179/64179 [==============================] - 35s 551us/step - loss: 1.4238 - accuracy: 0.3877 - val_loss: 1.3620 - val_accuracy: 0.4145\n",
            "Epoch 6/30\n",
            "64179/64179 [==============================] - 35s 550us/step - loss: 1.3562 - accuracy: 0.4299 - val_loss: 1.2502 - val_accuracy: 0.4956\n",
            "Epoch 7/30\n",
            "64179/64179 [==============================] - 35s 549us/step - loss: 1.2963 - accuracy: 0.4624 - val_loss: 1.1602 - val_accuracy: 0.5457\n",
            "Epoch 8/30\n",
            "64179/64179 [==============================] - 35s 550us/step - loss: 1.2385 - accuracy: 0.4898 - val_loss: 1.0850 - val_accuracy: 0.5726\n",
            "Epoch 9/30\n",
            "64179/64179 [==============================] - 35s 550us/step - loss: 1.1927 - accuracy: 0.5149 - val_loss: 1.0433 - val_accuracy: 0.6030\n",
            "Epoch 10/30\n",
            "64179/64179 [==============================] - 35s 549us/step - loss: 1.1544 - accuracy: 0.5333 - val_loss: 1.0116 - val_accuracy: 0.6221\n",
            "Epoch 11/30\n",
            "64179/64179 [==============================] - 35s 546us/step - loss: 1.1203 - accuracy: 0.5474 - val_loss: 0.9549 - val_accuracy: 0.6391\n",
            "Epoch 12/30\n",
            "64179/64179 [==============================] - 35s 549us/step - loss: 1.0877 - accuracy: 0.5630 - val_loss: 0.9120 - val_accuracy: 0.6628\n",
            "Epoch 13/30\n",
            "64179/64179 [==============================] - 36s 559us/step - loss: 1.0621 - accuracy: 0.5767 - val_loss: 0.8821 - val_accuracy: 0.6733\n",
            "Epoch 14/30\n",
            "64179/64179 [==============================] - 36s 556us/step - loss: 1.0403 - accuracy: 0.5866 - val_loss: 0.8273 - val_accuracy: 0.6994\n",
            "Epoch 15/30\n",
            "64179/64179 [==============================] - 35s 553us/step - loss: 1.0172 - accuracy: 0.5971 - val_loss: 0.8030 - val_accuracy: 0.7151\n",
            "Epoch 16/30\n",
            "64179/64179 [==============================] - 36s 554us/step - loss: 0.9976 - accuracy: 0.6066 - val_loss: 0.7708 - val_accuracy: 0.7296\n",
            "Epoch 17/30\n",
            "64179/64179 [==============================] - 35s 553us/step - loss: 0.9758 - accuracy: 0.6169 - val_loss: 0.7340 - val_accuracy: 0.7452\n",
            "Epoch 18/30\n",
            "64179/64179 [==============================] - 36s 557us/step - loss: 0.9567 - accuracy: 0.6241 - val_loss: 0.7093 - val_accuracy: 0.7562\n",
            "Epoch 19/30\n",
            "64179/64179 [==============================] - 36s 557us/step - loss: 0.9425 - accuracy: 0.6305 - val_loss: 0.7044 - val_accuracy: 0.7629\n",
            "Epoch 20/30\n",
            "64179/64179 [==============================] - 36s 555us/step - loss: 0.9270 - accuracy: 0.6383 - val_loss: 0.6689 - val_accuracy: 0.7680\n",
            "Epoch 21/30\n",
            "64179/64179 [==============================] - 39s 609us/step - loss: 0.9168 - accuracy: 0.6437 - val_loss: 0.6771 - val_accuracy: 0.7812\n",
            "Epoch 22/30\n",
            "64179/64179 [==============================] - 36s 562us/step - loss: 0.8985 - accuracy: 0.6525 - val_loss: 0.6362 - val_accuracy: 0.7907\n",
            "Epoch 23/30\n",
            "64179/64179 [==============================] - 36s 561us/step - loss: 0.8945 - accuracy: 0.6520 - val_loss: 0.6279 - val_accuracy: 0.7917\n",
            "Epoch 24/30\n",
            "64179/64179 [==============================] - 36s 559us/step - loss: 0.8811 - accuracy: 0.6593 - val_loss: 0.6073 - val_accuracy: 0.8002\n",
            "Epoch 25/30\n",
            "64179/64179 [==============================] - 36s 559us/step - loss: 0.8746 - accuracy: 0.6623 - val_loss: 0.5886 - val_accuracy: 0.8072\n",
            "Epoch 26/30\n",
            "64179/64179 [==============================] - 36s 559us/step - loss: 0.8555 - accuracy: 0.6698 - val_loss: 0.5890 - val_accuracy: 0.8044\n",
            "Epoch 27/30\n",
            "64179/64179 [==============================] - 36s 563us/step - loss: 0.8523 - accuracy: 0.6712 - val_loss: 0.5690 - val_accuracy: 0.8208\n",
            "Epoch 28/30\n",
            "64179/64179 [==============================] - 36s 557us/step - loss: 0.8428 - accuracy: 0.6767 - val_loss: 0.5640 - val_accuracy: 0.8264\n",
            "Epoch 29/30\n",
            "64179/64179 [==============================] - 36s 557us/step - loss: 0.8393 - accuracy: 0.6779 - val_loss: 0.5458 - val_accuracy: 0.8238\n",
            "Epoch 30/30\n",
            "64179/64179 [==============================] - 36s 562us/step - loss: 0.8262 - accuracy: 0.6854 - val_loss: 0.5322 - val_accuracy: 0.8348\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WVf_GbyKhURc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 573
        },
        "outputId": "11ff60a5-a0a3-4e90-9b29-32a56acf99f3"
      },
      "source": [
        "plot_loss(model_glove_2lstm_b_hist)\n",
        "plot_acc(model_glove_2lstm_b_hist)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3hUVf7H8fc3PaSHBBISQui9RxBsIBaqKCKKDdQVccW661r2Zy+7q+hid8GOAiJgQxBERZrSe5MeAgmkEJIA6ef3xx0glAAJM7mZzPf1PPMwc++dO9/7jM4n955zzxFjDEoppTybl90FKKWUsp+GgVJKKQ0DpZRSGgZKKaXQMFBKKYWGgVJKKTQMlDorEUkUESMiPuew7XARWVAVdSnlTBoGqkYRkZ0iUigiUSctX+n4QU+0p7KKhYpSVU3DQNVEO4ChR1+ISFugln3lKFX9aRiommg8cHuZ18OAz8puICJhIvKZiKSLyC4R+T8R8XKs8xaR0SKSISLbgX6nee+HIpIqIntE5EUR8T6fgkWknoh8JyJZIrJVRO4us66LiCwTkRwR2ScirzuWB4jI5yKSKSLZIrJUROqeTx3Kc2kYqJroDyBURFo6fqRvAj4/aZu3gDCgEXAZVnjc4Vh3N9Af6AgkAYNPeu8nQDHQxLHNVcBfzrPmSUAKUM/xeS+LyOWOdW8AbxhjQoHGwGTH8mGOY6gP1AZGAkfOsw7loTQMVE119OzgSmAjsOfoijIB8YQxJtcYsxN4DbjNsckQYIwxZrcxJgv4V5n31gX6Ag8ZYw4ZY/YD/3Xsr1JEpD5wEfCYMSbfGLMK+IDjZzdFQBMRiTLG5Blj/iizvDbQxBhTYoxZbozJqWwdyrNpGKiaajxwMzCcky4RAVGAL7CrzLJdQJzjeT1g90nrjmrgeG+q49JMNvA/oM551FoPyDLG5JZTz11AM2CT41JQf8fy8cAsYJKI7BWRV0TE9zzqUB5Mw0DVSMaYXVgNyX2BaSetzsD6q7pBmWUJHD97SMW69FJ23VG7gQIgyhgT7niEGmNan0e5e4FIEQk5XT3GmC3GmKFYgfMfYIqIBBljiowxzxljWgHdsS5t3Y5SlaBhoGqyu4DLjTGHyi40xpRgXXd/SURCRKQB8AjH2xUmAw+ISLyIRACPl3lvKjAbeE1EQkXES0Qai8hlFajL39H4GyAiAVg/+ouAfzmWtXPU/jmAiNwqItHGmFIg27GPUhHpKSJtHZe9crACrrQCdSh1jIaBqrGMMduMMcvKWX0/cAjYDiwAJgAfOdaNw7r8shpYwalnFrcDfsAG4AAwBYitQGl5WA29Rx+XY3WFTcQ6S/gaeMYYM8exfW9gvYjkYTUm32SMOQLEOD47B6td5DesS0dKVZjo5DZKKaX0zEAppZSGgVJKKQ0DpZRSaBgopZQC3G70xKioKJOYmGh3GUop5VaWL1+eYYyJLm+924VBYmIiy5aV11tQKaXU6YjIrjOt18tESimlNAyUUkppGCillMIN2wyUUqqiioqKSElJIT8/3+5SXC4gIID4+Hh8fSs2gK2GgVKqxktJSSEkJITExERExO5yXMYYQ2ZmJikpKTRs2LBC79XLREqpGi8/P5/atWvX6CAAEBFq165dqTMgDQOllEeo6UFwVGWP02PCIOtQIc9/v4G8gmK7S1FKqWrHY8JgwdYMPlm0g35vzmfV7uyzv0EppZwkMzOTDh060KFDB2JiYoiLizv2urCw8IzvXbZsGQ888IDLa/SYBuRr2tcjJjSAh79cxeD3FvHwlc0YeVljvL0849RRKWWf2rVrs2rVKgCeffZZgoOD+fvf/35sfXFxMT4+p/85TkpKIikpyeU1uuzMQEQ+EpH9IrLuDNv0EJFVIrJeRH5zVS1HdWkYyYwHL6FP21henbWZoeP+YE/2EVd/rFJKnWL48OGMHDmSrl278o9//IMlS5bQrVs3OnbsSPfu3dm8eTMAc+fOpX///oAVJHfeeSc9evSgUaNGvPnmm06rx5VnBp8AbwOfnW6liIQD7wK9jTHJIlLHhbUcExboy5s3daBHs2ie/nYdfcbM4+VBbenfrl5VfLxSymbPfb+eDXtznLrPVvVCeWZA6wq/LyUlhUWLFuHt7U1OTg7z58/Hx8eHOXPm8OSTTzJ16tRT3rNp0yZ+/fVXcnNzad68Offee2+F7yk4HZeFgTFmnogknmGTm4Fpxphkx/b7XVXLyUSE6zvHk5QYwUNfrmLUhJX8uimd5wa2JtjfY66cKaVsdsMNN+Dt7Q3AwYMHGTZsGFu2bEFEKCoqOu17+vXrh7+/P/7+/tSpU4d9+/YRHx9/3rXY+cvXDPAVkblACPCGMaa8s4gRwAiAhIQEpxXQoHYQk+/pxls/b+HtX7eybFcWY27sQMeECKd9hlKqeqnMX/CuEhQUdOz5U089Rc+ePfn666/ZuXMnPXr0OO17/P39jz339vamuNg5PSTt7E3kA3QG+gFXA0+JSLPTbWiMGWuMSTLGJEVHlzsc95nl7IVv/goH95yw2Nfbi0euas6X93SjuMQw+P3fefuXLZSUmsp9jlJKVcLBgweJi4sD4JNPPqnyz7czDFKAWcaYQ8aYDGAe0N51n7YU1k6BtzrD3P9A4eETVl+QaDUu92sby+jZfzLo3YWsSD7gsnKUUqqsf/zjHzzxxBN07NjRaX/tV4QY47q/gB1tBtONMW1Os64lVgPz1YAfsAS4yRhTbu8jgKSkJFPpyW0O7IKfnoYN30BoPFz5HLS5HsrcsWeM4fs1qbz0wwb25RRwXcc4HuvdgpiwgMp9plLKdhs3bqRly5Z2l1FlTne8IrLcGFNuH1VXdi2dCPwONBeRFBG5S0RGishIAGPMRuBHYA1WEHxwtiA4bxENYMinMHwG1IqEqXfBR71hz4qydXNN+3r88rcejOrZhB/WptJz9Fze+nkL+UUlLi1PKaXs4tIzA1c4rzODskpLYNUX8PPzcCgDOtwCvZ6GkLonbLY76zAvz9jIzHVpxIUH8s9+LenTJsZjxjlRqibQMwMbzwyqPS9v6HQ73L8Cut8Pa76EtzrB/Neh6PiIf/Uja/HerZ2ZcHdXQgJ8+OsXK7hx7B+s33vQxuKVUsq5PDcMjgoIhategPsWQ8PL4Ofn4J0usGXOCZt1bxzF9Psv5sVr27BlXy7931rAE9PWkplXYFPhSinlPBoGR9VuDEMnwO3fgm8t+OJ6mPMslBxv1ffx9uLWCxsw9+89uaN7Q75atpueo+eycGuGfXUrpZQTaBicrFEPGDEXOt8BC/4Lnw6w7lEoI6yWL08PaMWPD11CbFggwz9ewrer9pxmZ0op5R40DE7HNwAGjIFB4yB1Nbx/CWz75ZTNmtQJYfLIbnRMiODBSasYN2+7DcUqpaq7nj17MmvWrBOWjRkzhnvvvfe02/fo0QOndJSpAA2DM2k3xDpLCIqG8YPg15etXkhlhAX68tmdXejbNoaXZmzkxekbKNW7l5VSZQwdOpRJkyadsGzSpEkMHTrUpopOpWFwNtHN4O6fof1Q+O0/MP46yDtxTL0AX2/eGtqJ4d0T+WDBDh78chUFxXpPglLKMnjwYH744YdjE9ns3LmTvXv3MnHiRJKSkmjdujXPPPOMrTXqEJ3nwi8IrnsPEi+CH/4G718Mgz+CxIuPbeLtJTwzoBUxYQH8e+YmMnIL+N/tnQkNOP+hZZVSTjTzcUhb69x9xrSFPv8ud3VkZCRdunRh5syZDBw4kEmTJjFkyBCefPJJIiMjKSkpoVevXqxZs4Z27do5t7ZzpGcGFdHxVvjLz+AfYjUszxsNpaXHVosIIy9rzOtD2rN0ZxZD3v+dfTn5Z9ihUspTlL1UdPQS0eTJk+nUqRMdO3Zk/fr1bNiwwbb69MygomLaWO0I3z8Iv7wAyX/AkM/Ar9axTQZ1iicq2J97P1/OoHcX8emdXWhSJ9i2kpVSZZzhL3hXGjhwIA8//DArVqzg8OHDREZGMnr0aJYuXUpERATDhw8nP9++Px71zKAy/EPg+g+h72jYOge+/SucNKzHpc2i+fKebhQUlzD4/UUs35VlU7FKqeogODiYnj17cueddzJ06FBycnIICgoiLCyMffv2MXPmTFvr0zCoLBHocjdc8Sys/9q6ZHSSNnFhTLv3IsIDfbl53GJ+3VRlk7kppaqhoUOHsnr1aoYOHUr79u3p2LEjLVq04Oabb+aiiy6ytTbPHajOWYyBr++xxja68XNoOeCUTTLzChj28RJ2pB/i21EX0aROiA2FKuW5dKA6HajO9URgwJsQ1xmm3QNpp47CXTvYn3G3JxHo580945eTV1D1E1copdSZaBg4g28A3DTBGvRu4lBrSOyTxIYF8tbQTuzMPMyjX63G3c7IlFI1m4aBs4TEWIFwaD98eRsUF56ySbfGtXmsd3Nmrktj3HwdukKpquQpf4BV9jg1DJwprhMMfAeSF8GMv5/Swwjg7ksa0bdtDP+euYlF23S0U6WqQkBAAJmZmTU+EIwxZGZmEhBQ8Wl69T4DZ2s7GPathwWvQ9020HXECatFhFcGt2dzWi73T1jJ9AcuJjYs0KZilfIM8fHxpKSkkJ6ebncpLhcQEEB8fHyF36e9iVyhtBS+vAX+nAW3TbOGxT7J1v25DHx7Ic1iQpg04kL8fbyrvEyllOfQ3kR28PKCQWMhqhlMHgaZ207ZpEmdEF69oT0rk7N5cfpGG4pUSqnjNAxcxT8Ehk4E8bJ6GOWfOmdy37ax3HNpI8b/sYupy1NsKFIppSwaBq4U2dAatyhrG0z9yylzIQA8enVzLmwUyZNfr2X93lMDQymlqoKGgas1vAT6vAJbZlsD253Ex9uLt2/uREQtP+79fAUHDxfZUKRSytNpGFSFC+6CzsOtOZU3/3jK6qhgf969tROpB4/w0JcrdaY0pVSV0zCoKr3/AzHtrHGMspNPWd0pIYKnB7Tm183pvPXLVhsKVEp5Mg2DquIbADd8AqYUvrrjtHco39o1gUGd4hjz85/8uC6t6mtUSnksDYOqVLsxDHwb9iyDOafOdyoivHxdWzrUD+ehL1eyene2DUUqpTyRy8JARD4Skf0icuownidud4GIFIvIYFfVUq20GghdR8If78KG705ZHeDrzbjbk4gK9ucvny0j5cBhG4pUSnkaV54ZfAL0PtMGIuIN/AeY7cI6qp8rX7CGvP52FGSdOmBdVLA/Hw+/gPyiEu76ZBk5+drDSCnlWi4LA2PMPOBscz3eD0wFPGsKMB8/GPyxNRfCV8Oh6NR5T5vWDeH9WzuzLT2P+75YQVFJadXXqZTyGLa1GYhIHHAd8N45bDtCRJaJyLIaM9BURAO47n1IXQ2znjztJhc1ieLl69oyf0sGz3y3vsaPuKiUso+dDchjgMeMMWf9k9cYM9YYk2SMSYqOjq6C0qpI8z7Q/QFY9iGsnXLaTYZcUJ+/9mjMhMXJOgeCUspl7BzCOgmYJCIAUUBfESk2xnxjY01Vr9fTsHsJfP8gxLaHqKanbPL3q5qzK/Mw/5q5iYTIWvRuE2tDoUqpmsy2MwNjTENjTKIxJhGYAvzV44IAwNsXBn8EPv7WCKeFp/Ye8vISXhvS3tHldBWrtMupUsrJXNm1dCLwO9BcRFJE5C4RGSkiI131mW4rLM4a8nr/Bpj56Gk3OdrlNDrEn798upTdWdrlVCnlPDq5TXXyy4sw71W49j3ocPNpN9m6P5fr3l1ETGgAU//andAA3youUinljnRyG3fS4wlIvASmPwIHdp12kyZ1QvjfrZ3ZkXFIu5wqpZxGw6A68fK2upuaUvjtlXI3694kipcHWV1OH5+6Vkc5VUqdNw2D6iYs3hryevUEyNhS7mZDkurz0BVNmboihae+Xaf3ICilzouGQXV08SPgEwi/vnzGzR7s1ZSRlzXmi8XJPD99gwaCUqrSNAyqo+BouHAkrJ8GaeWP8yciPNa7OXdclMjHC3fynx83ayAopSpFw6C66n4/+IfBry+dcTMR4en+rbilawLv/7aNN34u/9KSUkqVR8OgugqMsAJh8wxIOXNXWhHhhYFtGNw5njFztvDuXJ0pTSlVMRoG1dmFI6FWbfjlhbNu6uUl/Of6dlzTvh6v/LiZDxfsqIIClVI1hYZBdeYfYjUmb58LO+afdXNvL+H1Ie3p0yaGF6ZvYPwfp79XQSmlTqZhUN1dcBeExFp3J59D47CPtxdv3NSRXi3q8NQ365i8dHcVFKmUcncaBtWdbyBc+nfY/QdsnXNOb/Hz8eKdWzpxSdMoHpu2hm9X7XFxkUopd6dh4A463g7hCVbbwTl2HQ3w9WbsbUl0bRjJI5NXM2NtqouLVEq5Mw0Dd+DjZ41blLoaNn5/zm8L9PPmw2EX0KF+OA9MXMms9WkuLFIp5c40DNxFuxshqpl130FpyTm/Lcjfh4/vuIC28WHc98UKDQSl1GlpGLgLL2/r7CB9E6ybWqG3hgb48umdXY4Fwo/rNBCUUifSMHAnra6Fum2tMYtKiir01tAAXz5zBMKoCRoISqkTaRi4Ey8vuPz/4MAOWPVFhd8eckogaKOyUsqiYeBuml0N8RdY8x0U5Vf47UcDoV18GKMmrNRAUEoBGgbuR8Q6O8jZA8s/qdQuQhxtCEcDYaZ2O1XK42kYuKNGPazpMeePhsJDldrF0UBoXz+c+ydqICjl6TQM3FWvp+FQOvzxXqV3ERLgyyd3XED7+uGM0kBQyqNpGLir+l2gRX+YNxqytld6N0fPEDpoICjl0TQM3FnfV8HbF75/8JyHqTidYH+fEwJBxzJSyvNoGLiz0Hpw5XOwYx6s/Py8dnU0EDo3iODBSat46YcNFJeUOqlQpVR1p2Hg7joNhwYXwex/Qu753UgW7O/D53d15fZuDRg3fwe3fbiEjLwC59SplKrWNAzcnZcXDHjTuudgxqPnvTs/Hy+eH9iG0Te0Z0XyAQa8tYBVu7OdUKhSqjrTMKgJoppAj8dh43cVGtX0TAZ3jmfqvd3x9hKGvP87k5YkO2W/SqnqyWVhICIfich+EVlXzvpbRGSNiKwVkUUi0t5VtXiE7vdDTFv44e9wxDl/ybeJC+P7URfTtVEkj09byxPT1lBQfO4jpiql3Icrzww+AXqfYf0O4DJjTFvgBWCsC2up+bx94Zq3rXsPfnrKabuNCPLjkzu6cF/Pxkxcspsh7//O3uwjTtu/Uqp6cFkYGGPmAVlnWL/IGHPA8fIPIN5VtXiMeh2g+yhY8Rls/81pu/X2Eh69ugXv39qZbemHGPDWAhZty3Da/pVS9qsubQZ3ATPLWykiI0RkmYgsS09Pr8Ky3FCPJyCyEXz/ABQeduque7eJ4Zv7LiK8li+3fbiEsfO2UVpa+fsblFLVh+1hICI9scLgsfK2McaMNcYkGWOSoqOjq644d+QbaPUuOrAT5r7s9N03qRPMt6Mu5sqWdXl5xiZu+WAxyZnODR2lVNWzNQxEpB3wATDQGJNpZy01SsNLoNMw+P0d2LPC6bsP9vfhvVs78e9BbVm75yBXj5nHxwt36FmCUm7MtjAQkQRgGnCbMeZPu+qosa58HoLqwHf3V3hWtHMhItzUJYHZD19Kl4aRPPf9Bm4c+zs7Mio3iqpSyl6u7Fo6EfgdaC4iKSJyl4iMFJGRjk2eBmoD74rIKhFZ5qpaPFJgOPR7Dfatg4VvuOxj6oUH8skdF/Dq4HZsTsul95h5fDB/OyV6lqCUWxFzHgOc2SEpKcksW6a5cc4m3w6bf4R7F0JUU5d+1L6cfP759VrmbNxPx4RwXh3cjiZ1Qlz6mUqpcyMiy40xSeWtt70BWblYn1etRuXv7odS1w48Vzc0gHG3J/HGTR3YkXGIvm8u4L2523TAO6XcgIZBTRdSF65+GZJ/t2ZGczERYWCHOGY/fCmXN6/Df37cxKD3FrExNcfln62UqjwNA0/Q4WZodyP8+jJs+6VKPrJOSADv3dqJd27uxJ4DR+j35nye+XYdBw87vzFbKXX+NAw8gQj0/y9Et4Apd8HBlCr6WKFfu1h+/ttl3NK1AeP/2EXP1+YycUmyNjArVc1oGHgKvyC4cbzVzXTyMCgurLKPDq/lxwvXtmH6/ZfQJDqYJ6at5dp3FrJ814Gzv1kpVSU0DDxJVFO49h3Ys8yaDKeKtaoXypf3XMgbN3Vgf24+17+3iEcmr2J/bn6V16KUOpGGgadpNRC6jYIlY2HtlCr/+KMNzL/8rQf39mjM96v3cvno3xg3bzuFxdrrSCm7aBh4oiuehYRuVnfT/ZtsKSHI34fHerdg9sOXcUFiBC/N2EifN+Yx708diFApO5xTGIhIkIh4OZ43E5FrRMTXtaUpl/H2hcEfg18wTL4NCnJtK6VhVBAf39GFD4clUVxquP2jJdz5yVK27revJqU80bmeGcwDAkQkDpgN3IY1eY1yV6GxMPgjyNxqnSHYfCd6r5Z1mf3wpTzRpwVLd2Rx9Zj5PP3tOjLzCmytSylPca5hIMaYw8Ag4F1jzA1Aa9eVpapEw0ug1zOw/mtY/L7d1eDv4809lzVm7qM9uLlLAl8sTqbH6LmMnbdNp9tUysXOOQxEpBtwC/CDY5m3a0pSVeqiB6FFf5j9f5C82O5qAKgd7M8L17bhxwcvIalBBC/P2MQVr//GD2tScbextJRyF+caBg8BTwBfG2PWi0gj4FfXlaWqjAgMfAfC6sNXwyCv+jTgNq0bwsd3dGH8XV0I8vPhvgkrGPz+76xM1vsTlHK2Co9a6mhIDjbG2DLYjI5a6iJpa+GDK6B+F7jtG/CqXid+JaWGr5btZvTsP8nIK+Ca9vX421XNaFA7yO7SlHILThm1VEQmiEioiAQB64ANIvKos4pU1UBMW+j3OuyYB989AEVH7K7oBN5e1mQ6cx/twaieTZi1Po0eo+cy4rNlLN6eqZePlDpP53RmICKrjDEdROQWoBPwOLDcGNPO1QWeTM8MXOyXF2HeqxDdEgZ/CHWrZz+BfTn5fLpoJxOWJJN9uIg2caHcdXFD+rWth5+P3j6j1MnOdmZwrmGwHugATADeNsb8JiKrjTHtnVfqudEwqAJb58DX90L+QbjqRehyt9W2UA0dKSxh2soUPlqwg23ph6gT4s+w7onc3CWBiCA/u8tTqtpwVhg8ADwGrAb6AQnA58aYS5xV6LnSMKgieenw7X2wZRY0vRqufReCouyuqlylpYbftqTz0YIdzN+SQYCvF4M6xXPnRYk625pSOCkMytmxjzGmuNKVVZKGQRUyBpaMs7qdBobDte9Bk152V3VWm9Ny+XjhDqat3ENhcSmXNovmpgvq06tlHfx9qlfDuFJVxVlnBmHAM8CljkW/Ac8bYw46pcoK0DCwwb711jwI6RutQe56PQ0+/nZXdVaZeQV8sTiZCYuTScvJJyzQl4Ed6jG4czxt48KQanrpSylXcFYYTMXqRfSpY9FtQHtjzCCnVFkBGgY2KToCs5+CpeOsnkfXfwTRzeyu6pyUlBoWbs3gq+UpzFqfRmFxKc3qBjO4czzXdoyjTkiA3SUq5XLOCoNVxpgOZ1tWFTQMbLZphtWWUHQEev8LOg0DL/fpvXPwSBHT1+xlyvIUViZn4+0lXNYsmsGd4/UykqrRnBUGvwOPGmMWOF5fBIw2xnRzWqXnSMOgGshJha/vgR2/QWwHuOoFaHjp2d9XzWzdn8fUFSlMW5HCvpwCwgJ9ua5jHEO7JNA8RhudVc3irDBoD3wGhDkWHQCGGWPWOKXKCtAwqCZKS2HNl9Z9CTkpVo+jK5+DOi3trqzCSkoNC7Zm8NWy3cxan0ZRiaFTQjg3d21Av7axBPrp2YJyf07tTSQioQDGmBwRecgYM8YJNVaIhkE1U5QPS/4H816DwlzocAv0fBJC69ldWaVk5hUwbcUeJi5JZnvGIUICfBjUMY6hXRNoERNqd3lKVZoru5YmG2MSKl1ZJWkYVFOHs2DeaGs6TS8f6HafNSJqgHv+gBpjWLwji4lLkpm5No3CklI6JoQztEsC/dvFUsvPx+4SlaoQV4bBbmNM/UpXVkkaBtXcgZ3w8wuwbgrUioIej0Pn4dbsam7qwKFCpq5IYeKSZLalHyLE34eBHetxTfs4khpE4OWlXVRV9WfbmYGIfAT0B/YbY9qcZr0AbwB9gcPAcGPMirN9roaBm9izAn56GnbOh8jGMGCMWzYyl2WMYdmuA0xcnMwPa1MpKC4lJjSAvm1j6dculk4J4Xrvgqq2zisMRCQXON0GAgQaY8o9VxaRS4E84LNywqAvcD9WGHQF3jDGdC23GAcNAzdiDGz5CWY9Cdm7YMh4aN7b7qqc4lBBMXM27mP6mlR+25xOYUkpceGB9GsXS7+2sbSL15vaVPXisjODc/zwRGB6OWHwP2CuMWai4/VmoIcxJvVM+9QwcENHDsD4QdacCTd8DC0H2F2RU+XkFzFngxUM87ekU1RiqB8ZSL+29ejfLpbW9UI1GJTtqnMYTAf+XebehZ+Bx4wxp/zSi8gIYARAQkJC5127drmsZuUi+Qfh88GwZzlc/wG0qfKb16vEwcNFzFqfxvS1qSzcmkFJqaFJHetu5+s6xlE3VO92VvaoEWFQlp4ZuLGCXPjiBti9GK77H7QbYndFLpV1qJCZ61KZtmIPy3cdwEvgUsfdzle0rEuAr96/oKrO2cLAzv5xe4CyvZHiHctUTeUfArdOhQk3wrQRUFIEHW+xuyqXiQzy45auDbilawO2px+923kPoyasJDTAh2s61OOGzvW1fUFVC3aeGfQDRnG8AflNY0yXs+1TzwxqgMLD8OUtsO0X6D8Gku6wu6IqU1JqWLQtgynLU/hxXRoFxaU0LXMZqY5eRlIuYttlIhGZCPQAooB9WENg+wIYY953dC19G+iN1bX0jrNdIgINgxqjKB8m3wZbZkPf0dZsah4mJ7+IH9akMmV5Cst3HUAEuiRG0rdtLL3bxGj7gnIqW9sMXEHDoAYpLoCv7oDNP8DVL1t3LXuo7el5fLNqLz+uS+XPfXmIQFKDCPq0sYKhXnig3SUqN6dhoKq3kiKYehds+BaueBYuftjuimy3dX8uM9amMYnK4kMAABXGSURBVGNtKpvScgHomBBO3zax9GkbQ3xELZsrVO5Iw0BVfyXF1pDY66ZAjyfhsn+ANqgC1hnDzHVWMKzfmwNA+/gwejSvQ9dGkXRKiNBeSeqcaBgo91BaYk2as3oitLoWrnkTAsLO/j4PsivzEDPXpTFzbSpr9hzEGPD1FtrHh9O1USRdG9amc4MIgvx1ED11Kg0D5T5KS2HRG9ZAd2Hx1t3KcZ3trqpaOnikiOW7sli8PYvFO7JYu+cgJaUGby+hTVwYFzaMpEvDSJISIwkLdN9BApXzaBgo95O82GpHyE2z2hG63aeXjc7iUEExy3cdYMmOLBbvyGT17oMUlpTiJdA2PpxLmkRxUZMoOjUI16k9PZSGgXJPh7Pgu/th03RrFrVr34Og2nZX5Tbyi0pYmZzN79szWbg1g1W7sykpNQT6etOlYSQXN4ni4qZRNK8bokNwewgNA+W+jLEmy5n9f9bcCIM/hAbd7a7KLeXkF7F4exYLt2Ywf0s629IPARAV7Ef3xlYwdEoIJ7F2ED7eXjZXq1xBw0C5v72rYMod1sQ5PZ6ESx4BL73UcT5SDx5h4dZMFmxJZ8HWTDLyCgDw8/GiSXQwLWJCaO54tIgJpW6ovw6Z4eY0DFTNUJAL0x+GtV9Zk+QMGgchMXZXVSMYY9iyP491ew6yOS2XTWm5bE7LJS0n/9g2YYG+NK9rhUObuFB6Nq+jQ2e4GQ0DVXMYAys/hxmPgn+wNYxFq4HauOwi2YcL2ZyWy+Z9xwNic1oueQXFiEDH+uH0bhPD1a1jaFA7yO5y1VloGKiaZ/9GmHIX7F8PsR3g8v+DJldoKFQBYwyb9+Uye/0+Zq1PO3YjXIuYEK5qHcPVrevSKlYn86mONAxUzVRSDGsnw9x/QXYy1O9qhYKbz7PsbnZnHWbW+jRmr9/H0l1ZGAP1IwO5ulUMV7eJoWVsKEF+3hoO1YCGgarZigth1efw26uQuxcaXmaFQv2zjoaunCwjr4A5G6wzhoVbMyksKQXAx0sIC/QlrJYv4YG+hAX6El7Lz1oW6Et4LV+iQ/zpUD+cuPBADQ4X0TBQnqEoH5Z9BPNfg8MZ1r0Jl/8TYtvbXZlHys0vYv6WDHZnHebgkSKyjxRx8EgRBw8XkX2k0Fp2uIjc/OIT3hcTGkDnBhF0bhBBUmIELWND8dWurk6hYaA8S0GedW/CwjcgPxtaXgM9n4Q6Le2uTJ1GcUkpufnF7Mk+wvJdB1i26wDLd2ax96DVkynQ15v29cNIahBJ58QIOiVE6PAalaRhoDzTkWz44134/R0ozrfmXG472O6q1Dna6wiHo48NqTmUlBpEoE6IP7FhgdQLDyA2LJDYMMe/4QHUCwskOsQfb72r+hQaBsqzHcqEybfDroUwYAx0Hm53RaoSDhUUs3p3Nst3HWBX1mFSDx4h9WA+qdn5HCkqOWFbby+hbog/iVFBXNmqLn3axBITpvdEaBgoVXjYCoStP8FVL0H3UXZXpJzEGMPBI0Xszc4nLecIe7PzjwXF+j05bN5nTQ7UuUEEfdvG0seDZ43TMFAKrF5H0+6GDd/AZY9Dj8f1vgQPsC09j5lrU/lhbRobU617Ijx11jgNA6WOKi2B7x6wuqJeeB9c/ZIGggfZkXGIGWtTT5k1rm/bWJrHhBAS4EOwvy/BAT4E+1uPmtT2oGGgVFmlpTDrCVj8PnS6HfqP0UHvPNCuzEPH5pleu+dgudvV8vO2giHAhxB/H8Jr+dE4OpimdYNpWieYpnVCCKvlHr2bNAyUOpkx8OvLMO8VaD3I6mnk42d3Vcome7OtNoa8gmLy8ovJK7Dufzj+uphcx/OMvAK2px86odE6OsSfpnWCaVY3hCZ1rJBoUieYyCC/anUD3dnCQCdLVZ5HxLohzT8YfnoaCg/BkE/B1zMbFj1dvfDACjUql5Ya9mQfYcv+XLbsy2PLfuvx1bLdHCo8HhI+XkJIgA8hAb6Of30IDfA99jrUsS4qxI8O9SNIrF3L1vDQMFCe66IHwT8Epj8CX9wAQydar5U6Ay8voX5kLepH1uLyFnWPLTfGkHowny3789i6P4/MvAJy84vJybfONHLzi0jOOnxsWV5BMWUvzEQG+dEpIYJODcLpnBBBu/hwAv2q7hKmXiZSas1X8PU9UK8D3PyVTq+pqkRpqSGvsJi92UdYmWzdQ7Fi1wG2Z1iz0Pl4Ca3rhdLJMTxHp4SI8+oWq20GSp2LTT/AV8PByxfaDYEud0Pd1nZXpTxQ1qFCViYfv/t6dUo2+UXWoH93X9KQf/ZrVan9ahgoda7S1llDWKydAiUFkNAduvwFWgzQBmZlm6KSUjam5rB81wFaxoZyYaPKnbnaGgYi0ht4A/AGPjDG/Puk9QnAp0C4Y5vHjTEzzrRPDQPlcoezYOV4WPohZO+C4LrWMBadh0NoPburU6pSbAsDEfEG/gSuBFKApcBQY8yGMtuMBVYaY94TkVbADGNM4pn2q2GgqkxpCWydA0vGWf+KF7TsDxf8BRIv0RvWlFuxs2tpF2CrMWa7o5BJwEBgQ5ltDBDqeB4G7HVhPUpVjJc3NLvaemRtt+ZLWDEeNnwLdVpBr6ehWW8NBVUjuHLWiDhgd5nXKY5lZT0L3CoiKcAM4P7T7UhERojIMhFZlp6e7opalTqzyEZw1Yvwt00w8B0oKYSJN8GnA2DvSrurU+q82T2F0FDgE2NMPNAXGC8ip9RkjBlrjEkyxiRFR0dXeZFKHeMbCB1vhb/+AX1Hw/4NMLYHTBsB2bvP+nalqitXhsEeoH6Z1/GOZWXdBUwGMMb8DgQAUS6sSSnn8Pa1up8+sBIufhjWfwNvJ8Gc5yA/x+7qlKowV4bBUqCpiDQUET/gJuC7k7ZJBnoBiEhLrDDQ60DKfQSEwRXPwv3LodVAWPA6vNnRanQuKbK7OqXOmcvCwBhTDIwCZgEbgcnGmPUi8ryIXOPY7G/A3SKyGpgIDDfuduODUgDh9WHQWBgx15pvecbf4d1usGkG6H/Syg3oTWdKOZsx8OePMPspyNwCjXpC//9CZEO7K1Me7GxdS+1uQFaq5hGB5n3gr79Dn1chZZl1lrBgjF46UtWWhoFSruLtC11HwKgl0KQXzHkGxvaEPSvsrkypU2gYKOVqofXgpi/gxs/hUDp80At+fAIK8uyuTKljNAyUqiotB1hnCZ3vsAbEe/dC+HO23VUpBWgYKFW1AsKg/+tw5yzwC4IJN8CUOyFvv92VKQ+nYaCUHRIuhHvmQY8nYeP31g1ryz+B4kK7K1MeSsNAKbv4+EOPx2DkQqjTGr5/EMa0hbn/gdx9dlenPIyGgVJ2i24Gw3+wptyMaQNzX4b/toapd1vdUpWqAq4cwlopda68vKDZVdYjY4s1nMWqCbB2MsR1hi73QOtrrbMJpVxA70BWqroqyIVVE2HJWOtO5qBoqydS0p0QGmt3dcrN6BzISrm70lLY/qsVCn/OsibdiWoOQVGORzTUioKg2mWeR1uvA8J18h0F2DvTmVLKGby8rDuYm/SyZlxb/ql1KelQujWxzqEMKChn2OzASLj8n9YZhZd31dat3IqGgVLuJLIRXPncqcuLC+BwphUQhzKsx+EM2DwTfvib1f7Q73Wo16Hqa1ZuQcNAqZrAx98a9iK03onLL/wrrP0KZj0J43pClxHQ80nr5jelytCupUrVZCLQbgiMWmY1PC/+H7zdBdZN1XkW1Ak0DJTyBIHh0O81uPtnCImxhsAYfx1kbqvYfoyB0hLX1KhspZeJlPIkcZ3h7l9g6YfwywvWYHkXPwwXPwK+AdY2xlhtDlnbrAbrTMe/Wdshawdg4KoXodPt2lOpBtGupUp5qtw0mPVPWDcFIhpajcuZ26wf/MLc49uJN4QnWI3XkY1g/0bYtQBa9IcBb1pdWFW1p/cZKKXObNuv8NNTUHjI8YPf+PgPf+3GEFYffPyOb19aCn+8Az8/b3VdvfZdq9urqtY0DJRSrpG2Fqb+BdI3QdeRcMVzxy81qWpH50BWSrlGTFsYMdcaN2nx+zC2B6Sts7koVVkaBkqpyvMNhL6vwC1T4UiWdS/DoretS0nKrWgYKKXOX9Mr4N5F0ORKmP1PGH8t5Oy1uypVAdq1VCnlHEFRcNMXsOIz+PFxeLcbdLgFapdplA6L1zGSqikNA6WU84hA52GQeDFMfwiWfQjF+cfXe/tBRGKZXksNrR5L0S1OHUpDVSkNA6WU89VuDMO+t9oOclMdN6yVvYltB2z/DYqPHH9PfBdr6IzW11lnGapKaddSpZQ9SkshL80Kh5QlsHYK7N9g3eTWpBe0HQIt+oJfkN2V1gi23mcgIr2BNwBv4ANjzL9Ps80Q4FnAAKuNMTefaZ8aBkrVYGnrrKk+106BnD3gWwta9LOCoXFP8Pa1u0K3ZVsYiIg38CdwJZACLAWGGmM2lNmmKTAZuNwYc0BE6hhj9p9pvxoGSnmA0lJIXmQNv73+G8jPtmZwa30dtBkE9btqQ3QF2TnTWRdgqzFmu6OQScBAYEOZbe4G3jHGHAA4WxAopTyEl5fVCJ14MfR5BbbOgTWTYeV4WDoOgutaZwwtr7G20TOG8+bKMIgDdpd5nQJ0PWmbZgAishDrUtKzxpgfXViTUsrd+PhbP/wt+kFBrjUP9MbvYPUkWPYRBEZA875WMDTuaW2vKszu3kQ+QFOgBxAPzBORtsaY7LIbicgIYARAQkJCVdeolKou/EOg7WDrUXgYtv0MG7+HjdNh1RfgFwLNroZW10AjRxuDKbWG5TalZR5lXmMgqI51NuLBXBkGe4D6ZV7HO5aVlQIsNsYUATtE5E+scFhadiNjzFhgLFhtBi6rWCnlPvxqQcsB1qO4EHb8Bhu+hU0/WMNyV0RIrKM94nprzgcPnKfBlQ3IPlgNyL2wQmApcLMxZn2ZbXpjNSoPE5EoYCXQwRiTWd5+tQFZKXVGJcVW43PKMutHXbyOPyj72vHclML2ubDlJygpsOZuaHO99ajbpsYEg91dS/sCY7DaAz4yxrwkIs8Dy4wx34mIAK8BvYES4CVjzKQz7VPDQCnlEvkHHWcVU605HkwJRDWDNoOtHkxRTe2u8LzofAZKKVVRhzJh47ewdirsWggYiGlnNVSHxUFQtOMRZf3rBjfGaRgopdT5yNlr3euwbirsKee3x7fW8WA4+ojrbIVHSN2qrbccGgZKKeUsRUfgULrjkVHO83TISYXDGYBA/S6OrrH9rTGbbKJhoJRSVc0Ya5yljdNh03RIW2Mtj255/J6Jeh2rtHFaw0AppeyWnQybZljBsGuR1TgdGmddRmrexxpewz/YpSVoGCilVHVyOAv+/NHqubT1Z2sYb/G25pRO6AYJF1qPkBinfqyGgVJKVVeFh60zhd1/QPIf1r0RR+d4iGhYJhy6WV1bz+Oykp0D1SmllDoTv1rW/NFNr7BeFxda7QvJv1vhsGUWrJ5grQuMhEsege73u6QUDQOllKoufPwgPsl6dL/faojO3GoFQ/If1rAZrvpol+1ZKaXU+RGxLg9FNYVOt7n0ozx7mD6llFKAhoFSSik0DJRSSqFhoJRSCg0DpZRSaBgopZRCw0AppRQaBkoppXDDsYlEJB3YVcm3RwEZTiynOqhpx1TTjgdq3jHVtOOBmndMpzueBsaY6PLe4HZhcD5EZNmZBmpyRzXtmGra8UDNO6aadjxQ846pMsejl4mUUkppGCillPK8MBhrdwEuUNOOqaYdD9S8Y6ppxwM175gqfDwe1WaglFLq9DztzEAppdRpaBgopZTynDAQkd4isllEtorI43bX4wwislNE1orIKhFxu4mhReQjEdkvIuvKLIsUkZ9EZIvj3wg7a6yoco7pWRHZ4/ieVolIXztrrAgRqS8iv4rIBhFZLyIPOpa75fd0huNx5+8oQESWiMhqxzE951jeUEQWO37zvhQRvzPuxxPaDETEG/gTuBJIAZYCQ40xG2wt7DyJyE4gyRjjljfLiMilQB7wmTGmjWPZK0CWMebfjtCOMMY8ZmedFVHOMT0L5BljRttZW2WISCwQa4xZISIhwHLgWmA4bvg9neF4huC+35EAQcaYPBHxBRYADwKPANOMMZNE5H1gtTHmvfL24ylnBl2ArcaY7caYQmASMNDmmjyeMWYekHXS4oHAp47nn2L9j+o2yjkmt2WMSTXGrHA8zwU2AnG46fd0huNxW8aS53jp63gY4HJgimP5Wb8jTwmDOGB3mdcpuPl/AA4GmC0iy0VkhN3FOEldY0yq43kaUNfOYpxolIiscVxGcotLKicTkUSgI7CYGvA9nXQ84MbfkYh4i8gqYD/wE7ANyDbGFDs2OetvnqeEQU11sTGmE9AHuM9xiaLGMNY1zJpwHfM9oDHQAUgFXrO3nIoTkWBgKvCQMSan7Dp3/J5Oczxu/R0ZY0qMMR2AeKwrIS0qug9PCYM9QP0yr+Mdy9yaMWaP49/9wNdY/xG4u32O67pHr+/ut7me82aM2ef4n7UUGIebfU+O69BTgS+MMdMci932ezrd8bj7d3SUMSYb+BXoBoSLiI9j1Vl/8zwlDJYCTR2t637ATcB3Ntd0XkQkyNEAhogEAVcB6878LrfwHTDM8XwY8K2NtTjF0R9Nh+two+/J0Tj5IbDRGPN6mVVu+T2Vdzxu/h1Fi0i443kgVkeZjVihMNix2Vm/I4/oTQTg6Co2BvAGPjLGvGRzSedFRBphnQ0A+AAT3O2YRGQi0ANruN19wDPAN8BkIAFrqPIhxhi3aZAt55h6YF1+MMBO4J4y19urNRG5GJgPrAVKHYufxLrO7nbf0xmOZyju+x21w2og9sb6A3+yMeZ5x2/EJCASWAncaowpKHc/nhIGSimlyucpl4mUUkqdgYaBUkopDQOllFIaBkoppdAwUEophYaBUqcQkZIyo1eucuYotyKSWHZEU6WqC5+zb6KUxzniuLVfKY+hZwZKnSPH/BGvOOaQWCIiTRzLE0XkF8cgZz+LSIJjeV0R+doxzvxqEenu2JW3iIxzjD0/23HXqFK20jBQ6lSBJ10murHMuoPGmLbA21h3tAO8BXxqjGkHfAG86Vj+JvCbMaY90AlY71jeFHjHGNMayAaud/HxKHVWegeyUicRkTxjTPBplu8ELjfGbHcMdpZmjKktIhlYE6YUOZanGmOiRCQdiC87BIBj2OSfjDFNHa8fA3yNMS+6/siUKp+eGShVMaac5xVRdnyYErTtTlUDGgZKVcyNZf793fF8EdZIuAC3YA2EBvAzcC8cm3wkrKqKVKqi9C8SpU4V6Jg16qgfjTFHu5dGiMgarL/uhzqW3Q98LCKPAunAHY7lDwJjReQurDOAe7EmTlGq2tE2A6XOkaPNIMkYk2F3LUo5m14mUkoppWcGSiml9MxAKaUUGgZKKaXQMFBKKYWGgVJKKTQMlFJKAf8PmoriAXvgbXcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3hUddbA8e/JpFFCSUJNgNCLgJQAChZAUKzYBRuoa0ext7Wgrvu6rrprW3exggVEEUEFOwpigRB66BAglCQQSCGkznn/uIOGkIQAmUwycz7PM8/MLXPn3AzcM/dXRVUxxhgT2IJ8HYAxxhjfs2RgjDHGkoExxhhLBsYYY7BkYIwxBksGxhhjsGRgAoSIxImIikhwJfYdKyI/V0dcxtQUlgxMjSMiySJSICLRpdYv8VzQ43wT2SGx1BeRHBGZ4+tYjKkKlgxMTbUZGH1wQUR6AHV9F85hLgHygeEi0rw6P7gydzfGHC1LBqameg+4tsTyGGByyR1EpKGITBaRdBHZIiKPikiQZ5tLRJ4Xkd0isgk4t4z3viUiO0Vku4j8TURcRxHfGOC/wHLg6lLHPkVEfhGRfSKyTUTGetbXEZEXPLFmisjPnnWDRSSl1DGSRWSY5/UEEflERN4XkSxgrIj0F5FfPZ+xU0ReFZHQEu8/QUS+FZEMEUkVkUdEpLmI5IpIVIn9+nj+fiFHce7GD1kyMDXVb0ADEenquUiPAt4vtc8rQEOgHXA6TvK4zrPtRuA8oDcQD1xa6r3vAkVAB88+ZwJ/qUxgItIGGAx84HlcW2rbHE9sTYBewFLP5ueBvsBAIBJ4AHBX5jOBkcAnQCPPZxYDdwPRwMnAGcBtnhgigO+Ar4CWnnP8XlV3AT8Cl5c47jXAVFUtrGQcxl+pqj3sUaMeQDIwDHgU+D9gBPAtEAwoEAe4gAKgW4n33Qz86Hn9A3BLiW1net4bDDTDKeKpU2L7aGCu5/VY4OcK4nsUWOp5HYNzYe7tWX4YmFHGe4KAA8CJZWwbDKSU9TfwvJ4AzDvC3+yug5/rOZcl5ex3BbDA89oF7AL6+/o7t4fvH1b2aGqy94B5QFtKFRHh/CIOAbaUWLcF5+IMzi/ibaW2HdTG896dInJwXVCp/StyLfAGgKpuF5GfcIqNlgCtgI1lvCcaCC9nW2UcEpuIdAJexLnrqYuT5BZ7NpcXA8BM4L8i0hboDGSq6sJjjMn4ESsmMjWWqm7BqUg+B/i01ObdQCHOhf2g1sB2z+udOBfFktsO2oZzZxCtqo08jwaqesKRYhKRgUBH4GER2SUiu4ABwJWeit1tQPsy3robyCtn235KVI57isWalNqn9PDCrwNrgI6q2gB4BDiY2bbhFJ0dRlXzgGk49RzX4CRcYywZmBrvBmCoqu4vuVJVi3Euas+ISISnrP4e/qxXmAbcKSKxItIYeKjEe3cC3wAviEgDEQkSkfYicnol4hmDU2TVDac+oBfQHagDnI1Tnj9MRC4XkWARiRKRXqrqBt4GXhSRlp4K7pNFJAxYB4SLyLmeitxHgbAjxBEBZAE5ItIFuLXEti+AFiJyl4iEef4+A0psn4xTFHYBlgyMhyUDU6Op6kZVTShn8x04v6o3AT8DH+JccMEpxvkaWAYkcvidxbVAKJAE7MWpnG1RUSwiEo5T+fqKqu4q8diMc1Edo6pbce5k7gUycCqPT/Qc4j5gBbDIs+0fQJCqZuJU/r6Jc2ezHzikdVEZ7gOuBLI95/rRwQ2qmg0MB87HqRNYDwwpsX0BTsV1oufuyxhE1Sa3MSbQiMgPwIeq+qavYzE1gyUDYwKMiPTDKepq5bmLMMaKiYwJJCIyCacPwl2WCExJdmdgjDHG7gyMMcZQ+zqdRUdHa1xcnK/DMMaYWmXx4sW7VbV0/5U/1LpkEBcXR0JCeS0NjTHGlEVEKmxGbMVExhhjLBkYY4yxZGCMMYZaWGdQlsLCQlJSUsjLy/N1KF4XHh5ObGwsISE2F4kxpur4RTJISUkhIiKCuLg4SgxJ7HdUlT179pCSkkLbtm19HY4xxo/4RTFRXl4eUVFRfp0IAESEqKiogLgDMsZUL79IBoDfJ4KDAuU8jTHVy2+SgTHG+K3d62Hu/0Fqktc+wi/qDHxtz549nHHGGQDs2rULl8tFkyZOR7+FCxcSGhpa7nsTEhKYPHkyL7/8crXEaoypJfZtg5XTYeUnsGsFIFC/CTTr5pWPs2RQBaKioli6dCkAEyZMoH79+tx3331/bC8qKiI4uOw/dXx8PPHx8dUSpzGmhstJg1WfOQlg2+/Ouph4OOv/4ISLoEGF8y8dF0sGXjJ27FjCw8NZsmQJgwYNYtSoUYwfP568vDzq1KnDO++8Q+fOnfnxxx95/vnn+eKLL5gwYQJbt25l06ZNbN26lbvuuos777zT16dijDmSglxIXwNpqyEtCXavA1co1GsC9aKhbrTnOarEuihwhcCBvbD6CycBbJ4H6oam3WDoY9D9EoisnpaDfpcMnvx8FUk7sqr0mN1aNuCJ8484V/phUlJS+OWXX3C5XGRlZTF//nyCg4P57rvveOSRR5g+ffph71mzZg1z584lOzubzp07c+utt1qfAmNqCncxZGyGtFVO+f3B572bnYs4QHA4RHUEdyFs/Q0OZPy5rbTwhk4icRdC4zg45R4nAXipKKgifpcMapLLLrsMl8sFQGZmJmPGjGH9+vWICIWFhWW+59xzzyUsLIywsDCaNm1KamoqsbGx1Rm2Maa03Rvgx/+DNV9C0QHPSoHIds6Fu8elzq/5Zic464Jcf77XXQwH9kHubtifDvt3e17vcZ6Dw6HbhRDTB3zYWtDvksGx/IL3lnr16v3x+rHHHmPIkCHMmDGD5ORkBg8eXOZ7wsLC/njtcrkoKirydpjGmPLsTYafnoNlU5yLdq8rnYt2027QpAuE1j3yMYJcUC/KeTTp7PWQj5XfJYOaKjMzk5iYGADeffdd3wZjjKlY5naY/zwkTgZxwYBb4ZS7ndY8fsr6GVSTBx54gIcffpjevXvbr31jaqqcNJjzELzcGxLfg75jYfxSGPF3v04EUAvnQI6Pj9fSk9usXr2arl27+iii6hdo52vMUcnNgJQEyN7ptNqp19R5rt8UQuuV/54FL8HCiVCUD71Gw2kPQOM21Ru7F4nIYlUttx27V4uJRGQE8BLgAt5U1WdLbW8NTAIaefZ5SFVnezMmY4wfKSqA1JXOxX97AqQsgoxN5e8fUtfTtLOJkxzqRUNQCCyfBgU50OMyGPwQRLWvvnOoIbyWDETEBbwGDAdSgEUiMktVS/anfhSYpqqvi0g3YDYQ562YjDG1XGYKbFsI2xc7F/4dS6E439lWvznExkOfayG2HzRqDbl7ICfd04onzWnJk5PmLO/b5hznwD7odCYM+Ss0Ddw7bm/eGfQHNqjqJgARmQqMBEomAwUaeF43BHZ4MR5jTG1zYB8k/wyb5sLGuZCx0VkfHA4tekH/G50EENsPGsQc3jSzUevqj7mW8mYyiAG2lVhOAQaU2mcC8I2I3AHUA4aVdSARuQm4CaB1a/tyjfFbRQXOL/5Nc2HTj84vd3VDSD2IOwX6/QXanAzNuju9d02V8XXT0tHAu6r6goicDLwnIt1VD+2up6oTgYngVCD7IE5jzLEoLnI6aRXlQ1Feiec8KMz7c93eZCcBJC+Awv0gQRDTF069F9oNcX75B5c/4KM5ft5MBtuBViWWYz3rSroBGAGgqr+KSDgQDaR5MS5jjLfkpMOGb2HdV06xTv5RDA0T2d5pxdNuMMSdCnUaeStKUwZvJoNFQEcRaYuTBEYBV5baZytwBvCuiHQFwoF0L8bkFUOGDOGhhx7irLPO+mPdv//9b9auXcvrr79+2P6DBw/m+eeft9FKTe2nCqmrnIv/uq+cVj2oU5nbbSQ0agPBYU4Zf0i483xw+Y/ncKd1T8MYX59NQPNaMlDVIhEZB3yN02z0bVVdJSJPAQmqOgu4F3hDRO7GqUweq7Wt4wMwevRopk6dekgymDp1Ks8995wPozLGSwoPwOb5ngTwNWSlOOtb9oHBD0Ons6DFiT4dZ8ccPa/WGXj6DMwute7xEq+TgEHejKE6XHrppTz66KMUFBQQGhpKcnIyO3bsYMqUKdxzzz0cOHCASy+9lCeffNLXoRpzbFSd8fUXvQWrP3fqAULqQfshMPhB6HgmRDT3dZTmOPi6ArnqzXnIMytQFWreA85+ttzNkZGR9O/fnzlz5jBy5EimTp3K5ZdfziOPPEJkZCTFxcWcccYZLF++nJ49e1ZtbMZ4U3620yEr4W2nc1dYA6dcv8u50OYUp+jH+AX/SwY+crCo6GAyeOutt5g2bRoTJ06kqKiInTt3kpSUZMnA1A6pSZDwFiz7CAqynR9E578E3S+FsPq+js54gf8lgwp+wXvTyJEjufvuu0lMTCQ3N5fIyEief/55Fi1aROPGjRk7dix5eXk+ic2YSikqgNWznKKgrb+AKwy6XwzxNzgdu6wOwK/5XzLwkfr16zNkyBCuv/56Ro8eTVZWFvXq1aNhw4akpqYyZ86ccucwMMYnCvbD7vWQvhZ2LYflHznDNDSOg+FPQ6+rnDH4TUCwZFCFRo8ezUUXXcTUqVPp0qULvXv3pkuXLrRq1YpBg2p9PbmprXIznDl509c6j91rIX0dZG79c5+gYKcSuN8N0G4oBNno9oHGkkEVuvDCCynZMra8SWx+/PHH6gnIBJ7sXbA9EXYkOkM57FrpDNB2UHAdiO4ArfpDn2ucmbeiOztTNVoP34BmycCY2urAXtixxHPx9zxne8Z6FJczNWOnM53pGaM7Q5NO0LC1/eo3ZbJkYExtUZQPG3+ApJlOm/+S4/ZHdXAGcovp43T+at6jcvPzGuPhN8lAVZEAaO1QCztom+NRXATJ82DldKezV14m1GkMbQZB76udwdxa9LJxfMxx84tkEB4ezp49e4iKivLrhKCq7Nmzh/Bw6+jj19xu2PabkwBWfQa5uyE0ArqeB90vcQZys+GbTRXzi2QQGxtLSkoK6em1boy7oxYeHk5sbKyvwzBVTdWp9F35qfPI3uFU9nYe4SSADsOtt6/xKr9IBiEhIbRt29bXYRhz9NxuWDsb5j0HO5c58/F2HA7dn4ZOI6y3r6k2fpEMjKl13G5YPRPmPe+M+dO4LZz3LzjhIqdOwJhqZsnAmOrkLoZVM2DePyF9DUR1hIsmOkVBLvvvaHzH/vUZUx2Ki2DFxzD/edizAZp0hUvecu4Egly+js4YSwbGeFVRASyfCvNfcOb5bdYDLp8MXc63zl+mRrFkYIw35GdD4nvw238gc5vTF2DUFOh8to3+aWokSwbGVKWsnfD7fyHhHcjPdDqHnfui00LIkoCpwSwZGFMVUpPg11edWcG0GLpeAAPvcOYBMKYWsGRgzLFShc0/wS+vwIbvIKQuxF8PJ90KkdbvxdQulgyMOVrFhU7z0F9edubbrtcUhj7mJIK6kb6OzphjYsnAmMoqKoBlU5yWQfu2OMNCX/AK9LjchoowtZ4lA2OOpCgflrwPP//LaRnUsg+c/Q/oeJY1DzV+w6vJQERGAC8BLuBNVX221PZ/AUM8i3WBpqpqY/GamqEwDxInO0kgewfE9oPz/g0dzrCWQcbveC0ZiIgLeA0YDqQAi0RklqomHdxHVe8usf8dQG9vxWNMpRXkwuJ3YcFLkLMLWp8MF/7HGTrakoDxU968M+gPbFDVTQAiMhUYCSSVs/9o4AkvxmNMxQr2Q8LbsOBlZ97guFPhkjedGcQsCRg/581kEANsK7GcAgwoa0cRaQO0BX7wYjzGlG/r7zDtWudOoN1gOO1diBvk46CMqT41pQJ5FPCJqhaXtVFEbgJuAmjdunV1xmUCQeJ78OU90CAGrv8aWp/k64iMqXbebAqxHWhVYjnWs64so4Ap5R1IVSeqaryqxjdp0qQKQzQBrbgI5jwEs8ZBm4Fw4w+WCEzA8uadwSKgo4i0xUkCo4ArS+8kIl2AxsCvXozFmEPlZsAn18GmH+Gk22D40zafgAloXvvXr6pFIjIO+BqnaenbqrpKRJ4CElR1lmfXUcBUVVVvxWLMIdJWw5TRkLUdRr4Gva/2dUTGlEtV2bInl0XJGfRt05h2TbwzFapXfwqp6mxgdql1j5danuDNGIw5xNo5MP0vzjhCY7+EVv19HZExhygsdpO0I4tFyRkkJO8lYctedufkA/DXc7rWzmRgTI2h6gwj8cPfoMWJMOpDaBjj66iMITuvkMSt+1icnMGi5L0s3baPA4VOW5pWkXU4rWM08XGRxMc1poOXEgFYMjCBoCAXZt4Oqz6FHpc54wmF1PF1VMbP5BcVszFtP+tSs9mQlkN2XiEHCos5UOgmr7CYvMJiDhQUe9YVk1/o5kBhMftyC3ArBAmc0LIhV/RrRT/Pxb9Zg+ob88qSgfFfqrA9Eb64yxlddNiTMGi8dSAzx6XYrWzZ41z01+7KYV1qNmt2ZZG8J5dit1P16QoSIsKDqRPiItzzqBMSRJ1QFw3rhBAe6qJOiPOIqh9KfJtIerVuRP0w312SLRkY/+J2Q8oiWD0LkmZB5lYIawBXfgSdzvJ1dKaGySssZtWOLPbuLyC3sJjc/CJyC4rJLShif4HzS35/ftEf29Ky89mQlkN+kRtwfle0iaxLp2YRnNOjBZ2aRdC5eQRxUfUIDa5dgxhaMjC1n7sYtv7qXPxXz4LsneAKhfZDYfBD0OUcqNPY11GaGmBPTj6LtziVsouSM1i5PZPC4rIbMgYHCXVDXdQLC6ZuqIu6ocFE1Q9jYPsoOjWLoEvzBnRoWp86oa5qPgvvsGRgaqfiIkieD0kzYc0XsD8dgsOhwzDodiF0OhPCG/o6SuNDqsrm3ftJ2LKXhOQMErbsZVP6fgBCXUH0jG3I9ae0pW/rxjRvGP7HBb9eaDB1Ql217pf98bJkYGqfzfPg05udYaVD6jkX/m4jocNwCPNeawtTc7jdSkZuAenZ+aRl55OWlUd6Tj5pWfmk5+STnpXPxvQc9uwvAKBR3RDi2zTmsr6t6BfXmO4xDQkP8Y9f9FXFkoGpPdzFMO+f8NM/ILI9XPG+cydgLYP82q7MPBZv2cviLXtZum0v2/cdYHdOwR+VtSXVDwumaUQYTSLCGNy5KfFxjYlv05j2TeoTFGQNBypiycDUDtm7nM5iyfOh5yg49wW7C/BDRcVu1uzK/qNcP3GLc/EHCAt2inZO7diEphFhnot+OE0bhP2RAOqG2iXtWNlfztR8G76HT2+CwlwY+R/ofZWvIzIVyCss5vfNGfy2aQ8HCooJDhJcLnGeRXAFBRHsElx/LAsZ+ws8v/z/7HDVrEEY8W0inXL9No3p1qJBwJXjVydLBqbmKi6CH/8O81+EJl3gsnehaRdfR2VKUVXWpeYwb10689an8/vmDAqK3IS4hPAQF8VupcituD3PZXEFCV1bRHB5fCx94yLp26YxLRuGI9YnpNpYMjA1U+Z2mH6D02S0z7Uw4h8QWtfXURmPjP0FzF+fzvz1u5m/Pp3ULGfsnA5N63P1gDac2imak9pGHdbsUlVxq9Nxy0kSbtxuCAsJsgpdH7NkYGqedV/DjFuguAAufhN6XubriALavtwC1qXmsDY1m/Wp2SzZuo+VOzJRhYZ1QjilYzSndYzm1I5NaNmo4sp8EcElzp2AwxJATWHJwNQcRQXww1PwyyvQrIdTLBTdwddRBYysvELWp2Y7F/5d2axPc16nZ+f/sU/9sGC6tWjA3cM6cVqnJvSIaVjiwm5qM0sGxvfcbkia4YwomrEJ+v0FznwGQqpvkK5Ao6psTN/Pws0ZLNy8h0XJf7baAagT4qJjs/qc1rEJnZvXp2OzCDo1i7ByfD9mycD4jips/B6+exJ2LYem3eDKj51OZKZKud3K2tRsft+0h4XJGSzcnMHuHKdDVnT9MPq3bcxVJ7WmU1NnbJ2YRnWsXX6AsWRgfCMlAb6b4PQbaNQaLpoIPS6FICtDrgq5BUUk7chiydZ9/L45g0XJGWQeKASgZcNwTu3YhP5tIxnQNpK20fXs176xZGCqWdoa+OFpZzyhek3g7H9C37EQHOrryGqt/flFJO3MYkVKJiu3Z7JieyYb03M42IqzbXQ9RpzQ3Ln4t4sktrG1yjKHs2Rgqse+bfDjs7DsQ2c8oSGPwkm3Wi/io5RXWMzK7ZksL3Hh35Cew8EZxJtGhNEjpiHn9GhBj5iG9IxtSNNqnCDF1F6WDIx35WXCT8/BwomAwEm3wSn3QL0oX0dWK6Rm5ZHoGZdn8da9hwy5XPrC3yO2YbXOjGX8iyUD4x2qsGwqfPu4M7x0r6ucuQUatfJ1ZDVWyXF5Fm/ZS+LWvaTs/XNcnhNjG3HDKe3o07oRvVo1sl/8pkpZMjBVb+dymH0/bPsNYuLhqmnQsrevo6qRMnML+XZ1Kl+t3MUvG3eTW3DouDzXDbJxeUz1sGRgqs6BfTD3GVj0pjOz2AWvOncEQXYRK2l3Tj7frEplzsqd/LpxD0VupWXDcC7pE0u/tjYuj/ENSwbm+LndTsXwt0/AgQyIvwGG/tWmmixhZ+YBvlq5izkrd5GQnIFbIS6qLn85tR0jujfnxNiGdvE3PuXVZCAiI4CXcAYgeVNVny1jn8uBCYACy1T1Sm/GZKrYjqUw+z5nEvpWA+CcGdCip6+j8jmnh28O361O46uVu1i6bR8AnZrVZ9zQjpzdvTldmkdYAjA1hteSgYi4gNeA4UAKsEhEZqlqUol9OgIPA4NUda+INPVWPKaK5WU6PYcT3oZ60XDh686kMwFcJFRQ5Ob3zXv4fnUaP6xJY2tGLgA9Yhpy/1mdGdG9Oe2bWFNaUzN5886gP7BBVTcBiMhUYCSQVGKfG4HXVHUvgKqmeTEeU1Vy0uC9iyAtCQbcDIMfhjqNfB2VT6Rn5/PjWufiP29dOvsLigkLDuKUDtHcfHo7hnRuesSRPI2pCbyZDGKAbSWWU4ABpfbpBCAiC3CKkiao6lelDyQiNwE3AbRu3dorwZpKytwOky+ArB1w9XRoP9TXEVWrgxO5fLNqF9+vSWNZyj5UoXmDcEb2juGMLk0Z2D76sHH8janpfF2BHAx0BAYDscA8EemhqvtK7qSqE4GJAPHx8WVPlWS8L2MTTB7ptBq6+lNoc7KvI6oWqsqqHVnMWbmTOSt2sWn3fgBObNWIe4Z1YmjXpnRr0cDK/02tdsRkICLnA1+qqvsoj70dKNnDKNazrqQU4HdVLQQ2i8g6nOSw6Cg/y3hb+lonERTlwZhZft9vQFVZlpLJnBU7mb1yJ9syDuAKEk5q58zJe+YJzWgaYZ2+jP+ozJ3BFcC/RWQ68LaqrqnksRcBHUWkLU4SGAWUbin0GTAaeEdEonGKjTZV8vimuuxc5tQRBAXD2NnQrJuvI/IKt1tJ3LqX2St28dXKnezIzCM4SBjUIZpxQzowvFtzIuvZgHrGPx0xGajq1SLSAOei/a6IKPAOMEVVsyt4X5GIjAO+xqkPeFtVV4nIU0CCqs7ybDtTRJKAYuB+Vd1z/Kdlqsy2hfD+pRAW4dwRRLX3dURVKr+omF837uGbpFS+S0olLTufUFcQp3WK5t4zOzOsazMa1g3xdZjGeJ2oVq4IXkSigGuAu4DVQAfgZVV9xXvhHS4+Pl4TEhKq8yMD16afYMpoiGgG187ym3GFsvIKmbsmjW+SUvlpbTo5+UXUDXVxeqcmjOjenKFdmhIRbgnA+BcRWayq8eVtr0ydwQXAdTgX/8lAf1VNE5G6OM1EqzUZmGqy7mv46BrnTuCaGRDR3NcRHZedmQf4LimVb5JS+W3THgqLlej6YZx/YgvO7Nack9tHER5iLYBM4KpMncElwL9UdV7JlaqaKyI3eCcs41OrZsD0v0DzHk6robqRvo7omOTkFzF9cQqfJqawLCUTgHbR9ZwK4G7N6d2qkU3taIxHZZLBBGDnwQURqQM0U9VkVf3eW4EZH1nyAcwa5wwtceU0CG/g64iO2ubd+5n0SzKfLE4hJ7+IHjENeWBEZ87s1pwOTa0HsDFlqUwy+BgYWGK52LOun1ciMr6h6sxE9tOz0G4IjPoAQuv5OqpKc7uV+Rt28+6Czcxdm06ISzivZ0vGDIyjV6vA7B1tzNGoTDIIVtWCgwuqWiAi1r7OnxQegJm3w8rpzpDT5/0LgsN8HVWlHCwKmvRLMpt276dJRBh3D+vE6AGtrB+AMUehMskgXUQu8DQFRURGAru9G5apNtmpMPVK2L4Yhj0Jg8ZDLehJm7x7P5N+TebjBKcoqFerRrw0qhdnd29hk8AYcwwqkwxuAT4QkVcBwRlv6FqvRmWqx64V8OEoZw6CK96Hruf5OqIjWpeazSs/bOCL5TsIDhLO7dGCMQPj6N3a5k4w5nhUptPZRuAkEanvWc7xelTG+9bOgU9ugPCGcP1X0OJEX0dUoVU7Mnn1hw3MWbmLeqEubj6tPdcPirN5gI2pIpUaqE5EzgVOAMIPDsalqk95MS7jLarw66vwzWPQsheMmgINWvg6qnIt27aPV35Yz3er04gID+bOoR24blBbGtuwEMZUqcp0OvsvUBcYArwJXAos9HJcxhuKCuDLe2DJe9DtQmdCmtC6vo6qTAnJGbz8wwbmrUunUd0Q7hneiTED42hYx3oGG+MNlbkzGKiqPUVkuao+KSIvAHO8HZipYrkZMO1aSJ4Ppz3gTEhTw2YlU1V+25TBy9+v59dNe4iqF8qDI7pwzcltqB/m69HWjfFvlfkflud5zhWRlsAeoOaWK5jDZWyG9y+GzBS4+A3oebmvIzpEYbGbr1bu4q2fN7N02z6aRITx6LlduXJAa+qGWhIwpjpU5n/a5yLSCPgnkIgzcf0bXo3KVB13MXx6o3NnMOYLaF16sjnfyTxQyNSFW5n0SzI7MvOIi6rL0yNP4LL4VjZOkDHVrMJkICJBwPeemcemi8gXQLiqZlZLdOb4LXoLUhbBRRNrTCJI3r2fdxZs5uPFKeQWFHNyuyieGtmdoV2a2lhBxvhIhclAVd0i8hrQ27OcD+RXR2CmCmSmwPdPQvszfF40pGdEwqgAABimSURBVKr8vjmDt37ezHerUwkOEi44MYbrT4njhJYNfRqbMaZyxUTfi8glwKda2ckPjO+pwpf3grrhvBd91qvY7VY+X76DifM2sWpHFo3rhjBuSAeuOamN9REwpgapTDK4GbgHKBKRPJxeyKqqtW84y0Cyagas+wrO/Bs0jvNJCIu3ZPDk50ksT8mkQ9P6/N/FPbiod4zVBxhTA1WmB3JEdQRiqtCBvTDnQWjRCwbcWu0fv2PfAZ6ds4ZZy3bQrEEY/7riREaeGGP1AcbUYJXpdHZaWetLT3ZjapBvHoPcPXD1dHBVX9PMAwXF/G/eRv7700ZU4Y6hHbjl9PbUsz4CxtR4lflfen+J1+FAf2AxMNQrEZnjs3m+08N40Hho0bNaPlJV+Xz5Tp6dvZodmXmc26MFD53dhVaRNbN3szHmcJUpJjq/5LKItAL+7bWIzLErPACfj3fqCE5/qFo+ckVKJk9+voqELXs5oWUD/nVFLwa0i6qWzzbGVJ1juX9PAbpWdSCmCsz7J2RshGtnen3MoYz9BTw7ZzUfL04hsm4oz17cg8viW+GyegFjaqXK1Bm8gtPrGCAI6IXTE9nUJLtWwoKXnJnK2g326kct3baPW99fzO6cfG48tR3jhnagQbgNIGdMbVaZO4OEEq+LgCmqusBL8Zhj4S6GWXdAeCOnKakXTVm4lSdmrqJpgzBm3DaI7jHWYcwYf1CZZPAJkKeqxQAi4hKRuqqae6Q3isgI4CXABbypqs+W2j4WZ8yj7Z5Vr6rqm0cRvwFYOBF2JMIlb0HdSK98RF5hMU/MXMVHCds4rVMTXrqil80pYIwfqVQPZGAYcHCGszrAN8DAit4kIi7gNWA4Tj3DIhGZpapJpXb9SFXHHVXU5k/7tsL3T0OH4dD9Eq98RMreXG59P5EV2zO5c2gHxg/rZHUDxviZyiSD8JJTXapqjohUpnayP7BBVTcBiMhUYCRQOhmYY3VwyAnw2pAT89enc+eUJRS5lTevjWdYt2ZV/hnGGN+rzOwm+0Wkz8EFEekLHKjE+2KAbSWWUzzrSrtERJaLyCeeZquHEZGbRCRBRBLS09Mr8dEBYuV0WP8NnPEYNGpdpYdWVV6bu4Exby+kaUQ4s8adYonAGD9WmTuDu4CPRWQHzrhEzYErqujzP8epkM4XkZuBSZTRmU1VJwITAeLj422wPIDCPPjqYYjpC/1vqtJDZ+cVcu+0ZXyTlMoFJ7bk2Ut62CQzxvi5ynQ6WyQiXYDOnlVrVbWwEsfeDpT8pR/LnxXFB4+9p8Tim8BzlTiuAVj9OexPg4v/B0FVN/DbutRsbnlvMVsycnn8vG5cNygO8dGIp8aY6nPEYiIRuR2op6orVXUlUF9EbqvEsRcBHUWkrYiEAqOAWaWOXXL6zAuA1ZUPPcAlToJGbaDt4Co75Nw1aVz02gKy8or48C8DuP6UtpYIjAkQlakzuNEz0xkAqroXuPFIb1LVImAc8DXORX6aqq4SkadE5ALPbneKyCoRWQbcCYw92hMISHs2OhPb97mmyia1n/xrMjdMWkTbJvX44o5TbEgJYwJMZQqCXSIiBye28TQZrVQDc1WdDcwute7xEq8fBh6ufLgGgMTJIC7odfVxH6rYrfztyyTeWZDMsK7NeHl0L6sfMCYAVeZ//VfARyLyP8/yzcAc74VkKlRcCEs/hE5nQYMWR96/Avvzixg/dQnfrU7jhlPa8sg5Xa3/gDEBqjLJ4EHgJuAWz/JynBZFxhfWznEqjvuMOa7DpGblcf27i1i9M4unR57ANSfHVU18xphaqTKtidwi8jvQHrgciAamezswU47EyRDRAjoMO+ZDJO3I4oZJi8g6UMhbY/oxpEvTKgzQGFMblZsMRKQTMNrz2A18BKCqQ6onNHOYfdtgw3dw2n3HPIPZ3DVpjPswkQZ1Qvj4loF0a2lTWRtjKr4zWAPMB85T1Q0AInJ3tURlyrbkfee59zXH9Pb3fk3miVmr6NayAW+N6UezBuFVF5sxplarKBlcjNM3YK6IfAVMxemBbHzBXewkg/ZDoHGbo3prsVt55svVvL1gs7UYMsaUqdxG6qr6maqOAroAc3GGpWgqIq+LyJnVFaDx2PA9ZKUcdcVxsVu5Z9pS3l6wmesHteV/1/S1RGCMOcwReyyp6n5V/dAzF3IssASnhZGpTomToG40dD6n0m8pdiv3f7yMmUt38OCILjx+fjdrOmqMKdNRdV9V1b2qOlFVz/BWQKYM2alOk9JeoyG4chPKuN3KQ9OX8+mS7dx3ZiduHdzey0EaY2qzqhnLwHjX0g9AiytdROR2K4/MWMHHi1O4a1hHxg3t6OUAjTG1nSWDms7tdvoWtBkE0Ue+qKsqj89aydRF27hjaAfGn2GJwBhzZJYMarrk+bB3c6XuClSVJz9P4v3ftnLL6e25Z3gnG3XUGFMplgxqusRJEN4Qul1Q4W6qytNfrObdX5K58dS2PDiisyUCY0ylWTKoyXIznElsel4BIXXK3U1VeXbOGt5esJnrBsXxyDldLREYY46KJYOabNlUKC6osIhIVfnn12v537xNXHNSGx4/r5slAmPMUbNkUFOpOkVEMX2hefdyd/vXd+v5z48bGd2/NU9ecIIlAmPMMbFkUFNtWwjpayq8K3jl+/W8/P16Lo+P5ZkLuxNkHcqMMcfIkkFNlTgJQutD90vK3PzVyp288O06Lu4dw7MX97REYIw5LpYMaqK8TFg1A7pfDGH1D9u8Zc9+7v94OSfGNuTZSywRGGOOnyWDmmjFJ1CYC33GHrYpr7CY2z9MRARevbIPocH2FRpjjp8NX1kTJU6CZt0hps9hm/72ZRIrt2fxxrXxtIqs64PgjDH+yH5W1jTbE2HnMqfiuFTLoFnLdvD+b1u56bR2DO/WzEcBGmP8kSWDmsLthoR3YPKFENYQel52yOaN6Tk8PH05fds05v6zOvsoSGOMv7Jioppg93r4fDxsWQBtToHzX4I6jf/YnFdYzO0fJBIaHMQro3sT4rIcboypWl69qojICBFZKyIbROShCva7RERUROK9GU+NU1QAP/0TXh8IqSvhgldg7BcQ3eGQ3SbMWsWaXdm8eEUvWjYqf1gKY4w5Vl67MxARF/AaMBxIARaJyCxVTSq1XwQwHvjdW7HUSNsWwed3QloSnHARjPgHRBxeD/BpYgpTF23j9iHtGdK5qQ8CNcYEAm/eGfQHNqjqJlUtAKYCI8vY72ngH0CeF2OpOfKzYfb98NZwpz/B6Klw2btlJoL1qdn8dcZKBrSN5O5hnao/VmNMwPBmMogBtpVYTvGs+4OI9AFaqeqXFR1IRG4SkQQRSUhPT6/6SKvL2jnw2gBY+Ab0vwlu/x06n13mrrkFRdz2QSL1wly8PLo3wVZPYIzxIp9VIItIEPAiMPZI+6rqRGAiQHx8vHo3Mi9Qhc9ug2UfQpOucMMkaNWvgt2VRz9byYb0HN6/YQDNGoRXY7DGmEDkzWSwHWhVYjnWs+6gCKA78KNnpM3mwCwRuUBVE7wYV/Vb9amTCAbeCUMfO+Kk9h8npPBp4nbGn9GRQR2iqylIY0wg82bZwyKgo4i0FZFQYBQw6+BGVc1U1WhVjVPVOOA3wP8SQWEefDsBmvWAYROOmAjW7MrisZkrGdQhijtt/mJjTDXx2p2BqhaJyDjga8AFvK2qq0TkKSBBVWdVfAQ/8dtrkLkVRs6CIFeFuxYVu7n/4+VEhAfz7yt647IB6Iwx1cSrdQaqOhuYXWrd4+XsO9ibsfhEdirMfxE6nwPtTj/i7u/+ksyK7Zm8emVvmkSEVUOAxhjjsCYq3jT3b1CUB8OfPuKu2zJyeeGbdZzRpSnn9mhRDcEZY8yfLBl4y64VkPie04S0VI/i0g62HgoSeOrC7jZ1pTGm2lky8AZV+PoRqNMITn/giLvPWraDn9alc99ZnYmx4SaMMT5gycAb1s6BzfNg8MOHDDhXlr37C3jq8yRObNWIa0+Oq574jDGmFBu1tKoVFcA3j0J0J4i//oi7PzN7NZkHCnn/4h7WesgY4zOWDKraojchYyNcOQ1cIRXuumDDbj5ZnMJtg9vTtUWDagrQGGMOZ8VEVSk3A356FtoNgY5nVrhrXmExj8xYQVxUXetcZozxObszqEo/PuuMSnrW3w+bsrK0l75fz5Y9uXx44wDCQyrujGaMMd5mdwZVJX2dU0TUZww061bhrkk7spg4bxOX9Y1lYHsbe8gY43uWDKrKt49BaD0Y8tcKdyt2Kw9/upzGdUP467ldqyk4Y4ypmCWDqrBxLqz7Ck69F+o3qXDXSb8ksywlk8fPP4FGdSsetM4YY6qLJYPj5S6Gr/8KjdrASbdWuGvK3lye/2YtQzo34fyeNuSEMabmsArk45U4GdJWwWWTILj8weVUlcc+WwnA0zbkhDGmhrE7g+ORlwVzn4HWA6FbWdM7/+mL5TuZuzade8/sTGzjutUUoDHGVI7dGRyPuX+H/elOB7MKfunv2HeACbNW0TO2IWMHxlVffMYYU0l2Z3CskhfA7/91RiWN6VPubnmFxdz83mLyi9y8ePmJNuSEMaZGsjuDY1GwH2beBo3bOFNZlkNVeXD6clbuyOSNa+Lp0DSi2kI0xpijYcngWHz3JOxNhrGznb4F5Zg4bxMzl+7g/rM6M6xbs+qLzxhjjpIVEx2tzfNh4f9gwC0QN6jc3eauTePZr9ZwXs8W3Da4fTUGaIwxR8+SwdHIz4GZt0NkOzijzKmcAdiYnsOdU5bQtXkDnru0pzUjNcbUeFZMdDS+ewL2bYXr5pRbPJSVV8iNkxMIdQXxxph46oban9gYU/PZnUFlbfrJGYjupNugzcll7lLsVsZPWcLWPbm8fnVfm8LSGFNr2M/WysjPhpnjILI9DH203N3++fVa5q5N55mLutO/bWQ1BmiMMcfHkkFlfPs4ZG6D67+G0LJ7D89cup3//rSRqwa05qoBbao5QGOMOT5eLSYSkREislZENojIQ2Vsv0VEVojIUhH5WUQqngjAFzbOhYS34eTbofWAMndZkZLJA58sp3/bSJ44/4RqDtAYY46f15KBiLiA14CzgW7A6DIu9h+qag9V7QU8B7zorXiOSV4WzLoDojqWWzyUnp3PTe8lEF0/jP9c1YfQYKuGMcbUPt68cvUHNqjqJlUtAKYCh4zmpqpZJRbrAerFeI7eN49C1na48HUIObwyOL+omFvfX8ze3AImXtuX6Prlj1pqjDE1mTfrDGKAbSWWU4DDyllE5HbgHiAUGFrWgUTkJuAmgNatW1d5oGXa8D0kToJB46FVv8M2F7uV+z5eTsKWvbx6ZW9OaNmweuIyxhgv8HmZhqq+pqrtgQeBMstiVHWiqsaranyTJhXPJFYl8jKd4qHozjD4kcM2u93KA58s5/NlO3jo7C6c17Ol92Myxhgv8mYy2A60KrEc61lXnqnAhV6Mp/K+egSyd3qKh8IP2aSqPDZzJdMTU7hrWEduOd2GmjDG1H7eTAaLgI4i0lZEQoFRwKySO4hIxxKL5wLrvRhP5cx/AZa+D6fcDbF9D9mkqjz9xWo++H0rt5zenvFndCznIMYYU7t4rc5AVYtEZBzwNeAC3lbVVSLyFJCgqrOAcSIyDCgE9gJjvBVPpSx8A75/CnpcBkMOLbFSVf759VreXrCZsQPjeHBEZxtzyBjjN7za6UxVZwOzS617vMTr8d78/KOy7COYfR90PscpHgo69KbplR828J8fNzK6f2ueOL+bJQJjjF/xeQVyjbD6C/jsVog7FS59B1whh2yeOG8jL367jov7xPCMTWZvjPFDlgw2zoVProOWvWH0lMMqjCf9kszfZ6/h3J4teO6SngTZtJXGGD8U2Mlg20KYepXTw/iqjyHs0Gkppy7cyhOzVjG8WzP+fUUvgl2B/ecyxvivwL267VoBH1wKEc3gmhlQ99BRRmcsSeHhGSs4vVMTXr2yNyGWCIwxfiwwr3B7NsJ7F0Fofbh2ppMQSpi9Yif3TlvGSW2j+N81fQkLdvkoUGOMqR6Blwz2bYPJI0EVrvkMGh06vMWcFTu5c8oS+rRuzJtj4gkPsURgjPF/gTWfQU4avHehMxrp2M+hSadDNs9ZsZNxU5ZwYmxD3rmuH/XCAuvPY4wJXIFztTuwD967GLJ2OHUELU48ZPOXy3dy59Ql9GrViHev60dEeEg5BzLGGP8TOMng11dh91oYPRVan3TIpi+W72D81KX0btWId6/vT327IzDGBJjAueqd/iB0GgGx8Yes/nzZDu76aCl9WjfinessERhjAlPgVCC7Qg5LBLM8iaBv68a8a4nAGBPAAicZlDJz6XbumrqEvm0aW2WxMSbgBeQVcObS7dz90VL6xUXyznX9qBsakH8GY4z5Q8DdGcxYksLdHy2lf1tLBMYYc1BAXQlnLEnh3mnLGNA2irfH9qNOqHUoM8YYCKA7g8+WbOeeacs4qZ0lAmOMKS1gkkFM4zoM79qMt8ZYIjDGmNICppioX1wk/eIij7yjMcYEoIC5MzDGGFM+SwbGGGMsGRhjjLFkYIwxBksGxhhjsGRgjDEGSwbGGGOwZGCMMQYQVfV1DEdFRNKBLcf49mhgdxWGUxP42zn52/mA/52Tv50P+N85lXU+bVS1SXlvqHXJ4HiISIKqxh95z9rD387J384H/O+c/O18wP/O6VjOx4qJjDHGWDIwxhgTeMlgoq8D8AJ/Oyd/Ox/wv3Pyt/MB/zunoz6fgKozMMYYU7ZAuzMwxhhTBksGxhhjAicZiMgIEVkrIhtE5CFfx3O8RCRZRFaIyFIRSfB1PMdCRN4WkTQRWVliXaSIfCsi6z3PjX0Z49Eo53wmiMh2z/e0VETO8WWMR0tEWonIXBFJEpFVIjLes75Wfk8VnE+t/Z5EJFxEForIMs85PelZ31ZEfvdc8z4SkdAKjxMIdQYi4gLWAcOBFGARMFpVk3wa2HEQkWQgXlVrbUcZETkNyAEmq2p3z7rngAxVfdaTtBur6oO+jLOyyjmfCUCOqj7vy9iOlYi0AFqoaqKIRACLgQuBsdTC76mC87mcWvo9iYgA9VQ1R0RCgJ+B8cA9wKeqOlVE/gssU9XXyztOoNwZ9Ac2qOomVS0ApgIjfRxTwFPVeUBGqdUjgUme15Nw/qPWCuWcT62mqjtVNdHzOhtYDcRQS7+nCs6n1lJHjmcxxPNQYCjwiWf9Eb+jQEkGMcC2Essp1PJ/ADhf9jcislhEbvJ1MFWomaru9LzeBTTzZTBVZJyILPcUI9WK4pSyiEgc0Bv4HT/4nkqdD9Ti70lEXCKyFEgDvgU2AvtUtcizyxGveYGSDPzRKaraBzgbuN1TROFX1CnDrO3lmK8D7YFewE7gBd+Gc2xEpD4wHbhLVbNKbquN31MZ51OrvydVLVbVXkAsTklIl6M9RqAkg+1AqxLLsZ51tZaqbvc8pwEzcP4B+INUT7nuwfLdNB/Hc1xUNdXzH9UNvEEt/J485dDTgQ9U9VPP6lr7PZV1Pv7wPQGo6j5gLnAy0EhEgj2bjnjNC5RksAjo6KldDwVGAbN8HNMxE5F6nsovRKQecCawsuJ31RqzgDGe12OAmT6M5bgdvGB6XEQt+548lZNvAatV9cUSm2rl91Te+dTm70lEmohII8/rOjgNZVbjJIVLPbsd8TsKiNZEAJ6mYv8GXMDbqvqMj0M6ZiLSDuduACAY+LA2no+ITAEG4wy3mwo8AXwGTANa4wxVfrmq1opK2XLOZzBO0YMCycDNJcraazwROQWYD6wA3J7Vj+CUs9e676mC8xlNLf2eRKQnTgWxC+cH/jRVfcpznZgKRAJLgKtVNb/c4wRKMjDGGFO+QCkmMsYYUwFLBsYYYywZGGOMsWRgjDEGSwbGGGOwZGDMYUSkuMTolUurcpRbEYkrOaqpMTVF8JF3MSbgHPB07TcmYNidgTGV5JlD4jnPPBILRaSDZ32ciPzgGeTsexFp7VnfTERmeMaZXyYiAz2HconIG56x57/x9Bo1xqcsGRhzuDqliomuKLEtU1V7AK/i9GgHeAWYpKo9gQ+Alz3rXwZ+UtUTgT7AKs/6jsBrqnoCsA+4xMvnY8wRWQ9kY0oRkRxVrV/G+mRgqKpu8gx2tktVo0RkN86EKYWe9TtVNVpE0oHYkkMAeIZN/lZVO3qWHwRCVPVv3j8zY8pndwbGHB0t5/XRKDk+TDFWd2dqAEsGxhydK0o8/+p5/QvOSLgAV+EMhAbwPXAr/DH5SMPqCtKYo2W/SIw5XB3PrFEHfaWqB5uXNhaR5Ti/7kd71t0BvCMi9wPpwHWe9eOBiSJyA84dwK04E6cYU+NYnYExleSpM4hX1d2+jsWYqmbFRMYYY+zOwBhjjN0ZGGOMwZKBMcYYLBkYY4zBkoExxhgsGRhjjAH+Hx8Q4v0ISd32AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7W8EPJTKnG_f",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "fa525d1b-14cb-4509-f5ed-10eb0d865a58"
      },
      "source": [
        "\n",
        "score = model_2lstm.evaluate(test_a_b, test_y_b, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.5319803232267428\n",
            "Test accuracy: 0.8332651853561401\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HZrW070WnSMn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_confusion = confusion_matrix(model_2lstm, test_a_b, test_y_b)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nJp2zqPxnWEA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "b2989acd-358a-4bfd-8296-78dc3444d17e"
      },
      "source": [
        "df_confusion"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>col_0</th>\n",
              "      <th>none</th>\n",
              "      <th>mild</th>\n",
              "      <th>moderate</th>\n",
              "      <th>moderately severe</th>\n",
              "      <th>severe</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>row_0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>none</th>\n",
              "      <td>2999</td>\n",
              "      <td>168</td>\n",
              "      <td>160</td>\n",
              "      <td>136</td>\n",
              "      <td>93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mild</th>\n",
              "      <td>181</td>\n",
              "      <td>2458</td>\n",
              "      <td>99</td>\n",
              "      <td>125</td>\n",
              "      <td>96</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>moderate</th>\n",
              "      <td>153</td>\n",
              "      <td>124</td>\n",
              "      <td>2477</td>\n",
              "      <td>126</td>\n",
              "      <td>77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>moderately severe</th>\n",
              "      <td>143</td>\n",
              "      <td>117</td>\n",
              "      <td>124</td>\n",
              "      <td>2334</td>\n",
              "      <td>91</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>severe</th>\n",
              "      <td>99</td>\n",
              "      <td>95</td>\n",
              "      <td>98</td>\n",
              "      <td>141</td>\n",
              "      <td>1956</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "col_0              none  mild  moderate  moderately severe  severe\n",
              "row_0                                                             \n",
              "none               2999   168       160                136      93\n",
              "mild                181  2458        99                125      96\n",
              "moderate            153   124      2477                126      77\n",
              "moderately severe   143   117       124               2334      91\n",
              "severe               99    95        98                141    1956"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5xERqORunZjd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#model_2lstm.save('/content/drive/My Drive/Depression_detect/model_glove_2lstm_b.h5')\n",
        "\n",
        "#with open('/content/drive/My Drive/Depression_detect/model_glove_2lstm_b_hist.json', 'w') as f:\n",
        "    #json.dump(model_glove_2lstm_b_hist.history, f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pn0OlHL_ndKh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "2b9ec1a5-7a68-453b-cfe5-3d39b3928e9a"
      },
      "source": [
        "sen = \"All is going right with the party, I'm happy to know new people\"\n",
        "test_model(sen, model_2lstm)\n",
        "sen = \"I want an ice cream and have some fries for lunch\"\n",
        "test_model(sen, model_2lstm)\n",
        "sen = \"I'm afraid of losing my work, I don't have any money\"\n",
        "test_model(sen, model_2lstm)\n",
        "sen = \"I'm worried about my future, I'm afraid of it\"\n",
        "test_model(sen, model_2lstm)\n",
        "sen = \"My father loves me\"\n",
        "test_model(sen, model_2lstm)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All is going right with the party, I'm happy to know new people\n",
            "[[90, 99, 563, 1, 18, 128, 14, 136, 39, 0]]\n",
            "moderately severe\n",
            "I want an ice cream and have some fries for lunch\n",
            "[[193, 2438, 2825, 6594, 4186, 0, 0, 0, 0, 0]]\n",
            "moderate\n",
            "I'm afraid of losing my work, I don't have any money\n",
            "[[1, 18, 780, 908, 108, 211, 0, 0, 0, 0]]\n",
            "none\n",
            "I'm worried about my future, I'm afraid of it\n",
            "[[1, 18, 1070, 566, 1, 18, 780, 0, 0, 0]]\n",
            "severe\n",
            "My father loves me\n",
            "[[339, 119, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
            "none\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aS-hZc7UPiuB",
        "colab_type": "text"
      },
      "source": [
        "## MODEL 3 Bidirectional LSTM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TFniIiuAn6L1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "outputId": "941927aa-9ab0-4f76-f05e-335c3d12ca55"
      },
      "source": [
        "# main model\n",
        "answer_inp = Input(shape=(windows_size, ))\n",
        "embedding_size_glove = 100\n",
        "answer_emb1 = Embedding(vocab_size_stop+1, embedding_size_glove, weights=[embedding_matrix_lp], input_length=windows_size, trainable=False)(answer_inp)\n",
        "\n",
        "\n",
        "bi_lstm =  Bidirectional (LSTM (embedding_size_glove,return_sequences=True,dropout=0.50),merge_mode='concat')(answer_emb1)\n",
        "model_bi1 = TimeDistributed(Dense(embedding_size_glove,activation='relu'))(bi_lstm) #TimeDistributed method is used to apply a Dense layer to each of the time-steps independently. We used Dropout and l2_reg regularizers to reduce overfitting.\n",
        "model_bi2 = Flatten()(model_bi1)\n",
        "model_bi3 = Dense(256,activation='relu')(model_bi2)\n",
        "output = Dense(5,activation='softmax')(model_bi3)\n",
        "model_bi = Model(answer_inp,output)\n",
        "model_bi.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy'])\n",
        "model_bi.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_3 (InputLayer)         (None, 10)                0         \n",
            "_________________________________________________________________\n",
            "embedding_3 (Embedding)      (None, 10, 100)           745000    \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, 10, 200)           160800    \n",
            "_________________________________________________________________\n",
            "time_distributed_1 (TimeDist (None, 10, 100)           20100     \n",
            "_________________________________________________________________\n",
            "flatten_2 (Flatten)          (None, 1000)              0         \n",
            "_________________________________________________________________\n",
            "dense_7 (Dense)              (None, 256)               256256    \n",
            "_________________________________________________________________\n",
            "dense_8 (Dense)              (None, 5)                 1285      \n",
            "=================================================================\n",
            "Total params: 1,183,441\n",
            "Trainable params: 438,441\n",
            "Non-trainable params: 745,000\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C2534Gm8j_jc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 782
        },
        "outputId": "01bf120f-2705-4b20-9c5b-0993e5c0e4fe"
      },
      "source": [
        "model_glove_bilstm = model_bi.fit(train_a_b, train_y_b, validation_data=(dev_a_b, dev_y_b), epochs=30, batch_size=64, shuffle=True, callbacks=[early_stopping])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 64179 samples, validate on 12836 samples\n",
            "Epoch 1/30\n",
            "64179/64179 [==============================] - 39s 604us/step - loss: 1.5946 - accuracy: 0.2473 - val_loss: 1.5790 - val_accuracy: 0.2725\n",
            "Epoch 2/30\n",
            "64179/64179 [==============================] - 38s 589us/step - loss: 1.5737 - accuracy: 0.2722 - val_loss: 1.5499 - val_accuracy: 0.2969\n",
            "Epoch 3/30\n",
            "64179/64179 [==============================] - 38s 586us/step - loss: 1.5495 - accuracy: 0.2983 - val_loss: 1.5145 - val_accuracy: 0.3278\n",
            "Epoch 4/30\n",
            "64179/64179 [==============================] - 37s 582us/step - loss: 1.5102 - accuracy: 0.3291 - val_loss: 1.4509 - val_accuracy: 0.3706\n",
            "Epoch 5/30\n",
            "64179/64179 [==============================] - 37s 580us/step - loss: 1.4489 - accuracy: 0.3710 - val_loss: 1.3726 - val_accuracy: 0.4159\n",
            "Epoch 6/30\n",
            "64179/64179 [==============================] - 37s 582us/step - loss: 1.3907 - accuracy: 0.4070 - val_loss: 1.2974 - val_accuracy: 0.4655\n",
            "Epoch 7/30\n",
            "64179/64179 [==============================] - 37s 581us/step - loss: 1.3334 - accuracy: 0.4392 - val_loss: 1.2182 - val_accuracy: 0.5090\n",
            "Epoch 8/30\n",
            "64179/64179 [==============================] - 41s 644us/step - loss: 1.2842 - accuracy: 0.4661 - val_loss: 1.1656 - val_accuracy: 0.5315\n",
            "Epoch 9/30\n",
            "64179/64179 [==============================] - 37s 584us/step - loss: 1.2366 - accuracy: 0.4914 - val_loss: 1.1339 - val_accuracy: 0.5538\n",
            "Epoch 10/30\n",
            "64179/64179 [==============================] - 37s 580us/step - loss: 1.1949 - accuracy: 0.5109 - val_loss: 1.0821 - val_accuracy: 0.5691\n",
            "Epoch 11/30\n",
            "64179/64179 [==============================] - 37s 577us/step - loss: 1.1588 - accuracy: 0.5315 - val_loss: 1.0442 - val_accuracy: 0.5888\n",
            "Epoch 12/30\n",
            "64179/64179 [==============================] - 37s 577us/step - loss: 1.1152 - accuracy: 0.5497 - val_loss: 1.0011 - val_accuracy: 0.6110\n",
            "Epoch 13/30\n",
            "64179/64179 [==============================] - 37s 576us/step - loss: 1.0818 - accuracy: 0.5649 - val_loss: 0.9906 - val_accuracy: 0.6095\n",
            "Epoch 14/30\n",
            "64179/64179 [==============================] - 37s 576us/step - loss: 1.0470 - accuracy: 0.5832 - val_loss: 0.9568 - val_accuracy: 0.6277\n",
            "Epoch 15/30\n",
            "64179/64179 [==============================] - 37s 576us/step - loss: 1.0107 - accuracy: 0.5993 - val_loss: 0.9447 - val_accuracy: 0.6313\n",
            "Epoch 16/30\n",
            "64179/64179 [==============================] - 37s 575us/step - loss: 0.9849 - accuracy: 0.6109 - val_loss: 0.9543 - val_accuracy: 0.6242\n",
            "Epoch 17/30\n",
            "64179/64179 [==============================] - 37s 577us/step - loss: 0.9522 - accuracy: 0.6270 - val_loss: 0.9146 - val_accuracy: 0.6453\n",
            "Epoch 18/30\n",
            "64179/64179 [==============================] - 38s 585us/step - loss: 0.9271 - accuracy: 0.6349 - val_loss: 0.8917 - val_accuracy: 0.6539\n",
            "Epoch 19/30\n",
            "64179/64179 [==============================] - 38s 594us/step - loss: 0.8984 - accuracy: 0.6495 - val_loss: 0.8881 - val_accuracy: 0.6582\n",
            "Epoch 20/30\n",
            "64179/64179 [==============================] - 38s 592us/step - loss: 0.8730 - accuracy: 0.6637 - val_loss: 0.8903 - val_accuracy: 0.6627\n",
            "Epoch 21/30\n",
            "64179/64179 [==============================] - 38s 594us/step - loss: 0.8517 - accuracy: 0.6695 - val_loss: 0.9022 - val_accuracy: 0.6551\n",
            "Epoch 22/30\n",
            "64179/64179 [==============================] - 38s 592us/step - loss: 0.8262 - accuracy: 0.6810 - val_loss: 0.9030 - val_accuracy: 0.6602\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZvjtNqpmK4d",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "45e1ec23-11d9-4a4a-96aa-92bb3247ca1c"
      },
      "source": [
        "# evaluate the model\n",
        "score = model_bi.evaluate(test_a_b, test_y_b, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.9042176990014968\n",
            "Test accuracy: 0.6584185361862183\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lBY2rOdrrKvh",
        "colab_type": "text"
      },
      "source": [
        "### Model 4 Using BiGRU "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RV8Vo0eLsdor",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import roc_auc_score\n",
        "\n",
        "from keras.models import Model\n",
        "from keras.layers import Input, Dense, Embedding, SpatialDropout1D, concatenate\n",
        "from keras.layers import GRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D\n",
        "from keras.preprocessing import text, sequence\n",
        "from keras.callbacks import Callback"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DIYFcmY1ubXy",
        "colab_type": "text"
      },
      "source": [
        "### AUC - ROC curve is a performance measurement for classification problem at various thresholds settings."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lalWggZmsPw8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class RocAucEvaluation(Callback):\n",
        "    def __init__(self, validation_data=(), interval=1):\n",
        "        super(Callback, self).__init__()\n",
        "\n",
        "        self.interval = interval\n",
        "        self.X_val, self.y_val = validation_data\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs={}):\n",
        "        if epoch % self.interval == 0:\n",
        "            y_pred = self.model.predict(self.X_val, verbose=0)\n",
        "            score = roc_auc_score(self.y_val, y_pred)\n",
        "            print(\"\\n ROC-AUC - epoch: %d - score: %.6f \\n\" % (epoch+1, score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9ZaaxKlqw-m",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        },
        "outputId": "908b0a5b-7c0d-4e59-bc01-c61042a785d9"
      },
      "source": [
        "# main model\n",
        "answer_inp = Input(shape=(windows_size, ))\n",
        "embedding_size_glove = 100\n",
        "answer_emb1 = Embedding(vocab_size_stop+1, embedding_size_glove, weights=[embedding_matrix_lp], input_length=windows_size, trainable=False)(answer_inp)\n",
        "\n",
        "x = SpatialDropout1D(0.2)(answer_emb1)\n",
        "x = Bidirectional(GRU(embedding_size_glove, return_sequences=True))(x)\n",
        "avg_pool = GlobalAveragePooling1D()(x)\n",
        "max_pool = GlobalMaxPooling1D()(x)\n",
        "conc = concatenate([avg_pool, max_pool])\n",
        "outp = Dense(5, activation=\"softmax\")(conc)\n",
        "    \n",
        "model_gru = Model(inputs=answer_inp, outputs=outp)\n",
        "model_gru.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
        "model_gru.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_4\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_4 (InputLayer)            (None, 10)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_4 (Embedding)         (None, 10, 100)      745000      input_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout1d_1 (SpatialDro (None, 10, 100)      0           embedding_4[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_2 (Bidirectional) (None, 10, 200)      120600      spatial_dropout1d_1[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_1 (Glo (None, 200)          0           bidirectional_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_1 (GlobalM (None, 200)          0           bidirectional_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 400)          0           global_average_pooling1d_1[0][0] \n",
            "                                                                 global_max_pooling1d_1[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "dense_9 (Dense)                 (None, 5)            2005        concatenate_1[0][0]              \n",
            "==================================================================================================\n",
            "Total params: 867,605\n",
            "Trainable params: 122,605\n",
            "Non-trainable params: 745,000\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t3Owc4aZsmCh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 64\n",
        "epochs = 30\n",
        "RocAuc = RocAucEvaluation(validation_data=(dev_a,dev_y), interval=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2LsTI5mCs788",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d1dcc039-deca-49b0-853e-bfdf765b91b6"
      },
      "source": [
        "hist = model_gru.fit(train_a, train_y, batch_size=batch_size, epochs=epochs, validation_data=(dev_a, dev_y),\n",
        "                 callbacks=[RocAuc], verbose=2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 98795 samples, validate on 19760 samples\n",
            "Epoch 1/30\n",
            " - 57s - loss: 1.4763 - accuracy: 0.3617 - val_loss: 1.4134 - val_accuracy: 0.4016\n",
            "\n",
            " ROC-AUC - epoch: 1 - score: 0.673696 \n",
            "\n",
            "Epoch 2/30\n",
            " - 53s - loss: 1.3791 - accuracy: 0.4203 - val_loss: 1.2791 - val_accuracy: 0.4897\n",
            "\n",
            " ROC-AUC - epoch: 2 - score: 0.764619 \n",
            "\n",
            "Epoch 3/30\n",
            " - 53s - loss: 1.2513 - accuracy: 0.4923 - val_loss: 1.1299 - val_accuracy: 0.5592\n",
            "\n",
            " ROC-AUC - epoch: 3 - score: 0.828097 \n",
            "\n",
            "Epoch 4/30\n",
            " - 53s - loss: 1.1308 - accuracy: 0.5519 - val_loss: 1.0039 - val_accuracy: 0.6262\n",
            "\n",
            " ROC-AUC - epoch: 4 - score: 0.870184 \n",
            "\n",
            "Epoch 5/30\n",
            " - 53s - loss: 1.0317 - accuracy: 0.5975 - val_loss: 0.9016 - val_accuracy: 0.6695\n",
            "\n",
            " ROC-AUC - epoch: 5 - score: 0.897007 \n",
            "\n",
            "Epoch 6/30\n",
            " - 53s - loss: 0.9500 - accuracy: 0.6332 - val_loss: 0.8216 - val_accuracy: 0.6997\n",
            "\n",
            " ROC-AUC - epoch: 6 - score: 0.915946 \n",
            "\n",
            "Epoch 7/30\n",
            " - 53s - loss: 0.8877 - accuracy: 0.6596 - val_loss: 0.7375 - val_accuracy: 0.7352\n",
            "\n",
            " ROC-AUC - epoch: 7 - score: 0.932660 \n",
            "\n",
            "Epoch 8/30\n",
            " - 54s - loss: 0.8355 - accuracy: 0.6804 - val_loss: 0.6652 - val_accuracy: 0.7671\n",
            "\n",
            " ROC-AUC - epoch: 8 - score: 0.943945 \n",
            "\n",
            "Epoch 9/30\n",
            " - 53s - loss: 0.7870 - accuracy: 0.7005 - val_loss: 0.6388 - val_accuracy: 0.7739\n",
            "\n",
            " ROC-AUC - epoch: 9 - score: 0.949937 \n",
            "\n",
            "Epoch 10/30\n",
            " - 54s - loss: 0.7567 - accuracy: 0.7126 - val_loss: 0.6537 - val_accuracy: 0.7647\n",
            "\n",
            " ROC-AUC - epoch: 10 - score: 0.952845 \n",
            "\n",
            "Epoch 11/30\n",
            " - 54s - loss: 0.7184 - accuracy: 0.7291 - val_loss: 0.5599 - val_accuracy: 0.8013\n",
            "\n",
            " ROC-AUC - epoch: 11 - score: 0.961253 \n",
            "\n",
            "Epoch 12/30\n",
            " - 58s - loss: 0.6971 - accuracy: 0.7366 - val_loss: 0.5329 - val_accuracy: 0.8134\n",
            "\n",
            " ROC-AUC - epoch: 12 - score: 0.965153 \n",
            "\n",
            "Epoch 13/30\n",
            " - 54s - loss: 0.6742 - accuracy: 0.7448 - val_loss: 0.4922 - val_accuracy: 0.8282\n",
            "\n",
            " ROC-AUC - epoch: 13 - score: 0.969153 \n",
            "\n",
            "Epoch 14/30\n",
            " - 54s - loss: 0.6528 - accuracy: 0.7525 - val_loss: 0.4867 - val_accuracy: 0.8303\n",
            "\n",
            " ROC-AUC - epoch: 14 - score: 0.970684 \n",
            "\n",
            "Epoch 15/30\n",
            " - 54s - loss: 0.6402 - accuracy: 0.7579 - val_loss: 0.4584 - val_accuracy: 0.8426\n",
            "\n",
            " ROC-AUC - epoch: 15 - score: 0.972938 \n",
            "\n",
            "Epoch 16/30\n",
            " - 54s - loss: 0.6262 - accuracy: 0.7633 - val_loss: 0.4677 - val_accuracy: 0.8357\n",
            "\n",
            " ROC-AUC - epoch: 16 - score: 0.974822 \n",
            "\n",
            "Epoch 17/30\n",
            " - 54s - loss: 0.6134 - accuracy: 0.7686 - val_loss: 0.4388 - val_accuracy: 0.8455\n",
            "\n",
            " ROC-AUC - epoch: 17 - score: 0.975861 \n",
            "\n",
            "Epoch 18/30\n",
            " - 54s - loss: 0.6000 - accuracy: 0.7739 - val_loss: 0.4111 - val_accuracy: 0.8557\n",
            "\n",
            " ROC-AUC - epoch: 18 - score: 0.978521 \n",
            "\n",
            "Epoch 19/30\n",
            " - 54s - loss: 0.5890 - accuracy: 0.7786 - val_loss: 0.4200 - val_accuracy: 0.8544\n",
            "\n",
            " ROC-AUC - epoch: 19 - score: 0.978647 \n",
            "\n",
            "Epoch 20/30\n",
            " - 54s - loss: 0.5803 - accuracy: 0.7821 - val_loss: 0.3928 - val_accuracy: 0.8656\n",
            "\n",
            " ROC-AUC - epoch: 20 - score: 0.980377 \n",
            "\n",
            "Epoch 21/30\n",
            " - 54s - loss: 0.5709 - accuracy: 0.7863 - val_loss: 0.3764 - val_accuracy: 0.8717\n",
            "\n",
            " ROC-AUC - epoch: 21 - score: 0.981702 \n",
            "\n",
            "Epoch 22/30\n",
            " - 54s - loss: 0.5623 - accuracy: 0.7891 - val_loss: 0.3990 - val_accuracy: 0.8599\n",
            "\n",
            " ROC-AUC - epoch: 22 - score: 0.980664 \n",
            "\n",
            "Epoch 23/30\n",
            " - 58s - loss: 0.5577 - accuracy: 0.7907 - val_loss: 0.3577 - val_accuracy: 0.8764\n",
            "\n",
            " ROC-AUC - epoch: 23 - score: 0.983401 \n",
            "\n",
            "Epoch 24/30\n",
            " - 54s - loss: 0.5505 - accuracy: 0.7949 - val_loss: 0.3493 - val_accuracy: 0.8851\n",
            "\n",
            " ROC-AUC - epoch: 24 - score: 0.983943 \n",
            "\n",
            "Epoch 25/30\n",
            " - 54s - loss: 0.5375 - accuracy: 0.7975 - val_loss: 0.3562 - val_accuracy: 0.8798\n",
            "\n",
            " ROC-AUC - epoch: 25 - score: 0.983334 \n",
            "\n",
            "Epoch 26/30\n",
            " - 54s - loss: 0.5360 - accuracy: 0.7998 - val_loss: 0.3397 - val_accuracy: 0.8842\n",
            "\n",
            " ROC-AUC - epoch: 26 - score: 0.985124 \n",
            "\n",
            "Epoch 27/30\n",
            " - 53s - loss: 0.5278 - accuracy: 0.8013 - val_loss: 0.3325 - val_accuracy: 0.8889\n",
            "\n",
            " ROC-AUC - epoch: 27 - score: 0.985522 \n",
            "\n",
            "Epoch 28/30\n",
            " - 53s - loss: 0.5267 - accuracy: 0.8024 - val_loss: 0.3468 - val_accuracy: 0.8799\n",
            "\n",
            " ROC-AUC - epoch: 28 - score: 0.984847 \n",
            "\n",
            "Epoch 29/30\n",
            " - 54s - loss: 0.5155 - accuracy: 0.8074 - val_loss: 0.3306 - val_accuracy: 0.8879\n",
            "\n",
            " ROC-AUC - epoch: 29 - score: 0.986227 \n",
            "\n",
            "Epoch 30/30\n",
            " - 54s - loss: 0.5104 - accuracy: 0.8098 - val_loss: 0.3232 - val_accuracy: 0.8908\n",
            "\n",
            " ROC-AUC - epoch: 30 - score: 0.986435 \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZOd4PwlXtGiw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "5140836d-a7e2-4722-9910-46ae6f5b360d"
      },
      "source": [
        "# evaluate the model\n",
        "score = model_gru.evaluate(test_a_b, test_y_b, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.20535954314892846\n",
            "Test accuracy: 0.938513994216919\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KS6u4DCr0W3g",
        "colab_type": "text"
      },
      "source": [
        "## Model - 5 CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b6V9g4oOOOsG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        },
        "outputId": "3e8d3578-5286-44fc-9892-fbd8f5906efc"
      },
      "source": [
        "filter_sizes = [1,2,3,5]\n",
        "num_filters = 36\n",
        "answer_inp = Input(shape=(windows_size, ))\n",
        "embedding_size_glove = 100\n",
        "answer_emb1 = Embedding(vocab_size_stop+1, embedding_size_glove, weights=[embedding_matrix_lp], input_length=windows_size, trainable=False)(answer_inp)\n",
        "x = Reshape((windows_size, embedding_size_glove, 1))(answer_emb1) #Initializers allow you to pre-specify an initialization strategy, encoded in the Initializer object, without knowing the shape and dtype of the variable being initialized.\n",
        "maxpool_pool = []\n",
        "for i in range(len(filter_sizes)):\n",
        "  conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embedding_size_glove),kernel_initializer='he_normal', activation='relu')(x)\n",
        "  maxpool_pool.append(MaxPool2D(pool_size=(windows_size - filter_sizes[i] + 1, 1))(conv))\n",
        "\n",
        "z = Concatenate(axis=1)(maxpool_pool)   \n",
        "z = Flatten()(z)\n",
        "z = Dropout(0.1)(z)\n",
        "\n",
        "outp = Dense(5, activation=\"softmax\")(z)\n",
        "\n",
        "model = Model(inputs=answer_inp, outputs=outp)\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "    \n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            (None, 10)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, 10, 100)      745000      input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "reshape_1 (Reshape)             (None, 10, 100, 1)   0           embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 10, 1, 36)    3636        reshape_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 9, 1, 36)     7236        reshape_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 8, 1, 36)     10836       reshape_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 6, 1, 36)     18036       reshape_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 1, 1, 36)     0           conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2D)  (None, 1, 1, 36)     0           conv2d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_3 (MaxPooling2D)  (None, 1, 1, 36)     0           conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_4 (MaxPooling2D)  (None, 1, 1, 36)     0           conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 4, 1, 36)     0           max_pooling2d_1[0][0]            \n",
            "                                                                 max_pooling2d_2[0][0]            \n",
            "                                                                 max_pooling2d_3[0][0]            \n",
            "                                                                 max_pooling2d_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "flatten_1 (Flatten)             (None, 144)          0           concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 144)          0           flatten_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 5)            725         dropout_1[0][0]                  \n",
            "==================================================================================================\n",
            "Total params: 785,469\n",
            "Trainable params: 40,469\n",
            "Non-trainable params: 745,000\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e9Ui0Dav1SZG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "4106ab38-bf7c-461d-a6af-98beba94e7bf"
      },
      "source": [
        "model_cnn =  model.fit(train_a, train_y, batch_size=64, epochs=30, validation_data=(dev_a, dev_y), verbose=2,callbacks=[early_stopping])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 98795 samples, validate on 19760 samples\n",
            "Epoch 1/30\n",
            " - 15s - loss: 1.4580 - accuracy: 0.3806 - val_loss: 1.3410 - val_accuracy: 0.4472\n",
            "Epoch 2/30\n",
            " - 15s - loss: 1.2535 - accuracy: 0.4903 - val_loss: 1.1902 - val_accuracy: 0.5317\n",
            "Epoch 3/30\n",
            " - 15s - loss: 1.0951 - accuracy: 0.5663 - val_loss: 1.0708 - val_accuracy: 0.5786\n",
            "Epoch 4/30\n",
            " - 15s - loss: 0.9823 - accuracy: 0.6151 - val_loss: 0.9857 - val_accuracy: 0.6177\n",
            "Epoch 5/30\n",
            " - 15s - loss: 0.9055 - accuracy: 0.6497 - val_loss: 0.9770 - val_accuracy: 0.6227\n",
            "Epoch 6/30\n",
            " - 15s - loss: 0.8471 - accuracy: 0.6724 - val_loss: 0.8788 - val_accuracy: 0.6635\n",
            "Epoch 7/30\n",
            " - 15s - loss: 0.8037 - accuracy: 0.6909 - val_loss: 0.8746 - val_accuracy: 0.6580\n",
            "Epoch 8/30\n",
            " - 15s - loss: 0.7680 - accuracy: 0.7047 - val_loss: 0.8537 - val_accuracy: 0.6692\n",
            "Epoch 9/30\n",
            " - 15s - loss: 0.7418 - accuracy: 0.7145 - val_loss: 0.7819 - val_accuracy: 0.7024\n",
            "Epoch 10/30\n",
            " - 19s - loss: 0.7177 - accuracy: 0.7247 - val_loss: 0.8304 - val_accuracy: 0.6779\n",
            "Epoch 11/30\n",
            " - 15s - loss: 0.6896 - accuracy: 0.7359 - val_loss: 0.7628 - val_accuracy: 0.7053\n",
            "Epoch 12/30\n",
            " - 15s - loss: 0.6753 - accuracy: 0.7420 - val_loss: 0.8017 - val_accuracy: 0.6957\n",
            "Epoch 13/30\n",
            " - 15s - loss: 0.6550 - accuracy: 0.7494 - val_loss: 0.7214 - val_accuracy: 0.7245\n",
            "Epoch 14/30\n",
            " - 15s - loss: 0.6365 - accuracy: 0.7559 - val_loss: 0.7239 - val_accuracy: 0.7249\n",
            "Epoch 15/30\n",
            " - 15s - loss: 0.6242 - accuracy: 0.7618 - val_loss: 0.6878 - val_accuracy: 0.7395\n",
            "Epoch 16/30\n",
            " - 15s - loss: 0.6152 - accuracy: 0.7654 - val_loss: 0.6973 - val_accuracy: 0.7355\n",
            "Epoch 17/30\n",
            " - 15s - loss: 0.6025 - accuracy: 0.7686 - val_loss: 0.7360 - val_accuracy: 0.7201\n",
            "Epoch 18/30\n",
            " - 15s - loss: 0.5901 - accuracy: 0.7752 - val_loss: 0.6763 - val_accuracy: 0.7438\n",
            "Epoch 19/30\n",
            " - 15s - loss: 0.5777 - accuracy: 0.7802 - val_loss: 0.6422 - val_accuracy: 0.7546\n",
            "Epoch 20/30\n",
            " - 15s - loss: 0.5674 - accuracy: 0.7852 - val_loss: 0.6368 - val_accuracy: 0.7599\n",
            "Epoch 21/30\n",
            " - 15s - loss: 0.5628 - accuracy: 0.7867 - val_loss: 0.6386 - val_accuracy: 0.7585\n",
            "Epoch 22/30\n",
            " - 15s - loss: 0.5537 - accuracy: 0.7881 - val_loss: 0.6235 - val_accuracy: 0.7664\n",
            "Epoch 23/30\n",
            " - 15s - loss: 0.5450 - accuracy: 0.7924 - val_loss: 0.6351 - val_accuracy: 0.7613\n",
            "Epoch 24/30\n",
            " - 15s - loss: 0.5403 - accuracy: 0.7931 - val_loss: 0.6105 - val_accuracy: 0.7708\n",
            "Epoch 25/30\n",
            " - 15s - loss: 0.5286 - accuracy: 0.7990 - val_loss: 0.6276 - val_accuracy: 0.7630\n",
            "Epoch 26/30\n",
            " - 15s - loss: 0.5252 - accuracy: 0.7993 - val_loss: 0.6156 - val_accuracy: 0.7723\n",
            "Epoch 27/30\n",
            " - 15s - loss: 0.5126 - accuracy: 0.8052 - val_loss: 0.6008 - val_accuracy: 0.7751\n",
            "Epoch 28/30\n",
            " - 15s - loss: 0.5119 - accuracy: 0.8049 - val_loss: 0.5880 - val_accuracy: 0.7799\n",
            "Epoch 29/30\n",
            " - 15s - loss: 0.5075 - accuracy: 0.8074 - val_loss: 0.5809 - val_accuracy: 0.7854\n",
            "Epoch 30/30\n",
            " - 15s - loss: 0.4983 - accuracy: 0.8116 - val_loss: 0.5953 - val_accuracy: 0.7748\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CxUj-ScM2GBu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "19d11727-0b7a-45db-d43a-4c0c86d2417e"
      },
      "source": [
        "# evaluate the model\n",
        "score = model.evaluate(test_a, test_y, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.5834497151177596\n",
            "Test accuracy: 0.7813302874565125\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LHyKyNx8QQls",
        "colab_type": "text"
      },
      "source": [
        "## Model 6- CNN + LSTM (Hybrid Model)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tF-vV2i9Eral",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "outputId": "7ec65d65-f437-40d6-a77b-f21bdbdbff80"
      },
      "source": [
        "lstm_cnn = Sequential()\n",
        "lstm_cnn.add(Embedding(vocab_size_stop+1, 100,weights=[embedding_matrix_lp],input_length=windows_size, trainable=False))\n",
        "lstm_cnn.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')) #We can use a smallish set of 32 features with a small filter length of 3. The pooling layer can use the standard length of 2 to halve the feature map size.\n",
        "lstm_cnn.add(MaxPooling1D(pool_size=2))\n",
        "lstm_cnn.add(LSTM(100))\n",
        "lstm_cnn.add(Dense(5, activation='softmax'))\n",
        "lstm_cnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "## Fit train data\n",
        "lstm_cnn.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_4 (Embedding)      (None, 10, 100)           745000    \n",
            "_________________________________________________________________\n",
            "conv1d_3 (Conv1D)            (None, 10, 32)            9632      \n",
            "_________________________________________________________________\n",
            "max_pooling1d_3 (MaxPooling1 (None, 5, 32)             0         \n",
            "_________________________________________________________________\n",
            "lstm_3 (LSTM)                (None, 100)               53200     \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 5)                 505       \n",
            "=================================================================\n",
            "Total params: 808,337\n",
            "Trainable params: 63,337\n",
            "Non-trainable params: 745,000\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1xOJG1qqQ1GB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 782
        },
        "outputId": "30a6e898-d871-4e84-fe1c-92ad1801d054"
      },
      "source": [
        "hist = lstm_cnn.fit(train_a_b, train_y_b, validation_data=(dev_a_b, dev_y_b), epochs=30, batch_size=64, shuffle=True, callbacks=[early_stopping])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 65939 samples, validate on 13189 samples\n",
            "Epoch 1/30\n",
            "65939/65939 [==============================] - 11s 171us/step - loss: 1.5373 - accuracy: 0.3087 - val_loss: 1.4574 - val_accuracy: 0.3735\n",
            "Epoch 2/30\n",
            "65939/65939 [==============================] - 11s 167us/step - loss: 1.3688 - accuracy: 0.4252 - val_loss: 1.3136 - val_accuracy: 0.4594\n",
            "Epoch 3/30\n",
            "65939/65939 [==============================] - 11s 166us/step - loss: 1.2345 - accuracy: 0.4990 - val_loss: 1.2511 - val_accuracy: 0.4927\n",
            "Epoch 4/30\n",
            "65939/65939 [==============================] - 11s 171us/step - loss: 1.1432 - accuracy: 0.5432 - val_loss: 1.1811 - val_accuracy: 0.5275\n",
            "Epoch 5/30\n",
            "65939/65939 [==============================] - 11s 167us/step - loss: 1.0780 - accuracy: 0.5733 - val_loss: 1.1505 - val_accuracy: 0.5377\n",
            "Epoch 6/30\n",
            "65939/65939 [==============================] - 11s 165us/step - loss: 1.0231 - accuracy: 0.5991 - val_loss: 1.1213 - val_accuracy: 0.5530\n",
            "Epoch 7/30\n",
            "65939/65939 [==============================] - 11s 164us/step - loss: 0.9724 - accuracy: 0.6212 - val_loss: 1.0706 - val_accuracy: 0.5802\n",
            "Epoch 8/30\n",
            "65939/65939 [==============================] - 11s 167us/step - loss: 0.9305 - accuracy: 0.6394 - val_loss: 1.0752 - val_accuracy: 0.5751\n",
            "Epoch 9/30\n",
            "65939/65939 [==============================] - 11s 165us/step - loss: 0.8860 - accuracy: 0.6576 - val_loss: 1.0159 - val_accuracy: 0.6023\n",
            "Epoch 10/30\n",
            "65939/65939 [==============================] - 11s 163us/step - loss: 0.8500 - accuracy: 0.6700 - val_loss: 1.0056 - val_accuracy: 0.6157\n",
            "Epoch 11/30\n",
            "65939/65939 [==============================] - 11s 165us/step - loss: 0.8121 - accuracy: 0.6883 - val_loss: 0.9913 - val_accuracy: 0.6214\n",
            "Epoch 12/30\n",
            "65939/65939 [==============================] - 11s 164us/step - loss: 0.7769 - accuracy: 0.7014 - val_loss: 1.0046 - val_accuracy: 0.6169\n",
            "Epoch 13/30\n",
            "65939/65939 [==============================] - 11s 165us/step - loss: 0.7444 - accuracy: 0.7153 - val_loss: 0.9979 - val_accuracy: 0.6246\n",
            "Epoch 14/30\n",
            "65939/65939 [==============================] - 11s 164us/step - loss: 0.7095 - accuracy: 0.7306 - val_loss: 0.9527 - val_accuracy: 0.6411\n",
            "Epoch 15/30\n",
            "65939/65939 [==============================] - 11s 165us/step - loss: 0.6796 - accuracy: 0.7410 - val_loss: 0.9625 - val_accuracy: 0.6373\n",
            "Epoch 16/30\n",
            "65939/65939 [==============================] - 11s 164us/step - loss: 0.6503 - accuracy: 0.7506 - val_loss: 0.9450 - val_accuracy: 0.6520\n",
            "Epoch 17/30\n",
            "65939/65939 [==============================] - 11s 165us/step - loss: 0.6183 - accuracy: 0.7651 - val_loss: 0.9615 - val_accuracy: 0.6530\n",
            "Epoch 18/30\n",
            "65939/65939 [==============================] - 11s 164us/step - loss: 0.5909 - accuracy: 0.7754 - val_loss: 0.9151 - val_accuracy: 0.6663\n",
            "Epoch 19/30\n",
            "65939/65939 [==============================] - 11s 165us/step - loss: 0.5662 - accuracy: 0.7846 - val_loss: 0.9118 - val_accuracy: 0.6735\n",
            "Epoch 20/30\n",
            "65939/65939 [==============================] - 11s 164us/step - loss: 0.5386 - accuracy: 0.7980 - val_loss: 0.9159 - val_accuracy: 0.6800\n",
            "Epoch 21/30\n",
            "65939/65939 [==============================] - 11s 164us/step - loss: 0.5168 - accuracy: 0.8055 - val_loss: 0.9193 - val_accuracy: 0.6772\n",
            "Epoch 22/30\n",
            "65939/65939 [==============================] - 11s 164us/step - loss: 0.4929 - accuracy: 0.8136 - val_loss: 0.9426 - val_accuracy: 0.6744\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D8J5CikETlss",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "5807ee86-4c3e-4bad-f2d5-f5bb4434ef66"
      },
      "source": [
        "# evaluate the model\n",
        "score = lstm_cnn.evaluate(test_a_b, test_y_b, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.9602536167393505\n",
            "Test accuracy: 0.6658926606178284\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4eU9eMlSFsy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "6f2633dd-bfa9-4adc-d1ab-65ff046a5bbf"
      },
      "source": [
        "plt.plot(hist.history['accuracy'])\n",
        "plt.plot(hist.history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'validation'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3gU57X48e9RR0IINZpE772JYgMGjAtu4N4LLsFxicsvzo1zk5vYTrm+9zqO4xLHJS5xwzY2Nq6YalwAI9ElikRVQR0JIVDd8/tjFhAggQAtu9Kez/Pw7O7M7OzZZTVn533fOa+oKsYYY/xXgLcDMMYY412WCIwxxs9ZIjDGGD9nicAYY/ycJQJjjPFzlgiMMcbPWSIwfkVE3hCRPzVy2x0icp6nYzLG2ywRGGOMn7NEYEwzJCJB3o7BtByWCIzPcTfJ/EpE1olIuYj8S0Tai8hXIlImIgtEJLrO9tNEJFVESkRkiYj0r7NuuIiscj/vfSDsqNe6VETWuJ/7o4gMaWSMl4jIahHZKyKZIvLYUevHu/dX4l4/w728lYj8VUR2ikipiHzvXjZJRLLq+RzOc99/TERmi8jbIrIXmCEio0Vkmfs1dovI8yISUuf5A0VkvogUi0ieiPyniHQQkf0iEltnuxEiUiAiwY1576blsURgfNVVwPlAH+Ay4CvgP4F4nO/tAwAi0gd4D3jIve5L4DMRCXEfFD8B3gJigA/d+8X93OHAa8DdQCzwEjBXREIbEV85cCvQFrgEuEdELnfvt6s73ufcMQ0D1rif9xQwEjjbHdN/AK5GfibTgdnu13wHqAUeBuKAs4ApwL3uGCKBBcDXQCegF7BQVXOBJcC1dfZ7CzBLVasbGYdpYSwRGF/1nKrmqWo28B2wQlVXq2oFMAcY7t7uOuALVZ3vPpA9BbTCOdCOBYKBZ1S1WlVnAyvrvMZM4CVVXaGqtar6JlDpft5xqeoSVV2vqi5VXYeTjCa6V98ILFDV99yvW6Sqa0QkALgDeFBVs92v+aOqVjbyM1mmqp+4X/OAqqao6nJVrVHVHTiJ7GAMlwK5qvpXVa1Q1TJVXeFe9yZwM4CIBAI34CRL46csERhflVfn/oF6Hrd23+8E7Dy4QlVdQCaQ4F6XrUdWVtxZ535X4JfuppUSESkBOrufd1wiMkZEFrubVEqBn+P8Mse9j631PC0Op2mqvnWNkXlUDH1E5HMRyXU3F/2lETEAfAoMEJHuOGddpar60ynGZFoASwSmucvBOaADICKCcxDMBnYDCe5lB3Wpcz8T+LOqtq3zL1xV32vE674LzAU6q2oU8E/g4OtkAj3reU4hUNHAunIgvM77CMRpVqrr6FLBLwKbgN6q2gan6axuDD3qC9x9VvUBzlnBLdjZgN+zRGCauw+AS0Rkiruz85c4zTs/AsuAGuABEQkWkSuB0XWe+wrwc/evexGRCHcncGQjXjcSKFbVChEZjdMcdNA7wHkicq2IBIlIrIgMc5+tvAY8LSKdRCRQRM5y90lsAcLcrx8M/A44UV9FJLAX2Cci/YB76qz7HOgoIg+JSKiIRIrImDrr/w3MAKZhicDvWSIwzZqqbsb5Zfsczi/uy4DLVLVKVauAK3EOeMU4/Qkf13luMvAz4HlgD5Dh3rYx7gWeEJEy4Pc4CengfncBF+MkpWKcjuKh7tWPAOtx+iqKgf8BAlS11L3PV3HOZsqBI0YR1eMRnARUhpPU3q8TQxlOs89lQC6QDkyus/4HnE7qVapat7nM+CGxiWmM8U8isgh4V1Vf9XYsxrssERjjh0RkFDAfp4+jzNvxGO+ypiFj/IyIvIlzjcFDlgQM2BmBMcb4PTsjMMYYP9fsClfFxcVpt27dvB2GMcY0KykpKYWqevS1KUAzTATdunUjOTnZ22EYY0yzIiINDhO2piFjjPFzlgiMMcbPWSIwxhg/1+z6COpTXV1NVlYWFRUV3g6lRQgLCyMxMZHgYJunxBh/0CISQVZWFpGRkXTr1o0jC02ak6WqFBUVkZWVRffu3b0djjHmDGgRTUMVFRXExsZaEmgCIkJsbKydXRnjR1pEIgAsCTQh+yyN8S8tomnIGGNaqr0V1azZVcLqXSVM6d+OQQlRTf4algiaQElJCe+++y733nvvST3v4osv5t1336Vt27YeiswY05zUupT0/DJW7yph9a49rN5VQkbBPlRBBGJah1gi8FUlJSX84x//OCYR1NTUEBTU8Ef85Zdfejo0Y4wPK9pX6Rz0M52D/trMEsqragGIDg9meJdopg/rxPAu0QxJjCIyzDMj+TyaCERkKvB3IBB4VVWfPGp9F+BNoK17m0dVtdkdHR999FG2bt3KsGHDCA4OJiwsjOjoaDZt2sSWLVu4/PLLyczMpKKiggcffJCZM2cCh8tl7Nu3j4suuojx48fz448/kpCQwKeffkqrVq28/M6MMU2lorqWjbv3si6r1Pm1n1nCzqL9AAQFCP07tuGqkYkM79KW4Z2j6Robfsb66zyWCNyTb7+AM11eFrBSROaqalqdzX4HfKCqL4rIAOBLoNvpvO7jn6WSlrP3dHZxjAGd2vCHywY2uP7JJ59kw4YNrFmzhiVLlnDJJZewYcOGQ8MvX3vtNWJiYjhw4ACjRo3iqquuIjY29oh9pKen89577/HKK69w7bXX8tFHH3HzzTc36fswxpwZlTW1bM4tY11WKRuyS1mXVcqWvDJqXE7Z//ZtQhnRJZqbxnRheJdoBnWKolVIoNfi9eQZwWggQ1W3AYjILGA6UDcRKNDGfT8KyPFgPGfM6NGjjxiD/+yzzzJnzhwAMjMzSU9PPyYRdO/enWHDhgEwcuRIduzYccbiNcacuqoaF1vyyljvPuCvzy5hc24Z1bXOQT86PJjBiW2Z3C+ewQltGZIYRae2vnW278lEkABk1nmcBYw5apvHgG9E5BdABHDe6b7o8X65nykRERGH7i9ZsoQFCxawbNkywsPDmTRpUr1j9ENDQw/dDwwM5MCBA2ckVmPMySk9UM2CtDxWZ+5hfVYpG3eXUVXrAqBNWBBDEtty14QeDEmIYlBCFInRrXx+SLa3O4tvAN5Q1b+KyFnAWyIySFVddTcSkZnATIAuXbp4Iczji4yMpKys/hn/SktLiY6OJjw8nE2bNrF8+fIzHJ0x5nRV1bj4dksBc1ZnsWBjPlU1LiJDgxiUEMXt47oxODGKIQlt6Rzj+wf9+ngyEWQDnes8TnQvq+tOYCqAqi4TkTAgDsivu5Gqvgy8DJCUlORzc2vGxsYybtw4Bg0aRKtWrWjfvv2hdVOnTuWf//wn/fv3p2/fvowdO9aLkRpjGktVWZtVypxVWcxdm8Oe/dXERIRw4+guXDE8gcEJUQQENL+Dfn08NmexiAQBW4ApOAlgJXCjqqbW2eYr4H1VfUNE+gMLgQQ9TlBJSUl69MQ0GzdupH///h54F/7LPlPjrzKL9/PJ6mzmrM5mW2E5IUEBnD+gPVcOT+CcPvEEBzbPggwikqKqSfWt89gZgarWiMj9wDycoaGvqWqqiDwBJKvqXOCXwCsi8jBOx/GM4yUBY4zxhNID1Xy1fjcfr87mp+3FAIzuHsPMc3pw0eCORLVq2ZV4PdpH4L4m4Mujlv2+zv00YJwnYzDGmPpU1tTy3ZZC5qzOZv7GPKpqXPSIj+CRC/owfVgCnWPCvR3iGePtzmJjjDkjKqprWZNZwoptxazYXkTKzj1U1riOaPcfkhjVLDt7T5clAmNMi1RRXcuqXXtYvq2YFduKWJ1ZQlWNCxEY0LENN43pyvjesUzo3Xzb/ZuKJQJjTItwoKqWlJ17WLG9iBXbilmTWUJVrYsAgYGdorh1bFfG9ohlVLcYosJbdpv/ybJEYIxplqpqXKzatYfv0gtYvq2YdVklVNcqAQKDE6KYMa4bY3vEkNQthjYeKtbWUlgi8ILWrVuzb98+cnJyeOCBB5g9e/Yx20yaNImnnnqKpKR6R3sB8MwzzzBz5kzCw51OLStrbVq6HYXlfJdewLdbClm2tZDyqloCA4TBCVHcOb4HY3rEkNQ12mNVOlsqSwRe1KlTp3qTQGM988wz3HzzzYcSgZW1Ni1NWUU1y7YWsTS9gKVbCtlV7FTrTIxuxfThCZzTO56ze8XaL/7TZImgCTz66KN07tyZ++67D4DHHnuMoKAgFi9ezJ49e6iuruZPf/oT06dPP+J5O3bs4NJLL2XDhg0cOHCA22+/nbVr19KvX78jag3dc889rFy5kgMHDnD11Vfz+OOP8+yzz5KTk8PkyZOJi4tj8eLFh8pax8XF8fTTT/Paa68BcNddd/HQQw+xY8cOK3dtfJrLpWzIKWXpFufAv2rXHmpcSnhIIGf1iOXO8d05p0883c5giWZ/0PISwVePQu76pt1nh8Fw0ZMNrr7uuut46KGHDiWCDz74gHnz5vHAAw/Qpk0bCgsLGTt2LNOmTWvwy/viiy8SHh7Oxo0bWbduHSNGjDi07s9//jMxMTHU1tYyZcoU1q1bxwMPPMDTTz/N4sWLiYuLO2JfKSkpvP7666xYsQJVZcyYMUycOJHo6Ggrd218TnWti4Ub8/ly/W6+zyikuLwKgIGd2vCzc3owoXccI7tGExrkvTLNLV3LSwReMHz4cPLz88nJyaGgoIDo6Gg6dOjAww8/zNKlSwkICCA7O5u8vDw6dOhQ7z6WLl3KAw88AMCQIUMYMmTIoXUffPABL7/8MjU1NezevZu0tLQj1h/t+++/54orrjhUBfXKK6/ku+++Y9q0aVbu2viMHYXlzFqZyeyULAr3VRIbEcLEPvGc0yeO8b3iiY8MPfFOTJNoeYngOL/cPemaa65h9uzZ5Obmct111/HOO+9QUFBASkoKwcHBdOvWrd7y0yeyfft2nnrqKVauXEl0dDQzZsw4pf0cZOWujTdVVNcyLzWXWT9lsmxbEYEBwuS+7bhhdGcm9oknyM/H83uLfepN5LrrrmPWrFnMnj2ba665htLSUtq1a0dwcDCLFy9m586dx33+Oeecw7vvvgvAhg0bWLduHQB79+4lIiKCqKgo8vLy+Oqrrw49p6Hy1xMmTOCTTz5h//79lJeXM2fOHCZMmNCE79aYk7Mlr4zHP0tl7H8v5MFZa8gq2c+vLuzLj4+ey6u3JTGlf3tLAl7U8s4IvGTgwIGUlZWRkJBAx44duemmm7jssssYPHgwSUlJ9OvX77jPv+eee7j99tvp378//fv3Z+TIkQAMHTqU4cOH069fPzp37sy4cYdLM82cOZOpU6fSqVMnFi9efGj5iBEjmDFjBqNHjwaczuLhw4dbM5A5o/ZX1fD52t28t3IXq3eVEBwoXDCwAzeM6sLZPWNbTAnnlsBjZag9xcpQnxn2mZpTtT6rlPdW7mLumhz2VdbQMz6CG0Z34coRicREhHg7PL/llTLUxhj/UF5Zw8odxfy4tYilWwrYlFtGWHAAFw/uyA2ju5DUNdqGevo4SwTGmJNSWVPLml0l/LC1iGVbC1m9q4QalxISGMCwLm354/SBTBuW0OJr+LckLSYRqKr96mgiza250HhWrUtJzSnlh4wiftxayModxVRUuw7V9LlrQg/G9YolqWsMrUJsrH9z1CISQVhYGEVFRcTGxloyOE2qSlFREWFhYd4OxXiJqpKRv48fMgr5cWsRy7cVsbeiBoA+7Vtzvbuzd0yPWPvV30K0iESQmJhIVlYWBQUF3g6lRQgLCyMxMdHbYZgzqGR/Fd9nFPLt5gKWpheQt7cSgM4xrbh4cEfO6hnLWT1jaRdpPxBaohaRCIKDg+nevbu3wzCm2ah1KWuzSli6pYBvtxSwNrMEl0KbsCAm9I5nQu84xvWK86vpGv1Zi0gExpgTy9tbcejA/31GISX7qxGBoYltuf/c3kzsE8/QxCi7sMsPWSIwpoWqrKklZccevnUf/DflOlehx0eGcl7/9kzsE8/4XnFE29h+v2eJwJgWJiO/jLeX7+KjlCzKKmsIDhRGdYvh0Yv6MbFPPP06RNqgCnMESwTGtADVtS7mp+Xx1rKdLNtWREhgABcN7sC0oZ0Y2yOWiFD7UzcNs2+HMc1YbmkF7/20i/d+2kV+WSUJbVvxH1P7cm1SZ+JaWxln0ziWCIxpZlSVZduKeGvZTr5Jy8OlysQ+8fz32K5M6tuOQCvmZk6SJQJjmom9FdV8nJLFW8t3srWgnLbhwdw1vjs3julC19gIb4dnmjFLBMb4uLScvby1fCefrM7mQHUtwzq35a/XDOWSIR0JC7aSDub0WSIwxgeV7q9m7rocZqdksTazhLDgAKYPTeDmsV0ZnBjl7fBMC2OJwBgfUVPr4ruMQmanZDE/LY+qGhf9OkTyX5cO4OoRiUSFW10f4xmWCIzxsoz8MmanZDNndRZ5eyuJDg/mxtFduHpkIgM7tbEx/8bjLBEY4wWl+6v5zN30syazxD2JezyPT0vk3H7tCQmyMg/mzPFoIhCRqcDfgUDgVVV98qj1fwMmux+GA+1Uta0nYzLGW2pdyvcZhXyYnMk37qafvu0j+d0l/Zk+LIH4SBv3b7zDY4lARAKBF4DzgSxgpYjMVdW0g9uo6sN1tv8FMNxT8RjjLZty9/LJ6pxDTT9tw4O5YVRnrknqbE0/xid48oxgNJChqtsARGQWMB1Ia2D7G4A/eDAeY86YrD37mbs2h7lrctiUW0ZggDCpTzyPXZbIuf3bERpkwz6N7/BkIkgAMus8zgLG1LehiHQFugOLGlg/E5gJ0KVLl6aN0pgmUlxexRfrd/Pp6mySd+4BYGTXaJ6YPpCLB3e0kg/GZ/lKZ/H1wGxVra1vpaq+DLwMkJSUZBPqGp9RXlnDgo15fLomh6VbCqhxKb3bteZXF/Zl2tBONrGLaRY8mQiygc51Hie6l9XneuA+D8ZiTJOprnXxXXoBn6zOYX5aHgeqa+kUFcZdE3owfVgnK/Nsmh1PJoKVQG8R6Y6TAK4Hbjx6IxHpB0QDyzwYizGnbePuvby9fCdfrt/Nnv3VtA0P5soRCUwflkBS12gCrNibaaY8lghUtUZE7gfm4QwffU1VU0XkCSBZVee6N70emKWq1uRjfNKG7FKeXZjON2l5tAoO5PwB7Zk+rBMTesfbeH/TIkhzO/4mJSVpcnKyt8MwfmBdVgnPLkxnwcZ82oQFccf47tx+dncr9WCaJRFJUdWk+tb5SmexMT5j9a49PLswncWbC4hqFcwvz+/DbeO60SbMEoABaquhZBcEBkN4LASHQzPvE7JEYIxbys5inlmQznfphUSHB/OrC/ty61ldibQE4J9UoTQT8jdCXqpzm58GhVugturwdoGhEB7jJIVW0c5teAy0iqnnfrRzPyzKp5KHJQLj91ZsK+LZRen8kFFEbEQIj17Uj1vGdrV5fv3J/mL3wT7N+ZeX5hz4q8oOb9MmEdoPgF5TIL4fuGphfxEcKHZu9+9x7uenOfs7UAzqqv/1JNCdNA4miRjn8THLYo5cFhzmkbdv33Tjlw5O9/j3Bems2F5MXOtQfntxf24a24XwEPuzaLFcLijeCrvXwu41kLvBOXDvyzu8TVhbaD8Qhl4P7fo799v1d37Fn+xrVZY6SWF/cZ2kUXzsbUmmE9P+Yqg50PA+L34KRv/s1N77cdg33viVmloX32cU8o/FW/lpRzHtIkP5/aUDuGF0F1qFWNmHFqW2Ggo2uw/67n+566G63FkfGOIc4HtOcX7pt+sP7QZCZIemabYJCDj8Kz+2Z+OfV33gcJI4sOfIhJE46vTjqoclAtPi1bqUn7YX88X6HL7ekEvhvio6tAnj8WkDuW5UZ5vusSWoroD81CMP+nlpUFvprA+OgA6DYfjN0HEodBziNO8E+mD/T3AriEpw/p0hlghMi1TrUlbuKOaLdbv5akMuhfsqaRUcyLn923HJ4I5MscJvzV9pNqz/EFI/dpp4DlaoCYtyDvZjZkLHYc79mB4QYP/fDbFEYFoMl0tJ3rmHL9bl8OWGXArKKgkLDuDcfu24ZHAnJveLt/b/5q5iL2ycC2tnwY7vAXWaS8Y/7P6lPxTadvGpETnNgf1VmGbN5VJW7drD5+t289WG3eTtrSQ0yH3wH9KRyX3b2eif5q62GjIWwrpZsPkrqKlwfuFPehSGXOvcN6fF/kJMs5SRX8a7KzL5cv1ucvdWEBIUwOS+8VwypBNT+tnBv9lThaxkWPe+0/Szv8gZhz/iVhhyHSSMtF/9Tcj+WkyzsnH3Xp5flMGXG3YTHBDAxL7x/GZIP6b0b09rO/g3f0VbnXb/de9D8TYICoO+FzsH/15TfLNztwWwvxzTLGzILuW5RenMS82jdWgQ907qyZ3jexATEeLt0MxBqlCwCcoLnCtva6qcUTsHb+tbVlPpNP3UVjoXdGWtBAS6T4AJv4T+l538+H1z0iwRGJ+2JrOE5xams3BTPpFhQTw4pTd3jLPCbz6lohTWfQCr3nTG6TeWBEJQqPMrPzDUGb9/3uMw+GqISvRcvOYYlgiMT0rZWcyzCzP4dksBbcODeeSCPtx6thV+8xmqzq/3lDdhw0fO1bAdhjhXvsb3cy7WCgpxDvBBoc7jo5fZcE6fYYnA+JS6dX9iIkL49dR+3HJWV2v/9xUH9ji//lPecEozhLSGodfByBnQabi3ozOnyP66jNepKsu2FvH3hVb3xyepwq5lzq//tE+c4ZudRsBlf4dBV0FopLcjNKfJ/sqM16gqS9MLeXZhOik799C+TSh/uMyp+2NlH05DeSEkvwar3gLUaXuP7Oj+1+HwbZtOzm1om/qHYu4vhrXvOb/+C7dASCQMuwlG3uZcuGVaDEsE5oxTVRZtyufZhemszSqlU1QYf5w+kGuSrO7PaSnYDMv/4Vx1W1MBPc+FiHZQtttZt20JVO499nnB4ccmi7Jc5wre2irnyt3pL8DAKyAk4oy/LeN5lgjMGeNyKfM35vHswnRSc/aSGN2K/75yMFeNSLS5f0+VqnOAX/YCZMx3OmKHXg9j74V2/Y7dvnKfU3K5bLdzsC/bDXt3H36cneLcDwx12v1H3AYdBp3pd2XOMEsExuNcLuWrDbk8tyidTblldIsN5/+uHsLlwxMIDrQEcEpqKmH9bCcB5KdCRDxM/i0k3QERcQ0/L7S18+94ZZFVnX8B9n/jLywRGI+pdSmfr8vh+UUZpOfvo0d8BH+7biiXDelEkCWAU3Ow/f+nV6A836mfP/0FGHR1081eJWLlG/yMJQLT5GpqXXy6JocXFmewrbCcPu1b8+wNw7lkcEcCA+wAc0qObv/vdT6cdR/0mGQHbXPaLBGYJlNd62LOqmyeX5zBruL99OsQyYs3jeDCgR0IsARwcly1zsE/ZxWkfQrp3zh1dw62/8f39XaEpgWxRGBOW1WNiw9TMvnH4q1klxxgUEIbXr5lJOf1b99yEkBNFezZ7lxQ1SbBGXrZVFfGqkLJLqejNmcVZK+CnDWHp1SMaNe49n9jTpElAnNalm4p4LG5qWwrLGdY57b86fJBTOobjzTX5or9xVCY7oybL9xy+P6eHYdnwAIICHKSQduuENXZmQylrfs2qrNTK6ehSpnlhc7B/tCBP8UpswzOaJ2OQ5wpFRNGQsIIiOlpHbfGoywRmFOSXXKAP36WxtepuXSLDedftyVxbr92zSMB1NZAyU4oyjj2gH/wgAxObZzYXs7wyUFXQlwfaBUDe7OcX/Almc7t9m9hbw6gdV5EnERxMEm06Qh7djoJoHSXe5MApy5P34ucK3UTRkK7AU49HmPOIEsE5qRU1tTy6nfbeW5ROgCPXNCHuyb08P0LwVwu2Pm9U+c+be6RF1aFxzkH+X6XOrdxfSCut3MAb2zzT00V7M12EkNp5pGJInO5M79uVCIkjnTm0u00wrk6N7S1Z96vMSfBEoFptCWb83n8szS2F5YzdWAHfndpfxKjw70d1vHlpTkH//UfOgfqkNbQfxp0GwexvZ0DfnjM6b9OUAjEdHf+1UfVRvcYn9WoRCAiHwP/Ar5SVZdnQzK+JmvPfv74eRrzUvPoHhfBm3eMZmKfeG+H1bC9u2HDbFj7PuStd+re9zoPzn/Cme0qxAvJy5KA8WGNPSP4B3A78KyIfAi8rqqbPReW8QUV1bW8snQbLyzJQBB+dWFf7prQndAgH2wGqiyDjZ87E5xv+xZQp839ov+FgVdCax9OXMZ4WaMSgaouABaISBRwg/t+JvAK8LaqVtf3PBGZCvwdCAReVdUn69nmWuAxnJ62tap646m8EdO0Fm/K57HPUtlZtJ+LB3fgt5cMIKFtK2+HdaTaGti22LnIatMXzuQobbvCOb+CIdc6zT7GmBNqdB+BiMQCNwO3AKuBd4DxwG3ApHq2DwReAM4HsoCVIjJXVdPqbNMb+A0wTlX3iEi7U38rpilkFu/n8c/SWLAxjx7xEbx152gm9PaxX9OqsPJV+PZ/nPlxw9rCsBtgyPXQebQ1wxhzkhrbRzAH6Au8BVymqrvdq94XkeQGnjYayFDVbe59zAKmA2l1tvkZ8IKq7gFQ1fyTfwumKVTXuvjnkq08vziDwADh0Yv6cce47r5XFXR/Mcz9BWz6HLpPhDF3O+UWbMilMaessWcEz6rq4vpWqGpSA89JADLrPM4Cxhy1TR8AEfkBp/noMVX9+ugdichMYCZAly5dGhmyaayM/H08/P4a1meXcsngjvzu0v50jPKxZiCAncvgo7ucMsoX/sUptWC//o05bY1NBANEZLWqlgCISDRwg6r+owlevzdO01IisFREBh98nYNU9WXgZYCkpCQ9eifm1Kgq/162k798uZHwkEBevGkEFw3u6O2wjuWqhe+ehiV/cfoA7vzGueLWGNMkGpsIfqaqLxx84G7P/xnOaKKGZAOd6zxOdC+rKwtY4e5s3i4iW3ASw8pGxmVOUd7eCh75cC3fpRcyqW88/3vVENq1aaIyxk2pLBc+/hlsX+qUWr70bxDWxttRGdOiNDYRBIqIqKrCoY7gEzXKrgR6i0h3nARwPXD0iKBPcEYhvS4icThNRdsaG7w5NV+s281/zllPZU0tf7x8EDeP6eKbpSHSF8Ccu6GqHKY979Tf8cU4jS5AMLMAABfYSURBVGnmGpsIvsbpGH7J/fhu97IGqWqNiNwPzMNp/39NVVNF5AkgWVXnutddICJpQC3wK1Utaniv5nSUHqjmsbmpzFmdzdDEKP523TB6xPtgiYOaKlj0R/jxWWfilWtet7LLxniQuH/kH38jkQCcg/8U96L5ONcF1Db8LM9ISkrS5OSGBiqZhvy4tZBHPlhLXlklvzi3F/dN7uWb00QWb4eP7nQqcibdCRf+GYJ9sOPamGZGRFIaGtzT2AvKXMCL7n+mGamoruWpeZt59fvtdI+L4KN7zmZY57ZN+yKqUFHijOc/naabDR/DZw8CAtf+GwZMb7IQjTENa+x1BL2B/wYGAId6FFW1h4fiMk0gNaeUh99fw5a8fdwytiu/ubgf4SFNXGewLBc+numUYg6OcIquRXdz33Y/fBvVGQIbeO2q/fD1o7DqTUgcBVf9C6K7Nm2cxpgGNfao8DrwB+BvwGScukM+2K5gwJk0/uWl23h6/mbahofw+u2jmNzXAxdtp8+HOT+H6v0w4RGnU3fPdqe2f/p8qK08vG1AkFPWuW5yiOnuVAP96tdQsBHGP+zMxNXQhC7GGI9obCJopaoL3SOHdgKPiUgK8HsPxmZOQXbJAR6etYafdhRz0aAO/OWKwURHNPFVtzVVsOgJ+PE5aD8Irn7t2M5clwvKdkPxNic5FG8/fJuVDJWlh7eNiIebP4ZeUzDGnHmNTQSV7g7jdPdIoGzAB4eb+LcV24q4951VVNa4ePraoVwxPKHph4UWb4PZdzpTLI66Cy74U/2duQEBEJXg/Os+4ch1qs7cv8Xbndm+upxt1UGN8aLGJoIHgXDgAeCPOM1Dt3kqKHPy3l6+k8fmptIlNpxXbk2ipyeGha6fDZ895Bzkr3sb+l92avsRcSaDCY8BRjZpiMaYk3fCROC+eOw6VX0E2IfTP2B8RFWNi8c+S+XdFbuY3Deev98wnDZhTdzGXlUOX/0HrH4bOo+Bq1512vuNMS3CCROBqtaKyPgzEYw5OYX7Krn37VX8tKOYn0/sya8u7EtgQBM3BeVugNm3Ox3AEx6BSb9pePSPMaZZauxf9GoRmQt8CJQfXKiqH3skKnNCqTmlzPx3CoX7Kvn79cOYPiyhaV/gYM3/eb+FVm3h1k+hx8SmfQ1jjE9obCIIA4qAc+ssU8ASgRd8vi6HRz5cS3R4CLN/fjaDE6Oa9gXq1vzvdT5c/qJ15hrTgjX2ymLrF/ABLpfy9PwtPL84g6Su0bx480jiI0Ob9kV2LXdGBe3Lgwv+7NT8D7BLRoxpyRp7ZfHrOGcAR1DVO5o8IlOvsopqHn5/DQs25nP9qM48Pn1g004iv78Ylr8I3/0V2naGO+c5k78bY1q8xjYNfV7nfhhwBZDT9OGY+mwvLOdn/05me2E5T0wfyC1juzbd9QHZKbDyX7DhI6ipsJr/xvihxjYNfVT3sYi8B3zvkYjMEZZuKeD+d1cRGCC8fecYzuoZe/o7rT7gHPhXvgo5q50aQcNudKp9dhh0+vs3xjQrpzoOsDfggeI15iBV5V/fb+cvX26kT/tIXrk1ic4x4ae306KtkPyacz1ARQnE94OLn4Ih19kZgDF+rLF9BGUc2UeQC/zaIxEZXC7l0Y/X8UFyFlMHduCv1w4lIvQUc3ZtDaTPc379b13kFH/rf5lTHqLrOJvxyxjT6KahSE8HYg77v28280FyFr84txcPn9eHgFO5SGxfvlPWOfkNp55PZCensueIWyGyQ5PHbIxpvhp7RnAFsEhVS92P2wKTVPUTTwbnj2anZPHikq3cOKYL/+/8PifXKVyx15nkPfVjSJsLrmroMRkuehL6XGRXBBtj6tXYI8MfVHXOwQeqWiIif8CZfN40kZU7ivnNx+s4u2csj08beOIk4HLB7tWQschp9sn6CVw1EBYFo2dC0h0Q1+vMBG+MabYamwjqu6LIfl42oczi/dz9VgqJ0eH846YRDc8nvHe3c9DfuhC2LoYDxc7yjkPh7Aecmv6JoyGoiecgMMa0WI09mCeLyNPAC+7H9wEpngnJ/5RVVHPnmyupdSn/ui2JtuF1DuLVFbDrR8hY6CSA/DRneev20OdC6DkFekyyEhDGmFPW2ETwC+C/gPdxRg/Nx0kG5jTVupQH3lvNtoJy/n3HaHrEt3YKvq15x5nMfecPzoVegSHQ5Sw4/wnoea4zM5iN+DHGNIHGjhoqBx71cCx+6c9fbGTx5gL+fMUgzu4V57T7f/0o/PQSxPaCkTOcX/3dxkFIhLfDNca0QI0dNTQfuEZVS9yPo4FZqnqhJ4Nr6d5dsYvXftjO7eO6cdOYrs6Y/7m/gLXvwln3O9NA2q9+Y4yHNbZpKO5gEgBQ1T0iYlcWn4Yftxby+083MLFPPL+9uD/UVMJHd8LGz5zx/uf8ypKAMeaMaGwicIlIF1XdBSAi3ainGqlpnO2F5dzz9iq6x0Xw3I3DCao9ALNugm2LYeqTMPYeb4dojPEjjU0EvwW+F5FvAQEmADM9FlULVrq/mjvfWElggPCv20bRRsvhrWshayVM/wcMv8nbIRpj/ExjO4u/FpEknIP/apwLyQ54MrCWqLrWxb3vppC5Zz/v3DWWLqHl8OYVkL8JrnkDBkz3dojGGD/U2M7iu4AHgURgDTAWWMaRU1ea41BVHv8slR8yivi/q4cwOmY/vD4dSrPhxlnQ6zxvh2iM8VONnYPwQWAUsFNVJwPDgZLjP8XU9eaPO3h7+S7untiDa7pXwWtTncJwt8yxJGCM8arGJoIKVa0AEJFQVd0E9D3Rk0RkqohsFpEMETnmOgQRmSEiBSKyxv3vrpMLv3lYsjmfJz5P4/wB7fn1sBonCVTvhxmfQ9ezvB2eMcbPNbazOMtdcfQTYL6I7AF2Hu8JIhKIU5LifCALWCkic1U17ahN31fV+08y7mYjPa+MX7y7mr4d2vDs+GoC3rzSmRHs1i8gvo+3wzPGmEZ3Fl/hvvuYiCwGooCvT/C00UCGqm4DEJFZwHTg6ETQYhWXV3Hnm8mEBgfy1uQDtHpvBrRuB7d+CtFdvR2eMcYAjW8aOkRVv1XVuapadYJNE4DMOo+z3MuOdpWIrBOR2SLSub4dichMEUkWkeSCgoKTDdlr/uerTeSWVvDBxCLiPr3JOfjf8bUlAWOMTznpRNDEPgO6qeoQnEJ2b9a3kaq+rKpJqpoUH988qmzuKtrPR6uyeLJXGj0W3QMdBsOML2x2MGOMz/FkIsgG6v7CT3QvO0RVi1S10v3wVWCkB+M5o15YnMGtgfO4cucT0PVspzkoPMbbYRljzDE8mQhWAr1FpLuIhADXA3PrbiAiHes8nAZs9GA8Z8yuwnK6rn2K3we+AX0vgZtmQ6hN+2yM8U0em2VMVWtE5H5gHhAIvKaqqSLyBJCsqnOBB0RkGlADFAMzPBXPGVNbTd7bd3Jv4FfsH3wL4Zc/Y3MFG2N8mqg2r9pxSUlJmpyc7O0w6ldVzoF3bqbVzkUs7ngnk2f+1SqIGmN8goikqGpSfeu83VnccpQXwZuXEbpzCf9VexcDb/iLJQFjTLNgbRZNYc8OePsqXCVZ3FP9MJ3GXkW7NmHejsoYYxrFEsHp2r0O3rkaaip5sctfWZIey3cTe3o7KmOMaTRrGjod276F1y+GgCCyr5jD05tjuGlMVzsbMMY0K5YITtWGj+DtqyAqEe6cz9/WBRIUIPx8Yg9vR2aMMSfFEsGpWP5PmH0nJI6CO75iR3Vb5qzOtrMBY0yzZH0EJ0MVFjwGPzwD/S6Fq16F4FY899la52xgkp0NGGOaH0sEjVVbDXN/AWvfg6Q74OKnICCQHYXlfLImmxlnd6NdpJ0NGGOaH0sEjVG5Dz68DTIWwOTfwTmPHLpG4LlFGQQHCndb34AxppmyRHAiLpfTKZz1E1z2LIy87dCq7YXlzFmdxR3jutvZgDGm2bJEcCJbF0Hmcrj0mSOSAMBzi9IJCQpgpp0NGGOaMRs1dCIr/gmtO8Cwm45YvL2wnE9WZ3PzmK52NmCMadYsERxPYTpkzIdRd0JQyBGrnlvonA3cbVcRG2OaOUsEx/PTyxAYAiNnHLF4W8E+PlmTzS1juxIfGeqd2IwxpolYImhIRSmseRcGXeVMOF/H84synL6Bc+xswBjT/FkiaMjqd6BqH4y5+4jFW91nA7ee1c3OBowxLYIlgvq4auGnl6DzWOg0/IhVh88GbKSQMaZlsERQn/T5zhwD9ZwNfOo+G4hrbWcDxpiWwRJBfVa8CG0SoP9lRyx+bmE6oUGBdjZgjGlRLBEcLX8TbFviDBkNDD60OCN/H3PX5nDrWV3tbMAY06JYIjjaTy9BUBiMmHHE4ucXOWcDP7OzAWNMC2OJoK4De2DtLBh8DUTEHlp86GzgbDsbMMa0PJYI6lr1FlTvP6aT+Dn32cDMCXY2YIxpeSwRHOSqhZ9ega7jocPgQ4uLy6v4bG0ON43pQqydDRhjWiBLBAdt/hJKd8HYnx+xeEFaHi6F6cMSvBSYMcZ4liWCg1a8BFFdoM9FRyyel5pLQttWDEpo46XAjDHGsywRAORugB3fwei7IPDwFA37Kmv4LqOQCwa2R9wzkhljTEtjiQCcOQeCw2HErUcsXrI5n6oaFxcO7OClwIwxxvMsEZQXwfoPYch10Cr6iFXzUvOIjQhhVLcYLwVnjDGeZ4lg1ZtQU3HMkNHKmloWb8rnvP7tCQywZiFjTMvl0UQgIlNFZLOIZIjIo8fZ7ioRURFJ8mQ8x6itgZWvQveJ0K7/Eat+zChiX2UNUwdZs5AxpmXzWCIQkUDgBeAiYABwg4gMqGe7SOBBYIWnYmnQps9gbzaMveeYVfNSc2kdGsTZvWLreaIxxrQcnjwjGA1kqOo2Va0CZgHT69nuj8D/ABUejKV+K16C6G7Q+4IjFte6lPlpeUzqG09oUOAZD8sYY84kTyaCBCCzzuMs97JDRGQE0FlVvzjejkRkpogki0hyQUFB00SXswZ2LYPRMyHgyIN98o5iisqrrFnIGOMXvNZZLCIBwNPAL0+0raq+rKpJqpoUHx/fNAGseAmCI2D4zcesmpeaR0hQAJP6tqvnicYY07J4MhFkA53rPE50LzsoEhgELBGRHcBYYO4Z6TDeVwAbZsOwGyEs6ohVqsq81FzG94qjdWhQAzswxpiWw5OJYCXQW0S6i0gIcD0w9+BKVS1V1ThV7aaq3YDlwDRVTfZgTI6UN6C26pghowCpOXvJLjnAVLuIzBjjJzyWCFS1BrgfmAdsBD5Q1VQReUJEpnnqdU+opsoZMtrrPIjrfczqeam5BAhM6W/NQsYY/+DRtg9V/RL48qhlv29g20mejOWQjXNhXy6Meb7e1V9vyGVUtxgrOW2M8Rv+d2Xxin9CbC/oOeWYVdsK9pGev89GCxlj/Ip/JYKsFMhaCaPvhoBj3/q81DwALrD+AWOMH/GvRLDinxASCcNuqHf116m5DEmMIqFtqzMcmDHGeI//JIKyXEid41w3EBp5zOrc0grWZpZYyWljjN/xn0SQ8ga4amD0z+pd/U1aLgAXDmx/BoMyxhjv858rpkbPhPi+ENuz3tVfb8ilZ3wEvdode7ZgjDEtmf+cEYTHwMAr6l21p7yKFduLrVnIGOOX/CcRHMfCTfnUutQSgTHGL1kiwGkW6hgVxpDEqBNvbIwxLYzfJ4Lyyhq+Sy/gwoEdELEpKY0x/sfvE8G3WwqorHFZs5Axxm/5fSKYl5pLdHgwo7pFezsUY4zxCr9OBFU1LhZtzOe8/u0JCvTrj8IY48f8+uj349ZCyiprrMicMcav+XUimJeaR0RIION6xXk7FGOM8Rq/TQS1LmV+Wh6T+rYjLDjwxE8wxpgWym8TwapdeyjcV8mF1ixkjPFzfpsI5m3IJSQwgMl9470dijHGeJVfJgJVZV5aLuN6xRIZFuztcIwxxqv8MhGk7d5LZvEBu4jMGGPw00QwLzWPAIHzBtjcA8YY45+JYEMuSd1iiGsd6u1QjDHG6/wuEewoLGdzXpk1CxljjJvfJYJ5qc6UlBdYs5AxxgB+mAi+Ts1lUEIbOseEezsUY4zxCX6VCPL2VrB6VwkXDrBmIWOMOcivEsE3aXkAdjWxMcbU4VeJYN6GXHrERdC7XWtvh2KMMT7DbxJB6f5qlm8r4gKbktIYY47gN4lg4aY8alxqcw8YY8xRPJoIRGSqiGwWkQwRebSe9T8XkfUiskZEvheRAZ6KJTIsmPMHtGdIQpSnXsIYY5olUVXP7FgkENgCnA9kASuBG1Q1rc42bVR1r/v+NOBeVZ16vP0mJSVpcnKyR2I2xpiWSkRSVDWpvnWePCMYDWSo6jZVrQJmAdPrbnAwCbhFAJ7JSsYYYxoU5MF9JwCZdR5nAWOO3khE7gP+HxACnFvfjkRkJjAToEuXLk0eqDHG+DOvdxar6guq2hP4NfC7BrZ5WVWTVDUpPt4mkjHGmKbkyUSQDXSu8zjRvawhs4DLPRiPMcaYengyEawEeotIdxEJAa4H5tbdQER613l4CZDuwXiMMcbUw2N9BKpaIyL3A/OAQOA1VU0VkSeAZFWdC9wvIucB1cAe4DZPxWOMMaZ+nuwsRlW/BL48atnv69x/0JOvb4wx5sS83llsjDHGuzx2QZmniEgBsPMUnx4HFDZhOC2RfUbHZ5/PidlndHze+ny6qmq9wy6bXSI4HSKS3NCVdcZhn9Hx2edzYvYZHZ8vfj7WNGSMMX7OEoExxvg5f0sEL3s7gGbAPqPjs8/nxOwzOj6f+3z8qo/AGGPMsfztjMAYY8xRLBEYY4yf85tEcKLZ0vydiOyoM1uczfwDiMhrIpIvIhvqLIsRkfkiku6+jfZmjN7UwOfzmIhku79Ha0TkYm/G6G0i0llEFotImoikisiD7uU+9T3yi0Tgni3tBeAiYABwgyenxWzGJqvqMF8b4+xFbwBHz5j3KLBQVXsDC92P/dUbHPv5APzN/T0a5i4z489qgF+q6gBgLHCf+9jjU98jv0gENGK2NGOOpqpLgeKjFk8H3nTffxM/Lp3ewOdj6lDV3aq6yn2/DNiIM2mXT32P/CUR1DdbWoKXYvFVCnwjIinuGeFM/dqr6m73/VygvTeD8VH3i8g6d9OR3zadHU1EugHDgRX42PfIXxKBObHxqjoCp/nsPhE5x9sB+Tp1xl7b+OsjvQj0BIYBu4G/ejcc3yAirYGPgIeOmqvdJ75H/pIITna2NL+jqtnu23xgDk5zmjlWnoh0BHDf5ns5Hp+iqnmqWquqLuAV7HuEiATjJIF3VPVj92Kf+h75SyI44Wxp/kxEIkQk8uB94AJgw/Gf5bfmcngCpduAT70Yi885eHBzuwI//x6JiAD/Ajaq6tN1VvnU98hvrix2D2N7hsOzpf3ZyyH5DBHpgXMWAM5kRe/a5wMi8h4wCadscB7wB+AT4AOgC0459GtV1S87TBv4fCbhNAspsAO4u05buN8RkfHAd8B6wOVe/J84/QQ+8z3ym0RgjDGmfv7SNGSMMaYBlgiMMcbPWSIwxhg/Z4nAGGP8nCUCY4zxc5YIjDmDRGSSiHzu7TiMqcsSgTHG+DlLBMbUQ0RuFpGf3DX1XxKRQBHZJyJ/c9eVXygi8e5th4nIcnehtTkHC62JSC8RWSAia0VklYj0dO++tYjMFpFNIvKO++pTY7zGEoExRxGR/sB1wDhVHQbUAjcBEUCyqg4EvsW5khbg38CvVXUIzhWkB5e/A7ygqkOBs3GKsIFTgfIhnLkxegDjPP6mjDmOIG8HYIwPmgKMBFa6f6y3wikK5gLed2/zNvCxiEQBbVX1W/fyN4EP3bWbElR1DoCqVgC49/eTqma5H68BugHfe/5tGVM/SwTGHEuAN1X1N0csFPmvo7Y71foslXXu12J/h8bLrGnImGMtBK4WkXZwaH7Zrjh/L1e7t7kR+F5VS4E9IjLBvfwW4Fv3bFRZInK5ex+hIhJ+Rt+FMY1kv0SMOYqqponI73BmbAsAqoH7gHJgtHtdPk4/AjhlhP/pPtBvA253L78FeElEnnDv45oz+DaMaTSrPmpMI4nIPlVt7e04jGlq1jRkjDF+zs4IjDHGz9kZgTHG+DlLBMYY4+csERhjjJ+zRGCMMX7OEoExxvi5/w9pM1GtzDSZ1wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GGEayO89TuOZ",
        "colab_type": "text"
      },
      "source": [
        "![alt text](https://)## Model 7- BiLSTM + Attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k3sFk5WJTtxG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        },
        "outputId": "f0c5e5fc-5e6c-4b25-b88f-139abe685765"
      },
      "source": [
        "from keras import backend as K\n",
        "lstm_units = 128\n",
        "\n",
        "in_text = Input(shape=(windows_size, ))\n",
        "embedding_layer = Embedding(vocab_size_stop+1,\n",
        "                        100,\n",
        "                        weights=[embedding_matrix_lp],\n",
        "                        input_length=windows_size,\n",
        "                        trainable=False)\n",
        "x = embedded_sequences = embedding_layer(in_text)\n",
        "bilstm = Bidirectional(LSTM(lstm_units,return_sequences=True, dropout=0.3, recurrent_dropout=0.25))(x)  \n",
        "attention = TimeDistributed(Dense(1,activation='tanh'))(bilstm)\n",
        "attention = Flatten()(attention)\n",
        "attention = Activation('softmax')(attention)\n",
        "attention = RepeatVector(2*lstm_units)(attention)\n",
        "attention = Permute([2, 1])(attention)\n",
        "sent_representation = multiply([bilstm, attention])\n",
        "out_text = Lambda(lambda xin: K.sum(xin, axis= 1), output_shape=(2*lstm_units,), name='sent')(sent_representation) \n",
        "\n",
        "output_ = Dense(5, activation='softmax')(out_text) \n",
        "\n",
        "bilstm_attn = Model(inputs=in_text, outputs= output_) \n",
        "bilstm_attn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "bilstm_attn.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_4\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_5 (InputLayer)            (None, 10)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_5 (Embedding)         (None, 10, 100)      745000      input_5[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_19 (Bidirectional (None, 10, 256)      234496      embedding_5[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "time_distributed_1 (TimeDistrib (None, 10, 1)        257         bidirectional_19[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "flatten_1 (Flatten)             (None, 10)           0           time_distributed_1[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "activation_1 (Activation)       (None, 10)           0           flatten_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "repeat_vector_6 (RepeatVector)  (None, 256, 10)      0           activation_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "permute_6 (Permute)             (None, 10, 256)      0           repeat_vector_6[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "multiply_5 (Multiply)           (None, 10, 256)      0           bidirectional_19[0][0]           \n",
            "                                                                 permute_6[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "sent (Lambda)                   (None, 256)          0           multiply_5[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "dense_6 (Dense)                 (None, 5)            1285        sent[0][0]                       \n",
            "==================================================================================================\n",
            "Total params: 981,038\n",
            "Trainable params: 236,038\n",
            "Non-trainable params: 745,000\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KA3RCgrvThch",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3669f687-ef16-49e3-8209-b2156796adda"
      },
      "source": [
        "history = bilstm_attn.fit(train_a_b, train_y_b, validation_data=(dev_a_b, dev_y_b), epochs=30, batch_size=64, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 65969 samples, validate on 13195 samples\n",
            "Epoch 1/30\n",
            "65969/65969 [==============================] - 67s 1ms/step - loss: 1.5746 - accuracy: 0.2735 - val_loss: 1.5460 - val_accuracy: 0.2947\n",
            "Epoch 2/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 1.5315 - accuracy: 0.3152 - val_loss: 1.4772 - val_accuracy: 0.3541\n",
            "Epoch 3/30\n",
            "65969/65969 [==============================] - 66s 998us/step - loss: 1.4539 - accuracy: 0.3724 - val_loss: 1.3581 - val_accuracy: 0.4390\n",
            "Epoch 4/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 1.3484 - accuracy: 0.4341 - val_loss: 1.2382 - val_accuracy: 0.5060\n",
            "Epoch 5/30\n",
            "65969/65969 [==============================] - 67s 1ms/step - loss: 1.2444 - accuracy: 0.4919 - val_loss: 1.1044 - val_accuracy: 0.5736\n",
            "Epoch 6/30\n",
            "65969/65969 [==============================] - 67s 1ms/step - loss: 1.1513 - accuracy: 0.5395 - val_loss: 0.9962 - val_accuracy: 0.6255\n",
            "Epoch 7/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 1.0735 - accuracy: 0.5746 - val_loss: 0.8989 - val_accuracy: 0.6709\n",
            "Epoch 8/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 1.0040 - accuracy: 0.6115 - val_loss: 0.8172 - val_accuracy: 0.7094\n",
            "Epoch 9/30\n",
            "65969/65969 [==============================] - 66s 998us/step - loss: 0.9419 - accuracy: 0.6378 - val_loss: 0.7395 - val_accuracy: 0.7411\n",
            "Epoch 10/30\n",
            "65969/65969 [==============================] - 66s 999us/step - loss: 0.8922 - accuracy: 0.6594 - val_loss: 0.6744 - val_accuracy: 0.7704\n",
            "Epoch 11/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.8450 - accuracy: 0.6798 - val_loss: 0.6217 - val_accuracy: 0.7919\n",
            "Epoch 12/30\n",
            "65969/65969 [==============================] - 66s 997us/step - loss: 0.7999 - accuracy: 0.6992 - val_loss: 0.5634 - val_accuracy: 0.8119\n",
            "Epoch 13/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.7672 - accuracy: 0.7121 - val_loss: 0.5220 - val_accuracy: 0.8287\n",
            "Epoch 14/30\n",
            "65969/65969 [==============================] - 66s 994us/step - loss: 0.7361 - accuracy: 0.7251 - val_loss: 0.4909 - val_accuracy: 0.8412\n",
            "Epoch 15/30\n",
            "65969/65969 [==============================] - 66s 995us/step - loss: 0.7082 - accuracy: 0.7360 - val_loss: 0.4578 - val_accuracy: 0.8487\n",
            "Epoch 16/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.6871 - accuracy: 0.7447 - val_loss: 0.4206 - val_accuracy: 0.8636\n",
            "Epoch 17/30\n",
            "65969/65969 [==============================] - 66s 1000us/step - loss: 0.6608 - accuracy: 0.7546 - val_loss: 0.3991 - val_accuracy: 0.8720\n",
            "Epoch 18/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.6397 - accuracy: 0.7632 - val_loss: 0.3775 - val_accuracy: 0.8809\n",
            "Epoch 19/30\n",
            "65969/65969 [==============================] - 66s 996us/step - loss: 0.6244 - accuracy: 0.7687 - val_loss: 0.3503 - val_accuracy: 0.8924\n",
            "Epoch 20/30\n",
            "65969/65969 [==============================] - 66s 996us/step - loss: 0.6065 - accuracy: 0.7746 - val_loss: 0.3292 - val_accuracy: 0.8990\n",
            "Epoch 21/30\n",
            "65969/65969 [==============================] - 66s 998us/step - loss: 0.5967 - accuracy: 0.7788 - val_loss: 0.3101 - val_accuracy: 0.9053\n",
            "Epoch 22/30\n",
            "65969/65969 [==============================] - 66s 998us/step - loss: 0.5786 - accuracy: 0.7866 - val_loss: 0.3026 - val_accuracy: 0.9044\n",
            "Epoch 23/30\n",
            "65969/65969 [==============================] - 66s 1000us/step - loss: 0.5651 - accuracy: 0.7935 - val_loss: 0.2862 - val_accuracy: 0.9106\n",
            "Epoch 24/30\n",
            "65969/65969 [==============================] - 66s 998us/step - loss: 0.5542 - accuracy: 0.7966 - val_loss: 0.2705 - val_accuracy: 0.9199\n",
            "Epoch 25/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.5395 - accuracy: 0.8016 - val_loss: 0.2569 - val_accuracy: 0.9235\n",
            "Epoch 26/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.5350 - accuracy: 0.8035 - val_loss: 0.2484 - val_accuracy: 0.9267\n",
            "Epoch 27/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.5226 - accuracy: 0.8084 - val_loss: 0.2380 - val_accuracy: 0.9287\n",
            "Epoch 28/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.5115 - accuracy: 0.8128 - val_loss: 0.2226 - val_accuracy: 0.9338\n",
            "Epoch 29/30\n",
            "65969/65969 [==============================] - 66s 1ms/step - loss: 0.5037 - accuracy: 0.8156 - val_loss: 0.2200 - val_accuracy: 0.9335\n",
            "Epoch 30/30\n",
            "65969/65969 [==============================] - 66s 993us/step - loss: 0.4956 - accuracy: 0.8188 - val_loss: 0.2074 - val_accuracy: 0.9382\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vPwJPiW4Us8F",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "8f1b0762-d741-4e6c-8e32-ac4c03a786ed"
      },
      "source": [
        "# evaluate the model\n",
        "score = bilstm_attn.evaluate(test_a_b, test_y_b, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.207232202045702\n",
            "Test accuracy: 0.9411764740943909\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IE8GlAwAAP1g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.layers import Dense, Input,LSTM,Embedding, Dropout, Activation, Conv1D,RepeatVector, Permute, Multiply\n",
        "from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\n",
        "from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\n",
        "from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\n",
        "from keras.optimizers import Adam\n",
        "from keras import backend as K\n",
        "from keras.models import Model\n",
        "from keras.engine.topology import Layer\n",
        "from keras import initializers, regularizers, constraints, optimizers, layers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZqGscCHAy8u6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "class AttentionLayer(Layer):\n",
        "    \"\"\"\n",
        "    Attention operation, with a context/query vector, for temporal data.\n",
        "    Supports Masking.\n",
        "    Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]\n",
        "    \"Hierarchical Attention Networks for Document Classification\"\n",
        "    by using a context vector to assist the attention\n",
        "    # Input shape\n",
        "        3D tensor with shape: `(samples, steps, features)`.\n",
        "    # Output shape\n",
        "        2D tensor with shape: `(samples, features)`.\n",
        "    How to use:\n",
        "    Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
        "    The dimensions are inferred based on the output shape of the RNN.\n",
        "    Note: The layer has been tested with Keras 2.0.6\n",
        "    Example:\n",
        "        model.add(LSTM(64, return_sequences=True))\n",
        "        model.add(AttentionWithContext())\n",
        "        # next add a Dense layer (for classification/regression) or whatever...\n",
        "    \"\"\"\n",
        "    def __init__(self, **kwargs):\n",
        "        self.init = initializers.get('glorot_uniform')\n",
        "        super(AttentionLayer, self).__init__(**kwargs)\n",
        "    \n",
        "    def build(self, input_shape):\n",
        "        assert len(input_shape) == 3\n",
        "        \n",
        "        self.W = self.add_weight(name='Attention_Weight',\n",
        "                                 shape=(input_shape[-1], input_shape[-1]),\n",
        "                                 initializer=self.init,\n",
        "                                 trainable=True)\n",
        "        self.b = self.add_weight(name='Attention_Bias',\n",
        "                                 shape=(input_shape[-1], ),\n",
        "                                 initializer=self.init,\n",
        "                                 trainable=True)\n",
        "        self.u = self.add_weight(name='Attention_Context_Vector',\n",
        "                                 shape=(input_shape[-1], 1),\n",
        "                                 initializer=self.init,\n",
        "                                 trainable=True)\n",
        "        super(AttentionLayer, self).build(input_shape)\n",
        "\n",
        "    def compute_mask(self, input, input_mask=None):\n",
        "        # do not pass the mask to the next layers\n",
        "        return None\n",
        "    \n",
        "    def call(self, x):\n",
        "        # refer to the original paper\n",
        "        # link: https://www.cs.cmu.edu/~hovy/papers/16HLT-hierarchical-attention-networks.pdf\n",
        "        \n",
        "        u_it = K.tanh(K.dot(x, self.W) + self.b)\n",
        "        \n",
        "       #Through structure the vector applied  is used as\n",
        "        #Make attention value into probability distribution through\n",
        "        a_it = K.dot(u_it, self.u)\n",
        "        a_it = K.squeeze(a_it, -1)\n",
        "        a_it = K.softmax(a_it)\n",
        "        \n",
        "        return a_it\n",
        "        \n",
        "    def compute_output_shape(self, input_shape):\n",
        "        return (input_shape[0], input_shape[1])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Q844pvxI__m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "answer_inp = Input(shape=(windows_size, ))\n",
        "embedding_size_glove = 100\n",
        "answer_emb1 = Embedding(vocab_size_stop+1, embedding_size_glove, weights=[embedding_matrix_lp], input_length=windows_size, trainable=False)(answer_inp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "94L8kk3dCDSp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2efc10b5-1980-41f8-f87d-8033b6a20124"
      },
      "source": [
        "lstm_dim=50\n",
        "bilstm = Bidirectional(LSTM(lstm_dim, return_sequences=True))(answer_emb1)  #Input should be three dimensional\n",
        "# bilstm = LSTM(2*lstm_dim, return_sequences=True)(input_data) \n",
        "# bilstm_output = Dense(1)(bilstm)\n",
        "\n",
        "attention_layer = AttentionLayer()(bilstm)\n",
        "print(attention_layer)\n",
        "repeated_word_attention = RepeatVector(lstm_dim * 2)(attention_layer)\n",
        "repeated_word_attention = Permute([2, 1])(repeated_word_attention)\n",
        "sentence_representation = Multiply()([bilstm, repeated_word_attention])\n",
        "sentence_representation = Lambda(lambda x: K.sum(x, axis=1))(sentence_representation) #total summation of the multiplied bilstm and attention \n",
        "\n",
        "bilstm_output = Dense(5,activation='softmax')(sentence_representation)\n",
        "\n",
        "model = Model(inputs=[answer_inp],\n",
        "            outputs=[bilstm_output])\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy'])\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tensor(\"attention_layer_7/Softmax:0\", shape=(?, 10), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VY-LNm4PMJzw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        },
        "outputId": "fbc31a94-646c-4cdf-e475-15a354ecb411"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_4 (InputLayer)            (None, 10)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_4 (Embedding)         (None, 10, 100)      745000      input_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_18 (Bidirectional (None, 10, 100)      60400       embedding_4[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "attention_layer_7 (AttentionLay (None, 10)           10200       bidirectional_18[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "repeat_vector_5 (RepeatVector)  (None, 100, 10)      0           attention_layer_7[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "permute_5 (Permute)             (None, 10, 100)      0           repeat_vector_5[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "multiply_4 (Multiply)           (None, 10, 100)      0           bidirectional_18[0][0]           \n",
            "                                                                 permute_5[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lambda_4 (Lambda)               (None, 100)          0           multiply_4[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 5)            505         lambda_4[0][0]                   \n",
            "==================================================================================================\n",
            "Total params: 816,105\n",
            "Trainable params: 71,105\n",
            "Non-trainable params: 745,000\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bxM2yMamQr_e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "b52f1d42-16c5-4630-c327-e3e83e42d69d"
      },
      "source": [
        "history = model.fit(train_a_b, train_y_b, validation_data=(dev_a_b, dev_y_b), epochs=30, batch_size=64, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 65969 samples, validate on 13195 samples\n",
            "Epoch 1/30\n",
            "65969/65969 [==============================] - 23s 349us/step - loss: 1.5496 - accuracy: 0.2982 - val_loss: 1.4944 - val_accuracy: 0.3444\n",
            "Epoch 2/30\n",
            "65969/65969 [==============================] - 22s 339us/step - loss: 1.4263 - accuracy: 0.3918 - val_loss: 1.3625 - val_accuracy: 0.4337\n",
            "Epoch 3/30\n",
            "65969/65969 [==============================] - 22s 336us/step - loss: 1.2535 - accuracy: 0.4949 - val_loss: 1.2268 - val_accuracy: 0.5076\n",
            "Epoch 4/30\n",
            "65969/65969 [==============================] - 22s 335us/step - loss: 1.0769 - accuracy: 0.5841 - val_loss: 1.0899 - val_accuracy: 0.5773\n",
            "Epoch 5/30\n",
            "65969/65969 [==============================] - 22s 334us/step - loss: 0.9249 - accuracy: 0.6551 - val_loss: 1.0196 - val_accuracy: 0.6090\n",
            "Epoch 6/30\n",
            "65969/65969 [==============================] - 22s 335us/step - loss: 0.7952 - accuracy: 0.7120 - val_loss: 0.8914 - val_accuracy: 0.6676\n",
            "Epoch 7/30\n",
            "65969/65969 [==============================] - 24s 358us/step - loss: 0.6892 - accuracy: 0.7554 - val_loss: 0.8334 - val_accuracy: 0.6919\n",
            "Epoch 8/30\n",
            "65969/65969 [==============================] - 22s 333us/step - loss: 0.6038 - accuracy: 0.7891 - val_loss: 0.7772 - val_accuracy: 0.7148\n",
            "Epoch 9/30\n",
            "65969/65969 [==============================] - 22s 336us/step - loss: 0.5314 - accuracy: 0.8192 - val_loss: 0.7022 - val_accuracy: 0.7501\n",
            "Epoch 10/30\n",
            "65969/65969 [==============================] - 22s 337us/step - loss: 0.4678 - accuracy: 0.8432 - val_loss: 0.6865 - val_accuracy: 0.7528\n",
            "Epoch 11/30\n",
            "65969/65969 [==============================] - 22s 340us/step - loss: 0.4161 - accuracy: 0.8621 - val_loss: 0.6507 - val_accuracy: 0.7721\n",
            "Epoch 12/30\n",
            "65969/65969 [==============================] - 22s 341us/step - loss: 0.3699 - accuracy: 0.8792 - val_loss: 0.6195 - val_accuracy: 0.7883\n",
            "Epoch 13/30\n",
            "65969/65969 [==============================] - 23s 342us/step - loss: 0.3284 - accuracy: 0.8952 - val_loss: 0.6053 - val_accuracy: 0.8003\n",
            "Epoch 14/30\n",
            "65969/65969 [==============================] - 23s 342us/step - loss: 0.2977 - accuracy: 0.9047 - val_loss: 0.5680 - val_accuracy: 0.8129\n",
            "Epoch 15/30\n",
            "65969/65969 [==============================] - 23s 348us/step - loss: 0.2647 - accuracy: 0.9184 - val_loss: 0.5772 - val_accuracy: 0.8129\n",
            "Epoch 16/30\n",
            "65969/65969 [==============================] - 23s 343us/step - loss: 0.2410 - accuracy: 0.9256 - val_loss: 0.5435 - val_accuracy: 0.8233\n",
            "Epoch 17/30\n",
            "65969/65969 [==============================] - 22s 339us/step - loss: 0.2167 - accuracy: 0.9340 - val_loss: 0.5183 - val_accuracy: 0.8340\n",
            "Epoch 18/30\n",
            "65969/65969 [==============================] - 22s 337us/step - loss: 0.1948 - accuracy: 0.9417 - val_loss: 0.5654 - val_accuracy: 0.8242\n",
            "Epoch 19/30\n",
            "65969/65969 [==============================] - 22s 338us/step - loss: 0.1791 - accuracy: 0.9469 - val_loss: 0.5135 - val_accuracy: 0.8377\n",
            "Epoch 20/30\n",
            "65969/65969 [==============================] - 22s 339us/step - loss: 0.1594 - accuracy: 0.9539 - val_loss: 0.5032 - val_accuracy: 0.8421\n",
            "Epoch 21/30\n",
            "65969/65969 [==============================] - 22s 337us/step - loss: 0.1524 - accuracy: 0.9547 - val_loss: 0.5040 - val_accuracy: 0.8471\n",
            "Epoch 22/30\n",
            "65969/65969 [==============================] - 22s 336us/step - loss: 0.1364 - accuracy: 0.9615 - val_loss: 0.5154 - val_accuracy: 0.8440\n",
            "Epoch 23/30\n",
            "65969/65969 [==============================] - 22s 338us/step - loss: 0.1256 - accuracy: 0.9652 - val_loss: 0.4880 - val_accuracy: 0.8496\n",
            "Epoch 24/30\n",
            "65969/65969 [==============================] - 22s 338us/step - loss: 0.1142 - accuracy: 0.9683 - val_loss: 0.4736 - val_accuracy: 0.8603\n",
            "Epoch 25/30\n",
            "65969/65969 [==============================] - 22s 337us/step - loss: 0.1054 - accuracy: 0.9710 - val_loss: 0.4816 - val_accuracy: 0.8571\n",
            "Epoch 26/30\n",
            "65969/65969 [==============================] - 22s 335us/step - loss: 0.1042 - accuracy: 0.9710 - val_loss: 0.4986 - val_accuracy: 0.8565\n",
            "Epoch 27/30\n",
            "65969/65969 [==============================] - 22s 336us/step - loss: 0.0861 - accuracy: 0.9776 - val_loss: 0.4807 - val_accuracy: 0.8668\n",
            "Epoch 28/30\n",
            "65969/65969 [==============================] - 22s 335us/step - loss: 0.0910 - accuracy: 0.9756 - val_loss: 0.4644 - val_accuracy: 0.8668\n",
            "Epoch 29/30\n",
            "65969/65969 [==============================] - 22s 339us/step - loss: 0.0792 - accuracy: 0.9794 - val_loss: 0.4568 - val_accuracy: 0.8700\n",
            "Epoch 30/30\n",
            "65969/65969 [==============================] - 22s 341us/step - loss: 0.0721 - accuracy: 0.9819 - val_loss: 0.4931 - val_accuracy: 0.8665\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GG1aF_TdQyOy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "265249a8-4ec3-4a52-f6f8-b9846b267e37"
      },
      "source": [
        "# evaluate the model\n",
        "score = model.evaluate(test_a_b, test_y_b, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.502733766115568\n",
            "Test accuracy: 0.8649114370346069\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dt5mW9UTT39G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}