Tagged_LSTM/Model_Final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AhaWewB3itRZ"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import pickle\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import boto3\n",
    "import pandas as pd\n",
    "from sagemaker import get_execution_role\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, Embedding, LSTM, GRU\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.layers import Dropout, SpatialDropout1D, Input, concatenate\n",
    "from tensorflow.core.protobuf import rewriter_config_pb2\n",
    "from tensorflow.keras.backend import set_session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 125
    },
    "colab_type": "code",
    "id": "UD6iZMIALJ8H",
    "outputId": "0e3ccf63-f1a3-46c1-e9f7-391ff31a155b"
   },
   "outputs": [],
   "source": [
    "role = get_execution_role()\n",
    "bucket='sagemaker-tagger'\n",
    "data_key1 = 'update_twelve_three.csv'\n",
    "data_location1 = 's3://{}/{}'.format(bucket, data_key1)\n",
    "\n",
    "df = pd.read_csv(data_location1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "qm1usu18VrKN",
    "outputId": "da4abaa4-95d2-4a07-aa2d-ae0a195540e1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Dear AVRAHAM JACOBSOHN,  This is to confirm that a card payment for 142.48 was made to your Grange account ending in 1173 using the card ending in 2308.  Confirmation Number: 6786423  Scheduled Payment Date: 11/08/2019  Payment Amount: 142.48  Last 4 of Card: 2308  Please contact Grange Insurance at 1-800-425-1100 if you have any questions.  Sincerely,  Grange Insurance  Please note:  This email message was sent from a notification-only address that cannot accept incoming email.  Please do not reply to this message.  '"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Message'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LxhryVWcdwzX"
   },
   "outputs": [],
   "source": [
    "# Change the name of the column with From + Subject + Message\n",
    "\n",
    "df.rename(columns={'text':'Combined_Text'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VEhxUZs4GxTl"
   },
   "outputs": [],
   "source": [
    "# Function to clean the text prior to tokenizing\n",
    "\n",
    "def clean_text(text):\n",
    "    # Perform a few cleaning steps to remove non-alphabetic characters\n",
    "    \n",
    "    text = text.replace(\"\\n\", \" \").replace(\"\\r\", \" \")\n",
    "\n",
    "    text = text.strip(\" \")\n",
    "    \n",
    "    punc_list = '!@#$%^&*()+?-_=:.<>[]{}/\\~\",©' + '1234567890'\n",
    "    t = str.maketrans(dict.fromkeys(punc_list, \" \"))\n",
    "    text = text.translate(t)\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MqwHdyjfWM1U"
   },
   "outputs": [],
   "source": [
    "# Clean the From Message and Subject before separating to different models\n",
    "\n",
    "df['Clean_Message'] = df['Message'].apply(clean_text)\n",
    "\n",
    "df['Clean_From'] = df['From'].apply(clean_text)\n",
    "\n",
    "df['Clean_Subject'] = df['Subject'].apply(clean_text)\n",
    "\n",
    "df['Combined_Text'] = df['Combined_Text'].apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 534
    },
    "colab_type": "code",
    "id": "bS_B87XnWOk0",
    "outputId": "ac38643d-3b0d-4caa-f8b8-7ffc9504d73a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>From</th>\n",
       "      <th>Message</th>\n",
       "      <th>Subject</th>\n",
       "      <th>Tags</th>\n",
       "      <th>UID</th>\n",
       "      <th>Combined_Text</th>\n",
       "      <th>tag_list</th>\n",
       "      <th>first_tag</th>\n",
       "      <th>sender_name</th>\n",
       "      <th>sender_email</th>\n",
       "      <th>domain_name</th>\n",
       "      <th>isNoReply</th>\n",
       "      <th>Clean_Message</th>\n",
       "      <th>Clean_From</th>\n",
       "      <th>Clean_Subject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;grangepayments@westernunionspeedpay.com&gt;</td>\n",
       "      <td>Dear AVRAHAM JACOBSOHN,  This is to confirm th...</td>\n",
       "      <td>Grange Payment Confirmation</td>\n",
       "      <td>Finance</td>\n",
       "      <td>31780</td>\n",
       "      <td>grangepayments westernunionspeedpay com  Gran...</td>\n",
       "      <td>['Finance']</td>\n",
       "      <td>Finance</td>\n",
       "      <td>NaN</td>\n",
       "      <td>grangepayments@westernunionspeedpay.com</td>\n",
       "      <td>westernunionspeedpay</td>\n",
       "      <td>False</td>\n",
       "      <td>Dear AVRAHAM JACOBSOHN   This is to confirm th...</td>\n",
       "      <td>grangepayments westernunionspeedpay com</td>\n",
       "      <td>Grange Payment Confirmation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chase &lt;no.reply.alerts@chase.com&gt;</td>\n",
       "      <td>This is an Alert to help manage your account ...</td>\n",
       "      <td>Your Debit Card Transaction</td>\n",
       "      <td>Finance</td>\n",
       "      <td>31779</td>\n",
       "      <td>Chase  no reply alerts chase com  Your Debit C...</td>\n",
       "      <td>['Finance']</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Chase</td>\n",
       "      <td>no.reply.alerts@chase.com</td>\n",
       "      <td>chase</td>\n",
       "      <td>True</td>\n",
       "      <td>This is an Alert to help manage your account e...</td>\n",
       "      <td>Chase  no reply alerts chase com</td>\n",
       "      <td>Your Debit Card Transaction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Amazon Web Services &lt;no-reply-aws@amazon.com&gt;</td>\n",
       "      <td>Please let us know if we helped resolve your i...</td>\n",
       "      <td>Resolved 6559329691: Limit Increase: SageMaker</td>\n",
       "      <td>Productivity</td>\n",
       "      <td>31738</td>\n",
       "      <td>Amazon Web Services  no reply aws amazon com  ...</td>\n",
       "      <td>['Productivity']</td>\n",
       "      <td>Productivity</td>\n",
       "      <td>Amazon Web Services</td>\n",
       "      <td>no-reply-aws@amazon.com</td>\n",
       "      <td>amazon</td>\n",
       "      <td>True</td>\n",
       "      <td>Please let us know if we helped resolve your i...</td>\n",
       "      <td>Amazon Web Services  no reply aws amazon com</td>\n",
       "      <td>Resolved             Limit Increase  SageMaker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Lambda Labs &lt;noreply@github.com&gt;</td>\n",
       "      <td>Youve been added to the Labs 18 - Tagger team ...</td>\n",
       "      <td>Bernie Durfee added you to the Lambda Labs tea...</td>\n",
       "      <td>Productivity</td>\n",
       "      <td>31693</td>\n",
       "      <td>Lambda Labs  noreply github com  Bernie Durfee...</td>\n",
       "      <td>['Productivity']</td>\n",
       "      <td>Productivity</td>\n",
       "      <td>Lambda Labs</td>\n",
       "      <td>noreply@github.com</td>\n",
       "      <td>github</td>\n",
       "      <td>True</td>\n",
       "      <td>Youve been added to the Labs      Tagger team ...</td>\n",
       "      <td>Lambda Labs  noreply github com</td>\n",
       "      <td>Bernie Durfee added you to the Lambda Labs tea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Amazon Web Services &lt;no-reply-aws@amazon.com&gt;</td>\n",
       "      <td>Hello,  We haven't heard back from you regard...</td>\n",
       "      <td>Attention required on case 6559329691: Limit I...</td>\n",
       "      <td>Productivity</td>\n",
       "      <td>31684</td>\n",
       "      <td>Amazon Web Services  no reply aws amazon com  ...</td>\n",
       "      <td>['Productivity']</td>\n",
       "      <td>Productivity</td>\n",
       "      <td>Amazon Web Services</td>\n",
       "      <td>no-reply-aws@amazon.com</td>\n",
       "      <td>amazon</td>\n",
       "      <td>True</td>\n",
       "      <td>Hello   We haven't heard back from you regardi...</td>\n",
       "      <td>Amazon Web Services  no reply aws amazon com</td>\n",
       "      <td>Attention required on case             Limit I...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            From  \\\n",
       "0      <grangepayments@westernunionspeedpay.com>   \n",
       "1              Chase <no.reply.alerts@chase.com>   \n",
       "2  Amazon Web Services <no-reply-aws@amazon.com>   \n",
       "3               Lambda Labs <noreply@github.com>   \n",
       "4  Amazon Web Services <no-reply-aws@amazon.com>   \n",
       "\n",
       "                                             Message  \\\n",
       "0  Dear AVRAHAM JACOBSOHN,  This is to confirm th...   \n",
       "1   This is an Alert to help manage your account ...   \n",
       "2  Please let us know if we helped resolve your i...   \n",
       "3  Youve been added to the Labs 18 - Tagger team ...   \n",
       "4   Hello,  We haven't heard back from you regard...   \n",
       "\n",
       "                                             Subject          Tags    UID  \\\n",
       "0                        Grange Payment Confirmation       Finance  31780   \n",
       "1                        Your Debit Card Transaction       Finance  31779   \n",
       "2     Resolved 6559329691: Limit Increase: SageMaker  Productivity  31738   \n",
       "3  Bernie Durfee added you to the Lambda Labs tea...  Productivity  31693   \n",
       "4  Attention required on case 6559329691: Limit I...  Productivity  31684   \n",
       "\n",
       "                                       Combined_Text          tag_list  \\\n",
       "0   grangepayments westernunionspeedpay com  Gran...       ['Finance']   \n",
       "1  Chase  no reply alerts chase com  Your Debit C...       ['Finance']   \n",
       "2  Amazon Web Services  no reply aws amazon com  ...  ['Productivity']   \n",
       "3  Lambda Labs  noreply github com  Bernie Durfee...  ['Productivity']   \n",
       "4  Amazon Web Services  no reply aws amazon com  ...  ['Productivity']   \n",
       "\n",
       "      first_tag          sender_name                             sender_email  \\\n",
       "0       Finance                  NaN  grangepayments@westernunionspeedpay.com   \n",
       "1       Finance                Chase                no.reply.alerts@chase.com   \n",
       "2  Productivity  Amazon Web Services                  no-reply-aws@amazon.com   \n",
       "3  Productivity          Lambda Labs                       noreply@github.com   \n",
       "4  Productivity  Amazon Web Services                  no-reply-aws@amazon.com   \n",
       "\n",
       "            domain_name  isNoReply  \\\n",
       "0  westernunionspeedpay      False   \n",
       "1                 chase       True   \n",
       "2                amazon       True   \n",
       "3                github       True   \n",
       "4                amazon       True   \n",
       "\n",
       "                                       Clean_Message  \\\n",
       "0  Dear AVRAHAM JACOBSOHN   This is to confirm th...   \n",
       "1  This is an Alert to help manage your account e...   \n",
       "2  Please let us know if we helped resolve your i...   \n",
       "3  Youve been added to the Labs      Tagger team ...   \n",
       "4  Hello   We haven't heard back from you regardi...   \n",
       "\n",
       "                                      Clean_From  \\\n",
       "0       grangepayments westernunionspeedpay com    \n",
       "1              Chase  no reply alerts chase com    \n",
       "2  Amazon Web Services  no reply aws amazon com    \n",
       "3               Lambda Labs  noreply github com    \n",
       "4  Amazon Web Services  no reply aws amazon com    \n",
       "\n",
       "                                       Clean_Subject  \n",
       "0                        Grange Payment Confirmation  \n",
       "1                        Your Debit Card Transaction  \n",
       "2     Resolved             Limit Increase  SageMaker  \n",
       "3  Bernie Durfee added you to the Lambda Labs tea...  \n",
       "4  Attention required on case             Limit I...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 603
    },
    "colab_type": "code",
    "id": "0fNuPHE1SdLi",
    "outputId": "2d681018-0d84-46e4-c74b-b1b1dd8c526b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Finance']                      5604\n",
       "['Entertainment']                1505\n",
       "['Shopping']                     1065\n",
       "['Personal', 'Other']             685\n",
       "['Other']                         514\n",
       "['Productivity']                  514\n",
       "['Personal', 'Productivity']      382\n",
       "['Social']                        146\n",
       "['Productivity', 'Events']        130\n",
       "['Entertainment', 'Finance']      102\n",
       "['Events']                         95\n",
       "['Personal', 'Shopping']           95\n",
       "['Personal', 'Events']             95\n",
       "['Travel']                         92\n",
       "['Personal', 'Finance']            69\n",
       "['Personal', 'Travel']             25\n",
       "['Personal', 'Entertainment']      19\n",
       "['Shopping', 'Finance']            13\n",
       "['Entertainment', 'Shopping']      10\n",
       "['Events', 'Productivity']          8\n",
       "['Travel', 'Finance']               5\n",
       "['Productivity', 'Finance']         4\n",
       "['Personal', 'Social']              4\n",
       "['Shopping', 'Entertainment']       4\n",
       "['Social', 'Productivity']          4\n",
       "['Shopping', 'Productivity']        2\n",
       "['Finance', 'Travel']               2\n",
       "['Productivity', 'Shopping']        1\n",
       "['Finance', 'Events']               1\n",
       "['Finance', 'Productivity']         1\n",
       "['Events', 'Entertainment']         1\n",
       "['Finance', 'Shopping']             1\n",
       "Name: tag_list, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tag_list'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 603
    },
    "colab_type": "code",
    "id": "ph6GGRbdSyzm",
    "outputId": "13939edc-b2f3-4f2e-db23-d9061a09d496"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Finance                    5604\n",
       "Entertainment              1505\n",
       "Shopping                   1065\n",
       "Personal, Other             685\n",
       "Productivity                514\n",
       "Other                       514\n",
       "Personal, Productivity      382\n",
       "Social                      146\n",
       "Productivity, Events        130\n",
       "Entertainment, Finance      102\n",
       "Events                       95\n",
       "Personal, Events             95\n",
       "Personal, Shopping           95\n",
       "Travel                       92\n",
       "Personal, Finance            69\n",
       "Personal, Travel             25\n",
       "Personal, Entertainment      19\n",
       "Shopping, Finance            13\n",
       "Entertainment, Shopping      10\n",
       "Events, Productivity          8\n",
       "Travel, Finance               5\n",
       "Social, Productivity          4\n",
       "Shopping, Entertainment       4\n",
       "Personal, Social              4\n",
       "Productivity, Finance         4\n",
       "Shopping, Productivity        2\n",
       "Finance, Travel               2\n",
       "Finance, Events               1\n",
       "Productivity, Shopping        1\n",
       "Finance, Productivity         1\n",
       "Finance, Shopping             1\n",
       "Events, Entertainment         1\n",
       "Name: Tags, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Tags'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "colab_type": "code",
    "id": "kj2T37nOSmu_",
    "outputId": "8b48f09d-f4b6-4885-977d-04fbba43e133"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    9824\n",
       "True     1374\n",
       "Name: Personal, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating Personal Column\n",
    "\n",
    "df['Personal'] = df['Tags'].str.contains('Personal')\n",
    "df['Personal'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "gaBzqwYYG0jA",
    "outputId": "04cee3d0-a0f8-4d15-ef3a-f7ea41139a0a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 3732402\n"
     ]
    }
   ],
   "source": [
    "# Applying length of CLEAN messages as feature\n",
    "\n",
    "df['Clean_Text_Length'] = df['Clean_Message'].apply(len)\n",
    "print(df['Clean_Text_Length'].min(), df['Clean_Text_Length'].max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "colab_type": "code",
    "id": "dp5Bj6JnG2Se",
    "outputId": "a7f4e25a-03e6-4993-d937-5ee411586cd4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     10544\n",
       "False      654\n",
       "Name: Clean_Text_Length, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Most lengths are below 5000 even with a max of 370k length\n",
    "\n",
    "clean_text_length = df['Clean_Text_Length'] < 5000\n",
    "clean_text_length.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sEAN2_5PG39c"
   },
   "outputs": [],
   "source": [
    "# lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "# def regnltk_tokenize(text):\n",
    "#     text = clean_text(text)\n",
    "#     words = regexp_tokenize(text, pattern = '\\s+', gaps = True)\n",
    "#     return [lemmatizer.lemmatize(word.lower()) for word in words if (len(word) >= 3)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dpfetY0tZNiF"
   },
   "outputs": [],
   "source": [
    "# Remove too small and too large texts from the messages before tokenizing\n",
    "\n",
    "message_values = df['Combined_Text'].values\n",
    "\n",
    "tokens = []\n",
    "\n",
    "for i in range(len(message_values)):\n",
    "    value = message_values[i]\n",
    "    value = value.split(\" \")\n",
    "    value = [x for x in value if len(x) > 3 and len(x) < 20]\n",
    "    value = \" \".join(value)\n",
    "    tokens.append(value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6ERBtrKuIZHw"
   },
   "source": [
    "## Start of Keras Work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "QaH-6EDDG65r",
    "outputId": "f1753a97-70c0-4e42-8e03-3a489975ff70"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 407235 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "# Starting tokenizing from keras preprocessing\n",
    "\n",
    "tokenizer = Tokenizer(num_words=5000, filters='!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~', lower=True, split=' ')\n",
    "tokenizer.fit_on_texts(tokens)\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "# saving\n",
    "with open('final_tokenizer.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "l7ImvnQAPopT",
    "outputId": "b1171f85-33fd-49c6-d8b1-dc4ba1f1fb5e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10078, 15) (10078,)\n",
      "(1120, 15) (1120,)\n"
     ]
    }
   ],
   "source": [
    "# Split the data into train/test prior to prepping for each model\n",
    "\n",
    "X = df.drop(columns=['tag_list', 'first_tag'])\n",
    "\n",
    "Y = df['first_tag']\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.10, random_state = 42)\n",
    "print(X_train.shape,Y_train.shape)\n",
    "print(X_test.shape,Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Inv2Xxd5e9yH"
   },
   "source": [
    "#### Format all of the inputs for each Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "fkUoeZlRPjCi",
    "outputId": "5537b812-44b2-44f9-a3ea-a8d203ce3f62"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data tensor: (10078, 5000)\n",
      "Shape of data tensor: (1120, 5000)\n"
     ]
    }
   ],
   "source": [
    "# Format the inputs for Model 1 of predicting tag from message body\n",
    "\n",
    "Message_X_train = tokenizer.texts_to_sequences(X_train['Clean_Message'].values)\n",
    "Message_X_train = pad_sequences(Message_X_train, maxlen=5000)\n",
    "print('Shape of data tensor:', Message_X_train.shape)\n",
    "\n",
    "Message_X_test = tokenizer.texts_to_sequences(X_test['Clean_Message'].values)\n",
    "Message_X_test = pad_sequences(Message_X_test, maxlen=5000)\n",
    "print('Shape of data tensor:', Message_X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "TkHlIFgZT_w3",
    "outputId": "1324624c-505c-4bcb-ea10-e4f326104800"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data tensor: (10078, 150)\n",
      "Shape of data tensor: (1120, 150)\n"
     ]
    }
   ],
   "source": [
    "# Format the inputs for Model 2 of predicting tag from message subject\n",
    "\n",
    "# First need to get length of subject messages\n",
    "\n",
    "clean_subject_length = df['Clean_Subject'].apply(len)\n",
    "(clean_subject_length < 150).value_counts()\n",
    "\n",
    "# Most are below 150 in length, will use this to pad\n",
    "\n",
    "Subject_X_train = tokenizer.texts_to_sequences(X_train['Clean_Subject'].values)\n",
    "Subject_X_train = pad_sequences(Subject_X_train, maxlen=150)\n",
    "print('Shape of data tensor:', Subject_X_train.shape)\n",
    "\n",
    "Subject_X_test = tokenizer.texts_to_sequences(X_test['Clean_Subject'].values)\n",
    "Subject_X_test = pad_sequences(Subject_X_test, maxlen=150)\n",
    "print('Shape of data tensor:', Subject_X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "colab_type": "code",
    "id": "Dpyf6fJYX9Kz",
    "outputId": "8b7427b2-fe29-4210-bb4b-1c0a0343a4f3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39.714770494731205\n",
      "Shape of data tensor: (10078, 100)\n",
      "Shape of data tensor: (1120, 100)\n"
     ]
    }
   ],
   "source": [
    "# Format the inputs for Model 3 of predicting tag from message sender\n",
    "\n",
    "# First need to get length of From messages sender\n",
    "\n",
    "clean_from_length = df['Clean_From'].apply(len)\n",
    "mean = clean_from_length.mean()\n",
    "print(mean)\n",
    "(clean_from_length < 100).value_counts()\n",
    "\n",
    "# Most are below 100 in length, will use this to pad\n",
    "\n",
    "From_X_train = tokenizer.texts_to_sequences(X_train['Clean_From'].values)\n",
    "From_X_train = pad_sequences(From_X_train, maxlen=100)\n",
    "print('Shape of data tensor:', From_X_train.shape)\n",
    "\n",
    "From_X_test = tokenizer.texts_to_sequences(X_test['Clean_From'].values)\n",
    "From_X_test = pad_sequences(From_X_test, maxlen=100)\n",
    "print('Shape of data tensor:', From_X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "MlNViWFYHHLA",
    "outputId": "de9b62f6-deac-4350-9107-a7f0990fe792"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of label tensor: (10078, 8)\n",
      "Shape of label tensor: (1120, 8)\n"
     ]
    }
   ],
   "source": [
    "# Format the Y inputs with encoding\n",
    "\n",
    "Encoded_Y_train = pd.get_dummies(Y_train).values\n",
    "print('Shape of label tensor:', Encoded_Y_train.shape)\n",
    "\n",
    "Encoded_Y_test = pd.get_dummies(Y_test).values\n",
    "print('Shape of label tensor:', Encoded_Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "colab_type": "code",
    "id": "oBpVjSd2PI0K",
    "outputId": "a797ac3d-9a51-4d88-b2c4-c1c41a84055c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Entertainment', 'Events', 'Finance', 'Other', 'Productivity',\n",
      "       'Shopping', 'Social', 'Travel'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Use these columns for the labels when decoding the neural net\n",
    "\n",
    "print(pd.get_dummies(df['first_tag']).columns)\n",
    "\n",
    "labels = ['Entertainment', 'Events', 'Finance', 'Other', 'Productivity',\n",
    "       'Shopping', 'Social', 'Travel']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mVfjt_nPE2qq"
   },
   "source": [
    "#### Setting up the Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Mm_tf0BvuDF-"
   },
   "outputs": [],
   "source": [
    "# LSTM function\n",
    "\n",
    "# 94.64 acc\n",
    "# Increased second layer GRU to 100 and all dropouts to 0.3\n",
    "# Also changed validation from test to validation split and increased epochs 3-5\n",
    "\n",
    "from tensorflow.core.protobuf import rewriter_config_pb2\n",
    "from tensorflow.keras.backend import set_session\n",
    "tf.keras.backend.clear_session()  # For easy reset of notebook state.\n",
    "\n",
    "config_proto = tf.ConfigProto()\n",
    "off = rewriter_config_pb2.RewriterConfig.OFF\n",
    "config_proto.graph_options.rewrite_options.arithmetic_optimization = off\n",
    "session = tf.Session(config=config_proto)\n",
    "set_session(session)\n",
    "\n",
    "def create_lstm(X):\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(50000, 100, input_length=X.shape[1]))\n",
    "    model.add(LSTM(100, dropout=0.5, recurrent_dropout=0.2, return_sequences=True))\n",
    "    model.add(GRU(100, dropout=0.5, recurrent_dropout=0.2, return_sequences=True))\n",
    "    model.add(GRU(50, dropout=0.5, recurrent_dropout=0.2))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SWsOg6J35KQN"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/amazonei_tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/amazonei_tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/keras/backend.py:4010: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "# Model for message, subject, and sender\n",
    "\n",
    "message_model = create_lstm(Message_X_train)\n",
    "subject_model = create_lstm(Subject_X_train)\n",
    "from_model = create_lstm(From_X_train)\n",
    "\n",
    "# Combine them all for the activation input\n",
    "\n",
    "combined_input = concatenate([message_model.output, subject_model.output, from_model.output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 798
    },
    "colab_type": "code",
    "id": "M93mOgWzDzUr",
    "outputId": "8c8e8244-dc54-48e8-88f9-8e7676d5a6a5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "embedding_input (InputLayer)    (None, 5000)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1_input (InputLayer)  (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2_input (InputLayer)  (None, 100)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 5000, 100)    5000000     embedding_input[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 150, 100)     5000000     embedding_1_input[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 100, 100)     5000000     embedding_2_input[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     (None, 5000, 100)    80400       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 150, 100)     80400       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   (None, 100, 100)     80400       embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "gru (GRU)                       (None, 5000, 100)    60300       lstm[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "gru_2 (GRU)                     (None, 150, 100)     60300       lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "gru_4 (GRU)                     (None, 100, 100)     60300       lstm_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "gru_1 (GRU)                     (None, 50)           22650       gru[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "gru_3 (GRU)                     (None, 50)           22650       gru_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "gru_5 (GRU)                     (None, 50)           22650       gru_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 150)          0           gru_1[0][0]                      \n",
      "                                                                 gru_3[0][0]                      \n",
      "                                                                 gru_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 8)            1208        concatenate[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 15,491,258\n",
      "Trainable params: 15,491,258\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# End activation of model\n",
    "\n",
    "x = Dense(8, activation='softmax')(combined_input)\n",
    "\n",
    "model = Model(inputs=[message_model.input, subject_model.input, from_model.input], outputs=x)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 425
    },
    "colab_type": "code",
    "id": "uu2MMQDCE97p",
    "outputId": "7099cc0e-9479-48c0-c15e-5ad3c76ae26d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] training model...\n",
      "Train on 9070 samples, validate on 1008 samples\n",
      "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/amazonei_tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/5\n",
      "9070/9070 [==============================] - 1848s 204ms/sample - loss: 0.7757 - acc: 0.7405 - val_loss: 0.3283 - val_acc: 0.8919\n",
      "Epoch 2/5\n",
      "9070/9070 [==============================] - 1858s 205ms/sample - loss: 0.3012 - acc: 0.9045 - val_loss: 0.2583 - val_acc: 0.9147\n",
      "Epoch 3/5\n",
      "9070/9070 [==============================] - 1833s 202ms/sample - loss: 0.2114 - acc: 0.9344 - val_loss: 0.2180 - val_acc: 0.9415\n",
      "Epoch 4/5\n",
      "9070/9070 [==============================] - 1836s 202ms/sample - loss: 0.1503 - acc: 0.9536 - val_loss: 0.2481 - val_acc: 0.9276\n",
      "Epoch 5/5\n",
      "9070/9070 [==============================] - 1828s 202ms/sample - loss: 0.1085 - acc: 0.9656 - val_loss: 0.2458 - val_acc: 0.9325\n",
      "[INFO] predicting house prices...\n"
     ]
    }
   ],
   "source": [
    "# train the model\n",
    "print(\"[INFO] training model...\")\n",
    "\n",
    "epochs = 5\n",
    "\n",
    "batch_size=64\n",
    "\n",
    "history = model.fit(\n",
    "\t[Message_X_train, Subject_X_train, From_X_train], Encoded_Y_train,\n",
    "\tvalidation_split=0.1,\n",
    "\tepochs=epochs, batch_size=batch_size,\n",
    "  callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)]\n",
    "  )\n",
    " \n",
    "# make predictions on the testing data\n",
    "print(\"[INFO] predicting house prices...\")\n",
    "preds = model.predict([Message_X_test, Subject_X_test, From_X_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CI0RbAIZHKTB"
   },
   "outputs": [],
   "source": [
    "## Side notes to look back into, especially attention layer\n",
    "\n",
    "# GRU layers can be added as well as LSTM, GRU > LSTM\n",
    "\n",
    "# attention models or attention transformed models\n",
    "\n",
    "# clustering in different forms for unsupervised training and then try to label them\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 407
    },
    "colab_type": "code",
    "id": "RrX30PQpHLh2",
    "outputId": "e8d332d3-0454-4e7c-b486-0d2ccf399178"
   },
   "outputs": [],
   "source": [
    "# epochs = 1\n",
    "# batch_size = 64\n",
    "\n",
    "# history = model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size,validation_split=0.1,callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NXxmsFaoCm6m"
   },
   "outputs": [],
   "source": [
    "# Saving model and weights\n",
    "\n",
    "\n",
    "#---------------------------\n",
    "## Change name each time ##\n",
    "#---------------------------\n",
    "\n",
    "from tensorflow.keras.models import model_from_json\n",
    "model_json = model.to_json()\n",
    "with open(\"combined_model4.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "model.save_weights(\"combined_weights4.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 196
    },
    "colab_type": "code",
    "id": "rNZV1AJ5MQwi",
    "outputId": "ea35bf24-08ff-4a43-a4e0-0614d993bd2a"
   },
   "outputs": [],
   "source": [
    "# from tensorflow.keras.models import model_from_json\n",
    "\n",
    "# # load json and create model\n",
    "# json_file = open('combined_model2.json', 'r')\n",
    "# loaded_model_json = json_file.read()\n",
    "# json_file.close()\n",
    "# loaded_model = model_from_json(loaded_model_json)\n",
    "\n",
    "# # load weights into new model\n",
    "# loaded_model.load_weights(\"combined_weights2.h5\")\n",
    "# print(\"Loaded model from disk\")\n",
    "\n",
    "# obj = {\n",
    "#     \"id\": \"3023\",\n",
    "#     \"sender\": \"normstormin@gmail.com\",\n",
    "#     \"subject\": \"Re: Prague travel\",\n",
    "#     \"message\": \"ok    On Mon, Mar 27, 2017 at 4:56 PM, <samuelithian@gmail.com> wrote:    > July 17th  >  >  >  > Sent from Mail <https://go.microsoft.com/fwlink/?LinkId=3D550986> for  > Windows 10  >  >  >  > *From: *norman hepner <normstormin@gmail.com>  > *Sent: *Monday, March 27, 2017 4:55 PM  > *To: *Samuel Hepner <samuelithian@gmail.com>  > *Subject: *Re: Prague travel  >  >  >  > when do you need to depart by?  >  >  >  > On Mon, Mar 27, 2017 at 4:53 PM, <samuelithian@gmail.com> wrote:  >  > I just reviewed the information on the flight date and it says that  > students must make their reservations to arrive on July 19th between 9am  > and 1pm at the Vlaclav Havel airport and thats where there will =  be people  > to pick me up. They also say that I should have a round trip ticket due t=  o  > previous experiences with students being denied boarding.  >  >  >  > Sent from Mail <https://go.microsoft.com/fwlink/?LinkId=3D550986> for  > Windows 10  >  >  >  >  >  >  >  \"\n",
    "# }\n",
    "\n",
    "\n",
    "# def format_inputs(text, maxlength):\n",
    "#     text = clean_text(text)\n",
    "#     seq = tokenizer.texts_to_sequences([text])\n",
    "#     padded = pad_sequences(seq, maxlen=maxlength)\n",
    "#     return padded\n",
    "\n",
    "\n",
    "# def get_tags(text, model):\n",
    "#     labels = ['Entertainment', 'Events', 'Finance', 'Other', 'Productivity', 'Shopping', 'Social', 'Travel']\n",
    "#     padded = text\n",
    "#     pred = model.predict(padded)\n",
    "#     print(pred)\n",
    "#     if (np.argmax(pred)) >= .70:\n",
    "#         best = np.argmax(pred)\n",
    "#         prediction = labels[best]\n",
    "#         return prediction\n",
    "#     else:\n",
    "#         best = np.argmax(pred)\n",
    "#         best_pred = labels[best]\n",
    "#         pred = np.where(pred == np.argmax(pred), 0, pred)\n",
    "#         second = np.argmax(pred)\n",
    "#         second_pred = labels[second]\n",
    "#         return [best_pred[0], second_pred[0]]\n",
    "\n",
    "# def getTags(preds_proba, encoder):\n",
    "#     classes = encoder.classes_\n",
    "#     if preds_proba.max() >= .75:\n",
    "#         best = np.where(preds_proba == preds_proba.max())\n",
    "#         prediction = classes[best]\n",
    "#         return [prediction[0]]\n",
    "#     else:\n",
    "#         best = np.where(preds_proba == preds_proba.max())\n",
    "#         best_pred = classes[best]\n",
    "#         preds_proba = np.where(preds_proba==preds_proba.max(), 0, preds_proba)\n",
    "#         second = np.where(preds_proba== preds_proba.max())\n",
    "#         second_pred = classes[second]\n",
    "#         return [best_pred[0], second_pred[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sender = format_inputs(obj['sender'], 100)\n",
    "# subject = format_inputs(obj['subject'], 150)\n",
    "# message = format_inputs(obj['message'], 5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.set_printoptions(formatter={'float': lambda x: \"{0:0.9f}\".format(x)})\n",
    "# result = get_tags([message, subject, sender], loaded_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JQBOKLxiHg8d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1120/1120 [==============================] - 88s 78ms/sample - loss: 0.1832 - acc: 0.9464\n",
      "Test set\n",
      "  Loss: 0.183\n",
      "  Accuracy: 0.946\n"
     ]
    }
   ],
   "source": [
    "accr = model.evaluate([Message_X_test, Subject_X_test, From_X_test],Encoded_Y_test)\n",
    "print('Test set\\n  Loss: {:0.3f}\\n  Accuracy: {:0.3f}'.format(accr[0],accr[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_uEhz3oaYi6r"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt81PWd7/HXJ/d7Qq4QEkiAcBdSCIi3KioatIX22HrU2u1lLe1ubbd2a9Vzuu2251Laeqxtbeuq62533dW6uluxRQUrqLUqRAXlnnCTJEBCuIVLyO17/phJMgmBDDDJb2byfj4e83Dm9/vOzCc/nHe++f6+8/2Zcw4REYkuMV4XICIioadwFxGJQgp3EZEopHAXEYlCCncRkSikcBcRiUIKdxGRKKRwl6hnZrvM7Fqv6xAZSgp3EZEopHCXYcvMvmRmNWZ20MyWmVmhf7uZ2U/NrMHMjprZB2Y23b/vBjPbZGbNZlZnZt/y9qcQ6Z/CXYYlM7sa+CFwMzAK2A085d99HfBRYCKQ6W/T5N/3j8CXnXPpwHTglSEsWyRocV4XIOKRzwCPO+feBTCz+4BDZlYCtAHpwGRgjXNuc8Dz2oCpZrbeOXcIODSkVYsEST13Ga4K8fXWAXDOHcPXOx/tnHsFeAj4JdBgZo+YWYa/6U3ADcBuM3vVzC4Z4rpFgqJwl+GqHhjb9cDMUoEcoA7AOfdz59xsYCq+4Zm7/dvXOucWA/nA74Cnh7hukaAo3GW4iDezpK4b8CTwBTMrN7NE4P8CbzvndpnZHDO72MzigeNAC9BpZglm9hkzy3TOtQFHgU7PfiKRs1C4y3CxHDgZcLsK+DvgWWAvMB64xd82A3gU33j6bnzDNT/x7/sssMvMjgJfwTd2LxJ2TBfrEBGJPuq5i4hEIYW7iEgUUriLiEQhhbuISBQK6huqZlYJ/AyIBR5zzi3ts38M8Bsgy9/mXufc8rO9Zm5urispKTmfmkVEhq133nnngHMub6B2A4a7mcXi+6beAqAWWGtmy5xzmwKafQd42jn3azObim/aWcnZXrekpISqqqqB3l5ERAKY2e6BWwU3LDMXqHHO7XDOteJbXGlxnzYO39xg8C20VB9soSIiEnrBhPtoYE/A41r/tkB/D9xuZrX4eu1f6++FzGyJmVWZWVVjY+N5lCsiIsEI1QnVW4F/ds4V4VtU6V/N7LTXds494pyrcM5V5OUNOGQkIiLnKZgTqnVAccDjIv+2QH8JVAI45970r92RCzSEokgRkS5tbW3U1tbS0tLidSmDKikpiaKiIuLj48/r+cGE+1qgzMxK8YX6LcBtfdp8CFwD/LOZTQGSAI27iEjI1dbWkp6eTklJCWbmdTmDwjlHU1MTtbW1lJaWntdrDDgs45xrB+4EXgI245sVs9HMfmBmi/zN/hb4kpmtx7fa3uedFq0RkUHQ0tJCTk5O1AY7gJmRk5NzQX+dBDXP3T9nfXmfbd8NuL8JuOy8qxAROQfRHOxdLvRnjLhvqG6sP8LSF7agPwxERM4s4sK9atchHn51O6u3aUhfRIbe4cOH+dWvfnXOz7vhhhs4fPjwIFTUv4gL91vnjmFsTgo/emELHZ3qvYvI0DpTuLe3t5/1ecuXLycrK2uwyjpNxIV7QlwM37puElv2NfO79/rOyBQRGVz33nsv27dvp7y8nDlz5nDFFVewaNEipk6dCsAnPvEJZs+ezbRp03jkkUe6n1dSUsKBAwfYtWsXU6ZM4Utf+hLTpk3juuuu4+TJkyGvM6gTquHmxotG8chrO3hg5TZunDGKpPhYr0sSEQ98//mNbKo/GtLXnFqYwfc+Pu2M+5cuXcqGDRtYt24dq1ev5sYbb2TDhg3dUxYff/xxsrOzOXnyJHPmzOGmm24iJyen12tUV1fz5JNP8uijj3LzzTfz7LPPcvvtt4f054i4njtATIxx38LJ1B0+yb++GdQaOiIig2Lu3Lm95qL//Oc/Z+bMmcybN489e/ZQXV192nNKS0spLy8HYPbs2ezatSvkdUVkzx3g0gm5fHRiHg+tquHmimIyU87vW1wiErnO1sMeKqmpqd33V69ezcsvv8ybb75JSkoKV111Vb9z1RMTE7vvx8bGDsqwTET23LvcWzmZoy1t/OrVGq9LEZFhIj09nebm5n73HTlyhBEjRpCSksKWLVt46623hri6HhHbcwff2NgnykfzT2/s4nOXlFCYlex1SSIS5XJycrjsssuYPn06ycnJFBQUdO+rrKzk4YcfZsqUKUyaNIl58+Z5Vqd59WWgiooKF4qLdew5eIJr/t+rLC4v5CefnhmCykQknG3evJkpU6Z4XcaQ6O9nNbN3nHMVAz03oodlAIqzU/iLS8by7Lu1bN3X/59KIiLDTcSHO8BX508gNTGOH7+4xetSRETCQlSE+4jUBP7qqvH8cUsDb+9o8rocERHPRUW4A3zh0lJGZiSx9EUtKiYiEjXhnpwQy10Lynjvw8O8tHGf1+WIiHgqasId4KZZRUzIT+PHL26lraPT63JERDwTVeEeFxvDPZWT2XHgOE9X7fG6HBGJQue75C/Agw8+yIkTJ0JcUf+iKtwBrp2ST8XYETz4cjUnWs++BKeIyLmKlHCP6G+o9sfMuO+Gydz06zd57PWdfP2aMq9LEpEoErjk74IFC8jPz+fpp5/m1KlTfPKTn+T73/8+x48f5+abb6a2tpaOjg7+7u/+jv3791NfX8/8+fPJzc1l1apVg1pn1IU7wOyx2Vw3tYB/eHU7n7l4DDlpiQM/SUQizwv3wr4PQvuaIy+ChUvPuDtwyd8VK1bwzDPPsGbNGpxzLFq0iNdee43GxkYKCwv5wx/+APjWnMnMzOSBBx5g1apV5ObmhrbmfkTdsEyXb1dOpqW9k1+8okXFRGRwrFixghUrVvCRj3yEWbNmsWXLFqqrq7noootYuXIl99xzD6+//jqZmZlDXltQPXczqwR+BsQCjznnlvbZ/1Ngvv9hCpDvnBu660n1Y0J+GjdXFPNvb+/mC5eVMDYndeAniUhkOUsPeyg457jvvvv48pe/fNq+d999l+XLl/Od73yHa665hu9+97tDWtuAPXcziwV+CSwEpgK3mtnUwDbOubucc+XOuXLgF8B/Dkax5+ob15YRG2Pcv2Kb16WISJQIXPL3+uuv5/HHH+fYsWMA1NXV0dDQQH19PSkpKdx+++3cfffdvPvuu6c9d7AF03OfC9Q453YAmNlTwGJg0xna3wp8LzTlXZiCjCTuuHwcD62qYckV47ioaOj/NBKR6BK45O/ChQu57bbbuOSSSwBIS0vjiSeeoKamhrvvvpuYmBji4+P59a9/DcCSJUuorKyksLBw0E+oDrjkr5l9Cqh0zt3hf/xZ4GLn3J39tB0LvAUUOec6+tm/BFgCMGbMmNm7dw/+JfKOtrRx5Y9XMbUwgyf+8mLMbNDfU0QGj5b89WbJ31uAZ/oLdgDn3CPOuQrnXEVeXl6I37p/GUnxfO3qMt6oaeL16gND8p4iIl4LJtzrgOKAx0X+bf25BXjyQosKtc/MG0PRiGSWvrCFzk4tKiYi0S+YcF8LlJlZqZkl4AvwZX0bmdlkYATwZmhLvHCJcbHcff0kNu09ynPrz/R7SUQixXBY+fVCf8YBw9051w7cCbwEbAaeds5tNLMfmNmigKa3AE+5MD3qH59RyLTCDO5/aRun2vsdNRKRCJCUlERTU1NUB7xzjqamJpKSks77NSL+Gqrn4vXqRj77j2v4zo1TuOOKcUP63iISGm1tbdTW1tLS0uJ1KYMqKSmJoqIi4uPje20P9oRqVC4/cCZXlOVxRVkuD62q4eY5xWQkxQ/8JBEJK/Hx8ZSWlnpdRtiL2uUHzuSeyskcPtHGw6u3e12KiMigGXbhPn10JovLC3n8jZ3sOxLdf9aJyPA17MId4FvXTaKj0/Hgy1qWQESi07AM9+LsFG6fN5anq/ZQ0zA06zyIiAylYRnuAHfOn0BKQhw/enGr16WIiITcsA33nLREvnLlOFZu2k/VroNelyMiElLDNtwBvnh5KfnpifzwhS1R/YUIERl+hnW4pyTE8Y1rJ/LO7kOs2LTf63JEREJmWIc7wM0VRYzLS+XHL26hvaPT63JEREJi2Id7XGwM375+Mtsbj/Mf79R6XY6ISEgM+3AHuH5aAbPGZPHTlds42apFxUQk8incATPjvhum0NB8isff2Ol1OSIiF0zh7jenJJtrpxTw8OrtHDze6nU5IiIXROEe4J7KSRxvbeehV2q8LkVE5IIo3AOUFaTz6dnF/Otbu9hz8ITX5YiInDeFex/fWFBGjBn/b4WWJRCRyKVw72NUZjJfvLyU362rZ0PdEa/LERE5Lwr3fnzlyvFkpcTzoxe3eF2KiMh5Ubj3IzM5njvnT+D16gP8qfqA1+WIiJyzoMLdzCrNbKuZ1ZjZvWdoc7OZbTKzjWb276Etc+h99pKxjM5KZumLm+ns1KJiIhJZBgx3M4sFfgksBKYCt5rZ1D5tyoD7gMucc9OAbwxCrUMqMS6Wv71uIhvqjvL8+/VelyMick6C6bnPBWqcczucc63AU8DiPm2+BPzSOXcIwDnXENoyvfGJ8tFMGZXB/Su20tquRcVEJHIEE+6jgT0Bj2v92wJNBCaa2Rtm9paZVfb3Qma2xMyqzKyqsbHx/CoeQjExxj2Vk9hz8CT/9vZur8sREQlaqE6oxgFlwFXArcCjZpbVt5Fz7hHnXIVzriIvLy9Ebz24rpyYx6Xjc/jFKzU0t7R5XY6ISFCCCfc6oDjgcZF/W6BaYJlzrs05txPYhi/sI56Zce/CyRw83sojr+3wuhwRkaAEE+5rgTIzKzWzBOAWYFmfNr/D12vHzHLxDdNETRLOKMriYzNG8djrO2k42uJ1OSIiAxow3J1z7cCdwEvAZuBp59xGM/uBmS3yN3sJaDKzTcAq4G7nXNNgFe2Fb103ibaOTh78Y7XXpYiIDMi8ujB0RUWFq6qq8uS9z9f3ntvAE29/yIq7Psr4vDSvyxGRYcjM3nHOVQzUTt9QPQdfu6aMpLgYfvKiFhUTkfCmcD8HuWmJLPnoeF7cuI93dh/yuhwRkTNSuJ+jO64oJTctkR+9sAWvhrRERAaicD9HqYlx/M21ZazZdZA/bo6KL+KKSBRSuJ+HW+YUU5qbyo9e3EKHFhUTkTCkcD8P8bEx3H39JKobjvHsO7VelyMichqF+3laOH0kM4uzeGDlNlraOrwuR0SkF4X7eTIz7ls4mX1HW/inN3Z5XY6ISC8K9wswb1wOV0/O51erazh0vNXrckREuincL9C3Kydx7FQ7v1pd43UpIiLdFO4XaPLIDG6aVcRv/ryb2kMnvC5HRARQuIfEXQsmgsEDK7d5XYqICKBwD4nRWcl84dIS/uu9OjbVH/W6HBERhXuo/PVVE0hPjOPHL23xuhQREYV7qGSmxPPV+RNYvbWRP28/4HU5IjLMKdxD6HOXllCYmcRSLSomIh5TuIdQUnwsdy2YyPu1R/jDB3u9LkdEhjGFe4j9t1lFTCpI5ycvbaWto9PrckRkmFK4h1hsjHHPwknsbjrBk2s+9LocERmmFO6DYP6kfC4uzeZnL1dz7FS71+WIyDCkcB8EZsa9CyfTdLyVR1/b4XU5IjIMBRXuZlZpZlvNrMbM7u1n/+fNrNHM1vlvd4S+1MjykTEjuOGikTz6+g4amlu8LkdEhpkBw93MYoFfAguBqcCtZja1n6a/dc6V+2+PhbjOiHT39ZM51d7JL/6oRcVEZGgF03OfC9Q453Y451qBp4DFg1tWdCjNTeXWucU8ueZDdh447nU5IjKMBBPuo4E9AY9r/dv6usnM3jezZ8ysuL8XMrMlZlZlZlWNjY3nUW7k+fo1ZSTExXD/S1u9LkVEhpFQnVB9Hihxzs0AVgK/6a+Rc+4R51yFc64iLy8vRG8d3vLTk7jjinH84YO9rNtz2OtyRGSYCCbc64DAnniRf1s351yTc+6U/+FjwOzQlBcdlnx0HDmpCfxw+WYtSyAiQyKYcF8LlJlZqZklALcAywIbmNmogIeLgM2hKzHypSXG8fVrynh750FWbx0ew1Ei4q0Bw9051w7cCbyEL7Sfds5tNLMfmNkif7Ovm9lGM1sPfB34/GAVHKlunTuGsTkpLH1hCx2d6r2LyOAyr4YJKioqXFVVlSfv7ZXn19fztSff4/5Pz+RTs4u8LkdEIpCZveOcqxionb6hOoRuvGgUM4oyeWDFVlraOrwuR0SimMJ9CMXEGPdWTqb+SAv/8uYur8sRkSimcB9il07I5cqJefxy1XaOnGjzuhwRiVIKdw/cUzmZoy1t/OpVLUsgIoND4e6BqYUZfLJ8NP/0xi7qD5/0uhwRiUIKd49887qJ4OCnK7d5XYqIRCGFu0eKRqTwF5eM5dl3a9m6r9nrckQkyijcPfTV+RNITYzjxy9u8boUEYkyCncPjUhN4K+uGs8ftzTw9o4mr8sRkSiicPfYFy8rZWRGEj98YYsWFRORkFG4eywpPpa7FpSxbs9hXtywz+tyRCRKKNzDwE2ziijLT+PHL22lraPT63JEJAoo3MNAXGwM366czM4Dx/nt2j0DP0FEZAAK9zBx7ZR85pSM4MGXqzl+qt3rckQkwincw4SZce/CKRw4dop//NNOr8sRkQincA8js8eO4PppBfzDq9s5cOzUwE8QETkDhXuY+XblZFraO3noFS0qJiLnT+EeZsbnpXFzRTH/9vZudjcd97ocEYlQCvcwdNe1ZcTFxPCTl7Z6XYqIRCiFexjKz0jiLy8v5ffv7+X92sNelyMiESiocDezSjPbamY1ZnbvWdrdZGbOzAa8eKuc3ZevHMeIlHiWalkCETkPA4a7mcUCvwQWAlOBW81saj/t0oG/Ad4OdZHDUXpSPF+7uow/b2/iteoDXpcjIhEmmJ77XKDGObfDOdcKPAUs7qfd/wJ+BLSEsL5h7TPzxlCcnczSF7bQ2aneu4gEL5hwHw0Efie+1r+tm5nNAoqdc38IYW3DXmJcLN+6bhKb9x7lufV1XpcjIhHkgk+omlkM8ADwt0G0XWJmVWZW1djYeKFvPSx8fEYh00dncP9L22hp6/C6HBGJEMGEex1QHPC4yL+tSzowHVhtZruAecCy/k6qOucecc5VOOcq8vLyzr/qYSQmxri3cgp1h0/yxFu7vS5HRCJEMOG+Figzs1IzSwBuAZZ17XTOHXHO5TrnSpxzJcBbwCLnXNWgVDwMXV6WyxVluTy0qoYjJ9u8LkdEIsCA4e6cawfuBF4CNgNPO+c2mtkPzGzRYBcoPvdUTubwiTb+4dXtXpciIhEgLphGzrnlwPI+2757hrZXXXhZ0tf00ZksLi/k8Td28heXlDAyM8nrkkQkjOkbqhHkW9dNoqPT8dOV27wuRUTCnMI9ghRnp3D7vLH8xzt7qN7f7HU5IhLGFO4R5mtXl5GSEMePXtSiYiJyZgr3CJOdmsBXrhzHy5v3s3bXQa/LEZEwpXCPQF+8vJT89ER+uHyzFhUTkX4p3CNQSkIcdy2YyLsfHmbFpv1elyMiYUjhHqE+PbuI8Xmp/PjFLbR3dHpdjoiEGYV7hIqLjeHblZPZ3nic/3in1utyRCTMKNwj2HVTC5g9dgQ/XbmNE63tXpcjImFE4R7BzIx7F06mofkUj/9pp9fliEgYUbhHuDkl2Vw7pYCHX93BweOtXpcjImFC4R4F7qmcxInWdn7xSrXXpYhImFC4R4GygnQ+PbuYJ97azZ6DJ7wuR0TCgMI9Sty1YCIxZty/QssSiEgkhnt7K+hbmacZmZnEFy8v5bl19WyoO+J1OSLiscgL93VPwP0T4T+XwLonoXmf1xWFja9cOZ6slHh+9OIWr0sREY8FdbGOsJIzAcZdCTV/hPd/69uWPw3Gz/fdxlwKCSne1uiRzOR47pw/gf/9h828Xt3IFWW6Tq3IcGVeLTxVUVHhqqou4DKrnZ2w/wPY/orv9uFb0NEKsYkw9hIYf7Xvlj8NYiLvD5Tzdaq9g6vvf5WslHiev/NyYmLM65JEJITM7B3nXMWA7SI23PtqPQG7/9wT9o2bfdtT82DcfH/Yz4f0kaF7zzD1X+/Vctdv1/OzW8pZXD7a63JEJISGX7j3dbQedqz2h/0qOHHAtz1/ak/QR+kQTmen48Zf/Iljp9p4+ZtXkhgX63VJIhIiCvdA3UM4q/xDOG/2DOGMmdczhFMwPWqGcF7d1sjnHl/Ddz82lS9eXup1OSISIiENdzOrBH4GxAKPOeeW9tn/FeCrQAdwDFjinNt0ttcc0nDvK3AIZ8cqaPCX2j2EM9/334xR3tQXAs45PvPY22zee5RXvz2fjKR4r0sSkRAIWbibWSywDVgA1AJrgVsDw9vMMpxzR/33FwF/7ZyrPNvrehrufR3d6wv5KBvCeb/2MIseeoM750/gW9dP8rocEQmBYMM9mKmQc4Ea59wO/ws/BSwGusO9K9j9UoHI+pZRxigov8136+yE/Rt6TsyueQTefAhiE2DMJT1hX3BR2A/hzCjK4mMzRvHo6ztobmljUfloZo3JwkwzaESiXTA9908Blc65O/yPPwtc7Jy7s0+7rwLfBBKAq51zp61iZWZLgCUAY8aMmb179+6Q/BCDqvUEfPjnnvH6riGclFz/3Pqrw3oIp+FoC3///EZe3txAa3snRSOSWTSzkMXlo5k0Mt3r8kTkHIVyWCaocA9ofxtwvXPuc2d73bAaljkXR/f2zMLZsQqON/q2503pOTE7NvyGcJpb2lixcT/Pra/njZoDdHQ6JhWks6i8kEUzCynODq96RaR/oQz3S4C/d85d7398H4Bz7odnaB8DHHLOZZ7tdSM23AN1DeF0jdfvfhM6TgUM4fh79mE2hHPg2CmWf7CXZevqqdp9CICPjMli8cxCbpxRSF56oscVisiZhDLc4/CdUL0GqMN3QvU259zGgDZlXcMwZvZx4HsDvXlUhHtfvYZwVkGD/xB1DeF0zcTJKPS2zgC1h07w/Pq9PLeuji37mokxuGxCLh+fWUjl9JGaZSMSZkI9FfIG4EF8UyEfd879HzP7AVDlnFtmZj8DrgXagEPAnYHh35+oDPe+BhzCme8fwkn1tMwu2/Y3s2xdPcvW1/PhwRMkxMUwf1Iei8tHc/XkfJLi9WUoEa/pS0zhprPT15PvmoXTawgn8ItU3g/hOOdYt+cwy9bX8/v399LYfIq0xDium1bAopmFXD4hl7jY8BlmEhlOFO7hrvWE75uyXXPrA4dwxl3V07P3eAino9Px1o4mlq2rZ/mGvTS3tJOTmsANF41icXkhs8aM0OJkIkNI4R5pmvcFrIXzSsAQzuQ+s3C8G8I51d7Bq1sbeW59PX/cvJ+Wtk5GZyXz8Zm+GTdTRqVrDr3IIFO4R7JeQzirfEsldA3hFF/cE/YjZ3g2hHPsVDsrN+1j2bp6Xqv2Ta0sy09j0cxCFpUXMjYnPM4jiEQbhXs0aTsZsJxx4BBOTu+1cDK9Wd734PHW7qmVa3YdBGBmcRaLZhby8RmjyM9I8qQukWikcI9mvYZwVsHxBt/2riGccfOh5DJPhnDqDp/k9+t9M2421h8lxmDeuBwWlxdSOW0UmSmaWilyIRTuw4VzsD9wFk4/QzilV0L+5CEP+5qGYyxbX8+ydXXsajpBQmwMV07KY9HMQq6dUkBygqZWipwrhftw1Xay9yyc/Rt69mUWQ24Z5E70/3eS735aPgziiVDnHB/UHeG5dfX8/v169h89RUpCLNdNLWBx+WguL8slXlMrRYKicBef5v2+sD9QDQe2+W/V0Ha8p01iZp/Qn+i7ZZdCbGiHUTo6HWt2HmTZ+jqWf7CPIyfbGJESzw0XjWLRzELmlGRraqXIWSjc5cyc812GsDvsA0K/eW9Pu5g4yB53eujnlkHSWZcOCkpreyevbWtk2fp6Vm7az8m2DkZlJnVPrZxWmKGpleHMOWg54vt/xnWCxYLF+G8GMYGPu26xPfstpp82gfv1b98fhbucn5aj0FTd09Nv3Oq7f3A7dLb3tEsrCAj7gPDPGH1e0zNPtLazctN+lq2r59VtjbR3OsblpbJ45mgWlRdSmquplUOuo83XCThS67/tCbjvv7U2D2IBZ/sFYL23xfT5xWL9/GKJ6fu8/tqc6ZfRhfzC6uc2+WNQNPv8jorCXUKqow0O7e7dyz+wDQ5s9fXeusSnQM6EntDP6xriGQ/xwU2JPHS8lRc27GPZ+jre3nkQ52BGUSaLZhbysRmFjMzU1MoL5hycPNQnsPuEd/M+TrvuTkqub8ptZjFkFvlu6aN8f+W5Tv/NgesIeOy/dXYE7O88Q5s+j/u2OW1/3zYu4H36tnH91HKWW2ef1z3jz+TOUMtZfqYb7oeKL5zXP53CXYaGc75v054W+tvg8B56wsFgxNjTe/q5kyA154wvv+9IC79/v57n1tXzQd0RzODi0mwWl49m4fSRZKUkDMmPGXHaT8HRuj497T7h3Xai93NiE3sCOzC8ux5nFIbddQqGI4W7eK/1hG8458A2aAwI/6ZqaG/paZec3f+4ftZYiO25EuSORv/UyvX17Gg8TnysceXEPD4+s5AFUwtISQjmqpFRwDk40dTPMEnA42P7T39eav7Zwzs1V+PcEUDhLuGrs9MXRH1n8BzY2rOmDvjm6meP7xna8Ye+y5nAxgOdLFtfz/Pr69l7pIXk+FgWTPWtWvnRiXkkxEXw1Mq2k3Ck7szhfbSu9y9HgLhkyCo+PbwzRvf8N8hhMQlvCneJTCcOQlPN6cM8B3f6xiq7ZIz2B30Zu6yIVw5k8dudSWw7mUZmcgI3XDSSRTNHc3FpmE2t7Oz0/QI7bZgk4P6JA32eZJA+8vSeduD95BHqdQ8TCneJLu2tcGhn7xk8XeEfMGOjPS6Vurhi1p3MY2v7KJqSx1IyaRaXz61g+pi8wZ9a2Xp84F53R2vv5ySk9QnuPuGdXghxOrcgPgp3GR6c883q6NPT72zcSkxzfXezdhdDfcxIWkdMIGfsdEaMmdYztp88Irj36uzwjWX36mnX9X588mDv51iML5zPFt5Jmep1S9DsxhauAAAMgUlEQVSCDfdhcgZKopYZZIzy3cZd2b05BuBUMzTVcKJuMzu3vsex2s1kHthJStOf4b2AOfup+b1P6GYVBwydBPS8j9b3nusPvm/3doV00Zx+et2jep0UFhkq6rnLsLL/aAt/WF/L2++to23fZsZbPfMymrgosYHcll3EtBzqaRwT55v+l3GmXvfokHxTV+RcaFhGZAC7Dhzn+fX1PLe+npqGY8TFGAvHxfPJcY4Zk8rIHTXG921DkTCicBcJknOOzXubu6dW1h0+CcDYnBTmlGQztySbipIRlOamaq0b8VxIw93MKoGfAbHAY865pX32fxO4A2gHGoEvOud2n+01Fe4Sjjo7HRvqj7Bm50HW7DxI1e5DHDzum92Sm5bInJIRvsAvzWbKqAxiw2mapQwLIQt3M4sFtgELgFpgLXCrc25TQJv5wNvOuRNm9lfAVc65/36211W4SyRwzrG98Rhrdh5i7S5f4Hf17NMS45g1dgRz/YE/sziLpHgN48jgCuVsmblAjXNuh/+FnwIWA93h7pxbFdD+LeD2cytXJDyZGRPy05mQn85tF48BoP7wye6gr9p1iPtXbAMgITaGGUWZzCn1DeXMGjuCzGRdVlC8EUy4jwb2BDyuBS4+S/u/BF7ob4eZLQGWAIwZMybIEkXCS2FWMovLR7O43HdB8sMnWqna5e/Z7zrIo6/t4Nert2MGkwrSmVua3T2UU6CLhcsQCekEXDO7HagAruxvv3PuEeAR8A3LhPK9RbySlZLAtVMLuHZqAQAnWzt4b88h1vqHcp55p5Z/edN3CmpMtv8kbalvKEcnaWWwBBPudUBxwOMi/7ZezOxa4H8CVzrnToWmPJHIk5wQy6Xjc7l0fC4AbR2dbKo/2j2Us2prA8++WwucfpJ28sh04nQ9WQmBYE6oxuE7oXoNvlBfC9zmnNsY0OYjwDNApXOuOpg31glVGa58J2mPs3bXQdbu9A3l1B7qOUn7kTFZzC3JZk5pNuU6SSt9hHoq5A3Ag/imQj7unPs/ZvYDoMo5t8zMXgYuArouwPmhc27R2V5T4S7SY++Rk6zZedAf+IfYut+3GFpCbAwXFWV2D+XMHputk7TDnL7EJBLB+p6k/aD2CO2dTidpReEuEk0CT9JW7T7IO7sPcaLVt759cXZy9zdp55RmM04naaOaVoUUiSJ9T9K2d3Syae/R7qGc1Vsb+c93ffMcctMSqBib3T3ffsoonaQdjtRzF4kCZztJm5oQ6/8mrU7SRgMNy4gMc2c6SRsfa8woyqKixBf4FWOzyUzRSdpIoXAXkV66T9Lu9vXuP6g7QltHz0naOSU9QzkjM3WSNlwp3EXkrE62drBuz2Ffz36XTtJGCp1QFZGzSk6I5ZLxOVwyPgc4/STtq/2cpK0oGcHc0mymjsrQSdowp567iPSr70natbsPsudg75O0c/wXMpk6KoOslASPKx4eNCwjIiG398hJ1u465Av7XQfZur+ZrgjJTUtkQn4qZfnplBWkMSEvjQkFaeSlJWpIJ4Q0LCMiITcqM5lFM5NZNLMQgCMn2nh3zyGq9zdT03CM6oZj/O69OppPtXc/JzM5ngn5aZTlpzHBfysrSKcwM0mhP4jUcxeRkHLO0dB8iur9x6hu6An9moZj3ZcsBEhJiO0Oe1/4p1OWn0ZxdoouX3gW6rmLiCfMjIKMJAoykri8LLfXvqZjp6hpOEZN4zGq9/sC/881Td0nbgES4mIYl5tKWUE6E/LSKCvw9frH5qSSEKeTuMFSuIvIkMlJSyQnLZGLx+X02n60pc0X+v5b9f5m1u05xPPr67vbxMYYJTkpPb38gjTG5/luyQn6xm1fCncR8VxGUjyzxoxg1pgRvbafbO1ge6M/8AOGeF7e3EBHp29I2QyKR6ScNq4/IT+N9KTh+81bhbuIhK3khFimj85k+ujMXttPtXewu+lEr3H9moZj/Kn6AK0dnd3tRmYk+Wbu9BnXH5Ea/dM2Fe4iEnES42KZWJDOxIJ0YFT39vaOTvYcOumbvdN4jJr9vp7+b9fu6f72LUBOaoJ/1k6af1w/nQn5aeSnR8+0TYW7iESNuNgYSnNTKc1N5bqA7Z2djvojJ6luOMb2Bv/J3MZjLFtXz9GWnmmb6Ulx3UM7ZfnpTPCH/+isZGIibAaPpkKKyLDlnKOx+VSv6ZpdwzwHjvVM20yO7z1ts2t8f0x2ypAvw6CpkCIiAzAz8jOSyM9I4tIJvadtHjre2mvKZnVDM2/taOK/3guYtun/S2FCQVqvHn9JbgqJcd7O4FG4i4j0Y0RqAnNSfdeqDdTc0sb2xuM9vfz9x9hQd4TlH+ztXoohNsYYm53SM67vD/2hnLYZVLibWSXwMyAWeMw5t7TP/o8CDwIzgFucc8+EulARkXCQnhRPeXEW5cVZvba3tPVM26wJGNd/ZUsD7Z09w99FI5K5+/pJLC4fPah1DhjuZhYL/BJYANQCa81smXNuU0CzD4HPA98ajCJFRMJdUnws0wozmVbYe9pma3snHx487p+26Qv+vLTEQa8nmJ77XKDGObcDwMyeAhYD3eHunNvl39fZ3wuIiAxXCXExTMhPZ0J+OguH8H2DOc07GtgT8LjWv+2cmdkSM6sys6rGxsbzeQkREQnCkM7hcc494pyrcM5V5OXlDeVbi4gMK8GEex1QHPC4yL9NRETCVDDhvhYoM7NSM0sAbgGWDW5ZIiJyIQYMd+dcO3An8BKwGXjaObfRzH5gZosAzGyOmdUCnwb+wcw2DmbRIiJydkHNc3fOLQeW99n23YD7a/EN14iISBjQZU1ERKKQwl1EJAp5tiqkmTUCu8/z6bnAgRCWEyqq69yornMXrrWprnNzIXWNdc4NOJfcs3C/EGZWFcySl0NNdZ0b1XXuwrU21XVuhqIuDcuIiEQhhbuISBSK1HB/xOsCzkB1nRvVde7CtTbVdW4Gva6IHHMXEZGzi9Seu4iInIXCXUQkCoV1uJtZpZltNbMaM7u3n/2JZvZb//63zawkTOr6vJk1mtk6/+2OIarrcTNrMLMNZ9hvZvZzf93vm9msMKnrKjM7EnC8vttfuxDXVGxmq8xsk5ltNLO/6afNkB+vIOvy4nglmdkaM1vvr+v7/bQZ8s9jkHV58nn0v3esmb1nZr/vZ9/gHi/nXFje8F2vdTswDkgA1gNT+7T5a+Bh//1bgN+GSV2fBx7y4Jh9FJgFbDjD/huAFwAD5gFvh0ldVwG/H+JjNQqY5b+fDmzr599xyI9XkHV5cbwMSPPfjwfeBub1aePF5zGYujz5PPrf+5vAv/f37zXYxyuce+7dl/dzzrUCXZf3C7QY+I3//jPANWZmYVCXJ5xzrwEHz9JkMfAvzuctIMvMRoVBXUPOObfXOfeu/34zvhVP+15hbMiPV5B1DTn/MTjmfxjvv/WdjTHkn8cg6/KEmRUBNwKPnaHJoB6vcA73YC7v193G+ZYmPgLkhEFdADf5/5R/xsyK+9nvhZBdMnEQXOL/0/oFM5s2lG/s/3P4I/h6fYE8PV5nqQs8OF7+IYZ1QAOw0jl3xuM1hJ/HYOoCbz6PDwLfBs50belBPV7hHO6R7HmgxDk3A1hJz29n6d+7+NbLmAn8AvjdUL2xmaUBzwLfcM4dHar3HcgAdXlyvJxzHc65cnzLe881s+lD8b4DCaKuIf88mtnHgAbn3DuD/V5nEs7hHszl/brbmFkckAk0eV2Xc67JOXfK//AxYPYg1xSssLxkonPuaNef1s537YB4M8sd7Pc1s3h8Afpvzrn/7KeJJ8droLq8Ol4B738YWAVU9tnlxedxwLo8+jxeBiwys134hm6vNrMn+rQZ1OMVzuEezOX9lgGf89//FPCK85+d8LKuPuOyi/CNm4aDZcBf+GeBzAOOOOf2el2UmY3sGms0s7n4/r8c1FDwv98/Apudcw+codmQH69g6vLoeOWZWZb/fjKwANjSp9mQfx6DqcuLz6Nz7j7nXJFzrgRfRrzinLu9T7NBPV5BXYnJC865djPrurxfLPC481/eD6hyzi3D9yH4VzOrwXfC7pYwqevr5rsEYbu/rs8Pdl0AZvYkvpkUuea77OH38J1gwjn3ML6rad0A1AAngC+ESV2fAv7KzNqBk8AtQ/BL+jLgs8AH/vFagP8BjAmoy4vjFUxdXhyvUcBvzCwW3y+Tp51zv/f68xhkXZ58HvszlMdLyw+IiEShcB6WERGR86RwFxGJQgp3EZEopHAXEYlCCncRkSikcBcRiUIKdxGRKPT/AeVfcR+dqXC+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.title('Loss')\n",
    "plt.plot(history.history['loss'], label='train')\n",
    "plt.plot(history.history['val_loss'], label='test')\n",
    "plt.legend()\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "486-J91eYnjE"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VeW97/HPL/NIZoYQMjAFHJjFAQcUEcSx7bkea+nRDpe2amvr0Npzree0595br7XWeo7W03psbav1eGx7ihUFUdA6gwwKSEKYMiGEDAQyD7/7x1pJNiEhO5Bk7eH3fr3yYu+11977l6X7u588z1rPI6qKMcaY8BDhdQHGGGNGjoW+McaEEQt9Y4wJIxb6xhgTRiz0jTEmjFjoG2NMGLHQN8aYMGKhb0KGiKwXkVoRifW6FmMClYW+CQkikg9cBChw7Qi+b9RIvZcxQ8FC34SKfwDeA34D3Ny1UUTiReSnIrJfRI6IyFsiEu8+dqGIvCMidSJSJiK3uNvXi8hXfV7jFhF5y+e+ishtIrIL2OVu+7n7GvUi8qGIXOSzf6SI/KOI7BaRo+7jE0TkMRH5qe8vISIrReQ7w3GAjAELfRM6/gF4xv1ZIiJj3O0PAXOBC4B04LtAp4jkAS8D/wpkAbOALYN4v+uBc4Ez3Psb3NdIB54F/ktE4tzH7gQ+DywDRgFfBhqBp4HPi0gEgIhkApe7zzdmWFjom6AnIhcCecDzqvohsBu4yQ3TLwN3qGqFqnao6juq2gLcBKxV1T+oapuqVqvqYEL/x6pao6pNAKr6e/c12lX1p0AsUOju+1XgPlUtUsdWd98PgCPAIne/G4H1qnrwNA+JMf2y0Deh4GZgjaoedu8/627LBOJwvgR6m9DPdn+V+d4RkbtF5BO3C6kOSHHff6D3ehpY7t5eDvzuNGoyZkA2CGWCmts/fwMQKSKfuptjgVRgHNAMTAK29npqGTC/n5dtABJ87o/tY5/u6Wnd/vvv4rTYt6tqp4jUAuLzXpOAbX28zu+BbSIyE5gO/Hc/NRkzJKylb4Ld9UAHTt/6LPdnOvA3nH7+p4CHRSTbHVA93z2l8xngchG5QUSiRCRDRGa5r7kF+KyIJIjIZOArA9SQDLQDVUCUiNyP03ff5UngX0RkijhmiEgGgKqW44wH/A74Y1d3kTHDxULfBLubgV+raqmqftr1A/wb8AXgXuBjnGCtAf4fEKGqpTgDq3e527cAM93X/BnQChzE6X55ZoAaVgOvAMXAfpy/Lny7fx4GngfWAPXAfwDxPo8/DZyNde2YESC2iIox3hKRi3G6efLUPpBmmFlL3xgPiUg0cAfwpAW+GQkW+sZ4RESmA3U4A86PeFyOCRPWvWOMMWHEWvrGGBNGAu48/czMTM3Pz/e6DGOMCSoffvjhYVXNGmi/gAv9/Px8Nm7c6HUZxhgTVERkvz/7WfeOMcaEEQt9Y4wJIxb6xhgTRgKuT78vbW1tlJeX09zc7HUpwy4uLo6cnByio6O9LsUYE4KCIvTLy8tJTk4mPz8fERn4CUFKVamurqa8vJyCggKvyzHGhKCg6N5pbm4mIyMjpAMfQETIyMgIi79ojDHeCIrQB0I+8LuEy+9pjPFGUHTvGGNMKGpu66CsppF91Y3sr24gPiaSL5ybN6zvaaHvp7q6Op599lluvfXWQT1v2bJlPPvss6Smpg5TZcaYQFbf3EZpdSP7qxvZV91Aade/NY0cOHJ8V+7s3FQL/UBRV1fH448/fkLot7e3ExXV/2FctWrVcJdmjPGQqlLd0Mp+t7Xe/W+NE/Q1Da3H7Z+ZFEt+RgLnT8ogPyORvIwE8jISyUtPIDVh+M/as9D307333svu3buZNWsW0dHRxMXFkZaWxs6dOykuLub666+nrKyM5uZm7rjjDlasWAH0TCtx7NgxrrzySi688ELeeecdxo8fz1/+8hfi4+MHeGdjjNc6O5VP65t9WuqNlNY0sO9wI6U1jRxrae/eVwSyU+LJy0hgyZljyctIID8jgdx0J+ATY72N3aAL/R++uJ0dlfVD+ppnZI/in64586T7PPDAA2zbto0tW7awfv16rrrqKrZt29Z9auVTTz1Feno6TU1NnHPOOXzuc58jIyPjuNfYtWsXf/jDH/jVr37FDTfcwB//+EeWL18+pL+LMebUtHV0Ul7b5NNa72mxl9Y00tre2b1vdKQwIS2BvIwE5heku611p8WekxZPbFSkh7/JyQVd6AeK+fPnH3cu/aOPPsqf//xnAMrKyti1a9cJoV9QUMCsWc7a23PnzmXfvn0jVq8xBppaO9hf03BcV0xpjdPHXlnXTEdnz/oi8dGR5GUkMCkrkUXTRpObkUB+RiK56Qlkp8YTGRGcZ9oFXegP1CIfKYmJid23169fz9q1a3n33XdJSEhg4cKFfZ5rHxsb2307MjKSpqamEanVmHBypLGN/TUNThdMdde/TrAfOtpy3L6pCdHkpScwe0Ia189y+9bdVntWUmxInkIddKHvleTkZI4ePdrnY0eOHCEtLY2EhAR27tzJe++9N8LVGRM+VJWqYy3Hd8H4dMXUNbYdt//o5FjyMxK5eGqW07eekUh+RgJ56YmkjMDAaaCx0PdTRkYGCxYs4KyzziI+Pp4xY8Z0P7Z06VKeeOIJpk+fTmFhIeedd56HlRoT/Do6lcq6pu6ul66Weld3TGNrR/e+EQLj0+LJS0/kqrPH9ZwNk5FAbnoCCTEWc74Cbo3cefPmae9FVD755BOmT5/uUUUjL9x+XxOeWto7+h44rW6krLaRto6ebIqJjGBCerzTp97Vt+7+Oz41npiooJlcYNiIyIeqOm+g/ewr0BgzbBpa2t3WeYN71WlPsFceacK3zZkYE0leRiKFY5O5wj3VsavVPnZUXNAOnAYaC31jzJBoae9gR2U9m0rr2FRay5bSOirqjj9ZIS0hmryMRM7JTyM3I8fpW3eDPSMxJiQHTgONX6EvIkuBnwORwJOq+kCvx/OAp4AsoAZYrqrl7mMdwMfurqWqeu0Q1W6M8dCBI01s2l/H5tJaNpXWsq2yvvtc9uyUOGbnpXHTublOqKc73TEp8eE3cBpoBgx9EYkEHgMWA+XABhFZqao7fHZ7CPitqj4tIpcBPwa+6D7WpKqzhrhuY8wIam7rYHtlfXfAby6t6543JiYqgrPHp3Dz+XnMyU1jdm4aY1PiPK7Y9Meflv58oERV9wCIyHPAdYBv6J8B3OneXgf891AWaYwZOapK5ZFmNu13wn1TaS07Kutp7XBa8eNT45mXn87sCanMyUvjjHGjbCA1iPgT+uOBMp/75cC5vfbZCnwWpwvoM0CyiGSoajUQJyIbgXbgAVU94QtBRFYAKwByc3MH/UsYY05dc1sH2yqOsKm01umuKavlYL1zEVNsVAQzclL40oJ8ZuemMSc3ldGjrBUfzIZqIPdu4N9E5BbgTaAC6DqRNk9VK0RkIvC6iHysqrt9n6yqvwR+Cc4pm0NU05A61amVAR555BFWrFhBQkLCMFRmjP9UlfLapu4ums2ltew4UN99euSE9HjOm5jR3YqfPm4U0ZHWig8l/oR+BTDB536Ou62bqlbitPQRkSTgc6pa5z5W4f67R0TWA7OB40I/GPQ3tbI/HnnkEZYvX26hb0ZcU2sHH3e34mvZXFZHlTsVQVx0BDNyUvnKhROZk5vKrNxURidbKz7U+RP6G4ApIlKAE/Y3Ajf57iAimUCNqnYC38c5kwcRSQMaVbXF3WcB8OAQ1j9ifKdWXrx4MaNHj+b555+npaWFz3zmM/zwhz+koaGBG264gfLycjo6OvjBD37AwYMHqays5NJLLyUzM5N169Z5/auYEKWqlNV0teJr2VRaxycH6ml3JxHLy0jgwsmZzM5NZU5uGoVjk60VH4YGDH1VbReR24HVOKdsPqWq20XkR8BGVV0JLAR+LCKK071zm/v06cC/i0gnznq8D/Q662fwXr4XPv144P0GY+zZcOUDJ93Fd2rlNWvW8MILL/DBBx+gqlx77bW8+eabVFVVkZ2dzUsvvQQ4c/KkpKTw8MMPs27dOjIzM4e2bhPWGlvb+ai8py9+S1kth485C3bER0cyc0IKKy6eyJzcNGblppKZFDvAK5pw4FefvqquAlb12na/z+0XgBf6eN47wNmnWWPAWbNmDWvWrGH27NkAHDt2jF27dnHRRRdx11138b3vfY+rr76aiy66yONKTahQVfZXN3b3xW8qrWXnp0e7pwIuyHQmFOsabC0ck0yUteJNH4LvitwBWuQjQVX5/ve/z9e+9rUTHtu0aROrVq3ivvvuY9GiRdx///19vIIJOh3tULEREMiaCvFpw/p2DS3tbC2v6x5s3VxaR7W77F5iTCQzJ6TyjUsmMScvlVkT0khPjBnWekzoCL7Q94jv1MpLlizhBz/4AV/4whdISkqioqKC6Oho2tvbSU9PZ/ny5aSmpvLkk08e91zr3gkyLcdg92uwcxXsWg1NtT2PJY6GrELInHr8v8njnPXyBkFV2Xu4gU2ldd198UWf1tO1nsfErEQunTa6uy9+6phkm4fGnDILfT/5Tq185ZVXctNNN3H++ecDkJSUxO9//3tKSkq45557iIiIIDo6ml/84hcArFixgqVLl5KdnW0DuYHu6EEoftkJ+j3roaPFadVPWQKFV0JUHBwugqpi59+PX4CWIz3Pjx0FmVMgs9D5iyCz0PkySMuHCGcJvWMt7Wwtq+s+m2ZzaS217hzwSbFRzJqQyu2XTmZ2XhqzJ6SSmmCteDN0bGrlABRuv6/nqoph51+haBWUbwQUUvNg2lVQuAxyz4fIftpHqnDsIFQVweFi91/3S+HYp927dUREUxUzgeKOcWxuGkNJZzYlOh4yp3B23mi3Lz6NyaOTrBVvTolNrWxMfzo7oHwD7HzJCfrqEmf7uFlw6T86YT/6DP+6aUQgeazzM/ES6pvb3FZ8HUX7y6gv28G4tv1MkkqmdVYyPWovF0a9SwTuItvHIqAiD1oLoa6rq8j9KyEuZfiOgQlbFvomPLQ1we51UPQSFL0CjYchIhoKLoJzv+606FPGD+olOzuV3VXHus+m2VRay65Dx1B1vgumjE5i9lkXMyfP6YuflJVERIRAW7PzRePbTVRVDLtfh47WnjdIGnt8F1HXuEHSmEGPGxjTJWhCX1XDYq7tQOtuC2oN1VD8itOi3/06tDc5fe5TFjut+cmXD6o1fbS5zZkr3u2L31JaS31zOwCj4qKYnZvGVWdnM9u9unVUXD/TCEfHwdiznB9fnR1Qu8+nm6gYqnbC1ueg1Wd95tgUny8Dn39T87rHDYzpT1CEflxcHNXV1WRkZIR08Ksq1dXVxMXZpfCnrHq302WzcxWUvQfaCaPGw+zlMG0Z5F0IUf4PjDa0tLP2k4O8uPUAbxQfoq1DEYHCMclcNWNcd1/8xMxEpxV/OiIiIWOS81N4Zc92VTh6oNe4QTHsWgNbft+zX1QcZEw+8YyijMkQZRdmGUdQDOS2tbVRXl5Oc3OzR1WNnLi4OHJycoiOtsUm/NLZCZWbewZiq3Y628ec7YR84TIYN3NQ3SHNbR28UVzFi1sree2TQzS1dTBmVCxXz8jm0sLRzJyQQnJ/rfiR1lTr00Xk86VQVwq4n22JcM4e6n1GUeZUiBvlZfXhRRVaj0FzPbQchZZ693Z9z+2EdKeBcgpCaiA3OjqagoICr8swgaK9Bfa+6Q7EvuycJSORkHcBzL3FCfq0vEG9ZFtHJ+/srmbllkrWbP+Uoy3tpCfG8Lm547lmRjbn5Keffkt+OMSnQe65zo+v1kZ33KDXGUUla6GzrWe/5Ow+uooKITHLxg18dbS5AX3ECezusD46wPZetxmgkZ09+5RD319BEfrG0FQLxWucgdiS15wWU0wSTF4EhVc5/fQJ6YN6yc5O5YN9Nby4tZKXt31KTUMrybFRLDlrLNfMzGbBpIzgncogJgHGzXB+fHW0u+MGvf4y2PKMc0y7xKX2uvjM/VJIyYWIIDomqtDaMIiAru97e7sfvQyRsc5fTrGjIDbZuZ040bnfe3vsKJ/tyT23oxOH/ZAERfeOCVN1pU7ffNFLsO9t0A7nzJXCK52gL7jYGRQdBFVla/kRXtxayV8/quRgfQtx0RFcPn0M18zM5pKpWcRFh+FgqCrUV554RtHhImio6tkvKh4yJ594RlH6pEGNlfilo80N3iO9ukO6bvezvXe4a+cAbyTHB29ft/sN7mTnZIDYZM/HTUKqe8eECVU4sLVnIPagO5tq1jRYcIdzxk32nEG3NFWVooNHeXFrJS9uPUBpTSMxkRFcUpjFNTOzWTRtNImxYf5REHFOWU0ZD5MuO/6xxpqevwi6uorKPoBtPnMsSiSkF5x4RlFM8skD+oQWts/29qaB646MOTGg0/JPEuJ9tLBjkoLrr5fTZC19462ONtj3Vk/Q15c7A48TzusZiM2YdEovvfdwgxv0lew6dIzICOGCSRlcMzObJWeOJSU+QAZjg1VrAxzedeK4Qc1u6Gwf+PkxvVvMvW+n9LPdJ7jtrKRu1tI3gau5HkpedScye9X5Mzwq3mlhXvp9mLoUEk9tcrrKuib++pHTov+4wpkTZ35+Ov9y3ZlcefY4m1N+KMUkQvYs58dXRxvU7HW+BNpb+u4aiUkOq9Z1ILHQNyOjvtJtzb8Ee//mnEGSkAlnXOP0z09c6Aw+noKqoy28vO0AL26tZMM+ZybMGTkp/K9l07lqxjiyU+OH7vcwA4uMdrp2sqZ6XYnpg4W+GR6qcGhHz0Bs5WZne/okOO/rTtBPmH/KV5AeaWxj9fZPefGjSt4uOUynwtQxSdx9xVSunpFNfubwnwVhTDCy0DdDp6MdSt/tadHX7Xe255wDi/7JGYjNnHrK53/3XB1byRvFVbR1KHkZCdy6cDLXzMymcGzyEP4yxoQmC31zelqOOfPaFK1y5rlpqnXOV564EC66E6ZeCcljTvnlm9s6WF9UxYsfVfLaJwdpbutk7Kg4bj4/n2tnZXP2+JSQnprDmKFmoW8Gr6+FRuJSnQHYactg0iKITTrll2/r6OTtksO8uPVA99WxGYkx/I+5E7hmZjbz8tIC8+pYY4KAhb7xT1Wx0ze/c5UzFz0KqblwzlcGXmjED31eHRsXxVL36tgLgvnqWGMCiIW+6dtAC40ULoMxZ57W/Cx9XR0bHx3J5WeM4ZoZ47ikMIvYqDC8OtaYYWShb3q0NTndNTtfcvrnG6pOe6GR3rqujl25pZIXP6qkrKbpuKtjL58+moQY+9/SmOFin65w17XQSNEqZ0C2rfG0FhrpT39Xx37rsilcYVfHGjNiLPTDTUcbVHzotOj3rIey93sWGpn1hVNaaKQ/FXVNvNT76tiCdP7l+rO48qyxdnWsMR6w0A91nZ3ORVJ733BCfv877hS64lw+f9HdTot+kAuN9Kfr6tiVWyrZuN+5OnZmTgr3XeVcHTsuxa6ONcZLFvqhqHa/E/B733AWG+maGjdjMsy8EQougfwLBz3/fH+6ro5dubWSd3Y7V8cWjknm7iumcs3MbPIy7OpYYwKFhX4oaKjuacnvfcNZJAMgaawzidnEhU7Qn+Yg7HFv2c/VsbddOpmrZ9jVscYEKgv9YNRyzJnuoCvkP3XnnY8d5bTgz7vVCfmswiFd8q6vq2PHpcRxywX5XDPTro41JhhY6AeD7sFXtzVfvsGZpTIyBiacC5fdBxMvdc6hP40LpPpiV8caE1os9AORKhzc7nbZvAH73+4ZfB03E86/DSZe4iw0corTEZ9MR6eyYV8NK7dW8vLHB6htbLOrY40JERb6gaJ2v0+/fK/B1xl/7/TLD+Hga29dV8eu3FLJSx/3XB27+Axn7diLp2ba1bHGhAALfa90Db52BX334OsYp6tm4kKnNZ+SM2wlqCo7P3XXjvW5OnZh19qxdnWsMSHHPtEjpbUB9r8Le9YdP/gak+xOc/ANJ+iHePC133LaO1n+5Pt8sK+GyAhhweRMuzrWmDBgoT9cOtqgYlPPGTZlH5w4+FqwELJnD/ngqz/+8EEpH+yr4Z4lhdx4zgQy7OpYY8KCX2kjIkuBnwORwJOq+kCvx/OAp4AsoAZYrqrl7mM3A/e5u/5vVX16iGoPLF3LA3adYXPC4Kt7GmXu+cMy+DoYja3t/OvrJZxbkM6tCyfZaZbGhJEBQ19EIoHHgMVAObBBRFaq6g6f3R4CfquqT4vIZcCPgS+KSDrwT8A8QIEP3efWDvUv4om6UncOG7dvvmvwNX0SzLjBHXy9aNgGX0/Vr9/ex+FjLfz7F+da4BsTZvxp6c8HSlR1D4CIPAdcB/iG/hnAne7tdcB/u7eXAK+qao373FeBpcAfTr90DzRUw743e4K+dq+zPXG0O/h6idOaT53gaZknU9fYyhNv7Oby6WOYm5fmdTnGmBHmT+iPB8p87pcD5/baZyvwWZwuoM8AySKS0c9zT5gLQERWACsAcnNz/a19+HUNvu5d74T8px8522OSndMnz/2aO/g6bUQGX4fCL97YzbGWdu5ZUuh1KcYYDwzVCOLdwL+JyC3Am0AF0OHvk1X1l8AvAebNm6dDVNPgnWzwNWc+XHqf05rPnuPJ4OvpOljfzG/e3sf1s8bb3DjGhCl/kqsC8O2vyHG3dVPVSpyWPiKSBHxOVetEpAJY2Ou560+j3qHlO/i69w3Y95bP4OsMOM89jTIABl+HwqOv7aJTle9cPtXrUowxHvEn9DcAU0SkACfsbwRu8t1BRDKBGlXtBL6PcyYPwGrg/4pIV+fxFe7j3qkr7TnDZu+b0HDI2Z4+0Rl8LbgECi4OuMHX07XvcAP/uaGMm87NJTcj+L/AjDGnZsDQV9V2EbkdJ8AjgadUdbuI/AjYqKorcVrzPxYRxeneuc19bo2I/AvOFwfAj7oGdUdM9+Br15WvvoOv7sDrxEsgNYDGEobBw68WEx0Zwe2XTfa6FGOMh0TVuy70vsybN083btx46i/Q2tAz7fCeritf1R18XdAzt/zo6UEz+Hq6dlTWs+zRv3Hrwkl8d+k0r8sxxgwDEflQVecNtF/wjUb250gF/GmFs+ZrZxtERDtXvl76j07QZ8+GyPCcXuChNUWkxEfztUsmeV2KMcZjoRP6iVlO2J/3dZ/BV1umb8O+Gl7feYjvLZ1mc+oYY0Io9KNi4CtrvK4ioKgqD76yk9HJsdxyQb7X5RhjAoCthBHC1hdVsWFfLd9cNIX4GJsL3xhjoR+yOjuVB1cXkZeRwI3nBO60EMaYkWWhH6L++vEBPjlQz52LpxJtSxsaY1yWBiGoraOTn64pYtrYZK6Zke11OcaYAGKhH4Ke31jG/upG7llSSEREeFyLYIzxj4V+iGlu6+DR13YxNy+Ny6aN9rocY0yAsdAPMU+/s4+D9S18b+k0WyDFGHMCC/0QcqSpjcfX72ZhYRbzC0JrwjhjzNCw0A8hv3pzD0ea2rj7ClsgxRjTNwv9EFF1tIWn3t7L1TPGcdb4FK/LMcYEKAv9EPHYuhJa2ju5y1r5xpiTsNAPAWU1jTzz/n5umDeBgkybZM4Y0z8L/RDws7XFRIhwx6IpXpdijAlwFvpBrvjgUf68uYKbL8hnbEqc1+UYYwKchX6Qe2h1EUkxUXzDFkgxxvjBQj+IbS6tZc2Og6y4eCJpiTFel2OMCQIW+kHKWSCliMykGL58YYHX5RhjgoSFfpB6q+Qw7+6p5rZLJ5MYGzoLoBljhpeFfhBSVX6yuojxqfHcdG6u1+UYY4KIhX4QemXbp3xUfoTvLJ5KbJQtg2iM8Z+FfpBp7+jkJ2uKmDI6ic/MHu91OcaYIGOhH2T+tKmCPVUN3HVFIZG2QIoxZpAs9INIc1sHj6wtZuaEVJacOcbrcowxQchCP4g8834plUea+d6SQlsgxRhzSiz0g8SxlnYeW1fChZMzuWByptflGGOClIV+kHjyb3uoaWjlniU2dbIx5tRZ6AeBmoZWnvzbXpaeOZaZE1K9LscYE8Qs9IPA4+tKaGxt5+4lU70uxRgT5Cz0A1xlXRO/fW8/n5uTw+TRyV6XY4wJchb6Ae7na3eBwrcXWyvfGHP6LPQD2O6qY/zXh2V84bxcxqfGe12OMSYEWOgHsIfXFBMfHcltl072uhRjTIjwK/RFZKmIFIlIiYjc28fjuSKyTkQ2i8hHIrLM3Z4vIk0issX9eWKof4FQ9XH5EV76+ABfuWgimUmxXpdjjAkRA07ELiKRwGPAYqAc2CAiK1V1h89u9wHPq+ovROQMYBWQ7z62W1VnDW3Zoe/B1TtJS4jmf15kC6QYY4aOPy39+UCJqu5R1VbgOeC6XvsoMMq9nQJUDl2J4efd3dX8bddhbl04meS4aK/LMcaEEH9CfzxQ5nO/3N3m65+B5SJSjtPK/6bPYwVut88bInJRX28gIitEZKOIbKyqqvK/+hCkqjy4eidjR8XxxfPzvC7HGBNihmog9/PAb1Q1B1gG/E5EIoADQK6qzgbuBJ4VkVG9n6yqv1TVeao6Lysra4hKCk5rPznE5tI6vn35FOKibYEUY8zQ8if0K4AJPvdz3G2+vgI8D6Cq7wJxQKaqtqhqtbv9Q2A3YCec96OjU/nJ6p1MzEzk7+bmeF2OMSYE+RP6G4ApIlIgIjHAjcDKXvuUAosARGQ6TuhXiUiWOxCMiEwEpgB7hqr4UPOXLRUUHzzGnVdMJSrSzqY1xgy9Ac/eUdV2EbkdWA1EAk+p6nYR+RGwUVVXAncBvxKR7+AM6t6iqioiFwM/EpE2oBP4uqrWDNtvE8Ra2zv52dpizswexbKzxnldjjEmRA0Y+gCqugpngNZ32/0+t3cAC/p43h+BP55mjWHhuQ2llNU08fSXzybClkE0xgwT60MIAI2t7Tz6WgnnFqRz8RRbIMUYM3ws9APAr9/ex+FjLXx36TRbBtEYM6ws9D1W19jKE2/s5vLpo5mbl+Z1OcaYEGeh77En3tjDsZZ27rZlEI0xI8BC30MH65v5zTt7uX7WeKaNPeGaNWOMGXIW+h569LVdtHco37ncrlczxowMC32P7K9u4D83lPH5+bnkZiR4XY6DBEmjAAAOM0lEQVQxJkxY6Hvk4VeLiY6M4JuX2QIpxpiRY6HvgR2V9azcWsmXFuQzelSc1+UYY8KIhb4HHlpTRHJsFF+7eJLXpRhjwoyF/gjbuK+G13ce4usLJ5GSYAukGGNGloX+CFJVHnyliKzkWL50gS2DaIwZeRb6I2h9cRUf7KvhW4umEB9jC6QYY0aehf4I6exUfvJKEbnpCfz9vAkDP8EYY4aBhf4I+evHB9hxoJ47F08lJsoOuzHGG5Y+I6Cto5OH1xQxbWwy187M9rocY0wYs9AfAf+1sZx91Y3cs6TQFkgxxnjKQn+YNbd18PPXipmbl8Zl00Z7XY4xJsxZ6A+zp9/Zx8H6Fr67pNAWSDHGeM5CfxjVN7fxizd2c8nULM6dmOF1OcYYY6E/nH715h7qGtu4xxZIMcYECAv9YVJ1tIX/eGsvV88Yx1njU7wuxxhjAAv9YfPYuhJa2ju56wpr5RtjAoeF/jAoq2nkmff3c8O8HAoyE70uxxhjulnoD4NH1u4iQoRvLZridSnGGHMcC/0htuvgUf68uZybL8hnXEq81+UYY8xxLPSH2ENrikiMieIbl9gCKcaYwGOhP4Q2l9ayevtB/ufFE0lLjPG6HGOMOYGF/hD6yeoiMhJj+PKFtkCKMSYwWegPkbd2Head3dXcftlkkmKjvC7HGGP6ZKE/BFSVB1fvZHxqPDedm+t1OcYY0y8L/SHwyrZP+aj8CN++fAqxUbYMojEmcFnon6b2jk4eWlPE5NFJfHZOjtflGGPMSVnon6Y/ba5gd1UDd19RSKQtkGKMCXB+hb6ILBWRIhEpEZF7+3g8V0TWichmEflIRJb5PPZ993lFIrJkKIv3WnNbBz9fu4uZE1JZcuYYr8sxxpgBDRj6IhIJPAZcCZwBfF5Ezui1233A86o6G7gReNx97hnu/TOBpcDj7uuFhGfeL6WirskWSDHGBA1/WvrzgRJV3aOqrcBzwHW99lFglHs7Bah0b18HPKeqLaq6FyhxXy/oHWtp57F1JSyYnMGCyZlel2OMMX7xJ/THA2U+98vdbb7+GVguIuXAKuCbg3huUPqPv+2lpqGV7y6Z5nUpxhjjt6EayP088BtVzQGWAb8TEb9fW0RWiMhGEdlYVVU1RCUNn5qGVn71tz0sPXMsMyekel2OMcb4zZ9grgAm+NzPcbf5+grwPICqvgvEAZl+PhdV/aWqzlPVeVlZWf5X75HH15XQ2NrO3Uumel2KMcYMij+hvwGYIiIFIhKDMzC7stc+pcAiABGZjhP6Ve5+N4pIrIgUAFOAD4aqeC9U1jXx2/f289k5OUwenex1OcYYMygDThKjqu0icjuwGogEnlLV7SLyI2Cjqq4E7gJ+JSLfwRnUvUVVFdguIs8DO4B24DZV7RiuX2YkPPraLlD49uW2QIoxJvj4NTOYqq7CGaD13Xa/z+0dwIJ+nvt/gP9zGjUGjD1Vx/ivD8v5h/PzyElL8LocY4wZNLsidxB++moxsVER3HbpZK9LMcaYU2Kh76dtFUd46aMDfPXCAjKTYr0uxxhjTomFvp8eXF1EakI0X714otelGGPMKbPQ98N7e6p5s7iK2xZOZlRctNflGGPMKbPQH4Cq8uArOxk7Ko4vnp/ndTnGGHNaLPQHsPaTQ2wqreOOy6cQFx0yc8UZY8KUhf5JdHQqD60uoiAzkf8x1xZIMcYEPwv9k1i5tYKig0e564qpREXaoTLGBD9Lsn60tnfy8KvFnJk9imVnjfO6HGOMGRIW+v14bkMpZTVN3LOkkAhbBtEYEyIs9PvQ2NrOo6+VML8gnUumBv6sn8YY4y8L/T78+u19HD7WwveW2jKIxpjQYqHfy5HGNv79jd1cPn00c/PSvS7HGGOGlIV+L0+8uZujLe3cvaTQ61KMMWbIWej7OFTfzK/f3st1M7OZNnbUwE8wxpggY6Hv49HXd9Heody52Fr5xpjQZKHv2l/dwHMflPH5+bnkZtgCKcaY0GSh7/rZq8VERQrfvMwWSDHGhC4LfeCTA/X8ZWslX1pQwOhRcV6XY4wxw8ZCH3hodRHJsVF8/eJJXpdijDHDKuxDf+O+Gl7beYivL5xESoItkGKMCW1hHfrOAilFZCXH8qULCrwuxxhjhl1Yh/764io+2FfDty6bTHyMLZBijAl9YRv6nZ3KT14pYkJ6PH9/Tq7X5RhjzIgI29B/6eMD7DhQz12LC4mJCtvDYIwJM2GZdm0dzgIp08Ymc+3MbK/LMcaYEROWof/Ch+XsPdzA3VfYAinGmPASdqHf3NbBI2uLmZObyqLpo70uxxhjRlTYhf5v393HwfoWvrd0mi2QYowJO2EV+vXNbTy+fjeXTM3i3IkZXpdjjDEjLqxC/8k391DX2MY9tkCKMSZMhU3oVx1t4cm39nLVjHGcNT7F63KMMcYTYRP6j60roaW9k7sWT/W6FGOM8UxYhH55bSPPvl/KDfNymJiV5HU5xhjjmbAI/UfW7gKBby2a4nUpxhjjKb9CX0SWikiRiJSIyL19PP4zEdni/hSLSJ3PYx0+j60cyuL9sevgUf60qZybz89jXEr8SL+9McYElKiBdhCRSOAxYDFQDmwQkZWquqNrH1X9js/+3wRm+7xEk6rOGrqSB+ehNUUkxETxjYW2DKIxxvjT0p8PlKjqHlVtBZ4DrjvJ/p8H/jAUxZ2uLWV1rN5+kBUXTyQ9McbrcowxxnP+hP54oMznfrm77QQikgcUAK/7bI4TkY0i8p6IXH/KlZ6Cn6zeSUZiDF++0BZIMcYY8KN7Z5BuBF5Q1Q6fbXmqWiEiE4HXReRjVd3t+yQRWQGsAMjNHZq57d/adZi3S6q5/+ozSIod6l/TGGOCkz8t/Qpggs/9HHdbX26kV9eOqla4/+4B1nN8f3/XPr9U1XmqOi8rK8uPkk5OVfnJ6p2MT43nC+fZAinGGNPFn9DfAEwRkQIRicEJ9hPOwhGRaUAa8K7PtjQRiXVvZwILgB29nzvUVm//lK3lR/j25VOIjbJlEI0xpsuA/R6q2i4itwOrgUjgKVXdLiI/AjaqatcXwI3Ac6qqPk+fDvy7iHTifME84HvWz3Do6FQeWlPM5NFJfHZOznC+lTHGBB2/OrtVdRWwqte2+3vd/+c+nvcOcPZp1Ddof9pUTsmhYzyxfA6RtkCKMcYcJ6SuyG1p7+CRtbuYmZPCkjPHel2OMcYEnJAK/WfeK6Wironv2gIpxhjTp5AJ/WMt7Ty2roQFkzNYMDnT63KMMSYghcwJ7I0t7ZyTn87XF07yuhRjjAlYIRP6o0fF8cQX53pdhjHGBLSQ6d4xxhgzMAt9Y4wJIxb6xhgTRiz0jTEmjFjoG2NMGLHQN8aYMGKhb4wxYcRC3xhjwogcPxOy90SkCth/Gi+RCRweonKGktU1OFbX4FhdgxOKdeWp6oCrUAVc6J8uEdmoqvO8rqM3q2twrK7BsboGJ5zrsu4dY4wJIxb6xhgTRkIx9H/pdQH9sLoGx+oaHKtrcMK2rpDr0zfGGNO/UGzpG2OM6YeFvjHGhJGgDH0RWSoiRSJSIiL39vF4rIj8p/v4+yKSHyB13SIiVSKyxf356gjV9ZSIHBKRbf08LiLyqFv3RyIyJ0DqWigiR3yO1/0jVNcEEVknIjtEZLuI3NHHPiN+zPysa8SPmYjEicgHIrLVreuHfewz4p9JP+vy5DPpvnekiGwWkb/28djwHS9VDaofIBLYDUwEYoCtwBm99rkVeMK9fSPwnwFS1y3Av3lwzC4G5gDb+nl8GfAyIMB5wPsBUtdC4K8eHK9xwBz3djJQ3Md/yxE/Zn7WNeLHzD0GSe7taOB94Lxe+3jxmfSnLk8+k+573wk829d/r+E8XsHY0p8PlKjqHlVtBZ4Druu1z3XA0+7tF4BFIiIBUJcnVPVNoOYku1wH/FYd7wGpIjIuAOryhKoeUNVN7u2jwCfA+F67jfgx87OuEeceg2Pu3Wj3p/cZIiP+mfSzLk+ISA5wFfBkP7sM2/EKxtAfD5T53C/nxP/xu/dR1XbgCJARAHUBfM7tDnhBRCYMc03+8rd2L5zv/nn+soicOdJv7v5ZPRunlejL02N2krrAg2PmdlVsAQ4Br6pqv8drBD+T/tQF3nwmHwG+C3T28/iwHa9gDP1g9iKQr6ozgFfp+SY3fduEM5/ITOBfgf8eyTcXkSTgj8C3VbV+JN/7ZAaoy5NjpqodqjoLyAHmi8hZI/G+A/GjrhH/TIrI1cAhVf1wuN+rL8EY+hWA77dxjrutz31EJApIAaq9rktVq1W1xb37JDB3mGvylz/HdMSpan3Xn+equgqIFpHMkXhvEYnGCdZnVPVPfeziyTEbqC4vj5n7nnXAOmBpr4e8+EwOWJdHn8kFwLUisg+nG/gyEfl9r32G7XgFY+hvAKaISIGIxOAMcqzstc9K4Gb39t8Br6s7IuJlXb36fK/F6ZMNBCuBf3DPSDkPOKKqB7wuSkTGdvVjish8nP9fhz0o3Pf8D+ATVX24n91G/Jj5U5cXx0xEskQk1b0dDywGdvbabcQ/k/7U5cVnUlW/r6o5qpqPkxOvq+ryXrsN2/GKGooXGUmq2i4itwOrcc6YeUpVt4vIj4CNqroS54PxOxEpwRkovDFA6vqWiFwLtLt13TLcdQGIyB9wzurIFJFy4J9wBrVQ1SeAVThno5QAjcCXAqSuvwO+ISLtQBNw4wh8eYPTEvsi8LHbHwzwj0CuT21eHDN/6vLimI0DnhaRSJwvmedV9a9efyb9rMuTz2RfRup42TQMxhgTRoKxe8cYY8wpstA3xpgwYqFvjDFhxELfGGPCiIW+McaEEQt9Y4wJIxb6xhgTRv4/Io9hXfGucI0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title('Accuracy')\n",
    "plt.plot(history.history['acc'], label='train')\n",
    "plt.plot(history.history['val_acc'], label='test')\n",
    "plt.legend()\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "S7Nu6Ir1YrHQ"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Model_Test.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "conda_amazonei_tensorflow_p36",
   "language": "python",
   "name": "conda_amazonei_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}