04_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from IPython.display import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data as data_utils\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"http://karpathy.github.io/assets/rnn/charseq.jpeg\" width=\"500\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Image(url=\"http://karpathy.github.io/assets/rnn/charseq.jpeg\", width=500,)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Input: seq_length x batch_size x input_size (embedding_dimension in this case)\n",
    "Output: seq_length x batch_size x hidden_size\n",
    "last_hidden_state: batch_size, hidden_size\n",
    "last_cell_state: batch, hidden_size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How does it look like?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./rnn_data/names/Arabic.txt\", encoding='utf-8') as f:\n",
    "    d = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Khoury\\n',\n",
       " 'Nahas\\n',\n",
       " 'Daher\\n',\n",
       " 'Gerges\\n',\n",
       " 'Nazari\\n',\n",
       " 'Maalouf\\n',\n",
       " 'Gerges\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Guirguis\\n',\n",
       " 'Baba\\n',\n",
       " 'Sabbagh\\n',\n",
       " 'Attia\\n',\n",
       " 'Tahan\\n',\n",
       " 'Haddad\\n',\n",
       " 'Aswad\\n',\n",
       " 'Najjar\\n',\n",
       " 'Dagher\\n',\n",
       " 'Maloof\\n',\n",
       " 'Isa\\n',\n",
       " 'Asghar\\n',\n",
       " 'Nader\\n',\n",
       " 'Gaber\\n',\n",
       " 'Abboud\\n',\n",
       " 'Maalouf\\n',\n",
       " 'Zogby\\n',\n",
       " 'Srour\\n',\n",
       " 'Bahar\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Hanania\\n',\n",
       " 'Daher\\n',\n",
       " 'Tuma\\n',\n",
       " 'Nahas\\n',\n",
       " 'Saliba\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Handal\\n',\n",
       " 'Baba\\n',\n",
       " 'Amari\\n',\n",
       " 'Bahar\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Said\\n',\n",
       " 'Khouri\\n',\n",
       " 'Tahan\\n',\n",
       " 'Baba\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Guirguis\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Seif\\n',\n",
       " 'Dagher\\n',\n",
       " 'Bahar\\n',\n",
       " 'Gaber\\n',\n",
       " 'Harb\\n',\n",
       " 'Seif\\n',\n",
       " 'Asker\\n',\n",
       " 'Nader\\n',\n",
       " 'Antar\\n',\n",
       " 'Awad\\n',\n",
       " 'Srour\\n',\n",
       " 'Shadid\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Hanania\\n',\n",
       " 'Kalb\\n',\n",
       " 'Shadid\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Masih\\n',\n",
       " 'Ghanem\\n',\n",
       " 'Haddad\\n',\n",
       " 'Isa\\n',\n",
       " 'Antoun\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Dagher\\n',\n",
       " 'Najjar\\n',\n",
       " 'Malouf\\n',\n",
       " 'Nahas\\n',\n",
       " 'Naser\\n',\n",
       " 'Saliba\\n',\n",
       " 'Shamon\\n',\n",
       " 'Malouf\\n',\n",
       " 'Kalb\\n',\n",
       " 'Daher\\n',\n",
       " 'Maalouf\\n',\n",
       " 'Wasem\\n',\n",
       " 'Kanaan\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Boutros\\n',\n",
       " 'Moghadam\\n',\n",
       " 'Masih\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Aswad\\n',\n",
       " 'Cham\\n',\n",
       " 'Assaf\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Sabbag\\n',\n",
       " 'Mifsud\\n',\n",
       " 'Gaber\\n',\n",
       " 'Shammas\\n',\n",
       " 'Tannous\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Rahal\\n',\n",
       " 'Cham\\n',\n",
       " 'Ghanem\\n',\n",
       " 'Ghanem\\n',\n",
       " 'Naser\\n',\n",
       " 'Baba\\n',\n",
       " 'Shamon\\n',\n",
       " 'Almasi\\n',\n",
       " 'Basara\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Bata\\n',\n",
       " 'Wasem\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Deeb\\n',\n",
       " 'Touma\\n',\n",
       " 'Asfour\\n',\n",
       " 'Deeb\\n',\n",
       " 'Hadad\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Touma\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Nahas\\n',\n",
       " 'Haddad\\n',\n",
       " 'Arian\\n',\n",
       " 'Kouri\\n',\n",
       " 'Deeb\\n',\n",
       " 'Toma\\n',\n",
       " 'Halabi\\n',\n",
       " 'Nazari\\n',\n",
       " 'Saliba\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Hadad\\n',\n",
       " 'Baba\\n',\n",
       " 'Mansour\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Antar\\n',\n",
       " 'Deeb\\n',\n",
       " 'Morcos\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Amari\\n',\n",
       " 'Wasem\\n',\n",
       " 'Ganim\\n',\n",
       " 'Tuma\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Hadad\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Nader\\n',\n",
       " 'Said\\n',\n",
       " 'Ganim\\n',\n",
       " 'Daher\\n',\n",
       " 'Ganem\\n',\n",
       " 'Tuma\\n',\n",
       " 'Boutros\\n',\n",
       " 'Aswad\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Daher\\n',\n",
       " 'Toma\\n',\n",
       " 'Boutros\\n',\n",
       " 'Kanaan\\n',\n",
       " 'Antar\\n',\n",
       " 'Gerges\\n',\n",
       " 'Kouri\\n',\n",
       " 'Maroun\\n',\n",
       " 'Wasem\\n',\n",
       " 'Dagher\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Bishara\\n',\n",
       " 'Ba\\n',\n",
       " 'Cham\\n',\n",
       " 'Kalb\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Bitar\\n',\n",
       " 'Hadad\\n',\n",
       " 'Moghadam\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Antar\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Koury\\n',\n",
       " 'Nahas\\n',\n",
       " 'Kouri\\n',\n",
       " 'Maroun\\n',\n",
       " 'Nassar\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Haik\\n',\n",
       " 'Ghanem\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Salib\\n',\n",
       " 'Cham\\n',\n",
       " 'Bata\\n',\n",
       " 'Touma\\n',\n",
       " 'Antoun\\n',\n",
       " 'Antar\\n',\n",
       " 'Bata\\n',\n",
       " 'Botros\\n',\n",
       " 'Shammas\\n',\n",
       " 'Ganim\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Seif\\n',\n",
       " 'Moghadam\\n',\n",
       " 'Ba\\n',\n",
       " 'Tannous\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Seif\\n',\n",
       " 'Salib\\n',\n",
       " 'Hadad\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Halabi\\n',\n",
       " 'Essa\\n',\n",
       " 'Bahar\\n',\n",
       " 'Kattan\\n',\n",
       " 'Boutros\\n',\n",
       " 'Nahas\\n',\n",
       " 'Sabbagh\\n',\n",
       " 'Kanaan\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Said\\n',\n",
       " 'Botros\\n',\n",
       " 'Najjar\\n',\n",
       " 'Toma\\n',\n",
       " 'Bata\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Halabi\\n',\n",
       " 'Tannous\\n',\n",
       " 'Kouri\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Kassis\\n',\n",
       " 'Haddad\\n',\n",
       " 'Tuma\\n',\n",
       " 'Mansour\\n',\n",
       " 'Antar\\n',\n",
       " 'Kassis\\n',\n",
       " 'Kalb\\n',\n",
       " 'Basara\\n',\n",
       " 'Rahal\\n',\n",
       " 'Mansour\\n',\n",
       " 'Handal\\n',\n",
       " 'Morcos\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Hadad\\n',\n",
       " 'Morcos\\n',\n",
       " 'Kouri\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Almasi\\n',\n",
       " 'Awad\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Koury\\n',\n",
       " 'Asker\\n',\n",
       " 'Maroun\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Sabbag\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Shamon\\n',\n",
       " 'Assaf\\n',\n",
       " 'Boutros\\n',\n",
       " 'Malouf\\n',\n",
       " 'Nassar\\n',\n",
       " 'Qureshi\\n',\n",
       " 'Ghanem\\n',\n",
       " 'Srour\\n',\n",
       " 'Almasi\\n',\n",
       " 'Qureshi\\n',\n",
       " 'Ghannam\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Najjar\\n',\n",
       " 'Kassab\\n',\n",
       " 'Shadid\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Morcos\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Isa\\n',\n",
       " 'Ba\\n',\n",
       " 'Baz\\n',\n",
       " 'Asker\\n',\n",
       " 'Seif\\n',\n",
       " 'Asghar\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Deeb\\n',\n",
       " 'Essa\\n',\n",
       " 'Qureshi\\n',\n",
       " 'Abboud\\n',\n",
       " 'Ganem\\n',\n",
       " 'Haddad\\n',\n",
       " 'Koury\\n',\n",
       " 'Nassar\\n',\n",
       " 'Abadi\\n',\n",
       " 'Toma\\n',\n",
       " 'Tannous\\n',\n",
       " 'Harb\\n',\n",
       " 'Issa\\n',\n",
       " 'Khouri\\n',\n",
       " 'Mifsud\\n',\n",
       " 'Kalb\\n',\n",
       " 'Gaber\\n',\n",
       " 'Ganim\\n',\n",
       " 'Boulos\\n',\n",
       " 'Samaha\\n',\n",
       " 'Haddad\\n',\n",
       " 'Sabbag\\n',\n",
       " 'Wasem\\n',\n",
       " 'Dagher\\n',\n",
       " 'Rahal\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Antar\\n',\n",
       " 'Asghar\\n',\n",
       " 'Mansour\\n',\n",
       " 'Awad\\n',\n",
       " 'Boulos\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Deeb\\n',\n",
       " 'Abadi\\n',\n",
       " 'Nazari\\n',\n",
       " 'Daher\\n',\n",
       " 'Gerges\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Gaber\\n',\n",
       " 'Amari\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Nazari\\n',\n",
       " 'Saliba\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Nazari\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Shamon\\n',\n",
       " 'Abboud\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Tahan\\n',\n",
       " 'Safar\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Srour\\n',\n",
       " 'Gaber\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Attia\\n',\n",
       " 'Safar\\n',\n",
       " 'Said\\n',\n",
       " 'Ganem\\n',\n",
       " 'Nader\\n',\n",
       " 'Asghar\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Said\\n',\n",
       " 'Antar\\n',\n",
       " 'Botros\\n',\n",
       " 'Nader\\n',\n",
       " 'Ghannam\\n',\n",
       " 'Asfour\\n',\n",
       " 'Tahan\\n',\n",
       " 'Mansour\\n',\n",
       " 'Attia\\n',\n",
       " 'Touma\\n',\n",
       " 'Najjar\\n',\n",
       " 'Kassis\\n',\n",
       " 'Abboud\\n',\n",
       " 'Bishara\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Safar\\n',\n",
       " 'Khoury\\n',\n",
       " 'Nazari\\n',\n",
       " 'Sabbag\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Kouri\\n',\n",
       " 'Bitar\\n',\n",
       " 'Zogby\\n',\n",
       " 'Ghanem\\n',\n",
       " 'Assaf\\n',\n",
       " 'Abadi\\n',\n",
       " 'Arian\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Khoury\\n',\n",
       " 'Morcos\\n',\n",
       " 'Shamon\\n',\n",
       " 'Wasem\\n',\n",
       " 'Abadi\\n',\n",
       " 'Antoun\\n',\n",
       " 'Baz\\n',\n",
       " 'Naser\\n',\n",
       " 'Assaf\\n',\n",
       " 'Saliba\\n',\n",
       " 'Nader\\n',\n",
       " 'Mikhail\\n',\n",
       " 'Naser\\n',\n",
       " 'Daher\\n',\n",
       " 'Morcos\\n',\n",
       " 'Awad\\n',\n",
       " 'Nahas\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Malouf\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Ghannam\\n',\n",
       " 'Shadid\\n',\n",
       " 'Gaber\\n',\n",
       " 'Koury\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Shamon\\n',\n",
       " 'Boutros\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Arian\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Abadi\\n',\n",
       " 'Kassab\\n',\n",
       " 'Nahas\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Mansour\\n',\n",
       " 'Samaha\\n',\n",
       " 'Wasem\\n',\n",
       " 'Seif\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Saliba\\n',\n",
       " 'Cham\\n',\n",
       " 'Bahar\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Essa\\n',\n",
       " 'Shamon\\n',\n",
       " 'Asfour\\n',\n",
       " 'Bitar\\n',\n",
       " 'Cham\\n',\n",
       " 'Tahan\\n',\n",
       " 'Tannous\\n',\n",
       " 'Daher\\n',\n",
       " 'Khoury\\n',\n",
       " 'Shamon\\n',\n",
       " 'Bahar\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Ghannam\\n',\n",
       " 'Kassab\\n',\n",
       " 'Zogby\\n',\n",
       " 'Basara\\n',\n",
       " 'Shammas\\n',\n",
       " 'Arian\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Mifsud\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Arian\\n',\n",
       " 'Kassis\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Kassis\\n',\n",
       " 'Harb\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Boulos\\n',\n",
       " 'Asghar\\n',\n",
       " 'Shamon\\n',\n",
       " 'Kanaan\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Kassab\\n',\n",
       " 'Tahan\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Kassis\\n',\n",
       " 'Qureshi\\n',\n",
       " 'Basara\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Haik\\n',\n",
       " 'Attia\\n',\n",
       " 'Maroun\\n',\n",
       " 'Kassis\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Harb\\n',\n",
       " 'Assaf\\n',\n",
       " 'Kattan\\n',\n",
       " 'Antar\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Touma\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Boulos\\n',\n",
       " 'Baz\\n',\n",
       " 'Issa\\n',\n",
       " 'Shamon\\n',\n",
       " 'Shadid\\n',\n",
       " 'Deeb\\n',\n",
       " 'Sabbag\\n',\n",
       " 'Wasem\\n',\n",
       " 'Awad\\n',\n",
       " 'Mansour\\n',\n",
       " 'Saliba\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Arian\\n',\n",
       " 'Bishara\\n',\n",
       " 'Dagher\\n',\n",
       " 'Bishara\\n',\n",
       " 'Koury\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Naser\\n',\n",
       " 'Nader\\n',\n",
       " 'Antar\\n',\n",
       " 'Gerges\\n',\n",
       " 'Handal\\n',\n",
       " 'Hanania\\n',\n",
       " 'Shadid\\n',\n",
       " 'Gerges\\n',\n",
       " 'Kassis\\n',\n",
       " 'Essa\\n',\n",
       " 'Assaf\\n',\n",
       " 'Shadid\\n',\n",
       " 'Seif\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Baba\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Sabbagh\\n',\n",
       " 'Isa\\n',\n",
       " 'Najjar\\n',\n",
       " 'Tannous\\n',\n",
       " 'Hanania\\n',\n",
       " 'Ganem\\n',\n",
       " 'Gerges\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Mifsud\\n',\n",
       " 'Nahas\\n',\n",
       " 'Bishara\\n',\n",
       " 'Bishara\\n',\n",
       " 'Abadi\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Masih\\n',\n",
       " 'Isa\\n',\n",
       " 'Attia\\n',\n",
       " 'Kalb\\n',\n",
       " 'Essa\\n',\n",
       " 'Boulos\\n',\n",
       " 'Basara\\n',\n",
       " 'Halabi\\n',\n",
       " 'Halabi\\n',\n",
       " 'Dagher\\n',\n",
       " 'Attia\\n',\n",
       " 'Kassis\\n',\n",
       " 'Tuma\\n',\n",
       " 'Gerges\\n',\n",
       " 'Ghannam\\n',\n",
       " 'Toma\\n',\n",
       " 'Baz\\n',\n",
       " 'Asghar\\n',\n",
       " 'Zogby\\n',\n",
       " 'Aswad\\n',\n",
       " 'Hadad\\n',\n",
       " 'Dagher\\n',\n",
       " 'Naser\\n',\n",
       " 'Shadid\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Zogby\\n',\n",
       " 'Abboud\\n',\n",
       " 'Tannous\\n',\n",
       " 'Khouri\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Ganem\\n',\n",
       " 'Maalouf\\n',\n",
       " 'Isa\\n',\n",
       " 'Maroun\\n',\n",
       " 'Issa\\n',\n",
       " 'Khouri\\n',\n",
       " 'Harb\\n',\n",
       " 'Nader\\n',\n",
       " 'Awad\\n',\n",
       " 'Nahas\\n',\n",
       " 'Said\\n',\n",
       " 'Baba\\n',\n",
       " 'Totah\\n',\n",
       " 'Ganim\\n',\n",
       " 'Handal\\n',\n",
       " 'Mansour\\n',\n",
       " 'Basara\\n',\n",
       " 'Malouf\\n',\n",
       " 'Said\\n',\n",
       " 'Botros\\n',\n",
       " 'Samaha\\n',\n",
       " 'Safar\\n',\n",
       " 'Tahan\\n',\n",
       " 'Botros\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Handal\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Malouf\\n',\n",
       " 'Bishara\\n',\n",
       " 'Aswad\\n',\n",
       " 'Khouri\\n',\n",
       " 'Baz\\n',\n",
       " 'Asker\\n',\n",
       " 'Toma\\n',\n",
       " 'Koury\\n',\n",
       " 'Gerges\\n',\n",
       " 'Bishara\\n',\n",
       " 'Boulos\\n',\n",
       " 'Najjar\\n',\n",
       " 'Aswad\\n',\n",
       " 'Shamon\\n',\n",
       " 'Kouri\\n',\n",
       " 'Srour\\n',\n",
       " 'Assaf\\n',\n",
       " 'Tannous\\n',\n",
       " 'Attia\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Kattan\\n',\n",
       " 'Asghar\\n',\n",
       " 'Amari\\n',\n",
       " 'Shadid\\n',\n",
       " 'Said\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Masih\\n',\n",
       " 'Antar\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Shadid\\n',\n",
       " 'Masih\\n',\n",
       " 'Handal\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Kassis\\n',\n",
       " 'Salib\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Totah\\n',\n",
       " 'Koury\\n',\n",
       " 'Totah\\n',\n",
       " 'Mustafa\\n',\n",
       " 'Sabbagh\\n',\n",
       " 'Moghadam\\n',\n",
       " 'Toma\\n',\n",
       " 'Srour\\n',\n",
       " 'Almasi\\n',\n",
       " 'Totah\\n',\n",
       " 'Maroun\\n',\n",
       " 'Kattan\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Mikhail\\n',\n",
       " 'Nazari\\n',\n",
       " 'Boutros\\n',\n",
       " 'Guirguis\\n',\n",
       " 'Gaber\\n',\n",
       " 'Kassis\\n',\n",
       " 'Masih\\n',\n",
       " 'Hanania\\n',\n",
       " 'Maloof\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Cham\\n',\n",
       " 'Hadad\\n',\n",
       " 'Tahan\\n',\n",
       " 'Bitar\\n',\n",
       " 'Arian\\n',\n",
       " 'Gaber\\n',\n",
       " 'Baz\\n',\n",
       " 'Mansour\\n',\n",
       " 'Kalb\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Attia\\n',\n",
       " 'Antar\\n',\n",
       " 'Asfour\\n',\n",
       " 'Said\\n',\n",
       " 'Essa\\n',\n",
       " 'Koury\\n',\n",
       " 'Hadad\\n',\n",
       " 'Tuma\\n',\n",
       " 'Moghadam\\n',\n",
       " 'Sabbagh\\n',\n",
       " 'Amari\\n',\n",
       " 'Dagher\\n',\n",
       " 'Srour\\n',\n",
       " 'Antoun\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Maroun\\n',\n",
       " 'Tuma\\n',\n",
       " 'Nahas\\n',\n",
       " 'Hanania\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Amari\\n',\n",
       " 'Sabbagh\\n',\n",
       " 'Said\\n',\n",
       " 'Cham\\n',\n",
       " 'Asker\\n',\n",
       " 'Nassar\\n',\n",
       " 'Bitar\\n',\n",
       " 'Said\\n',\n",
       " 'Dagher\\n',\n",
       " 'Safar\\n',\n",
       " 'Khouri\\n',\n",
       " 'Totah\\n',\n",
       " 'Khoury\\n',\n",
       " 'Salib\\n',\n",
       " 'Basara\\n',\n",
       " 'Abboud\\n',\n",
       " 'Baz\\n',\n",
       " 'Isa\\n',\n",
       " 'Cham\\n',\n",
       " 'Amari\\n',\n",
       " 'Mifsud\\n',\n",
       " 'Hadad\\n',\n",
       " 'Rahal\\n',\n",
       " 'Khoury\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Basara\\n',\n",
       " 'Totah\\n',\n",
       " 'Ghannam\\n',\n",
       " 'Koury\\n',\n",
       " 'Malouf\\n',\n",
       " 'Zogby\\n',\n",
       " 'Zogby\\n',\n",
       " 'Boutros\\n',\n",
       " 'Nassar\\n',\n",
       " 'Handal\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Maloof\\n',\n",
       " 'Abadi\\n',\n",
       " 'Maroun\\n',\n",
       " 'Mifsud\\n',\n",
       " 'Kalb\\n',\n",
       " 'Amari\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Boutros\\n',\n",
       " 'Masih\\n',\n",
       " 'Kattan\\n',\n",
       " 'Haddad\\n',\n",
       " 'Arian\\n',\n",
       " 'Nazari\\n',\n",
       " 'Assaf\\n',\n",
       " 'Attia\\n',\n",
       " 'Wasem\\n',\n",
       " 'Gerges\\n',\n",
       " 'Asker\\n',\n",
       " 'Tahan\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Shadid\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Attia\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Aswad\\n',\n",
       " 'Deeb\\n',\n",
       " 'Tannous\\n',\n",
       " 'Totah\\n',\n",
       " 'Cham\\n',\n",
       " 'Baba\\n',\n",
       " 'Najjar\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Handal\\n',\n",
       " 'Awad\\n',\n",
       " 'Guirguis\\n',\n",
       " 'Awad\\n',\n",
       " 'Ganem\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Khoury\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Moghadam\\n',\n",
       " 'Mikhail\\n',\n",
       " 'Ghannam\\n',\n",
       " 'Guirguis\\n',\n",
       " 'Tannous\\n',\n",
       " 'Kanaan\\n',\n",
       " 'Handal\\n',\n",
       " 'Khoury\\n',\n",
       " 'Kalb\\n',\n",
       " 'Qureshi\\n',\n",
       " 'Najjar\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Gerges\\n',\n",
       " 'Nassar\\n',\n",
       " 'Tahan\\n',\n",
       " 'Hadad\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Salib\\n',\n",
       " 'Wasem\\n',\n",
       " 'Bitar\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Attia\\n',\n",
       " 'Awad\\n',\n",
       " 'Totah\\n',\n",
       " 'Deeb\\n',\n",
       " 'Touma\\n',\n",
       " 'Botros\\n',\n",
       " 'Nazari\\n',\n",
       " 'Nahas\\n',\n",
       " 'Kouri\\n',\n",
       " 'Ghannam\\n',\n",
       " 'Assaf\\n',\n",
       " 'Asfour\\n',\n",
       " 'Sarraf\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Toma\\n',\n",
       " 'Asghar\\n',\n",
       " 'Abboud\\n',\n",
       " 'Issa\\n',\n",
       " 'Sabbag\\n',\n",
       " 'Sabbagh\\n',\n",
       " 'Isa\\n',\n",
       " 'Koury\\n',\n",
       " 'Kattan\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Rahal\\n',\n",
       " 'Kalb\\n',\n",
       " 'Naser\\n',\n",
       " 'Masih\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Dagher\\n',\n",
       " 'Asker\\n',\n",
       " 'Maroun\\n',\n",
       " 'Dagher\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Botros\\n',\n",
       " 'Sleiman\\n',\n",
       " 'Harb\\n',\n",
       " 'Tahan\\n',\n",
       " 'Tuma\\n',\n",
       " 'Said\\n',\n",
       " 'Hadad\\n',\n",
       " 'Samaha\\n',\n",
       " 'Harb\\n',\n",
       " 'Cham\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Haik\\n',\n",
       " 'Malouf\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Harb\\n',\n",
       " 'Malouf\\n',\n",
       " 'Ghanem\\n',\n",
       " 'Cham\\n',\n",
       " 'Asghar\\n',\n",
       " 'Samaha\\n',\n",
       " 'Khouri\\n',\n",
       " 'Nassar\\n',\n",
       " 'Rahal\\n',\n",
       " 'Baz\\n',\n",
       " 'Kalb\\n',\n",
       " 'Rahal\\n',\n",
       " 'Gerges\\n',\n",
       " 'Cham\\n',\n",
       " 'Sayegh\\n',\n",
       " 'Shadid\\n',\n",
       " 'Morcos\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Qureshi\\n',\n",
       " 'Ganim\\n',\n",
       " 'Shadid\\n',\n",
       " 'Khoury\\n',\n",
       " 'Boutros\\n',\n",
       " 'Hanania\\n',\n",
       " 'Antoun\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Deeb\\n',\n",
       " 'Samaha\\n',\n",
       " 'Awad\\n',\n",
       " 'Asghar\\n',\n",
       " 'Awad\\n',\n",
       " 'Saliba\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Mikhail\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Mikhail\\n',\n",
       " 'Cham\\n',\n",
       " 'Halabi\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Kattan\\n',\n",
       " 'Nazari\\n',\n",
       " 'Safar\\n',\n",
       " 'Morcos\\n',\n",
       " 'Khoury\\n',\n",
       " 'Essa\\n',\n",
       " 'Nassar\\n',\n",
       " 'Haik\\n',\n",
       " 'Shadid\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Najjar\\n',\n",
       " 'Arian\\n',\n",
       " 'Botros\\n',\n",
       " 'Daher\\n',\n",
       " 'Saliba\\n',\n",
       " 'Saliba\\n',\n",
       " 'Kattan\\n',\n",
       " 'Hajjar\\n',\n",
       " 'Nader\\n',\n",
       " 'Daher\\n',\n",
       " 'Nassar\\n',\n",
       " 'Maroun\\n',\n",
       " 'Harb\\n',\n",
       " 'Nassar\\n',\n",
       " 'Antar\\n',\n",
       " 'Shammas\\n',\n",
       " 'Toma\\n',\n",
       " 'Antar\\n',\n",
       " 'Koury\\n',\n",
       " 'Nader\\n',\n",
       " 'Botros\\n',\n",
       " 'Bahar\\n',\n",
       " 'Najjar\\n',\n",
       " 'Maloof\\n',\n",
       " 'Salib\\n',\n",
       " 'Malouf\\n',\n",
       " 'Mansour\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Atiyeh\\n',\n",
       " 'Kanaan\\n',\n",
       " 'Bishara\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Saliba\\n',\n",
       " 'Tuma\\n',\n",
       " 'Mifsud\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Assaf\\n',\n",
       " 'Nassar\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Bitar\\n',\n",
       " 'Isa\\n',\n",
       " 'Halabi\\n',\n",
       " 'Shamon\\n',\n",
       " 'Qureshi\\n',\n",
       " 'Bishara\\n',\n",
       " 'Maalouf\\n',\n",
       " 'Srour\\n',\n",
       " 'Boulos\\n',\n",
       " 'Safar\\n',\n",
       " 'Shamoun\\n',\n",
       " 'Ganim\\n',\n",
       " 'Abadi\\n',\n",
       " 'Koury\\n',\n",
       " 'Shadid\\n',\n",
       " 'Zogby\\n',\n",
       " 'Boutros\\n',\n",
       " 'Shadid\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Isa\\n',\n",
       " 'Totah\\n',\n",
       " 'Salib\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Gaber\\n',\n",
       " 'Antar\\n',\n",
       " 'Antar\\n',\n",
       " 'Najjar\\n',\n",
       " 'Fakhoury\\n',\n",
       " 'Malouf\\n',\n",
       " 'Salib\\n',\n",
       " 'Rahal\\n',\n",
       " 'Boulos\\n',\n",
       " 'Attia\\n',\n",
       " 'Said\\n',\n",
       " 'Kassis\\n',\n",
       " 'Bahar\\n',\n",
       " 'Bazzi\\n',\n",
       " 'Srour\\n',\n",
       " 'Antar\\n',\n",
       " 'Nahas\\n',\n",
       " 'Kassis\\n',\n",
       " 'Samaha\\n',\n",
       " 'Quraishi\\n',\n",
       " 'Asghar\\n',\n",
       " 'Asker\\n',\n",
       " 'Antar\\n',\n",
       " 'Totah\\n',\n",
       " 'Haddad\\n',\n",
       " 'Maloof\\n',\n",
       " 'Kouri\\n',\n",
       " 'Basara\\n',\n",
       " 'Bata\\n',\n",
       " 'Antar\\n',\n",
       " 'Shammas\\n',\n",
       " 'Arian\\n',\n",
       " 'Gerges\\n',\n",
       " 'Seif\\n',\n",
       " 'Almasi\\n',\n",
       " 'Tuma\\n',\n",
       " 'Shamoon\\n',\n",
       " 'Khoury\\n',\n",
       " 'Hakimi\\n',\n",
       " 'Abboud\\n',\n",
       " 'Baz\\n',\n",
       " 'Seif\\n',\n",
       " 'Issa\\n',\n",
       " 'Nazari\\n',\n",
       " 'Harb\\n',\n",
       " 'Shammas\\n',\n",
       " 'Amari\\n',\n",
       " 'Totah\\n',\n",
       " 'Malouf\\n',\n",
       " 'Sarkis\\n',\n",
       " 'Naser\\n',\n",
       " 'Zogby\\n',\n",
       " 'Handal\\n',\n",
       " 'Naifeh\\n',\n",
       " 'Cham\\n',\n",
       " 'Hadad\\n',\n",
       " 'Gerges\\n',\n",
       " 'Kalb\\n',\n",
       " 'Shalhoub\\n',\n",
       " 'Saliba\\n',\n",
       " 'Tannous\\n',\n",
       " 'Tahan\\n',\n",
       " 'Tannous\\n',\n",
       " 'Kassis\\n',\n",
       " 'Shadid\\n',\n",
       " 'Sabbag\\n',\n",
       " 'Tahan\\n',\n",
       " 'Abboud\\n',\n",
       " 'Nahas\\n',\n",
       " 'Shamoun\\n',\n",
       " ...]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Slusarski\n"
     ]
    }
   ],
   "source": [
    "import unicodedata\n",
    "import string\n",
    "\n",
    "all_letters = string.ascii_letters + \" .,;'\"\n",
    "\n",
    "# 유니코드 문자열을 일반 ASCII로 변환. http://stackoverflow.com/a/518232/2809427 \n",
    "def unicodeToAscii(s):\n",
    "    return ''.join(\n",
    "        c for c in unicodedata.normalize('NFD', s)\n",
    "        if unicodedata.category(c) != 'Mn'\n",
    "        and c in all_letters\n",
    "    )\n",
    "\n",
    "print(unicodeToAscii('Ślusàrski'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {'X': [], 'y': []}\n",
    "for base_dir,_,fnames in os.walk('./rnn_data/names/'):\n",
    "    for fname in fnames:\n",
    "        with open(os.path.join(base_dir, fname), encoding='utf-8') as f:\n",
    "            dd = f.readlines()\n",
    "        data['X'].extend([unicodeToAscii(d.split(\"\\n\")[0]) for d in dd])\n",
    "        data['y'].extend([fname.split(\".txt\")[0]]*len(dd))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Arabic',\n",
       " 'Chinese',\n",
       " 'Czech',\n",
       " 'Dutch',\n",
       " 'English',\n",
       " 'French',\n",
       " 'German',\n",
       " 'Greek',\n",
       " 'Irish',\n",
       " 'Italian',\n",
       " 'Japanese',\n",
       " 'Korean',\n",
       " 'Polish',\n",
       " 'Portuguese',\n",
       " 'Russian',\n",
       " 'Scottish',\n",
       " 'Spanish',\n",
       " 'Vietnamese'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(data['y'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Khoury'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['X'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN input matrix를 만들기 위한 조건\n",
    "* one-hot vector이므로 character의 사전\n",
    "* character set input이므로 각 word의 character의 개수\n",
    "* max 길이에 맞춰서 padding\n",
    "* Input: \n",
    "    * seq_length x batch_size x input_size (embedding_dimension in this case)\n",
    "    * batch_size x seq_length x input_size (embedding_dimension in this case) (batch_first = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_dict = dict()\n",
    "char_len = []\n",
    "\n",
    "for x in data['X']:\n",
    "    for xx in x:\n",
    "        first_char = char_dict.get(xx, None)\n",
    "        if first_char == None:\n",
    "            char_dict[xx] = len(char_dict.keys())\n",
    "        else:\n",
    "            continue\n",
    "    char_len.append(len(x))\n",
    "        \n",
    "input_dim = len(char_dict.keys()) # num of set of characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'K': 0,\n",
       " 'h': 1,\n",
       " 'o': 2,\n",
       " 'u': 3,\n",
       " 'r': 4,\n",
       " 'y': 5,\n",
       " 'N': 6,\n",
       " 'a': 7,\n",
       " 's': 8,\n",
       " 'D': 9,\n",
       " 'e': 10,\n",
       " 'G': 11,\n",
       " 'g': 12,\n",
       " 'z': 13,\n",
       " 'i': 14,\n",
       " 'M': 15,\n",
       " 'l': 16,\n",
       " 'f': 17,\n",
       " 'B': 18,\n",
       " 'b': 19,\n",
       " 'S': 20,\n",
       " 'A': 21,\n",
       " 't': 22,\n",
       " 'T': 23,\n",
       " 'n': 24,\n",
       " 'H': 25,\n",
       " 'd': 26,\n",
       " 'w': 27,\n",
       " 'j': 28,\n",
       " 'I': 29,\n",
       " 'Z': 30,\n",
       " 'm': 31,\n",
       " 'k': 32,\n",
       " 'W': 33,\n",
       " 'C': 34,\n",
       " 'Q': 35,\n",
       " 'R': 36,\n",
       " 'F': 37,\n",
       " 'c': 38,\n",
       " 'E': 39,\n",
       " 'Y': 40,\n",
       " 'J': 41,\n",
       " 'L': 42,\n",
       " 'O': 43,\n",
       " 'P': 44,\n",
       " ' ': 45,\n",
       " 'X': 46,\n",
       " 'p': 47,\n",
       " 'v': 48,\n",
       " 'x': 49,\n",
       " 'U': 50,\n",
       " 'V': 51,\n",
       " 'q': 52,\n",
       " \"'\": 53,\n",
       " ',': 54}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_len = max(char_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# padding -> 0 vector 로\n",
    "padded_X = []\n",
    "zero_vector = [0]*input_dim\n",
    "for x in data['X']:\n",
    "    char_list = []\n",
    "    for xx in x:\n",
    "        char_vector = [0]*input_dim\n",
    "        char_idx = char_dict[xx]\n",
    "        char_vector[char_idx]=1\n",
    "        char_list.append(char_vector)\n",
    "    char_list.extend([zero_vector]*(seq_len - len(x)))\n",
    "    padded_X.append(char_list)\n",
    "\n",
    "padded_X = np.array(padded_X, dtype=np.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20074, 19, 55)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "padded_X.shape # batch, seq_len, input_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data num: 20074\n",
      "seq len: 19\n",
      "input dim: 55\n"
     ]
    }
   ],
   "source": [
    "print(\"data num: {}\".format(padded_X.shape[0]))\n",
    "print(\"seq len: {}\".format(padded_X.shape[1]))\n",
    "print(\"input dim: {}\".format(padded_X.shape[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang2idx = dict()\n",
    "idx2lang = dict()\n",
    "for y in set(data['y']):\n",
    "    idx = len(lang2idx)\n",
    "    lang2idx[y] = idx\n",
    "    idx2lang[idx] = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Dutch': 0,\n",
       " 'Italian': 1,\n",
       " 'Greek': 2,\n",
       " 'Polish': 3,\n",
       " 'Vietnamese': 4,\n",
       " 'English': 5,\n",
       " 'German': 6,\n",
       " 'Portuguese': 7,\n",
       " 'Scottish': 8,\n",
       " 'Chinese': 9,\n",
       " 'French': 10,\n",
       " 'Czech': 11,\n",
       " 'Arabic': 12,\n",
       " 'Russian': 13,\n",
       " 'Irish': 14,\n",
       " 'Korean': 15,\n",
       " 'Japanese': 16,\n",
       " 'Spanish': 17}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lang2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = []\n",
    "for yy in data['y']:\n",
    "    y.append(lang2idx[yy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20074,)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(LSTM, self).__init__()\n",
    "        self.lstm = nn.LSTM(input_size=input_dim, hidden_size=128, batch_first=True)\n",
    "        self.out = nn.Linear(128, len(lang2idx.keys()))\n",
    "        if use_cuda:\n",
    "            self.lstm = self.lstm.cuda()\n",
    "            self.out = self.out.cuda()\n",
    "            \n",
    "    def forward(self, x, seq_len):\n",
    "        \"\"\"\n",
    "        x: (batch_size, seq(time), input_dim)\n",
    "        seq_len: (batch_size)\n",
    "        \"\"\" \n",
    "        h, _ = self.lstm(x)\n",
    "        # Implementation of the last relevant output indexing\n",
    "        idx = (seq_len - 1).view(-1,1,1).expand(h.size(0), 1, h.size(2)) # (batch_size, seq, hidden_dim)\n",
    "        last_h = h.gather(1, idx).squeeze(dim=1) # (batch_size, seq, hidden_dim) -> 하나의 batch는 마지막 output vector로 구성된 matrix\n",
    "#         print(last_h.size()) # [batch, 128] \n",
    "        o = self.out(last_h) # (batch_size, out_dim)\n",
    "        return F.softmax(o, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LSTM()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### prepare batch generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffle_idx = list(range(len(data['y'])))\n",
    "np.random.shuffle(shuffle_idx)\n",
    "trn_num = int(len(data['y'])*0.8)\n",
    "trn_idxs = shuffle_idx[:trn_num]\n",
    "val_idxs = shuffle_idx[trn_num:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4015"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(val_idxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_X, trn_y, trn_seq_len = torch.from_numpy(padded_X[trn_idxs]), torch.from_numpy(y[trn_idxs]).long(), torch.from_numpy(np.array(char_len)[trn_idxs])\n",
    "trn = data_utils.TensorDataset(trn_X, trn_y, trn_seq_len)\n",
    "trn_loader = data_utils.DataLoader(trn, batch_size=batch_size, shuffle=True, drop_last=False)\n",
    "\n",
    "val_X, val_y, val_seq_len = torch.from_numpy(padded_X[val_idxs]), torch.from_numpy(y[val_idxs]).long(), torch.from_numpy(np.array(char_len)[val_idxs])\n",
    "val = data_utils.TensorDataset(val_X, val_y, val_seq_len)\n",
    "val_loader = data_utils.DataLoader(val, batch_size=batch_size, shuffle=True, drop_last=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 50\n",
    "num_batches = len(trn_loader)\n",
    "val_ckpt = 250\n",
    "learning_rate = 0.005\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1/50 | step: 250/502 | trn_loss: 2.5279 | trn_acc: 45.89% | val_loss: 2.5018 | val_acc: 47.74%\n",
      "epoch: 1/50 | step: 500/502 | trn_loss: 2.5119 | trn_acc: 46.95% | val_loss: 2.4984 | val_acc: 47.97%\n",
      "epoch: 2/50 | step: 250/502 | trn_loss: 2.5014 | trn_acc: 47.95% | val_loss: 2.4752 | val_acc: 50.37%\n",
      "epoch: 2/50 | step: 500/502 | trn_loss: 2.4884 | trn_acc: 49.24% | val_loss: 2.4755 | val_acc: 50.37%\n",
      "epoch: 3/50 | step: 250/502 | trn_loss: 2.4725 | trn_acc: 50.88% | val_loss: 2.4824 | val_acc: 49.90%\n",
      "epoch: 3/50 | step: 500/502 | trn_loss: 2.4713 | trn_acc: 51.04% | val_loss: 2.4546 | val_acc: 52.26%\n",
      "epoch: 4/50 | step: 250/502 | trn_loss: 2.4942 | trn_acc: 48.73% | val_loss: 2.4997 | val_acc: 47.87%\n",
      "epoch: 4/50 | step: 500/502 | trn_loss: 2.4940 | trn_acc: 48.70% | val_loss: 2.4774 | val_acc: 50.15%\n",
      "epoch: 5/50 | step: 250/502 | trn_loss: 2.4783 | trn_acc: 50.30% | val_loss: 2.4644 | val_acc: 51.54%\n",
      "epoch: 5/50 | step: 500/502 | trn_loss: 2.4801 | trn_acc: 50.12% | val_loss: 2.4638 | val_acc: 51.61%\n",
      "epoch: 6/50 | step: 250/502 | trn_loss: 2.4804 | trn_acc: 50.10% | val_loss: 2.4595 | val_acc: 52.03%\n",
      "epoch: 6/50 | step: 500/502 | trn_loss: 2.4664 | trn_acc: 51.54% | val_loss: 2.4622 | val_acc: 51.59%\n",
      "epoch: 7/50 | step: 250/502 | trn_loss: 2.4696 | trn_acc: 51.21% | val_loss: 2.4408 | val_acc: 53.75%\n",
      "epoch: 7/50 | step: 500/502 | trn_loss: 2.4490 | trn_acc: 53.24% | val_loss: 2.4293 | val_acc: 54.96%\n",
      "epoch: 8/50 | step: 250/502 | trn_loss: 2.4231 | trn_acc: 56.04% | val_loss: 2.3852 | val_acc: 59.67%\n",
      "epoch: 8/50 | step: 500/502 | trn_loss: 2.3637 | trn_acc: 61.96% | val_loss: 2.3248 | val_acc: 65.40%\n",
      "epoch: 9/50 | step: 250/502 | trn_loss: 2.3170 | trn_acc: 66.67% | val_loss: 2.2989 | val_acc: 68.20%\n",
      "epoch: 9/50 | step: 500/502 | trn_loss: 2.2881 | trn_acc: 69.75% | val_loss: 2.2869 | val_acc: 69.32%\n",
      "epoch: 10/50 | step: 250/502 | trn_loss: 2.2711 | trn_acc: 71.41% | val_loss: 2.2669 | val_acc: 71.28%\n",
      "epoch: 10/50 | step: 500/502 | trn_loss: 2.2574 | trn_acc: 72.70% | val_loss: 2.2538 | val_acc: 72.77%\n",
      "epoch: 11/50 | step: 250/502 | trn_loss: 2.2499 | trn_acc: 73.41% | val_loss: 2.2482 | val_acc: 73.21%\n",
      "epoch: 11/50 | step: 500/502 | trn_loss: 2.2383 | trn_acc: 74.59% | val_loss: 2.2507 | val_acc: 72.97%\n",
      "epoch: 12/50 | step: 250/502 | trn_loss: 2.2298 | trn_acc: 75.39% | val_loss: 2.2428 | val_acc: 73.64%\n",
      "epoch: 12/50 | step: 500/502 | trn_loss: 2.2304 | trn_acc: 75.31% | val_loss: 2.2314 | val_acc: 74.95%\n",
      "epoch: 13/50 | step: 250/502 | trn_loss: 2.2179 | trn_acc: 76.62% | val_loss: 2.2343 | val_acc: 74.33%\n",
      "epoch: 13/50 | step: 500/502 | trn_loss: 2.2229 | trn_acc: 76.04% | val_loss: 2.2259 | val_acc: 75.25%\n",
      "epoch: 14/50 | step: 250/502 | trn_loss: 2.2103 | trn_acc: 77.31% | val_loss: 2.2222 | val_acc: 75.97%\n",
      "epoch: 14/50 | step: 500/502 | trn_loss: 2.2098 | trn_acc: 77.45% | val_loss: 2.2215 | val_acc: 75.77%\n",
      "epoch: 15/50 | step: 250/502 | trn_loss: 2.2059 | trn_acc: 77.74% | val_loss: 2.2213 | val_acc: 75.67%\n",
      "epoch: 15/50 | step: 500/502 | trn_loss: 2.2014 | trn_acc: 78.12% | val_loss: 2.2185 | val_acc: 75.97%\n",
      "epoch: 16/50 | step: 250/502 | trn_loss: 2.2012 | trn_acc: 78.24% | val_loss: 2.2171 | val_acc: 76.24%\n",
      "epoch: 16/50 | step: 500/502 | trn_loss: 2.1925 | trn_acc: 79.06% | val_loss: 2.2144 | val_acc: 76.39%\n",
      "epoch: 17/50 | step: 250/502 | trn_loss: 2.1889 | trn_acc: 79.36% | val_loss: 2.2203 | val_acc: 75.84%\n",
      "epoch: 17/50 | step: 500/502 | trn_loss: 2.1968 | trn_acc: 78.56% | val_loss: 2.2159 | val_acc: 76.41%\n",
      "epoch: 18/50 | step: 250/502 | trn_loss: 2.1880 | trn_acc: 79.54% | val_loss: 2.2112 | val_acc: 77.01%\n",
      "epoch: 18/50 | step: 500/502 | trn_loss: 2.1896 | trn_acc: 79.39% | val_loss: 2.2132 | val_acc: 76.66%\n",
      "epoch: 19/50 | step: 250/502 | trn_loss: 2.1805 | trn_acc: 80.20% | val_loss: 2.2154 | val_acc: 76.31%\n",
      "epoch: 19/50 | step: 500/502 | trn_loss: 2.1906 | trn_acc: 79.27% | val_loss: 2.2159 | val_acc: 76.19%\n",
      "epoch: 20/50 | step: 250/502 | trn_loss: 2.1766 | trn_acc: 80.61% | val_loss: 2.2144 | val_acc: 76.56%\n",
      "epoch: 20/50 | step: 500/502 | trn_loss: 2.1849 | trn_acc: 79.84% | val_loss: 2.2105 | val_acc: 76.76%\n",
      "epoch: 21/50 | step: 250/502 | trn_loss: 2.1738 | trn_acc: 80.85% | val_loss: 2.2147 | val_acc: 76.54%\n",
      "epoch: 21/50 | step: 500/502 | trn_loss: 2.1859 | trn_acc: 79.74% | val_loss: 2.2144 | val_acc: 76.49%\n",
      "epoch: 22/50 | step: 250/502 | trn_loss: 2.1744 | trn_acc: 80.83% | val_loss: 2.2124 | val_acc: 76.81%\n",
      "epoch: 22/50 | step: 500/502 | trn_loss: 2.1799 | trn_acc: 80.29% | val_loss: 2.2109 | val_acc: 76.79%\n",
      "epoch: 23/50 | step: 250/502 | trn_loss: 2.1716 | trn_acc: 81.14% | val_loss: 2.2062 | val_acc: 77.16%\n",
      "epoch: 23/50 | step: 500/502 | trn_loss: 2.1770 | trn_acc: 80.65% | val_loss: 2.2154 | val_acc: 76.36%\n",
      "epoch: 24/50 | step: 250/502 | trn_loss: 2.1735 | trn_acc: 80.90% | val_loss: 2.2071 | val_acc: 77.06%\n",
      "epoch: 24/50 | step: 500/502 | trn_loss: 2.1680 | trn_acc: 81.50% | val_loss: 2.2089 | val_acc: 76.88%\n",
      "epoch: 25/50 | step: 250/502 | trn_loss: 2.1668 | trn_acc: 81.58% | val_loss: 2.2108 | val_acc: 76.74%\n",
      "epoch: 25/50 | step: 500/502 | trn_loss: 2.1751 | trn_acc: 80.74% | val_loss: 2.2087 | val_acc: 76.96%\n",
      "epoch: 26/50 | step: 250/502 | trn_loss: 2.1630 | trn_acc: 81.95% | val_loss: 2.2084 | val_acc: 77.03%\n",
      "epoch: 26/50 | step: 500/502 | trn_loss: 2.1710 | trn_acc: 81.20% | val_loss: 2.2054 | val_acc: 77.28%\n",
      "epoch: 27/50 | step: 250/502 | trn_loss: 2.1648 | trn_acc: 81.74% | val_loss: 2.2077 | val_acc: 76.96%\n",
      "epoch: 27/50 | step: 500/502 | trn_loss: 2.1664 | trn_acc: 81.66% | val_loss: 2.2092 | val_acc: 76.93%\n",
      "epoch: 28/50 | step: 250/502 | trn_loss: 2.1641 | trn_acc: 81.84% | val_loss: 2.2071 | val_acc: 77.16%\n",
      "epoch: 28/50 | step: 500/502 | trn_loss: 2.1678 | trn_acc: 81.50% | val_loss: 2.2047 | val_acc: 77.33%\n",
      "epoch: 29/50 | step: 250/502 | trn_loss: 2.1629 | trn_acc: 81.97% | val_loss: 2.2057 | val_acc: 77.28%\n",
      "epoch: 29/50 | step: 500/502 | trn_loss: 2.1654 | trn_acc: 81.70% | val_loss: 2.2080 | val_acc: 77.16%\n",
      "epoch: 30/50 | step: 250/502 | trn_loss: 2.1668 | trn_acc: 81.53% | val_loss: 2.2061 | val_acc: 77.28%\n",
      "epoch: 30/50 | step: 500/502 | trn_loss: 2.1605 | trn_acc: 82.23% | val_loss: 2.2070 | val_acc: 77.21%\n",
      "epoch: 31/50 | step: 250/502 | trn_loss: 2.1654 | trn_acc: 81.76% | val_loss: 2.2068 | val_acc: 77.28%\n",
      "epoch: 31/50 | step: 500/502 | trn_loss: 2.1596 | trn_acc: 82.33% | val_loss: 2.2075 | val_acc: 77.18%\n",
      "epoch: 32/50 | step: 250/502 | trn_loss: 2.1535 | trn_acc: 82.85% | val_loss: 2.2068 | val_acc: 77.11%\n",
      "epoch: 32/50 | step: 500/502 | trn_loss: 2.1686 | trn_acc: 81.39% | val_loss: 2.2080 | val_acc: 77.06%\n",
      "epoch: 33/50 | step: 250/502 | trn_loss: 2.1575 | trn_acc: 82.45% | val_loss: 2.2047 | val_acc: 77.46%\n",
      "epoch: 33/50 | step: 500/502 | trn_loss: 2.1630 | trn_acc: 81.91% | val_loss: 2.2046 | val_acc: 77.18%\n",
      "epoch: 34/50 | step: 250/502 | trn_loss: 2.1497 | trn_acc: 83.25% | val_loss: 2.2054 | val_acc: 77.33%\n",
      "epoch: 34/50 | step: 500/502 | trn_loss: 2.1674 | trn_acc: 81.51% | val_loss: 2.2036 | val_acc: 77.55%\n",
      "epoch: 35/50 | step: 250/502 | trn_loss: 2.1623 | trn_acc: 82.00% | val_loss: 2.2031 | val_acc: 77.65%\n",
      "epoch: 35/50 | step: 500/502 | trn_loss: 2.1542 | trn_acc: 82.81% | val_loss: 2.2036 | val_acc: 77.53%\n",
      "epoch: 36/50 | step: 250/502 | trn_loss: 2.1601 | trn_acc: 82.25% | val_loss: 2.2075 | val_acc: 77.01%\n",
      "epoch: 36/50 | step: 500/502 | trn_loss: 2.1562 | trn_acc: 82.64% | val_loss: 2.2086 | val_acc: 77.06%\n",
      "epoch: 37/50 | step: 250/502 | trn_loss: 2.1570 | trn_acc: 82.59% | val_loss: 2.2060 | val_acc: 77.26%\n",
      "epoch: 37/50 | step: 500/502 | trn_loss: 2.1599 | trn_acc: 82.33% | val_loss: 2.2017 | val_acc: 77.88%\n",
      "epoch: 38/50 | step: 250/502 | trn_loss: 2.1565 | trn_acc: 82.61% | val_loss: 2.2015 | val_acc: 77.80%\n",
      "epoch: 38/50 | step: 500/502 | trn_loss: 2.1491 | trn_acc: 83.45% | val_loss: 2.1980 | val_acc: 78.22%\n",
      "epoch: 39/50 | step: 250/502 | trn_loss: 2.1466 | trn_acc: 83.65% | val_loss: 2.1977 | val_acc: 78.20%\n",
      "epoch: 39/50 | step: 500/502 | trn_loss: 2.1488 | trn_acc: 83.36% | val_loss: 2.1997 | val_acc: 77.90%\n",
      "epoch: 40/50 | step: 250/502 | trn_loss: 2.1456 | trn_acc: 83.66% | val_loss: 2.2052 | val_acc: 77.55%\n",
      "epoch: 40/50 | step: 500/502 | trn_loss: 2.1409 | trn_acc: 84.17% | val_loss: 2.1967 | val_acc: 78.10%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 41/50 | step: 250/502 | trn_loss: 2.1398 | trn_acc: 84.28% | val_loss: 2.1984 | val_acc: 78.00%\n",
      "epoch: 41/50 | step: 500/502 | trn_loss: 2.1405 | trn_acc: 84.25% | val_loss: 2.1957 | val_acc: 78.42%\n",
      "epoch: 42/50 | step: 250/502 | trn_loss: 2.1401 | trn_acc: 84.29% | val_loss: 2.1931 | val_acc: 78.50%\n",
      "epoch: 42/50 | step: 500/502 | trn_loss: 2.1339 | trn_acc: 84.97% | val_loss: 2.1925 | val_acc: 78.72%\n",
      "epoch: 43/50 | step: 250/502 | trn_loss: 2.1368 | trn_acc: 84.58% | val_loss: 2.1903 | val_acc: 78.92%\n",
      "epoch: 43/50 | step: 500/502 | trn_loss: 2.1310 | trn_acc: 85.17% | val_loss: 2.1986 | val_acc: 78.00%\n",
      "epoch: 44/50 | step: 250/502 | trn_loss: 2.1381 | trn_acc: 84.41% | val_loss: 2.1942 | val_acc: 78.32%\n",
      "epoch: 44/50 | step: 500/502 | trn_loss: 2.1293 | trn_acc: 85.31% | val_loss: 2.1963 | val_acc: 78.17%\n",
      "epoch: 45/50 | step: 250/502 | trn_loss: 2.1313 | trn_acc: 85.14% | val_loss: 2.1939 | val_acc: 78.52%\n",
      "epoch: 45/50 | step: 500/502 | trn_loss: 2.1323 | trn_acc: 85.05% | val_loss: 2.1928 | val_acc: 78.70%\n",
      "epoch: 46/50 | step: 250/502 | trn_loss: 2.1337 | trn_acc: 84.85% | val_loss: 2.1919 | val_acc: 78.60%\n",
      "epoch: 46/50 | step: 500/502 | trn_loss: 2.1287 | trn_acc: 85.42% | val_loss: 2.1936 | val_acc: 78.65%\n",
      "epoch: 47/50 | step: 250/502 | trn_loss: 2.1327 | trn_acc: 85.00% | val_loss: 2.1906 | val_acc: 78.87%\n",
      "epoch: 47/50 | step: 500/502 | trn_loss: 2.1266 | trn_acc: 85.55% | val_loss: 2.1948 | val_acc: 78.77%\n",
      "epoch: 48/50 | step: 250/502 | trn_loss: 2.1257 | trn_acc: 85.70% | val_loss: 2.1913 | val_acc: 78.77%\n",
      "epoch: 48/50 | step: 500/502 | trn_loss: 2.1278 | trn_acc: 85.49% | val_loss: 2.1940 | val_acc: 78.55%\n",
      "epoch: 49/50 | step: 250/502 | trn_loss: 2.1244 | trn_acc: 85.79% | val_loss: 2.1951 | val_acc: 78.42%\n",
      "epoch: 49/50 | step: 500/502 | trn_loss: 2.1294 | trn_acc: 85.34% | val_loss: 2.1918 | val_acc: 78.82%\n",
      "epoch: 50/50 | step: 250/502 | trn_loss: 2.1242 | trn_acc: 85.84% | val_loss: 2.1906 | val_acc: 78.92%\n",
      "epoch: 50/50 | step: 500/502 | trn_loss: 2.1271 | trn_acc: 85.51% | val_loss: 2.1954 | val_acc: 78.35%\n",
      "finish Training\n"
     ]
    }
   ],
   "source": [
    "trn_loss_list = []\n",
    "val_loss_list = []\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    trn_loss_batch = 0.0\n",
    "    trn_corr_batch = 0\n",
    "    for i, trn_data in enumerate(trn_loader):\n",
    "        trn_X, trn_y, trn_seq_len = trn_data\n",
    "        trn_seq_len = trn_seq_len.type(torch.LongTensor)\n",
    "        if use_cuda:\n",
    "            trn_X = trn_X.float().cuda()\n",
    "            trn_y = trn_y.cuda()\n",
    "            trn_seq_len = trn_seq_len.cuda()\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        model_y = model(trn_X, trn_seq_len)\n",
    "        loss = criterion(model_y, trn_y)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        trn_loss_batch += loss\n",
    "        trn_corr = len(trn_y[trn_y == model_y.argmax(dim=1)])\n",
    "        trn_corr_batch += trn_corr\n",
    "        \n",
    "        if (i+1) % val_ckpt == 0:\n",
    "            with torch.no_grad():\n",
    "                val_loss_all = 0.0\n",
    "                val_corr_all = 0\n",
    "                for j, val_data in enumerate(val_loader):\n",
    "                    val_X, val_y, val_seq_len = val_data\n",
    "                    val_seq_len = val_seq_len.type(torch.LongTensor)\n",
    "                    if use_cuda:\n",
    "                        val_X = val_X.float().cuda()\n",
    "                        val_y = val_y.cuda()\n",
    "                        val_seq_len = val_seq_len.cuda()\n",
    "\n",
    "                    val_model_y = model(val_X, val_seq_len)\n",
    "                    val_loss = criterion(val_model_y, val_y)\n",
    "                    val_loss_all += val_loss\n",
    "                    \n",
    "                    val_corr = len(val_y[val_y == val_model_y.argmax(dim=1)])\n",
    "                    val_corr_all += val_corr\n",
    "        \n",
    "            trn_loss_summary = trn_loss_batch / val_ckpt\n",
    "            trn_acc_summary = trn_corr_batch / (val_ckpt*batch_size) * 100\n",
    "            val_loss_summary = val_loss_all / len(val_loader)\n",
    "            val_acc_summary = val_corr_all / (len(val_loader)*batch_size) * 100\n",
    "            \n",
    "            print(\"epoch: {}/{} | step: {}/{} | trn_loss: {:.4f} | trn_acc: {:.2f}% | val_loss: {:.4f} | val_acc: {:.2f}%\".format(\n",
    "                epoch+1, num_epochs, i+1, num_batches, trn_loss_summary, trn_acc_summary, val_loss_summary, val_acc_summary\n",
    "            ))\n",
    "\n",
    "            trn_loss_list.append(trn_loss_summary)\n",
    "            val_loss_list.append(val_loss_summary)\n",
    "\n",
    "            trn_loss_batch = 0.0\n",
    "            trn_corr_batch = 0\n",
    "        \n",
    "print(\"finish Training\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f0891529470>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6wAAAIMCAYAAADxUCEeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl83VWd//HXydY0S5s2S1vSJd0XWmhpgVK2soPIJihujI46jssoKLj81Bkdl1FmcZhRER1hZnRQnGETUFYpOwVaKHTf9zXd07Rpm+T8/rhpabYmbZPcm+T1fDzyuMn5nu+9n9uHf/D2nO/5hBgjkiRJkiSlmrRkFyBJkiRJUlMMrJIkSZKklGRglSRJkiSlJAOrJEmSJCklGVglSZIkSSnJwCpJkiRJSkkGVkmSJElSSjKwSpIkSZJSkoFVkiRJkpSSDKySJEmSpJSUkewCmlJUVBTLysqSXYYkSZIkqY3Nnj17a4yxuDVzUzKwlpWVMWvWrGSXIUmSJElqYyGE1a2d65ZgSZIkSVJKMrBKkiRJklKSgVWSJEmSlJJS8hlWSZIkSerODh48yLp166iqqkp2KcctOzubgQMHkpmZedzvYWCVJEmSpBSzbt068vPzKSsrI4SQ7HKOWYyRbdu2sW7dOoYOHXrc7+OWYEmSJElKMVVVVRQWFnbKsAoQQqCwsPCEV4gNrJIkSZKUgjprWD2kLeo3sEqSJEmS6tm5cyd33nlnssswsEqSJEmS6msusFZXV3doHR66JEmSJEmq5+tf/zrLly9n4sSJZGZmkp2dTZ8+fVi0aBFPPfUUV1xxBeeccw6vvPIKpaWl/OEPf6Bnz55tXoeBVZIkSZJS2N8/Op8FG3a36XuOO6kX377q5Gav/+hHP2LevHnMmTOH5557jiuvvJJ58+YxdOhQVq1axdKlS/nd737Hf/zHf/CBD3yABx54gI9+9KNtWiO4JViSJEmS1IIzzjijXnuaoUOHMnHiRAAmT57MqlWr2uVzXWGVJEmSpBR2tJXQjpKbm1vv7x49ehz+PT09nX379rXL57rCKkmSJEmqJz8/n4qKimSX4QqrJEmSJKm+wsJCzj77bMaPH0/Pnj3p169fUuowsEqSJEmSGvntb3/b5HhZWRnz5s07/Pdtt93WbjW4JViSJEmSlJIMrJIkSZKklGRgPQ4VVQeTXYIkSZIkdXkG1mP0nUfmc92dr1BbG5NdiiRJkiR1aQbWYzRpcAHLtuzhmYWbk12KJEmSJHVpBtZjdOWEAQzq25OfP7+cGF1llSRJkqT2YmA9RhnpaXz63GG8tWYnr6/cnuxyJEmSJCnp8vLy2uV9DazH4f1TBlGYm8Vdzy9PdimSJEmS1GUZWI9DdmY6H59WxozF5SzcuDvZ5UiSJElSm/r617/Oz372s8N/f+c73+H73/8+F110EaeddhoTJkzgD3/4Q7vXkdHun9BF3XTWEH7+/HJ+8fxy7vjgpGSXI0mSJKmrevzrsGlu275n/wlwxY+avXzjjTdyyy238PnPfx6A//3f/+XJJ5/ki1/8Ir169WLr1q1MnTqVq6++mhBC29Z2BFdYj1NBThYfOmMwj76zkbXb9ya7HEmSJElqM5MmTWLLli1s2LCBt99+mz59+tC/f3++8Y1vcMopp3DxxRezfv16Nm9u3+4prrCegE+dO5Rfv7qKu19ayXeuPjnZ5UiSJEnqio6yEtqe3v/+93P//fezadMmbrzxRu69917Ky8uZPXs2mZmZlJWVUVVV1a41uMJ6Agb07sk1E0u57401bK88kOxyJEmSJKnN3Hjjjdx3333cf//9vP/972fXrl2UlJSQmZnJjBkzWL16dbvXYGA9QZ85fxhVB2v571dWJbsUSZIkSWozJ598MhUVFZSWljJgwAA+8pGPMGvWLCZMmMCvf/1rxowZ0+41uCX4BI0oyefisf3471dX8dfnDyMny39SSZIkSV3D3LnvHvZUVFTEq6++2uS8PXv2tMvnu8LaBj47fRg79x7kvtfXJrsUSZIkSeoyDKxtYPKQvpxR1pe7X1rJwZraZJcjSZIkSV2CgbWNfGb6MNbv3Mejb29IdimSJEmS1CUYWNvIBaNLGN0vn7ueX05tbUx2OZIkSZI6uRg7d65oi/oNrG0khMBfnz+MJZv3MGPxlmSXI0mSJKkTy87OZtu2bZ02tMYY2bZtG9nZ2Sf0Ph5p24auOvUk/uWpJdz1/HIuGtsv2eVIkiRJ6qQGDhzIunXrKC8vT3Ypxy07O5uBAwee0HsYWNtQZnoaf3XuUL7z6AIeeXsDV596UrJLkiRJktQJZWZmMnTo0GSXkXRuCW5jH506hEmDC/jmQ3NZv3NfssuRJEmSpE7LwNrGMtLTuOPGidTWRr70+znUeACTJEmSJB0XA2s7GFKYy99fM57XV27nFy8sT3Y5kiRJktQpGVjbyfWnlXLlKQP48VNLeGfdzmSXI0mSJEmdjoG1nYQQ+IdrJ1Cc34Nb7pvD3gPVyS5JkiRJkjoVA2s76p2TyY8/MJGV2yr53mMLk12OJEmSJHUqBtZ2dtbwQv76vOH87vU1PDl/U7LLkSRJkqROw8DaAb58ySjGl/bi6w+8w+bdVckuR5IkSZI6BQNrB8jKSOPfPjiJfQdruO3/3qbWVjeSJEmS1KIWA2sIYVAIYUYIYUEIYX4I4eYm5kwPIewKIcyp+/m7I65dHkJYHEJYFkL4elt/gQ63bXni5xgNL87jb987jheXbuU/X1nV9nVJkiRJUhfTmhXWauDWGOM4YCrw+RDCuCbmvRhjnFj3812AEEI68DPgCmAc8KFm7u08nvsR/GQy3PcRWPv6Md364TMGc/HYftz++CKWbq5opwIlSZIkqWtoMbDGGDfGGN+s+70CWAiUtvL9zwCWxRhXxBgPAPcB1xxvsSnh0u/DubfCqpfg7kvg7kth4WNQW9virSEEbr9+Aj2z0vnuYwuI0a3BkiRJktScY3qGNYRQBkwCXmvi8lkhhLdDCI+HEE6uGysF1h4xZx2tD7upKb8fXPS38KX5cPntULERfv8R+NnpMOs/4eDRD1UqzOvBLReP5MWlW5mxeEsHFd3A+jfhf26AA5XJ+XxJkiRJaoVWB9YQQh7wAHBLjHF3g8tvAkNijKcCPwEePtZCQgifDiHMCiHMKi8vP9bbO16PPJj6GfjCW3D93ZCVC4/dAneMh1d+AkdZPf3o1CEMK87l+48t5GBNyyuzbW7RH2HZ07DkyY7/bEmSJElqpVYF1hBCJomwem+M8cGG12OMu2OMe+p+/xOQGUIoAtYDg46YOrBurJEY4y9jjFNijFOKi4uP8WskUXoGTLgBPv08fOxRKBkLT30L1s9u9pbM9DT+9spxrNhayW9eXd2BxdYpX5R4nf9Qx3+2JEmSJLVSa04JDsDdwMIY44+bmdO/bh4hhDPq3ncb8AYwMoQwNISQBXwQeKStik8pIcDQ8+ADv4a0DFh49K85fXQx540q5o5nlrCj8kAHFVnnUGBd+jTs39Oxny1JkiRJrdSaFdazgZuAC49oW/OeEMJnQgifqZtzAzAvhPA28O/AB2NCNfA3wJMkDmv63xjj/Hb4HqmjZ59EcF346FG3BYcQ+NaVY6k8UMO/PrOk4+qr3g/bV8DgaVC9D5a6LViSJElSaspoaUKM8SUgtDDnp8BPm7n2J+BPx1VdZzX2KnjsS7BlAfQ7udlpo/rl85EzB3Pva2v46NQhjOqX3/61bVsGsRam/CVsX57YFjz++vb/XEmSJEk6Ri0GVh2H0VfCY19OrLIeJbAC3HLxKB5+az3fe2wBv/7EGdTtrG4/dduBv/riQS6sPZMLFz7B3/zqeWoycshID2SkpZGRHsjJyuC2S0dRmNejfeuRJEmSpGYcU1sbtVJ+Pxg8NRFYW9A3N4ubLx7Fi0u38tziDjgdecsiaknjiU15zMw+lywOMGrXy2zaXcXqbXtZtGk3b6/dye/fWMPPn1ve/vVIkiRJUjMMrO1l7FWweR5sazn03TR1CMOKcvneHxe0e5ub2i2LWEs/Ljh5MN/5m09CXn9uK53PH794Lk/cch5/vnU6z33lAq6ZWMpvX1/T8QdCSZIkSVIdA2t7GfPexOuix1qcmpWRxjevHMuK8kr+Z2b7trmp2riARTWlXDG+P6Slw7irYdkzsL+i3rzPTh/O3gM1/Perq9q1HkmSJElqjoG1vfQZAgNObdW2YIALx5Rw7sgi7nhmafutalYfoMeulawMgzh/VEli7OTroLoKltQ/LXhUv3wuHtuP/3plFZX7q9unHkmSJEk6CgNrexp7Fax7A3ZvaHFqos3NOCqqDvJvf17aLuXUbF1GOjX0GDCWnlnpicFBUyGvf+K04AY+d8Fwdu49yO9eX9Mu9UiSJEnS0RhY29PYqxOvi/7Yqumj++fz4TMH85uZq1m6uaLlG47RioWzARg6bvK7g2lpMO4aWPp0o23Bpw3uw9RhffnViyvZX13T5vVIkiRJ0tEYWNtT8WgoGgULH2n1LV+6eBQ5menc0Q6rrOuXvEVtDEyZfGb9CydfBzX7YfETje753PQRbNpdxcNvrW/zeiRJkiTpaAys7W3sVbDqZajc1qrphXk9+OhZQ3h87kZWba1sszJqayPVmxeyNXMAeXn59S8OOhPyB8CChxvdd+7IIsaX9uKu51dQUxvbrB5JkiRJaomBtb2NvQpiDSx5vNW3/OXZZWSkp/HLF1e0WRlvr9vJoOo1VBeOanzxyG3BVbvrXQoh8LnpI1i5tZIn5m1qs3okSZIkqSUG1vY2YCL0HtTq04IBSvKzuWHyQO6ftY4tu6vapIyn3lnH0LCRvmWnND1h3LWJbcENTgsGuOzk/gwryuXO55YRo6uskiRJkjqGgbW9hZBYZV3+bKNDjY7m0+cOo7q2lnteXnXCJcQYmTtvDlmhhuyTTm560qFtwU2cFpyeFvjM+cOZv2E3LyzdesL1SJIkSVJrGFg7wtiroOYALH2q1beUFeVyxYQB3DtzNburDp7Qxy/YuJvc3csSfxSPbnpSWlpilXXZM422BQNcO6mUAb2zuXPGshOqRZIkSZJay8DaEQadCbnFx7QtGOCz5w+nYn819848og9qzUF4+tvw1v+0+n2emLeJ0WnrEn8UNfEM6yEnH9oW3Pi04KyMND517jBeW7md2at3tPqzJUmSJOl4GVg7Qlo6jLkSljwFB1v/TOr40t6cO7KIu19aSdXBGjiwF+77CLx8Bzx/O7TyedI/zd3I1PytUDAYsnKbnzjwDMg/qcltwQAfOmMQfXIy+flzrrJKkiRJan8ZyS6g2xh7Fcz+L1gxA0Zf0erbPjt9OB/+j9d45LUFfGDJV2DNTBh6Pqx8HnashL7Djnr/0s0VLC+vZEzRBigee/QPO3Ra8Ky7E9uCs3vVu5yTlcHHpw3lX59ZwuJNFYzun9/k29TWRpaV72HDzn3sO1DDvoM17D1QU+/3qoM1jOqXz4fPHNzqfwtJkiRJ3YuBtaOUnQc9eie2BR9DYD1rWCHnn1TDpGc/SmQ94YZ7oP8E+OkUWD6jxcD6+LxNpFNDn32roPjylj/w5OvgtZ/D4sfh1BsbXf7YtCH88oXl/Py5ZdzxwUkAVNfUMn/Dbl5fuZ3XVm5n1urt7Nzb/HO3WelppKcFqqprmDiogHEn9Wp2riRJkqTuy8DaUTKyYPTlsPhPiedQ0zNbdVvYuZo793+TULOZ1875OVPHvy+xFbjXQFjxHJz+yaPe//i8TVxRWkXYdgCKx7T8gQNPf3dbcBOBtSAniw+fOZh7Xl7FoL45zFm7k9mrd7D3QA0AZYU5XDquH2cMLWRoUS45Wen0zEwnJyud7LrfM9PT2LX3IOf+47P881OLuefjp7fq30KSJElS92Jg7Uhjr4J3fg+rX4Zh01uev3k+/OZ95NRU8Te532PVwv48dnEkhADDp8PCx6C2JvGMbBNWba1k4cbdfPmMCtgGlLQisKalJQ5feuNXULULsns3mvKpc4fxm5mr+cmzyxjTP58bJg/kjKF9OaOsLyW9slv+DKB3TiafnT6C259YxOsrt3PG0L6tuk+SJElS9+GhSx1p+EWQ0bN1pwWveQ3+8woIgfCJJzj/gvcwf8NuXjzUB3XYBVC1EzbOafYtHp+3CYAzcrckBo52QvCRxl+faMMz+7+avNyvVzZ/vnU6c/7uEp645Ty+e8143nvKSa0Oq4d8fFoZ/Xr14PYnFhFbeYCUJEmSpO7DwNqRsnJg5MWw6I9QW5sYq62F/XtgzxbYsQo2L4C598Ovr4GcIvjEk1AylmsmnUS/Xj246/nlifuGnp94XT6j2Y97Yt5GThnYm957lkPvQdCj6UOSGhk4BUZeBi/8M+wpb3JKaUFPCnKyWvnFm9YzK52bLxrF7NU7eGbhlhN6L0mSJEldj4G1o429Gio2wj8Ohe/3g+/2gR+Wwj+PhH87FX5+FjzwSSgaCZ94AvoMAaBHRjqfOmcYryzfxpy1OyGvGPpNSDzH2oT1O/fx9rpdXD6+P5QvguLRx1bnpd+Hg3thxvdP8Asf3funDGRoUS7/9OQiampdZZUkSZL0Lp9h7Whj3gtTP5fYcpuZk/jJOvSaC5k9ISsfhkxLjB/hQ2cO5ifPLuWu55Zz102TE8+xvvYLOFDZqL/qE3Xbga8YVwIvLX13Rba1ikfB6X8Fr/8CTv9U4mTidpCZnsZtl47m8799k4ffWs/1kwe2y+dIkiRJ6nwMrB0tKwcu/+Fx3ZrXI4OPTSvjpzOWsbx8D8OHXQCv/ARWv5rYanyEx+duZEz/fIZmbIXqqtadENzQ9K/BO/fBE/8PPvYohHBcdbfkivH9mVDamx8/vYT3njqAHhlNHyIlSZIkqXtxS3An87FpZWSlp/EXd7/Ol2b2pCZksuL1x1i2ZQ/VNYnnYrfsrmL2mh1cMX4AbFmUuPF4AmvPPnDBN2HVi4nnbttJWlrgq5ePZv3Ofdw7c027fY4kSZKkzsUV1k6mKK8Hd9w4kf+bvY5ZGyqYWT2Kvov/zBVzLyEzPTCsKI+eWenECFdM6A9L6k4kLm7lCcENTf7LRIubp74FIy+BjB5t92WOcO7IYs4eUchPZyzj/VMGkp/duj61kiRJkrouV1g7oSsmDOCej5/Oi1+9kNMvuo6xaWv42dWlfPKcYQzq25Mdew9wzogiRpbkQfli6FXaZD/VVknPgMv+AXasTDwv246+etkYtlce4FcvrmzXz5EkSZLUObjC2slljbwInvs+V+Yt5cpp72884XhOCG5oxEV1bW7+CU79UOKE4nZw6qACrhjfn1+9uIKbzhpCUV7Tq7lVB2t47J2NzFi0hb997zj69z62/q+SJEmSOgdXWDu7AacmnjVd0UQ/1tpa2Lrk+J5fbeiyH3RIm5vbLhtNVXUtP312WaNrW3ZX8eOnFnP2j57ltv97mz/O3cj/zFzdrvVIkiRJSh4Da2eXlg5Dz0v0Y40N+pjuWpMImSe6wgqJvrBnfBre/DVsmnvi79eM4cV5vH/yQO59bTVrt+8FYM7andx831tM+9Gz/GTGMiYNLuDeT53JuSOLeHjOemrt3ypJkiR1SW4J7gqGXQAL/gBbl9Y/XKl8ceK1eGzbfM75X4W3f9fubW5uvngkD721nq/c/zb7q2t5a81O8npkcNNZQ/jYWWWUFSV6zm6pqOJLv3+bWat3cMbQvu1SiyRJkqTkcYW1Kxg2PfG64rn64+WHWtoc5wnBDXVQm5sBvXvy8bPLmLliOzsqD/Cdq8bx6v+7kG9fdfLhsApw6bj+9MxM56G31rdbLZIkSZKSx8DaFfQdCn3KGj/HumUR5PVPBM22MvkvE8/EPvUtqN7fdu/bwG2Xjubhz5/Ns7dO5+NnD22yzU1ujwwuH9+fP76zgaqDNe1WiyRJkqTkcEtwVzHsAph7P9RUJ1rRQGKFtaQNDlw60qE2N//zPvjnkZBbArlFkFNY91r07uvgM6Fg8HF9TGZ6GhMHFbQ479pJpTz01nqeW7yFy8cPOK7PkiRJkpSaDKxdxbDpMPs/Yf3sRFCMMfEM62k3tf1njbgIrvsFrJsFleWwdxtsWw5rX0v8HmsT89KzYOrn4NxbIbtX29cBnD28kOL8Hjz45noDqyRJktTFGFi7iqHnASGxLXjwmbBrLRysbJsTgpty6gcTPw3V1sC+nVCxAV79Gbx8B8z5LVz0tzDxI4lTjVtSWwMrX4Cda+C0vzjq4U4Z6WlcfepJ/PrVVezce4CCnKzj/06SJEmSUorPsHYVOX3hpInvHrx0+ITgNt4S3JK0dMgthP4T4Lq74K+eTTxj+8gX4Jfnw6qXmr4vRlg3Gx7/OvzLGPjNtfDoF2Ht6y1+5HWTSjlYE3nsnY1t/GUkSZIkJZOBtSsZdgGsewP2VxxxQnAHB9aGSifDJ56EG+5JrLz+15Xw+4/C9pWJ61uXwox/gJ+cBr+6EGbdnVghft+vICMb5t3f4kecfFIvRpbk8bCnBUuSJEldiluCu5LhF8BLP4ZVLycCa25JYuU12UKA8dfD6PfAqz+FF/8VljwJhSNhy3wgJLY0n/NlGHsV9Kw7bGnRozD/Ibjsh+8eJNXk2weuO62Uf3xiMWu27WVwYU7HfC9JkiRJ7coV1q5k0JmQ0TPxHOuWRe33/OrxyuwJ530FvjAbTvlA4iCmy/4BvrwQPvZI4oConkecDDz+hsShTqteaPGtr5lYCmBPVkmSJKkLcYW1K8noAUOmwfIZsHtD04cipYJeA+Can7U8b+Sl0KMXzH0Ahl941KmlBT2ZOqwvD89ZzxcvGkE4ykFNkiRJkjoHV1i7mmHTYetiOFDR9j1YO1pmNox5Lyx8FA5WtTj9ukmlrNxayZy1OzugOEmSJEntzcDa1Qy/4N3fk33gUluYcD3s3wXLnm5x6hUTBpCVkebhS5IkSVIXYWDtakpOhtzixO9dIbAOnQ45RTC35dOCe2VncsnYfjz6zkYO1tS2f22SJEmS2pWBtatJS4MRF0P+AMgtSnY1Jy49A06+DpY8kWjX04LrJpWyvfIALywp74DiJEmSJLUnA2tXdPkP4S//lOwq2s6EG6C6Cha1/J3OG1VMn5xMHnRbsCRJktTptRhYQwiDQggzQggLQgjzQwg3H2Xu6SGE6hDCDUeM1YQQ5tT9PNJWhesoevaBvsOSXUXbGXgG9B4E81reFpyVkcZVp57EMws2s7vqYAcUJ0mSJKm9tGaFtRq4NcY4DpgKfD6EMK7hpBBCOnA78FSDS/tijBPrfq4+4YrV/aSlwfj3wfJnoXJbi9OvnVTK/upanpi7qQOKkyRJktReWgysMcaNMcY3636vABYCpU1M/QLwALClTSuUAMbfALXVsODhFqdOGlRAWWEOD7ktWJIkSerUjukZ1hBCGTAJeK3BeClwHfDzJm7LDiHMCiHMDCFce5x1qrvrPwGKRsO8B1qcGkLg2kmlzFy5jQ0793VAcZIkSZLaQ6sDawghj8QK6i0xxt0NLt8BfC3G2FQvkSExxinAh4E7QgjDm3n/T9cF21nl5Z7wqgZCSBy+tPoV2NXyyul1k0qJER59e0MHFCdJkiSpPbQqsIYQMkmE1XtjjA82MWUKcF8IYRVwA3DnodXUGOP6utcVwHMkVmgbiTH+MsY4JcY4pbi4+Fi/h7qD8dcDEeY39T/B+oYU5jKmfz7PLnKHuiRJktRZteaU4ADcDSyMMf64qTkxxqExxrIYYxlwP/C5GOPDIYQ+IYQede9TBJwNLGiz6tW9FA6HkybB3JZPCwaYPrqE2at3UOFpwZIkSVKn1JoV1rOBm4ALj2hP854QwmdCCJ9p4d6xwKwQwtvADOBHMUYDq47f+Btg4xzYuqzFqReMLqa6NvLysq0dUJgkSZKktpbR0oQY40tAaO0bxhg/fsTvrwATjqsyqSnj3wdPfSvRk3X614869bQhfcjvkcGMReVcPn5ABxUoSZIkqa0c0ynBUtL1OgmGnJ3YFhzjUadmpqdxzsginl9STmxhriRJkqTUY2BV5zPhBti2FDa90+LUC0aXsGl3FYs2VXRAYZIkSZLakoFVnc+4ayAto1WHL50/OnHi9IzFnhYsSZIkdTYGVnU+OX1h+EUw70Gobar177v69cpm7IBePLfY3r6SJElSZ2NgVec04QbYvQ6W/7nFqdNHFzN79Q52295GkiRJ6lQMrOqcxl4NfYfDn26DA3uPOvWC0SXU1EZeWmp7G0mSJKkzMbCqc8rMhqv/HXasghk/OOrU0wYXkJ+dwXM+xypJkiR1KgZWdV5l58Dkj8PMO2H97GanZaSnca7tbSRJkqROx8Cqzu2S70JeP3jki1DT/DOq00eXsHn3fhZutL2NJEmS1FkYWNW5ZfeGK/8FNs+Dl+9odtr0Uba3kSRJkjobA6s6vzFXwsnXwfP/COVLmpxS0iubcQN68bztbSRJkqROw8CqruGKf4TMHHjkC832Zr1gTDGz1+xg1z7b20iSJEmdgYFVXUNeCVz+Q1g7E2bd3eSU6ba3kSRJkjoVA6u6jlM/BMMvhGe+AzvXNro8aVABvWxvI0mSJHUaBlZ1HSHAe++AGOGPX068HiHR3qaY52xvI0mSJHUKBlZ1LX2GwEV/C0ufgrn3N7o8fXQx5RX7mb9hdxKKkyRJknQsDKzqes74NJROgce/CpX1n1c9f3Sivc3zSzwtWJIkSUp1BlZ1PWnpcM1PYd8OmHVPvUsl+dmcfFIvn2OVJEmSOgEDq7qmkrHQdyhsmtvo0gWjS3hzzU527bW9jSRJkpTKDKzqukrGQfmiRsPTRxdTUxt5cZnbgiVJkqRUZmBV11U8BrYth+r99YYnHm5vY2CVJEmSUpmBVV1XyViINbBtWb3hjPQ0zhtVzPNLyqmttb2NJEmSlKoMrOq6isckXrcsbHRp+ugSyiv2s2Cj7W0kSZKkVGVgVddVNBJCepOB9fxRifY2nhYsSZIkpS4Dq7qujB5QOLzJg5eK83swpn8+s1bvSEJhkiRJklrDwKqurXhMkyusAGWFuazbsa+DC5IkSZLUWgZWdW0lY2H7CjjYOJgO7NOTdTv2EqMHL0k4+NVZAAAgAElEQVSSJEmpyMCqrq1kLBBh65JGlwb26UnVwVq27jnQ8XVJkiRJapGBVV1b8djE65bGz7EO7JMDwLodezuyIkmSJEmtZGBV11Y4HNIyobzxc6wD+/YE8DlWSZIkKUUZWNW1pWdC4YgmD14qLTCwSpIkSanMwKqur2Rsk4E1PzuTgpxMtwRLkiRJKcrAqq6vZCzsXA0HKhtdSpwU7AqrJEmSlIoMrOr6isckXssXN7o0sCDHFVZJkiQpRRlY1fWVHDopuPG24EF9Eyus9mKVJEmSUo+BVV1fn6GQ3qPpk4L75LC/2l6skiRJUioysKrrS8+AolHN9GI9dFKw24IlSZKkVGNgVfdQMgbKmwqsOQCs9eAlSZIkKeUYWNU9FI+BXWuhane94VJXWCVJkqSUZWBV91AyLvHa4KTgvB4Z9MnJtLWNJEmSlIIMrOoeSg61tmn64CUDqyRJkpR6DKzqHgrKIKNnswcvuSVYkiRJSj0GVnUPaWlQPAq2LGh0aWCfnqy3F6skSZKUcgys6j5KxjV5UvCgvolerOV79iehKEmSJEnNMbCq+ygeAxUbYd/OesPv9mL1OVZJkiQplRhY1X2UjE28NlhlPdSL1cAqSZIkpZYWA2sIYVAIYUYIYUEIYX4I4eajzD09hFAdQrjhiLGPhRCW1v18rK0Kl45Zcd1JwQ2eYy0tsBerJEmSlIoyWjGnGrg1xvhmCCEfmB1CeDrGWO+/+kMI6cDtwFNHjPUFvg1MAWLdvY/EGHe02TeQWqv3IMjKa3RScG6PDPrmZrF2uyuskiRJUippcYU1xrgxxvhm3e8VwEKgtImpXwAeALYcMXYZ8HSMcXtdSH0auPyEq5aOR1oaFI9upherrW0kSZKkVHNMz7CGEMqAScBrDcZLgeuAnze4pRRYe8Tf62g67Eodo3hss71Y1/sMqyRJkpRSWh1YQwh5JFZQb4kx7m5w+Q7gazHG2uMtJITw6RDCrBDCrPLy8uN9G+noSsZA5Rao3FZveGCfHNbt3Edtrb1YJUmSpFTRqsAaQsgkEVbvjTE+2MSUKcB9IYRVwA3AnSGEa4H1wKAj5g2sG2skxvjLGOOUGOOU4uLiY/gK0jE4fFJw/W3BA/v05EB1LVvtxSpJkiSljNacEhyAu4GFMcYfNzUnxjg0xlgWYywD7gc+F2N8GHgSuDSE0CeE0Ae4tG5MSo7iusC6pX5gHVTX2mat24IlSZKklNGaU4LPBm4C5oYQ5tSNfQMYDBBjvKu5G2OM20MI3wPeqBv6boxx+wnUK52YXidBj15N9GJ9t7XN5CF9klGZJEmSpAZaDKwxxpeA0No3jDF+vMHf9wD3HHNlUnsIIdGPtcEKa+nhwOoKqyRJkpQqjumUYKlLKBmbCKzx3QOWcrIyKMzNMrBKkiRJKcTAqu6nZCzs2w6V9U+jtherJEmSlFoMrOp+isckXrc0PCk4xxVWSZIkKYUYWNX9lDR9UvDAPj1Zv8NerJIkSVKqMLCq+8nrBz37NN2LtaaWcnuxSpIkSSnBwKruJ4REP9YtDVrb9E30YvU5VkmSJCk1GFjVPZWMSaywHnFS8CBb20iSJEkpxcCq7ql4LFTtgoqNh4dKCw6tsBpYJUmSpFRgYFX31MTBSz2z0inKy3JLsCRJkpQiDKzqng4F1vL6z7GW2tpGkiRJShkGVnVPuUWQU9RkaxsDqyRJkpQaDKzqvopGwvaV9YbsxSpJkiSlDgOruq/eg2DnmnpDA/vkcKCmli0V9mKVJEmSks3Aqu6rYDDsXg811YeHBh5ubePBS5IkSVKyGVjVfRUMglgDFRsODw3qY2sbSZIkKVUYWNV9FQxOvO5ce3jIFVZJkiQpdRhY1X31PhRY332ONTsznaK8Hq6wSpIkSSnAwKruq/fAxOuutfWGbW0jSZIkpQYDq7qvzGzI6w87V9cbTgRWtwRLkiRJyWZgVfdW0HRrm/U77cUqSZIkJZuBVd1bweB6hy5BYoX1YE20F6skSZKUZAZWdW+9B8GudVBbe3jIk4IlSZKk1GBgVfdWMBhqD8KeTYeHBtqLVZIkSUoJBlZ1bwWNW9scWmFdu90VVkmSJCmZDKzq3g4H1nefY83OTKc4316skiRJUrIZWNW9HerF2lRrm52usEqSJEnJZGBV95aVCzlFsKvhScE5rrBKkiRJSWZglZrsxdqTDTv3UWMvVkmSJClpDKzSUXuxViWpKEmSJEkGVqn3oMSW4PjuaqqtbSRJkqTkM7BKBUOgugoqyw8PHWpts26HBy9JkiRJyWJglQoGJV6PeI61tKAusG53hVWSJElKFgOrdLgX67utbbIz0ymxF6skSZKUVAZWqfehFdbGBy+tdUuwJEmSlDQGVim7F2QXNNHaJsfAKkmSJCWRgVWCxLbgXfVXWIcW5bJ+xz6qDtYkqShJkiSpezOwSlDXi7X+CuvwkjxqI6zcWpmkoiRJkqTuzcAqQV1grd+LdURxHgDLtuxJVlWSJElSt2ZglSBx8NLBSti7/fDQsOJcQoDl5QZWSZIkKRkMrBK829pm17vbgrMz0xnUJ8cVVkmSJClJDKwSQMGh1jb1n2MdUZJnYJUkSZKSxMAqwbsrrA16sQ4vzmXF1kpqamMTN0mSJElqTwZWCRJ9WLPym1xhPVBdyzr7sUqSJEkdzsAqAYTQZGubESWeFCxJkiQli4FVOqRgEOyqvyV4RHE+4EnBkiRJUjIYWKVDmlhh7Z2TSVFeD1dYJUmSpCRoMbCGEAaFEGaEEBaEEOaHEG5uYs41IYR3QghzQgizQgjnHHGtpm58Tgjhkbb+AlKb6T0I9u+GfTvrDQ8vzjWwSpIkSUmQ0Yo51cCtMcY3Qwj5wOwQwtMxxgVHzPkz8EiMMYYQTgH+FxhTd21fjHFi25YttYPDJwWvgZ4Fh4dHlOTx6NsbiDESQkhScZIkSVL30+IKa4xxY4zxzbrfK4CFQGmDOXtijIf6fuQC9gBR53OoF2vD51hL8thdVU35nv1JKEqSJEnqvo7pGdYQQhkwCXitiWvXhRAWAX8EPnHEpey6bcIzQwjXnkCtUvsqGJJ4beak4OVbKju6IkmSJKlba3VgDSHkAQ8At8QYdze8HmN8KMY4BrgW+N4Rl4bEGKcAHwbuCCEMb+b9P10XbGeVl5cf05eQ2kROIWTmwM7GK6wAyzwpWJIkSepQrQqsIYRMEmH13hjjg0ebG2N8ARgWQiiq+3t93esK4DkSK7RN3ffLGOOUGOOU4uLi1n8Dqa2EkDh4aefqesP9e2WTm5XOcg9ekiRJkjpUa04JDsDdwMIY44+bmTOibh4hhNOAHsC2EEKfEEKPuvEi4GxgQVPvIaWEgsGNnmENITC8JM+TgiVJkqQO1ppTgs8GbgLmhhDm1I19AxgMEGO8C7ge+IsQwkFgH3Bj3YnBY4FfhBBqSYTjHzU4XVhKLQWDYP2sRsMjivN4Zfm2JBQkSZIkdV8tBtYY40vAUXt5xBhvB25vYvwVYMJxVyd1tILBsG8H7K+AHvmHh4eX5PHgW+vZs7+avB6t+f95JEmSJJ2oYzolWOryete1tmnm4CWfY5UkSZI6joFVOlIzrW2GF9edFGxglSRJkjqMgVU6UkHdCmuDg5eGFOaQkRZsbSNJkiR1IAOrdKTcEkjv0ai1TWZ6GmVFua6wSpIkSR3IwCodKS0Neg9s9AwrJE4KXu4KqyRJktRhDKxSQwWDGz3DComDl1Zv28uB6tokFCVJkiR1PwZWqaGCQY2eYQUYXpJLTW1k9bbKJBQlSZIkdT8GVqmhgsFQWQ4H9tYbHlGc6Mvqc6ySJElSxzCwSg31Hpx43bWu3vDwklzAwCpJkiR1FAOr1FBBXWBt8BxrTlYGpQU9PXhJkiRJ6iAGVqmhw71YGx+8NLwkz16skiRJUgcxsEoN5Q+AtIwmTwoeXpzL8i2V1NbGJBQmSZIkdS8GVqmhtPS6XqxNt7bZd7CGDbv2JaEwSZIkqXsxsEpN6T0IdjZubTOiOA/w4CVJkiSpIxhYpaYUDGl2hRUMrJIkSVJHMLBKTSkYBHs2QfX+esOFeT3ok5PJ8vLKJBUmSZIkdR8GVqkpBU33YgUYXpzHcldYJUmSpHZnYJWa0ruutU0z24JtbSNJkiS1PwOr1JRDK6zNBNbtlQfYXnmgg4uSJEmSuhcDq9SUXidBSINdjU8KHu7BS5IkSVKHMLBKTUnPhF6lTa+w1rW2We62YEmSJKldGVil5jTTi7W0oCfZmWmusEqSJEntzMAqNadPGWxbCjHWG05LCwwryjOwSpIkSe3MwCo1p+xsqCyHTXMbXRpRYmCVJEmS2puBVWrOiIsTr8uebnypJI/1O/ex90B1BxclSZIkdR8GVqk5+f2h/ymwtOnACrCivLKjq5IkSZK6DQOrdDQjL4W1r8O+HfWGh3tSsCRJktTuDKzS0Yy8BGINLJ9Rb7isKIe0YC9WSZIkqT0ZWKWjKZ0C2QWw7Jl6wz0y0hlSmGtglSRJktqRgVU6mvQMGH5h4jnW2tp6l4YXe1KwJEmS1J4MrFJLRl4KlVtg0zv1hkeU5LFqWyXVNbXN3ChJkiTpRBhYpZaMuCjx2uC04LED8jlYE1m4sSIJRUmSJEldn4FVakleCZw0qVE/1rOGFQLwyvKtyahKkiRJ6vIMrFJrjLgE1r0Be7cfHirplc2IkjxeWb4tiYVJkiRJXZeBVWqNkZdCrIXlz9Ybnja8kDdWbeegz7FKkiRJbc7AKrVG6WnQs2+j51inDS9k74Ea3lm3M0mFSZIkSV2XgVVqjbT0xOFLy56p197mzKGFhACvLHNbsCRJktTWDKxSa424BPZuhY1vHR7qk5vFuAG9fI5VkiRJagcGVqm1RlwEBFj6TL3hs4YVMnvNDqoO1iSnLkmSJKmLMrBKrZVblHiWdelT9YanjSjkQHUtb67ekaTCJEmSpK7JwCodi5GXwvrZUPnuFuDTy/qSnhbcFixJkiS1MQOrdCxGXAJEWP7nw0P52ZmcMrA3ryzfmry6JEmSpC7IwCodi5MmQU5Rk+1t3lm3iz37q5NUmCRJktT1GFilY5GWdkR7m3cPWZo2vIjq2sgbq7YnsThJkiSpazGwSsdq5KWwbztseLe9zeQhfchKT+NVn2OVJEmS2oyBVTpWwy+EkFZvW3B2ZjqTBhf4HKskSZLUhloMrCGEQSGEGSGEBSGE+SGEm5uYc00I4Z0QwpwQwqwQwjlHXPtYCGFp3c/H2voLSB0upy+UTmnc3mZ4EfM37Gbn3gNJKkySJEnqWlqzwloN3BpjHAdMBT4fQhjXYM6fgVNjjBOBTwC/Aggh9AW+DZwJnAF8O4TQp62Kl5Jm5CWJLcF7yg8PTRtRSIwwc4XPsUqSJEltocXAGmPcGGN8s+73CmAhUNpgzp4YY6z7Mxc49PtlwNMxxu0xxh3A08DlbVW8lDQjG7e3OXVgAT0z03nVbcGSJElSmzimZ1hDCGXAJOC1Jq5dF0JYBPyRxCorJILt2iOmraNB2JU6pf6nQm5xvedYszLSOH1oX15d4cFLkiRJUltodWANIeQBDwC3xBh3N7weY3woxjgGuBb43rEWEkL4dN3zr7PKy8tbvkFKprQ0GHFJor1NzcHDw9OGF7Jk8x7KK/YnsThJkiSpa2hVYA0hZJIIq/fGGB882twY4wvAsBBCEbAeGHTE5YF1Y03d98sY45QY45Ti4uJWFS8l1YTroWonvPqzw0PThhcCuMoqSZIktYHWnBIcgLuBhTHGHzczZ0TdPEIIpwE9gG3Ak8ClIYQ+dYctXVo3JnV+Iy6G0VfCcz+CHasBOPmk3uRnZ/gcqyRJktQGWrPCejZwE3BhXduaOSGE94QQPhNC+EzdnOuBeSGEOcDPgBtjwnYS24PfqPv5bt2Y1DW85x8TPVn/9BWIkfS0wJlDC3lluSuskiRJ0onKaGlCjPElILQw53bg9mau3QPcc1zVSamu90C44Bvw1Ddh4SMw7hqmDS/kmYWbWbdjLwP75CS7QkmSJKnTOqZTgiU14czPQP8J8PjXoGo300bUPcfqKqskSZJ0Qgys0olKz4D3/htUbIIZP2BUST6FuVkevCRJkiSdIAOr1BYGTobTPwmv/5K0TXOYOryQV5dvI8aY7MokSZKkTsvAKrWVi/4Ocovh0Vs4e1gBG3dVsWrb3mRXJUmSJHVaBlaprWT3hst/CBvncOmeRwF4xfY2kiRJ0nEzsEpt6eT3wfCLKHz9n5iQX2l7G0mSJOkEGFilthQCXPkvhNqDfD/7f5i5fBu1tT7HKkmSJB0PA6vU1voOhfO+wqkVz3Pqvpks2lSR7IokSZKkTsnAKrWHaV+kunA038v8Lx56bUmyq5EkSZI6JQOr1B4yssh4779QGrZy8K3fsbvqYLIrkiRJkjodA6vUXsrOoarPKK7ief73jbXJrkaSJEnqdAysUnsJgewpH2Vy2lKeffllajx8SZIkSTomBlapPZ1yI5E0pu15mmcWbk52NZIkSVKnYmCV2lN+f+KIC7kh42X+88Xlya5GkiRJ6lQMrFI7S5v4YfqzlbQ1LzF/w65klyNJkiR1GgZWqb2NvpLYoxcfzHiR/3x5VbKrkSRJkjoNA6vU3jKzCeOv5/KMN/jznOWUV+xPdkWSJElSp2BglTrCxA+TVVvFxbzKb19bk+xqJEmSpE7BwCp1hIGnQ9/hfDLvVX4zczX7q2uSXZEkSZKU8gysUkcIASZ+iDH755JduZbH3t6Y7IokSZKklGdglTrKKR8kEvirXq9xz8sriTEmuyJJkiQppRlYpY5SMIgw9Dzel/YCCzbs5I1VO5JdkSRJkpTSDKxSR5r4YfL3rWd69nLueWllsquRJEmSUpqBVepIY6+CrDxuKZ7FUws2sXb73mRXJEmSJKUsA6vUkbJyYdw1TNg1g57hAL9+dVWyK5IkSZJSloFV6mgTP0zagT18ZfAS7ntjLZX7q5NdkSRJkpSSDKxSRxs8DQoG8760F6ioqua+N9YmuyJJkiQpJRlYpY6Wlganfoj8DS9z5ZAafjZjGRVVB5NdlSRJkpRyDKxSMpz6QQKRbw2ay/bKA/zi+RXJrkiSJElKOQZWKRn6DoPB0xiw8iGuOmUAv3ppBZt3VyW7KkmSJCmlGFilZJn4Idi2lG+eWklNbeSOZ5YkuyJJkiQppRhYpWQZdy1k5dH/nZ/z0alD+P0ba1m6uSLZVUmSJEkpw8AqJUt2LzjnS7D4j3xpxGZyszK4/YnFya5KkiRJShkGVimZzvo89B5Mr+f+js+eX8YzCzfz+srtya5KkiRJSgkGVimZMnvCJd+BzXP5VN4r9OvVg3/400JijMmuTJIkSUo6A6uUbCe/DwadSdbz/8BXLyhlztqdPD5vU7KrkiRJkpLOwColWwhw+Q+hcgvX7fk9o/rl8U9PLuZgTW2yK5MkSZKSysAqpYLSyXDKjaTNvJNvn5vHyq2V3Pf6mmRXJUmSJCWVgVVKFRd9G0Ia01b+hDOH9uWOZ5ayZ391squSJEmSksbAKqWK3qVw9s2E+Q/x/dP2sK3yAL98YUWyq5IkSZKSxsAqpZKzvwj5JzHyrR/w3gn9+I8XVrBld1Wyq5IkSZKSwsAqpZKsXLj427DhLb49ZB4Ha2r59iPzqam1zY0kSZK6HwOrlGomfABOOo3i137ENy4ZzOPzNnHb/71taJUkSVK3Y2CVUk1aWqLNTcVGPsEj3HbpKB56a72hVZIkSd2OgVVKRYOnwsnvg5f/nb+ZnMOtlyRC61cMrZIkSepGDKxSqrrk7yHWwgOf4gtnFnDrJaN48K31fOV+Q6skSZK6hxYDawhhUAhhRghhQQhhfgjh5ibmfCSE8E4IYW4I4ZUQwqlHXFtVNz4nhDCrrb+A1GUVDIZrfgob3oRfnMsXRmzly5eM4sE3Da2SJEnqHlqzwloN3BpjHAdMBT4fQhjXYM5K4PwY4wTge8AvG1y/IMY4McY45YQrlrqTUz4An3waMrLhv67kiz2f4EsXjeTBN9fz1fvfMbRKkiSpS2sxsMYYN8YY36z7vQJYCJQ2mPNKjHFH3Z8zgYFtXajUbQ04Bf76eRjzHnjqW9y87e/52vT+PPDmOr72wFFCa/UBWPwEzLwLaqo7tmZJkiSpDWQcy+QQQhkwCXjtKNM+CTx+xN8ReCqEEIFfxBgbrr5Kakl2b/jAb2DmnfD03/HZ3vPoNfX7fHPmOjLT0/jh+yYk5tVUw6oXYd4DsPBRqNqZGN88F67+KYSQvO8gSZIkHaNWB9YQQh7wAHBLjHF3M3MuIBFYzzli+JwY4/oQQgnwdAhhUYzxhSbu/TTwaYDBgwcfw1eQuokQ4KzPQ+lk+L+/5CNzP0mfMV/m86/XckPRaiZXPAcLHobKcsjKhzFXwvjrYd3r8MI/QY/ecNkPDK2SJEnqNEKMLT8DF0LIBB4Dnowx/riZOacADwFXxBiXNDPnO8CeGOM/H+3zpkyZEmfN8nwmqVmVW+GBT8GKGewO+fSKFcSMnoRRlyVC6shLILNnYm6M8MTX4bW74IJvwvlfTW7tkiRJ6tZCCLNbe75RiyusIYQA3A0sPEpYHQw8CNx0ZFgNIeQCaTHGirrfLwW+25rCJB1FbhF89AF4+d+oXj6Lm5eW0e/Ua/nGdWc0nhsCXPZDqNoNM34APXrB1M90fM2SJEnSMWrNluCzgZuAuSGEOXVj3wAGA8QY7wL+DigE7kzkW6rrEnM/4KG6sQzgtzHGJ9r0G0jdVVo6nPtl+p4LBX+Yx3/MXM3lk3dw2uA+TcxNg6t/Avt3wxNfg+xeMPHDHV+zJEmSdAxatSW4o7klWDo2e/ZXc8mPn6dXdiaPfuEcsjKaOQC8ej/89kZY+Tx84Ncw9qqOLVSSJEnd3rFsCW5NH1ZJKS6vRwbfu2Y8izdX8Mv/z95dx1dZ/n8cf93rDpZsY4zBxqiN7hQQVBSVEEXBbgX92vE1vj8buxAVG1BCEUQaBemRCxhj1GDdvVP3749rlLCCBRuf5+NxHudw7rruM4W9z3Vdn2t9UuU72tjDpJ8gsCcsuBOS1jVcI4UQQgghhKglCaxCNBMjOvpxTZeWfLT2IIcyiyrf0c4ZJv8C3uEwbzIkb2u4RgohhBBCCFELEliFaEZeuq4jDjZWPLsoBouliuH+jp5w6yJw9YOfxkN2Fb2yQgghhBBCNBIJrEI0I76uDjx/TQe2Hs7h5+jkqnd29YPbflOvFz8EFkv9N1AIIYQQQohakMAqRDMzsWcr+oa24PVl+8goKKt6Z8/WasmbY5sh+uuGaaAQQgghhBA1JIFViGZG0zTeuDGScpOFl5fEVX9A11ug7XBY9RLkHq3/BgohhBBCCFFDEliFaIbaeDszbXgYy2LSWBWfXvXOmgbXfqCel0yDS3CpKyGEEEIIcXmSwCpEM3Xv4FAi/F158bdYMgvLq97ZIxhGvAyH1sHunxqieUIIIYQQQlRLAqsQzZSttRUzJkSRV2rg7u+2U2IwVX1Az7ug9QBY8RwUpjVMI4UQQgghhKiCBFYhmrHOge58fHN3Yk7k8+jcXZjMVVQCtrKC6z4GUzksfVyGBgshhBBCiEYngVWIZm5kRz9eua4Tq/dl8PKSOPSqgqhXWxj2HCT8AXGLGq6RQgghhBBCnIcEViEuA7f1C+G+IaH8uOUYM/8+VPXOfR+CgG6w7Ckozq56X5MB9i+DjP1111ghhBBCCCEqSGAV4jLx9KgIro0K4K3l+1m8+0TlO1rbwNhPoSwflj99/n0yD8DKF+C9DjDvZvhmNGQl1k/DhRBCCCHEZUsCqxCXCSsrjRkTIunTpgVPzN/D5qQqek/9OsHgJyBmPiT8qd4zFMOun+DrUfBpL9jyOQT3hRtmgZUN/HgjFFazhI4QQgghhBC1oFU5n62R9OzZU4+Ojm7sZgjRLOWXGBk/cxNpBWUsfKA/4X6u59/RZIBZQ6E0B8JHQ8wCMBSCVzvoPgWibgYXX7XviR3w7RjwDoPbl4G9S4PdjxBCCCGEaFo0Tduh63rPmuwrPaxCXGbcnWz55o5eONhac/vsbaQXlJ13PwM2ZA1/D70oA33PPOhwLdyxHB6OhgHTTodVgMAeMOFbSIuF+VPBbGyYmxFCCCGEEM2a9LAKcZmKPZHPTV9sJtDTkV4hLcguMpBdXE52kYGsonIKytS6raFaCqP7RPHU9b2rP+mOb2HJNOh2K1z3CWha/d6EEEIIIYRocmrTw2pT340RQlyaOge689mtPZg+bxfLY9PwcrHDy9mejgFueLvY4+Vsh5eLPRsPtuSLbWmM7VtIe/9Khg+f1ON2yD8B698G91Yw9JkGuRchhBBCCNE8SQ+rEKJKucUGhr37FxH+rsy9py9adb2mug6/PQh75sB1H6v5rkIIIYQQQlSQOaxCiDrj6WzHk6Pas+VQDkv3plZ/gKbBdR9B2ytgyXRIXFX/jRRCCCGEEM2SBFYhRLUm9Qqmc6Abr/2xj+JyU/UHWNvCxO/BryP8MhVSdtV/I4UQQgghRLMjgVUIUS1rK41XrutMWkEZn6w7WLOD7F1h8gJwagGzR8PyZ6EwrX4bKoQQQgghmhUJrEKIGunR2pPxPYL4asMhDmUW1ewgV3+440/oPA62fgEfRsGfz0hwFUIIIYQQNSKBVQhRY0+PjsDBxpqXl8RT44JtHq3g+s/gkWjoPB62zYIPIuHPp6GgBnNihRBCCCHEZUsCqxCixnxc7XlsZDjrD2SyKj692v3zSgw8OX8PD8/ZicWjDVz/KTyyAyInwvavVI/rsich96iqLiyEEEIIIcQZZFkbIUStmMwWrvnoH4oNJlY/PgQHW+vz7rdmXzrPLIohs7AcgHcnRDGuR9DpHXKPwIZ3YfccsJjAyhacfcDFRz07+57x2gdsHcHGEWwdwKbiYesINvZg66zmyhlVwTsAACAASURBVFa35I4QQgghhGh0tVnWRgKrEKLWNidlc/OXW5g+IozpI8LP2pZfauTVJfEs3HmcCH9X3hkfxYuLYzmeW8raJ4bg5mB79slyj8L+P6A4A4oyoTjzjNcZYDbUrFE+ERB1M0TeBG4t6+hOa8FiBqvzh3chhBBCCHGaBFYhRL17ZO4uVsalsfrxIbRq4QTAuoQMnl0YQ2ZROQ8ObcsjV4RhZ2PF3uN5jP10I3cNaMMLYzrW/CK6DuUFUJwFxlIwlYOpFIxlYKp4GEuhLA/2LYXkLaBZqTVgu94C7a9RPbL1Sddh4wew7g3oditc8YLq7RVCCCGEEOclgVUIUe9S80sZ/u7fDArz5p0JUby2dB8/RycT5uvCuxOjiAzyOGv/ZxbuZcGO4yyfPoh2vq7106jsJNgzF3bPhYLjYO8OnW9U4TWoV90PGTYUw+KHIO5XCOgOqbvB0RNGvAJdJ4OVlAkQQgghhPg3CaxCiAbx+V9JvLV8P94uduQUG7hvSFumDQ8777zW7KJyhs34i8ggD364qzdafc43tVjgyHoVXPf9DsYScGwB/l0qHpHq2TsMrG2rP9/55ByGeZMhcx8MfwkGTIP0WPjjCdXTG9QLrp4BAV3r9t6EEEIIIZo4CaxCiAZRbjJz3ccbMVkszJgQRbdgzyr3/3bjYV5eEs/nk7tzVZcGmmdaXgj7lsCxzZAWA+nxYFaFoLC2B98OKrwG94UO14KDe/XnTFoL8+9Qr8fPhnbDT2+zWGDvPFj1XyjJhp53wRXPq55XIYQQQgghgVUI0XAMJgs2VhpWVtX3mJrMFsZ8/A+FZarCsKNdIxQpMpsgO1GF17S96jl1L5TmqMrDEddA5CQ1D9ba5uxjdR02fQyrXwKfDjDpR2gRev7rlObButdh+5eqd3fkKxB1S+MMEzYZ1DxfYwm4BZ17X0IIIYQQDUgCqxDikrXlUDaTZm3h0eFhPD4yvPoDGoKuw4kdsGcexC6A0ly1rE6XCRB1kxpCbCyF3x9R2zuOhbGfgb1L9edO3QvLnoDkrdCqL4x5H/xqUXiqJgrTIHGluoeSHNX+0jwVUktzwVB0el8bB/DrBC2j1H21jATfTvVfnOpSsX8ZFJyA3vc0dkuEEEKIy5YEViHEJe2RubtYEZfG6seGEOzl1NjNOZvJoMLf3nmQsBwsRvCtCJgZ+2D4f2HgY7Ur4GSxwJ45sPJFVfW438Mw5Gmwu8B7t1ggZSccWAGJKyB1j3rfsQW4+Krhx46e4OBx+rWjB1jbQWaC6llO3Qvl+eo4zVotC9QyCrpNhpCBtW9T/nFY+39wfPvZ6+XaOp5+tnUEJy9oM0QNwbaxv7D7v1DHtsC3Y9TP9MrXoP/DDXt9IYQQQgASWIUQl7jU/FKumPE3A8O8+XJKjf6uahwlORC3CPb8rHrlrv0QwkZe+PmKs9Xc1t0/gkewKsoUPqpmx5bmQtI6FaYTV0FJllrCJ6g3hF8JYaNUz2lNg7SuQ+6R0+E1bS8cj1ZDo9uNhBEvqbm91TEUw8YPYeNHoFtUWyyWiuWHKh4nlx8ylqrzW0xg6wRtBkPb4WoOsFfbmrX7QuWfgFlDwd5VzVvev1TNP+48rn6vK4QQQohzSGAVQlzyPvvrIG8vT+C7O3szJNynsZvTsI5shKWPQVaCKvQ0+i1wDzx7H0OxKhR1eD0c+ruiF1VXvaXtRqiA2m543a75aiyFbbNgw3tQlq+GRA97Dlq0OXdfiwVifoHVL0NhKnS6Uc3T9Qiu+hrlRXBkAxxcrR65R9T7nm3U/bS9AgJ7gqtfHd5XGXxzFWQlwj1rwKM1/HADnIiGWxdBm0F1dy0hhBBCVEsCqxDiklduMjP6gw1owPLpg7GzuczWLDUZYNNHsP4dsLKBK16All3h8N8qoB7froauWtmqJXLaDIbQoep1fRdNKs2DjR/AlpmqN7TnHTD4STXcGNTQ2uXPQMoutf7s6DfUEN8LkZ2kqi4fXA2HN4CxWL3v4q+GKLeMrHiOAvdWtV9LV9fhtwfVkOxJc1RRLVA91rNHQ0Eq3Pmn6p0WQgghRIOQwCqEaBLW7c/gjm+38+xVEdw3pJ6HhF6qcg6rokwHV1e8oalwFjpEhdTgfmDn3DhtK0iFv9+Cnd+reaj9HlQBM24RuAbAiJdVL2xdVT42lavCUal7Tj8y96uhxqDm5AZ0hd73ng6e1dkyE5Y/DUOfhaHPnL0tLxm+HglocPcqcA+6uPaX5MCRf+D4NmjRVg0fv9hzCiGEEM2QBFYhRJNx17fbWbM/A1cHGzyd7PB0ssW94tnTyQ4PJ1t8XR24JrIl7o62jd3c+qHrkLRGDckNGXjprdmadRDW/g/if1MFlQZMgwGPNkyQNpaqtXNTd6sAe3g95B6GDtfB1e+Aq3/lxx5eD99fD+Gj4aYfzx+s02LVcGG3QLhzuSpOVVPlhXB0s+oVP7xeLZGEropY6Wa1j2/HiiHcI1WVaBu7Wt2+EEII0RxJYBVCNBk5xQbmbjtGZmE5uSUGckuM5JUYyC0xkFdspLDcBICbgw33D23L7f1DcLKTdUQbRXYS2LnU7fzS2jIbVZGnv99Wvb4jX4HuU88No7lHVZElZx+4ezU4uFV+zkN/w4/joFUfuHVh5Uv8GErU8kRH/lEB9cQOFUyt7aFVb1X9uM0gNUw6J0kVx0pcqYZQW4xg56p6zsNGqqrMto5g66yqRZ98XV+B1lQO6bFwYqd6pMeCT3sIu1IVvnL2qp/riubPWAbmcnBwb+yWCCGaEAmsQohmw2i2sD+1kPdXH2Dt/gy8Xex55Ip23Nw7+PKb9ypOy06CJdNUAafg/qqCs0/Fur6GEph9JeQeg3vWgne76s+3dz4suhs63QDjZqsAbChRw3sPb1Ah9cQOFTw1awjsoYZstxmswqqtY+XnLi9UofjgKkhcDQXHK9/XykYFV6+20GMqdB5fs/V+z6TrqsDUiWjV5hM7Ve+vxai2O/uont/0OFVtGg0Cu6vw2m4kBHSru2HelbUvLUbNgfbvrOZuW1nX3/XqmsUC+clqiai8oxAyCHwjGrtV9ctihrxj6v+77INnPJLUZ6Fp6v+FzuNVIbnajFQQQlyWJLAKIZql6CM5vL0igW2HcwjydGT6iHBu6BaItVUtC/GI5kHXYdePsPIFMJbAoCdg4HT47QGIXQS3/KKW2ampjR/BqhdVBebyArXMz8mAGtBNDdcOGQTBfdTyOBfa5swEtUySsVS121Bc8bpYhWRjiQrJGXGqVzZyIvS8U4W7yphNcGwT7F8GCX+ocAGqRzygmwqkAd1V0HYPUgHDYoHUXSpEJ65U4RYdnLxVxeb2V6kQWxdDv03lKvQn/KkeZ4Z2ezdo3V8FnpBB4Ne58sCs61Ccpe4vP1n1uNvYqV5uGzvV637ytbW9WlKpKB0K06AoDQrTK54rXhtLVDExFz81vNzVXxX8cvUD15bq55xzWM2lzjpQ8ZyojjtT2yug70PquT7DfkM7Hg3rXldfDJkNp9+3d1Nfqni1Uw9TuZrbnntErffcbiR0GQfhV134etNCiGZNAqsQotnSdZ0NiVm8syKBmBP5tPN14T8jwxnd2R+tthVkRfNQlKGqFscuBGdfKM6A4S/BoMdrdx5dhxXPw9aZqrhTyEAIGXxxAfVC6bqqFB09W4Vvc7lac7fnHaoX2NZRLRGUtBb2/wGJK1TlY2t7aDtMhc1WfcE7rOa9l8XZ6nyJK9Wc6pJsNWc5bCR0ul4F+dr09pbkqHMlLIODa8FQqM7X9oqK9vVR6/8e2aACek6SOs7BQ332wf3Ufeclnw6oeclqjd8L5eBREUorAqqtExRnqqWZCtNVuD3ZE/1vbkFqGPXJh3d7dY7YhbDtSxWEvcOhz/0QNanxiqXVhbQYWPsaHPgTnLwg6mY1jP1kQHX2Prdit66r3vzYBeq/2aI0NVog4hr1pUu7EbWv8i2EaLYksAohmj1d11kem8aMlQkkZRbTK8STV8d2pkPLKuYqiubtwEr480kVhG744sJ/OTYZLq3iSCU5sGcuRH8D2YkqdLWMUnNjzeXqz+2vgvZXqzBY2yHE52Mxw9FNqtBW/O/qSwAbBxU6Ol4P7UerEG82qjCZc0g9spNOv849rCo8u/ipwlftr1ZzeCsbPp1/4vT84CPrT/cSO7ZQ6/ue+XBvBR6tVAA2VcyhNBkqniseZgNY21b0mFaE1MrmJ5+6b4sK/oWpKnCV5YNniAqiVX1pYTKoz2rzp6pAmIMH9LhdVbT+9xrLVX3mRRmq9z3/OBSkqNcWk/pvOmTg6aWl6kvmAfjrdYj7FezdYcAjKoDX9gsbixmOboSYBRC/GMry1D2MflP19jc1FrOq5N56QN38/yWEkMAqhLh8mC0686OTeXtFAvmlRqb0a81jI8Nxc2imFYVF1U7+m9Yce3J0XQW66NmQEQ+hw1TvVXC/+l2b12JW4Th+Mez7XYU5a3sVAvOPn66IDGoIcos20CIUfDqoIdktL3BObGG66qVsSgFB11Vhri2fwb4lgKYCtrWtWlPZ2qbi2VbNV7a2VUPCC1LU52oxnX0+GwfQrE4PQfYOV6EpZKB6VFUluzZyDqslrPb+rL4E6PsA9H+4biqWmwywdx6seVX1ZnedDMP/W3dtr2+F6bDoHlUNvM0QmLzg0vpCS4gmSgKrEOKyk1diYMbKBH7aegwvZ3ueuzqCG7oFyjBhIeqSxaIKUcUvVvNAW4SquYwtQtXD2ad5fllwIXKPwo5v1DBmi1HNM7aYKl4bK16bVCh1C1Q9sW4BauixW4Caa+zoqb4wSN0DR/9RX1gc3ayGV4Mantuqrxqi6+Cm5pY6eJzx2k19iWAsUT3HpXkVzxWPsjz1czywXAXoXnfDwMfU+epaWQGsfwe2fA429jD4Cej7oHp9qUpaC4vuVcPvIyfCzu/U8OjrP5f/zoW4SHUaWDVNawV8D/gBOjBL1/UP/7XPZOBpQAMKgQd0Xd9TsW008CFgDXyl6/qb1TVKAqsQ4kLFHM/nxcWx7E7Oo3dIC169vhMR/jJMWAjRTJhNau7v0Y0V1at3qvBZ2dzbymhWKtw6eqih5IP+o4JyfctOUoXSEpaBZxu48v/USIFLKQCaTWpo9Ib31NzdCd+qStB/vw3rXoPBT8IVLzR2K4Vo0uo6sLYEWuq6vlPTNFdgB3C9ruvxZ+zTH9in63qupmlXAS/rut5H0zRr4AAwEjgObAduPvPY85HAKoS4GBaLzvwdybz5534KykxM7RfCxF5B2NtYY2djhb2NFXY2VthZq9fSCyuEaNJ0Xc3lLStQFa7LCqA8v+K5UA2tdvRQPbaOniqo2rs1bkXjpLWw/FlVeTlkEAT1UhWGrW0rnv/1OriPmk9c3/KPw8K74dhm6D4FRr91utKxrsOSR2Hn92oprR631397hGim6nVIsKZpi4FPdF1fVcl2TyBW1/VATdP6ocLrqIptzwLouv5GVdeQwCqEqAt5JQbeWZHAnG3HqOqvOjtrK3qGePLV1J442dXjXEAhhBCnmU1q2PT6GWpN4H/P4T2TZgWdx8GA6VUv8XQxEpbDb/erIdtjPoDICedpsxHmToKkdXDzvNotnSWEOKXeAqumaSHAeqCzrusFlezzBBCh6/rdmqaNB0brun53xbbbgD66rj9c1XUksAoh6tKB9EIS04swmM2UGy0YzBYMJgvlJvVcUGbku01HGN7Bj5m39pB1XYUQojFYLCq0mg0VD6N6NhTB7jmq4JihSK0PPPAxtX5vXShMh00fweZPwL8LTPhOzc2uTHkhfHO1Gt58xx9qrWMhRK3US2DVNM0F+Bt4Tdf1RZXsMwz4DBio63p2bQKrpmn3AvcCBAcH9zh69GiN2iWEEHXhu01HeOn3OO4a2IYXx3Rs7OYIIYT4t9Jc2P6VKtxUkq0KTg18TAXY2gxvNhSr4lWH1qme0ow49X7ve2Hk/6pf/ghUsaqvRqqh2HevapjhyvXNWKqGPZ8cAi1EParzwKppmi2wFFih6/p7lewTCfwKXKXr+oGK92RIsBCiyXhlSRzfbDzCq2M7MaVfSGM3RwghxPkYSmDXj6pXND8ZfDtCpxvUvNyTSyHZuajXds7qdWmeCqiH/lJLD5kNanmm4L7QdphaY9i/S+3akZkAX48EZ1+4ayU4tTi9TdfVWsTJ21Vl7ePb1ZJBXcaroc1n7nspKMmBb65SSw8Nfwm63da4c5xFs1fXRZc04DsgR9f16ZXsEwysBabour7pjPdtUEWXhgMnUEWXbtF1Pa6qa0pgFUI0BrNF574folm7P4Ovp/ZiWIRvnZ27xGDC0dZaCjwJIURdMRshdiFs/FCtTVwTfl2g7VC1jnFwv4vvTTyyEX64HgJ7qMrBx7efDqnFmWofOxe1vThL9eZa2ape4aibIHx04y/tYyyF78dCym41P/jEDtXeq2dAYPfGbZtotuo6sA4ENgAxgKXi7eeAYABd12dqmvYVMA44OY7XdLIBmqZdDXyAWtZmtq7rr1XXKAmsQojGUlxuYuIXmzmSVcz8+/vTMeDil8TZeDCLu77bzoQerXh1bCcJrUIIUZd0vWKua7Ga42ooVmunnnxtKFYVh0MGgkvdfRF5SswCWHjX6T97tYOg3tCql3r27QBW1mpbWgzsmaeOKUoDB3fVOxw5SfX2NvS/D2YT/HIbJPwJE7+DDtfB3p9h5YsqcPe4HYb/t/oe4eJsVfk5I159ziGDwMauQW5BNE31WiW4IUhgFUI0pvSCMq7/dCO6Dr89NAB/9xrMZ6rEjqO53Pb1Vqw1jcJyEy9d25E7BrSpw9YKIYRodEc3Q1m+Wp7H2av6/S1mNTx578+wbwkYS8AjWA0X7jwe/DrVf3jVdVg6HXZ8C1e9A33uPb2tLB/+ehO2fgEObmqYcPcpp4O3xQwpuyBxFRxcpdYD5oxMYe8O4aOgw7XQbrgami3UFynZiep1y6718zM2lqqfXbvh0GZw3Z+/jkhgFUKIixSfUsCEmZsI8Xbml/v64Wxf++Vu4lLymTRrC17Odvx8Xz9e+C2WNfvS+fr2XgxrXw/f8gshhGh6yotg/1LY+4sKsboZvNufnu9aVcXii/HXm/DXGzDoP6oX9XzS42HZE3B0o6qG3O02OLpJ9aaW5gAaBPWEdiMhbAT4RMDh9SqEJyxThbJsHKDtcOgwRg2BvtTm755kNqqiXtu/hpABMGAatAit/Xl0HYoyIOsAZCVAVqKa75yVCAXHT+8X0A36PQwdx6oRAHXh8HpYMg1yDsGQp2HYc3Vz3noggVUIIerAuv0Z3PXddq6I8OWL23rWarmbgxlF3PTFZuxtrPjl/n4EeTpRXG5i/MzNJOeUsOjB/oT7udZj64UQQjQ5xVkQ/xvELIRjFWVhWnZV4bXDteDe6nQv58WI/kb1rnadDGM/rbqnT9fVEOaVL6hhzM4+qkhVuxHQ9orKA6jZpO5h31IVyAtOqPV0PUPUsGmvdiqMn3ztGnDhhZ7K8uF4tJpDbCiGqJvBr4YV/3UdDqyAlc9D9kFoGQUZ+8FiVL3dAx+r2bmyDsLeeeqLh7wzVjuxdQbvMPBpr56920Nxhqp2nX0Q3AKhz33QfSo4elzY/ZfmqmHcu34AzzZw7QcQOvTCztVAJLAKIUQd+WHzEV5cHMetfYN5cUxH7G2q/0UhOaeECTM3Y7LozL+/H228Tw+FSskrZeynG7G3sWLxQwPwcmnkYhtCCCEuTfknIG6RCoupuyve1MDRE5y8wNlbPZ98uPiqebD+UVUHv/3L4OfJqtfz5rk1790rL4L84+AdXvtgqesVQ4hXQuZ+FdSyk9RQ6JNsHFWA9WgNHq3APajiEayenX3UdXVd9SAmb614bK8ouqWrz8fKRoXNVn2h112qB7OywlbpcbDiOdWz7RUGo15TBbGK0tW6vNtng7EYIsbAoMdVMaozFWern9GeuapYFRqEDlE9yT7t1WflFnj+LwQsFvV5bP4EjmxQwbb7bdDnfmhRw6lDug7xi2HZk2qpp/4Pw5BnmsTSRBJYhRCiDv3f0ni++ucwLd0duGdQKJN6t8LJ7vxDhNMLypgwczP5pUZ+vq8vEf7nFm3ak5zHxC820yXQnZ/u6VOjECyEEOIylp2khuEWZ6pe2JLs04+Tf9bNal8XPwgbqUJT6FCwP2M0z7Gt8P11aimg25c27txSXYfC1IrwWhFgsxIh75harshQdPb+1vbgHqh6U0uy1Xv27mpIcqs+0Kq3CpRmI+z+CaJnq6WFnLyg263Q447TQbA4C9a9pubv2rvB0GdVuP13eC/JUfN4t86EsjxVXXrgdLVM0t6fVeC0mMCvM0TepHrC3QJq/1mk7oHNn0HsAjU/OHwUBPYEn3DVI9si9NwiVvkn1HDthGWqV/i6j9VzEyGBVQgh6pCu62xIzOKTdQfZdjiHFs523DWwDbf2bY274+l/3HKKDdz0xWZS8kr56Z6+dG1V+dCepXtTeHjOLm7sFsi7E6PqvXKwruscyymhtZcUvhBCiGbHYlHDTJPWQeIKOLgWyvPVEjohAyBsFPhGwPw7VIC7a6Xqob1U6boKpvnJqlc3//jpIGvrpMJpqz4qzFXW22uxqLV3o2erKsi6WfUqB3SDbbNU726vu9Vcz+rm1ZYXqvNs+kR9zgAu/hA5QVV49u9cN/ddkArbv4SY+ep+T7KyUaHVO1zNE7ZxUMs5WUxqnmrfB8G69rU2GpMEViGEqCfbj+Tw6bqD/JWQiau9DVP6t+bOAW2wtbFi8pdbOZBeyHd39qZvaPVVIj9cncj7qw/w5Kj2PDSsXZX75pcYQeOsgFwbs9Yn8fqy/Xw4qStjuwZe0DmEEEI0EWYjHNuiwuuBlar4D4CzrwqrNR1y2lwUpMDO71WPamGqCvBX/p/qwawNY6kqKOXsDW2G1M184sqcrCiceUANo846oIo35RxS4Tt0KIz5oMn+LCWwCiFEPYs9kc/nfyWxLDYVexsrAtwdSc4tYdaUnjWuAKzrOtPm7eb3PSnMvLU7ozu3BMBktrA/rZBdyXnsOpbL7uQ8DmUW4+tqz+8PD6z1Mjs7j+UyceZmzLpOkKcjax4fip3NBRa2EEII0fTkHFZDikMGqrmVlyuzCQpT1BJCTZXJoObYugc1/Lq9dUgCqxBCNJCkzCI+/yuJFbFpvDkukmsiW9bq+DKjmZu/3MK+1AJu6d2a2JR8Yo7nU2pUc5G8Xezo2sqTTgFufLXhEG19Xfjlvn442NbsW938EiNXf7QBTYOnRkfw6NxdvDq2E1P6hdT2VoUQQggh6oQEViGEaGC6rl/wPNTMwnJu+Gwj6QVldApwp1uwB92CPenWyoMgT8dT510dn849P0RzbWQAH07qWu31dF3ngR93snpfOgse6E9UkDuTZm0hKbOIv58cdkFrywohhBBCXKzaBFb5bUUIIerAxRRN8nG1Z/XjQwCq7Dkd0dGPJ0e15+3lCUS0dOXBoVXPe/1hy1GWx6Xx/NUdThWAevqqCG78bBOz/znMI8PDLrjNF8pktmBjLcORhRBCCFEz8luDEEJcAhxsrWs0zPeBIW25LiqAd1YksDo+vdL94lLy+b+l+xjW3oe7Bp4uyNA92JMrO/oxa/0hcooNddL2mvri7yT6vrGG5JyS6ncWQgghhEACqxBCNCmapvH2+Ei6BLozbd4uDqQXnrNPUbmJh+fswtPZlncndsXK6uze3ydHtafYYOKzdQdrfN38EiPXf7qRlxbHXlC7D2cV8+6qA2QVGXju1xguxekoQgghhLj0SGAVQogmxsHWmlm39cTJ3oa7v4sm94yeUl3XeeHXGI5mF/PRpG60cLY75/gwP1fGdQ/i+y1HOZFXWu31yoxm7v5+O7uT8/hu81FWVdGzez66rvPfxbHYW1vx6BXt2JCYxcKdJ2p1DiGEEEJcniSwCiFEE+Tv7sCs23qQVlDGgz/txGi2ADB/x3F+253C9BHh9KliLdjpI8NBhw9XH6jyOmaLzqNzdxF9NJcPbupKh5ZuPPdrzFkhuTq/70lhQ2IWT45uz/QR4fQK8eR/S+PJKCyr8TmEEEIIcXmSwCqEEE1Ut2BP3rihC5sPZfO/pfEkphfy0uI4+rf14qFhVRdkCvRw5LZ+rVmw4zgHM84dVgyqZ/TFxbGsjE/npTEdub5bIO9OiCK32MDLS+Jq1Mb8UiP/W7qPyCB3JvdpjZWVxpvjIik1mnn595qdQwghhBCXLwmsQgjRhI3rEcS9g0P5fvNRJs3agpOdNR/c1BVrq+qrFj80rB1Odja8syLhvNs/XnuQOVuP8cDQttw+QBVu6hjgxqPDw1i8O4XlsanVXmPGigRyist57foup9rU1seF6SPCWBaTVqNzCCGEEOLyJYFVCCGauKdHRzC0vQ/ZxQbeu6krvm4ONTquhbMd9w4OZUVcOruO5Z61bd62Y7y36gA3dg/kqVHtz9r2wNC2dA504/lfY8kuKq/0/LuT8/hx61Gm9AuhS5D7WdvuGRRKpwA3XvgtjvwSY43aW24y89PWo6TUYN6tEEIIIZoHCaxCCNHEWVtpfHFbD1Y9Npgh4T61OvaugW3wdrHjreX7T1XuXRWfznO/xjAk3Ie3xkWes8asrbUV707oSkGZkf8uPv+wXpPZwvO/xuDras9/rgw/Z7uttRVvjYskt8TA//0RX207j2QVM/7zzTz/ayyTv9ra4EvyCCGEEKJxSGAVQohmwN7GmjA/11of52xvwyNXhLHlUA7rE7PYcTSHh+fspEugO59N7o6t9fn/mWjv78r0EeH8EZPK0r0p52z/fvNR4lIKeOnaTrg62J73HJ0D3blvcCjzdxxnQ2JmpW38fU8KYz7+h6PZxTw5qj0peaXc+e12Sg3mWt+vEEIIIZoWCaxCCHGZu7l3MEGejry6JI67voumpbsDIb3jQAAAIABJREFUs2/vhbO9TZXH3Tc4lKhWHrz4WyyZhaeHBqfll/HuygSGtvfhqs7+VZ7j0eFhhPo48+yiGIrLTWdtKzWYeWbhXh6du4v2/q4smzaIh4a148NJ3dhzPI+H5+zEVFEduaZyiw0k55TU6hghhBBCNB4JrEIIcZmzs7HiP1eGk5RZjI2VFd/f2QcvF/tqj7OxtuLdCZEUG8y88FvMqSHFryyJw2TRefW6zucMJ/43B1tr3h4XyYm8UmasPF38KTG9kLGf/sO87ck8OLQt8+7tS5CnEwCjO/vz6nWdWLM/gxd+iz113eosj01l6Iy/GPT2Ou78djvRR3JqdJwQQgghGk/VX58LIYS4LIyNCiQ1v4zhEX4EeznV+Lh2vq48cWU4ry/bz+97UnB1sOHP2DSeHNW+xufpGdKCKX1b8+2mI4yJbElSRjH//T0WF3sbvr+zN4PPMy/3tn4hpBWU8em6JPzdHZg+4tx5sieVGc38b2k8P209RmSQO0PDffhhy1HGz9xMrxBPHhjalmHtfasN13Ulo7AMV3tbHO2sG+R6QgghRFOm1fSb6YbUs2dPPTo6urGbIYQQogbMFp0JMzeRlFmMi70NjnbWLHt0EHY2NR/EU1RuYtT768ktMVBiMDOgnRfv39QVX9fKKx7rus4T8/eycOdx3rixCzf3Dj5nn4S0Qh6Zu5MD6UXcNziU/1zZHjsbK0oMJn7ensyX6w+Rkl9GhL8rDwxtyzVdWmJTybzduhCXks9NX2yhU4Ab8+7t22AhWQghhLiUaJq2Q9f1njXZV4YECyGEuCjWVhozJkRRZjRzIq+U/7u+c63CKoCLvQ1vjuuCtabxxJXhfH9nnyrDKoCmabw5rgtDwn14/tcY1uxLP7VN13V+2HKU6z75h5xiI9/f2Ztnr+5wql1OdjbcMaANfz81jHcnRGG26Eybt5uhM/5iztZjWCx1/2Vuck4Jt3+zHYuus/VwDgt3nqjzawghhBDNjfSwCiGEqBOr4tNJyStlav+QCz6Hruu17nUsLjcxadYWEjMKmXNPX9p4OfP0wr2sjE9nSLgP706MwruaObkWi86a/Rl89tdBdh3LY3C4D+9OiMLHtfq5vDWRU2xg/OebyCoqZ8ED/Xlm4V6OZJew9j9D8HCyq5NrCCGEEE1FbXpYJbAKIYRo8jILyxk/cxMFpUYcbK3JKirn6dER3DmgDVZWNQ/Auq7z09Zj/G9pPK4Otrx/UxSDwmq3tu2/lRrM3PLVFuJSCvjp7j70CmnBvtQCxnz8DxN7BvHGjZEXdX4hhBCiqZEhwUIIIS4rPq72fHdHb6ytNBxsrVn0wADuHhRaq7AKapjxrX1bs/jhAXg42TJl9jbeWr4fYy2XzznJZLbwyNyd7E7O46NJXekV0gKADi3duKN/CHO3JbPjaG6tzrn9SA7/WxpPQZnxgtokhBBCNCXSwyqEEKLZKCgz4mBjXes5tOdTajDz6tI45m5LpluwBx9N6karFjWvoKzrOs/9GsPcbcm8OrYTU/qFnLW9qNzEyPf+xsPJjiUPD6hRsafdyXnc8uUWSgxm2vm68PXUnrT2cq7trQkhhBCNSnpYhRBCXJbcHGzrJKwCONpZ88aNkXx8czcOphdx9Ucb+GNvao2P/2jNQeZuU+vI/jusgio09dK1HdmXWsC3m45Ue76EtEJu/2YbXi52fHxzN7KKyhn76UY2JWXV4q5EXboUv/QXQojmRgKrEEIIUYVrowJYNm0QoT4uPDRnJ08v2Mv6A5mk5pdWGlh+3n6M91cf4MbugTw5qn2l5x7VyZ9h7X14f9UBUvNLK93vWHYJt329FTtrK366qy/XRgWw+KEB+LjYM+Xrbfy45ehF32dt5JcaySsxNOg1LzXpBWUMm/EXi3Yeb+ymCCFEsyZDgoUQQogaMJotzFiZwKz1hzj5T6ervQ3t/FwI93UlzM+Fdr4u5JcaefyXPQxo583XU3tiW81Q32PZJYx8/2+uiPDl81t7nLM9vaCMCTM3U1Bm5Jf7+hHu53pqW2GZkUfn7mJdQia39W3Nf6/tWO31LkZOsYFZ6w/xXUWP8FOj2zO1X0it5wpXR9d1CkpNuDvZ1ul569IrS+L4ZuMRnO2sWT59cK2Gi9eV9IIyPlidyO39Q2jv71r9AUIIcYmQKsFCCCFEPckuKudAehGJGYUkphdxIL2QgxlFZBef7nHsEujOvHv74mxvU6NzfrI2kRkrD/DN7b0YFuF76v3cYgM3zdrMidxSfrqnL11beZxzrNmi89by/cxaf4j+bb34bHL38y6VY7boHMwoYs/xPBLSCmnv58oVHXyrXfIHIK/EwJcbDvHtxiOUGM1cGxlAQZmRvxIy6R3SgrfHRxLifWFzac0WncNZxcSl5BOXUnDqOa/EyHVRAbw9PhIHW+sLOnd9ySgoY9Db6+jf1ovtR3KJDHLnx7v61Hlwr86rS+KZvfEwdtZWPDYynHsHh2LdwG0QQogLIYFVCCGEaGDZReUkZhSRklfKFRG+tVpftdxk5qoPN2A0W1g5fQiOdtYUlZuY/NVWNcf1jl70b+td5TkW7DjOc4tiaOnhwNdTe2Jnbc2e43nsPZ7HnuR8YlPyKTGYAbC11jCadTQNegR7MqKjHyM7+tHWx+Wsc+aXGPn6n0PM3niEonITYyJbMm14GGF+rui6zoIdx3l1aTxGs4WnR0fUqLfVZLaw+VA2a/ZlEHMin32pBafaZWdtRXt/VzoFuOFga823m47Qo7Uns27rgVcNgvWZ/knMotRoZmRHv1odVxP/tzSebzYdYc3jQ9hyKJtnFsWct7BWfSouN9H3jTX0DmmBrbUVy+PS6B7swYwJUYT+6+cohBCXGgmsQgghRBOzKSmLW77cysPD2vHwFe2489vtbD2cw8xbe9Q4dO04msN9P+wgq+h0b6+djRUdW7rRtZUHkUHuRAZ50Mbbmf1pBayKT2dVfDpxKQUAhPo4M7KDH8MifNlyKJuv/zlMYZmJqzr7M21EGBH+budcMy2/jGcX7WVdFb2tZovO1sPZLN2byvLYNHKKDTjZWdM5wJ2OAW50CnCjc6A77XxdzhrSvCwmlcd+3o2fmwOzb+9FO9/qg1husYFXl8bz664TANzaN5gXx3TE3qZuemmzisoZ+NZaru7SkvcmdkXXdW7/ZjvbDuewfPqgBqva/NPWozz/aywLH+hH92BPft+Twou/xWIwW3hmdART6mGothBC1BUJrEIIIUQT9NjPu1m6N4VeIS3YlJTN+zdFcUO3oFqd40ReKT9tOUqgpyNRQR6E+7lWWzk5Ja+U1ftUeN1yKBujWf1ucGVHP6aPCKdjwLlB9Uy6rrNw5wleWRKH0WzhqVERTOnXmp3H8li6N4VlMWlkFZXjZGfN8A5+jIlsyZBwnxoN9d11LJd7vo/GYLIw89Ye9G9XeU/z8thUXvgtjrwSAw8Oa0e50cwX6w8R1cqDzyZ3J9DDsdrrVeeNZfv4csMhVj0+5FSPdGp+KVe+v54O/m7Mu7dvvQdFXdcZ/cEGbKw1lj4yEE1T10svKOOZherLg36hXrw9PrJR5tYKIUR1JLAKIYQQTVBmYTnD3/2LgjITr1zXian9Qxq8DQVlRjYdzKZVC0c6BbjX6tgze1ud7KwpMZixt7FieAdfrukSwBURvjja1b6nMzmnhDu/3c7hrGJev7ELE3u2Omt7VlE5Ly2O44+YVDoFuPHO+KhTIXt5bCpPzN+LrbXGh5O6MTjcp9bXPym7qJyBb63jyk5+fDip21nbFuw4zhPz9/DimI7cNbBNtefKLzXyyu9xONhZ8/oNXWrVjq2Hsrlp1hbeHhfJxF5nfxa6rvNLdDL/W7oPXdd5cUxHburV6lSoFUKIS4EEViGEEKKJ2pSURUZBOdd3C2zsplwQXdf5ddcJ/jmYxZBwH0Z08Ktx8amqFJQZeeinnWxIzOLBoW154sr2aBr8vieFl3+Po7jczLQRYdw7OPScSsmHMot44MedHMgo5PER4Tw0rN0F9YK+vXw/n/+dxKrHBtPO9+yqvLquc8/30WxIzGLZtEHnzAc+085juTwyZxcn8tRSRj/e1YeBYVXPUT7TQz/tZGNSFlueHV5pL/Xx3BKenL+XzYeyuXNAG14c00FCqxDikiGBVQghhBDNjtFs4b+L45i77RhXd/HHYLKwel8G3YI9eGd85Dkh8kwlBhPPLoph8e4Urojw5f2JXWu1bE5usYGBb61lWIQvn9zS/bz7ZBSUMfL99YT6OLPg/v7nVOy1WHRmbTjEjBUJ+Ls78N7Ervxn/m6c7Wz449FBNarwm5ZfxoC31nL3wDY8e3WHKve1WHReXRrPt5uO8NTo9jw4tF2N71cIIepTbQJr/S3WJoQQQghRh2ytrXj9hs48f3UH/oxN45+DWbxwTQcW3N+/yrAK4GRnwwc3deXVsZ3YkJjJmE82EHsiv8bXnr3xMMUGM48OD6t0H183B14d24ldx/L4csOhs7ZlFZVz+7fbefPP/Yzq5M8fjw6id5sWPD06gv1phSzYkVyjdszZehSLrnNr39bV7mtlpfHfMR0Z2zWAt5cnMG/bsRpd4yRd1/lu0xHeXZlA7hnLNtWl3GIDi3efIDmnpF7OL4Ro+i5+jI4QQgghRAPRNI17BofSM8QTbxf7WhUV0jSNKf1C6BzozkM/7WTc55t4e3wkY7tWPfw6v8TItxuPcHUXf8L9qg7G10UF8GdMGu+tPMDwCF/C/FzZdDCLaT/vJr/UyGs3dOaW3sGnhude06Uls4MPM2PlAcZEBlQ5fNpgsjBnWzJXtPet8X1bWWm8Mz6KvBIjz/0ag4eTHaM7+1d7XHG5iacW7OWPmFQAvt14hHsGh3LnwDa4XOQQb4tFZ2NSFvO2J7MqLh2D2YKVBld3acm9g0OJDDp3vWEhxOVLhgQLIYQQ4rKTVVTOgz/uZNuRnFNzYiub1/r+qgN8uCaRP6cNokPLqismnzz3le+vJ8jTkSHhPnyy7iCh3s58ckv38x6/42gu4z7fxKPDw3h8ZHil5128+wTT5u3muzt7M6SWxaNKDCZu+XIr8akFfHdHb/q19ap038NZxdz3QzQHM4p45qoIhoT78t6qBFbEpdPC2Y4Hh7bl1r6ta1Tl+UwpeaXMjz7OL9HJnMgrxcPJlhu6BTK6kz9r92cwZ+sxCstN9A1twX2D2zK0vY/MuxWimZI5rEIIIYQQ1TCYLLz0eyxztyUzooMvH0zqdk7vYUGZkQFvrqV/Wy++uK1Gv1sBag3ZB3/aCcD4HkG8OrYTTnaV90w+PGcnq/el89cTw/B3dzjvPuM+30ROsYE1jw+5oKJRucUGJnyxmbT8Mubd25fOgedWgV67P51p83ZjY6Xx8c3dzyoGtTs5jxkrEvjnYBYt3R14dHgY43sEnVPk6qSichNp+WXsTytgfvRx1idmouswsJ03N/VqxciOfmeF3sIyI/O2JTN742FS88sI93PhnkGhjO0aWO3STEKIpkUCqxBCCCFEDei6zvebj/Lq0nja+jjz1ZReBHudHm770ZpE3lt1gKWPDDxvwKvKVxsO4e/uwJjIgGr3Tc4pYfi7f3Nd1wBmTIg6Z3vsiXzGfPwP/x3TkTtrsGxOZVLzSxn32SYMZgsL7u9PiLczoIbpfrz2IB+sOUDHlm7MvLVHpcOONx3M4p2VCew6lkcbb2cm9mxFYZmRtIIy0gvKSMsvI72gnKJy06ljWro7MKFHEBN6tqp2OLPBZGHp3hRmrT/E/rRC/NzseenaTlzdpeUF37cQ4tIigVUIIYQQohY2HsziwZ92YqXBZ5N70K+tF4VlRga+tY5eIZ58NbVXvbfhjWX7mLXhEEsePjccP7VgD0v2pLLlueG4O9a8uvH5HMwoYsLMTbg42LDw/v442Fnz+M97WL0vnRu7B/L6DV2qHe6r6zpr9mUwY2UC+9MKsbHS8HW1x8/dAX83B/zcHPCveB3k6Ui3YM8aVUH+9zXWJ2bx3soEYk7k897Erk12uSchxNkksAohhBBC1NKRrGLu/j6aI1nFvHRdJwpKjbyzIoHfHx7QIIWA8kuNDH1nHRH+bsy5p8+p+Zu5xQb6vrGGcT2CeP2GLnVyrd3Jedzy5RaCWzhhMFk4llPCC9d0YGr/kFrNG7VYdPJKjXg42l7QMOWaKDGYuOvbaLYezubdiVHc0C2oXq4jhGg4sqyNEEIIIUQthXg7s+jB/gwK8+bF32L5YPUBhrX3abCqte6OtkwfEc7mQ9ms2Zdx6v35O5IpN1mY0q/6pWxqqmsrD764rQdJmUUUlJmYc09fbh/QptZFjqysNFo429VbWAW1JNHs23vRp40Xj/+yh0U7j9fbtYQQl55qA6umaa00TVunaVq8pmlxmqZNO88+EZqmbdY0rVzTtCf+te2Ipmkxmqbt1jRNuk2FEEIIcclyc7Dlq6m9uG9IKNZWGtNHVF61tz7c0ieYUB9nXv9zH0azBbNF54ctR+nTpgUR/tVXKK6NQWE+/PHoIFZMV2vCXsoc7ayZfXsv+oV68Z/5e1i4Q0KrEJeLmvSwmoD/6LreEegLPKRpWsd/7ZMDPArMqOQcw3Rd71rTbl8hhBBCiMZibaXx7FUdiHl5FFGtGnZNUFtrK567qgOHMouZu+0YfyVkkJxTytT+IfVyvXA/V7xc7Ovl3HXN0c6ar6f2YkBbb55YsIcFElqFuCxUG1h1XU/VdX1nxetCYB8Q+K99MnRd3w4Y66WVQgghhBANrLLlWurb8A6+9Av14oPVicz8Owl/NwdGdvRrlLZcahztrPlqak8GtPXmyQV7mB+d3NhNEkLUs1r9TaxpWgjQDdhai8N0YKWmaTs0Tbu3NtcTQgghhLjcaJrG89d0ILfEwPYjudzSJ7jRwvOlyMFWhdaB7bx5auFeftkuoVWI5qzGf/tpmuYCLASm67peUItrDNR1vTtwFWo48eBKzn+vpmnRmqZFZ2Zm1uL0QgghhBDNS+dAd8Z3D8LOxopJvVs1dnMuOQ621nw5RYXWpxft5eM1iWQUlF3QuUoMJuJTCrBYLr2VM4QQNVzWRtM0W2ApsELX9feq2O9loEjX9fPOZa1u+0myrI0QQgghLndlRjNp+WWEeDs3dlMuWWVGMw/P2cnqfRloGvQKacE1XVpyVWd/fN0cKj3uRF4pa/els2Z/BpuSsjGYLIR6O3P7gBDGdQ/C2d6mAe9CiMtPna7Dqqn65t8BObquT69m35c5I5BqmuYMWOm6XljxehXwqq7ry6s6jwRWIYQQQghRU4nphfwRk8qymFQOpBep8Nq6BVd38eeqLi3xdrFnd3Iea/ens2ZfBvvTCgFo7eXE8Ag/2vo688v2ZPYcz8fVwYabewczpV9rgjydGvnOhGie6jqwDgQ2ADGApeLt54BgAF3XZ2qa5g9EA24V+xQBHQFv4NeKY2yAObquv1ZdoySwCiGEEEKIC3Ewo5A/9qaxLCaVhPRCNA1c7W0oKDNhbaXRs7UnIzr4cUUHX0K9nU+tPavrOjuP5fHNxsP8GZuGruuM6uTPnQPb0LO1Z63XqBVCVK5OA2tjkMAqhBBCCCEu1sGMIpbFpHI8t4SBYT4MCfPB3cm22uNS8kr5fvNR5m47Rn6pkS6B7jxzVQQD2nk3QKuFaP4ksAohhBBCCHGRSgwmft11glnrD3E0u4Rx3YN4/poOtHC2a+ymCdGk1SawSo10IYQQQgghzsPJzobJfVqzYvpgHh7WjsW7TzD83b9YtPM4l2KnjxDNkQRWIYQQQgghquBga80To9rzx6ODaOPtzOO/7OG2r7dxNLu4sZsmRLMngVUIIYQQQogaaO/vyoL7+/O/6zuzJzmPK99fz2d/HcRotlR/sBDigsgcViGEEEIIIWopvaCMl3+P48/YNCL8XZnctzVtvJwJ8XYiwN0RK6uaVRXWdZ3CchNFZSac7WxwtrfGxlr6lETzJkWXhBBCCCGEaACr4tN5+fc4TuSVnnrP3saK1l5OhHg508bbmRBvZ+xtrMgoLCejoJz0wjIyCsrIKCwnvaCMMuPZPbT2NlY426vw6mxng4u9DX5uDjxzVQStWsjasKLpk8AqhBBCCCFEA7FYdNILyzicVcyRrBKOZBdXvC7maE4JBtPpQOpib4Ovqz2+bvb4ujrgV/Hs4mBDicFMcblJPQwmisvNFFX8OeZEPnbWVsya0oMerVs04t0KcfFqE1ht6rsxQgghhBBCNGdWVhot3R1p6e5I/7ZnbzNbdFLySjGaLfi5OeBsf2G/fh/KLOLOb7dz86ytvD0+kuu7BdZBy2suJa+UgxlFDA73adDrCiED5IUQQgghhKgn1lYarVo4EerjcsFhFSDUx4VfHxxA99YeTP95N++uTMBiaZiRkglphYz9dCNTZm/jt10nGuSaQpwkgVUIIYQQQogmwNPZju/v7MNNPVvx8dqDPDJ3F6UGc5XH7Est4KXFsfT8v1Xc90M0+aXGWl0z5ng+k2ZtRgO6B3vw1IK9RB/JuYi7EKJ2ZA6rEEIIIYQQTYiu63y14TCv/7mPyEB3vpzSE183h1Pbi8pNLNmTwrztyexJzsPO2ooB7bzYkJhFoKcjM2/tQYeWbtVeJ/pIDnd8sx03R1vm3NMHd0dbrv90I4VlJn57aIAUgBIXTIouCSGEEEII0cytik9n2rxduDva8tXUnhjNOvO2HeP3PSmUGMyE+bowqXcwN3YLxNPZjh1Hc3jgx50UlBl588aq58FuPJjF3d9F09LdgR/v7kOAhyMASZlF3PDpRvzdHVj4QH9cHWwb6nZFMyKBVQghhBBCiMtAXEo+d38XTXpBGRYdHG2tGRPZkkm9g+ke7IGmnb0ebEZhGQ/P2cW2wzlM7dea56/piJ3N2bMEV8en8+CcnbTxcubHu/vg42p/1vaNB7OYOnsbA8O8+WpKz3pdN7bMaCYhrZD41AL2pRYQ3MKJqf1DsJW1aps0CaxCCCGEEEJcJjIKyvjsryTC/Fy4Liqg2l5Po9nCW3/u56t/DtOjtSefTe6OX8WQ4qV7U5g+bzcdA9z47o7eeDrbnfccc7cd49lFMdzeP4SXr+t00feg6zqZReUqnKYUEJ9aQHxKAUmZRZysLeVoa02p0UyXQHf+v717D7K6vO84/v6yXFeEXWBBYdcFZSGgCeCKlSjUiaZe4oTWeKmN0VhTO21nYp1GY5KaTKaX2Blj2g6tEwdNjXVsEjBAncbGplQ0CilUQcQACq7souyFRRcO7IV9+sc5plu5rpFzfsy+XzPM2d/vnMP5zplnvruf83ue53z7ullMm3Bqv15j0853Wbm5mZvm1XpluMQMrJIkSZKO6skNO7lryQbKhw5m0e/N4c3dOe5euoH62koe/vzcY4a6v3xyE4uf285fLDybz82bfMzX6+1NNHd08kbbPhra9vFGWy5/25q/3ddnA6mJo4czc+IoZp4+qnA7murKETz1ytv8+bKN7D3Qwx2fnMYfzJ9yzCu8O3bn+PZPN7N8/U5SgrrxI1l883nUjj3luN4nffgMrJIkSZKOaeuuDv7w0XU07M5xsDcxv24c3/1cPeVDj/0VPAd7E7d9fy3/taWF731+7iHf0ZpSYnvrPp7d2sqzW1tYvW03ezt7fnX/kLKgprKc2rHl1I49hdqx5UyfcCozTh91xCu7AK17O7ln2UZ+svFtZtdUcN+1s5g6fuQhj2vb28mila/xz6sbKBsU3HLhFObUVHDX0g2kBP/42XO5cOq4frxb+rAYWCVJkiQdl44D3Xx9+SsE8NdXf5ThQ8qO+7l7O3u45oHnaWrfzxN//HHGjxrOC6+38syWfEhtbN8PQO3Yci6aOo4Zp49iciGcTqwYQdmgOMYrHF5KiSc3vMU9yzeS6zrInb81nd+/aAplg4J9nT089Nx2Hly1jVxXD9fPreH2S6Zx2uj8tOeGtn184ZG1bGvdx9evmslN82oPWeurE8vAKkmSJKkomvbsZ+Gin9PZfZBc90EO9iZGDhvMvLPGsmBaFQvqxp2w6bfNHQf42o838vSmXdTXVnL52afx3VXbaN3byWVnT+DOy6Yzdfyha107DnRzxw9e4j9ebeaG82v45qfPOWTzqWJLKfH4L3aw7KUm7rh0GvPOGlvSek4kA6skSZKkolm/Yw/3/XQzs2sqmF9XxZwzKoq2k29KieUv7eQbK17hnf3dnD95DF++4iPU11Ye9Xm9vYn7n97CopWvMXdyJQ/cWM+4kcOO+pxj6T7Yy6CIfl85bmzPcffSl3nutVbKh+Y3l7r1wil86bLp/brifbIwsEqSJEkaUJo7DvBGa465kyv7NcV3xfqd3Pmj9YwbOYwHb6rn7Imj+/W6DW37eGZLC89sbuGFbW2UDy3j5nmTufGC2qOuxYV82H5szZt8699eBeArV87gd+ZM4t6f/JJHVzdQN34k37l+NudMOr6aensTP3+9lfGnDmf6af3bRbmYDKySJEmSdJxebnyH2x5dS3uui/raSmoqy6kZU0515QiqK8upGTOCqpHDiMivkX3h9TZWbW3hmS0tNLTlgPw63QV1VTS251i5uYXhQwZxTX01t150JlPGHTolesfuHF9euoHnX2/jwqljuffqj1EzpvxX9z+zpYW7lqynbW8Xt19Sxx9dfNYRd0TesTvHj9Y1snRdI0179nPD+Wfwras/emLerA+BgVWSJEmS+qG54wD3/ftmtuzaS2N7jta9Xf/v/mGDBzGxYgSN7Tm6DyZGDCnj42eN5TenV7GgrorJfULp1l0dLH52Oz9+sYnu3l4+OWMCty04k/raSlKCx36Rv6o6KIKvXjmDG86vOexV4T25Lu5Z/gr/un4ns2squP+6WZxZld8R+UD3QZ7a+DY/XLuD519vIwLm11Vx3XnVXDpjQqanEhtYJUmSJOnXkOvqoal9Pzvac+xhGNScAAAF/UlEQVTYvZ/G9hxNe/ZTXVnOxdOqqJ9cybDBRw+FzR0HePSFBh5d3cCeXDezayoYNngQa7bvZn7dOO79zMeYVDHimLWsWL+Te5ZtpLPnIF+8pI6m9v2sWL+TjgM9nDGmnGvrq/lMfTUTj+P/ygIDqyRJkiRlRK6rh6XrGln83HZ27+3ia5+awfVzD39V9Uh2vXuAO5dsYNWW/HTjK885nWvPq+E3poxh0Af8eqBSMbBKkiRJUsb09uaz1wcNmCklNjS+w5SqUxg1fMiHWVpR9SewDj7RxUiSJEmSPnhQfU9EMKum4kOq5uRQ2m/HlSRJkiTpCAyskiRJkqRMMrBKkiRJkjLJwCpJkiRJyiQDqyRJkiQpkwyskiRJkqRMMrBKkiRJkjLJwCpJkiRJyiQDqyRJkiQpkwyskiRJkqRMMrBKkiRJkjLJwCpJkiRJyiQDqyRJkiQpkwyskiRJkqRMMrBKkiRJkjLJwCpJkiRJyiQDqyRJkiQpkwyskiRJkqRMMrBKkiRJkjLJwCpJkiRJyqRIKZW6hkNERAvQUOo6jmIc0FrqIqQCx6OyxPGoLHE8Kkscj8qSUo/H2pRS1fE8MJOBNesiYm1K6bxS1yGB41HZ4nhUljgelSWOR2XJyTQenRIsSZIkScokA6skSZIkKZMMrB/Mg6UuQOrD8agscTwqSxyPyhLHo7LkpBmPrmGVJEmSJGWSV1glSZIkSZlkYO2niLg8IjZHxGsRcXep69HAEhE1EbEyIjZFxCsRcXvh/JiIeDoithZuK0tdqwaGiCiLiBcj4snC8ZSIWFPokT+IiKGlrlEDQ0RURMSSiPhlRLwaEfPsjSqViLij8Ht6Y0Q8HhHD7Y8qpoh4OCKaI2Jjn3OH7YmR9/eFsbkhIs4tXeWHMrD2Q0SUAf8AXAHMBG6IiJmlrUoDTA/wZymlmcAFwJ8UxuDdwM9SSnXAzwrHUjHcDrza5/hvgO+klKYC7cCtJalKA9HfAU+llD4CzCI/Lu2NKrqImAR8ETgvpXQOUAb8LvZHFdc/AZe/79yReuIVQF3h323AA0Wq8bgYWPvnfOC1lNK2lFIX8C/AwhLXpAEkpfRWSul/Cj93kP+DbBL5cfhI4WGPAL9dmgo1kERENfApYHHhOIBPAEsKD3EsqigiYjSwAHgIIKXUlVLag71RpTMYGBERg4Fy4C3sjyqilNIqYPf7Th+pJy4Evp/yVgMVEXF6cSo9NgNr/0wCdvQ5biyck4ouIiYDc4A1wISU0luFu94GJpSoLA0sfwvcBfQWjscCe1JKPYVje6SKZQrQAnyvMEV9cUScgr1RJZBSagLuA94kH1TfAdZhf1TpHaknZjrjGFilk1BEjASWAn+aUnq3730pv/W323/rhIqIq4DmlNK6Utcikb+adS7wQEppDrCP903/tTeqWArrAheS/yBlInAKh07NlErqZOqJBtb+aQJq+hxXF85JRRMRQ8iH1cdSSk8UTu96b+pG4ba5VPVpwLgQ+HREvEF+ecQnyK8hrChMgQN7pIqnEWhMKa0pHC8hH2DtjSqFS4HtKaWWlFI38AT5nml/VKkdqSdmOuMYWPvnv4G6wi5vQ8kvoF9R4po0gBTWCD4EvJpSur/PXSuAmws/3wwsL3ZtGlhSSl9JKVWnlCaT74X/mVL6LLASuKbwMMeiiiKl9DawIyKmF05dAmzC3qjSeBO4ICLKC7+33xuP9keV2pF64grgpsJuwRcA7/SZOlxykb8arOMVEVeSX7dVBjycUvqrEpekASQiLgKeBV7m/9YNfpX8OtYfAmcADcB1KaX3L7SXToiIuBj4Ukrpqog4k/wV1zHAi8CNKaXOUtangSEiZpPfAGwosA24hfwH8/ZGFV1EfBO4nvzu/i8CXyC/JtD+qKKIiMeBi4FxwC7gG8AyDtMTCx+sLCI/dT0H3JJSWluKug/HwCpJkiRJyiSnBEuSJEmSMsnAKkmSJEnKJAOrJEmSJCmTDKySJEmSpEwysEqSJEmSMsnAKkmSJEnKJAOrJEmSJCmTDKySJEmSpEz6X4LO1tdLY+0MAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x648 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(16,9))\n",
    "plt.plot(range(len(trn_loss_list)), trn_loss_list)\n",
    "plt.plot(range(len(val_loss_list)), val_loss_list)\n",
    "plt.legend(['trn', 'val'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}