classification_text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center> <font color=\"blue\"><h1> French Web Domain Classification </h1> </center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center> <h3> Done by :Saif Eddine GHRIBI & Mohamed Skander HELLAL & Ramzi Charradi </h3> </center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 116
    },
    "colab_type": "code",
    "id": "_K3OseQNwn8M",
    "outputId": "ef46898e-f848-4a7a-ea2b-921bc0c30eac"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<p style=\"color: red;\">\n",
       "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
       "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
       "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
       "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split,cross_val_score\n",
    "from keras.layers import *\n",
    "from keras.models import *\n",
    "from keras.callbacks import *\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "plt.style.use('ggplot')\n",
    "from warnings import simplefilter,filterwarnings\n",
    "# ignore all future warnings\n",
    "filterwarnings(\"ignore\")\n",
    "simplefilter(action='ignore', category=FutureWarning)\n",
    "simplefilter(action='ignore', category=DeprecationWarning)\n",
    "\n",
    "\n",
    "from src.preprocess import read_data, pre_process_text, filter_text\n",
    "from src.k_core import get_k_core\n",
    "from src.word2vec import Word2Vec, BagOfWords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OSVygiPUwn8Q"
   },
   "source": [
    "### Setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lrP4-Isuwn8R"
   },
   "outputs": [],
   "source": [
    "# path of files to read :\n",
    "train_data = \"data/train.csv\"\n",
    "test_data  = \"data/test.csv\"\n",
    "texts_path = \"data/text/text\"\n",
    "kaggle  = pd.read_csv(\"./data/kaggle.csv\")\n",
    "\n",
    "# vocab path for word2vec\n",
    "fr_embeddings = Path('data/cc.fr.300.vec.gz')\n",
    "en_embeddings = Path('data/cc.en.300.vec.gz')\n",
    "import urllib.request as tr\n",
    "if not en_embeddings.exists():\n",
    "    tr.urlretrieve('https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.en.300.vec.gz', en_embeddings)\n",
    "if not fr_embeddings.exists():\n",
    "    tr.urlretrieve('https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fr.300.vec.gz', fr_embeddings)\n",
    "\n",
    "# preprocessing :\n",
    "num_words = 512\n",
    "do_stem = False\n",
    "do_tokenize = False\n",
    "\n",
    "# text filters\n",
    "min_word_length = 4\n",
    "min_text_length = 20\n",
    "\n",
    "# k_core decomposition\n",
    "do_kcore = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "R9TW5NQSwn8U"
   },
   "source": [
    "# Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 330
    },
    "colab_type": "code",
    "id": "ub2n7S0fwn8U",
    "outputId": "c93c6c7b-b9e9-4e38-8943-f0775b68baa9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading training data ...\n",
      "reading test data ...\n",
      "reading text files ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 28003/28003 [04:44<00:00, 98.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished\n"
     ]
    }
   ],
   "source": [
    "# read data\n",
    "df, test_df = read_data(train_data, test_data, texts_path)\n",
    "# drop empty rows\n",
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H1VjiDGYwn8Y",
    "outputId": "acd2ed05-7aba-4861-d15b-4f703bf7ca19"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing training data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 1994/1994 [06:37<00:00,  5.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing test data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 560/560 [01:50<00:00,  5.07it/s]\n"
     ]
    }
   ],
   "source": [
    "# preprocess data\n",
    "tqdm.pandas()\n",
    "print(\"Preprocessing training data...\")\n",
    "df[\"text\"] = df[\"text\"].progress_apply( lambda x : pre_process_text(x, num_words, do_stem, do_tokenize))\n",
    "print(\"Preprocessing test data...\")\n",
    "test_df[\"text\"] = test_df[\"text\"].progress_apply( lambda x : pre_process_text(x, num_words, do_stem, do_tokenize))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l6IXhtEQwn8a",
    "outputId": "784b5011-647c-46db-fe45-055e07a41559"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filtering training data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██████████████████▋                                                            | 471/1994 [02:20<08:03,  3.15it/s]"
     ]
    }
   ],
   "source": [
    "# filtering data\n",
    "tqdm.pandas()\n",
    "print(\"Filtering training data...\")\n",
    "df[\"text\"] = df[\"text\"].progress_apply( lambda x : filter_text(x, min_word_length = 4 , min_text_length = 15,dataset = \"train\"))\n",
    "df = df[df.text!=\"No text\"]\n",
    "df = df.reset_index(drop=True)\n",
    "print(\"Filtering test data...\")\n",
    "test_df[\"text\"] = test_df[\"text\"].progress_apply( lambda x : filter_text(x, min_word_length = 4 , min_text_length = 0,dataset =\"test\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wDbBRMc8wn8c"
   },
   "outputs": [],
   "source": [
    "# get k core decomposition to pick most relevant words\n",
    "if do_kcore:\n",
    "    df = get_k_core(df)\n",
    "    test_df =  get_k_core(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YZe5aFM9wn8f"
   },
   "outputs": [],
   "source": [
    "# convert labels\n",
    "classes = {\"business/finance\":0,\"education/research\":1,\"entertainment\":2,\"health/medical\":3,\"news/press\":4, \\\n",
    "           \"politics/government/law\":5, \"sports\":6, \"tech/science\":7}\n",
    "for i in range(len(df)):\n",
    "    df.label[i] = classes[df.label[i]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TDflSw56wn8h"
   },
   "outputs": [],
   "source": [
    "#save files\n",
    "df.to_csv(\"./data/processed_train.csv\", index=False)\n",
    "test_df.to_csv(\"./data/processed_test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QKTmLchvwn8k"
   },
   "source": [
    "### TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w8wpSD1ewn8k"
   },
   "outputs": [],
   "source": [
    "# Load data \n",
    "df = pd.read_csv(\"./data/train_preprocessed.csv\")\n",
    "test_df = pd.read_csv(\"./data/test_preprocessed.csv\")\n",
    "df = df.dropna()\n",
    "X = df.text.values\n",
    "y = df.label.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eB6PICXgwn8n"
   },
   "outputs": [],
   "source": [
    "# train test split\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "       X, y, test_size=0.25, random_state=70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eCzRO9LTwn8p"
   },
   "outputs": [],
   "source": [
    "# create tfidf vectors\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vec = TfidfVectorizer(decode_error='ignore', strip_accents='unicode', encoding='latin-1', min_df=10, max_df=1000)\n",
    "X_train_tfidf = vec.fit_transform(X_train)\n",
    "X_test_tfidf = vec.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "AfvqPs4Qwn8r",
    "outputId": "5ce3f785-1343-498d-c12d-3aa7086dd8c2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Penalty: l2\n",
      "Best C: 21.544346900318832\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# Create logistic regression\n",
    "logistic = LogisticRegression(multi_class=\"auto\")\n",
    "\n",
    "# Create regularization penalty space\n",
    "penalty = ['l1', 'l2']\n",
    "\n",
    "# Create regularization hyperparameter space\n",
    "C = np.logspace(0, 4, 10)\n",
    "\n",
    "# Create hyperparameter options\n",
    "hyperparameters = dict(C=C, penalty=penalty)\n",
    "\n",
    "# Create grid search using 5-fold cross validation\n",
    "clf = GridSearchCV(logistic, hyperparameters, cv=5, verbose=0)\n",
    "\n",
    "# Fit grid search\n",
    "best_model = clf.fit(X_train_tfidf, y_train)\n",
    "\n",
    "# View best hyperparameters\n",
    "print('Best Penalty:', best_model.best_estimator_.get_params()['penalty'])\n",
    "print('Best C:', best_model.best_estimator_.get_params()['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "R4Rr8YElwn8u",
    "outputId": "901387a6-7f87-4f6d-d486-e2615df4861b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy score :  0.5788336933045356\n",
      "Loss : 1.195474602345714\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "y_pred = best_model.predict(X_test_tfidf)\n",
    "print(\"accuracy score : \",accuracy_score(y_test, y_pred))\n",
    "print(\"Loss :\", log_loss(y_test, best_model.predict_proba(X_test_tfidf)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hkSfuPG9tD8N"
   },
   "outputs": [],
   "source": [
    "# Write predictions to a file\n",
    "test_tfidif = vec.transform(test_df.text)\n",
    "prediction_ = best_model.predict_proba(test_tfidif)\n",
    "test_hosts = list(kaggle.Host.values)\n",
    "import csv\n",
    "with open('tfidf.csv', 'w') as csvfile:\n",
    "    writer = csv.writer(csvfile, delimiter=',')\n",
    "    lst = kaggle.columns.to_list()\n",
    "    writer.writerow(lst)\n",
    "    for i,test_host in enumerate(test_hosts):\n",
    "        lst = prediction_[i].tolist()\n",
    "        lst.insert(0, test_host)\n",
    "        writer.writerow(lst)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DI6RqJ3g3vWo"
   },
   "source": [
    "### FastText "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To generate FastText, it is just by uncommenting the code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-lYrLQRv3-FA"
   },
   "outputs": [],
   "source": [
    "\n",
    "# Installing FastText : \n",
    "\"\"\"\n",
    "!rm -r fastText/\n",
    "!git clone https://github.com/facebookresearch/fastText.git\n",
    "!cd fastText/\n",
    "!pip install .\n",
    "!cd ..\n",
    "\"\"\"\n",
    "#Loading weights\n",
    "\"\"\"\n",
    "fasttext.util.download_model('fr', if_exists='ignore') \n",
    "ft = fasttext.load_model('cc.fr.300.bin')\n",
    "\"\"\"\n",
    "#Load preprocessed data\n",
    "\"\"\"\n",
    "train = pd.read_csv('train_preprocessed.csv')\n",
    "test = pd.read_csv(\"test_preprocessed.csv\")\n",
    "\"\"\"\n",
    "# create train_data and test_data: \n",
    "\"\"\"\n",
    "train_data = []\n",
    "for x in train.dropna().text:\n",
    "    train_data.append(ft.get_sentence_vector(x,))\n",
    "test_data = []\n",
    "for x in test.dropna().text:\n",
    "    test_data.append(ft.get_sentence_vector(x,))\n",
    "train_data = np.array(train_data)\n",
    "test_data = np.array(test_data)\n",
    "y_train  = transform_one_hot(  train.dropna().label.values)\n",
    "\"\"\"\n",
    "#save weights \n",
    "\"\"\"\n",
    "np.save(\"train_data_fasttext.npy\",train_data)\n",
    "np.save(\"train_labels_fastext.npy\",y_train)\n",
    "np.save(\"test_data_fasttext.npy\",test_data)\n",
    "\"\"\"\n",
    "#Load weights : \n",
    "train_data = np.load(\"saved_files/train_data_fasttext.npy\")\n",
    "y_train = np.load(\"saved_files/train_labels_fastext.npy\")\n",
    "test_data = np.load(\"saved_files/test_data_fasttext.npy\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "colab_type": "code",
    "id": "fy8CrNlY79zL",
    "outputId": "365f9bac-34cb-4ed3-b570-6935bed28192"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of train data is (1849, 300)\n",
      "The shape of train data labels is (1849, 8)\n",
      "The shape of test data is (560, 300)\n"
     ]
    }
   ],
   "source": [
    "print(\"The shape of train data is \"+str(np.shape(train_data)))\n",
    "print(\"The shape of train data labels is \"+str(np.shape(y_train)))\n",
    "print(\"The shape of test data is \"+str(np.shape(test_data)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 516
    },
    "colab_type": "code",
    "id": "oNQ1mZvC9Zyn",
    "outputId": "a65403f7-ac7f-4a14-d16e-2568d7332747"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_1 (Conv1D)            (None, 297, 128)          640       \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 297, 128)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 38016)             0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 8)                 304136    \n",
      "=================================================================\n",
      "Total params: 304,776\n",
      "Trainable params: 304,776\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#Creating neural network \n",
    "r,c = train_data.shape\n",
    "modelfastText = Sequential()\n",
    "modelfastText.add(Conv1D(128, 4, activation='relu', padding='valid',input_shape=(c,1)))\n",
    "modelfastText.add(Dropout(0.5))\n",
    "modelfastText.add(Flatten())\n",
    "modelfastText.add(Dense(8, activation='softmax'))  #multi-label (k-hot encoding)\n",
    "modelfastText.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "tG7kEBLt_HlY",
    "outputId": "be60c7d1-ffb3-4d87-e610-b5929b3408f8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3576: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3005: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "Train on 1571 samples, validate on 278 samples\n",
      "Epoch 1/600\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
      "\n",
      " - 13s - loss: 2.0786 - categorical_accuracy: 0.2342 - val_loss: 2.0450 - val_categorical_accuracy: 0.3022\n",
      "Epoch 2/600\n",
      " - 0s - loss: 2.0478 - categorical_accuracy: 0.2743 - val_loss: 1.9980 - val_categorical_accuracy: 0.3022\n",
      "Epoch 3/600\n",
      " - 0s - loss: 2.0046 - categorical_accuracy: 0.2763 - val_loss: 1.9335 - val_categorical_accuracy: 0.3022\n",
      "Epoch 4/600\n",
      " - 0s - loss: 1.9462 - categorical_accuracy: 0.2928 - val_loss: 1.8580 - val_categorical_accuracy: 0.3022\n",
      "Epoch 5/600\n",
      " - 0s - loss: 1.8797 - categorical_accuracy: 0.2890 - val_loss: 1.7886 - val_categorical_accuracy: 0.3201\n",
      "Epoch 6/600\n",
      " - 0s - loss: 1.8218 - categorical_accuracy: 0.3081 - val_loss: 1.7502 - val_categorical_accuracy: 0.3741\n",
      "Epoch 7/600\n",
      " - 0s - loss: 1.7983 - categorical_accuracy: 0.3361 - val_loss: 1.7514 - val_categorical_accuracy: 0.2950\n",
      "Epoch 8/600\n",
      " - 0s - loss: 1.8122 - categorical_accuracy: 0.2947 - val_loss: 1.7568 - val_categorical_accuracy: 0.2950\n",
      "Epoch 9/600\n",
      " - 0s - loss: 1.8242 - categorical_accuracy: 0.2941 - val_loss: 1.7460 - val_categorical_accuracy: 0.2950\n",
      "Epoch 10/600\n",
      " - 0s - loss: 1.8140 - categorical_accuracy: 0.2941 - val_loss: 1.7304 - val_categorical_accuracy: 0.2950\n",
      "Epoch 11/600\n",
      " - 0s - loss: 1.7953 - categorical_accuracy: 0.2941 - val_loss: 1.7218 - val_categorical_accuracy: 0.2950\n",
      "Epoch 12/600\n",
      " - 0s - loss: 1.7812 - categorical_accuracy: 0.2941 - val_loss: 1.7219 - val_categorical_accuracy: 0.2950\n",
      "Epoch 13/600\n",
      " - 0s - loss: 1.7766 - categorical_accuracy: 0.2941 - val_loss: 1.7263 - val_categorical_accuracy: 0.2950\n",
      "Epoch 14/600\n",
      " - 0s - loss: 1.7771 - categorical_accuracy: 0.2941 - val_loss: 1.7304 - val_categorical_accuracy: 0.3058\n",
      "Epoch 15/600\n",
      " - 0s - loss: 1.7787 - categorical_accuracy: 0.3004 - val_loss: 1.7318 - val_categorical_accuracy: 0.3273\n",
      "Epoch 16/600\n",
      " - 0s - loss: 1.7781 - categorical_accuracy: 0.3132 - val_loss: 1.7297 - val_categorical_accuracy: 0.3525\n",
      "Epoch 17/600\n",
      " - 0s - loss: 1.7758 - categorical_accuracy: 0.3272 - val_loss: 1.7243 - val_categorical_accuracy: 0.3777\n",
      "Epoch 18/600\n",
      " - 0s - loss: 1.7713 - categorical_accuracy: 0.3501 - val_loss: 1.7163 - val_categorical_accuracy: 0.3921\n",
      "Epoch 19/600\n",
      " - 0s - loss: 1.7648 - categorical_accuracy: 0.3635 - val_loss: 1.7066 - val_categorical_accuracy: 0.3849\n",
      "Epoch 20/600\n",
      " - 0s - loss: 1.7580 - categorical_accuracy: 0.3666 - val_loss: 1.6967 - val_categorical_accuracy: 0.3921\n",
      "Epoch 21/600\n",
      " - 0s - loss: 1.7507 - categorical_accuracy: 0.3775 - val_loss: 1.6880 - val_categorical_accuracy: 0.4029\n",
      "Epoch 22/600\n",
      " - 0s - loss: 1.7457 - categorical_accuracy: 0.3794 - val_loss: 1.6814 - val_categorical_accuracy: 0.4065\n",
      "Epoch 23/600\n",
      " - 0s - loss: 1.7425 - categorical_accuracy: 0.3781 - val_loss: 1.6768 - val_categorical_accuracy: 0.3993\n",
      "Epoch 24/600\n",
      " - 0s - loss: 1.7403 - categorical_accuracy: 0.3768 - val_loss: 1.6733 - val_categorical_accuracy: 0.3957\n",
      "Epoch 25/600\n",
      " - 0s - loss: 1.7391 - categorical_accuracy: 0.3749 - val_loss: 1.6697 - val_categorical_accuracy: 0.3921\n",
      "Epoch 26/600\n",
      " - 0s - loss: 1.7358 - categorical_accuracy: 0.3781 - val_loss: 1.6657 - val_categorical_accuracy: 0.3885\n",
      "Epoch 27/600\n",
      " - 0s - loss: 1.7307 - categorical_accuracy: 0.3743 - val_loss: 1.6615 - val_categorical_accuracy: 0.3885\n",
      "Epoch 28/600\n",
      " - 0s - loss: 1.7250 - categorical_accuracy: 0.3717 - val_loss: 1.6580 - val_categorical_accuracy: 0.3885\n",
      "Epoch 29/600\n",
      " - 0s - loss: 1.7192 - categorical_accuracy: 0.3717 - val_loss: 1.6555 - val_categorical_accuracy: 0.3957\n",
      "Epoch 30/600\n",
      " - 0s - loss: 1.7144 - categorical_accuracy: 0.3781 - val_loss: 1.6535 - val_categorical_accuracy: 0.3993\n",
      "Epoch 31/600\n",
      " - 0s - loss: 1.7108 - categorical_accuracy: 0.3819 - val_loss: 1.6511 - val_categorical_accuracy: 0.4173\n",
      "Epoch 32/600\n",
      " - 0s - loss: 1.7067 - categorical_accuracy: 0.3896 - val_loss: 1.6476 - val_categorical_accuracy: 0.4281\n",
      "Epoch 33/600\n",
      " - 0s - loss: 1.7022 - categorical_accuracy: 0.3978 - val_loss: 1.6423 - val_categorical_accuracy: 0.4353\n",
      "Epoch 34/600\n",
      " - 0s - loss: 1.6980 - categorical_accuracy: 0.3978 - val_loss: 1.6356 - val_categorical_accuracy: 0.4424\n",
      "Epoch 35/600\n",
      " - 0s - loss: 1.6922 - categorical_accuracy: 0.3947 - val_loss: 1.6282 - val_categorical_accuracy: 0.4388\n",
      "Epoch 36/600\n",
      " - 0s - loss: 1.6859 - categorical_accuracy: 0.3991 - val_loss: 1.6209 - val_categorical_accuracy: 0.4317\n",
      "Epoch 37/600\n",
      " - 0s - loss: 1.6810 - categorical_accuracy: 0.3966 - val_loss: 1.6143 - val_categorical_accuracy: 0.4245\n",
      "Epoch 38/600\n",
      " - 0s - loss: 1.6763 - categorical_accuracy: 0.3927 - val_loss: 1.6083 - val_categorical_accuracy: 0.4209\n",
      "Epoch 39/600\n",
      " - 0s - loss: 1.6723 - categorical_accuracy: 0.3959 - val_loss: 1.6027 - val_categorical_accuracy: 0.4281\n",
      "Epoch 40/600\n",
      " - 0s - loss: 1.6674 - categorical_accuracy: 0.3966 - val_loss: 1.5969 - val_categorical_accuracy: 0.4353\n",
      "Epoch 41/600\n",
      " - 0s - loss: 1.6608 - categorical_accuracy: 0.4017 - val_loss: 1.5913 - val_categorical_accuracy: 0.4460\n",
      "Epoch 42/600\n",
      " - 0s - loss: 1.6558 - categorical_accuracy: 0.4017 - val_loss: 1.5861 - val_categorical_accuracy: 0.4424\n",
      "Epoch 43/600\n",
      " - 0s - loss: 1.6494 - categorical_accuracy: 0.4074 - val_loss: 1.5816 - val_categorical_accuracy: 0.4496\n",
      "Epoch 44/600\n",
      " - 0s - loss: 1.6433 - categorical_accuracy: 0.4067 - val_loss: 1.5771 - val_categorical_accuracy: 0.4388\n",
      "Epoch 45/600\n",
      " - 0s - loss: 1.6387 - categorical_accuracy: 0.4061 - val_loss: 1.5718 - val_categorical_accuracy: 0.4388\n",
      "Epoch 46/600\n",
      " - 0s - loss: 1.6322 - categorical_accuracy: 0.4144 - val_loss: 1.5651 - val_categorical_accuracy: 0.4424\n",
      "Epoch 47/600\n",
      " - 0s - loss: 1.6272 - categorical_accuracy: 0.4112 - val_loss: 1.5574 - val_categorical_accuracy: 0.4353\n",
      "Epoch 48/600\n",
      " - 0s - loss: 1.6208 - categorical_accuracy: 0.4201 - val_loss: 1.5496 - val_categorical_accuracy: 0.4532\n",
      "Epoch 49/600\n",
      " - 0s - loss: 1.6131 - categorical_accuracy: 0.4169 - val_loss: 1.5421 - val_categorical_accuracy: 0.4496\n",
      "Epoch 50/600\n",
      " - 0s - loss: 1.6083 - categorical_accuracy: 0.4106 - val_loss: 1.5349 - val_categorical_accuracy: 0.4532\n",
      "Epoch 51/600\n",
      " - 0s - loss: 1.6029 - categorical_accuracy: 0.4169 - val_loss: 1.5278 - val_categorical_accuracy: 0.4604\n",
      "Epoch 52/600\n",
      " - 0s - loss: 1.5959 - categorical_accuracy: 0.4125 - val_loss: 1.5207 - val_categorical_accuracy: 0.4604\n",
      "Epoch 53/600\n",
      " - 0s - loss: 1.5881 - categorical_accuracy: 0.4227 - val_loss: 1.5141 - val_categorical_accuracy: 0.4640\n",
      "Epoch 54/600\n",
      " - 0s - loss: 1.5833 - categorical_accuracy: 0.4284 - val_loss: 1.5079 - val_categorical_accuracy: 0.4676\n",
      "Epoch 55/600\n",
      " - 0s - loss: 1.5776 - categorical_accuracy: 0.4316 - val_loss: 1.5016 - val_categorical_accuracy: 0.4748\n",
      "Epoch 56/600\n",
      " - 0s - loss: 1.5705 - categorical_accuracy: 0.4398 - val_loss: 1.4946 - val_categorical_accuracy: 0.4748\n",
      "Epoch 57/600\n",
      " - 0s - loss: 1.5633 - categorical_accuracy: 0.4500 - val_loss: 1.4874 - val_categorical_accuracy: 0.4748\n",
      "Epoch 58/600\n",
      " - 0s - loss: 1.5584 - categorical_accuracy: 0.4418 - val_loss: 1.4806 - val_categorical_accuracy: 0.4820\n",
      "Epoch 59/600\n",
      " - 0s - loss: 1.5517 - categorical_accuracy: 0.4360 - val_loss: 1.4742 - val_categorical_accuracy: 0.4748\n",
      "Epoch 60/600\n",
      " - 0s - loss: 1.5444 - categorical_accuracy: 0.4430 - val_loss: 1.4680 - val_categorical_accuracy: 0.4856\n",
      "Epoch 61/600\n",
      " - 0s - loss: 1.5391 - categorical_accuracy: 0.4488 - val_loss: 1.4616 - val_categorical_accuracy: 0.5036\n",
      "Epoch 62/600\n",
      " - 0s - loss: 1.5323 - categorical_accuracy: 0.4628 - val_loss: 1.4551 - val_categorical_accuracy: 0.5072\n",
      "Epoch 63/600\n",
      " - 0s - loss: 1.5260 - categorical_accuracy: 0.4710 - val_loss: 1.4482 - val_categorical_accuracy: 0.5036\n",
      "Epoch 64/600\n",
      " - 0s - loss: 1.5204 - categorical_accuracy: 0.4653 - val_loss: 1.4409 - val_categorical_accuracy: 0.5036\n",
      "Epoch 65/600\n",
      " - 0s - loss: 1.5156 - categorical_accuracy: 0.4698 - val_loss: 1.4335 - val_categorical_accuracy: 0.5072\n",
      "Epoch 66/600\n",
      " - 0s - loss: 1.5098 - categorical_accuracy: 0.4736 - val_loss: 1.4266 - val_categorical_accuracy: 0.5144\n",
      "Epoch 67/600\n",
      " - 0s - loss: 1.5031 - categorical_accuracy: 0.4723 - val_loss: 1.4201 - val_categorical_accuracy: 0.5144\n",
      "Epoch 68/600\n",
      " - 0s - loss: 1.4962 - categorical_accuracy: 0.4742 - val_loss: 1.4142 - val_categorical_accuracy: 0.5180\n",
      "Epoch 69/600\n",
      " - 0s - loss: 1.4905 - categorical_accuracy: 0.4806 - val_loss: 1.4085 - val_categorical_accuracy: 0.5180\n",
      "Epoch 70/600\n",
      " - 0s - loss: 1.4862 - categorical_accuracy: 0.4780 - val_loss: 1.4028 - val_categorical_accuracy: 0.5252\n",
      "Epoch 71/600\n",
      " - 0s - loss: 1.4798 - categorical_accuracy: 0.4850 - val_loss: 1.3967 - val_categorical_accuracy: 0.5288\n",
      "Epoch 72/600\n",
      " - 0s - loss: 1.4756 - categorical_accuracy: 0.4876 - val_loss: 1.3903 - val_categorical_accuracy: 0.5360\n",
      "Epoch 73/600\n",
      " - 0s - loss: 1.4677 - categorical_accuracy: 0.4908 - val_loss: 1.3839 - val_categorical_accuracy: 0.5360\n",
      "Epoch 74/600\n",
      " - 0s - loss: 1.4630 - categorical_accuracy: 0.4908 - val_loss: 1.3778 - val_categorical_accuracy: 0.5432\n",
      "Epoch 75/600\n",
      " - 0s - loss: 1.4560 - categorical_accuracy: 0.4882 - val_loss: 1.3721 - val_categorical_accuracy: 0.5468\n",
      "Epoch 76/600\n",
      " - 0s - loss: 1.4544 - categorical_accuracy: 0.4882 - val_loss: 1.3667 - val_categorical_accuracy: 0.5468\n",
      "Epoch 77/600\n",
      " - 0s - loss: 1.4497 - categorical_accuracy: 0.4965 - val_loss: 1.3614 - val_categorical_accuracy: 0.5683\n",
      "Epoch 78/600\n",
      " - 0s - loss: 1.4419 - categorical_accuracy: 0.5041 - val_loss: 1.3557 - val_categorical_accuracy: 0.5683\n",
      "Epoch 79/600\n",
      " - 0s - loss: 1.4339 - categorical_accuracy: 0.5060 - val_loss: 1.3499 - val_categorical_accuracy: 0.5683\n",
      "Epoch 80/600\n",
      " - 0s - loss: 1.4318 - categorical_accuracy: 0.5016 - val_loss: 1.3443 - val_categorical_accuracy: 0.5647\n",
      "Epoch 81/600\n",
      " - 0s - loss: 1.4279 - categorical_accuracy: 0.5022 - val_loss: 1.3390 - val_categorical_accuracy: 0.5683\n",
      "Epoch 82/600\n",
      " - 0s - loss: 1.4237 - categorical_accuracy: 0.5035 - val_loss: 1.3340 - val_categorical_accuracy: 0.5719\n",
      "Epoch 83/600\n",
      " - 0s - loss: 1.4145 - categorical_accuracy: 0.5137 - val_loss: 1.3293 - val_categorical_accuracy: 0.5755\n",
      "Epoch 84/600\n",
      " - 0s - loss: 1.4078 - categorical_accuracy: 0.5156 - val_loss: 1.3246 - val_categorical_accuracy: 0.5719\n",
      "Epoch 85/600\n",
      " - 0s - loss: 1.4033 - categorical_accuracy: 0.5143 - val_loss: 1.3198 - val_categorical_accuracy: 0.5719\n",
      "Epoch 86/600\n",
      " - 0s - loss: 1.4007 - categorical_accuracy: 0.5226 - val_loss: 1.3149 - val_categorical_accuracy: 0.5719\n",
      "Epoch 87/600\n",
      " - 0s - loss: 1.4002 - categorical_accuracy: 0.5162 - val_loss: 1.3101 - val_categorical_accuracy: 0.5719\n",
      "Epoch 88/600\n",
      " - 0s - loss: 1.3926 - categorical_accuracy: 0.5067 - val_loss: 1.3053 - val_categorical_accuracy: 0.5647\n",
      "Epoch 89/600\n",
      " - 0s - loss: 1.3885 - categorical_accuracy: 0.5213 - val_loss: 1.3007 - val_categorical_accuracy: 0.5647\n",
      "Epoch 90/600\n",
      " - 0s - loss: 1.3820 - categorical_accuracy: 0.5213 - val_loss: 1.2962 - val_categorical_accuracy: 0.5647\n",
      "Epoch 91/600\n",
      " - 0s - loss: 1.3777 - categorical_accuracy: 0.5251 - val_loss: 1.2918 - val_categorical_accuracy: 0.5647\n",
      "Epoch 92/600\n",
      " - 0s - loss: 1.3725 - categorical_accuracy: 0.5213 - val_loss: 1.2875 - val_categorical_accuracy: 0.5647\n",
      "Epoch 93/600\n",
      " - 0s - loss: 1.3690 - categorical_accuracy: 0.5277 - val_loss: 1.2832 - val_categorical_accuracy: 0.5647\n",
      "Epoch 94/600\n",
      " - 0s - loss: 1.3660 - categorical_accuracy: 0.5296 - val_loss: 1.2791 - val_categorical_accuracy: 0.5683\n",
      "Epoch 95/600\n",
      " - 0s - loss: 1.3613 - categorical_accuracy: 0.5436 - val_loss: 1.2752 - val_categorical_accuracy: 0.5719\n",
      "Epoch 96/600\n",
      " - 0s - loss: 1.3562 - categorical_accuracy: 0.5334 - val_loss: 1.2714 - val_categorical_accuracy: 0.5719\n",
      "Epoch 97/600\n",
      " - 0s - loss: 1.3522 - categorical_accuracy: 0.5353 - val_loss: 1.2674 - val_categorical_accuracy: 0.5683\n",
      "Epoch 98/600\n",
      " - 0s - loss: 1.3511 - categorical_accuracy: 0.5341 - val_loss: 1.2634 - val_categorical_accuracy: 0.5683\n",
      "Epoch 99/600\n",
      " - 0s - loss: 1.3445 - categorical_accuracy: 0.5404 - val_loss: 1.2593 - val_categorical_accuracy: 0.5683\n",
      "Epoch 100/600\n",
      " - 0s - loss: 1.3396 - categorical_accuracy: 0.5398 - val_loss: 1.2554 - val_categorical_accuracy: 0.5719\n",
      "Epoch 101/600\n",
      " - 0s - loss: 1.3377 - categorical_accuracy: 0.5379 - val_loss: 1.2520 - val_categorical_accuracy: 0.5755\n",
      "Epoch 102/600\n",
      " - 0s - loss: 1.3356 - categorical_accuracy: 0.5430 - val_loss: 1.2486 - val_categorical_accuracy: 0.5719\n",
      "Epoch 103/600\n",
      " - 0s - loss: 1.3335 - categorical_accuracy: 0.5417 - val_loss: 1.2451 - val_categorical_accuracy: 0.5719\n",
      "Epoch 104/600\n",
      " - 0s - loss: 1.3262 - categorical_accuracy: 0.5449 - val_loss: 1.2415 - val_categorical_accuracy: 0.5755\n",
      "Epoch 105/600\n",
      " - 0s - loss: 1.3231 - categorical_accuracy: 0.5430 - val_loss: 1.2378 - val_categorical_accuracy: 0.5791\n",
      "Epoch 106/600\n",
      " - 0s - loss: 1.3202 - categorical_accuracy: 0.5449 - val_loss: 1.2343 - val_categorical_accuracy: 0.5791\n",
      "Epoch 107/600\n",
      " - 0s - loss: 1.3171 - categorical_accuracy: 0.5493 - val_loss: 1.2312 - val_categorical_accuracy: 0.5863\n",
      "Epoch 108/600\n",
      " - 0s - loss: 1.3142 - categorical_accuracy: 0.5525 - val_loss: 1.2281 - val_categorical_accuracy: 0.5863\n",
      "Epoch 109/600\n",
      " - 0s - loss: 1.3114 - categorical_accuracy: 0.5500 - val_loss: 1.2248 - val_categorical_accuracy: 0.5863\n",
      "Epoch 110/600\n",
      " - 0s - loss: 1.3018 - categorical_accuracy: 0.5506 - val_loss: 1.2216 - val_categorical_accuracy: 0.5899\n",
      "Epoch 111/600\n",
      " - 0s - loss: 1.3011 - categorical_accuracy: 0.5525 - val_loss: 1.2186 - val_categorical_accuracy: 0.5899\n",
      "Epoch 112/600\n",
      " - 0s - loss: 1.2997 - categorical_accuracy: 0.5582 - val_loss: 1.2156 - val_categorical_accuracy: 0.5935\n",
      "Epoch 113/600\n",
      " - 0s - loss: 1.2930 - categorical_accuracy: 0.5602 - val_loss: 1.2125 - val_categorical_accuracy: 0.5899\n",
      "Epoch 114/600\n",
      " - 0s - loss: 1.2924 - categorical_accuracy: 0.5576 - val_loss: 1.2095 - val_categorical_accuracy: 0.5935\n",
      "Epoch 115/600\n",
      " - 0s - loss: 1.2932 - categorical_accuracy: 0.5474 - val_loss: 1.2065 - val_categorical_accuracy: 0.6007\n",
      "Epoch 116/600\n",
      " - 0s - loss: 1.2828 - categorical_accuracy: 0.5525 - val_loss: 1.2037 - val_categorical_accuracy: 0.6007\n",
      "Epoch 117/600\n",
      " - 0s - loss: 1.2822 - categorical_accuracy: 0.5589 - val_loss: 1.2008 - val_categorical_accuracy: 0.6043\n",
      "Epoch 118/600\n",
      " - 0s - loss: 1.2774 - categorical_accuracy: 0.5538 - val_loss: 1.1979 - val_categorical_accuracy: 0.6043\n",
      "Epoch 119/600\n",
      " - 0s - loss: 1.2759 - categorical_accuracy: 0.5602 - val_loss: 1.1953 - val_categorical_accuracy: 0.6043\n",
      "Epoch 120/600\n",
      " - 0s - loss: 1.2737 - categorical_accuracy: 0.5582 - val_loss: 1.1927 - val_categorical_accuracy: 0.6043\n",
      "Epoch 121/600\n",
      " - 0s - loss: 1.2705 - categorical_accuracy: 0.5646 - val_loss: 1.1902 - val_categorical_accuracy: 0.6043\n",
      "Epoch 122/600\n",
      " - 0s - loss: 1.2636 - categorical_accuracy: 0.5652 - val_loss: 1.1875 - val_categorical_accuracy: 0.6043\n",
      "Epoch 123/600\n",
      " - 0s - loss: 1.2630 - categorical_accuracy: 0.5678 - val_loss: 1.1848 - val_categorical_accuracy: 0.6043\n",
      "Epoch 124/600\n",
      " - 0s - loss: 1.2624 - categorical_accuracy: 0.5608 - val_loss: 1.1821 - val_categorical_accuracy: 0.6043\n",
      "Epoch 125/600\n",
      " - 0s - loss: 1.2591 - categorical_accuracy: 0.5621 - val_loss: 1.1796 - val_categorical_accuracy: 0.6115\n",
      "Epoch 126/600\n",
      " - 0s - loss: 1.2575 - categorical_accuracy: 0.5608 - val_loss: 1.1774 - val_categorical_accuracy: 0.6151\n",
      "Epoch 127/600\n",
      " - 0s - loss: 1.2553 - categorical_accuracy: 0.5576 - val_loss: 1.1753 - val_categorical_accuracy: 0.6151\n",
      "Epoch 128/600\n",
      " - 0s - loss: 1.2512 - categorical_accuracy: 0.5646 - val_loss: 1.1732 - val_categorical_accuracy: 0.6115\n",
      "Epoch 129/600\n",
      " - 0s - loss: 1.2490 - categorical_accuracy: 0.5672 - val_loss: 1.1707 - val_categorical_accuracy: 0.6187\n",
      "Epoch 130/600\n",
      " - 0s - loss: 1.2463 - categorical_accuracy: 0.5742 - val_loss: 1.1682 - val_categorical_accuracy: 0.6187\n",
      "Epoch 131/600\n",
      " - 0s - loss: 1.2435 - categorical_accuracy: 0.5691 - val_loss: 1.1660 - val_categorical_accuracy: 0.6115\n",
      "Epoch 132/600\n",
      " - 0s - loss: 1.2405 - categorical_accuracy: 0.5614 - val_loss: 1.1639 - val_categorical_accuracy: 0.6079\n",
      "Epoch 133/600\n",
      " - 0s - loss: 1.2398 - categorical_accuracy: 0.5691 - val_loss: 1.1617 - val_categorical_accuracy: 0.6079\n",
      "Epoch 134/600\n",
      " - 0s - loss: 1.2323 - categorical_accuracy: 0.5691 - val_loss: 1.1594 - val_categorical_accuracy: 0.6115\n",
      "Epoch 135/600\n",
      " - 0s - loss: 1.2295 - categorical_accuracy: 0.5792 - val_loss: 1.1570 - val_categorical_accuracy: 0.6115\n",
      "Epoch 136/600\n",
      " - 0s - loss: 1.2274 - categorical_accuracy: 0.5684 - val_loss: 1.1552 - val_categorical_accuracy: 0.6115\n",
      "Epoch 137/600\n",
      " - 0s - loss: 1.2285 - categorical_accuracy: 0.5703 - val_loss: 1.1537 - val_categorical_accuracy: 0.6115\n",
      "Epoch 138/600\n",
      " - 0s - loss: 1.2250 - categorical_accuracy: 0.5729 - val_loss: 1.1518 - val_categorical_accuracy: 0.6151\n",
      "Epoch 139/600\n",
      " - 0s - loss: 1.2218 - categorical_accuracy: 0.5665 - val_loss: 1.1497 - val_categorical_accuracy: 0.6151\n",
      "Epoch 140/600\n",
      " - 0s - loss: 1.2202 - categorical_accuracy: 0.5780 - val_loss: 1.1476 - val_categorical_accuracy: 0.6115\n",
      "Epoch 141/600\n",
      " - 0s - loss: 1.2216 - categorical_accuracy: 0.5754 - val_loss: 1.1454 - val_categorical_accuracy: 0.6151\n",
      "Epoch 142/600\n",
      " - 0s - loss: 1.2131 - categorical_accuracy: 0.5761 - val_loss: 1.1433 - val_categorical_accuracy: 0.6187\n",
      "Epoch 143/600\n",
      " - 0s - loss: 1.2085 - categorical_accuracy: 0.5767 - val_loss: 1.1415 - val_categorical_accuracy: 0.6151\n",
      "Epoch 144/600\n",
      " - 0s - loss: 1.2098 - categorical_accuracy: 0.5729 - val_loss: 1.1403 - val_categorical_accuracy: 0.6115\n",
      "Epoch 145/600\n",
      " - 0s - loss: 1.2046 - categorical_accuracy: 0.5780 - val_loss: 1.1389 - val_categorical_accuracy: 0.6115\n",
      "Epoch 146/600\n",
      " - 0s - loss: 1.2095 - categorical_accuracy: 0.5818 - val_loss: 1.1367 - val_categorical_accuracy: 0.6151\n",
      "Epoch 147/600\n",
      " - 0s - loss: 1.2037 - categorical_accuracy: 0.5742 - val_loss: 1.1347 - val_categorical_accuracy: 0.6187\n",
      "Epoch 148/600\n",
      " - 0s - loss: 1.2047 - categorical_accuracy: 0.5767 - val_loss: 1.1327 - val_categorical_accuracy: 0.6187\n",
      "Epoch 149/600\n",
      " - 0s - loss: 1.1992 - categorical_accuracy: 0.5754 - val_loss: 1.1313 - val_categorical_accuracy: 0.6115\n",
      "Epoch 150/600\n",
      " - 0s - loss: 1.2038 - categorical_accuracy: 0.5786 - val_loss: 1.1292 - val_categorical_accuracy: 0.6115\n",
      "Epoch 151/600\n",
      " - 0s - loss: 1.2011 - categorical_accuracy: 0.5716 - val_loss: 1.1277 - val_categorical_accuracy: 0.6187\n",
      "Epoch 152/600\n",
      " - 0s - loss: 1.2016 - categorical_accuracy: 0.5742 - val_loss: 1.1262 - val_categorical_accuracy: 0.6151\n",
      "Epoch 153/600\n",
      " - 0s - loss: 1.1861 - categorical_accuracy: 0.5837 - val_loss: 1.1250 - val_categorical_accuracy: 0.6115\n",
      "Epoch 154/600\n",
      " - 0s - loss: 1.1863 - categorical_accuracy: 0.5767 - val_loss: 1.1233 - val_categorical_accuracy: 0.6151\n",
      "Epoch 155/600\n",
      " - 0s - loss: 1.1855 - categorical_accuracy: 0.5786 - val_loss: 1.1221 - val_categorical_accuracy: 0.6151\n",
      "Epoch 156/600\n",
      " - 0s - loss: 1.1844 - categorical_accuracy: 0.5792 - val_loss: 1.1211 - val_categorical_accuracy: 0.6151\n",
      "Epoch 157/600\n",
      " - 0s - loss: 1.1819 - categorical_accuracy: 0.5761 - val_loss: 1.1195 - val_categorical_accuracy: 0.6223\n",
      "Epoch 158/600\n",
      " - 0s - loss: 1.1859 - categorical_accuracy: 0.5754 - val_loss: 1.1173 - val_categorical_accuracy: 0.6187\n",
      "Epoch 159/600\n",
      " - 0s - loss: 1.1839 - categorical_accuracy: 0.5729 - val_loss: 1.1151 - val_categorical_accuracy: 0.6223\n",
      "Epoch 160/600\n",
      " - 0s - loss: 1.1853 - categorical_accuracy: 0.5710 - val_loss: 1.1140 - val_categorical_accuracy: 0.6223\n",
      "Epoch 161/600\n",
      " - 0s - loss: 1.1830 - categorical_accuracy: 0.5792 - val_loss: 1.1138 - val_categorical_accuracy: 0.6187\n",
      "Epoch 162/600\n",
      " - 0s - loss: 1.1802 - categorical_accuracy: 0.5805 - val_loss: 1.1131 - val_categorical_accuracy: 0.6223\n",
      "Epoch 163/600\n",
      " - 0s - loss: 1.1728 - categorical_accuracy: 0.5805 - val_loss: 1.1114 - val_categorical_accuracy: 0.6259\n",
      "Epoch 164/600\n",
      " - 0s - loss: 1.1687 - categorical_accuracy: 0.5824 - val_loss: 1.1096 - val_categorical_accuracy: 0.6259\n",
      "Epoch 165/600\n",
      " - 0s - loss: 1.1712 - categorical_accuracy: 0.5792 - val_loss: 1.1080 - val_categorical_accuracy: 0.6223\n",
      "Epoch 166/600\n",
      " - 0s - loss: 1.1711 - categorical_accuracy: 0.5742 - val_loss: 1.1069 - val_categorical_accuracy: 0.6223\n",
      "Epoch 167/600\n",
      " - 0s - loss: 1.1630 - categorical_accuracy: 0.5818 - val_loss: 1.1059 - val_categorical_accuracy: 0.6259\n",
      "Epoch 168/600\n",
      " - 0s - loss: 1.1696 - categorical_accuracy: 0.5856 - val_loss: 1.1047 - val_categorical_accuracy: 0.6259\n",
      "Epoch 169/600\n",
      " - 0s - loss: 1.1643 - categorical_accuracy: 0.5799 - val_loss: 1.1039 - val_categorical_accuracy: 0.6259\n",
      "Epoch 170/600\n",
      " - 0s - loss: 1.1605 - categorical_accuracy: 0.5863 - val_loss: 1.1023 - val_categorical_accuracy: 0.6259\n",
      "Epoch 171/600\n",
      " - 0s - loss: 1.1598 - categorical_accuracy: 0.5805 - val_loss: 1.1008 - val_categorical_accuracy: 0.6295\n",
      "Epoch 172/600\n",
      " - 0s - loss: 1.1517 - categorical_accuracy: 0.5792 - val_loss: 1.1003 - val_categorical_accuracy: 0.6331\n",
      "Epoch 173/600\n",
      " - 0s - loss: 1.1580 - categorical_accuracy: 0.5754 - val_loss: 1.0993 - val_categorical_accuracy: 0.6259\n",
      "Epoch 174/600\n",
      " - 0s - loss: 1.1513 - categorical_accuracy: 0.5952 - val_loss: 1.0987 - val_categorical_accuracy: 0.6187\n",
      "Epoch 175/600\n",
      " - 0s - loss: 1.1568 - categorical_accuracy: 0.5818 - val_loss: 1.0978 - val_categorical_accuracy: 0.6223\n",
      "Epoch 176/600\n",
      " - 0s - loss: 1.1523 - categorical_accuracy: 0.5831 - val_loss: 1.0969 - val_categorical_accuracy: 0.6259\n",
      "Epoch 177/600\n",
      " - 0s - loss: 1.1517 - categorical_accuracy: 0.5805 - val_loss: 1.0956 - val_categorical_accuracy: 0.6295\n",
      "Epoch 178/600\n",
      " - 0s - loss: 1.1504 - categorical_accuracy: 0.5843 - val_loss: 1.0937 - val_categorical_accuracy: 0.6295\n",
      "Epoch 179/600\n",
      " - 0s - loss: 1.1479 - categorical_accuracy: 0.5913 - val_loss: 1.0922 - val_categorical_accuracy: 0.6259\n",
      "Epoch 180/600\n",
      " - 0s - loss: 1.1532 - categorical_accuracy: 0.5805 - val_loss: 1.0919 - val_categorical_accuracy: 0.6259\n",
      "Epoch 181/600\n",
      " - 0s - loss: 1.1463 - categorical_accuracy: 0.5869 - val_loss: 1.0921 - val_categorical_accuracy: 0.6259\n",
      "Epoch 182/600\n",
      " - 0s - loss: 1.1430 - categorical_accuracy: 0.5856 - val_loss: 1.0919 - val_categorical_accuracy: 0.6259\n",
      "Epoch 183/600\n",
      " - 0s - loss: 1.1454 - categorical_accuracy: 0.5850 - val_loss: 1.0899 - val_categorical_accuracy: 0.6295\n",
      "Epoch 184/600\n",
      " - 0s - loss: 1.1425 - categorical_accuracy: 0.5856 - val_loss: 1.0882 - val_categorical_accuracy: 0.6259\n",
      "Epoch 185/600\n",
      " - 0s - loss: 1.1380 - categorical_accuracy: 0.5875 - val_loss: 1.0871 - val_categorical_accuracy: 0.6223\n",
      "Epoch 186/600\n",
      " - 0s - loss: 1.1342 - categorical_accuracy: 0.5933 - val_loss: 1.0864 - val_categorical_accuracy: 0.6223\n",
      "Epoch 187/600\n",
      " - 0s - loss: 1.1337 - categorical_accuracy: 0.5824 - val_loss: 1.0863 - val_categorical_accuracy: 0.6259\n",
      "Epoch 188/600\n",
      " - 0s - loss: 1.1357 - categorical_accuracy: 0.5882 - val_loss: 1.0857 - val_categorical_accuracy: 0.6223\n",
      "Epoch 189/600\n",
      " - 0s - loss: 1.1349 - categorical_accuracy: 0.5888 - val_loss: 1.0852 - val_categorical_accuracy: 0.6259\n",
      "Epoch 190/600\n",
      " - 0s - loss: 1.1299 - categorical_accuracy: 0.5843 - val_loss: 1.0848 - val_categorical_accuracy: 0.6187\n",
      "Epoch 191/600\n",
      " - 0s - loss: 1.1344 - categorical_accuracy: 0.5818 - val_loss: 1.0840 - val_categorical_accuracy: 0.6187\n",
      "Epoch 192/600\n",
      " - 0s - loss: 1.1254 - categorical_accuracy: 0.5863 - val_loss: 1.0828 - val_categorical_accuracy: 0.6223\n",
      "Epoch 193/600\n",
      " - 0s - loss: 1.1335 - categorical_accuracy: 0.5843 - val_loss: 1.0817 - val_categorical_accuracy: 0.6223\n",
      "Epoch 194/600\n",
      " - 0s - loss: 1.1320 - categorical_accuracy: 0.5945 - val_loss: 1.0809 - val_categorical_accuracy: 0.6223\n",
      "Epoch 195/600\n",
      " - 0s - loss: 1.1239 - categorical_accuracy: 0.5952 - val_loss: 1.0803 - val_categorical_accuracy: 0.6259\n",
      "Epoch 196/600\n",
      " - 0s - loss: 1.1262 - categorical_accuracy: 0.5958 - val_loss: 1.0800 - val_categorical_accuracy: 0.6259\n",
      "Epoch 197/600\n",
      " - 0s - loss: 1.1261 - categorical_accuracy: 0.5939 - val_loss: 1.0793 - val_categorical_accuracy: 0.6223\n",
      "Epoch 198/600\n",
      " - 0s - loss: 1.1187 - categorical_accuracy: 0.5926 - val_loss: 1.0789 - val_categorical_accuracy: 0.6187\n",
      "Epoch 199/600\n",
      " - 0s - loss: 1.1275 - categorical_accuracy: 0.5926 - val_loss: 1.0784 - val_categorical_accuracy: 0.6223\n",
      "Epoch 200/600\n",
      " - 0s - loss: 1.1172 - categorical_accuracy: 0.6022 - val_loss: 1.0774 - val_categorical_accuracy: 0.6295\n",
      "Epoch 201/600\n",
      " - 0s - loss: 1.1188 - categorical_accuracy: 0.5964 - val_loss: 1.0768 - val_categorical_accuracy: 0.6223\n",
      "Epoch 202/600\n",
      " - 0s - loss: 1.1200 - categorical_accuracy: 0.5850 - val_loss: 1.0766 - val_categorical_accuracy: 0.6223\n",
      "Epoch 203/600\n",
      " - 0s - loss: 1.1210 - categorical_accuracy: 0.5971 - val_loss: 1.0763 - val_categorical_accuracy: 0.6223\n",
      "Epoch 204/600\n",
      " - 0s - loss: 1.1182 - categorical_accuracy: 0.5926 - val_loss: 1.0751 - val_categorical_accuracy: 0.6259\n",
      "Epoch 205/600\n",
      " - 0s - loss: 1.1111 - categorical_accuracy: 0.5863 - val_loss: 1.0740 - val_categorical_accuracy: 0.6295\n",
      "Epoch 206/600\n",
      " - 0s - loss: 1.1105 - categorical_accuracy: 0.5983 - val_loss: 1.0735 - val_categorical_accuracy: 0.6223\n",
      "Epoch 207/600\n",
      " - 0s - loss: 1.1167 - categorical_accuracy: 0.5990 - val_loss: 1.0738 - val_categorical_accuracy: 0.6223\n",
      "Epoch 208/600\n",
      " - 0s - loss: 1.1034 - categorical_accuracy: 0.6034 - val_loss: 1.0741 - val_categorical_accuracy: 0.6295\n",
      "Epoch 209/600\n",
      " - 0s - loss: 1.1113 - categorical_accuracy: 0.5952 - val_loss: 1.0735 - val_categorical_accuracy: 0.6295\n",
      "Epoch 210/600\n",
      " - 0s - loss: 1.1075 - categorical_accuracy: 0.5920 - val_loss: 1.0716 - val_categorical_accuracy: 0.6223\n",
      "Epoch 211/600\n",
      " - 0s - loss: 1.1033 - categorical_accuracy: 0.5926 - val_loss: 1.0699 - val_categorical_accuracy: 0.6223\n",
      "Epoch 212/600\n",
      " - 0s - loss: 1.1114 - categorical_accuracy: 0.5869 - val_loss: 1.0695 - val_categorical_accuracy: 0.6295\n",
      "Epoch 213/600\n",
      " - 0s - loss: 1.1063 - categorical_accuracy: 0.5882 - val_loss: 1.0702 - val_categorical_accuracy: 0.6259\n",
      "Epoch 214/600\n",
      " - 0s - loss: 1.0977 - categorical_accuracy: 0.5913 - val_loss: 1.0704 - val_categorical_accuracy: 0.6223\n",
      "Epoch 215/600\n",
      " - 0s - loss: 1.0964 - categorical_accuracy: 0.6053 - val_loss: 1.0705 - val_categorical_accuracy: 0.6223\n",
      "Epoch 216/600\n",
      " - 0s - loss: 1.1061 - categorical_accuracy: 0.5939 - val_loss: 1.0699 - val_categorical_accuracy: 0.6223\n",
      "Epoch 217/600\n",
      " - 0s - loss: 1.1045 - categorical_accuracy: 0.5958 - val_loss: 1.0683 - val_categorical_accuracy: 0.6259\n",
      "Epoch 218/600\n",
      " - 0s - loss: 1.1001 - categorical_accuracy: 0.6041 - val_loss: 1.0669 - val_categorical_accuracy: 0.6259\n",
      "Epoch 219/600\n",
      " - 0s - loss: 1.1054 - categorical_accuracy: 0.5913 - val_loss: 1.0665 - val_categorical_accuracy: 0.6259\n",
      "Epoch 220/600\n",
      " - 0s - loss: 1.0988 - categorical_accuracy: 0.6015 - val_loss: 1.0669 - val_categorical_accuracy: 0.6259\n",
      "Epoch 221/600\n",
      " - 0s - loss: 1.0991 - categorical_accuracy: 0.5945 - val_loss: 1.0684 - val_categorical_accuracy: 0.6295\n",
      "Epoch 222/600\n",
      " - 0s - loss: 1.0963 - categorical_accuracy: 0.6041 - val_loss: 1.0683 - val_categorical_accuracy: 0.6187\n",
      "Epoch 223/600\n",
      " - 0s - loss: 1.0952 - categorical_accuracy: 0.5996 - val_loss: 1.0669 - val_categorical_accuracy: 0.6187\n",
      "Epoch 224/600\n",
      " - 0s - loss: 1.0914 - categorical_accuracy: 0.6085 - val_loss: 1.0652 - val_categorical_accuracy: 0.6259\n",
      "Epoch 225/600\n",
      " - 0s - loss: 1.0931 - categorical_accuracy: 0.5977 - val_loss: 1.0642 - val_categorical_accuracy: 0.6295\n",
      "Epoch 226/600\n",
      " - 0s - loss: 1.0960 - categorical_accuracy: 0.6003 - val_loss: 1.0649 - val_categorical_accuracy: 0.6187\n",
      "Epoch 227/600\n",
      " - 0s - loss: 1.0894 - categorical_accuracy: 0.5952 - val_loss: 1.0658 - val_categorical_accuracy: 0.6223\n",
      "Epoch 228/600\n",
      " - 0s - loss: 1.0892 - categorical_accuracy: 0.6022 - val_loss: 1.0653 - val_categorical_accuracy: 0.6295\n",
      "Epoch 229/600\n",
      " - 0s - loss: 1.0828 - categorical_accuracy: 0.6098 - val_loss: 1.0647 - val_categorical_accuracy: 0.6259\n",
      "Epoch 230/600\n",
      " - 0s - loss: 1.0852 - categorical_accuracy: 0.5990 - val_loss: 1.0641 - val_categorical_accuracy: 0.6295\n",
      "Epoch 231/600\n",
      " - 0s - loss: 1.0827 - categorical_accuracy: 0.6015 - val_loss: 1.0639 - val_categorical_accuracy: 0.6187\n",
      "Epoch 232/600\n",
      " - 0s - loss: 1.0860 - categorical_accuracy: 0.6060 - val_loss: 1.0627 - val_categorical_accuracy: 0.6259\n",
      "Epoch 233/600\n",
      " - 0s - loss: 1.0812 - categorical_accuracy: 0.6047 - val_loss: 1.0622 - val_categorical_accuracy: 0.6259\n",
      "Epoch 234/600\n",
      " - 0s - loss: 1.0948 - categorical_accuracy: 0.5958 - val_loss: 1.0619 - val_categorical_accuracy: 0.6187\n",
      "Epoch 235/600\n",
      " - 0s - loss: 1.0839 - categorical_accuracy: 0.6060 - val_loss: 1.0629 - val_categorical_accuracy: 0.6259\n",
      "Epoch 236/600\n",
      " - 0s - loss: 1.0879 - categorical_accuracy: 0.5907 - val_loss: 1.0629 - val_categorical_accuracy: 0.6331\n",
      "Epoch 237/600\n",
      " - 0s - loss: 1.0772 - categorical_accuracy: 0.6098 - val_loss: 1.0630 - val_categorical_accuracy: 0.6259\n",
      "Epoch 238/600\n",
      " - 0s - loss: 1.0810 - categorical_accuracy: 0.6009 - val_loss: 1.0632 - val_categorical_accuracy: 0.6187\n",
      "Epoch 239/600\n",
      " - 0s - loss: 1.0748 - categorical_accuracy: 0.6053 - val_loss: 1.0611 - val_categorical_accuracy: 0.6115\n",
      "Epoch 240/600\n",
      " - 0s - loss: 1.0760 - categorical_accuracy: 0.6079 - val_loss: 1.0595 - val_categorical_accuracy: 0.6331\n",
      "Epoch 241/600\n",
      " - 0s - loss: 1.0780 - categorical_accuracy: 0.6060 - val_loss: 1.0597 - val_categorical_accuracy: 0.6331\n",
      "Epoch 242/600\n",
      " - 0s - loss: 1.0783 - categorical_accuracy: 0.6066 - val_loss: 1.0609 - val_categorical_accuracy: 0.6223\n",
      "Epoch 243/600\n",
      " - 0s - loss: 1.0766 - categorical_accuracy: 0.6047 - val_loss: 1.0616 - val_categorical_accuracy: 0.6223\n",
      "Epoch 244/600\n",
      " - 0s - loss: 1.0737 - categorical_accuracy: 0.6003 - val_loss: 1.0604 - val_categorical_accuracy: 0.6223\n",
      "Epoch 245/600\n",
      " - 0s - loss: 1.0735 - categorical_accuracy: 0.6073 - val_loss: 1.0599 - val_categorical_accuracy: 0.6223\n",
      "Epoch 246/600\n",
      " - 0s - loss: 1.0763 - categorical_accuracy: 0.6060 - val_loss: 1.0600 - val_categorical_accuracy: 0.6295\n",
      "Epoch 247/600\n",
      " - 0s - loss: 1.0731 - categorical_accuracy: 0.6060 - val_loss: 1.0584 - val_categorical_accuracy: 0.6331\n",
      "Epoch 248/600\n",
      " - 0s - loss: 1.0701 - categorical_accuracy: 0.6066 - val_loss: 1.0576 - val_categorical_accuracy: 0.6187\n",
      "Epoch 249/600\n",
      " - 0s - loss: 1.0658 - categorical_accuracy: 0.6028 - val_loss: 1.0582 - val_categorical_accuracy: 0.6187\n",
      "Epoch 250/600\n",
      " - 0s - loss: 1.0715 - categorical_accuracy: 0.6085 - val_loss: 1.0598 - val_categorical_accuracy: 0.6367\n",
      "Epoch 251/600\n",
      " - 0s - loss: 1.0677 - categorical_accuracy: 0.6143 - val_loss: 1.0599 - val_categorical_accuracy: 0.6367\n",
      "Epoch 252/600\n",
      " - 0s - loss: 1.0616 - categorical_accuracy: 0.6155 - val_loss: 1.0582 - val_categorical_accuracy: 0.6187\n",
      "Epoch 253/600\n",
      " - 0s - loss: 1.0643 - categorical_accuracy: 0.6130 - val_loss: 1.0565 - val_categorical_accuracy: 0.6187\n",
      "Epoch 254/600\n",
      " - 0s - loss: 1.0699 - categorical_accuracy: 0.6066 - val_loss: 1.0571 - val_categorical_accuracy: 0.6187\n",
      "Epoch 255/600\n",
      " - 0s - loss: 1.0653 - categorical_accuracy: 0.6130 - val_loss: 1.0570 - val_categorical_accuracy: 0.6295\n",
      "Epoch 256/600\n",
      " - 0s - loss: 1.0683 - categorical_accuracy: 0.6123 - val_loss: 1.0569 - val_categorical_accuracy: 0.6187\n",
      "Epoch 257/600\n",
      " - 0s - loss: 1.0613 - categorical_accuracy: 0.6117 - val_loss: 1.0570 - val_categorical_accuracy: 0.6187\n",
      "Epoch 258/600\n",
      " - 0s - loss: 1.0655 - categorical_accuracy: 0.6117 - val_loss: 1.0574 - val_categorical_accuracy: 0.6151\n",
      "Epoch 259/600\n",
      " - 0s - loss: 1.0591 - categorical_accuracy: 0.6174 - val_loss: 1.0567 - val_categorical_accuracy: 0.6223\n",
      "Epoch 260/600\n",
      " - 0s - loss: 1.0641 - categorical_accuracy: 0.6136 - val_loss: 1.0564 - val_categorical_accuracy: 0.6187\n",
      "Epoch 261/600\n",
      " - 0s - loss: 1.0592 - categorical_accuracy: 0.6092 - val_loss: 1.0562 - val_categorical_accuracy: 0.6151\n",
      "Epoch 262/600\n",
      " - 0s - loss: 1.0552 - categorical_accuracy: 0.6047 - val_loss: 1.0564 - val_categorical_accuracy: 0.6187\n",
      "Epoch 263/600\n",
      " - 0s - loss: 1.0595 - categorical_accuracy: 0.6117 - val_loss: 1.0567 - val_categorical_accuracy: 0.6187\n",
      "Epoch 264/600\n",
      " - 0s - loss: 1.0466 - categorical_accuracy: 0.6194 - val_loss: 1.0562 - val_categorical_accuracy: 0.6151\n",
      "Epoch 265/600\n",
      " - 0s - loss: 1.0500 - categorical_accuracy: 0.6174 - val_loss: 1.0556 - val_categorical_accuracy: 0.6187\n",
      "Epoch 266/600\n",
      " - 0s - loss: 1.0567 - categorical_accuracy: 0.6162 - val_loss: 1.0549 - val_categorical_accuracy: 0.6187\n",
      "Epoch 267/600\n",
      " - 0s - loss: 1.0581 - categorical_accuracy: 0.6098 - val_loss: 1.0542 - val_categorical_accuracy: 0.6151\n",
      "Epoch 268/600\n",
      " - 0s - loss: 1.0579 - categorical_accuracy: 0.6130 - val_loss: 1.0555 - val_categorical_accuracy: 0.6187\n",
      "Epoch 269/600\n",
      " - 0s - loss: 1.0489 - categorical_accuracy: 0.6225 - val_loss: 1.0561 - val_categorical_accuracy: 0.6223\n",
      "Epoch 270/600\n",
      " - 0s - loss: 1.0528 - categorical_accuracy: 0.6264 - val_loss: 1.0567 - val_categorical_accuracy: 0.6187\n",
      "Epoch 271/600\n",
      " - 0s - loss: 1.0495 - categorical_accuracy: 0.6073 - val_loss: 1.0545 - val_categorical_accuracy: 0.6151\n",
      "Epoch 272/600\n",
      " - 0s - loss: 1.0514 - categorical_accuracy: 0.6111 - val_loss: 1.0538 - val_categorical_accuracy: 0.6187\n",
      "Epoch 273/600\n",
      " - 0s - loss: 1.0469 - categorical_accuracy: 0.6028 - val_loss: 1.0547 - val_categorical_accuracy: 0.6223\n",
      "Epoch 274/600\n",
      " - 0s - loss: 1.0537 - categorical_accuracy: 0.6073 - val_loss: 1.0551 - val_categorical_accuracy: 0.6223\n",
      "Epoch 275/600\n",
      " - 0s - loss: 1.0461 - categorical_accuracy: 0.6174 - val_loss: 1.0546 - val_categorical_accuracy: 0.6151\n",
      "Epoch 276/600\n",
      " - 0s - loss: 1.0493 - categorical_accuracy: 0.6085 - val_loss: 1.0548 - val_categorical_accuracy: 0.6151\n",
      "Epoch 277/600\n",
      " - 0s - loss: 1.0478 - categorical_accuracy: 0.6194 - val_loss: 1.0555 - val_categorical_accuracy: 0.6223\n",
      "Epoch 278/600\n",
      " - 0s - loss: 1.0429 - categorical_accuracy: 0.6206 - val_loss: 1.0549 - val_categorical_accuracy: 0.6223\n",
      "Epoch 279/600\n",
      " - 0s - loss: 1.0482 - categorical_accuracy: 0.6155 - val_loss: 1.0528 - val_categorical_accuracy: 0.6151\n",
      "Epoch 280/600\n",
      " - 0s - loss: 1.0411 - categorical_accuracy: 0.6206 - val_loss: 1.0524 - val_categorical_accuracy: 0.6151\n",
      "Epoch 281/600\n",
      " - 0s - loss: 1.0467 - categorical_accuracy: 0.6123 - val_loss: 1.0536 - val_categorical_accuracy: 0.6187\n",
      "Epoch 282/600\n",
      " - 0s - loss: 1.0339 - categorical_accuracy: 0.6232 - val_loss: 1.0546 - val_categorical_accuracy: 0.6151\n",
      "Epoch 283/600\n",
      " - 0s - loss: 1.0374 - categorical_accuracy: 0.6149 - val_loss: 1.0553 - val_categorical_accuracy: 0.6223\n",
      "Epoch 284/600\n",
      " - 0s - loss: 1.0344 - categorical_accuracy: 0.6123 - val_loss: 1.0543 - val_categorical_accuracy: 0.6187\n",
      "Epoch 285/600\n",
      " - 0s - loss: 1.0418 - categorical_accuracy: 0.6060 - val_loss: 1.0531 - val_categorical_accuracy: 0.6151\n",
      "Epoch 286/600\n",
      " - 0s - loss: 1.0424 - categorical_accuracy: 0.6174 - val_loss: 1.0524 - val_categorical_accuracy: 0.6115\n",
      "Epoch 287/600\n",
      " - 0s - loss: 1.0411 - categorical_accuracy: 0.6155 - val_loss: 1.0530 - val_categorical_accuracy: 0.6151\n",
      "Epoch 288/600\n",
      " - 0s - loss: 1.0371 - categorical_accuracy: 0.6244 - val_loss: 1.0535 - val_categorical_accuracy: 0.6187\n",
      "Epoch 289/600\n",
      " - 0s - loss: 1.0350 - categorical_accuracy: 0.6206 - val_loss: 1.0538 - val_categorical_accuracy: 0.6115\n",
      "Epoch 290/600\n",
      " - 0s - loss: 1.0382 - categorical_accuracy: 0.6117 - val_loss: 1.0536 - val_categorical_accuracy: 0.6151\n",
      "Epoch 291/600\n",
      " - 0s - loss: 1.0379 - categorical_accuracy: 0.6111 - val_loss: 1.0529 - val_categorical_accuracy: 0.6187\n",
      "Epoch 292/600\n",
      " - 0s - loss: 1.0323 - categorical_accuracy: 0.6162 - val_loss: 1.0522 - val_categorical_accuracy: 0.6151\n",
      "Epoch 293/600\n",
      " - 0s - loss: 1.0381 - categorical_accuracy: 0.6168 - val_loss: 1.0516 - val_categorical_accuracy: 0.6187\n",
      "Epoch 294/600\n",
      " - 0s - loss: 1.0328 - categorical_accuracy: 0.6149 - val_loss: 1.0518 - val_categorical_accuracy: 0.6115\n",
      "Epoch 295/600\n",
      " - 0s - loss: 1.0380 - categorical_accuracy: 0.6213 - val_loss: 1.0526 - val_categorical_accuracy: 0.6151\n",
      "Epoch 296/600\n",
      " - 0s - loss: 1.0341 - categorical_accuracy: 0.6200 - val_loss: 1.0533 - val_categorical_accuracy: 0.6187\n",
      "Epoch 297/600\n",
      " - 0s - loss: 1.0323 - categorical_accuracy: 0.6117 - val_loss: 1.0533 - val_categorical_accuracy: 0.6151\n",
      "Epoch 298/600\n",
      " - 0s - loss: 1.0309 - categorical_accuracy: 0.6232 - val_loss: 1.0540 - val_categorical_accuracy: 0.6151\n",
      "Epoch 299/600\n",
      " - 0s - loss: 1.0287 - categorical_accuracy: 0.6143 - val_loss: 1.0535 - val_categorical_accuracy: 0.6187\n",
      "Epoch 300/600\n",
      " - 0s - loss: 1.0254 - categorical_accuracy: 0.6194 - val_loss: 1.0528 - val_categorical_accuracy: 0.6187\n",
      "Epoch 301/600\n",
      " - 0s - loss: 1.0362 - categorical_accuracy: 0.6181 - val_loss: 1.0524 - val_categorical_accuracy: 0.6223\n",
      "Epoch 302/600\n",
      " - 0s - loss: 1.0282 - categorical_accuracy: 0.6155 - val_loss: 1.0521 - val_categorical_accuracy: 0.6079\n",
      "Epoch 303/600\n",
      " - 0s - loss: 1.0329 - categorical_accuracy: 0.6257 - val_loss: 1.0519 - val_categorical_accuracy: 0.6187\n",
      "Epoch 304/600\n",
      " - 0s - loss: 1.0319 - categorical_accuracy: 0.6200 - val_loss: 1.0509 - val_categorical_accuracy: 0.6187\n",
      "Epoch 305/600\n",
      " - 0s - loss: 1.0269 - categorical_accuracy: 0.6238 - val_loss: 1.0516 - val_categorical_accuracy: 0.6187\n",
      "Epoch 306/600\n",
      " - 0s - loss: 1.0309 - categorical_accuracy: 0.6200 - val_loss: 1.0530 - val_categorical_accuracy: 0.6187\n",
      "Epoch 307/600\n",
      " - 0s - loss: 1.0286 - categorical_accuracy: 0.6174 - val_loss: 1.0532 - val_categorical_accuracy: 0.6151\n",
      "Epoch 308/600\n",
      " - 0s - loss: 1.0324 - categorical_accuracy: 0.6257 - val_loss: 1.0524 - val_categorical_accuracy: 0.6187\n",
      "Epoch 309/600\n",
      " - 0s - loss: 1.0240 - categorical_accuracy: 0.6213 - val_loss: 1.0522 - val_categorical_accuracy: 0.6151\n",
      "Epoch 310/600\n",
      " - 0s - loss: 1.0295 - categorical_accuracy: 0.6289 - val_loss: 1.0518 - val_categorical_accuracy: 0.6115\n",
      "Epoch 311/600\n",
      " - 0s - loss: 1.0176 - categorical_accuracy: 0.6187 - val_loss: 1.0509 - val_categorical_accuracy: 0.6115\n",
      "Epoch 312/600\n",
      " - 0s - loss: 1.0206 - categorical_accuracy: 0.6289 - val_loss: 1.0518 - val_categorical_accuracy: 0.6187\n",
      "Epoch 313/600\n",
      " - 0s - loss: 1.0242 - categorical_accuracy: 0.6232 - val_loss: 1.0530 - val_categorical_accuracy: 0.6151\n",
      "Epoch 314/600\n",
      " - 0s - loss: 1.0312 - categorical_accuracy: 0.6149 - val_loss: 1.0535 - val_categorical_accuracy: 0.6115\n",
      "Epoch 315/600\n",
      " - 0s - loss: 1.0164 - categorical_accuracy: 0.6251 - val_loss: 1.0517 - val_categorical_accuracy: 0.6151\n",
      "Epoch 316/600\n",
      " - 0s - loss: 1.0218 - categorical_accuracy: 0.6123 - val_loss: 1.0512 - val_categorical_accuracy: 0.6151\n",
      "Epoch 317/600\n",
      " - 0s - loss: 1.0190 - categorical_accuracy: 0.6295 - val_loss: 1.0516 - val_categorical_accuracy: 0.6115\n",
      "Epoch 318/600\n",
      " - 0s - loss: 1.0216 - categorical_accuracy: 0.6238 - val_loss: 1.0525 - val_categorical_accuracy: 0.6115\n",
      "Epoch 319/600\n",
      " - 0s - loss: 1.0194 - categorical_accuracy: 0.6200 - val_loss: 1.0528 - val_categorical_accuracy: 0.6187\n",
      "Epoch 320/600\n",
      " - 0s - loss: 1.0197 - categorical_accuracy: 0.6340 - val_loss: 1.0523 - val_categorical_accuracy: 0.6151\n",
      "Epoch 321/600\n",
      " - 0s - loss: 1.0172 - categorical_accuracy: 0.6187 - val_loss: 1.0518 - val_categorical_accuracy: 0.6187\n",
      "Epoch 322/600\n",
      " - 0s - loss: 1.0193 - categorical_accuracy: 0.6238 - val_loss: 1.0525 - val_categorical_accuracy: 0.6115\n",
      "Epoch 323/600\n",
      " - 0s - loss: 1.0113 - categorical_accuracy: 0.6308 - val_loss: 1.0530 - val_categorical_accuracy: 0.6151\n",
      "Epoch 324/600\n",
      " - 0s - loss: 1.0170 - categorical_accuracy: 0.6276 - val_loss: 1.0537 - val_categorical_accuracy: 0.6151\n",
      "Epoch 325/600\n",
      " - 0s - loss: 1.0133 - categorical_accuracy: 0.6327 - val_loss: 1.0525 - val_categorical_accuracy: 0.6115\n",
      "Epoch 326/600\n",
      " - 0s - loss: 1.0159 - categorical_accuracy: 0.6225 - val_loss: 1.0514 - val_categorical_accuracy: 0.6115\n",
      "Epoch 327/600\n",
      " - 0s - loss: 1.0125 - categorical_accuracy: 0.6257 - val_loss: 1.0510 - val_categorical_accuracy: 0.6115\n",
      "Epoch 328/600\n",
      " - 0s - loss: 1.0168 - categorical_accuracy: 0.6219 - val_loss: 1.0516 - val_categorical_accuracy: 0.6151\n",
      "Epoch 329/600\n",
      " - 0s - loss: 1.0081 - categorical_accuracy: 0.6302 - val_loss: 1.0538 - val_categorical_accuracy: 0.6151\n",
      "Epoch 330/600\n",
      " - 0s - loss: 1.0127 - categorical_accuracy: 0.6327 - val_loss: 1.0547 - val_categorical_accuracy: 0.6187\n",
      "Epoch 331/600\n",
      " - 0s - loss: 1.0041 - categorical_accuracy: 0.6340 - val_loss: 1.0545 - val_categorical_accuracy: 0.6187\n",
      "Epoch 332/600\n",
      " - 0s - loss: 1.0115 - categorical_accuracy: 0.6264 - val_loss: 1.0524 - val_categorical_accuracy: 0.6115\n",
      "Epoch 333/600\n",
      " - 0s - loss: 1.0143 - categorical_accuracy: 0.6225 - val_loss: 1.0513 - val_categorical_accuracy: 0.6115\n",
      "Epoch 334/600\n",
      " - 0s - loss: 1.0066 - categorical_accuracy: 0.6283 - val_loss: 1.0514 - val_categorical_accuracy: 0.6187\n",
      "Epoch 335/600\n",
      " - 0s - loss: 1.0099 - categorical_accuracy: 0.6238 - val_loss: 1.0520 - val_categorical_accuracy: 0.6151\n",
      "Epoch 336/600\n",
      " - 0s - loss: 1.0060 - categorical_accuracy: 0.6384 - val_loss: 1.0532 - val_categorical_accuracy: 0.6187\n",
      "Epoch 337/600\n",
      " - 0s - loss: 1.0093 - categorical_accuracy: 0.6327 - val_loss: 1.0542 - val_categorical_accuracy: 0.6151\n",
      "Epoch 338/600\n",
      " - 0s - loss: 1.0168 - categorical_accuracy: 0.6206 - val_loss: 1.0533 - val_categorical_accuracy: 0.6187\n",
      "Epoch 339/600\n",
      " - 0s - loss: 1.0085 - categorical_accuracy: 0.6327 - val_loss: 1.0520 - val_categorical_accuracy: 0.6151\n",
      "Epoch 340/600\n",
      " - 0s - loss: 1.0063 - categorical_accuracy: 0.6181 - val_loss: 1.0511 - val_categorical_accuracy: 0.6151\n",
      "Epoch 341/600\n",
      " - 0s - loss: 1.0110 - categorical_accuracy: 0.6276 - val_loss: 1.0499 - val_categorical_accuracy: 0.6187\n",
      "Epoch 342/600\n",
      " - 0s - loss: 1.0068 - categorical_accuracy: 0.6264 - val_loss: 1.0492 - val_categorical_accuracy: 0.6115\n",
      "Epoch 343/600\n",
      " - 0s - loss: 1.0071 - categorical_accuracy: 0.6270 - val_loss: 1.0512 - val_categorical_accuracy: 0.6151\n",
      "Epoch 344/600\n",
      " - 0s - loss: 1.0070 - categorical_accuracy: 0.6283 - val_loss: 1.0540 - val_categorical_accuracy: 0.6151\n",
      "Epoch 345/600\n",
      " - 0s - loss: 1.0057 - categorical_accuracy: 0.6359 - val_loss: 1.0547 - val_categorical_accuracy: 0.6187\n",
      "Epoch 346/600\n",
      " - 0s - loss: 1.0037 - categorical_accuracy: 0.6289 - val_loss: 1.0535 - val_categorical_accuracy: 0.6187\n",
      "Epoch 347/600\n",
      " - 0s - loss: 0.9992 - categorical_accuracy: 0.6295 - val_loss: 1.0506 - val_categorical_accuracy: 0.6151\n",
      "Epoch 348/600\n",
      " - 0s - loss: 0.9971 - categorical_accuracy: 0.6334 - val_loss: 1.0506 - val_categorical_accuracy: 0.6115\n",
      "Epoch 349/600\n",
      " - 0s - loss: 1.0006 - categorical_accuracy: 0.6283 - val_loss: 1.0493 - val_categorical_accuracy: 0.6115\n",
      "Epoch 350/600\n",
      " - 0s - loss: 1.0027 - categorical_accuracy: 0.6314 - val_loss: 1.0518 - val_categorical_accuracy: 0.6115\n",
      "Epoch 351/600\n",
      " - 0s - loss: 1.0066 - categorical_accuracy: 0.6270 - val_loss: 1.0555 - val_categorical_accuracy: 0.6187\n",
      "Epoch 352/600\n",
      " - 0s - loss: 1.0135 - categorical_accuracy: 0.6225 - val_loss: 1.0562 - val_categorical_accuracy: 0.6151\n",
      "Epoch 353/600\n",
      " - 0s - loss: 1.0037 - categorical_accuracy: 0.6353 - val_loss: 1.0544 - val_categorical_accuracy: 0.6151\n",
      "Epoch 354/600\n",
      " - 0s - loss: 0.9908 - categorical_accuracy: 0.6353 - val_loss: 1.0518 - val_categorical_accuracy: 0.6187\n",
      "Epoch 355/600\n",
      " - 0s - loss: 1.0075 - categorical_accuracy: 0.6442 - val_loss: 1.0504 - val_categorical_accuracy: 0.6187\n",
      "Epoch 356/600\n",
      " - 0s - loss: 0.9966 - categorical_accuracy: 0.6251 - val_loss: 1.0516 - val_categorical_accuracy: 0.6115\n",
      "Epoch 357/600\n",
      " - 0s - loss: 1.0032 - categorical_accuracy: 0.6219 - val_loss: 1.0532 - val_categorical_accuracy: 0.6151\n",
      "Epoch 358/600\n",
      " - 0s - loss: 1.0009 - categorical_accuracy: 0.6334 - val_loss: 1.0531 - val_categorical_accuracy: 0.6151\n",
      "Epoch 359/600\n",
      " - 0s - loss: 1.0044 - categorical_accuracy: 0.6276 - val_loss: 1.0536 - val_categorical_accuracy: 0.6151\n",
      "Epoch 360/600\n",
      " - 0s - loss: 0.9888 - categorical_accuracy: 0.6416 - val_loss: 1.0528 - val_categorical_accuracy: 0.6115\n",
      "Epoch 361/600\n",
      " - 0s - loss: 0.9953 - categorical_accuracy: 0.6257 - val_loss: 1.0527 - val_categorical_accuracy: 0.6151\n",
      "Epoch 362/600\n",
      " - 0s - loss: 0.9994 - categorical_accuracy: 0.6334 - val_loss: 1.0538 - val_categorical_accuracy: 0.6187\n",
      "Epoch 363/600\n",
      " - 0s - loss: 0.9898 - categorical_accuracy: 0.6321 - val_loss: 1.0539 - val_categorical_accuracy: 0.6151\n",
      "Epoch 364/600\n",
      " - 0s - loss: 0.9943 - categorical_accuracy: 0.6365 - val_loss: 1.0530 - val_categorical_accuracy: 0.6187\n",
      "Epoch 365/600\n",
      " - 0s - loss: 0.9859 - categorical_accuracy: 0.6391 - val_loss: 1.0520 - val_categorical_accuracy: 0.6115\n",
      "Epoch 366/600\n",
      " - 0s - loss: 0.9908 - categorical_accuracy: 0.6346 - val_loss: 1.0521 - val_categorical_accuracy: 0.6187\n",
      "Epoch 367/600\n",
      " - 0s - loss: 0.9981 - categorical_accuracy: 0.6232 - val_loss: 1.0531 - val_categorical_accuracy: 0.6187\n",
      "Epoch 368/600\n",
      " - 0s - loss: 0.9800 - categorical_accuracy: 0.6429 - val_loss: 1.0552 - val_categorical_accuracy: 0.6115\n",
      "Epoch 369/600\n",
      " - 0s - loss: 0.9909 - categorical_accuracy: 0.6302 - val_loss: 1.0545 - val_categorical_accuracy: 0.6187\n",
      "Epoch 370/600\n",
      " - 0s - loss: 0.9892 - categorical_accuracy: 0.6276 - val_loss: 1.0540 - val_categorical_accuracy: 0.6115\n",
      "Epoch 371/600\n",
      " - 0s - loss: 0.9920 - categorical_accuracy: 0.6321 - val_loss: 1.0541 - val_categorical_accuracy: 0.6115\n",
      "Epoch 372/600\n",
      " - 0s - loss: 0.9935 - categorical_accuracy: 0.6378 - val_loss: 1.0545 - val_categorical_accuracy: 0.6043\n",
      "Epoch 373/600\n",
      " - 0s - loss: 0.9907 - categorical_accuracy: 0.6314 - val_loss: 1.0529 - val_categorical_accuracy: 0.6151\n",
      "Epoch 374/600\n",
      " - 0s - loss: 0.9812 - categorical_accuracy: 0.6416 - val_loss: 1.0529 - val_categorical_accuracy: 0.6115\n",
      "Epoch 375/600\n",
      " - 0s - loss: 0.9841 - categorical_accuracy: 0.6346 - val_loss: 1.0537 - val_categorical_accuracy: 0.6151\n",
      "Epoch 376/600\n",
      " - 0s - loss: 0.9917 - categorical_accuracy: 0.6391 - val_loss: 1.0547 - val_categorical_accuracy: 0.6115\n",
      "Epoch 377/600\n",
      " - 0s - loss: 0.9892 - categorical_accuracy: 0.6314 - val_loss: 1.0528 - val_categorical_accuracy: 0.6115\n",
      "Epoch 378/600\n",
      " - 0s - loss: 0.9873 - categorical_accuracy: 0.6384 - val_loss: 1.0525 - val_categorical_accuracy: 0.6115\n",
      "Epoch 379/600\n",
      " - 0s - loss: 0.9878 - categorical_accuracy: 0.6384 - val_loss: 1.0531 - val_categorical_accuracy: 0.6115\n",
      "Epoch 380/600\n",
      " - 0s - loss: 0.9842 - categorical_accuracy: 0.6378 - val_loss: 1.0543 - val_categorical_accuracy: 0.6115\n",
      "Epoch 381/600\n",
      " - 0s - loss: 0.9768 - categorical_accuracy: 0.6410 - val_loss: 1.0544 - val_categorical_accuracy: 0.6079\n",
      "Epoch 382/600\n",
      " - 0s - loss: 0.9859 - categorical_accuracy: 0.6404 - val_loss: 1.0541 - val_categorical_accuracy: 0.6115\n",
      "Epoch 383/600\n",
      " - 0s - loss: 0.9910 - categorical_accuracy: 0.6353 - val_loss: 1.0541 - val_categorical_accuracy: 0.6115\n",
      "Epoch 384/600\n",
      " - 0s - loss: 0.9841 - categorical_accuracy: 0.6391 - val_loss: 1.0534 - val_categorical_accuracy: 0.6007\n",
      "Epoch 385/600\n",
      " - 0s - loss: 0.9865 - categorical_accuracy: 0.6397 - val_loss: 1.0524 - val_categorical_accuracy: 0.6115\n",
      "Epoch 386/600\n",
      " - 0s - loss: 0.9787 - categorical_accuracy: 0.6334 - val_loss: 1.0528 - val_categorical_accuracy: 0.6115\n",
      "Epoch 387/600\n",
      " - 0s - loss: 0.9825 - categorical_accuracy: 0.6308 - val_loss: 1.0546 - val_categorical_accuracy: 0.6115\n",
      "Epoch 388/600\n",
      " - 0s - loss: 0.9757 - categorical_accuracy: 0.6334 - val_loss: 1.0551 - val_categorical_accuracy: 0.6115\n",
      "Epoch 389/600\n",
      " - 0s - loss: 0.9785 - categorical_accuracy: 0.6334 - val_loss: 1.0549 - val_categorical_accuracy: 0.6079\n",
      "Epoch 390/600\n",
      " - 0s - loss: 0.9859 - categorical_accuracy: 0.6372 - val_loss: 1.0533 - val_categorical_accuracy: 0.6079\n",
      "Epoch 391/600\n",
      " - 0s - loss: 0.9699 - categorical_accuracy: 0.6378 - val_loss: 1.0532 - val_categorical_accuracy: 0.6079\n",
      "Epoch 392/600\n",
      " - 0s - loss: 0.9790 - categorical_accuracy: 0.6391 - val_loss: 1.0540 - val_categorical_accuracy: 0.6115\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00392: early stopping\n"
     ]
    }
   ],
   "source": [
    "#Splitting data into training and validation \n",
    "X_tra, X_val, y_tra, y_val = train_test_split(train_data.reshape(r,c,1),y_train , train_size=0.85)\n",
    "#train a single  model\n",
    "earlyStopping = EarlyStopping(monitor='val_loss', patience=50,min_delta=1e-5,restore_best_weights=True ,verbose=2, mode='min')\n",
    "modelfastText.compile(loss='categorical_crossentropy',\n",
    "                    optimizer='adam',\n",
    "                    metrics=['categorical_accuracy'])\n",
    "    \n",
    "hist = modelfastText.fit(X_tra, y_tra, batch_size=2048, epochs=600, validation_data=(X_val, y_val),callbacks=[earlyStopping],verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "lZsNyIUTQZah",
    "outputId": "68a01035-d197-4410-8c05-8c99b18106da"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation loss of best epoch\n",
      "1.0491794188131691\n"
     ]
    }
   ],
   "source": [
    "print(\"validation loss of best epoch\")\n",
    "print(  log_loss(y_val, modelfastText.predict_proba(X_val) ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 137,
     "referenced_widgets": [
      "761936f1bb1447bf9062ff97f3511293",
      "1593d0a1cb394b1aa60881f9c5f3397a",
      "46547a03c06443e79de78bc74fe1ee27",
      "8bc4f47a871b4b73b5020c49c272da17",
      "3b1eb411542044b487c566f4a846decd",
      "eaf8cf34468d4aeda1a4a645caa67d01",
      "cebc2c79376f4cbdb986483cc9cde515",
      "9fe91718136b438e9629c5358b1d1520"
     ]
    },
    "colab_type": "code",
    "id": "0FFPYPKkfPZy",
    "outputId": "9cd7a27f-aa2a-40a6-8e9f-208bc6d32daa"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "761936f1bb1447bf9062ff97f3511293",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Large dropout rate: 0.6 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.6 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.6 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.6 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#blending multiple FastText models to predict test labels\n",
    "n = 20\n",
    "preds = []\n",
    "earlyStopping = EarlyStopping(monitor='val_loss', patience=100,min_delta=1e-5,restore_best_weights=True ,verbose=0, mode='min')\n",
    "\n",
    "for i in tqdm_notebook(range(n)):\n",
    "    model = Sequential()\n",
    "    m = np.random.choice([64,32,50,100,20,300])\n",
    "    \n",
    "    model.add(Conv1D(m, 4, activation='relu', padding='valid',input_shape=(300,1)))\n",
    "    drop = np.random.choice([0.2,0.4,0.6])\n",
    "    model.add(Dropout(drop))\n",
    "    model.add(Flatten())\n",
    "   \n",
    "    model.add(Dense(8, activation='softmax'))  #multi-label (k-hot encoding)\n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "    \n",
    "    X_tra, X_val, y_tra, y_val = train_test_split(train_data.reshape(1849,300,1),y_train , train_size=0.80)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                    optimizer='adam',\n",
    "                    metrics=['categorical_accuracy'])\n",
    "\n",
    "\n",
    "    \n",
    "    hist = model.fit(X_tra, y_tra, batch_size=1024, epochs=600, validation_data=(X_val, y_val),callbacks=[earlyStopping],verbose=0)\n",
    "    y_test = model.predict_proba(np.array(test_data).reshape(560,300,1)) \n",
    "    \n",
    "    preds.append(y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 347
    },
    "colab_type": "code",
    "id": "a9kIkYLRatkS",
    "outputId": "a332b14c-8097-4035-f5d1-1f7d1ffed25f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>business/finance</th>\n",
       "      <th>education/research</th>\n",
       "      <th>entertainment</th>\n",
       "      <th>health/medical</th>\n",
       "      <th>news/press</th>\n",
       "      <th>politics/government/law</th>\n",
       "      <th>sports</th>\n",
       "      <th>tech/science</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27997</td>\n",
       "      <td>0.180167</td>\n",
       "      <td>0.457875</td>\n",
       "      <td>0.222623</td>\n",
       "      <td>0.000149</td>\n",
       "      <td>0.028417</td>\n",
       "      <td>0.019923</td>\n",
       "      <td>0.025170</td>\n",
       "      <td>0.065675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9316</td>\n",
       "      <td>0.233947</td>\n",
       "      <td>0.016154</td>\n",
       "      <td>0.568829</td>\n",
       "      <td>0.000108</td>\n",
       "      <td>0.076303</td>\n",
       "      <td>0.001297</td>\n",
       "      <td>0.053943</td>\n",
       "      <td>0.049419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27045</td>\n",
       "      <td>0.621927</td>\n",
       "      <td>0.002541</td>\n",
       "      <td>0.338860</td>\n",
       "      <td>0.007097</td>\n",
       "      <td>0.018335</td>\n",
       "      <td>0.000577</td>\n",
       "      <td>0.005273</td>\n",
       "      <td>0.005391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19805</td>\n",
       "      <td>0.038542</td>\n",
       "      <td>0.021738</td>\n",
       "      <td>0.008269</td>\n",
       "      <td>0.895884</td>\n",
       "      <td>0.004803</td>\n",
       "      <td>0.020098</td>\n",
       "      <td>0.001076</td>\n",
       "      <td>0.009590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>26580</td>\n",
       "      <td>0.061945</td>\n",
       "      <td>0.287197</td>\n",
       "      <td>0.189942</td>\n",
       "      <td>0.020335</td>\n",
       "      <td>0.096425</td>\n",
       "      <td>0.119118</td>\n",
       "      <td>0.003975</td>\n",
       "      <td>0.221063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>16886</td>\n",
       "      <td>0.074311</td>\n",
       "      <td>0.010211</td>\n",
       "      <td>0.815029</td>\n",
       "      <td>0.005123</td>\n",
       "      <td>0.050438</td>\n",
       "      <td>0.008206</td>\n",
       "      <td>0.011435</td>\n",
       "      <td>0.025248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>27931</td>\n",
       "      <td>0.163313</td>\n",
       "      <td>0.010612</td>\n",
       "      <td>0.745544</td>\n",
       "      <td>0.001582</td>\n",
       "      <td>0.038148</td>\n",
       "      <td>0.027122</td>\n",
       "      <td>0.009809</td>\n",
       "      <td>0.003871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8977</td>\n",
       "      <td>0.153492</td>\n",
       "      <td>0.294532</td>\n",
       "      <td>0.269260</td>\n",
       "      <td>0.082540</td>\n",
       "      <td>0.027831</td>\n",
       "      <td>0.039561</td>\n",
       "      <td>0.010942</td>\n",
       "      <td>0.121842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>20285</td>\n",
       "      <td>0.005407</td>\n",
       "      <td>0.015207</td>\n",
       "      <td>0.083071</td>\n",
       "      <td>0.000776</td>\n",
       "      <td>0.013367</td>\n",
       "      <td>0.011486</td>\n",
       "      <td>0.867571</td>\n",
       "      <td>0.003115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2427</td>\n",
       "      <td>0.116083</td>\n",
       "      <td>0.433328</td>\n",
       "      <td>0.190309</td>\n",
       "      <td>0.011710</td>\n",
       "      <td>0.025405</td>\n",
       "      <td>0.097031</td>\n",
       "      <td>0.006869</td>\n",
       "      <td>0.119264</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Host  business/finance  ...    sports  tech/science\n",
       "0  27997          0.180167  ...  0.025170      0.065675\n",
       "1   9316          0.233947  ...  0.053943      0.049419\n",
       "2  27045          0.621927  ...  0.005273      0.005391\n",
       "3  19805          0.038542  ...  0.001076      0.009590\n",
       "4  26580          0.061945  ...  0.003975      0.221063\n",
       "5  16886          0.074311  ...  0.011435      0.025248\n",
       "6  27931          0.163313  ...  0.009809      0.003871\n",
       "7   8977          0.153492  ...  0.010942      0.121842\n",
       "8  20285          0.005407  ...  0.867571      0.003115\n",
       "9   2427          0.116083  ...  0.006869      0.119264\n",
       "\n",
       "[10 rows x 9 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#predicting test labels : \n",
    "fasttext_df = pd.read_csv(\"./data/graph_baseline.csv\")\n",
    "y_test = np.mean(preds,axis=0)\n",
    "fasttext_df.loc[:,fasttext_df.columns[1:]] = y_test\n",
    "fasttext_df.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 500
    },
    "colab_type": "code",
    "id": "B7jzzKdOoMLA",
    "outputId": "f7961cb8-f13a-4b3e-8ae9-fdc0f9904505"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAHjCAYAAAAg4dmtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeUDVVf7/8edh3xRFNNcyzQ1QQXA3\nEUXNsRpzdLKsscYWs8x0Wqdf1rfFcdLSss2s0Wmblsm0cpsUl0zEBU3FfUnNXTAXNoX7+f1B3iRA\nAbl84PJ6/OXlfpb3PWK9Obw+5xjLsixERERERKo4D7sLEBERERGpCNQYi4iIiIigxlhEREREBFBj\nLCIiIiICqDEWEREREQHUGIuIiIiIAGqMRcTN3XXXXcTHx9tdRgE9evTgnnvuKfJ1WXruuee47rrr\ninxd1irqmIuIXI4aYxGptFJTU3n88cdp0aIFfn5+1KlTh+7du/PBBx+Qk5Njd3klMmvWLF599dVi\nHfvzzz9jjGHp0qXFOv7RRx9l1apVV1Bd4T766COMMQW+/tprr/HFF1+U+f1ERFzNy+4CRERK48CB\nA3Tr1g0vLy+ef/55oqKi8Pb2ZuXKlUyaNIk2bdoQGRlpd5nFFhISUubXdDgcWJZFUFAQQUFBZX79\nogQHB5fbvUREypJmjEWkUho5ciTZ2dkkJyczdOhQwsLCaNasGcOGDWPdunU0a9as0POSk5Pp168f\nderUISgoiPbt27NgwYJ8x8yZM4eoqCgCAgKoUaMGHTp0YP369QCcP3+esWPH0rBhQ3x9falXrx5D\nhgy5ZK379u3jhhtuwN/fn0aNGjF16tQCx/w+SrFixQq6du1KtWrVqFatGm3btmXhwoUANGrUCIC4\nuDiMMTRu3Bj4LSLx2Wef0bJlS3x8fNixY0eR0YlPPvmEJk2a4OfnR+/evfnpp5+c7xV2zooVKzDG\n8NNPP7F06VLuvPNOAIwxGGO46667gIJRCsuymDRpEk2aNMHHx4emTZsyZcqUfNdu3Lgx48aNY/To\n0YSEhHDVVVcxZsyYfDP/lxoTEZGyoBljEal00tLSmDdvHv/3f/9X6Oykt7c33t7ehZ57+vRpbr31\nViZNmoS3tzcffPABN998M5s3b6Z58+YcOXKEwYMH8+KLLzJ48GCysrJYv349Xl55/7mcOnUqn3/+\nOR999BFNmjTh6NGj/PDDD0XWalkWt9xyC56enixduhRfX18ee+wxkpOTi8z55uTkcPPNN3PXXXcx\nc+ZMADZv3kxAQACQ19y3a9eOL7/8ki5duuDp6ek899ChQ7z11lv8+9//pmbNmtSrV6/Qexw+fJi3\n3nqLzz//HMuyeOihhxg4cCDr1q0rNB7xe126dOGNN97goYce4vDhwwD4+/sXeuxbb73FM888w2uv\nvUZcXByLFy/mkUceoVq1agwfPtx53NSpU3niiSdISkpi/fr1DB06lIiICIYPH37ZMRERKQtqjEWk\n0tm1axcOh4OwsLASn9ujR498r1988UW++eYbvvjiC55++mkOHz7M+fPn+fOf/+yciW3VqpXz+H37\n9tG8eXNiY2MxxnD11VfTvn37Iu+3ePFi1q9fz/bt22nevDmQN1N79dVXF3nOmTNnOHnyJDfffLNz\n5vviGfDatWsDefGLunXr5js3KyuLDz/88JLXB8jIyGDmzJnO5vzDDz+kRYsWJCQk0KtXr0ueC+Dj\n4+P8oeT3NfzehAkTGDVqFPfdd5/zs2zfvp2XXnopX2N8/fXX8+STTzqPmTFjBosWLWL48OGXHRMR\nkbKgKIWIVDqWZZX63OPHjzNy5EhatmxJjRo1CAoKIiUlhX379gHQpk0b+vbtS0REBLfccguvvfYa\nBw4ccJ5/9913s2nTJq677jpGjBjBl19+yblz54q835YtWwgNDXU2xZDX2LZo0aLIc2rWrMk999xD\n37596devHxMmTGD79u3F+nxXXXXVZZviCzVcPGPdvHlzQkNDSUlJKdZ9iuv06dP8/PPPdO/ePd/X\nY2Nj+emnn8jIyHB+7feZ8Pr163P06FHgysZERKS41BiLSKXTrFkzPDw82LJlS4nPveuuu/j+++95\n+eWX+f7779mwYQORkZHO5tbT05P58+eTkJBA+/bt+fLLL2nevDnffvstkNe87d27l0mTJuHj48Po\n0aOJjIzk9OnTZfoZp0+fzrp16+jduzfLli0jIiKCadOmXfa8wMDAMrm/h4dHgR9Azp8/XybXLoqP\nj0++18YYHA6H83Vpx0REpLjUGItIpRMSEkK/fv144403OHXqVIH3z58/T3p6eqHnLl++nJEjR3Lz\nzTfTunVr6tWrx549e/IdY4yhQ4cO/P3vf2f58uXExsYyY8YM5/tBQUHccsstvP7666xdu5atW7ey\nbNmyQu8XFhbGiRMn2Llzp/NrJ06cKNZsZ0REBGPHjmX+/PkMHz6cd999F/itgczNzb3sNYpy/Phx\ndu/e7Xy9Y8cOTpw44Yyn1KlTh2PHjuW7R3Jycr5rFKeO6tWr07BhQ5YvX57v68uWLePaa68tcUa4\nqDERESkLaoxFpFJ666238Pb2Jjo6mk8++YQtW7awa9cuPvroI2JiYvI1ohdr0aIFH3/8MZs2bWLD\nhg3cdttt+Rq7lStX8sILL5CUlMT+/ftZvHgxGzdudDaMEydO5OOPPyYlJYW9e/fyr3/9C09Pz3xR\niYv16tWLtm3bcscdd7B69Wo2bNjA0KFDi3w4EPIy1E888QQrVqxg3759JCYm8v333ztrCA0NJSgo\niP/9738cOXKEkydPlnj8AgICuPvuu1m7di1r165l2LBhREZGOvPFcXFxZGRkMG7cOHbv3s0XX3zB\nm2++me8a1157LQBff/01x48f5+zZs4Xe66mnnmLq1KlMnz6dnTt3Mm3aNN5++23+/ve/F7vey42J\niEhZ0MN3IlIpXX311SQnJ/PPf/6T5557jv3791O9enVatWrFY489RkRERKHnzZgxg/vvv58OHTpw\n1VVX8fjjj+fLuQYHB5OYmMibb77JyZMnqVu3LkOHDuWZZ54B8mZAX331VXbu3InD4aBVq1Z8+eWX\nRWaGjTHMnj2b++67j+7duxMaGspjjz1GdnZ2kZ8tMDCQnTt3MmTIEI4fP06tWrXo378/kyZNAvJi\nDm+++SbPPvssr7zyCg0bNsy31Fpx1KtXj/vuu49BgwZx5MgRunbtyieffOJckaJFixZMnz6dF198\nkVdffZUePXowfvx4brvtNuc12rdvz+jRo7n//vs5fvw4w4YNc64YcbEHHniA9PR0xo8fz8iRI2nU\nqBETJkzI9+Dd5VxuTEREyoKxruQpFhERERERN6EohYiIiIgIaoxFRERERAA1xiIiIiIigBpjERER\nERFAjbGIiIiICKDGWEREREQEqEDrGB86dMiW+4aGhnLixAlb7u3ONK6uo7F1HY2t62hsXUdj6zoa\nW9exc2zr169f5HuaMRYRERERQY2xiIiIiAigxlhEREREBKhAGWMRERG5MpZlkZWVhcPhwBhjdzmV\n3tGjR8nOzra7DLfk6rG1LAsPDw/8/PxK9G9BjbGIiIibyMrKwtvbGy8v/e+9LHh5eeHp6Wl3GW6p\nPMY2JyeHrKws/P39i32OohQiIiJuwuFwqCkW+ZWXlxcOh6NE56gxFhERcROKT4jkV9J/E2qMRURE\npEykpaXRu3dvevfuTWRkJNHR0c7X586dK9Y1xowZw65du664lujoaE6dOnXF17mcAQMGsHnzZgBu\nv/12zp49W+Sx06ZNIysrq8j3L3z2nJwcWrVqVaI6Nm3axJIlS5yv58+fz9tvv12ia4gyxiIiIlJG\nQkJC+O677wB45ZVXCAwMZMSIEfmOsSzL+WBUYSZPnuzyOl3lk08+ueT77777LkOGDMHPz6/Ae7m5\nuc7PnpOTU+J7b9q0iW3bthEXFwdAv379SnwN0YyxiIiIuNjevXvp0aMHDz30EHFxcRw9epTHH3+c\nfv36ERcXl68ZvjADe2HWdPz48cTHx3PTTTcVulPa2bNnGT16NPHx8cTHx7NgwYICxwwbNowbbriB\nuLg4Z/Oak5PDqFGj6NWrFz179uT9998H8prXHj16EB8fz8iRIwtcKzMzk/vuu4/Y2FjuvffefCsr\nXJilPnv2LHfccQfx8fH07NmTb7/9lnfffZfU1FQGDhzIrbfe6vx848aNIz4+nvXr1+ebfQZ45pln\niIuLY8iQIaSlpeUbH4Bjx47RtWtXMjMzmTx5Ml999RW9e/fm22+/5ZNPPmHcuHEA7N+/n0GDBhEf\nH8+QIUOcuw2PGjWKcePGcfPNN9O5c2fmz59fsr9YN6QZYxERETfk+HQ61oG9ZXpN0+haPIbcW6pz\nd+3axWuvvUbbtm0BeOqpp6hZsyY5OTkMHjyY/v3707x583znnD59mk6dOvH3v/+d5557jk8//ZSH\nHnoo3zGvvPIKtWrVYtGiRViWVWh8YsqUKdSsWZPMzEz69evHH/7wB/bs2cPJkydZvHgxgPO8t99+\nm6SkJHx8fEhPTy9wrRkzZlCjRg2WLVvG5s2bC52ZXbx4MQ0bNuSjjz5yfo7q1aszbdo0Zs2aRXBw\nMDk5Oc7P9/zzzxe4xunTp4mJieGFF15g4sSJTJkypdDjAPz9/RkzZgzbtm1zHnPx7PXf//53br/9\ndgYOHMhHH33Es88+y/Tp0wE4ceIEc+bMYevWrYwYMaLKzzRrxlhERERc7pprrnE2xQBz5syhb9++\n3HDDDezcuZMdO3YUOMfPz4+ePXsC0KZNGw4cOFDgmO+//5677roLyHvQqkaNGgWOmT59unPW+fDh\nw+zbt4/GjRuze/dunnnmGZYuXUr16tUBaN68OaNGjWLWrFmFrvCxatUqBg4cCEBERAQtWrQocExY\nWBhLly5l/PjxrFmzxnnt3/Px8SmyEfXy8uKmm24CYODAgaxZs6bQ44pj/fr1/PGPfwRg0KBBrF69\n2vle3759McYQFhbGkSNHSn0Pd6EZYxERETdU2pldVwkICHD+ec+ePbz33nvMnTuX4OBgRo0aVehm\nDz4+Ps4/e3p6kpubW+L7Ll++nKSkJL755hv8/f0ZMGAA2dnZhISEsGjRIhISEpg5cybz5s3j5Zdf\n5pNPPiExMZH//e9/vPHGG3z33XclXm+3WbNmzJs3j4SEBMaPH09cXBwPP/xwgeNKsvnEheO8vLyw\nLAugTDbI8PX1df75wnWrMs0Yi4iISLk6e/YsQUFBVKtWjaNHj7J06dJSX6t79+7MnDkTyGvsfvnl\nl3zvnzlzhho1auDv78/27dv58ccfAUhNTcWyLG666SYeffRRNm3aRG5uLocPH6Zbt278v//3/0hN\nTSUzMzPf9Tp16sRXX30FQEpKCtu3by9Q0+HDhwkMDGTQoEHcf//9bNq0CYCgoKBLrlpxsZycHObN\nmwfA7Nmzad++PQANGzZk48aNAMydO9d5fGBgYKHRD4B27drxzTffADBr1iw6duxYrBqqIs0Yi4iI\nSLlq3bo1zZo1o3v37jRs2NDZ9JXG2LFjeeqpp+jZsyceHh48/vjj9OnTx/l+r169+Pjjj+nRowdN\nmzYlKioKgEOHDvG3v/0Ny7IwxvD000+Tk5PDgw8+SHp6Og6Hg5EjRxIUFJTvfnfffTejR48mNjaW\n5s2bExERUaCmLVu28I9//ANjDD4+PkyYMAGAoUOHMmTIEOrXr8/HH398yc9VvXp1Vq9ezSuvvEKd\nOnWcS6898MADPPDAA3zwwQfOFSgAunXrxjvvvEOfPn0KzE6/9NJLjB07ljfeeIPQ0FBeffXVEoxw\n1WKsCjJvfuEJyfIWGhpa6FOucmU0rq6jsXUdja3raGxd5+KxzcjIyBdZkCvj5eVVqqXT5PLKa2wL\n+zdRv379Io+v0lEKK+0EVgm3ChQRERER91RlG2MrZT2OJ/7K+W0b7S5FRERERCqAKtsY07QlePuQ\ntTLB7kpEREREpAKoso2x8fOHiHZkr1yqOIWIiIiIVN3GGMDEdMNx8gTs2mp3KSIiIiJis6rdGLdp\nDz4+WGtX2F2KiIiIiNisajfGfv74tuuClZyoOIWIiMgVGjRoUIHNOqZPn86TTz55yfOaNWsGwJEj\nR7j33sJ37Bs0aJBzc46iTJ8+Pd+GHHfeeSenTp0qRuWlc6FuV+vYsSNpaWkA3HzzzZc89vXXX7/k\n+xfG5MCBA87ttotr5cqV+bam/uCDD/jiiy9KdI2Krko3xgB+XeLgVJriFCIiIldowIABzJkzJ9/X\n5syZw4ABA4p1ft26dZk+fXqp7//ee+/la4w//PBDgoODS329iujrr7++5PtTp04t9OuWZeFwOK5o\nTBITE1m3bp3z9V/+8hcGDx5cqmtVVFW+MfaJ6QreilOIiIhcqf79+7N48WLOnTsHwIEDBzh69Cgd\nO3YkPT2dP//5z/Tt25devXqxcOHCAudfPIuZmZnJAw88QGxsLMOHDycrK8t53JNPPkm/fv2Ii4tj\n0qRJALz//vscPXqUwYMHM2jQICD/TOu0adPo2bMnPXv2dDbfBw4cIDY2lscee4y4uDhuu+22AltA\nAxw/fpzhw4cTHx9PfHx8vllToMjPlpGRwZ133kl8fDw9e/Z0/tAwfvx4evToQXx8PM8//3yB+6Wl\npXHbbbcRFxfHo48+ysV7sV2YpT569CgDBw6kd+/e9OzZk6SkJMaPH09WVha9e/fmoYce4sCBA1x/\n/fU8/PDD9OzZk0OHDuUbk5ycHB566CFiY2O59957nZ/94mN+/PFHBg0axIEDB/jwww+ZPn06vXv3\nJikpiVdeeYV33nkHgM2bN3PjjTcSHx/P8OHDnVtzDxo0iJdeeon+/fvTrVs3kpKSCn7jVCBVfkto\nD/8AaB2dF6cYcg/Gw9PukkRERK7Ye2uPsvdk1uUPLIFra/pxT8xVRb5fs2ZNIiMjWbJkCX379mXO\nnDncdNNNGGPw9fXl/fffp1q1aqSlpXHTTTfRp08fjDGFXuuDDz7A39+fZcuWsWXLFm644Qbne088\n8QQ1a9YkNzeXW2+9lS1btjB8+HDeffddvvjiC0JCQvJda+PGjXz++ed8++23WJbFjTfeSOfOnQkO\nDmbv3r28+eabTJw4kfvvv5958+bxpz/9Kd/5zzzzDJ06deL9998nNzeX9PT0fO8X9dmWLFlC3bp1\n+fDDDwE4ffo0aWlpzJ8/n+XLl2OMKTTqMXnyZDp06MCYMWNYtGgR//nPfwoc89VXXxEbG8vo0aPJ\nzc0lMzOTjh07MmPGDL777jsgr/Hfu3cvU6ZMITo6usA1du/ezSuvvEL79u0ZO3Ys//73vxkxYkSh\nfx+NGjXizjvvJDAw0HnMihW/TSo+8sgjvPDCC3Tu3JmJEyfy6quvOpv+nJwc5s6dy+LFi3n11Vf5\n7LPPCr1HRVDlZ4wBTHRXxSlERETKwMVxiotjFJZlMWHCBOLj47n11ls5cuQIx48fL/I6SUlJDBw4\nEICwsDBatWrlfO+bb76hb9++9O3bl+3bt7Nz585L1rR69WpuuOEGAgICCAwMpF+/fs6Zy0aNGhER\nEQFAmzZtOHDgQIHzf/jhB/7yl78A4OnpSfXq1fO9X9Rna9myJcuXL+ell14iKSmJ6tWrU716dXx9\nffnb3/7GvHnz8Pf3L3C/VatWOT97fHw8NWrUKHBMZGQkn3/+Oa+88gpbt24lKCio0M/esGHDQpti\nyNsauX379gAMHDiQ1atXF3rc5Zw+fZpTp07RuXNnAAYPHpxvZvgPf/gDkDe+P//8c6nuUV6q/Iwx\n5K1OYXn7YK39AdM8wu5yRERErtilZnZdqW/fvjz33HNs2rSJzMxM2rRpA8CsWbNITU1l/vz5eHt7\n07FjR7Kzs0t8/f379zNt2jTmzp1LjRo1eOSRR/LFLErK19fX+WdPT89SXauoz9a0aVMWLFhAQkIC\nL7/8Mt26dWPMmDHMnTuXFStWMHfuXGbMmFGqB9g6derEl19+yeLFixkzZgz33XdfoXnfgICAIq/x\n+9n6C6+9vLxw/LooQWn+jn7Px8cHyBvfnJycK76eK2nGmF83+7gQp3Dk2l2OiIhIpRUYGEiXLl0Y\nO3Zsvofuzpw5Q2hoKN7e3vzwww+XnTns2LEjs2fPBmDbtm1s3brVeR1/f3+qV6/O8ePHWbJkifOc\noKAgzp49W+i1Fi5cSGZmJhkZGSxYsICOHTsW+zN169aNDz74AIDc3FxOnz6d7/2iPtuRI0fw9/fn\nT3/6EyNGjGDTpk2kp6dz5swZevXqxXPPPceWLVsK3K9Tp0589dVXACQkJDjzuhf7+eefqV27NkOH\nDuX2229n06ZNAHh7e3P+/Plifa6DBw+ydu1aAGbPnu2cPW7YsCEbN24EYO7cuc7jAwMDCx3f6tWr\nExwc7Jwl/vLLL+nUqVOxaqho1Bj/ysR0U5xCRESkDAwYMIAtW7bka4wHDhzIjz/+SK9evfjvf//L\nddddd8lr/OUvfyE9PZ3Y2FgmTZrknHkODw8nIiKC7t278+CDDzqbOYChQ4cydOhQ58N3F7Ru3ZrB\ngwfTv39/brzxRm677TZnfKI4nn/+eVauXEmvXr244YYb2LFjR773i/ps27Zt48Ybb6R3795MnjyZ\n0aNHc/bsWYYNG0Z8fDy33HILzz77bIH7jRkzhqSkJOLi4pg/fz4NGjQocMzKlSvp3bs3ffr04euv\nv+aee+5xjkF8fDwPPfTQZT9X06ZN+fe//01sbCynTp1i2LBhAIwdO5Zx48bRr18/PD1/e/aqd+/e\nLFiwwPnw3cWmTJnCCy+8QHx8PCkpKYwZM+ay96+IjHXxo442OnTokC33DQ0N5cSJE1hZmTjG3onp\n1huP2++3pRZ3cmFcpexpbF1HY+s6GlvXuXhsMzIyLvmrcykZLy+vCv+r/8qqvMa2sH8T9evXL/J4\nzRj/6rc4xUrFKURERESqIDXGF8mLU5xUnEJERESkClJjfBHTOkabfYiIiIhUUWqML5IXp4jR6hQi\nIiIiVZAa498xMV0VpxARERGpgtQY/47iFCIiIiJVkxrj31GcQkREpHTS0tLo3bs3vXv3JjIykujo\naOfrc+fOFfs6n376KceOHXNhpb8ZNWoUCxYsAPLWD961a1eRx65YsYJ169aV+B7R0dGcOnWqWMd+\n8sknjBs3rsT3KKnly5fz17/+FYD58+fz9ttvF3nspk2b8m2k8nvJycnO9Zj/+c9/Mn369BLVMm3a\ntHw7Dt5+++2FbiRSHrQldCFMTFes5JWwcyu00BbRIiIixRESEsJ3330HwCuvvEJgYCAjRowo8XU+\n/fRTIiIiqFOnTqnqyMnJwcur5C3O5MmTL/n+Dz/8QEhICNHR0aWqq6Lq16/fJd/ftGkT27ZtIy4u\nrsB7OTk5tGvXjnbt2pX6/u+++y5DhgzBz88PyPvhwC5qjAthWsdg+fhgrVuBUWMsIiJyxT7//HP+\n/e9/c+7cOWJiYnjppZdwOByMGTOGLVu2YFkWQ4cOJTQ0lJSUFB544AH8/PyYO3cuPj4+zusMGDCA\nNm3akJiYiMPh4NVXX6Vt27b885//5ODBg+zbt4+rr76ayZMn8+KLL7JmzRqys7P561//yu23347D\n4eDpp5/mhx9+oH79+vl2dhswYAAvvvgiERERLFq0iEmTJpGTk0Pt2rX5xz/+wX/+8x88PT35/PPP\nGT9+PI0bN+app57i4MGDeHh48PzzzxMdHU1qaioPPvggR48ezbcz3+8tWrSIiRMnkpubS+3atfnP\nf/6T7/2FCxcydepUzp07R61atZg6dSqhoaGsWLGC5557DmMMHh4efPXVV5w6dYoHHniA9PR0cnNz\n+ec//1ng3osWLeL//u//CAgIyPfeJ598wrZt23j++eeZM2cOr732Gh4eHtSoUYMPP/yQyZMnk5WV\nRWJiIqNHjyYlJSXfWA8ePJiZM2fyr3/9C4DNmzdz4403cvLkSUaNGsWQIUNYvnx5vmMee+wxYmJi\nSE1NJTU1lYEDBxIaGspnn31GdHQ0CQkJBAcH89Zbb/Hf//4XgDvuuIO//vWv7N27l+HDhxMVFUVy\ncjINGjTgvffeczbWV0KNcSGMnz9E/BqnGHIvxsPz8ieJiIhUIJuTMzj9S9lGAqvX8CSiXcl31tu2\nbRsLFixgzpw5eHl58fjjjzNnzhyuueYaTp48yeLFiwE4deoUwcHBzJgxw9mgFubcuXN89913rFix\ngkcffdQ5S717926+/PJL/Pz8mDlzJqGhocydO5fs7GxuuukmYmNjWbduHQcOHGDp0qUcOXKEuLg4\n7rzzznzXP3bsGE899RRff/019erV4+TJk9SsWZPbbruNkJAQ7r33XgBGjBjBAw88QHR0NAcOHGDY\nsGEkJCQwadIkunTpwsMPP8zChQv5+OOPC3yGC/f46quvaNiwISdPnixwTMeOHenTpw/GGD744AOm\nTZvG008/zTvvvMPLL79Mu3btSE9Px9fXl1mzZtG7d28efPBBcnNz80UTADIzM3niiSf48ssvueaa\na7jvvvsKHdtXX32V//73v9SuXZtTp07h7+/PmDFjnI0zQEpKSr6xXr58eYG/7zlz5nD27Fn69u1L\nr169Cr0XwH333ce0adOYNWsWwcHB+d5LTk5m1qxZzJ07l9zcXPr370/nzp3x8/Nj9+7dvPnmm7Rq\n1Yp77rmHhQsX8sc//rHI+xSXGuMimJhuilNIheQ4/YvdJYiIlMj333/Pjz/+6PyVfVZWFvXq1SM2\nNpbdu3fzzDPP0KtXL2JjY4t1vQEDBgDQrVs3UlNTSU9PB6BPnz7OWcPly5ezc+dO5syZA8CZM2fY\ns2cPSUlJ/PGPf8TDw4P69evTuXPnAtdft24dXbp0oVGjRuTk5FCzZs0iP9fu3budr0+dOkVmZiZJ\nSUl8+OGHAPTt25egoKAi79GwYUOAQu9x8OBBRowYwfHjx8nOzqZJkyYAxMTEMG7cOAYOHMgf/vAH\nAgMDadu2LU888QTZ2dn07duX8PDwfNfasWMHTZo0oXHjxgDccsstzpnYi7Vv357Ro0dz4403XjJi\ncfFY/94NN9yAn58ffn5+dDz5ZMMAACAASURBVOrUiQ0bNuDr61vktYqyevVq+vfvj7+/v/O6SUlJ\nxMbG0rhxY1q1agVA69atOXDgQImvXxg1xkUwbRSnkIrH2rCK429PwGPc65gGV9tdjohUYKWZ2XUV\ny7K49dZbefzxxwu8t2jRIhISEpg5cybz5s3j5Zdfvuz1jDGFvg4I+O0zW5bF+PHjuf766/Mde+FB\nu7Ly+6hHWXr66acZNWoUvXr1Yvny5bz55psAPPLII/Tp04fFixdz00038dlnn9GtWzf++9//snjx\nYkaPHs3IkSMZOHBgie85ceJEkpOTWbRoETfccAMLFy4s9LiLx/pyjDF4eXnhcDicX8vOzi5xbRe7\neMw9PT3Jycm5outdoFUpimB8/fLiFOtWanUKqTAcy/8HDgfWxjV2lyIiUmzXX38933zzDWlpaUDe\n6hUHDx4kNTUVy7K46aabePTRR9m0aRMAQUFBzlngwnz99dcArFy5ktDQ0EKbtNjYWD744ANnw7Rr\n1y4yMzPp2LEjX3/9NQ6Hg8OHD7Nq1aoC58bExLBy5UrnLOSFmENQUFC+1RK6devGzJkzna83b94M\n5EUgvvrqKwD+97//FbrCwoV7/Pzzz/nucbEzZ85Qt25dLMviiy++cH79p59+IiwsjFGjRtG6dWt2\n797Nzz//TJ06dbjjjju49dZbnbVc0Lx5c/bu3cv+/fuxLIvZs2cXuB/Avn37iI6O5vHHHyc4OJgj\nR44QGBh4yb+P31u4cCHZ2dmkpqaSlJRE27ZtadiwITt27ODcuXP88ssvrFjx27K4vx/XCzp27Mj8\n+fPJzMwkPT2dhQsX0rFjx2LXURqaMb4ExSmkIrFOnYSU5Lw/b1kP/f5kc0UiIsXTqlUrxo4dy623\n3oplWXh5eTFhwgQ8PT3529/+hmVZGGN4+umnAfjzn//Mo48+WujDdwBeXl707t3b+fBdYe68804O\nHTpEnz59AKhVqxYzZszgxhtvZOXKlfTo0YMGDRoUusLEhYfthg0bhsPhoG7dunz00Uf07duX+++/\nn/nz5/PSSy8xfvx4nnzyST7//HNycnLo0qUL48eP59FHH2XkyJHMmjWLmJgY6tatW+Q97r77bizL\nct7jYmPHjuWee+6hRo0adO7c2bmE3TvvvMPq1avx8PCgVatWxMbGMmvWLN599128vLwICgri9ddf\nz3ctf39/JkyYwB133OF8+O7gwYMF6nruuec4cOAAlmXRvXt3WrZsSe3atXnnnXfo06cPDz/8cFF/\nzU4tWrTgT3/6EydPnuTRRx+ldu3aQF6sJC4ujmuuuYbWrVs7jx86dChDhgyhfv36fPbZZ86vR0VF\nMWDAAPr37w/AX/7yF1q1asXevXsvW0NpGcuyLJddvQQOHTpky31DQ0M5ceJEoe9Z2Vk4xt6B6RqP\nx+0lX26mKrvUuErpOP43G+uLf+ET05VzG5LwmPJJ3m82pMzo+9Z1NLauc/HYZmRklOhX3JXRxStH\nuJqXl1eZ/Ype8iuvsS3s30T9+vWLPL5YM8YPPvggfn5+eHh44OnpyYQJEzh79iyTJ0/m+PHj1K5d\nmzFjxhAUFIRlWcyYMYP169fj6+vLyJEjnWHxysb4+uVt9rFupVanENtZiUvg2uYE9B/MubU/wPZN\n0KboZYBERESkZIodpXj22WepXr268/Xs2bNp3bo1AwYMYPbs2cyePZs77riD9evXc+TIEV5//XV2\n7tzJe++9x/jx411SfHkw0d2w1ilOIfayDuyFn/dibh+BT1jbvG3Lt2zAqDEWkSqmqGysSFko9cN3\na9ascS6rEhsby5o1eQ8DrV27lu7du2OMoXnz5qSnpxcaKK8sTJsY8PHBWrvi8geLuIiVmACeXpj2\n3TA+vtA8HOvXvLGIiIiUjWI3xi+99BJPPPEEixYtAvLW6ruw5l6NGjWce4CnpaURGhrqPK9WrVrO\np1ArI2ecIlmrU4g9rNxcrKRl0LY9JijvtzYmvB0cOYiVeszm6kSkIqkgjw2JVBgl/TdRrCjFCy+8\nQEhICKdOneLFF18sEFo2xhRYU/ByFi1a5GyyJ0yYkK+ZLk9eXl6XvXdWXD9OrVtJ8LGD+ESUfi/w\nqqQ44yrFk71uJb+c/oXgPn/ELzQULy8vQrrGkfr5+wTu20lAizC7S3Qb+r51HY2t61w8tsYYHA4H\n3t7eNlflPry8tICXq7h6bM+fP09QUBC1atUq9jnFqigkJASA4OBg2rdvz65duwgODnZukXjy5Eln\n/jgkJCTfk8epqanO8y8WHx9PfHy887VdTysX50lpq3EL8PHhl8Xz8KirTRWKQ0+glx3HgtkQVJ0z\n1zTj7IkThIaGctK/GtSoxZnV35PRrqvdJboNfd+6jsbWdS4eW8uyyMrKIiMjo8QTVlKQr6/vFW9E\nIYVz9dhaloWHhwd+fn4F/ttzRatSZGVlYVkW/v7+ZGVlsXHjRgYNGkRMTAzLli1jwIABLFu2jPbt\n8x4CiomJYcGCBXTt2pWdO3cSEBBQ5FaKlUW+OMVtWp1Cyo+VcRZrQxKme1+M128zQMYYTHgk1vpV\nWLm5GE99T4pI3n8bLmyfK1dOP9C5TkUd28s2xqdOnWLSpEkA5Obm0q1bNyIjI2natCmTJ08mISHB\nuVwb5C3GnJyczMMPP4yPjw8jR4507ScoJx4x3XCsWwk7t0CL1pc/QaQMWGt/gJzzmM5xBd8Mbwc/\nLIafdkLTluVfnIiIiJu5bGN81VVXMXHixAJfr1atGuPGjSvwdWMM99xzT9lUV5G0vrA6xQ8YNcZS\nTqzEBKjXCK65rsB7plVbLGOwUtZj1BiLiIhcsVIv11bVaHUKKW/WscOwayumc89Cs4ImqDpcc13e\n9tAiIiJyxdQYl4BHTDc4/UtenELExazEJWAMplOPIo8x4VGwdwdWxtnyK0xERMRNqTEuiYviFCKu\nZDkceTGKVm0xNYteZsaERYHDAVs3lmN1IiIi7kmNcQkYXz9M6/aKU4jr7doCqccwnXte+rgmLcDP\nX3EKERGRMqDGuIRMTFfFKcTlrMQl4OuPiep0yeOMlxe0bIOVsl47XomIiFwhNcYl5YxTrLC7EnFT\nVnY21toVmOgueQ99XoYJj4LUY3D0UDlUJyIi4r7UGJeQM06xTnEKcQ1rwyrIysR0uUyM4lcmLCrv\nvBTFKURERK6EGuNSMDFd4cwp2JFidynihqxVS6BWHWgWXqzjTZ16ULuucsYiIiJXSI1xabSOAR9f\nrHVanULKlvVLKqRswHTqgfEo/j9PE94Otm/CyjnvwupERETcmxrjUsiLU8QoTiFlzkpaDpYD06mQ\nLaAvwYRHQnYW7N7mospERETcnxrjUjLtuylOIWXKsqy8tYubtsTUbVCyk1u0AU9PrJRk1xQnIiJS\nBagxLq0IxSmkjB3YAwf3lXi2GMD4B0CTFnoAT0RE5AqoMS4l4+urOIWUKWtlAnh55f02ohRMeDvY\nvwfr9C9lXJmIiEjVoMb4CihOIWXFysnBWr0c2nbABFYr1TWcy7Zt2VCWpYmIiFQZaoyvhOIUUlZS\n1sOZU3hcbgvoS7mmCQRVAy3bJiIiUipqjK+A8fXFtNFmH3LlHImLoVowhLcr9TWMhyemVSTWlg3a\nHlpERKQU1BhfIW32IVfKSj8LP67GdOiO8fK6souFR8Gpk3DwpzKpTUREpCpRY3ylLsQp1q6wuxKp\npKw130NOTrG3gL4U0yoy75panUJERKTE1BhfIWecIjkRK1dxCik5a9USaHANNGpyxdcyIaFQ/2o1\nxiIiIqWgxrgM/Ban2Gx3KVLJWEcOwu5tmM5xGGPK5JomPAp2bsHKzi6T64mIiFQVaozLglankFKy\nVi0B44HpGFtm1zRhUZBzXj+oiYiIlJAa4zKgOIWUhuVwYK1aCmFtMTVqld2Fm4eDlzeWlm0TEREp\nETXGZcTEdFOcQkpmZwqkHsNcydrFhTA+vtA8XDljERGRElJjXFYiohWnkBKxEhPAzx8T2anMr23C\no+DwAay042V+bREREXelxriMKE4hJWFlZ2GtXYmJ6Ybx9S3z6zu3h9assYiISLGpMS5DilNIcVnr\nV0F2JqZznGtu0OAaCA6BLRtcc30RERE3pMa4LClOIcVkJSZArTpwXZhLrm+MwYT9uj20tisXEREp\nFjXGZcj4+mLadlCcQi7JOpkKW3/EdO6J8XDhP8HwKMg4C/t2u+4eIiIibkSNcRkz0drsQy7NWrUU\nLAvTuYdL72PCosAYrJRkl95HRETEXagxLmsX4hRrFaeQgizLyotRXNcKU6e+S+9lqlWHq5vqATwR\nEZFiUmNcxpxxivWKU0gh9u+Gwwdc99Dd75jwKNizHSsjvVzuJyIiUpmpMXYBxSmkKNbKBPDyzlvB\npByYsChwOGDbxnK5n4iISGWmxtgVIqLB109xCsnHyjmPtXo5JrIjJiCofG7atAX4+mt7aBERkWJQ\nY+wCzs0+FKeQi21eB2dPl1uMAsB4eUPL1lgp67Esq9zuKyIiUhmpMXYRE6M4heTnSFwC1YIhvF25\n3teER8GJo3D8cLneV0REpLJRY+wqilPIRayzp+HHNZiOPTCenuV6b20PLSIiUjxqjF3E+Pwap0he\nqTiFYK1ZAbk5mC49y//mdepB6FVqjEVERC5DjbELmZiucPa04hSSt3Zxw8aYRteW+72NMXlxim2b\nsHLOl/v9RUREKgs1xq6kOIUA1pGfYe8OTGcbZot/ZcKiIDsT9my3rQYREZGKTo2xCylOIQBW4hLw\n8MB0jLWviJZtwMNDcQoREZFLUGPsYopTVG2Ww4G1agmEt8ME17StDhMQCE1aqDEWERG5BDXGruaM\nU6ywuxKxw/ZNkHaiXNcuLooJj4L9u7HOnLK7FBERkQpJjbGL/Ran0GYfVZGVmAD+gZi2HewuJS9n\nbFlYWzbYXYqIiEiFpMa4HDjjFNs32V2KlCMrKxMrORET0xXj42t3OdD4OggIAjXGIiIihVJjXB4u\nxCnWaXWKqsRavwqys2xdjeJixsMT06ot1hZtDy0iIlIYNcblQHGKqslKTIDadeG6VnaX8pvwKPgl\nDQ7tt7sSERGRCkeNcTkxMd0Up6hCrLTjsG0jplMcxhi7y3Ey4Re2h062uRIREZGKR41xeYloB77+\nilNUEdaqpWBZFWI1iouZkNpQrxFWinLGIiIiv6fGuJzkxSliFKeoAizLytvUo1kYpnZdu8spwIRH\nwc4UrHPZdpciIiJSoagxLkeKU1QRP+2EIz9XmIfufs+ERcH5c7Ajxe5SREREKhQ1xuVJcYoqwUpM\nAG8fTHRXu0spXPMI8PLC2qJd8ERERC6mxrgcGR9fTFutTuHOrPPnsVZ/j4nsmLcNcwVkfH2hWbi2\nhxYREfkdNcblzERrsw+3tmktpJ/BdKmYMYoLTHgUHNqPdTLV7lJEREQqDDXG5e1CnGLtCrsrERdw\nJC6B4JrQKtLuUi7JhP26bJviFCIiIk5qjMuZM06xXnEKd2OdOQ2b1mI69sB4etpdzqU1bJzXwCtO\nISIi4qTG2AZ5cYozsH2j3aVIGbLWLIfcnAq3dnFhjDGYsEisrRuwHPoBTUREBNQY28MZp9DqFO7E\nWpkAja7FNGxsdynFExaV9wPa/j12VyIiIlIhqDG2geIU7sc6tB/27arwD91dzITl5aC1OoWIiEge\nNcY2ydvsQ3EKd2ElLgEPD0yH7naXUmymeg24uglWSrLdpYiIiFQIaoztEh6lOIWbsBy5WKuWQkQ0\npnpNu8spERMeBXu2Y2Vm2F2KiIiI7dQY2yRfnCInx+5y5Eps2wS/pOJRCR66+z0TFgW5ufrNhYiI\nCGqMbeWMU+zQZh+VmZWYAAGB0LaD3aWUXNNW4OuHlbLB7kpERERsp8bYTopTVHpWVgZWciIm5nqM\nt4/d5ZSY8faG5hHKGYuIiKDG2FZ5cYoOilNUYta6RDiXXSnWLi6KCW8Hx49gHTtsdykiIiK28iru\ngQ6HgyeffJKQkBCefPJJjh07xpQpUzhz5gxNmjRh1KhReHl5cf78ed544w327NlDtWrVeOSRR6hT\np44rP0OlZmK6Yq1elhen+HWbXqk8rMQEqFMPmra0u5RSM+GRWORtD23q1LO7HBEREdsUe8Z43rx5\nNGjQwPn6o48+on///kydOpXAwEASEhIASEhIIDAwkKlTp9K/f38+/vjjsq/anShOUWlZqcdg+yZM\n5ziMMXaXU3pXNYBadZQzFhGRKq9YjXFqairJycn06tULAMuySElJoVOnTgD06NGDNWvWALB27Vp6\n9OgBQKdOndi8eTOWZbmgdPegOEXlZa1aCoDpVHljFPDr9tDhUbDtR30PiohIlVasxnjmzJnccccd\nzlmxM2fOEBAQgKenJwAhISGkpaUBkJaWRq1atQDw9PQkICCAM2fOuKJ2t2Fiump1ikrGsqy8TT2a\nR2BCr7K7nCtmwqIgKxP27rC7FBEREdtcNmO8bt06goODadKkCSkpKWV240WLFrFo0SIAJkyYQGho\naJlduyS8vLxsu/cFVmxvjv9rCr6b1lK9e29baykrFWFcXenc9s2cPHqQ6oOH4V/On9MVY+voFsfx\nd1/Gf+82gjpXnt37ypq7f9/aSWPrOhpb19HYuk5FHdvLNsbbt29n7dq1rF+/nnPnzpGZmcnMmTPJ\nyMggNzcXT09P0tLSCAkJAfJmj1NTU6lVqxa5ublkZGRQrVq1AteNj48nPj7e+frEiRNl+LGKLzQ0\n1LZ759OmPZmrlpL9p7sxXsV+JrLCqjDj6iKOBV+Bjw9nm7chvZw/p8vG9trmpK/5gaw+A8v+2pWE\nu3/f2klj6zoaW9fR2LqOnWNbv379It+7bJTi9ttv55133uHNN9/kkUceISIigocffpjw8HBWrVoF\nwNKlS4mJiQEgOjqapUuXArBq1SrCw8Mr94NJ5cS0/zVOsV1xiorOOn8ea/X3mMjOGP8Au8spMyYs\nCvbtwjp72u5SREREbFHqdYyHDh3Kt99+y6hRozh79iw9e/YEoGfPnpw9e5ZRo0bx7bffMnTo0DIr\n1q2FtwM/f6x1Wp2iwtu4BjLOYrr0tLuSMmXCo8CysLb+aHcpIiIitijR7+zDw8MJDw8H4KqrruIf\n//hHgWN8fHwYO3Zs2VRXhRhvH0ybDljJiVi3j3CLOIW7ciQmQI0QaNXG7lLKVuNmeVtbp6yH9tfb\nXY2IiEi50853FYhp3xXSFaeoyKzTv8DmdZiOPTAennaXU6aMpye0bIuVsl5LLIqISJWkxrgiUZyi\nwrNWL4fcXExn94pRXGDCo+CXVDh8wO5SREREyp0a4wrEePvkbfaRrM0+KiorcQlccx2mwdV2l+IS\nJjxvW3IrZb3NlYiIiJQ/NcYVjIlRnKKisg7ug/27MZ0r9053l2Jq1YG6DbC2qDEWEZGqR41xRXMh\nTrF2hd2VyO9YiQng6Ynp4N4bYJjwdrBjM9b5c3aXIiIiUq7UGFcwzjjF+lWKU1QgliMXa9UyiIjG\nVAu2uxyXMmGRcO4c7Cy7nS5FREQqAzXGFZDiFBXQlh/hVBoebvrQXT4tWoOnF1bKBrsrERERKVdq\njCsixSkqHCtxCQQEQZv2dpficsbXD65rhZWSbHcpIiIi5UqNcQWkOEXFYmVmYG1IxHS4HuPtbXc5\n5cKEt4OD+7B+SbO7FBERkXKjxriCMjHd8uIU2zbaXUqVZ637Ac6dc9u1iwtjwiMBtDqFiIhUKWqM\nK6rwKG32UUFYiUvgqgZwbXO7Syk/Da+FasGgnLGIiFQhaowrKMUpKgbr+BHYsRnTOQ5jjN3llBvj\n4YEJj8Lash7L4bC7HBERkXKhxrgCU5zCflbSUgBMJ/fd1KNIYVFw9jQc2Gt3JSIiIuVCjXFFpjiF\nrSzLyotRtGiNqVXb7nLKnQn7NWes1SlERKSKUGNcgRlvH0xkR8Up7LJ7Gxw7jOlSdR66u5gJrgkN\nr8VK0QN4IiJSNagxruBMdFfFKWxiJSaAjy+mXWe7S7GNCY+C3duwsjLsLkVERMTl1BhXdBfiFNrs\no1xZ589hrVmBadcF4xdgdzm2MeFRkJsD2zfbXYqIiIjLqTGu4Jxxig1JilOUpx9XQ2Y6pnMVfOju\nYteFgY+P4hQiIlIlqDGuBBSnKH+OlQlQMxRatra7FFsZb29o3lqNsYiIVAlqjCuD8CjwD1CcopxY\np09CSjKmUyzGw9PucmxnwqPg2KG8NZ1FRETcmBrjSkCbfZQvK2k5OBxVagvoSzHh7QCwtmgXPBER\ncW9qjCsJE9MNMs7Cth/tLsXtWYkJ0LgZpl4ju0upGOo2gJBQrC2KU4iIiHtTY1xZhF2IU2izD1ey\nft4LB/bqobuLGGPyZo23bsTKzbW7HBEREZdRY1xJGG9vxSnKgZW4BDy9MO27211KhWLCIiEzHfbu\nsLsUERERl1FjXIkoTuFaVm4u1qql0DoGU6263eVULK3agvHQ6hQiIuLW1BhXJopTuNaWDXD6Fzyq\n6BbQl2ICq0Hj67BSku0uRURExGXUGFcieXGKjopTuIiVmACB1aB1tN2lVEgmvB38tAsr/YzdpYiI\niLiEGuNKxsR0VZzCBayMdKwNSZgO12O8vO0up0Iy4ZFgOWCrvvdERMQ9qTGubMK02YcrWOt+gPPn\nMJ172V1KxXVti7zvPa1nLCIibkqNcSXzW5wiSXGKMmStTIC6DaHxdXaXUmEZT09o2QYrJRnLsuwu\nR0REpMypMa6EFKcoW9axw7BrC6ZLT4wxdpdToZnwdpB2Ao4ctLsUERGRMqfGuDJSnKJMWauWgDGY\njj3sLqXCM2GRAFqdQkRE3JIa40oof5zivN3lVGqWZeVt6tGyDSYk1O5yKjxTuy7Uqa+csYiIuCU1\nxpWUc7OPrRvtLqVy27UVThzFdNbaxcVlwqNg+yas8/qhTERE3Isa48oqLDIvTrFOcYorYSUmgK8f\nJqqT3aVUGiY8Cs5lw64tdpciIiJSptQYV1L5N/vQzF1pWOeysdauwLTrgvHzt7ucyqNFa/D00vbQ\nIiLidtQYV2J5cYp0xSlKydqQBJkZmM5xdpdSqRg/f2jaUo2xiIi4HTXGlZniFFfESlwCIaF5M6BS\nIiY8Cn7ei3XqpN2liIiIlBk1xpWY8fbGRCpOURrWL2mQsh7TKQ7joX8GJWXCowC0OoWIiLgVdQSV\nnIlWnKI0rNXLwHIoRlFajZpAtWDYojiFiIi4DzXGld2FOIU2+ygRK3EJXNscU7eh3aVUSsbDA9Mq\nEitlPZbDYXc5IiIiZUKNcSXnjFNsUJyiuKz9e+Dnn7R28ZUKj4Qzp+Dnn+yuREREpEyoMXYDilOU\njJW4BDy9MB2ut7uUSs2E/Zoz1uoUIiLiJtQYu4OwSPAPVJyiGKycHKykpdC2PSawmt3lVGqmRgg0\nuAYrJdnuUkRERMqEGmM3kBen6KA4RXFsWQ9nTuGhGEWZMOHtYNdWrOwsu0sRERG5YmqM3cRvcYof\n7S6lQrNWJkBQdYhoZ3cpbsGER0JuDmzfZHcpIiIiV0yNsbtwxil+sLuSCstKP4v142pMx1iMl7fd\n5biHZuHg7aP1jEVExC2oMXYTilNcnrVuBeSc19rFZch4+0DzcOWMRUTELagxdiMmRnGKS7FWJkC9\nRnB1U7tLcSsmvB0cOYiVeszuUkRERK6IGmN3ojhFkaxjh2D3NkyXnhhj7C7HrTi3h9aybSIiUsmp\nMXYjxkubfRTFSlwCxgPTsYfdpbifeo2gRi0sbQ8tIiKVnBpjN2NiuipO8TuWw5HXGLdqi6lZy+5y\n3I4xJm/WeOuPWLm5dpcjIiJSamqM3c2FOMUabfbhtHMLpB7TQ3euFB6V9wPZTzvtrkRERKTU1Bi7\nGWec4sckxSl+ZSUmgK8/Jqqz3aW4LdOqLRijnLGIiFRqaozdkOIUv7Gys7HW/YCJ6YLx9bW7HLdl\ngqrDNdcpZywiIpWaGmN3pDiFk7VhFWRlYjr3srsUt2fCo2DPDqyMs3aXIiIiUipqjN3Qb6tTKE5h\nJSZArTrQLMzuUtyeCYsCywFbN9pdioiISKmoMXZTJqYrZKZDFd6q1/olFbb8iOnUA+Ohb3WXa9IC\n/PwVpxARkUpL3YK70mYfWEnLwHJgOve0u5QqwXh5Qcu2WCnrsSzL7nJERERKTI2xm8oXpzhf9eIU\nlmXlbQHdtCXmqvp2l1NlmPBISD0GRw/ZXYqIiEiJqTF2Y6Z9t7w4xdYqGKfYvwcO7ddscTkz4e0A\nbQ8tIiKVkxpjd9aqLQRUzTiFlZgAXt6YmG52l1KlmNp1oXZd5YxFRKRSUmPsxvLiFJ2qXJzCysnB\nSlqGadsBExhkdzlVjglvB9s3VfkVUUREpPJRY+zmnKtTVKU4RUoynD2tGIVNTHgkZGfBrq12lyIi\nIlIiaozdnTNOUXU2+3CsTIBqwRAeZXcpVVOLNuDpqTiFiIhUOl6XO+DcuXM8++yz5OTkkJubS6dO\nnfjzn//MsWPHmDJlCmfOnKFJkyaMGjUKLy8vzp8/zxtvvMGePXuoVq0ajzzyCHXq1CmPzyKFcMYp\n1q/COn8e4+1td0kuZaWfgY2rMT3+kLd8mJQ74x8ATVrkPYA3cJjd5YiIiBTbZWeMvb29efbZZ5k4\ncSIvv/wyGzZsYMeOHXz00Uf079+fqVOnEhgYSEJCAgAJCQkEBgYydepU+vfvz8cff+zyDyGXVpXi\nFNaa7yEnB9M5zu5SqjQT3g7278E6/YvdpYiIiBTbZRtjYwx+fn4A5ObmkpubizGGlJQUOnXqBECP\nHj1Ys2YNAGvXrqVHjx4AdOrUic2bN2uxf7tVoTiFlbgEGlwDjZrYXUqVZn6NsVhVeOdFERGpfIr1\nu2aHw8ETTzzBkSNH6Nu3L1dddRUBAQF4enoCEBISQlpaGgBpaWnUqlULAE9PTwICAjhz5gzVq1fP\nd81FixaxaNEiACZMBVB8OwAAIABJREFUmEBoaGiZfaiS8PLysu3e5elUp1iyVy2nVnB1jLePy+9n\nx7jmHNxH6p7tBA17iMDatcv13uWpMnzPWjVrcrxaML67txJ84yC7yym2yjC2lZXG1nU0tq6jsXWd\nijq2xWqMPTw8mDhxIunp6UyaNIlDh658V6v4+Hji4+Odr0+cOHHF1yyN0NBQ2+5dnqyIGKyEeZxY\nvhjTtr3L72fHuDrmfQXGg4yIGDLd+O+00nzPtmxD1vpVnDt+HGOM3dUUS6UZ20pIY+s6GlvX0di6\njp1jW79+0TvilmhVisDAQMLDw9mxYwcZGRnk5uYCebPEISEhQN7scWpqKpAXvcjIyKBatWqlrV3K\nyoU4xTr3jFNYDgfWqiUQHompEWJ3OQJ5q4KcOgkHf7K7EhERkWK5bGN8+vRp0tPTgbwVKjZu3EiD\nBg0IDw9n1apVACxdupSYmBgAoqOjWbp0KQCrVq0iPDy80swWubPfNvtY7Z6bfezYDGnHMZ300F1F\nYcJ+zRlre2gREakkLhulOHnyJG+++SYOhwPLsujcuTPR0dE0bNiQKVOm8Omnn3LttdfSs2feZgo9\ne/bkjTfeYNSoUQQFBfHII4/8f/buOz6u8kr4+O+5UySNerMsyZYtW26SZUtyL9gUExKKQwJvglPe\nwG7iBJIQSNtNNi+BTSNkCWAIsEAW7y6sSTbsYiAkBGGwjS1XWe62XOQqF/Uyoza6z/vH2ALjojaj\nO+V8Px8+ISNp7tHD1cyZe89zTsB/CdE3avp89Pp3YU8FDEE5xVDSZe9BjAtVPNvqUMQ5KjkVsnJ8\nifENn7U6HCGEEKJXvSbGo0aN4pFHHrno8YyMDH71q19d9LjT6eS73/2uf6IT/jVpSk85xVDUGQ8V\n3dGO3roeNWM+yhlldTjiI1RBMfq9P6M7OlBR8t9GCCFEcJPJdxFE2R2o4tnoio1hVU6ht5VBR5uM\ngA5CKr8YvF5fqYsQQggR5CQxjjBq2nxo8/jKKcKEXr8K0jIgb5LVoYiPG18AdoeMhxZCCBESJDGO\nNOfLKcJk2Ieur4V9O1BzrkEZcjoHG+WMgvEFsgFPCCFESJBMIsL0lFNsD49yCr3xfdBaulEEMVVQ\nDKeOo+trrA5FCCGEuCJJjCNQuJRTaK193SjyJqGGZVodjrgMVVACSNs2IYQQwU8S40g0aQq44kK/\nnOLoQTh1XDbdBbusHEhKCfkPYkIIIcKfJMYRyFdOMSvkyyn0+lVgd6Cmz7M6FHEFSilUfjF6TwXa\n7LY6HCGEEOKyJDGOUB+WU4Tm7W3t7UJvXoMqno1yxVkdjuhNfhF4WuHIQasjEUIIIS5LEuNI1VNO\nsc7qSAZm51ZobUHNkU13oUDlF4NS0rZNCCFEUJPEOEKFejmFWbYKEpIgv9jqUEQfqPgEyBkrG/CE\nEEIENUmMI5iaHprlFLq1GXZsQc1aiLLZrA5H9JEqKIbD+9Eet9WhCCGEEJckiXEkmzg1JMsp9Oa1\n0O2VbhQhRuUXg2nCvh1WhyKEEEJckiTGEUzZ7SE57EOXvQcjclEjc60ORfTH2AkQFSN1xkIIIYKW\nJMYRTk2fF1LlFPrUCaiqlE13IUjZHTCxEL17G1prq8MRQgghLiKJcaTrKacIjWEfumwVGAZq1kKr\nQxEDoAqKofYM1JyyOhQhhBDiIpIYR7gPyyk2BX05hTa70Rveh4ISVGKy1eGIAVAFvi4i0p1CCCFE\nMJLEWIROOcX+XdBQK5vuQll6JqRlSGIshBAiKEliLEKmnEKvXwUxsaipM6wORQyQUsp31XjfTrQ3\nuO9QCCGEiDySGIuPlVN0Wh3OJen2NnT5etT0eShnlNXhiEFQ+cXQ0QaH9lsdihBCCHEBSYwF8JFy\niiC9xa3Ly6CzAzVXyihC3sQpYBjStk0IIUTQkcRY+Jwvp9ganMM+dNkqSB8OYydZHYoYJOWKhTET\npM5YCCFE0JHEWADBXU6h62pg/07UnGtRSlkdjvADVVAMxw6hW5qsDkUIIYToIYmx6KGmzw/Kcgq9\n4T3QGjX7aqtDEX6iCkpAa/SeCqtDEUIIIXpIYiw+NHEKxMYHVTmF1tqXGI8vQKUPtzoc4S+jxkJs\nPEhiLIQQIohIYix69JRTVGwMnnKKqko4fRI1W0ZAhxNl2FCTpqL3yHhoIYQQwUMSY3EBNW0etLcF\nTTmFLnsPHE5fmYcIL/lF0FgP1cesjkQIIYQAJDEWH3e+nCIIhn3ori70pjWo4tmoGJfV4Qg/+3A8\ndLnFkQghhBA+khiLCwRVd4qdm8HTipojZRThSKWkQ+ZIadsmhBAiaEhiLC4SLOUUZtl7kJgCk4os\njUMEjioohsrd6M4Oq0MRQgghJDEWlxAE5RS6pQl2bkHNWoiy2SyLQwSWyi8GbxdU7rY6FCGEEEIS\nY3GxYCin0JvWQne3jIAOd+Mng90h46GFEEIEBUmMxSV9WE5hzcYoXbYKcsagskdZcnwxNFRUFIzL\nlzpjIYQQQUESY3FpPeUUQz/sQ588BkcPoubI1eJIoAqKofoYuqHO6lCEEEJEOEmMxSVZWU6hy1aB\nYaBmLhjS4wpr9LRtk3IKIYQQFpPEWFyWmj5/yMsptNmN3vg+TJ6GSkgasuMKC2WPhsRky7ugCCGE\nEJIYi8ubUDj05RT7dkBjPYZsuosYSilUfhF6bwXa7LY6HCGEEBFMEmNxWcpuR5XMGdJyCr1+Fbhi\nYcqMITmeCBL5xdDaAkcPWx2JEEKICCaJsbiioexOods96G1lqBlXoRzOgB9PBA+V7xviInXGQggh\nrCSJsbiy8+UUmwNfTqG3rofOTulGEYFUQhLkjEFb1B5QCCGEAEmMRS96yil2BL6cQpe9B8OyYMyE\ngB5HBCdVUAyH96PbPFaHIoQQIkJJYix6NRTlFLr2DOzfiZpzDUqpgB1HBC+VXwzd3bB/h9WhCCGE\nCCB9YA8ty59Cd7RbHcpFJDEWvZtQCHGBLafQG94HQM2+OmDHEEEubxJERaN3V1gdiRBCiADRXi/m\ny8/Qvv5dq0O5JEmMRa98wz7Odafo7PD782utfWUUEwpRaRl+f34RGpTdARMKpc5YCCHCmH73dTh5\nlPivfhcVFW11OBeRxFj0iZo2DzraAjOE4fB+OFuNmnON/59bhBSVXww1p9FnT1kdihBCCD/TdWfR\nr6+AollEz7zK6nAuSRJj0TfnyykCMOxDl60CpxM1ba7fn1uEFhkPLYQQ4ct85XkAjDuWWhzJ5Uli\nLPokUOUUuqsTvXktqngOKtrlt+cVISojC1KHSZ2xEEKEGV2xESo2ohYvQaWmWx3OZUliLPpMTQ9A\nOcWOzeBxo2QEtODceOiCYti3He31Wh2OEEIIP9Ad7ZgrnoPsUajrFlsdzhVJYiz6bsIUv5dTmOtX\nQVIKTJzit+cUoU3lF/vaAx7eb3UoQggh/EC/8QrU12B88W6U3W51OFckibHoM2Wz+bWcQjc3wq6t\nqFlXowybHyIUYWHSFFCG1BkLIUQY0CePoktXouYtQo3LtzqcXkliLPrFn+UUetNqME0ZAS0uoFxx\nMGY8OhAdUIQQQgwZbZqYLz0DMS7UbXdaHU6fSGIs+qennOKDQT+VLnsPRuWhsnP8EJgIJyq/GI4e\nRLc2Wx2KEEKIAdLr34WDe1C33YmKT7A6nD6RxFj0y4flFJsHVU6hTxyBY4flarG4JFVQDFqj9263\nOhQhhBADoFua0a8uh7x81NzrrA6nzyQxFv3mj3IKXfYe2GyoIG3wLSw2ehy4YgMzUEYIIUTA6VeX\nQ5sH40t3o4zQSTdDJ1IRPAZZTqG7u9Eb34fC6aj4RP/GJsKCstlg0lT07m1ora0ORwghRD/oA3vQ\n60pRiz6Nyh5ldTj9Iomx6LdBl1PsrYCmBgwZAS2uQOUXQ2MdnDpudShCCCH6SHu9mC8/AynpqFvu\nsDqcfpPEWAxITznFrvJ+/6wuew9ccVA4IwCRiXDRMx5ayimEECJk6Hdfh5NHMZYsRUVFWx1Ov0li\nLAbmfDnF1v4N+9AeN3rbBtTMBSiHI0DBiXCgUofB8Gz07v5/+BJCCDH0dN1Z9OsroGgWqmiW1eEM\niCTGYkAGWk6ht66Drk6UlFGIPlAFJVC5G93VaXUoQgghemG+8jwAxh1LLY5k4CQxFgOmps/vdzmF\n3vAeDM+G3PEBjEyEC5VfBF2dcGC31aEIIYS4Al2xESo2ohYvQaWmWx3OgEliLAZuQiHEJfS5nELX\nnIbK3ajZ16CUCnBwIixMKAS7Hb27wupIhBBCXIbuaMdc8Rxkj0Jdt9jqcAZFEmMxYMpmQ5XMQW/f\n1KdyCr3hfVAKNVvKKETfqKhoyMuXOmMhhAhi+o1XoL4G44t3o+x2q8MZFEmMxaCoafOgo73Xcgqt\nNbpsFUwoDOlbLGLoqfxiOHkU3VhvdShCCCE+Rp88ii5diZq3CDUu3+pwBk0SYzE458spehv2cWgv\n1JyWTXei33ratu2Rtm1CCBFMtGlivvQMxLhQt91pdTh+IYmxGJSecoodV+5OodevAmcUqmTuEEYn\nwsKI0RCfCFJnLIQQQUWvfxcO7kHddicqPsHqcPyi10KQ2tpafve739HY2IhSikWLFnHjjTfS2trK\nY489Rk1NDenp6dx///3ExcWhtebFF19k27ZtREVFcc899zBmzJih+F2ERdS0eeg1b/vKKUrmXPR1\n3dmB3rIOVTIXFR1jQYQilCnDQBUUo3eVo00TZcjneSGEsJpuaUa/uhzy8lFzr7M6HL/p9R3GZrPx\n5S9/mccee4xf/OIXvP3225w4cYLXXnuNwsJCli1bRmFhIa+99hoA27Zt4/Tp0yxbtoylS5fywgsv\nBPyXEBbrpZxCb98MbW7U3GuHODARNvKLobUZjh+2OhIhhBDgS4rbPBhfujusLlj0+pskJyf3XPGN\niYkhOzub+vp6Nm/ezMKFCwFYuHAhmzdvBmDLli0sWLAApRTjx4/H7XbT0NAQwF9BWK23cgpdtgqS\n02DCZAuiE+FA5RcBMh5aCCGCgT6wB72uFLXo06jsUVaH41f9SvHPnj1LVVUVeXl5NDU1kZycDEBS\nUhJNTU0A1NfXk5aW1vMzqamp1NfLbvJw92F3iq0XPK6bGmB3OWr21SjDZlF0ItSpxGQYkSuJsRBC\nWEx7vZgvPwMp6ahb7rA6HL/rc7O59vZ2Hn30Ue68805cLtcFX1NK9XtgQ2lpKaWlpQA8/PDDFyTT\nQ8lut1t27HCi511Nze+TcOzcQtInFvesq3vdO7SaJik3fha7rLNfROo52zJjLp43/kBKbAxGTGxA\njhGpazsUZG0DR9Y2cGRtL+b+35dpPXmUxB/9mujsEQN+nmBd2z4lxl6vl0cffZSrrrqKWbNmAZCY\nmEhDQwPJyck0NDSQkODbjZiSkkJtbW3Pz9bV1ZGSknLRcy5atIhFixb1/P+P/sxQSktLs+zYYado\nFh0bV1NTfZL0rGxqa2vpLn0TcsfTGB0Hss5+EannrM6dCF4vdetXo6bODMgxInVth4KsbeDI2gaO\nrO2FdN1ZzFdegKJZtI6ZROsg1sbKtc3Kyrrs13otpdBa8+yzz5Kdnc3NN9/c8/j06dNZvXo1AKtX\nr2bGjBk9j69ZswatNZWVlbhcrp6SCxHe1PT5F5RT6ONVcKJKehcL/8jLB2eUlFMIIYRFzFeeB8C4\nY6nFkQROr1eM9+/fz5o1a8jJyeEHP/gBAEuWLOHWW2/lscceY9WqVT3t2gCKi4spLy/n3nvvxel0\ncs899wT2NxDBY/zkc90p1sEnFvs23dnsqBlXWR2ZCAPK4YAJhZIYCyGEBXTFBqjYiLr9zrCeYNtr\nYjxx4kT++Mc/XvJrDzzwwEWPKaX46le/OvjIRMjxdaeYi974PmabG71xNUyZjooLj6bfwnoqvwi9\ncwu65jQqfbjV4QghRETQHe2YK56H7FGo6xZbHU5AhU/jOREU1HRfd4qWf1sGzY0Yc6R3sfAfVVAC\ngN4jU/CEEGKo6DdegfoajC/ejbL3uW9DSJLEWPjXuXKK9tI3IC4eCqdZHZEIJ8OzISUNvbvc6kiE\nECIi6JNH0aUrUfMWocblWx1OwEliLPzqfDkFgJqxAGV3WByRCCdKKd9V43070N3dVocjhBBhTZsm\n5kvPQIwLddudVoczJCQxFn6n5l4LzijUgk9YHYoIQyq/CNo8ULXf6lCEECKs6fXvwsE9qNvvQsVH\nxn4hSYyF36mxExm24l3UiFyrQxHhaFIRKAO9W+qMhRAiUHRLM/rV5ZCXj4qg/UIRmxh3dWp2lXvo\n6jStDiUsKSNiTy0RYCo2DnLHSZ2xEEIEkH51ObR5ML50d0S9p0fOb/oxTQ1eqg52svqdM2itrQ5H\nCNEPKr8YjhxEu1usDkUIIcKOPrAHva4UtejTqOxRVoczpCI2MU7LcDC5KIbjR9zs29FudThCiH5Q\nBcWgTdi73epQhBAirGivF/PlZyAlHXXLHVaHM+QiNjEGGD3OyYSCBA7u6+B4VafV4Qgh+ip3PMTE\nSj9jIYTwM/3u63DyKMaSpaioaKvDGXIRnRgrpZh9VTppGXa2b/FQV+O1OiQhRB8omw0mTUHvLpdS\nKCGE8BNddxb9+goomoUqmmV1OJaI6MQYwLApps114Yo12LLOjadVeqMKEQpUfjHU18Lpk1aHIoQQ\nYcFc8RwAxh1LLY7EOhGfGAM4nQYzr4pFm7BprZuuLrkCJUSwU/lFANKdQggh/EBXbIDtm1CLl6BS\n060OxzKSGJ8TF29j2jwXrS0m5WVutCnJsQhOHe1yVwNApQ+HYVno3dusDkUIy2jTxGxutDoMEeJ0\nRzvmiuchexTqusVWh2MpSYw/Ij3DweSSGM6e8rJnu3SqEMFn3842/uv3VZw6IZtF4Vx3isqd6K4u\nq0MRYshpjxtz2UPU/N0t6J1brQ5HhDD9xitQX4PxxbtRdrvV4VhKEuOPGZ0XRe44J4crOzh6qMPq\ncIToUXWggwN7OrA7FBWbPLilHt6XGHd2wsE9VocixJDSZ09hPvxD2LcDW1oG5rMPow/LmHTRf/rk\nUXTpStT861Hj8q0Ox3KSGF9CflEM6cPt7NzaRu1ZuRIlrFd9vJNd5W1kZNn59OdzUEqxdb2H7u4I\nL/mZUAg2u5RTiIii9+/E/OX3obkR476HSH74OUhMxnzyn9GnTlgdnggh2jQxX3oGYlyo275idThB\nQRLjSzAMxbQ5scTGG2xZJ1fmhLVqz3rZtsFDcqqNkjmxJCQ6KJrpoqmhm93b2qwOz1IqOgbyJkli\nLCKGueZtzMcegPhEjB//BjVxCrakFIz7HgJlYD7+U3R9rdVhihCh178LB/egbr8LFZdgdThBQRLj\ny3A4FTOvigVg0xo3XZ2mxRGJSNTc2M3mD1pxxfk6p9jtCoDh2Q7GTozi6KFOTh6N7HpjlV8EJ6rQ\nTQ1WhyJEwOjubsw/vID+z9/BxCkYP/oNalhWz9fVsEyM+x4ETyvmEw+i3a3WBStCgm5pRr+6HPLy\nUXOutTqcoCGJ8RXExtmYPi8Wd6vJ1jIPpnSqEEPI4zbZuKYVu10xa0EczqgL/1wnFkaTnGZj+xYP\nrc2Re1dDFZQAyBQ8Eba0x4351M/Qpa+jrrsF49sPoFyxF32fyhmLcc+P4Wy17/s7ZZ+MuDz96nJo\n82B86W6UIengebISvUgbZmfK9BhqTnvZUxHZt63F0OnsMNmwuhWvVzNrQRyu2Iv/VM+X/Nhsii3r\n3Xi9EfrBbWQuxCfCHimnEOGnZ5Pd3u2oL9+DccfXfJMfL0NNmorx99+FQ/swn/sNujtyPzSLy9MH\n9qDXlaIWfRqVPcrqcIKKJMZ9kDMmijHjo6g60MmRg/IJXASW16vZtNZNm9tk5vw4EpIu/yYY4zIo\nnuWipclkV3lkfnBThoGaVITevQ1tSsmTCB8f32RnLPhkn35OTZ+PWvJ12L4J/dLTMjZdXEB7vZgv\nPQ0p6ahb7rA6nKAjiXEf5U+NZlimnV3lbdSclk4VIjBMU1Ne5qahvpuSOS5Sh/XeT3JYpoNx+VEc\nr+rkeFWE1hsXFEFLE5yosjoSIfziUpvs+sO45kbUzZ9Hf/AO+rWXAxSlCEW6dCVUH8NYshQVFW11\nOEFHEuM+UoaiZE4scQkGW9d7aG2R21PCv7TW7NjSxplqL4UlMWSOcPb5ZycURJM6zM6OrR6aGyPv\n3FT5xQDo3VJnLEJbb5vs+kMt/gLqqk+g3/oj5rtv+jlSEYp03VnfMI+iWaiiWVaHE5QkMe4Hh8PX\nqUIZvk4VndKpQvjR/l3tHK/qZFx+FKPzovr1s8pQlMx24XAotq534+2KrFunKikFskehd5dbHYoQ\nA9bXTXZ9pZRCffFuKJqN/sPzmJvX+jFaEYrMFc8BYNyx1OJIgpckxv3kivV1qmjzmGxdL50qhH8c\nOTfVLmeMkwmTB3ZrKzrGoGS2i9ZWkx1bPRFXV6gKSuDgXnSHjHMXoae/m+z6StlsGF/7nq/f9+8f\nk+4tEUxXbIDtm1CLl6BS060OJ2hJYjwAqel2pkx3UXvGy67ytohLQIR/VR/vZOe5qXaF02JQSg34\nudIyHEwoiObk0S6OHY6semNVUAzdXti/0+pQhOiXgW6y6yvljML41k9geDbm079CHz3o1+cXwU93\ntGOueB6yR6GuW2x1OEFNEuMBGpnrJO/cgIUjByIrARH+8/GpdoYx8KT4vHH5UaQP920UbWrw+iHK\nEDEuH5xOuSImQspgN9n1lXLF+QaAxMVjPvEQ+kx1QI4jgpN+4xWor/H1LLb3vqk7kkliPAgTp0ST\nkW1nV0UbZ09JpwrRP5ebajdYSimKZ7lwRim2rvfQFSH1xsrhhPGTpc5YhISLN9k9MuBNdn2lklJ9\no6O19o2OlmmREUGfPIouXYmafz0qL9/qcIKeJMaDoJSiZFYsCYkGW8vctETw9DHRP71NtRusqGiD\nkjmxeNwm2zdHTr2xyi+G0yfRdWetDkWIy7r0Jru4ITm2Gp6Nce8D0NKE+fiDaI97SI4rrKFNE/Ol\nZyDGhbrtK1aHExIkMR4ku0MxY34cNpvydarokE4V4so6O0w29jLVzh9S0+1MLIzm1PEujhyMjHIf\nVXC+bZtMwRPBKVCb7PpD5Y7H+MY/wqljmE//Et0VGa8PkUivfxcO7kHdfhcqLsHqcEKCJMZ+4Io1\nmDEvlvY2ky3r3JjdkXF1TvTf+al2nj5MtfOHsROjyMiys7uijca6CKg3zhwJSamSGIugpPfvCugm\nu/5Qk0tQd34H9u/E/P1v0abc8Qw3uqUZ/epyyMtHzbnW6nBChiTGfpKcZmfqTBd1Nd3s3CqdKsTF\neqba1fV9qt1gKaUomukiOlqxpcwT9r23lVK+q8b7tqO75Y1eBA/fJrv/F/BNdv1hzL4a9bm/h63r\n0Suel/etMKNfXQ5tHt+GO0PSvb6SlfKjEaOcjMuP4lhVJ4crO6wORwSRC6baTevfVLvBckYZTJvr\nu6NRsTEC6o0LisHjhiMHrI5ECEs22fWHcf2nUTd8Fv3+W+g//8HqcISf6AN70OtKUYs+jcoeZXU4\nIUUSYz+bMDmazBEO9lS0c6ZaOlUIn8FMtfOH5FQ7+VNjOFPt5fD+8P7QpiZNBaWknEJYzspNdv2h\nbvsKas616JX/hbn6r1aHIwZJe72YLz0NKemoW+6wOpyQI4mxnymlKJrlIjHZxtYyN82Ncjs30vVM\ntcsd+FQ7f8gd52T4CAd7d7RTXxu+9cYqLgFGj0PvkcRYWCcYNtn1lVIK9X+/BYXT0S8/iy5fb3VI\nYhB06UqoPoaxZCkqyrr3nFAliXEA2O2KGfN9fWk3feCmoz286zrF5V0w1W764KbaDZZSiqIZLmJc\nBlvXu+kI4w4qKr8IDleiPa1WhyIiUDBtsusrZbdjfP2HkDsO8/lH0ft3WR2SGABdd9Y3zKNoFqpo\nltXhhCRJjAMkxmUwc34sHe0mm9e56ZZOFREnEFPtBsvhVEyb66KzQ7NtQ/jWG6uCEtAm7N1hdSgi\nwphr/xZ0m+z6SkVFY3z7/0H6cMzf/Rx9vMrqkEQ/mSueA8C4Y6nFkYQuSYwDKCnVTvFMFw213ezY\nEr5JiLhYz1S7WP9OtfOHpBQ7BcUx1Jz2cnBvmNYb546H6BgppxBDRpvnNtn9x1NBucmur1Rcgm90\ndLQL84kH0TWnrQ5J9JGu2ADbN6EWL0GlplsdTsiSxDjAsnJ8daUnjnRxaF+YJiHiAhdMtVvo/6l2\n/jBqrJOsHAf7drVTezb86o2V3Q4Tp6J3b5MPpCLgtMeN+WTwb7LrK5WSjvGdB6Gryzcdr6XJ6pBE\nL3RHO+aK5yF7FOq6xVaHE9KC7x07DI3LjyIrx7fp6dQJmTAUzoZqqt1gKaWYOt1FbJxBeVl41sGr\ngiKoOwtnqq0ORYSxCzbZfSm4N9n1h8rO8ZVVNNZiPvEQur3N6pDEFeg3XoH6Gl/PYnvge+SHs+B8\n1w4z5zc9JaXY2LbRQ1ND+F2hExdOtZsxBFPtBsvuUEyfG0tXl6Z8gwdthteVVVVQAsh4aBE4PZvs\nmhp8m+wWBv8mu/5QeZMwlv4Qjh/GfOZhtFdakAYjffIounQlav71qLx8q8MJeZIYDxHbuU4VDoev\nU0V7W/hdoYtkH51qVzzbRdoQTLXzh4QkG4UlMdSe8VK5p93qcPxKpQ+H9OHo3eVWhyLC0AWb7P7p\nX0Jqk11/qKkzfa3c9mxDv7gMbcp7VzDRpon50jMQ40Ld9hWrwwkLkhgPoegY30asrg7N5g+kU0W4\n0Fqz8yNT7bJGDt1UO38YmetkxGgHlbs7qDkdXleEVEEJ7N+J7gqv30tYJ1w22fWHMW8R6rP/F71p\nNfq//03q9oOIXv8uHNyDuv0uXw93MWiSGA+xxGQ7xbNdNNZ3s32TdKoIB/t3tXPMwql2g6WUonCa\ni/gEg/INnrBnQD0YAAAgAElEQVS6m6EKiqGzAw7ttToUEQZ8m+x+Hjab7PpDffI21HW3oEtfR//1\nf6wORwC6pRn9p+WQl4+ac63V4YQNSYwtkDnCycTCaE4e6+JAuLbLihDBMtVusOx2xbR5sXR3a7aW\nuTHDpd54QiHYbNK2TQzah5vsKsJqk11fKaVQn/t71MwF6P/5d8x1pVaHFPH0q8uh3ePbcGdIOucv\nspIWyZsURfYoB/t3tlN9XDpVhKJTJ4Jnqp0/xCfYmDLNRX1NN/t3hUe9sYpxwdiJsgFPDEq4b7Lr\nK2UYqLu+A/lF6P94Cr19s9UhRSx9YA96XSlq0adR2aOsDiesSGJsEaUUU2e4SE71daporJdOFaGk\n9qyX8rLgmmrnDyNGO8kZ4+Tg3g7OVIdHXa7KL4Zjh9HNjVaHIkLQh5vsEjB+HL6b7PpK2R0Yd/8j\njByD+dyv0QelTGmoaa8X86WnISUddcsdVocTdiQxtpDN5utUERWl2CydKkJGME+184fJxTEkJBls\n2+jB4w79c1IVFAOg91RYHIkIJRdsspswBeNHv0FlhPcmu75S0S6Mex+ApDTfYJPqY1aHFFF06Uqo\nPoaxZCkqKnRL+IKVJMYWi4o2mHlVHF1dvh64Xm+Y1HaGqVCYajdYNrti2txY9LkWdGaod0/JGQNx\n8SB1xqKPLtpkd2/kbLLrK5WQ5Bsd7XD4puPV11gdUkTQdWd9wzyKZqGKZlkdTlgKv3f1EJSQZKNk\ndixNDd1USKeKoBUqU+38IS7extSZLhrqutm7I7TrjZVhQ00qQu+pkL8t0atI32TXHyp9uG90dLvH\nlxy7W6wOKeyZK54DwLhjqcWRhK/wfWfvg6ONwdMRYni2g/yp0Zw63kXl7tBORMJRqE2184eskU5G\n5zk5XNkR+qPMC4qhqQFOHLE6EhHE9P5dmL+STXb9oUbmYnzzJ1Bz2ldW0RE876vhRldsgO2bUIuX\noFLTrQ4nbEVsYlxe3cq9f67iN6sO0tYVHHWUYyZEMTLXSeXuDk4eC/FEJIyE6lQ7f8gviiEx2UbF\nJg/u1m6rwxkwlX++zljKKcSl9Wyyi5NNdv2lJkzG+Nr34PB+zH/9Ndorm8n9TXe0Y654HrJHoa5b\nbHU4YS1iE+PJGS5unZTCyp2nue+tKnaf9Vgd0rlBCzGkpPsSkYY6eXGx2gVT7UpCb6rdYNlsiulz\nXSgUW9d7QnZao0pOhawcadsmLiKb7PxDlcxFffFu2LnF18pNypb8Sr/xCtTX+HoW2yPn4owVIjYx\ndtoM7ioZxlO3FwLwT+8c48Xys3R2W3v12JeIxBIdbbD5AzdtnuC4mh2pLphqNy70ptr5gyvORtEs\nF00N3eypaLM6nAFTBcVwYLfc6hU9Lthkd+3NsslukIyFn0Qt/gK6bBX6f/7D6nDChj5xBF26EjX/\nelRevtXhhL2ITYzPK8pO5PEbc7lhXBKv7a3nu385wsE6a2t8fZ0qYun2SqcKKx056JtqNzLEp9r5\nw/BsB2MmRHHkYGfIlvmoghLweqFyl9WhiCBw0Sa7JUtlk50fqJs/j7r6U+i/vor5zkqrwwl52jQx\nX34GYlyo275idTgRIeITY4AYh8HdM4fz02tG4Ok0+cHbR1ixowavhWNx4xNtlMyNpbmpm20bpFPF\nUDt1opOdW31T7aaEwVQ7f5g0JZrkVBvbN3tobQnBeuNx+eBwSp2xkE12AaSUQi1ZCiVz0X/8PeaG\n960OKaTp9e/Cwb2o2+9CxSVYHU5EkMT4I0qy4lh2Uy4LRifwys46fvj2EY5Z2LkiI9NBwdRoTp/s\nYt9O6VQxVOrCdKrdYBmGr7+xYSi2rnPTHWJ3MpQzCsYVSJ1xhJNNdoGnDBvGV78LEwrRy5+Qv7kB\n0i3N6D8th7x81JxrrQ4nYkhi/DFxUTbun5vFP16VTa3by3f/coT/3VNHt0VXj3PHR/WM6D1xJDRv\nYYeS5sZuNn3QSkyYTrUbrBiXQfFsF81NJru2hV69sSooglPHZRhBBPJtsvu9bLIbIsrhxLjnx5CZ\ng/nMr9BVB6wOKeToV5dDu8e34c6QdG2oyEpfxpyceJbdnEtJVizLt9Xwk9JjnGoZ+sT0fKeK1GF2\ntm/2UF8rnSoC5aNT7WaH6VQ7f8jIdJA3KYpjhzs5HmIf1lRBCYBcwYowH26yWymb7IaQcsVifOen\nEJ+Iuewh9OmTVocUMvSBPeh1pahFn0Zlj7I6nIgi7/xXkBRt50cLsrlvTiZHGzu4760q/lLZMOT1\nvobha5kV4/J1qvC4pVOFv0XSVDt/mDA5mtR0Gzu3eGhpCqF646wcSEoBSYwjRs8muz3bUF+8WzbZ\nDTGVlIJx30OgFObjP0U31lkdUtDTXi/mS09DSjrqljusDifiyLt/L5RSXDMmkWU35zIxLYZnN5/h\nwfdOUOvpGtI4nFEGM66KxTQ1m9a24u0KrfrOYBaJU+0GyzAUJXNisdkVW9aHTucUpRQqvxi9dzva\nDKGEXgzIRZvsrv6U1SFFJJWR5bty3NriGx3tabU6pKCmS1dC9TGML3wdFRXZHZGsIIlxH6W5HDx4\n7Ui+MSODvWc93PtmFe8dbhrSq8fxCTamz42ltdmkfIMbbWHXjHARyVPtBis6xqBkjovWZpOdW0Ko\nc0p+EXha4chBqyMRAXTRJrtJU60OKaKpUXkY9/wITp/EfOrn6E7pJ34puu6sb5hH0SzU1JlWhxOR\nJDHuB6UUnxqfzBM35ZKTFMXjZad4eO1JGtuHru43fbiDguIYzlR72SudKgZFa83OrZE71c4f0jMc\nTJgczYmjXRyvCo16Y5VfDEpJ27YwJZvsgpfKL0L9/f1wcC/m84/KXZtLMFc8B4Bxx1KLI4lckhgP\nQGa8k18syuHO4nS2nHRz75tVlB1vGbLj546LYnSek0P7Ojh2WD51D9T+Xe0cOxzZU+38YdykKNIy\n7Owsb6OpIfjf6FR8AuSMlQ14YUi3eTCf+oVssgtixoyrUJ//GlRsQL/8bOjcaRoCumIDbN+EWrwE\nlZpudTgRq9f7xk8//TTl5eUkJiby6KOPAtDa2spjjz1GTU0N6enp3H///cTFxaG15sUXX2Tbtm1E\nRUVxzz33MGbMmID/ElawGYrP5KcyLSuOx8uqeXjNSa4encDXZmQQ5wx8jWpBcQytLSY7trYRG28j\nNV1KAPpDptr5jzIUJbNdrH67ha3r3Vz1iXgcjuBuc6cKitF/fRXtcaNcsVaHI/xA15zGfPJncOak\nb5Od1BMHLeO6mzGbG9Bv/TckJKM+/QWrQ7Kcbm/zXS3OHoW6brHV4US0Xq8YX3311fz4xz++4LHX\nXnuNwsJCli1bRmFhIa+99hoA27Zt4/Tp0yxbtoylS5fywgsvBCbqIJKTFMUjN4zmjsJU1hxt5t43\nq9h2yh3w4/qGLbhwxfo6Vbhbg/9KXbCQqXb+FxVtMG1OLB63yY7NwV9vrAqKwTRh3w6rQxF+oPfv\nwvzl92STXQhRt34JNW8R+s1XMN97y+pwLKfffAXqa309i+1yoctKvSbG+fn5xMVdeCtq8+bNLFy4\nEICFCxeyefNmALZs2cKCBQtQSjF+/HjcbjcNDQ0BCDu42A3FkinpPHLDKGIcBg+uOs6zm07T1hXY\ntmpOp28IBcCmtW66pFNFr2SqXeCkDrMzoTCa6uNdHD0Y5PXGYyZAVIzUGYcB2WQXmpRSqC9/E6bO\nRK/4V/TWdVaHZBl94gi69HXU/OtReflWhxPxBlRj3NTURHJyMgBJSUk0NTUBUF9fT1paWs/3paam\nUl9f74cwQ8O41Bgeu3E0t05K4a8HGrnvrSr2nPUE9Jhx8Tamz3XhbjEpL3NjSqeKy5KpdoGXNzGK\nYZl2dle00VgfvMNolN0BEwvRu7cF/dVtcWmyyS70KZsN42s/gLETMV94FL13u9UhDTltmpgvPwMx\nLtRtX7E6HEEfaox7o5Qa0K3o0tJSSktLAXj44YcvSKiHkt1u9/uxf/CJYVxf0MQv3jnAj985xh0l\n2Xxtziii7IHZ65iWBugmylbXUFUJs+Zbs5YfFYh1HYzWli7e/eAEDoeNG28dQVyCw+qQBizY1vbj\nrrsxhdf/cJxtG9tZ/LmRREUFZ19oz8yraHl+E8neDuyZI4DgX9tQ5s+1NT1umn77AJ1by4i56Xbi\n77oXZYvc28+hft6aDzxG/T/djfnMr0j6+e9wjJlgdUg9Ar22baVv0nxwLwnf+jExo8NzT9blBOt5\nO6BXksTERBoaGkhOTqahoYGEhAQAUlJSqK2t7fm+uro6UlJSLvkcixYtYtGiRT3//6M/N5TS0tIC\ncuwRUfDbG3JYvu0sK8pP8sGhGu6bk0VeamA2eqUNh9xxTvZsb8Lu6GTUWGu7LARqXQeis8Nk3but\ndHaazLs2nvbOJtqDI7R+8XR187eDjVQ2dHPj2DgmZ7isDumyimdFs25VK6v+coLp81xBWcetR48D\noP6DdzGuuQkIrvM23PhrbT++ya7z6k9R19DohwhDVzict/pb/w/96x9S/9D9GP/wa9SwTKtDAgK7\ntrqlGXP5k5CXT2vhTNwh/t+wv6w8b7OyLn93aUCXMKdPn87q1asBWL16NTNmzOh5fM2aNWitqays\nxOVy9ZRcRKIYh8HdM4fz02tG4Ok0+eHbR3hlRy3eAJU75BfFMCzTzs6tbdSeGdrJfMEqHKbaNbd7\neXl7DV977RAvltew/WQTPyk9xu+3nqHDG5zjwZPT7EyaGs3pk10crgzSloLpmZCWIW3bQoiulE12\n4UqlpGHc98/Q3e0bHd0c/vuT9KvLod3j23BnSPfcYGF78MEHH7zSNzz++OP84Q9/oK6ujtLSUlwu\nF9dddx0rV67k1VdfpbW1lbvuugun08nw4cOprKxk+fLlVFRU8PWvf/2yV4w/rqVl6PoAf5TL5cLj\nCWwdcGa8k+vGJFLr8fJmZQNbq91MGhZDYrR/b/0ppcjIdHD6ZBfHq7oYPsKBM8qaP7ahWNfemKZm\n63o3tWe7KZnjIiMrtMonatxdrNhRy2PrT7HjjIfizFi+MyeT+xdNpKaxlT9XNlJ2vIXxadGkuoLv\nd0tOtdHcaHLkYCfpGXZiXMH1wq+UgtMnoGIT6hO3ogxbUJy34Wqwa2uu/Rv6ud9AcirG936BOnfF\nXwTH660/qPgE1PjJ6PfeQu8uR81cgHJY+9oWqLXVB/agX3kOdcNnMWYt9PvzhwIrz9v4+PjLfk3p\nINl5Ul1dbclxh/pSftmxFp4+17Hii1PTWDwxBZufOyO4W7tZ+04rzijFVYvicDiHPiGx+tae1pod\nW9o4driTySUx5IbQAI8TTR28uqee1VVNaGDh6AQ+m59KTpLvdzi/thWn3CzbcIqGNi+3F6Tyuclp\nOGzBVbLQ1Wmy5m+tmFqz8BPxln1Quxy9bQPm07/E+P4vURMmW37ehrOBrq02u9H/vRxduhLyizC+\n/kMZ2vEx4Xbe6p1bMZ/6GUwoxPj2A5Ymx4FYW+31Yv7sPuhox3joKVRUZPbSD6tSCjFwc3LiefLm\nXEqyYlm+rYaflB7jVIt/W1vFxtmYMc/XU3bLek9Edqqo3P3hVLtQSYoP1LXx8JoTfOvNKj442swn\nxyfzr4vHct/crJ6k+KOKMmNZdlMuV+cm8Mdddfzw7SMcaQiuMeEOp8G0uS462zXbNgZhf+MJhWAY\n0rYtSF0wye6amzDu/akkxRFAFU5DfeVe2Lsd/W+Poc3gLBkbKF26EqqPYSxZGrFJcTDrtZRiqIRz\nKcXHRdsN5o+KZ3ick1WHm/jLgQbinDbyUqL9tknJFWsQHaOoquykq1MPeRmBlbdIjhzsYO/2dkbm\nOikoDu4BHlprdpzx8LuNp/nPiloa2rx8elIK35+fxbycBGIvMUXxo2vrtBnMHhlPbnIUa4428+b+\nRuyGYkJaDEaQ/N7RMQZOp6LqQCeGTQXVlEblcKL3VMDpkxgLbgibW9LBqL9rq2tOYz76E6iqRH3h\nGxi33CF1mJcRjuetGpkLUdHo0tfB3QKTp1nyWu7vtdV1Z9H/+ggUTse4ZYnfnjcUBWspRfC8Q0UY\npRTXjElkcoaLpzac4tnNZ9hwopVvzx5Omp/qRXPGRNHabHJofwdxCbaQuXI6GOen2g3LDO6pdqbW\nbDzeyqt76jhQ105ytI2vFKfzyXFJuBz93yA4e2Q8k9JjeGbTaf6jooaNJ1q5b04mWQnOAETff6Py\nnNTVeNm/s52UVDupw4LnpUcVFKFfX4FuaTrX+1BYTVfuwnzmV2BqjPsekqEdEcq44TO+0dF/ew0S\nk1E3fc7qkAbNXPEcAMYdSy2ORFyOfPy2WHqsgwevHck3ZmSw96yHe9+s4v2qJr/dcp40JZqMLDu7\nt7VRczq8O1V8dKrdtLnBOdWuq1tTeqiRb71ZxcNrT9LS0c3dMzN47taxfDY/dUBJ8XmJ0Xb+4aps\n7p+byYnmDu57q4o/72/ADILyBaUUU2b4RphvLXPT0R48t0ZVQQlo7btyLCxnrv0b5m8fkEl2AgB1\n252o2VejX3sJc+3frA5nUHTFBti+CbV4CSo13epwxGVIYhwElFJ8anwyT9yUS05SFI+tP8Wv156k\nqX3wk8OUoSiZHUtcgsGW9W5amrv9EHHw+ehUuxlBONWu3Wvyxr56vvH6IZ7ccBqnTfG9eVk8fcsY\nPjkuGafNP3+KSimuzk3kyZtyyR/m4rktZ3hw1XFq3NZ/KHI4FNPmxtLVpSnf4EEHS+37qLEQGw+S\nGFtKm92Yfzw/yW6yTLITACjD8NUbTy5B/+fTvuQyBOn2Nt/V4uxRqOsWWx2OuAJJjINIZryTXyzK\n4SvF6Ww+6ebbb1ax4fjga6/tDsXMq+IwDMXmtW46O4Lnap0/eNwmG9e0YrMpZi+MIyqIOh+0dHTz\nys5avvraIV7YepZhsQ4euHoEj31qNAtGJ/i9I8l5qS4HP71mBPfMHM7+2jbu/XMVqw77707EQCUm\n2ygsiaH2jJcDe4Ojv7EybKhJU9F7ZDy0VXo22b0jm+zExZTdjvH1f4DReZjP/Qu6crfVIfWbfvMV\nqK/19Sy2B08pmbhY8GQQAgCbofhsfiqPfWo0abF2frXmJI+tr6a1c3BXel2xBjPmx9LmCa9OFZ0d\nvqTY69XMXhiHKzY4Tuk6Txcvlp/lq68dYsWOWiamRfPw9Tn86hOjmJYdNyS1z0opbhiXxBM35jI6\nKYonyk7xyzUnaWwb/J2IwRiZ62TEKAf7d7VTEyyDaPKLoLEe77HDVkcScXTNacxf/QB2l6O+eDfG\nF76OsoXeIB4RWCo6BuPbD0BqOubvfo4+ccTqkPpMnziCLn0dNf96VF6+1eGIXgRHFiEukpMUxSM3\njOaOwlTWHGnm3jer2HbKPajnTEmzM2WGi7qzXnZubQv5q2M9U+1aTWbMjw2KqXbVzZ08teEUS1ce\n5vV99cwcEccTN47mJ1ePZNIwa0Y4D4938vNFOdxVks62ajff+nMV6441WxIL+BL2wuku4hIMyss8\ntLdZfwdDFZQA0Fmx0eJIIotMshP9oeITMO57CJxRmE88iK47a3VIvdKmifnyMxDjQt32FavDEX0g\niXEQsxuKJVPSeeSGUcQ4DB5cdZxnzw0HGaiRo53kTYri2OFOqg74t3/yUDJNTXmZm4a6bopnu0gb\nZu10pMP17Tyy9iT3vHGY96uauX5sIs/cMobvzctidLL1fSpthuLWSan89sbRDIt18Mjaah5dV01L\nhzU153a7YvrcWLq9vv+OVt/BUClpkDmSzm2SGA8V2WQnBkKlDvMlx50dvtHRLdZ9yO8Lvf5dOLgX\ndftdqLgEq8MRfSCFLiFgXGoMv/3UaP5rRy0r99az7ZSb78zJJH+AVyAnFkbT2myyu6KN2HiDjMzg\nGyd8JVprdm5t40y1l8klMWSNtKYlmdaaXWc9vLrb998kxm7wmfwUFk9MITkmOP+0chKjeOSGUfxp\ndx1/3FnLrjMevjVrONOyh76eMz7RRuF0FxUbPezf1c6kKTFDHsNHqYJiOlf/FfXS0+Bw+v5xOj/8\n93P/qEs8dsH3OZ1gd0od4WVosxv9p+Xod2SSnRgYlT0K45s/wXz8p5hP/jPG934elIMydEsz+k/L\nYVw+as61Vocj+kheuUNElN3grpJhzMyO44kNp/jxO8e4dVIKX5ia1u+OBkopime5WLeqlfL1buYv\niic+0foyhL46P9Uub5I1U+1Mrdl8spVXd9exv7adxGgbX56azifHJxF3iYEcwcZuKO4oTGN6VhxP\nlFXzz++f4Ia8JO4sSR9Uu7iBGDnaSX2Nl4N7O0hJt1v6IU3Nvhpjdznd5WXQ1en7p/viK+p9vrZt\nGBcly5dMoC/x2EXf53CiLpeIX/CYA2UE7zmo2zyYz/8L7NyCuuYm1Oe/KvXEYkDU+AKMpd/HfPph\nzGcfxvjmT4Luw6h+dTm0ezC+eI8MpwkhSgdJoWl1dbUlxw3FGfNtXSYvlp/l7YON5CQ6uW9uFmNT\n+v9puc1jsvadFmw2xfzr/dvNIVDreuRgBzu3tjEy18nUGUM7wMNratYeaeZ/9tRxrKmTYbEOPpOf\nwnVjEomyD92Lnj/XtrPb5L+21/La3nqGxTn4zuxMCjKGtha626v5oLSFtjbNwhviiXFZ9wby8bXV\n3d0fJsldndDZCd5z//uRx/RHv6frY1+/7Pd1QWfHh//e1fHhzw3mZdlmv0Ti7ABn1IVXvi94zAGO\nqJ7kGkdUz/+qSzx2wfc5nWB39Pq3mOTtoO5n34PTJ1BLvo7UE/tPKL6P+Yu59m/o/3gKNftq1F33\n+T0BHeja6gN7MB/5R9Qnb8OQ2uJLsvK8zcq6fCtISYxD+AWlvLqVJzecpqndy+cmp3H75FTs/Wz/\n1VDnZf17rSSl2JizMA7D5p9EMxDreupEJ1vWexg23M6M+UM3wKPDa1J6qInX9tZx1u0lJ9HJbQWp\nzB+V0O/19odArO2esx6eKDvFmdYubpmYzJempg9pst/a0s2av7WQkGhj7rVxlg1nCYbXA601dHs/\nliyf+/ePJdP6Ul+/xGO6q/Pcz3WdS9I7wNt1cQI/GB9PsO3nE2+HLxE/cQTd3Y3xjX+QemI/C4bz\n1krmn/+Ifu0l1Cduxfg/f+fX5x7I2mqvF/Nn90FHO8ZDTwVlmUcwCNbEOLjuO4h+KcmK48mbcnlu\nyxlW7Kxl08lW7pubSU5i38sLklPtFM10UV7mYcfWtiG/CttXdTW+qXZJyUM31a61s5u/VDbwxr4G\nmjq6mZAWw9emZzA9Ow4jCNdoMPKHuXj8xlz+fdtZXt/XQHm1m/vmZjIudWjqfuPibUyd4TsP9+5o\np6DI2npjKymlfEml3QExV75678+zUGt9cbJ8iSvk+oJk+nyyfvEV8gu+r7MDR95EvLfdJUM7hN+p\nG/8PNPlGR5sJyRg3fMbSeHTpSqg+hvGtn0hSHIIkMQ5xcVE2vjsvi9kj43hm0xm++9YRvlSUxi0T\nUvo8PCI7x0lrczeVuzuITzAYOzG4/pCbG7vZvNZNTKzBzAWBn2rX0Obl9X31/KWykTavSUlmLLcV\npFIwLDg/NPhLjMPgGzOHM2tkPE+WneKHbx/l9oJUPjc5DYef7iRcSXaOr9748P4OUtPtDM8OrU2h\noU4p9WGpxZW+b4DPnxzhVzVF4Cil4I6vQksT+k8vYsYnYsy1ZrObrjuLfuMVKJqFmjrTkhjE4Ehi\nHCbm5iSQn+7i6U2nebG8ho3HW/nOnEyGx/etY8P4gmhamk32bG8nLsFGRlZwJCXnp9oZNpi9MDag\nU+1Ot3Tyv3vrefdQE91aM2dkPLcXpDJmAPXboaw4M5ZlN+fywpYz/HFXHVtOtnLf3CxGJQV+o2N+\nUQwNdd1UbPSw4BNxuOJkY5YQonfKsMHf3Y92t6D/fRk6PgFVOH3I4zBXPAeAccfSIT+28A/ZJhlG\nkmLs/GhBNt+Zk8nRxg6+81YVfz3Q0KdBHkopima6SEy2sbXMTXOjNf1tP+riqXaBSZKONLTz6AfV\n3P3GYUoPNXHNmAR+d/MYfnhVdsQlxefFOW3cNzeLHy3Ips7j5bt/OcL/7K6jO8D9hm02xbS5LjSa\nLes9dHcHxRYIIUQIUA4Hxt0/ghG5mM/+Gn1o35AeX1dsgO2bUIuXoFLTh/TYoaajw+TIoVarw7gk\n24MPPvig1UEAtLS0WHJcl8uFx+Ox5NiBoJQiNzmaBaMTOFzfzpv7G9lf205hhqvXVlyGoRiW6eDk\n0U6qj3WSPco54LKFwa6r16vZuMZNa7PJzAWxJKf6/+bGnrMent10mn8rr+GMu4ubJiTz/flZLMxN\nJD4qeK9UDuU5OyIximvGJFLd0smfKxvZftrD5AxXQNfH6TSIizeoquzE26UZNoQt3MLt9SCYyNoG\njqzth5TDgSqehd7yAXpdKWrqTFR84oCfr69rq9vbMJ/8GaSkY9z5HWnPdhltHpPKXe1s2+DhyKFW\nRuc5sQW4PPJS4uPjL/s1SYzD9AUl1mnj6twEEqPtvHOwkbcPNpLqsjMqKeqKdbIOhyI13U7VwU7q\nznrJHuUc0Ea3wayraWq2rndTe7abkjkuMrL8N8BDa83WajdPbjjFKzvraOk0ub0gle/Ny2LWyPgh\n7+M7EEN9zkbbDebnxJMZ7+S9w028VdmAy2EjLzU6YDXX8Qk2vF2aqgOdxCcaQ9ZnO1xfD4KBrG3g\nyNpeSEXFoAqno9e/i960BjVtHqqXjayX0+fE+LWXYFc5xj0/QqVlDOhY4czd2s3e7e1s3+yhoa6b\nrBwH135yBMpmzQReSYyvIJxfUJRSjEuNYf6oBPbVtPHG/gaONnZQmOEi+gqtuKJjfFfsDld20uY2\nGZ7de4/Sjxvoup6fald9rIvJJTHk5PqnrrXb1HxwtIXH1p/i9X0NaOALU9K4f24WRZmxQ9qabLCs\nOGeVUnDYFBYAACAASURBVIxOjmZhbgJVDR38ubKBfTVtTM5wERugoSZpw+zUnvFy7HAnmSMdOANY\nX35eOL8eWE3WNnBkbS+mYuNRk6agV/8VvWMzauYCX0/ufurL2uoTR9DLl6HmLcK49uaBhhyWWpq6\n2V3Rxs4tbbQ0dTMy18m0OS5yxkSRkhpv2XkrifEVRMILSnyUjWtyE4l2GPz1QBOlh5rIjHcy4gpt\n3eITbSgFVQc6MWy+q8j9MdB1rdzdzuFK31S7cfmDr+/t7Db528FGHl1XzTuHmohz2rizOJ1vzspk\n0jCXJX2IB8vKc9blsLFwdALJMXZKDzXx1wONpMTYGd3LnYiBUIYiLcPBscOd1JzuYuTogd296I9I\neD2wiqxt4MjaXppKTEHljkevehNduQs1Y0G/p+P1trbaNDGffRg62jC++U8o59BPYw1GjfVedpa3\nsau8DU+ryei8KKbNjSV7lBOH03eRw8rzVhLjK4iUFxRDKSalu5g9Mp4dp928sb+BM62dTM5wXXak\ndEq6DXeLSVVlJwlJBvEJfb8yOJB1PXKwg73b2xmZ62Ry8eBao3m6unljXwP/8kE1HxxrITPeydIZ\nGXxtegZjU2L63MouGFl9ziqlyDt3J2J/bRtv7m+gquHcnQiHf6/qOpyK+EQbhys76ejQAW/hZvXa\nhjNZ28CRtb08lT4cNTwbXfo6+uQRX1lFP+p/e02M15XCe39GfeEbGHmT/BFySKur8bJji68ffUeb\nZuzEKErmxpI5wondceH7riTGvZDEeGgkRdtZNDYJpeCtykber2pmVFLUJdu6KeXbjFdzxsvRQ50M\ny7QTHdO3F5T+ruupE51UbGpjWKadktmuAV8ZbGz38t+76vjtumo2V7uZkBbDt2YN50tT0xmZ6P+r\nmlYIlnP2/J0Il9Pg7QNNvHOoieFxDkb2Y8BMX8TF2zBNX71xbJxBQlLg6o2DZW3Dkaxt4MjaXpnK\nyoG4RChdCQ11MHVmn98LrrS2uqUZ/btfwOg8jM9/NSzeXwZCa03NaS8Vmz1U7u7A26UZVxBNyZxY\nMjIdl93EH6yJsfQxjkB2Q/GFKenMyI7j8fWn+Omq43xqXBJ3lgy7qPbYZlfMmB/L2tIWNq11c9X1\n8X1OjvvKH1PtzrR2snJvPe8caqKrWzN7ZDy3FaQM2eS2SGUzFLdOSqUky3cu/XptNQtGt/L16RnE\n+bFzxYTJ0dTX+q5EJKbY+nX3QgghjGtuxGxuRL/5CiQmoz7z5UE/p351ObR7ML54T0R2odBac/pk\nFwf2dNDU0E10jKKgOIacMQPvaBUM5IpxBH/STnU5WDQ2kc5ukz9XNrL2aDN5KdGkx154u9ruUKQN\nc3DkUAe1Z7yM6EOnir6ua3NjNxtXu4l2Gcy5Jg6ns38vLscaO/i38rP8buNpDje0szA3ke/Ny+LG\nCcmkuoJjSIm/BeM5mxht57qxidiU4q8HGlhV1UxOopPMPg6Y6Y1SimHDHRyr6uRsdRcjcgNTbxyM\naxsuZG0Dw+vVdHXYMXVHxF6x7LMJk32jo0tfh9g41JgJvf7I5c5bXbkb/crzqBs+izFrYSCiDVqm\nqTl5tIvyDR6OHOzEblfkT42maIaLlHR7n1+bg/WKsSTGEf5ibTcUxVlxFGa42HCildf3NdDuNckf\ndmEdbnSMQVyCr1OF222SOeLKnSr6sq4et0nZ+60oBXOvjSMmpu9XAffXtvGvm8/wwtaznGrp5FPj\nk/n+vGyuGZNIQnR43wgJ1nPWUIrJGS6mZcVRfsp3LjW0eSnIiMFxmTr2/rA7FIlJvnrj9raBdUvp\nTbCubTiQtfWvjnaT/9/emcdHVZ3//33vbJnJngnZCAGyEvZNWfTnvlS//bZ0QfttbV1qlYIrSsWv\ntmCrorZS20qxWrT9Km212rpUQEVZXMK+J0ACCSSQjezL7HPP748hQ5aZkEB2zvv14pWZYebec5/7\nnOd+7rnPec7RQ756sLl76zlZ7MbrEVjC1A65nBIfiqLAhGmIk8fh0w8gfjjK8JGd/iaQ3wqPB+3F\nJ8FoQr37Z92e0DdY8XoFJUUuduXYKClyYQpRGD/FzIRpZqJi9CjdHKyQwvgsSGHcv8SFGbg6LZJG\np8Z/8mvZWtJIVqyZGPOZDh8eoUOng6J8F4qiYI0LHgzOZleX0yeKXS7BrCvCu/RoXAjB7rJmXtxa\nzuq9VdQ7PHx7bAwPXzqcWSPCe61s2EBjoPtsjEXPNWmReLyCD/Nr+fx4I6nRIcSFnf8IfmiYDvDl\nG5stKpHRPXtBGui2HcxI2/YMTY1eDu13sGebjaoKL7HxesZPjqG+1kFJkZuifCd1NR50egVLmCpH\nkduhKCrK5BmIglz47EOU1CyUYQlBvx9QGH/8b9i2GfXHD55VWA8FPB7BsQInu3JsnCx2ExquMmGa\nhXFTzERE6c/Jx4QQNHp1qN6BV8dYEV1ZL7gPKC0t7Zf9xsbGUlVV1S/7HqjsKm3iD1vKqXd4uGlC\nLN8dZ/WXNRNCsGebjRPH3EybbSFpROBH5Z3Z1esR5Gxsor7Wy4zLQ4mN61wweTXBlpJG3smr5miN\nE6tZzzezY7guPQpzD1dBGAwMJp/Nq7Txu5wyKprcfGNMND+YNOy8a0YLTbBlczM1VR7+3zXhPToZ\nbzDZdrAhbXt+1FR5OHrISflJN6oKyaOMpGaZCI/Q+W3b1OCluMhFSZELl1NgClEYMdrIiNFGwsIv\njIGDriJsTWjPPQpVlaiLnkIZmR7we+39VlRXov1iAYydjG7BY33V3H7B7dIoOuKiKN+JyymwxunJ\nyDYRG39uYhjA7tbYdKyetfl1VNs9rJqT1i/rCCQlJQX9PymMZbAOSJPTy8s7Kth0zJd3fP/sRFJO\nVxvwes8I20uuCiMqpuOoXTC7appgx5fNVJR6OhXWAG6vxoaiBv6dV01po5ukcAPfHmvlitERPfJo\nfrAy2HzW7tb46+5K1hbUkRxh5IHZiec9KdLp0Nj0USN6g8Jl14b32KPjwWbbwYS0bfcRmqC81M3R\nQ05qq70YjAqj0o2MzjBhCjkTA9vbVtMEFaVuSopcVJR5QPjKb6aMNpE4IniVgAsNUVeN9swj4HKi\nPvIsSnxHsdTett4Xn4SDe1F/+UcU67C+bG6f4XRoFOY7OXbEiccNcYl6MsaGEBN77k/oiuucrC2o\nZUNhA3aPxuhoE3OnjOCiODVoydjeRArjTpDBunO+Km5g5bYK7G6NH04extezotGpCk6HxuefNCIE\nXHpNOGZLW8cOZFchBPt22CkudDF+ipnRmYHLetndvkU53j1YQ43dQ1qMie+MszIzOXxQ1x/uKQar\nz+4ua+YPOWXUOjzMHW9l7rhYDLpzP59VlR5yNjaRNMLA1JmWHnlkPFhtOxiQtu06Ho/gxDEXhYed\nNDdpWEJVUrNMjBgdeLZ/Z7Z12DVKilwUF7mwNWnoDTA8xUhKqpHIaN0Fn2ohyk+iPfsIhJh94jgq\nps3/t7at2LMFbcXTKN+9DfX6b/dHc3sVu03j6GEnxUedeL2QmGwgY6zpnFPW3F7f0961BbXkVtrR\nqwqXjgznhoxosmJDGDZsWL/FBCmMO0EG67NTZ/fwx23lbD3RxNhhZu6flUhCuJGGOi9ffNpIWLiO\n2VeFtQnYgex6+ICd/Fwn6dkmsid2HDFscHj4T34tHx6upcmlMSHewnfGWZmc0DOiZ6gwmH22yeXl\nlR0VbCxqIDXaxAOzkxgZde51jwvyHBza72DCNDOj0s+/fvJgtu1AR9r27DgdGseOODl2xJcKERWj\nI22MicThhk4nNnXFtkIIqk95KS50UnbCjeaFiEiVEakmkkf2zZLrAxVRlI/2/OMwLBF10dMollD/\n/7XYVjjsaEsWgDkU9fHfDqkJd81NXo4cdFJyzAUCho80kJ4dcs5lMU81u/mooI5PjtZR5/ASH2bg\naxlRXNNucnx/xgQpjDtBBuuuIYRgQ1EDr+yoQBOC26fGcX16FJVlHrZ93kxisoFps88I2PZ2PXbE\nyf6ddkaMMjLp4rar2p1qdvPewRo+PlKH0yuYkRzGd8ZZyYqVNYgDMRR8NqekkZVby2l2a/xgYizf\nzI45p6cBQgi2fd5MVYWHS64OnNbTHYaCbQcq0rbBaWr0UnjYJ0w0L8Qn6UkbE0JMbNdGdLtrW7dL\n42Sxm+JCF/W1XlQVEoYbSEk1nlf+6GBG5O5G+8MvIX0s6v1LUAy+NL8W22pvv4b46N+ojzyDkj62\nn1vbMzTWeyk46OBksRtVgRGjjaRnm7CEdl8Qa0Kwp6yZtQV17DjZhBAwfXgYN2REMSUpFDWATw1U\nYTx0bnkkvYqiKFyVGsmEeAt/2FLGym0VbClp4t6ZCYydHELeHgeHDzgYM6GjmC074WL/Lt+qdhMv\nOiOKT9Q7+VdeDZuO1aMJuHxUBN8eZ/XnMkuGLrNGhJM9zMzKbeX8dc8ptp1s4v5Zid2ue6woCpNn\nWNj8USM7v7Jx2XXhGIwX3kVdMjjpbEJdb2IwqoxKNzEq3UR9rZeSIicnjrspLXFjtiikpJpIHmXE\nEnrhjCIr46ag3P4A4s/Po/15Oerdi1BU33kQJ44h1r+Pcum1Q0IU19V4KMjz+Z1OD6mZJtKyTOe0\neFeD08unR+tYV1BHeZObSJOOb4+1cn16VI9UIuoP5IixHMXoNpoQrCuo4y+7KtHrFH4yLY7oKgMl\nRW6mzLSQPNLot2v1KQ9bNjYREaVj1pW+dIuCajvv5FazpaQJg07h2vQo5oyJGbSdqK8ZSj4rhGDT\nsQZe3l6BRxPcNjWOGzKiuj1iVVPl4avPmogfbmD67HNPvRlKth1oSNv66OqEuu7QE7b1egXlJ9wU\nF7moqvAAMCxBT0qqkfgkA7rzmA8wmNDWv4d4cxXK5V9D+cFPibVaqXzkJ1B+AvVXK1HCIvq7iedM\n9SkPBXkOTpV7MBgURmUYSc00dTuNRghBfrWDtfm1fHG8EbcmGDvMzA2Z0cwaEdblyfFyxFgyZFAV\nhRszo5mSGMoLX5XxQk45s5PDmWUNZ+82G6FhKrGxvlXttn/ejDlU5eL/ZyG3ysbbudXsK7cRalD5\n7jgrXx8TTdQQX5BDEhxFUbhidCTj4y38YUs5f9pewdaSRu6ZmdhhBcbOiInVkz0xhLy9DooKXKQG\nmdgpkfQXgSbUjZ9qDjqhrq/R6RSGjzQyfKQRW9OZsm87v7JhMCokjzKSMtrYo+URByLqNd9Eq69D\nrHsHImNwjBgFRw6i3HbfoBTFQghOlfsEcU2VF6NJIXtiCCPTTRi6Wc3H4dHYfKyBtfm1FNY6CdGr\nXJMWydcyohgVHdJLR9D3yBFjOYpxXng1wXsHa1i9r4pog45v6WMxKApX35jEZ2tLEQIsYxXePVpD\nQbWD6BAd38iO4WsZUVgMQzvA9hZD1WdFy5OI3ZXoFIU7p8dz5eiILo/+CiHY/mUzlaW+fONoa/dv\nuIaqbQcCF6ptz3VCXXfoLdsKTXCqwkNxoYvyUjdCg6gYHSmpRpJSjN0WVoMFIQTitd8hcj4DowlG\npqE+/DSKOnhSS4QQlJ90U5DnpL7WS4hZIX1MCCNSu38jdqLeydqCOjYU1tPs1hgZZeKGjCguHx1x\nXtfxgTpiLIXxBRqse5rjdU5e+KqUmloP3zLEohMK6OBLQz0Hm+wkhBn41tgYrkqN7JeahUOJoe6z\nZY0ufp9TRt4pOzOSw5h/cQJR5q6JXJdLY/PHTSAEl10X3u1HhEPdtv3JhWbb851Q1x36wrZOh8aJ\n4y6KC100NWjodJA0wsiIVGOvHFN/IzwetD8+7atZ/PhvUYan9HeTuoSmCUqL3RQcdNDUoBEappKe\nbSJ5pBG1G+kwHk2w9UQja/Pr2F9hQ6/C7JQIbsiIInuYedCXx5TCuBMutGDdm7i9grcOVJGT28hM\nNYIvtAZCohS+PdbKJSmyBnFPcSH4rFcTfHC4hjf2VGE2qPz04nhmp3TtMWZdtYcvPmsiLkHPRZeG\ndiuAXwi27S8uFNv2x4S6vrStEIK6al+qxcliF14PhIarpJxeYe9c86QHIkLzEqPXUav1d0vOjtcr\nKClycfSQE1uzRnikSsbYEJKSu/dkosrm5uMjdXx8pJ5au4e4UD3Xp0dzTXpkj6c9SmF8FqQwHjoU\nVNv58qSTCVY9U5O6J0wkZ+dC8tniOicv5JRytMbJ5aMiuGt6PGGmswuMogInB3bZyZ4YQnp213Pf\nLiTb9jVD2ba9MaGuO/SXbT1uQWmJb/GQ2iovigLxSb6yb8MS9KhDYDBkoPutxyM4ftRJ4WEnDrsv\nVSdjbAjxSV0vu6cJwb5yG2sLatl2wldqbWpSKDdkRDM1KbTXBrUGqjCWs54kPU6G1cysrBEDOphI\nBgcpUSaeu34U/zxQxVsHqtlfYePemQlMTQrr9Hej0o1Un/JwaL+D6Fg91mEy1El6Hq9HUDKAJ9T1\nNnqDr7RbSqqJxgYvJYUuSo65KD/pJsR8ZsJeaLicT9LTuF0aRUdcFOU7cTkF1jg9ky82dasOdZPT\ny6eF9awrqKO00UW4Scec7BiuT48ioZulM4cS8mohkUgGNHpV4X8mDuOi4eG8kFPKExtOcH16FLdP\njcNsCDwapygKky6y0FDbyK6cZi67LnxIPeKV9C+BJtRNm23p0Ql1g43wCB1jJ5sZMzGEilLf4iFH\nDjk5ctCJNU5PymgjickGdBfADUNv4nRoFOY7OXbEiccNcYl6MsaGEBPbdTlXUG1nbX4dnx9vwOUV\nZMWaeXBCIrNTwuUcIKQwlkgkg4R0awjLbxjF6r1VvHewhj3lzdw/M5Fx8ZaA3zcYFKbNtvDF+iZ2\nb7Ux4zKZ1iM5P/pyQt1gRVUVEpONJCYbsds0ThzzpVrs3mpj/y4YnmIkJdVIZLS0WXew2zSOHnJw\nvNDne4kjDGRkm4iM7pqMc3o0Pj/ewLqCOgqqHZh0CleO9pVaS40ZOqXWegIpjCUSyaDBqFO5fWoc\nFyeH8fucMh5bX8w3xkRzy+RhAUc6IqP1jJ9qZt8OOwV5TjLHyQuApPv01wp1gx2zxTcBLD3bRHWl\nx1cb+ZiL40ddRESppIw2MXykodvVYy4kmpu8HDnouxlDwPCRBtKzQ7rse6UNLtYV1PJpYT1NLo3k\nCCN3TY/nitERhBql/wZCCmOJRDLoGBdn4YUbR/OX3ZW8d6iWnaXNPDA7kQxrxyXJU1J9+caHcx3E\nxOqIjZcrLErOTqAJdRljTX02oW4ooSgKsfEGYuMNuF0aJ4/7Vtg7sNtO3l47ickGRqQaiY3ren7s\nUKex3ktBnoOTJW5UBUamGkkbY8ISenYx69UE2042sS6/lj3lNnQKzBwRzo2Z0YyL65lSa0MZKYwl\nEsmgxFfGLYEZyWG8uKWcn310nLnjrdw0PhZ9qzxPRVGYOM1CfU0ju7bYuOy6cELMUthIAnOhT6jr\nbQxGlVEZJkZlmKiv9S0ecvK4m5PFbiyhKiNOl30zWy7MPlpX46Egz/d0QqeHtEwTqVmmLsWsGrvH\nV2qtoI5quwerRc8PJsZybXoU0V2sBS+RwlgikQxypiaF8fv/Gs0rOyt4c38120808cDsJEZGnVkW\nWm9QmH5JKJs/8YnjmZeHDolSUpKeQ06o63sio/VMmKZn7CRB2Uk3JYUuDh9wcDjXwbB4PSmpRhKS\nDN1amGKwUl3poeCgg1PlHgwGhcxxvqcTZ0szEUKwv8LGuoI6tpQ04hUwOTGUuy+KZ/rwMLl+wDkg\nhbFEIhn0hJl0PDg7iZnJ4azcVs7Ctcf4waRYvjkmxn9hCI/UMXGahT3bbOTnOhgzoWPaheTCI+CE\nuqwQYobJyWF9hU6vkDzSSPJII81NXkqKXJQUudj5lQ2jyfd/KalGwiOHVk6sEILKcg9H8hzUVHkx\nmhSyJ4YwMt101uW2m1xeNpwutXaiwUWYUeW/x/hKrSVFXLil1noCKYwlEsmQYVZKONlxZv64tZy/\n7j7FthNN3D8rkcTTNTlHjPblGxfkOYmJ1ROXKPONL1TkhLqBSWiYjjETzGSNC6Gy3Ddhr6jASWG+\nk2irjhGjjQxPMaI/i3AcyAghKDvh5shBJ/W1XkIsCuOnmkkZbTxrObvCGgdrC2rZVNSA0yvIsIZw\n/6xELkkJx6S/MNNPehopjCUSyZAiKkTPo5cNZ2NRA6/sqOD+D4u4fWocX8uIQlF8F6C6Gg+7tti4\n/PrwCzaX8UJETqgbPCiqQnySgfgkA07HmbJv+3bYyd1jJ2mEbxQ52jp4RvY1TXCy2M2Rgw6aGjRC\nw1QmXWQmeaSx03QRl1fjy+ONrC2o5XCVA6NO4bJREdyQEU26VVba6WmkMJZIJEMORVG4MjWSCQkW\n/rClnJe2V7ClpJF7ZiYyLNTA9Nm+fOOdOc3MvjJM5hsPceSEusGNKUQlbUwIqVkmaqt9K+ydLPGl\nW4SFq6SkGkkeZRywNzder6CkyLfgib1ZIzxSZeosC0nJneevlzW6+KigjvWF9TQ6vSSFG7lzWhxX\njo4kzCSfbPQWUhhLJJIhS6zFwNIrk1lXUMdruyq5/8Mi7pwez5WjI5g03cKuLTYO7XcwdpLMNx6K\n+CbUuTh2xDkkJtR5NUGtw0O1zUOVzU21zfda6OqI0HtJDDeQFG4kMdxIyBB8rK4oCjGxemJi9Yyb\nYqa0xEVxoYu8vQ4O7nMQn2QgJdXIsAT9gLjZ9XgEx486OXrIidPh87/xU0KJTwpels6rCXaWNrE2\nv47dZc0oCsxIDueGzCgmxlsGzej4YEYKY4lEMqRRFIUbMqOZnBjK73LK+F1OGVtKGpk/I4GRaUaO\nHvLlGycMl/nGQ4XBOKHO7RXU2N2nRa+HalvH17UOD5po+zujTsFi1FFn97T5PNqsJyncQOJpoZx0\nWjQnDBHRrDcopKSaSEk10djgpbjQxYljLspPugkxK/6yb6FhfT+y6nZpFBW4KMx34nYJrHF6psw0\ndVqnuc7u4ZOjdXxUUMcpm4cYs57vTYjl2vRIrBYZm/oSRQghzv613qe0tLRf9hsbG0tVVVW/7Hso\nI+3ae0jbnjteTfD+oRre2FuFxaDy0+nxePIVbM0al10XRsrIeGnbXqIv/HagTqhzejRq7GdGeQMJ\n3zqHt8PvQvQKsRYDVoseq8VArEWP1aJv81m4UWXYsGEUl1VQ3uimtNFFaaOLskY3ZY0uyhpdHbYd\nY9aTeFo0+0aYz4w0D+YJXJpXUFHmprjQRWW5BwRY4/SkjDaSmGw468S2QHTHb50OjcJ8J8cKnHg8\nEJeoJ2NsCDGxgccghRDknbKzNr+WnJJGPBpMTLBwQ0YUFyeHt6nHPhTpz2tZUlJS0P+TI8YSieSC\nQacqfGuslWlJYbyQU8qzX5ZyTXIkaY1mdn5lY3jygBgnkHSDYBPqRqV3bVGE88Xu1qi2n0lrOJPi\n4D4tej00ODuK3lCjSqzZJ3BTo02txO4Z4WsxqF0e4bYYdKTG6EiN6TgZy+b2+oVy6WmxXNboZvvJ\nJuoDiOa2I81Gv4ge6KJZ1SkkJhtJTDZit2mUHHNRUuhi91YbB3YpDB/pS7WIjO5Z6WO3aRw95OB4\noe8JRdIIA+nZpqD7sbm9bCxqYG1+LcX1LkINKjdkRvO1jCiSI0wBfyPpO6QwlkgkFxwpUSaeu34U\n/zxQxVsHqqkwuplRE8HWz0+RPBpCzMqAyFGUBKe3J9QJIbC5tQ75vG1e2900u7QOvw036Xyju2Y9\nWbHmdqO8eqxmA2ZD34lMi0FHWoyOtACiudnlpbzJTWnDacHc5KK0wc22AKLZ2mGkeeCKZrNFJXNs\nCBnZJqorfSvsFRe6OHbERUSUjpRUI8NHGjAaz73dzY1ejhzypewgIHmkkfRsE2FBnlAcq3WwtqCO\njUUNODwaaTEm7pmRwP8bFTEk0luGClIYSySSCxK9qvA/E4dx0fBwXsgpZX9jM+TC4VwAgWJQ0JnA\nGKJgClExW1RCw1QiQnWEh6tYLDp0F8CKXAONnphQJ4Sg0aW1SmcInOLg8HQUvVEhOqwWAwnhBsbF\nmzukOMSY9QNOJHZGqLFz0dw6JaMlRWPbiSbqnQFEc4QvlzkxzHj6tZGEMEO/2kNRFGLjDcTGG3C5\nNE4e96VaHNhlJ2+vncThvlFkayf5v+1pqPNy5KCDkyVuVAVGphpJGxOCJbTjcbq9Gl8VN7K2oI6D\np+wYdQqXjozghowoMqwhAzbn/UJGCmOJRHJBk24NYfkNo1i9p4p1+bWECpVQRUeoU8Xi1BHaqBKK\nDpPS8aLnRMOhaLh1Ao9OIAwCDKAzKuhDwGRWMRtVzAYVs77d31avjTpFXiDPQlcn1GlC0ODw+kVu\nG7FrP/Pa5W2bNqMqEB3iE7gjIk1MSQztkNsbYzZguIBuhkKNOtKtuoC1cltE85nUDBeljW62lgQQ\nzRa9fwJg6xSNvhbNRqPK6Axf3eq6Gg8lRS5OHHdxstiNJVRlRKqREaOMQWub11V7yD/ooOKkB50e\n0rJMpGYGTtmpaDpdau1oPfVOX8WQO6bGcVVqJOGy1NqARk6+kxOZegVp195D2rb3iLFaOVFeid2t\n+f55zvy1ObzYbQKnQ8NtF3hdIFyAG3ReBYNXwSA6XiAdQsOGl2ah0YyXZuGlmbavvYrArFcJCSKc\nOxPVgT4z6AbeiOW5+m1NlYejh52Un/BNqItN1mNJUmhUvAFHeWvsbtoP9OoU/CK3fVpDy+voEL1/\n+fDBxkCLCU0urz+PuazdZMD2+dZWi56kFqF8ehJgUriR+D4SzV6PbxW64iIX1ZUeUCAuQU9KqpH4\nRAPD4mI5fLCCgjwHVRUeDEaF0RlGRmeYMJrats+rCXaXNbM2v5adpb5SaxcND+OGzGgmJVhQ5c1v\nG+TkO4lEIhngqIqCxaDDYji3ER2vR+Cwa9jtGg6bwGbz0tSkYbNpOOwaLodPULdHqL4RZ7cqcAqf\najUNFAAAH7ZJREFUGG92eWnSvJzQnNR6vdR7vHja1+oKgl6llXDWEWJQAghoXWCh3U50hxjUPpkd\n79EEtXYPVc1uTjW7qanw4K1QMDpU3IrGUdXBbncTzYUaFJ75nUFVTgtcPdnDzFgt4R2Eb2SIToqS\nPiTMqCPDaibD2rE+eGvR3Hq0OaeksY1oVmg90mxsk9ucEG7A2EM3fzq9QvIo3wIhzU1eSop8C4fs\n+NKG0aQQEemgqtKJKUQhe2IIo9JNHZajrnd4WH+0no+O1FHR5CYqRMfc8VauS49iWKgstTbYkMJY\nIpFIegidXiE0XEdoeHBhrXkFDoeG3eYT0Q6bht2m4bCL0399r2mngVUDhJhVjCG+3GfVCBjAq/eJ\napcqcAgNu1frMOLt8Gi+iWR2T5vPu6izMahKl0aqW38W0u4zu85OYYXtTD6vve1Ib53dgwpkKGYm\nqKFEKnpswkOB0Y49wktMqJ4bzdFtRnljLXrCTQO3NrGkI52KZqeXsqZWornBNxnwq5JGGgOI5tYT\nAJPCfXnNCWHnLppDw3SMmWAmc1wIp8p9E/YcdsH4qWZSRhvblHsTQnCoys7a/Dq+LG7EownGx1v4\n0eRhzEgOv6BSboYaUhhLJBJJH6LqFCyhOiyhwb+jaQKnQ/hEc4t4tp95b6v1ief2iXCqqmI16wmx\nKJjNKiGhKiFmFbNFOf1XxWRSUFQFIQQurziTLtI6daSdqG6fVmJ3azQ4vVQ0u/3fd3i09lq+HUVt\n3lkMqj+9YXS4iXiHEVOdDjxgjlBIHWNiZIoR3QBMC5H0DmEmHRmmzkWzr3rGmRSNQKI59vRIc2K4\nkaSIM5MBuyqaVVUhPslAfJKhw+N+u1tj07F61hXUUVTrxGJQuT49kq9lRpMSKUutDQWkMJZIJJIB\nhqoqmC0KZotKdJDvCHFaPNvPjDi3FtK1NV4cJ9xo7fJtFQVM5tPC2aKe/uvbV5RZjzlaxRTS/XJ1\nmhA4Pe2FtheH2/dZiCUUo9fhT3GwGHSDcoU6Sf/QmWhudHr9KRmtUzS+Km6gsVU5Pb9ojuiYnhF/\nFtFcXO9kXX4tnxU2YPdojI42Mf/iBC4bFdGnpfckvY8UxhKJRDIIURSFELNvJDgqJvB3hBC4nC3i\nubWI9uVAN9R5qSx1422//oQCISG+bfvEs9JKRPvem8xqm3J1qqJgNvhSLuioXdqMvNVUecg93Oyf\nUDdQVqiTDE7CTTrCTWYyY4OL5tJ2wvmL4w00tRPNw0L1JJwWyv5c5mrB27uKOVBpR68qXJoSzg2Z\n0WTFylJrQxUpjCUSiWSIoigKptN1mCODDD0LIXC7BQ6b8I82txbSTQ1eqio0PO6OvzWafCPNIeaW\nv22FdIhZ9S+2oWmCshOufluhTnJhcjbR3L7cXFkA0RwfZuDWycO4Oi2SyBApm4Y6vXKG9+zZw2uv\nvYamaVx99dXMmTOnN3YjkUgkkvNEURSMRgWjESKigo/Yut3BJwvamzVqqry4XR2zjA1GBbNZQRNN\nNDV4enyFOonkXAk36cgymckKIJobTo80h4ZHkGR0yaomFxA9Low1TWPVqlU8/vjjWK1WHn30UaZP\nn05ycnJP70oikUgkfYTBoGAw6DpNd/B4Wonn0znPLekber2RrPFGEoYb5HLbkgFPhElHhMlMbGzk\ngKoRLel9elwYHzlyhISEBOLj4wGYPXs227dvl8JYIpFIhjh6vUJYuI6wAOXqBtoiFBKJRBKIHk/s\nqqmpwWq1+t9brVZqamp6ejcSiUQikUgkEkmP0m9Z5OvXr2f9+vUAPPPMM8TGxvZLO/R6fb/teygj\n7dp7SNv2HtK2vYe0be8hbdt7SNv2HgPVtj0ujGNiYqiurva/r66uJiamYy2ha665hmuuucb/vr8e\nscnHe72DtGvvIW3be0jb9h7Str2HtG3vIW3be/SnbZOSkoL+X4+nUqSlpVFWVkZlZSUej4evvvqK\n6dOn9/RuJBKJRCKRSCSSHqXHR4x1Oh133HEHTz31FJqmceWVVzJixIie3o1EIpFIJBKJRNKj9EqO\n8dSpU5k6dWpvbFoikUgkEolEIukV5HJDEolEIpFIJBIJUhhLJBKJRCKRSCSAFMYSiUQikUgkEgkg\nhbFEIpFIJBKJRAJIYSyRSCQSiUQikQBSGEskEolEIpFIJIAUxhKJRCKRSCQSCSCFsUQikUgkEolE\nAkhhLJFIJBKJRCKRAFIYSyQSiUQikUgkgBTGEolEIpFIJBIJIIWxRCKRSCQSiUQCgCKEEP3dCIlE\nIpFIJBKJpL+54EeMFy9e3N9NGJJIu/Ye0ra9h7Rt7yFt23tI2/Ye0ra9x0C17QUvjCUSiUQikUgk\nEpDCWCKRSCQSiUQiAUC3dOnSpf3diP4mNTW1v5swJJF27T2kbXsPadveQ9q295C27T2kbXuPgWhb\nOflOIpFIJBKJRCJBplJIJBKJRCKRSCRALwrjyspKHnroofPaRk1NDc8//3wPtejsPPLII7jdbnJy\ncnjwwQd54oknOHr0KK+++mqftaG7bNy4kVWrVvXoNrdt28aJEyf8799880327dt3ztvzeDw88sgj\nPdG0XmHp0qUcPXq0R7fZ3oZdJTc3l8OHD5/1ezt27ODdd989l6adN83NzXz00Uc9sq2eiBPQth+0\nt/3Zzm9Lv+9JVqxYwZYtWwB46aWXztkXnnnmmR5tV3/y9NNPU11d3d/NCEprP1m2bBnNzc0dfP18\nrkm1tbU8+eSTPdLWgU5lZSVffPFFh89b+tqCBQtoaGjoh5b1HF2N1QOd84nnreNcIM6lzz/++OPn\n1JaeZECPGMfExPTIRbMrVFZWEhMTg8Fg4LPPPuPuu+9myZIlpKWlcccdd/RJGwYK27dvb3Mhv/nm\nm5k4ceI5b+/QoUNkZWW1+UwIgaZp57zN7uL1evtsX9DRhl3B6/V2OdhOnz6dOXPmnGvzzovm5mY+\n/vjjftl3V+iO7Vv3+95i3rx5JCcn99r2BwMul4vGxkasVmuXvt/X/bU9jz76KKGhoR18/XyuSXv2\n7GHSpEk91cQu05dxtoVTp051EMZ90df6iu7E6oFOb8Xz7vb5FgbCzaO+Nzfu9Xr5/e9/T1FREcnJ\nydxzzz0sXLiQZcuWERERwdGjR3n99ddZunQpeXl5vPbaawAoisITTzxBY2Mjzz77LM8//zwbN25k\nx44dOJ1OKioquPjii7nlllsA2Lt3L2+99RYej4f4+Hjmz59PSEgIq1evZseOHeh0OiZOnMiPfvQj\ncnJyePvtt1FVFYvFwhNPPAH4gtbkyZN5++23OXToECtXrmT69OlMnTqVDz74gMWLF/PWW29RVVVF\nZWUlVVVV3Hjjjdx4440APPfcc1RXV+N2u7nxxhu55pprAPjhD3/IjTfeyK5duzAajSxatIioqCjq\n6up45ZVXqKysBODOO+8kKyuLzZs3s3btWjweDxkZGWRlZbFu3Tr/+zvvvJNNmzbx7rvvYrFYGDly\npD/QrFixgmnTpjFz5kz/vl9//XUA3n33XT7//HNUVWXy5Mn84Ac/YP369Xz66ad+u917770cO3aM\nHTt2kJeXxzvvvMNDDz3EO++849/u/v37ef311/F6vaSlpfGTn/wEg8HAggULuPzyy9m5cycej4eF\nCxcyfPjwNratrKzkqaeeIiMjg8LCQh599FFKS0u7fO4aGhp4+eWX/Xegt956K2PGjOHIkSO89tpr\nuN1ujEYj8+fPJykpiY0bN7J161YcDgeapvHEE08EtANATk4Of/7zn7HZbMybN4/s7OwO/tz+3Nx5\n553ceuutHc5vRUVFBxsCrFq1ioaGBkwmE3fffTfDhw9nxYoVGAwGjh07RkxMDIcPH0ZVVT7//HPu\nuOMOmpub+de//oXH4yE8PJx7772XqKgoNm7cyNGjR/nxj3/MihUrMJvNFBYWUldXxy233MLMmTPJ\nzc3lrbfeIjQ0lOLiYmbNmkVKSgpr1qzB5XKxaNEiEhISgto1mL//7W9/o7y8nEWLFjFx4kR++MMf\nnlec0DSNl156ifz8fGJiYvjZz35GTU1NQHvt2LEjoD1aOHz4cEDbBzu/Lb7Z0l+uu+46du/eTXR0\nNP/zP//DG2+8QVVVFbfddhvTp09H0zRWr15NXl4ebreb66+/nmuvvRYhBK+++ir79u0jNjYWvf5M\naF26dCk//OEPSUtLY8+ePfz9739H0zTCw8P5xS9+EdR/+4LKykqWLVtGVlbWWe2fmJjIvffey4sv\nvojNZuOOO+5gyZIljB07liVLljBv3jxqa2s7xHGz2Uxubi7jxo0DYMGCBcyaNYvdu3djNBq5//77\nSUhIaNMXsrKyuPnmm3n11VcpKSnB6/Uyd+5cLrroIkpKSvjjH/+Ix+NBCMFDDz1EdHQ0v/3tb6mp\nqUHTNL7zne+Qnp7O008/TWpqaptrUH5+fsAY1poFCxawbNmyDr5+/fXX+69JmqbxxhtvsHfvXhRF\n4eqrr+aGG24IGLvA52tz585F0zReffVVDhw4gNVqRa/Xc+WVVwaNr7m5uXz22WcsXLgQ8I1UtlyT\ngl37Wmy8f/9+vvGNb/DJJ5+Qnp5Obm5umz6wceNGtm3bhtPppLy8nP/+7//G4/GwefNmDAYDjz76\nKGFhYZSXlweNX4Fiz9/+9jdOnDjBokWLuPzyy/n617/epq+1JtC1Mycnh/z8fG699VbWrFnDmjVr\nePHFF6moqODFF1/kV7/6Vbf83OFwdPCP1atXB/TDyspKVq5cSWNjIxEREcyfP5/Y2Nizxuq6urqA\n2mKg097HIyMjycnJwe12c/HFF3PTTTcBsGnTJj744AMURSElJYV7770XgLy8PP7zn/+0Of9Amz4f\nqE8E00Ctdcv777/foS3BYpbRaKS8vJxXXnmFhoYGVFXlwQcfJCEhIeB2OkX0EhUVFWLu3Lni4MGD\nQgghVqxYId577z0xf/58UV9fL4QQ4siRI2LJkiVCCCGWLVvm/67dbhcej0dUVFSIhQsXCiGE2LBh\ng1iwYIFobm4WTqdT/PSnPxWnTp0S9fX14he/+IWw2+1CCCH+/e9/i3/+85+ioaFB3HfffULTNCGE\nEE1NTUIIIRYuXCiqq6vbfCaEEM8++6woLy8XQgixZMkSceTIESGEEAcOHBDLli0TQgjx5ptvisce\ne0y4XC5RX18vbr/9duF2u4UQQjQ2NgohhHA6nWLhwoWioaFBCCHE3Llzxfbt24UQQrz++uvi7bff\nFkIIsXz5cvGf//xHCCGE1+sVzc3NoqSkRCxbtsy/zd/+9rdi0aJF/vevvPKK2LBhg5g3b56or68X\nbrdbPP744+LPf/6zEEKIF198UeTk5PiP6ZZbbhFCCLFr1y7x2GOPCYfD0aatLW0UQoi///3vYs2a\nNQG30/Le6XSKefPmiZMnTwohhPjDH/7gP4b58+f7f79u3TqxcuVK/+8XL14sHA6HqKioEDfddJM4\nfPiwEEJ0+9y98MILfh85deqUeOCBB4QQQjQ3NwuPxyOEEGLv3r3i17/+td9n7r77bv/xBrPDkiVL\nxF//+lchhBA7d+4Uv/zlL0V72p+bV155RWzcuDHo+W1vwyeeeEKUlpYKIYTIz88XS5cu9X9v2bJl\nwuv1CiF8Pvbee+/5f9fY2Oi3w/r16/3t3LBhQ5vz/vzzzwuv1ytKSkrEPffcI4Tw+e6tt94qampq\nhMvlEnfddZd48803hRBCfPjhh+K1117r1K7B/L11vzxfKioqxM033yyKioqEEEI8//zzYtOmTUHt\n1VV7tLZ9Z+e3db+fO3eu2LVrlxBCiOeee0786le/Em63WxQVFYmHH35YCCHEJ5984j/HLpdLPPLI\nI6KiokJs2bJF/PKXvxRer1dUV1eLW2+91d+GlnhSX18v5s2bJyoqKvzHIkRw/20de3qL7tr/ySef\nFMXFxWLHjh1i8eLF4p133hEul0vMnz9fCBE4jgshxKpVq8T+/fuFEL5Y8c477wghhNi4caP/GNv3\nhdWrV4tNmzYJIXwx4L777hN2u12sWrVKbN68WQghhNvtFk6nU+Tk5LSJOc3NzQGvQW+//XbQGNY6\n7rdcp9r7euv3H330kfjNb37jP8bGxsagscvr9fp9KCcnRzz99NPC6/WK2tpacdttt3UaXz0ej5g3\nb54/Tr788sti06ZNQeNnS/vfffddf7uD9YENGzaIe+65R9hsNlFfXy9+9KMfiY8++kgIIcRrr73m\nt01n8StY7Gnvu637WmsdEOjaWVtbKxYvXiyEEOI3v/mNWLx4saiurhYbNmwQq1evFt0lkH8E88Nl\ny5aJDRs2CCGE+PTTT8Wzzz7rP9bOYnUwbTHQae3Te/bsES+99JLQNE14vV6xbNkykZubK4qLi8V9\n993X4ZwFO/9CnOnzwfpEIA0kxBndEqwtwWKWEEI8+uijYuvWrUIInz85HI6g2+mMXh0xtlqtjBkz\nBoDLLruMNWvWBP3umDFj+L//+z8uvfRSZsyYEXD4ffz48VgsFgCSk5OpqqqiubmZEydO8POf/xzw\n5bNmZmZisVgwGo2sXLmSadOmMW3aNACysrJYsWIFs2bNYsaMGf7fVFdXEx8ff9Zjmjp1KgaDAYPB\nQGRkJPX19VitVtasWcP27dsBqKqqoqysjPDwcPR6vX/fqamp/lzdAwcOcM899wD47zA3b95MUVER\njz76KODLSXO5XP73LpeLw4cPM27cOCIiIgCYNWsWZWVlnbZ5//79XHHFFZhMJgDCwsIAKCkp4R//\n+AfNzc04HI6zPuYrLS0lLi7OP5p1+eWX89FHH/Ff//VfAH57pqamsm3bNsCXkxcWFubfd2xsLJmZ\nmQAUFBR069zt37+/zSNym82Gw+HAZrOxYsUKysvLgbaPYSdOnOg/3mB2ALj44ov9bW+5g23NgQMH\n2pwbl8tFRERE0PPbGofDweHDh1m+fLn/M4/H4389c+ZMVDVwVlNNTQ0vvPACtbW1eDwe4uLiAn7v\noosuQlVVkpOTqa+v93+elpZGdHQ0AAkJCf6UmJSUFA4cOOC3SyC7QmB/72ni4uIYNWoU4LPhqVOn\ngtqrq/ZoT6Dz277f6/V6/4hWSkoKBoMBvV5PSkoKp06dAnxPp4qLi/15dTabjbKyMg4ePMgll1yC\nqqrExMQwfvz4Dm3Iz88nOzvb3+YW/+vMf/uC7tg/OzubgwcPUllZyZw5c/j0008ZO3YsaWlpQPA4\nfvjwYf/IKcAll1zi//vXv/7V/3nrvrBv3z527tzJBx98APj6XFVVFZmZmfzrX/+iurqaGTNmkJiY\nSEpKCq+//jpvvPEG06ZNIzs7m6ampg7XoHfeeafTGNYd9u3bx3XXXYdOpwN859Pr9QaMXQUFBWRk\nZAC+1LKW44yKivKPqnUWXydPnszOnTuZOXMmu3bt4pZbbiEvLy9g/Gxh9uzZbdobLMaNGzcOs9mM\n2WzGYrEwffp0wNcHiouLzxq/gsWe1nR2jQ107czMzMThcGC326muruaSSy4hLy+PQ4cO+Y+jOwTy\nDwjshwUFBTz88MOAz2dWr17t305nsTqQthhs7N27l3379vGzn/0M8F27ysvLOX78ODNnzvTrjtbX\nzmDnv6XPK4oSsE8E0kBdaUtsbGzAmGW326mpqfH7h9Fo7HQ7Y8eODWqHXhXGiqJ0eK+qKuJ0hbjW\nE17mzJnD1KlT2bVrFz//+c957LHHOjzeav1eVVW8Xi9CCCZMmMADDzzQYf9PP/00+/fvZ8uWLaxb\nt44lS5Zw1113UVBQwK5du1i8eDHPPPMMx44d8wfPs9H6EWlLG3Jzc9m/fz9PPvkkJpOJpUuX+o9N\np9P57dDy/WAIIbj88sv5/ve/D8DatWupra31vwffxKIW0dkenU7nzyfTNK1N8ArEihUrWLRoEaNG\njWLjxo3k5uZ2wQLBabFN6+Nsn1cXEhLif93dcyeE4KmnnvI7fAurVq1i3LhxLFq0iMrKyjaPsFpE\n8Nlo8S1VVQPm5LU/Ny20PFpqf9yt0TSN0NBQfv3rXwfcd2ubtOfVV1/l61//OtOnTyc3N5d//vOf\nnba/pa2BPlcUxf9eURT/cQazKwT2956mfb+ur68Paq+u2iPYPlqf34MHD7bp9637qqIoAf1ZCMHt\nt9/e4ZHw7t27u3q4HXjzzTeD+m9f0B37Z2dn8/HHH1NbW8tNN93E+++/T25url9oBIrjer3enzLQ\nQutrQ+vX7ePDQw891CGtJDk5mfT0dHbt2sWyZcu46667GD9+PM8++yy7du3iH//4BxMmTOCyyy7r\ncA2yWCw0NTWdo6XOjk6nCxi7du/eHTCNoKtccsklrFu3jrCwMNLS0jCbzZ3GT+gY+4LFuPbnv73f\nny1+BYs9rWnf11ro7NqZmZnJhg0bSEpKIjs7mw0bNpCfn9/mBqurJCUldfAPCO6HwegsVgfSFuHh\n4d1ua38zZ84crr322jafrV27Nuj3A53/ioqKNn0+UJ8417ZUVlZ28FmXy9Xt7XRGr06+q6qqIj8/\nH4AvvviCMWPGEBcXR2FhIUCb2Yzl5eWkpKQwZ84c0tLSOHnyZJf2kZmZyeHDh/2jLQ6Hg9LSUv9I\n4tSpU7nttts4fvy4fz8ZGRncfPPNREREUF1dzZ49e5gyZco5H6fNZiM0NBSTycTJkycpKCg4628m\nTJjgT3jXNA2bzcaECRPYsmWL/64rNTWVL7/80v++qamJ0aNHk5eXR2NjIx6Pp40Nhw0b5rftjh07\n/BfziRMnsnHjRpxOp387LbaKjo7G4/Hw+eef+7djNpux2+0d2pyUlERlZaXf1ps3b+70rgvo1Lbd\nPXcTJ05k3bp1/t8fO3YM8Nk/JiYG8FUnCEYwO3SF9uemqanJP4oYiNY2tFgsxMXFkZOTA/iCR0vb\nA/2uZbS2/bFt2rSpy+3tDsHsGoxg/tFTmM3moPbqij262r5z6feTJ0/m448/9t90tvhrdnY2OTk5\naJpGbW1twJvMzMxM/2grnPG/rvpvX9GZ/dPT08nPz/ePAo0aNYr169f7hXGgOB5IFH711Vf+vy0j\nqe2ZNGkSa9eu9V9si4qKAN9FNz4+nhtvvJHp06dz/PhxampqMBqNXHbZZXzjG9/wx8H216C0tLRu\nxbDOfGnixIl88skn/jjb1NQUNHYdOHDAL8aysrLYunUrmqZRV1fn95XO4uvYsWMpKiri008/9Y8E\nB4ufPU134lcL7e0WrK91du3Mzs7mgw8+IDs7m9GjR5Obm4vBYOgwstgVgvlHID/MzMz0f96iW4Id\nY+tYHUhbDAZan6tJkyaxYcMG/3HV1NRQX1/P+PHj2bJlC42NjcDZr52t+3ywPhFIA7UmWFs6Ow6r\n1eofOHS73Tidzm5vB3p5xDgpKYl169axcuVKhg8fznXXXUd6ejovvfQSb775ZpuAtGbNGnJzc1EU\nheTkZKZMmUJtbe1Z9xEREcGCBQv43e9+57/T/N73vofZbOa5557D7XYjhPDfZb7xxhv+1IPx48cz\ncuRI/vSnP3HzzTef83FOnjyZTz75hAcffJDExMSggb41t912Gy+//DKfffYZqqryk5/8hMzMTL73\nve/x5JNPIoRAp9Nx1VVXtXn/4x//mLlz5/L4449jsVj8jxMArr76an7961+zaNEiJk2a5B8xmDx5\nMseOHWPx4sXo9XqmTJnC97//fW6++Wb+93//l4iICDIyMvydY/bs2fzpT39i7dq1/gkfgH9i0PLl\ny/2TQzq7C9M0jfLycv8kvPZ099zdfvvtrFq1iocffhiv10t2djZ33XUX3/zmN1mxYgX/+te/mDp1\naqfnKZAdukJycnKHc/PjH/846Pfb2/C+++7jlVde8U8cu+SSS9qcuxamTZvG8uXL2b59O3fccQdz\n585l+fLlhIaGMn78+IBpHudLMLsGIzw8nKysLB566CEmT5583pPvAhHMXl2xRzD/bU9eXl63+/1V\nV11FZWWlv/xgREQEixYt4uKLL+bAgQM8+OCDbdKFWhMREcFdd93Fb37zG4QQRERE8POf/7zL/tuX\nBLO/wWDAarX6Y1x2djZffvklKSkpQOA4/vzzz3eo7NPU1MTDDz+MwWDg/vvvD9iG7373u/zlL3/h\n4YcfRghBXFwcixcvJicnh82bN6PT6YiKiuLb3/42R44c4Y033vCP9N95551Ax2vQ7bffTkZGRpdj\nWHtfv/766/3/d/XVV1NWVsbDDz+MXq/n6quvZsaMGR1iV0NDAwaDAbPZDPhSzvbv38/ChQuxWq2k\npqb608eCxVdVVZk6dSobN25kwYIFQPD42RsTN7sav1pISUlBVVX/5Ltgfa2za+eYMWOorq4mOzsb\nVVWxWq3nfGzFxcUd/GP58uUB/fCOO+7gj3/8I++//75/8l0g2sfqDz/8sIO2GAy09/FLL72Uxx57\nDPCNkN97772MGDGCb33rWyxduhRVVRk1apTfDwOxZ88ef5+32+0Br+fBNFALkyZN4uTJkx3aEiyV\nBeCee+7h5Zdf5q233kKn07Fw4cKg24mMjAxulC7kZg9pqqqqxFNPPdXfzRiSHDx4UPzpT3/q72ZI\nJB2Q/b5vaJmg2JrWE696k56cJHo+bNq0Sfz73/9u81nLhLmGhgZxzz33iNra2v5oWp8wUPtaX/nh\nhUagPj/YkEtCSyQSiaTPaCmF1jKRp7eorKz0l1YbaCxdupTm5mY8Hg/f/OY3ueKKK/q7SRccfeWH\nksGHFMYSiUQikUgkEgkDfOU7iUQikUgkEomkr5DCWCKRSCQSiUQiQQpjiUQikUgkEokEkMJYIpFI\nJBKJRCIBpDCWSCQSiUQikUgAKYwlEolEIpFIJBIA/j9pOk2puGqGNwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#class distribution\n",
    "plt.figure(figsize=(12,8))\n",
    "\n",
    "plt.title(\"Class distributions\")\n",
    "\n",
    "plt.plot(  fasttext_df.columns[1:] , np.bincount(np.argmax(y_train,axis=1)) ,label=\"Train class distribution\"  )\n",
    "\n",
    "plt.plot(  fasttext_df.columns[1:] , np.bincount(np.argmax(y_val,axis=1)) ,label=\"Validation class distribution\" )\n",
    "\n",
    "\n",
    "plt.plot(  fasttext_df.columns[1:] , np.bincount(np.argmax(y_test,axis=1)) , label= \"Test predicted class distribution\" )\n",
    "\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "dTZqUTFHWoG4",
    "outputId": "f2aafe51-8007-4bd1-f438-141d28624732"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the number of empty websites is 30\n"
     ]
    }
   ],
   "source": [
    "#postprocessing \n",
    "test_df = pd.read_csv(\"./data/test_preprocessed.csv\")\n",
    "k ,a= 0,[]\n",
    "for t in list(test_df.text):\n",
    "    if len(str(t).split())<2:\n",
    "        a.append(k)        \n",
    "    k+=1\n",
    "print(\"the number of empty websites is \"+str(len(a))) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "9mWWNK8pmEv_",
    "outputId": "95964208-1eeb-4d67-fbf5-56b564269e07"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.29421309, 0.10167658, 0.27582477, 0.0427258 , 0.04056247,\n",
       "       0.09410492, 0.02325581, 0.12763656])"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#compute Class frequency vector\n",
    "frequency_vector = np.bincount(np.argmax(y_train,axis=1))/np.sum(np.bincount(np.argmax(y_train,axis=1)))\n",
    "frequency_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "id": "tMY1IuYqm-te",
    "outputId": "b3ae7fc2-54ee-433a-e938-6d4982e986f9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>business/finance</th>\n",
       "      <th>education/research</th>\n",
       "      <th>entertainment</th>\n",
       "      <th>health/medical</th>\n",
       "      <th>news/press</th>\n",
       "      <th>politics/government/law</th>\n",
       "      <th>sports</th>\n",
       "      <th>tech/science</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18965</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>12970</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>8759</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>26547</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>19524</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Host  business/finance  ...    sports  tech/science\n",
       "17  18965          0.294213  ...  0.023256      0.127637\n",
       "24  12970          0.294213  ...  0.023256      0.127637\n",
       "51   8759          0.294213  ...  0.023256      0.127637\n",
       "91  26547          0.294213  ...  0.023256      0.127637\n",
       "99  19524          0.294213  ...  0.023256      0.127637\n",
       "\n",
       "[5 rows x 9 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replace the empty websites predictions with frequency vector\n",
    "fasttext_df.loc[a,fasttext_df.columns[1:]] = frequency_vector\n",
    "fasttext_df.iloc[a][:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iwuYE7fqhlqE"
   },
   "outputs": [],
   "source": [
    "fasttext_df.to_csv(\"fastText_df.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SE-GbX0Mwn8w"
   },
   "source": [
    "# Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3HhLmOzQtEAt"
   },
   "outputs": [],
   "source": [
    "# Load data \n",
    "df = pd.read_csv(\"./data/train_preprocessed.csv\")\n",
    "test_df = pd.read_csv(\"./data/test_preprocessed.csv\")\n",
    "df = df.dropna()\n",
    "X = df.text.values\n",
    "y = df.label.values\n",
    "# train test split\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "       X, y, test_size=0.25, random_state=70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "1__H-8Rlwn8x",
    "outputId": "c8410f40-2b2f-4315-ed47-4c19c700c847"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 50000 pretrained word vectors\n"
     ]
    }
   ],
   "source": [
    "w2v = Word2Vec(fr_embeddings, vocab_size=50000)\n",
    "sentence2vec = BagOfWords(w2v)\n",
    "\n",
    "# encode data with sentence2vec\n",
    "X_train_encode = [sentence2vec.encode(X_train[i]).tolist() for i in range(len(X_train))]\n",
    "X_test_encode = [sentence2vec.encode(X_test[i]).tolist() for i in range(len(X_test))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y53-4qTtwn80"
   },
   "outputs": [],
   "source": [
    "# Create logistic regression\n",
    "logistic = LogisticRegression(multi_class=\"auto\")\n",
    "\n",
    "# Create regularization penalty space\n",
    "penalty = ['l1', 'l2']\n",
    "\n",
    "# Create regularization hyperparameter space\n",
    "C = np.logspace(0, 4, 10)\n",
    "\n",
    "# Create hyperparameter options\n",
    "hyperparameters = dict(C=C, penalty=penalty)\n",
    "\n",
    "# Create grid search using 5-fold cross validation\n",
    "clf = GridSearchCV(logistic, hyperparameters, cv=5, verbose=0)\n",
    "\n",
    "# Fit grid search\n",
    "best_model = clf.fit(X_train_encode, y_train)\n",
    "\n",
    "# View best hyperparameters\n",
    "print('Best Penalty:', best_model.best_estimator_.get_params()['penalty'])\n",
    "print('Best C:', best_model.best_estimator_.get_params()['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "t0mWrxq2wn82"
   },
   "outputs": [],
   "source": [
    "y_pred = best_model.predict(X_test_encode)\n",
    "print(\"accuracy score : \",accuracy_score(y_test, y_pred))\n",
    "print(\"Loss :\", log_loss(y_test, best_model.predict_proba(X_test_encode)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rkbP3CtAwn84"
   },
   "source": [
    "# Bert, XLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 694
    },
    "colab_type": "code",
    "id": "rcD-vIa2-na_",
    "outputId": "97ab9a44-7782-455a-f583-7fa809186cf9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/13/33/ffb67897a6985a7b7d8e5e7878c3628678f553634bd3836404fef06ef19b/transformers-2.5.1-py3-none-any.whl (499kB)\n",
      "\r",
      "\u001b[K     |▋                               | 10kB 20.7MB/s eta 0:00:01\r",
      "\u001b[K     |█▎                              | 20kB 4.5MB/s eta 0:00:01\r",
      "\u001b[K     |██                              | 30kB 6.4MB/s eta 0:00:01\r",
      "\u001b[K     |██▋                             | 40kB 8.1MB/s eta 0:00:01\r",
      "\u001b[K     |███▎                            | 51kB 5.3MB/s eta 0:00:01\r",
      "\u001b[K     |████                            | 61kB 6.2MB/s eta 0:00:01\r",
      "\u001b[K     |████▋                           | 71kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████▎                          | 81kB 7.8MB/s eta 0:00:01\r",
      "\u001b[K     |██████                          | 92kB 8.6MB/s eta 0:00:01\r",
      "\u001b[K     |██████▋                         | 102kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████▏                        | 112kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████▉                        | 122kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |████████▌                       | 133kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▏                      | 143kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▉                      | 153kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▌                     | 163kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▏                    | 174kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▉                    | 184kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▌                   | 194kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▏                  | 204kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▉                  | 215kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▍                 | 225kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████                 | 235kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▊                | 245kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▍               | 256kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████               | 266kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▊              | 276kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▍             | 286kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████             | 296kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▊            | 307kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▍           | 317kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████           | 327kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▋          | 337kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▎         | 348kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████         | 358kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▋        | 368kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▎       | 378kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████       | 389kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▋      | 399kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▎     | 409kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████     | 419kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▋    | 430kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▏   | 440kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▉   | 450kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▌  | 460kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▏ | 471kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▉ | 481kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▌| 491kB 7.0MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 501kB 7.0MB/s \n",
      "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.11.15)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
      "\r",
      "\u001b[K     |▎                               | 10kB 21.9MB/s eta 0:00:01\r",
      "\u001b[K     |▋                               | 20kB 28.0MB/s eta 0:00:01\r",
      "\u001b[K     |█                               | 30kB 31.8MB/s eta 0:00:01\r",
      "\u001b[K     |█▎                              | 40kB 34.1MB/s eta 0:00:01\r",
      "\u001b[K     |█▋                              | 51kB 34.3MB/s eta 0:00:01\r",
      "\u001b[K     |██                              | 61kB 35.5MB/s eta 0:00:01\r",
      "\u001b[K     |██▏                             | 71kB 36.7MB/s eta 0:00:01\r",
      "\u001b[K     |██▌                             | 81kB 37.1MB/s eta 0:00:01\r",
      "\u001b[K     |██▉                             | 92kB 37.6MB/s eta 0:00:01\r",
      "\u001b[K     |███▏                            | 102kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███▌                            | 112kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███▉                            | 122kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████                            | 133kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████▍                           | 143kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████▊                           | 153kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████                           | 163kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████▍                          | 174kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████▊                          | 184kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████                          | 194kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████▎                         | 204kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████▋                         | 215kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████                         | 225kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████▎                        | 235kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████▋                        | 245kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████▉                        | 256kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████▏                       | 266kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████▌                       | 276kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████▉                       | 286kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▏                      | 296kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▌                      | 307kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▊                      | 317kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████                      | 327kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▍                     | 337kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▊                     | 348kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████                     | 358kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▍                    | 368kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▋                    | 378kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████                    | 389kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▎                   | 399kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▋                   | 409kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████                   | 419kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▎                  | 430kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▌                  | 440kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▉                  | 450kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▏                 | 460kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▌                 | 471kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▉                 | 481kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▏                | 491kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▍                | 501kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▊                | 512kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████                | 522kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▍               | 532kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▊               | 542kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████               | 552kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▎              | 563kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▋              | 573kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████              | 583kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▎             | 593kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▋             | 604kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████             | 614kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▏            | 624kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▌            | 634kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▉            | 645kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▏           | 655kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▌           | 665kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▉           | 675kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▏          | 686kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▍          | 696kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▊          | 706kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████          | 716kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▍         | 727kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▊         | 737kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████         | 747kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▎        | 757kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▋        | 768kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████        | 778kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▎       | 788kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▋       | 798kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████       | 808kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▏      | 819kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▌      | 829kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▉      | 839kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▏     | 849kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▌     | 860kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▉     | 870kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████     | 880kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▍    | 890kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▊    | 901kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████    | 911kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▍   | 921kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▊   | 931kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████   | 942kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▎  | 952kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▋  | 962kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████  | 972kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▎ | 983kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▋ | 993kB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▉ | 1.0MB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▏| 1.0MB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▌| 1.0MB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▉| 1.0MB 38.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 1.0MB 38.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
      "\u001b[K     |████████████████████████████████| 870kB 39.0MB/s \n",
      "\u001b[?25hCollecting tokenizers==0.5.2\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
      "\u001b[K     |████████████████████████████████| 3.7MB 38.8MB/s \n",
      "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.5)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.3)\n",
      "Requirement already satisfied: botocore<1.15.0,>=1.14.15 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.14.15)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers) (2.6.1)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers) (0.15.2)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=d8fa359b31500290b2c48b3f2b2592d621fab139078fde21c85ccd483d5895f0\n",
      "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sentencepiece, sacremoses, tokenizers, transformers\n",
      "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.5.2 transformers-2.5.1\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fPNRbelMjTcv"
   },
   "outputs": [],
   "source": [
    "# Load data \n",
    "df = pd.read_csv(\"./data/train_preprocessed.csv\")\n",
    "test_df = pd.read_csv(\"./data/test_preprocessed.csv\")\n",
    "X = df.text.values\n",
    "y = df.label.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OGcO0OBo-4Cm"
   },
   "outputs": [],
   "source": [
    "train_sentences = list(X)\n",
    "train_label = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7vkks0RMkaly"
   },
   "outputs": [],
   "source": [
    "train_sentences = [str(sentence) for sentence in train_sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EqSBk85ck_cg"
   },
   "outputs": [],
   "source": [
    "train_label = train_label.astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xtHIi7CP1vV9"
   },
   "outputs": [],
   "source": [
    "# set device to use GPU\n",
    "import torch\n",
    "device = torch.device(\"cuda\")\n",
    "\n",
    "# Models\n",
    "models = [\"Bert\", \"XLM\"]\n",
    "model = \"Bert\"\n",
    "\n",
    "# params\n",
    "MAX_LEN = 140\n",
    "batch_size = 16\n",
    "epochs = 4\n",
    "lr = 1e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102,
     "referenced_widgets": [
      "4cd778cd88fc4d06a61b08e003f1f6ea",
      "f9cf597b9113461392d75b3c75fbe130",
      "4c76d6507b3546b8b1ee277d78d262a6",
      "5f0349e3159e4add863f65c02f5e44f3",
      "c1187efe0c514100bd9325f50ad05636",
      "035d7228890f456a9af164bcb79afe62",
      "01c2ef36955049f6b8f4ed87d9b66015",
      "662983832d29456f9e9cfc29c17c78e9"
     ]
    },
    "colab_type": "code",
    "id": "ee3UWui58IqR",
    "outputId": "188faa85-f53b-4085-b1c2-466d27bd3fe3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading  tokenizer...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cd778cd88fc4d06a61b08e003f1f6ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=871891, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/1849 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1849/1849 [00:42<00:00, 28.16it/s]\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, XLMTokenizer\n",
    "\n",
    "# Load the model tokenizer.\n",
    "print('Loading  tokenizer...')\n",
    "if model == \"Bert\":\n",
    "    tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased', do_lower_case=True)\n",
    "if model == \"XLM\":\n",
    "    tokenizer = XLMTokenizer.from_pretrained('xlm-clm-enfr-1024', do_lower_case=True)\n",
    "\n",
    "\n",
    "\n",
    "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
    "input_ids = []\n",
    "\n",
    "# For every sentence...\n",
    "for sent in tqdm(train_sentences):\n",
    "    encoded_sent = tokenizer.encode(\n",
    "                        sent,                      # Sentence to encode.\n",
    "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
    "                        max_length = 512,          # Truncate all sentences.)\n",
    "    )\n",
    "    \n",
    "    # Add the encoded sentence to the list.\n",
    "    input_ids.append(encoded_sent)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "X9cnE5_0-ysE"
   },
   "outputs": [],
   "source": [
    "#`pad_sequences` utility .\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# Set the maximum sequence length.\n",
    "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", \n",
    "                          value=0, truncating=\"post\", padding=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "Hz8955sf_KVl",
    "outputId": "a5bc3bb4-e06c-4043-958a-9cadfcb7700d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1849/1849 [00:00<00:00, 10377.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating attention masks\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Create attention masks\n",
    "print(\"Creating attention masks\")\n",
    "attention_masks = []\n",
    "from tqdm import tqdm\n",
    "# For each sentence...\n",
    "for sent in tqdm(input_ids):\n",
    "    att_mask = [int(token_id > 0) for token_id in sent]\n",
    "    attention_masks.append(att_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KQSgeDLB_MvP"
   },
   "outputs": [],
   "source": [
    "# Use train_test_split to split our data into train and validation sets for\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Use 90% for training and 10% for validation.\n",
    "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, train_label, \n",
    "                                                            random_state=2018, test_size=0.2)\n",
    "# Do the same for the masks.\n",
    "train_masks, validation_masks, _, _ = train_test_split(attention_masks, train_label,\n",
    "                                             random_state=2018, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gECqQObo_S86"
   },
   "outputs": [],
   "source": [
    "# Convert all inputs and labels into torch tensors, the required datatype \n",
    "train_inputs = torch.tensor(train_inputs)\n",
    "validation_inputs = torch.tensor(validation_inputs)\n",
    "\n",
    "train_labels = torch.tensor(train_labels)\n",
    "validation_labels = torch.tensor(validation_labels)\n",
    "\n",
    "train_masks = torch.tensor(train_masks)\n",
    "validation_masks = torch.tensor(validation_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QT6JyT1y_WHc"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "# Create the DataLoader for our training set.\n",
    "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
    "\n",
    "# Create the DataLoader for our validation set.\n",
    "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
    "validation_sampler = SequentialSampler(validation_data)\n",
    "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 116,
     "referenced_widgets": [
      "0048433a2f4b4d979ab88ae4e1ee5a20",
      "0ea92569bcb048fda46fdb885407cc34",
      "940a05591b8045638a052327f63c60cb",
      "e104835252164712a1da835d1d84955a",
      "ed94d4ec0ca34826821f9adbafeef372",
      "e033fd7029e1478e981910c4e1059aec",
      "a5087301f0fe46e9af59324a2bb9f7cb",
      "d7a9332c1dc441ab9b9888bca0e8b0bb",
      "edeb6b95f1bf44bdb9963d009dee5f68",
      "6bc01417858a434589240d8e4bd83eeb",
      "2da7698f0eb445bdbbe77f42323f2a9a",
      "ad17c9e63dd64b2da665576f9842a443",
      "bab49b1a707d4f57aa12a0a016a86d94",
      "33a349fc98b94b40ba1479362cffed7a",
      "435ebcf1669f45c4a4abe13e805ca9a5",
      "91fa701dd6fb4e199896ee73b67fcfbb"
     ]
    },
    "colab_type": "code",
    "id": "_kOb9QY6_XRd",
    "outputId": "ab0deeac-a919-4858-dd3a-3344f4361978"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0048433a2f4b4d979ab88ae4e1ee5a20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=569, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "edeb6b95f1bf44bdb9963d009dee5f68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=672271273, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, XLMForSequenceClassification, DistilBertForSequenceClassification\n",
    "\n",
    "if model == \"Bert\":\n",
    "    model = BertForSequenceClassification.from_pretrained(\"bert-base-multilingual-uncased\", num_labels = 8, output_attentions = False, output_hidden_states = False)\n",
    "if model == \"XLM\":\n",
    "    model = XLMForSequenceClassification.from_pretrained('xlm-clm-enfr-1024', num_labels = 8, output_attentions = False, output_hidden_states = False)\n",
    "\n",
    "\n",
    "\n",
    "# Tell pytorch to run this model on the GPU.\n",
    "model.cuda()\n",
    "# Get all of the model's parameters as a list of tuples.\n",
    "params = list(model.named_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LXWa1af3Atm4"
   },
   "outputs": [],
   "source": [
    "from torch import optim\n",
    "from transformers import AdamW\n",
    "\n",
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr = lr, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
    "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
    "                )\n",
    "\n",
    "#optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wMr1tPbwA3Ux"
   },
   "outputs": [],
   "source": [
    "from transformers import get_linear_schedule_with_warmup,get_cosine_with_hard_restarts_schedule_with_warmup\n",
    "\n",
    "# Number of training epochs (authors recommend between 2 and 4)\n",
    "\n",
    "# Total number of training steps is number of batches * number of epochs.\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "\n",
    "# Create the learning rate scheduler.\n",
    "scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
    "                                            num_training_steps = total_steps)\n",
    "\n",
    "\n",
    "#get_linear_schedule_with_warmup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8DRXDWFuA6UK"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Function to calculate the accuracy of our predictions vs labels\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    #print(np.shape(preds))\n",
    "    #print(preds)\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "\n",
    "\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "def format_time(elapsed):\n",
    "    '''\n",
    "    Takes a time in seconds and returns a string hh:mm:ss\n",
    "    '''\n",
    "    # Round to the nearest second.\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "    \n",
    "    # Format as hh:mm:ss\n",
    "    return str(datetime.timedelta(seconds=elapsed_rounded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "NLc-EULrwn85",
    "outputId": "4f984c08-152e-4490-90cf-8d0e35caaa6f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "======== Epoch 1 / 4 ========\n",
      "Training...\n",
      "  Batch    40  of     93.    Elapsed: 0:00:34.\n",
      "  Batch    80  of     93.    Elapsed: 0:01:07.\n",
      "\n",
      "  Average training loss: 1.77\n",
      "  Training epcoh took: 0:01:18\n",
      "\n",
      "  Average validation loss: 1.42\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.51\n",
      "  Validation took: 0:00:06\n",
      "\n",
      "======== Epoch 2 / 4 ========\n",
      "Training...\n",
      "  Batch    40  of     93.    Elapsed: 0:00:34.\n",
      "  Batch    80  of     93.    Elapsed: 0:01:07.\n",
      "\n",
      "  Average training loss: 1.47\n",
      "  Training epcoh took: 0:01:18\n",
      "\n",
      "  Average validation loss: 1.21\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.59\n",
      "  Validation took: 0:00:06\n",
      "\n",
      "======== Epoch 3 / 4 ========\n",
      "Training...\n",
      "  Batch    40  of     93.    Elapsed: 0:00:34.\n",
      "  Batch    80  of     93.    Elapsed: 0:01:07.\n",
      "\n",
      "  Average training loss: 1.25\n",
      "  Training epcoh took: 0:01:18\n",
      "\n",
      "  Average validation loss: 1.17\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.57\n",
      "  Validation took: 0:00:06\n",
      "\n",
      "======== Epoch 4 / 4 ========\n",
      "Training...\n",
      "  Batch    40  of     93.    Elapsed: 0:00:34.\n",
      "  Batch    80  of     93.    Elapsed: 0:01:07.\n",
      "\n",
      "  Average training loss: 1.10\n",
      "  Training epcoh took: 0:01:18\n",
      "\n",
      "  Average validation loss: 1.07\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.60\n",
      "  Validation took: 0:00:06\n",
      "\n",
      "Training complete!\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "# This training code is based on the `run_glue.py` script here:\n",
    "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
    "\n",
    "\n",
    "# Set the seed value all over the place to make this reproducible.\n",
    "seed_val = 42\n",
    "\n",
    "random.seed(seed_val)\n",
    "np.random.seed(seed_val)\n",
    "torch.manual_seed(seed_val)\n",
    "torch.cuda.manual_seed_all(seed_val)\n",
    "\n",
    "# Store the average loss after each epoch so we can plot them.\n",
    "loss_values = []\n",
    "\n",
    "# For each epoch...\n",
    "for epoch_i in range(0, epochs):\n",
    "    \n",
    "    # ========================================\n",
    "    #               Training\n",
    "    # ========================================\n",
    "    \n",
    "    # Perform one full pass over the training set.\n",
    "\n",
    "    print(\"\")\n",
    "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
    "    print('Training...')\n",
    "\n",
    "    # Measure how long the training epoch takes.\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Reset the total loss for this epoch.\n",
    "    total_loss = 0\n",
    "    model.train()\n",
    "\n",
    "    # For each batch of training data...\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "        # Progress update every 40 batches.\n",
    "        if step % 40 == 0 and not step == 0:\n",
    "            # Calculate elapsed time in minutes.\n",
    "            elapsed = format_time(time.time() - t0)\n",
    "            \n",
    "            # Report progress.\n",
    "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        model.zero_grad()        \n",
    "\n",
    "        outputs = model(b_input_ids, \n",
    "                    token_type_ids=None, \n",
    "                    attention_mask=b_input_mask, \n",
    "                    labels=b_labels)\n",
    "        \n",
    "        loss = outputs[0]\n",
    "        total_loss += loss.item()\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        optimizer.step()\n",
    "        #scheduler.step()\n",
    "\n",
    "    # Calculate the average loss over the training data.\n",
    "    avg_train_loss = total_loss / len(train_dataloader)            \n",
    "    \n",
    "    # Store the loss value for plotting the learning curve.\n",
    "    loss_values.append(avg_train_loss)\n",
    "    print(\"\")\n",
    "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "    print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
    "\n",
    "    # ========================================\n",
    "    #               loss validation\n",
    "    # ========================================\n",
    "\n",
    "    # Reset the total loss for this epoch.\n",
    "    total_loss = 0\n",
    "    model.eval()\n",
    "\n",
    "    # For each batch of training data...\n",
    "    for step, batch in enumerate(validation_dataloader):\n",
    "\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        model.zero_grad()        \n",
    "\n",
    "        outputs = model(b_input_ids, \n",
    "                    token_type_ids=None, \n",
    "                    attention_mask=b_input_mask, \n",
    "                    labels=b_labels)\n",
    "        \n",
    "        loss = outputs[0]\n",
    "        total_loss += loss.item()\n",
    "\n",
    "    # Calculate the average loss over the training data.\n",
    "    avg_train_loss = total_loss / len(validation_dataloader)            \n",
    "    \n",
    "    # Store the loss value for plotting the learning curve.\n",
    "    loss_values.append(avg_train_loss)\n",
    "    print(\"\")\n",
    "    print(\"  Average validation loss: {0:.2f}\".format(avg_train_loss))\n",
    "        \n",
    "    # ========================================\n",
    "    #               Validation\n",
    "    # ========================================\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Running Validation...\")\n",
    "\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Put the model in evaluation mode--the dropout layers behave differently\n",
    "    # during evaluation.\n",
    "    model.eval()\n",
    "\n",
    "    # Tracking variables \n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "\n",
    "    # Evaluate data for one epoch\n",
    "    for batch in validation_dataloader:\n",
    "        \n",
    "        # Add batch to GPU\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        \n",
    "        # Unpack the inputs from our dataloader\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        \n",
    "        # Telling the model not to compute or store gradients, saving memory and\n",
    "        # speeding up validation\n",
    "        with torch.no_grad():        \n",
    "\n",
    "            outputs = model(b_input_ids, \n",
    "                            token_type_ids=None, \n",
    "                            attention_mask=b_input_mask)\n",
    "        \n",
    "\n",
    "        logits = outputs[0]\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        \n",
    "        # Calculate the accuracy for this batch of test sentences.\n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "        \n",
    "        # Accumulate the total accuracy.\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "        # Track the number of batches\n",
    "        nb_eval_steps += 1\n",
    "\n",
    "    # Report the final accuracy for this validation run.\n",
    "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
    "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
    "\n",
    "print(\"\")\n",
    "print(\"Training complete!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WPUXXQihCHqB"
   },
   "outputs": [],
   "source": [
    "torch.save(model, './models/Bert_last')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Y4ovHdXQCIS9"
   },
   "source": [
    "Bert : loss 1.20, acc 0.57 epoch 7, 100 words\n",
    "\n",
    "Bert  no warmup : loss 1.15, acc 0.65, epoch 3, 100 words, lr = 1.e-5\n",
    "\n",
    "XLM with clm, loss 1.32, acc 0.59 epoch 5 , 100 words\n",
    "\n",
    "Bert  cosine warmup : loss 1.08, acc 0.62, epoch 4, 100 words, lr = 1.e-5\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KTcP9Ry6Sx8c"
   },
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "orFXQ7tST7hW",
    "outputId": "66079d03-5d52-4ec4-a690-f08ce16548f7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of test sentences: 560\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "\n",
    "# Report the number of sentences.\n",
    "print('Number of test sentences: {:,}\\n'.format(test_df.shape[0]))\n",
    "\n",
    "# Create sentence and label lists\n",
    "sentences = test_df.text.values\n",
    "labels = np.zeros(test_df.shape[0])\n",
    "\n",
    "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
    "input_ids = []\n",
    "\n",
    "# For every sentence...\n",
    "for sent in sentences:\n",
    "    # `encode` will:\n",
    "    #   (1) Tokenize the sentence.\n",
    "    #   (2) Prepend the `[CLS]` token to the start.\n",
    "    #   (3) Append the `[SEP]` token to the end.\n",
    "    #   (4) Map tokens to their IDs.\n",
    "    encoded_sent = tokenizer.encode(\n",
    "                        sent,                      # Sentence to encode.\n",
    "                        add_special_tokens = True,\n",
    "                        max_length = 512,  # Add '[CLS]' and '[SEP]'\n",
    "                   )\n",
    "    \n",
    "    input_ids.append(encoded_sent)\n",
    "\n",
    "# Pad our input tokens\n",
    "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, \n",
    "                          dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
    "\n",
    "# Create attention masks\n",
    "attention_masks = []\n",
    "\n",
    "# Create a mask of 1s for each token followed by 0s for padding\n",
    "for seq in input_ids:\n",
    "  seq_mask = [float(i>0) for i in seq]\n",
    "  attention_masks.append(seq_mask) \n",
    "\n",
    "# Convert to tensors.\n",
    "prediction_inputs = torch.tensor(input_ids)\n",
    "prediction_masks = torch.tensor(attention_masks)\n",
    "prediction_labels = torch.tensor(labels)\n",
    "\n",
    "\n",
    "# Set the batch size.  \n",
    "batch_size = 8  \n",
    "\n",
    "# Create the DataLoader.\n",
    "prediction_data = TensorDataset(prediction_inputs, prediction_masks, prediction_labels)\n",
    "prediction_sampler = SequentialSampler(prediction_data)\n",
    "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "JRczhR_cUc5B",
    "outputId": "9adcf1bb-6dbe-4567-ddbe-3d202f06bba2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting labels for 560 test sentences...\n",
      "DONE.\n"
     ]
    }
   ],
   "source": [
    "# Prediction on test set\n",
    "\n",
    "print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs)))\n",
    "\n",
    "# Put model in evaluation mode\n",
    "model.eval()\n",
    "\n",
    "# Tracking variables \n",
    "predictions , true_labels = [], []\n",
    "\n",
    "# Predict \n",
    "for batch in prediction_dataloader:\n",
    "  # Add batch to GPU\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "  \n",
    "  # Unpack the inputs from our dataloader\n",
    "    b_input_ids, b_input_mask, b_labels = batch\n",
    "  \n",
    "  # Telling the model not to compute or store gradients, saving memory and \n",
    "  # speeding up prediction\n",
    "    with torch.no_grad():\n",
    "      # Forward pass, calculate logit predictions\n",
    "        outputs = model(b_input_ids, token_type_ids=None, \n",
    "                      attention_mask=b_input_mask)\n",
    "\n",
    "    logits = outputs[0]\n",
    "\n",
    "    # Move logits and labels to CPU\n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    label_ids = b_labels.to('cpu').numpy()\n",
    "  \n",
    "    # Store predictions and true labels\n",
    "    predictions.append(logits)\n",
    "    true_labels.append(label_ids)\n",
    "\n",
    "print('DONE.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hbnwKXHWcE9l"
   },
   "outputs": [],
   "source": [
    "from scipy.special import softmax\n",
    "def get_probs(vect):\n",
    "    res = []\n",
    "    #n,c = np.shape(vect)\n",
    "    for batch in vect:\n",
    "        for v in batch : \n",
    "            res.append(softmax(v))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nmspuwRoVgQP"
   },
   "outputs": [],
   "source": [
    "# Write predictions to a file\n",
    "prediction_ = get_probs(predictions)\n",
    "test_hosts = list(kaggle.Host.values)\n",
    "import csv\n",
    "with open('Bert07.csv', 'w') as csvfile:\n",
    "    writer = csv.writer(csvfile, delimiter=',')\n",
    "    lst = kaggle.columns.to_list()\n",
    "    writer.writerow(lst)\n",
    "    for i,test_host in enumerate(test_hosts):\n",
    "        lst = prediction_[i].tolist()\n",
    "        lst.insert(0, test_host)\n",
    "        writer.writerow(lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zS14AiGO8uDL"
   },
   "outputs": [],
   "source": [
    "#final model\n",
    "\n",
    "tfidf = pd.read_csv(\"tfidf.csv\")\n",
    "bert = pd.read_csv(\"Bert07.csv\")\n",
    "fasttext = pd.read_csv(\"fastText_df.csv\")\n",
    "\n",
    "final_pred = tfidf.copy()\n",
    "final_pred.loc[:,final_pred.columns[1:]] = 0.25 * bert.loc[:,bert.columns[1:]] + 0.7*fasttext.loc[:,fasttext.columns[1:]] + 0.05 *tfidf.loc[:,tfidf.columns[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "t6FpJ5LI-2U0"
   },
   "outputs": [],
   "source": [
    "#Replace the empty websites predictions with frequency vector\n",
    "final_pred.loc[a,final_pred.columns[1:]] = frequency_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 947
    },
    "colab_type": "code",
    "id": "7XVrrrlM_Xut",
    "outputId": "4635506f-2399-43dd-8606-400c67838f77"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>business/finance</th>\n",
       "      <th>education/research</th>\n",
       "      <th>entertainment</th>\n",
       "      <th>health/medical</th>\n",
       "      <th>news/press</th>\n",
       "      <th>politics/government/law</th>\n",
       "      <th>sports</th>\n",
       "      <th>tech/science</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18965</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>12970</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>8759</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>26547</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>19524</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>10861</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>18155</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>15151</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>13560</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>19885</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>2811</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>27886</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>11059</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>13690</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>15842</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>18399</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>24824</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>11534</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>5707</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>10728</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>20627</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>13445</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>27463</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>467</th>\n",
       "      <td>5193</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>17511</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509</th>\n",
       "      <td>26875</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>528</th>\n",
       "      <td>813</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>546</th>\n",
       "      <td>11125</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>549</th>\n",
       "      <td>26035</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>553</th>\n",
       "      <td>1387</td>\n",
       "      <td>0.294213</td>\n",
       "      <td>0.101677</td>\n",
       "      <td>0.275825</td>\n",
       "      <td>0.042726</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.094105</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.127637</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Host  business/finance  ...    sports  tech/science\n",
       "17   18965          0.294213  ...  0.023256      0.127637\n",
       "24   12970          0.294213  ...  0.023256      0.127637\n",
       "51    8759          0.294213  ...  0.023256      0.127637\n",
       "91   26547          0.294213  ...  0.023256      0.127637\n",
       "99   19524          0.294213  ...  0.023256      0.127637\n",
       "111  10861          0.294213  ...  0.023256      0.127637\n",
       "114  18155          0.294213  ...  0.023256      0.127637\n",
       "135  15151          0.294213  ...  0.023256      0.127637\n",
       "139  13560          0.294213  ...  0.023256      0.127637\n",
       "146  19885          0.294213  ...  0.023256      0.127637\n",
       "196   2811          0.294213  ...  0.023256      0.127637\n",
       "197  27886          0.294213  ...  0.023256      0.127637\n",
       "201  11059          0.294213  ...  0.023256      0.127637\n",
       "255  13690          0.294213  ...  0.023256      0.127637\n",
       "262  15842          0.294213  ...  0.023256      0.127637\n",
       "314  18399          0.294213  ...  0.023256      0.127637\n",
       "327  24824          0.294213  ...  0.023256      0.127637\n",
       "355  11534          0.294213  ...  0.023256      0.127637\n",
       "363   5707          0.294213  ...  0.023256      0.127637\n",
       "409  10728          0.294213  ...  0.023256      0.127637\n",
       "438  20627          0.294213  ...  0.023256      0.127637\n",
       "447  13445          0.294213  ...  0.023256      0.127637\n",
       "454  27463          0.294213  ...  0.023256      0.127637\n",
       "467   5193          0.294213  ...  0.023256      0.127637\n",
       "499  17511          0.294213  ...  0.023256      0.127637\n",
       "509  26875          0.294213  ...  0.023256      0.127637\n",
       "528    813          0.294213  ...  0.023256      0.127637\n",
       "546  11125          0.294213  ...  0.023256      0.127637\n",
       "549  26035          0.294213  ...  0.023256      0.127637\n",
       "553   1387          0.294213  ...  0.023256      0.127637\n",
       "\n",
       "[30 rows x 9 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_pred.iloc[a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "id": "q_Uw6o_N_Awm",
    "outputId": "88d85a5c-49f1-4ced-cd89-52151b1198d0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>business/finance</th>\n",
       "      <th>education/research</th>\n",
       "      <th>entertainment</th>\n",
       "      <th>health/medical</th>\n",
       "      <th>news/press</th>\n",
       "      <th>politics/government/law</th>\n",
       "      <th>sports</th>\n",
       "      <th>tech/science</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27997</td>\n",
       "      <td>0.149249</td>\n",
       "      <td>0.495470</td>\n",
       "      <td>0.193233</td>\n",
       "      <td>0.009428</td>\n",
       "      <td>0.027103</td>\n",
       "      <td>0.029247</td>\n",
       "      <td>0.022248</td>\n",
       "      <td>0.074022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9316</td>\n",
       "      <td>0.221549</td>\n",
       "      <td>0.018925</td>\n",
       "      <td>0.602724</td>\n",
       "      <td>0.003235</td>\n",
       "      <td>0.063607</td>\n",
       "      <td>0.003240</td>\n",
       "      <td>0.043145</td>\n",
       "      <td>0.043575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27045</td>\n",
       "      <td>0.650820</td>\n",
       "      <td>0.004086</td>\n",
       "      <td>0.298257</td>\n",
       "      <td>0.009769</td>\n",
       "      <td>0.018679</td>\n",
       "      <td>0.002360</td>\n",
       "      <td>0.009390</td>\n",
       "      <td>0.006639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19805</td>\n",
       "      <td>0.050291</td>\n",
       "      <td>0.032108</td>\n",
       "      <td>0.014941</td>\n",
       "      <td>0.840549</td>\n",
       "      <td>0.011753</td>\n",
       "      <td>0.023314</td>\n",
       "      <td>0.010395</td>\n",
       "      <td>0.016649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>26580</td>\n",
       "      <td>0.059731</td>\n",
       "      <td>0.333517</td>\n",
       "      <td>0.186301</td>\n",
       "      <td>0.024385</td>\n",
       "      <td>0.082237</td>\n",
       "      <td>0.118392</td>\n",
       "      <td>0.007928</td>\n",
       "      <td>0.187509</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Host  business/finance  ...    sports  tech/science\n",
       "0  27997          0.149249  ...  0.022248      0.074022\n",
       "1   9316          0.221549  ...  0.043145      0.043575\n",
       "2  27045          0.650820  ...  0.009390      0.006639\n",
       "3  19805          0.050291  ...  0.010395      0.016649\n",
       "4  26580          0.059731  ...  0.007928      0.187509\n",
       "\n",
       "[5 rows x 9 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gt2xdyA7-Dbt"
   },
   "outputs": [],
   "source": [
    "final_pred.to_csv(\"final_pred.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "COVJFQgz_n8q"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "classification_text.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0048433a2f4b4d979ab88ae4e1ee5a20": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_940a05591b8045638a052327f63c60cb",
       "IPY_MODEL_e104835252164712a1da835d1d84955a"
      ],
      "layout": "IPY_MODEL_0ea92569bcb048fda46fdb885407cc34"
     }
    },
    "01c2ef36955049f6b8f4ed87d9b66015": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "035d7228890f456a9af164bcb79afe62": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0ea92569bcb048fda46fdb885407cc34": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1593d0a1cb394b1aa60881f9c5f3397a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2da7698f0eb445bdbbe77f42323f2a9a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading",
      "description_tooltip": null,
      "layout": "IPY_MODEL_33a349fc98b94b40ba1479362cffed7a",
      "max": 672271273,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_bab49b1a707d4f57aa12a0a016a86d94",
      "value": 672271273
     }
    },
    "33a349fc98b94b40ba1479362cffed7a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3b1eb411542044b487c566f4a846decd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "435ebcf1669f45c4a4abe13e805ca9a5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "46547a03c06443e79de78bc74fe1ee27": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eaf8cf34468d4aeda1a4a645caa67d01",
      "max": 20,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3b1eb411542044b487c566f4a846decd",
      "value": 20
     }
    },
    "4c76d6507b3546b8b1ee277d78d262a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading",
      "description_tooltip": null,
      "layout": "IPY_MODEL_035d7228890f456a9af164bcb79afe62",
      "max": 871891,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c1187efe0c514100bd9325f50ad05636",
      "value": 871891
     }
    },
    "4cd778cd88fc4d06a61b08e003f1f6ea": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_4c76d6507b3546b8b1ee277d78d262a6",
       "IPY_MODEL_5f0349e3159e4add863f65c02f5e44f3"
      ],
      "layout": "IPY_MODEL_f9cf597b9113461392d75b3c75fbe130"
     }
    },
    "5f0349e3159e4add863f65c02f5e44f3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_662983832d29456f9e9cfc29c17c78e9",
      "placeholder": "​",
      "style": "IPY_MODEL_01c2ef36955049f6b8f4ed87d9b66015",
      "value": "100% 872k/872k [00:00&lt;00:00, 1.73MB/s]"
     }
    },
    "662983832d29456f9e9cfc29c17c78e9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6bc01417858a434589240d8e4bd83eeb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "761936f1bb1447bf9062ff97f3511293": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_46547a03c06443e79de78bc74fe1ee27",
       "IPY_MODEL_8bc4f47a871b4b73b5020c49c272da17"
      ],
      "layout": "IPY_MODEL_1593d0a1cb394b1aa60881f9c5f3397a"
     }
    },
    "8bc4f47a871b4b73b5020c49c272da17": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9fe91718136b438e9629c5358b1d1520",
      "placeholder": "​",
      "style": "IPY_MODEL_cebc2c79376f4cbdb986483cc9cde515",
      "value": "100% 20/20 [09:13&lt;00:00, 34.20s/it]"
     }
    },
    "91fa701dd6fb4e199896ee73b67fcfbb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "940a05591b8045638a052327f63c60cb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e033fd7029e1478e981910c4e1059aec",
      "max": 569,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ed94d4ec0ca34826821f9adbafeef372",
      "value": 569
     }
    },
    "9fe91718136b438e9629c5358b1d1520": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a5087301f0fe46e9af59324a2bb9f7cb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ad17c9e63dd64b2da665576f9842a443": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_91fa701dd6fb4e199896ee73b67fcfbb",
      "placeholder": "​",
      "style": "IPY_MODEL_435ebcf1669f45c4a4abe13e805ca9a5",
      "value": "100% 672M/672M [00:22&lt;00:00, 29.5MB/s]"
     }
    },
    "bab49b1a707d4f57aa12a0a016a86d94": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "c1187efe0c514100bd9325f50ad05636": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "cebc2c79376f4cbdb986483cc9cde515": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d7a9332c1dc441ab9b9888bca0e8b0bb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e033fd7029e1478e981910c4e1059aec": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e104835252164712a1da835d1d84955a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d7a9332c1dc441ab9b9888bca0e8b0bb",
      "placeholder": "​",
      "style": "IPY_MODEL_a5087301f0fe46e9af59324a2bb9f7cb",
      "value": "100% 569/569 [00:00&lt;00:00, 21.7kB/s]"
     }
    },
    "eaf8cf34468d4aeda1a4a645caa67d01": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ed94d4ec0ca34826821f9adbafeef372": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "edeb6b95f1bf44bdb9963d009dee5f68": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2da7698f0eb445bdbbe77f42323f2a9a",
       "IPY_MODEL_ad17c9e63dd64b2da665576f9842a443"
      ],
      "layout": "IPY_MODEL_6bc01417858a434589240d8e4bd83eeb"
     }
    },
    "f9cf597b9113461392d75b3c75fbe130": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}