03_HyperParameterSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import copy\n",
    "import logging\n",
    "#import torch\n",
    "\n",
    "#from tqdm.autonotebook import tqdm\n",
    "\n",
    "from data.data_loader import Dataset\n",
    "from data.germeval2017 import germeval2017_dataset\n",
    "\n",
    "from misc.preferences import PREFERENCES\n",
    "#from misc.visualizer import *\n",
    "from misc.run_configuration import get_default_params, randomize_params\n",
    "from misc import utils\n",
    "\n",
    "from optimizer import get_default_optimizer\n",
    "from criterion import NllLoss, LossCombiner\n",
    "\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.softmax_output import SoftmaxOutputLayerWithCommentWiseClass\n",
    "from models.transformer_tagger import TransformerTagger\n",
    "from models.jointAspectTagger import JointAspectTagger\n",
    "from models.transformer.train import Trainer\n",
    "import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "    data_root='./data/germeval2017',\n",
    "    data_train='train_v1.4.tsv',    \n",
    "    data_validation='dev_v1.4.tsv',\n",
    "    data_test='test_TIMESTAMP1.tsv',\n",
    "    early_stopping='highest_5_F1'\n",
    ")\n",
    "def load(hp, logger):\n",
    "    dataset = Dataset(\n",
    "        'germeval',\n",
    "        logger,\n",
    "        hp,\n",
    "        source_index=0,\n",
    "        target_vocab_index=2,\n",
    "        data_path=PREFERENCES.data_root,\n",
    "        train_file=PREFERENCES.data_train,\n",
    "        valid_file=PREFERENCES.data_validation,\n",
    "        test_file=PREFERENCES.data_test,\n",
    "        file_format='.tsv',\n",
    "        init_token=None,\n",
    "        eos_token=None\n",
    "    )\n",
    "    dataset.load_data(germeval2017_dataset, verbose=False)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model(dataset, hp, experiment_name):\n",
    "    loss = LossCombiner(4, dataset.class_weights, NllLoss)\n",
    "    transformer = TransformerEncoder(dataset.source_embedding,\n",
    "                                     hyperparameters=hp)\n",
    "    tagging_softmax = SoftmaxOutputLayerWithCommentWiseClass(hp.model_size, dataset.target_size)\n",
    "    model = JointAspectTagger(transformer, hp.model_size, 4, 20, dataset.target_names)\n",
    "    optimizer = get_default_optimizer(model, hp)\n",
    "    trainer = Trainer(\n",
    "                        model,\n",
    "                        loss,\n",
    "                        optimizer,\n",
    "                        hp,\n",
    "                        dataset,\n",
    "                        experiment_name,\n",
    "                        enable_tensorboard=False,\n",
    "                        verbose=False)\n",
    "    return trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = 'HyperParameterTest'\n",
    "use_cuda = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log path is  /data/home/felix/ABSA-Transformer/logs/HyperParameterTest/20190223/4\n"
     ]
    }
   ],
   "source": [
    "# get general logger just for search\n",
    "experiment_name = utils.create_loggers(experiment_name=experiment_name)\n",
    "logger = logging.getLogger(__name__)\n",
    "logger.info('Run hyper parameter random grid search for experiment with name ' + experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.get_current_git_commit()\n",
    "logger.info('Current commit: ' + utils.get_current_git_commit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_optim_iterations = 25\n",
    "logger.info('num_optim_iterations: ' + str(num_optim_iterations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_grid_search_ranges = {\n",
    "    'batch_size': (1, 80),\n",
    "    'num_encoder_blocks': (1, 10),\n",
    "    'pointwise_layer_size': (32, 4000),\n",
    "    'clip_comments_to': (12, 500),\n",
    "    'learning_rate': (0, 1e-2),\n",
    "    'learning_rate_factor': (1e-3, 4),\n",
    "    'learning_rate_warmup': (1000, 10000),\n",
    "    'optim_adam_beta1': (0.5, 0.99),\n",
    "    'optim_adam_beta2': (0.5, 0.99),\n",
    "    'dropout_rate': (0, 0.8),\n",
    "    'transformer_config': {\n",
    "        'transformer_heads': [1, 2, 3, 4, 5, 6, 10, 12, 15, 20]\n",
    "    }\n",
    "}\n",
    "logger.info(pprint.pformat(random_grid_search_ranges, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------------+\n",
      "|          Hyperparameters           |\n",
      "+-------------------------+----------+\n",
      "|        Parameter        |  Value   |\n",
      "+-------------------------+----------+\n",
      "|        batch_size       |    12    |\n",
      "|        model_size       |   300    |\n",
      "|    learning_rate_type   |   noam   |\n",
      "|      learning_rate      |    0     |\n",
      "|   learning_rate_warmup  |   4800   |\n",
      "|   learning_rate_factor  |    2     |\n",
      "|     optim_adam_beta1    |   0.9    |\n",
      "|     optim_adam_beta2    |   0.98   |\n",
      "|      early_stopping     |    5     |\n",
      "|         use_cuda        |   True   |\n",
      "|       n_enc_blocks      |    3     |\n",
      "|         n_heads         |    6     |\n",
      "|           d_k           |    50    |\n",
      "|           d_v           |    50    |\n",
      "|       dropout_rate      |   0.1    |\n",
      "|   pointwise_layer_size  |   2048   |\n",
      "| log_every_xth_iteration |    -1    |\n",
      "|        num_epochs       |    25    |\n",
      "|      embedding_type     | fasttext |\n",
      "|      embedding_name     |    6B    |\n",
      "|      embedding_dim      |   300    |\n",
      "|     clip_comments_to    |   100    |\n",
      "|         language        |    de    |\n",
      "|      use_stop_words     |   True   |\n",
      "|           seed          |   None   |\n",
      "+-------------------------+----------+\n"
     ]
    }
   ],
   "source": [
    "default_hp = get_default_params(use_cuda)\n",
    "default_hp.num_epochs = 25\n",
    "default_hp.seed = None\n",
    "logger.info(default_hp)\n",
    "print(default_hp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "+-------------------------------------------------+\n",
      "Optim Iteration: 0\n",
      "\n",
      "\n",
      "+-------------------------------------------------+\n",
      "|                 Hyperparameters                 |\n",
      "+-------------------------+-----------------------+\n",
      "|        Parameter        |         Value         |\n",
      "+-------------------------+-----------------------+\n",
      "|        batch_size       |           8           |\n",
      "|        model_size       |          300          |\n",
      "|    learning_rate_type   |          noam         |\n",
      "|      learning_rate      | 0.0011657210564608288 |\n",
      "|   learning_rate_warmup  |   7873.552714474266   |\n",
      "|   learning_rate_factor  |  0.15117973392094078  |\n",
      "|     optim_adam_beta1    |   0.5316564360707892  |\n",
      "|     optim_adam_beta2    |   0.5378184321445757  |\n",
      "|      early_stopping     |           5           |\n",
      "|         use_cuda        |          True         |\n",
      "|       n_enc_blocks      |           2           |\n",
      "|         n_heads         |           3           |\n",
      "|           d_k           |          100          |\n",
      "|           d_v           |          100          |\n",
      "|       dropout_rate      |   0.749227321713445   |\n",
      "|   pointwise_layer_size  |          3831         |\n",
      "| log_every_xth_iteration |           -1          |\n",
      "|        num_epochs       |           25          |\n",
      "|      embedding_type     |        fasttext       |\n",
      "|      embedding_name     |           6B          |\n",
      "|      embedding_dim      |          300          |\n",
      "|     clip_comments_to    |          409          |\n",
      "|         language        |           de          |\n",
      "|      use_stop_words     |          True         |\n",
      "|           seed          |          None         |\n",
      "+-------------------------+-----------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                           \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - DEBUG - 20 initialized\n",
      "pre_training - DEBUG - Initilize parameters with nn.init.xavier_uniform_\n",
      "pre_training - DEBUG - Tagger initialized\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|██████████| 2131/2131 [04:11<00:00, 10.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t2131\t263.58\t\t\u001b[32m33.29\t\t\u001b[32m0.229\u001b[0m\t\t0.795\t\t4.19m - 4.2m / 0.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3:   0%|          | 1/2131 [08:17<43:48:09, 74.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t4262\t130.82\t\t\u001b[32m44.45\t\t\u001b[37m0.217\u001b[0m\t\t0.727\t\t4.10m - 8.3m / 104.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 100%|██████████| 2131/2131 [12:23<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\t6393\t112.33\t\t\u001b[32m21.98\t\t\u001b[32m0.252\u001b[0m\t\t0.866\t\t4.11m - 12.4m / 102.5m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5:   0%|          | 1/2131 [16:26<43:48:09, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\t8524\t102.09\t\t\u001b[32m28.34\t\t\u001b[37m0.239\u001b[0m\t\t0.786\t\t4.04m - 16.4m / 102.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5: 100%|██████████| 2131/2131 [20:26<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\t10655\t89.87\t\t\u001b[32m24.66\t\t\u001b[32m0.253\u001b[0m\t\t0.829\t\t4.00m - 20.4m / 101.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6: 100%|██████████| 2131/2131 [24:26<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\t12786\t81.70\t\t\u001b[32m19.91\t\t\u001b[32m0.261\u001b[0m\t\t0.889\t\t4.00m - 24.4m / 100.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8:   0%|          | 1/2131 [28:25<43:48:09, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\t14917\t72.49\t\t\u001b[32m19.39\t\t\u001b[37m0.261\u001b[0m\t\t0.861\t\t3.97m - 28.4m / 100.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8: 100%|██████████| 2131/2131 [32:25<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\t17048\t69.26\t\t\u001b[32m19.78\t\t\u001b[32m0.266\u001b[0m\t\t0.881\t\t4.00m - 32.4m / 99.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9: 100%|██████████| 2131/2131 [36:24<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\t19179\t65.07\t\t\u001b[32m19.77\t\t\u001b[32m0.268\u001b[0m\t\t0.884\t\t3.98m - 36.4m / 100.5m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10: 100%|██████████| 2131/2131 [40:24<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\t21310\t64.51\t\t\u001b[32m16.38\t\t\u001b[32m0.269\u001b[0m\t\t0.897\t\t3.99m - 40.4m / 100.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12:   0%|          | 1/2131 [44:23<43:48:09, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\t23441\t59.72\t\t\u001b[32m18.51\t\t\u001b[37m0.268\u001b[0m\t\t0.870\t\t3.97m - 44.4m / 100.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12: 100%|██████████| 2131/2131 [48:22<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\t25572\t56.44\t\t\u001b[32m15.10\t\t\u001b[32m0.270\u001b[0m\t\t0.896\t\t3.98m - 48.4m / 100.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13: 100%|██████████| 2131/2131 [52:21<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\t27703\t53.62\t\t\u001b[32m16.23\t\t\u001b[32m0.270\u001b[0m\t\t0.881\t\t3.98m - 52.4m / 100.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15:   0%|          | 0/2131 [56:21<43:49:23, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\t29834\t48.56\t\t\u001b[32m16.82\t\t\u001b[37m0.267\u001b[0m\t\t0.873\t\t4.00m - 56.4m / 100.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15: 100%|██████████| 2131/2131 [1:00:21<00:00, 74.03s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\t31965\t44.18\t\t\u001b[32m12.69\t\t\u001b[32m0.279\u001b[0m\t\t0.905\t\t3.99m - 60.4m / 100.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17:   0%|          | 0/2131 [1:04:21<43:49:23, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\t34096\t40.73\t\t\u001b[32m12.10\t\t\u001b[37m0.277\u001b[0m\t\t0.902\t\t4.01m - 64.4m / 100.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18:   0%|          | 1/2131 [1:08:21<43:48:09, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\t36227\t36.98\t\t\u001b[32m11.91\t\t\u001b[37m0.273\u001b[0m\t\t0.905\t\t3.99m - 68.4m / 100.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18: 100%|██████████| 2131/2131 [1:12:18<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\t38358\t37.31\t\t\u001b[32m11.47\t\t\u001b[32m0.284\u001b[0m\t\t0.908\t\t3.95m - 72.3m / 100.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20:   0%|          | 1/2131 [1:16:16<43:48:09, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\t40489\t34.00\t\t\u001b[32m12.26\t\t\u001b[37m0.278\u001b[0m\t\t0.896\t\t3.97m - 76.3m / 100.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21:   0%|          | 2/2131 [1:20:15<43:46:55, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\t42620\t33.16\t\t\u001b[32m12.92\t\t\u001b[37m0.278\u001b[0m\t\t0.873\t\t3.98m - 80.3m / 100.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22:   0%|          | 2/2131 [1:24:13<43:46:55, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\t44751\t32.25\t\t\u001b[32m13.61\t\t\u001b[37m0.277\u001b[0m\t\t0.880\t\t3.97m - 84.2m / 100.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22: 100%|██████████| 2131/2131 [1:28:12<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\t46882\t32.29\t\t\u001b[32m12.76\t\t\u001b[32m0.286\u001b[0m\t\t0.883\t\t3.98m - 88.2m / 100.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24:   0%|          | 0/2131 [1:32:12<43:49:23, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\t49013\t32.39\t\t\u001b[32m12.93\t\t\u001b[37m0.281\u001b[0m\t\t0.894\t\t4.00m - 92.2m / 100.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24: 100%|██████████| 2131/2131 [1:36:13<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\t51144\t33.15\t\t\u001b[32m12.11\t\t\u001b[32m0.288\u001b[0m\t\t0.913\t\t4.00m - 96.2m / 100.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25: 100%|██████████| 2131/2131 [1:40:12<00:00, 74.03s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25\t53275\t33.05\t\t\u001b[32m12.05\t\t\u001b[32m0.288\u001b[0m\t\t0.910\t\t3.99m - 100.2m / 100.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------------------------------+\n",
      "Best Valid Result: 0.3268054296292514\n",
      "+-------------------------------------------------+\n",
      "\n",
      "\n",
      "###################################################\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------+\n",
      "Optim Iteration: 1\n",
      "\n",
      "\n",
      "+-------------------------------------------------+\n",
      "|                 Hyperparameters                 |\n",
      "+-------------------------+-----------------------+\n",
      "|        Parameter        |         Value         |\n",
      "+-------------------------+-----------------------+\n",
      "|        batch_size       |           78          |\n",
      "|        model_size       |          300          |\n",
      "|    learning_rate_type   |          noam         |\n",
      "|      learning_rate      | 0.0027784034103142275 |\n",
      "|   learning_rate_warmup  |    3198.2461104295    |\n",
      "|   learning_rate_factor  |   1.9678624417114878  |\n",
      "|     optim_adam_beta1    |   0.661937245881485   |\n",
      "|     optim_adam_beta2    |   0.792089380122234   |\n",
      "|      early_stopping     |           5           |\n",
      "|         use_cuda        |          True         |\n",
      "|       n_enc_blocks      |           2           |\n",
      "|         n_heads         |           3           |\n",
      "|           d_k           |          100          |\n",
      "|           d_v           |          100          |\n",
      "|       dropout_rate      |   0.6374810833593357  |\n",
      "|   pointwise_layer_size  |          204          |\n",
      "| log_every_xth_iteration |           -1          |\n",
      "|        num_epochs       |           25          |\n",
      "|      embedding_type     |        fasttext       |\n",
      "|      embedding_name     |           6B          |\n",
      "|      embedding_dim      |          300          |\n",
      "|     clip_comments_to    |          429          |\n",
      "|         language        |           de          |\n",
      "|      use_stop_words     |          True         |\n",
      "|           seed          |          None         |\n",
      "+-------------------------+-----------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                           \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - DEBUG - 20 initialized\n",
      "pre_training - DEBUG - Initilize parameters with nn.init.xavier_uniform_\n",
      "pre_training - DEBUG - Tagger initialized\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|██████████| 219/219 [03:25<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t219\t1170.77\t\t\u001b[32m113.63\t\t\u001b[32m0.168\u001b[0m\t\t0.490\t\t3.42m - 3.4m / 0.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 219/219 [06:51<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t438\t580.41\t\t\u001b[32m86.63\t\t\u001b[32m0.184\u001b[0m\t\t0.546\t\t3.44m - 6.9m / 85.5m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 100%|██████████| 219/219 [10:18<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\t657\t506.29\t\t\u001b[32m56.12\t\t\u001b[32m0.215\u001b[0m\t\t0.658\t\t3.44m - 10.3m / 86.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4: 100%|██████████| 219/219 [13:43<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\t876\t456.94\t\t\u001b[32m49.97\t\t\u001b[32m0.215\u001b[0m\t\t0.670\t\t3.42m - 13.7m / 86.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5: 100%|██████████| 219/219 [17:08<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\t1095\t416.07\t\t\u001b[32m75.05\t\t\u001b[32m0.218\u001b[0m\t\t0.652\t\t3.40m - 17.1m / 85.5m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6: 100%|██████████| 219/219 [20:33<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\t1314\t387.99\t\t\u001b[32m43.00\t\t\u001b[32m0.229\u001b[0m\t\t0.719\t\t3.43m - 20.6m / 85.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8:   0%|          | 0/219 [23:57<02:59,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\t1533\t346.41\t\t\u001b[32m49.74\t\t\u001b[37m0.218\u001b[0m\t\t0.669\t\t3.39m - 24.0m / 85.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8: 100%|██████████| 219/219 [27:21<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\t1752\t336.74\t\t\u001b[32m38.90\t\t\u001b[32m0.237\u001b[0m\t\t0.742\t\t3.40m - 27.4m / 85.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9: 100%|██████████| 219/219 [30:49<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\t1971\t309.31\t\t\u001b[32m40.83\t\t\u001b[32m0.249\u001b[0m\t\t0.765\t\t3.45m - 30.8m / 85.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10: 100%|██████████| 219/219 [34:12<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\t2190\t275.33\t\t\u001b[32m47.44\t\t\u001b[32m0.255\u001b[0m\t\t0.769\t\t3.39m - 34.2m / 86.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12:   0%|          | 0/219 [37:36<02:59,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\t2409\t258.79\t\t\u001b[32m30.05\t\t\u001b[37m0.254\u001b[0m\t\t0.780\t\t3.40m - 37.6m / 85.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12: 100%|██████████| 219/219 [40:59<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\t2628\t225.25\t\t\u001b[32m30.24\t\t\u001b[32m0.259\u001b[0m\t\t0.816\t\t3.38m - 41.0m / 85.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13: 100%|██████████| 219/219 [44:23<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\t2847\t196.45\t\t\u001b[32m19.19\t\t\u001b[32m0.260\u001b[0m\t\t0.834\t\t3.40m - 44.4m / 84.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15:   0%|          | 0/219 [47:48<02:59,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\t3066\t168.70\t\t\u001b[32m22.68\t\t\u001b[37m0.247\u001b[0m\t\t0.793\t\t3.40m - 47.8m / 85.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15: 100%|██████████| 219/219 [51:13<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\t3285\t123.22\t\t\u001b[32m18.88\t\t\u001b[32m0.261\u001b[0m\t\t0.799\t\t3.41m - 51.2m / 85.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17:   0%|          | 0/219 [54:35<02:59,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\t3504\t90.89\t\t\u001b[32m16.59\t\t\u001b[37m0.253\u001b[0m\t\t0.783\t\t3.37m - 54.6m / 85.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18:   0%|          | 0/219 [58:01<02:59,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\t3723\t60.43\t\t\u001b[32m13.75\t\t\u001b[37m0.254\u001b[0m\t\t0.780\t\t3.43m - 58.0m / 84.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18: 100%|██████████| 219/219 [1:01:25<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\t3942\t40.95\t\t\u001b[32m11.49\t\t\u001b[32m0.265\u001b[0m\t\t0.807\t\t3.40m - 61.4m / 85.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19: 100%|██████████| 219/219 [1:04:46<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\t4161\t31.68\t\t\u001b[32m12.57\t\t\u001b[32m0.269\u001b[0m\t\t0.801\t\t3.35m - 64.8m / 85.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21:   0%|          | 0/219 [1:08:08<02:59,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\t4380\t25.07\t\t\u001b[32m10.97\t\t\u001b[37m0.266\u001b[0m\t\t0.841\t\t3.36m - 68.1m / 84.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21: 100%|██████████| 219/219 [1:11:34<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\t4599\t27.35\t\t\u001b[32m9.67\t\t\u001b[32m0.284\u001b[0m\t\t0.877\t\t3.44m - 71.6m / 84.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22: 100%|██████████| 219/219 [1:14:54<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\t4818\t25.29\t\t\u001b[32m8.91\t\t\u001b[32m0.290\u001b[0m\t\t0.885\t\t3.32m - 74.9m / 85.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24:   0%|          | 0/219 [1:18:17<02:59,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\t5037\t21.90\t\t\u001b[32m9.65\t\t\u001b[37m0.284\u001b[0m\t\t0.870\t\t3.38m - 78.3m / 84.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25:   0%|          | 0/219 [1:21:35<02:59,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\t5256\t20.92\t\t\u001b[32m9.62\t\t\u001b[37m0.286\u001b[0m\t\t0.877\t\t3.31m - 81.6m / 85.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25: 100%|██████████| 219/219 [1:25:00<00:00,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25\t5475\t19.18\t\t\u001b[32m8.30\t\t\u001b[32m0.298\u001b[0m\t\t0.898\t\t3.41m - 85.0m / 84.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------------------------------+\n",
      "Best Valid Result: 0.3330782401219957\n",
      "+-------------------------------------------------+\n",
      "\n",
      "\n",
      "###################################################\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------+\n",
      "Optim Iteration: 2\n",
      "\n",
      "\n",
      "+------------------------------------------------+\n",
      "|                Hyperparameters                 |\n",
      "+-------------------------+----------------------+\n",
      "|        Parameter        |        Value         |\n",
      "+-------------------------+----------------------+\n",
      "|        batch_size       |          22          |\n",
      "|        model_size       |         300          |\n",
      "|    learning_rate_type   |         noam         |\n",
      "|      learning_rate      | 0.009900575093264441 |\n",
      "|   learning_rate_warmup  |  6359.775466708938   |\n",
      "|   learning_rate_factor  |   1.79306569436184   |\n",
      "|     optim_adam_beta1    |   0.9359787134166    |\n",
      "|     optim_adam_beta2    |  0.655227539448646   |\n",
      "|      early_stopping     |          5           |\n",
      "|         use_cuda        |         True         |\n",
      "|       n_enc_blocks      |          5           |\n",
      "|         n_heads         |          3           |\n",
      "|           d_k           |         100          |\n",
      "|           d_v           |         100          |\n",
      "|       dropout_rate      |  0.3910367119713505  |\n",
      "|   pointwise_layer_size  |         1417         |\n",
      "| log_every_xth_iteration |          -1          |\n",
      "|        num_epochs       |          25          |\n",
      "|      embedding_type     |       fasttext       |\n",
      "|      embedding_name     |          6B          |\n",
      "|      embedding_dim      |         300          |\n",
      "|     clip_comments_to    |         427          |\n",
      "|         language        |          de          |\n",
      "|      use_stop_words     |         True         |\n",
      "|           seed          |         None         |\n",
      "+-------------------------+----------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                           \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - DEBUG - 20 initialized\n",
      "pre_training - DEBUG - Initilize parameters with nn.init.xavier_uniform_\n",
      "pre_training - DEBUG - Tagger initialized\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|██████████| 775/775 [06:05<00:00,  2.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t775\t69218178.19\t\t\u001b[37m15332337.13\t\t\u001b[32m0.165\u001b[0m\t\t0.535\t\t6.09m - 6.1m / 0.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 775/775 [11:57<00:00,  2.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t1550\tnan\t\t\u001b[32mnan\t\t\u001b[32m0.240\u001b[0m\t\t0.944\t\t5.85m - 12.0m / 152.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4:   0%|          | 0/775 [17:44<04:43,  2.73it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\t2325\tnan\t\t\u001b[32mnan\t\t\u001b[37m0.240\u001b[0m\t\t0.944\t\t5.79m - 17.7m / 146.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5:   0%|          | 0/775 [23:34<04:43,  2.73it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\t3100\tnan\t\t\u001b[32mnan\t\t\u001b[37m0.240\u001b[0m\t\t0.944\t\t5.82m - 23.6m / 145.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6:   0%|          | 0/775 [29:25<04:43,  2.73it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\t3875\tnan\t\t\u001b[32mnan\t\t\u001b[37m0.240\u001b[0m\t\t0.944\t\t5.86m - 29.4m / 145.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7:   0%|          | 0/775 [35:11<04:43,  2.73it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\t4650\tnan\t\t\u001b[32mnan\t\t\u001b[37m0.240\u001b[0m\t\t0.944\t\t5.75m - 35.2m / 146.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7: 100%|██████████| 775/775 [40:54<00:00,  2.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\t5425\tnan\t\t\u001b[32mnan\t\t\u001b[37m0.240\u001b[0m\t\t0.944\t\t5.73m - 40.9m / 144.5m\n",
      "\n",
      "Valid Result: 0.3330782401219957\n",
      "\n",
      "\n",
      "\n",
      "###################################################\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------+\n",
      "Optim Iteration: 3\n",
      "\n",
      "\n",
      "+------------------------------------------------+\n",
      "|                Hyperparameters                 |\n",
      "+-------------------------+----------------------+\n",
      "|        Parameter        |        Value         |\n",
      "+-------------------------+----------------------+\n",
      "|        batch_size       |          22          |\n",
      "|        model_size       |         300          |\n",
      "|    learning_rate_type   |         noam         |\n",
      "|      learning_rate      | 0.004667062686953772 |\n",
      "|   learning_rate_warmup  |  7255.007208703063   |\n",
      "|   learning_rate_factor  |  2.464618265231109   |\n",
      "|     optim_adam_beta1    |  0.6767548445192131  |\n",
      "|     optim_adam_beta2    |  0.6568842895120443  |\n",
      "|      early_stopping     |          5           |\n",
      "|         use_cuda        |         True         |\n",
      "|       n_enc_blocks      |          4           |\n",
      "|         n_heads         |          6           |\n",
      "|           d_k           |          50          |\n",
      "|           d_v           |          50          |\n",
      "|       dropout_rate      | 0.06856115451591105  |\n",
      "|   pointwise_layer_size  |         911          |\n",
      "| log_every_xth_iteration |          -1          |\n",
      "|        num_epochs       |          25          |\n",
      "|      embedding_type     |       fasttext       |\n",
      "|      embedding_name     |          6B          |\n",
      "|      embedding_dim      |         300          |\n",
      "|     clip_comments_to    |          35          |\n",
      "|         language        |          de          |\n",
      "|      use_stop_words     |         True         |\n",
      "|           seed          |         None         |\n",
      "+-------------------------+----------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                           \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - DEBUG - 20 initialized\n",
      "pre_training - DEBUG - Initilize parameters with nn.init.xavier_uniform_\n",
      "pre_training - DEBUG - Tagger initialized\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|██████████| 775/775 [01:42<00:00, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t775\t82.77\t\t\u001b[32m26.13\t\t\u001b[32m0.214\u001b[0m\t\t0.794\t\t1.71m - 1.7m / 0.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 775/775 [03:25<00:00, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t1550\t50.83\t\t\u001b[32m13.18\t\t\u001b[32m0.245\u001b[0m\t\t0.871\t\t1.71m - 3.4m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4:   0%|          | 1/775 [05:09<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\t2325\t47.18\t\t\u001b[32m13.04\t\t\u001b[37m0.243\u001b[0m\t\t0.809\t\t1.71m - 5.2m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4: 100%|██████████| 775/775 [06:51<00:00, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\t3100\t43.89\t\t\u001b[32m9.28\t\t\u001b[32m0.246\u001b[0m\t\t0.848\t\t1.71m - 6.9m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6:   0%|          | 1/775 [08:34<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\t3875\t38.12\t\t\u001b[32m13.97\t\t\u001b[37m0.238\u001b[0m\t\t0.807\t\t1.71m - 8.6m / 42.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7:   0%|          | 1/775 [10:17<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\t4650\t32.81\t\t\u001b[32m12.44\t\t\u001b[37m0.235\u001b[0m\t\t0.800\t\t1.71m - 10.3m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8:   0%|          | 1/775 [11:59<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\t5425\t27.96\t\t\u001b[32m12.94\t\t\u001b[37m0.240\u001b[0m\t\t0.775\t\t1.71m - 12.0m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8: 100%|██████████| 775/775 [13:41<00:00, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\t6200\t21.26\t\t\u001b[32m7.71\t\t\u001b[32m0.262\u001b[0m\t\t0.863\t\t1.71m - 13.7m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9: 100%|██████████| 775/775 [15:24<00:00, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\t6975\t14.37\t\t\u001b[32m5.05\t\t\u001b[32m0.274\u001b[0m\t\t0.939\t\t1.71m - 15.4m / 42.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10: 100%|██████████| 775/775 [17:07<00:00, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\t7750\t14.66\t\t\u001b[32m6.27\t\t\u001b[32m0.274\u001b[0m\t\t0.913\t\t1.71m - 17.1m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12:   0%|          | 1/775 [18:50<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\t8525\t14.55\t\t\u001b[32m7.51\t\t\u001b[37m0.265\u001b[0m\t\t0.863\t\t1.71m - 18.8m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12: 100%|██████████| 775/775 [20:32<00:00, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\t9300\t14.32\t\t\u001b[32m6.21\t\t\u001b[32m0.277\u001b[0m\t\t0.899\t\t1.70m - 20.5m / 42.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14:   0%|          | 1/775 [22:14<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\t10075\t13.71\t\t\u001b[32m9.74\t\t\u001b[37m0.246\u001b[0m\t\t0.799\t\t1.69m - 22.2m / 42.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15:   0%|          | 1/775 [23:57<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\t10850\t13.90\t\t\u001b[32m6.20\t\t\u001b[37m0.272\u001b[0m\t\t0.896\t\t1.71m - 24.0m / 42.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15: 100%|██████████| 775/775 [25:38<00:00, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\t11625\t13.34\t\t\u001b[32m6.29\t\t\u001b[32m0.280\u001b[0m\t\t0.901\t\t1.69m - 25.6m / 42.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17:   0%|          | 1/775 [27:20<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\t12400\t13.18\t\t\u001b[32m8.12\t\t\u001b[37m0.260\u001b[0m\t\t0.849\t\t1.69m - 27.3m / 42.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18:   0%|          | 1/775 [29:03<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\t13175\t13.00\t\t\u001b[32m7.65\t\t\u001b[37m0.271\u001b[0m\t\t0.873\t\t1.71m - 29.1m / 42.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19:   0%|          | 1/775 [30:45<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\t13950\t12.69\t\t\u001b[32m7.74\t\t\u001b[37m0.266\u001b[0m\t\t0.850\t\t1.70m - 30.8m / 42.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20:   0%|          | 1/775 [32:27<00:58, 13.32it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\t14725\t12.89\t\t\u001b[32m7.90\t\t\u001b[37m0.269\u001b[0m\t\t0.854\t\t1.71m - 32.5m / 42.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20: 100%|██████████| 775/775 [34:09<00:00, 13.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\t15500\t12.70\t\t\u001b[32m9.34\t\t\u001b[37m0.256\u001b[0m\t\t0.834\t\t1.70m - 34.2m / 42.7m\n",
      "\n",
      "Valid Result: 0.3330782401219957\n",
      "\n",
      "\n",
      "\n",
      "###################################################\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------+\n",
      "Optim Iteration: 4\n",
      "\n",
      "\n",
      "+------------------------------------------------+\n",
      "|                Hyperparameters                 |\n",
      "+-------------------------+----------------------+\n",
      "|        Parameter        |        Value         |\n",
      "+-------------------------+----------------------+\n",
      "|        batch_size       |          19          |\n",
      "|        model_size       |         300          |\n",
      "|    learning_rate_type   |         noam         |\n",
      "|      learning_rate      | 0.006079812121101265 |\n",
      "|   learning_rate_warmup  |  9020.521735852282   |\n",
      "|   learning_rate_factor  | 0.05745533778662076  |\n",
      "|     optim_adam_beta1    |  0.783681944746232   |\n",
      "|     optim_adam_beta2    |  0.7054632733674455  |\n",
      "|      early_stopping     |          5           |\n",
      "|         use_cuda        |         True         |\n",
      "|       n_enc_blocks      |          5           |\n",
      "|         n_heads         |          2           |\n",
      "|           d_k           |         150          |\n",
      "|           d_v           |         150          |\n",
      "|       dropout_rate      |  0.590072940833766   |\n",
      "|   pointwise_layer_size  |         820          |\n",
      "| log_every_xth_iteration |          -1          |\n",
      "|        num_epochs       |          25          |\n",
      "|      embedding_type     |       fasttext       |\n",
      "|      embedding_name     |          6B          |\n",
      "|      embedding_dim      |         300          |\n",
      "|     clip_comments_to    |         401          |\n",
      "|         language        |          de          |\n",
      "|      use_stop_words     |         True         |\n",
      "|           seed          |         None         |\n",
      "+-------------------------+----------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                           \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - DEBUG - 20 initialized\n",
      "pre_training - DEBUG - Initilize parameters with nn.init.xavier_uniform_\n",
      "pre_training - DEBUG - Tagger initialized\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|██████████| 897/897 [05:17<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t897\t1187.78\t\t\u001b[32m85.03\t\t\u001b[32m0.185\u001b[0m\t\t0.547\t\t5.29m - 5.3m / 0.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 897/897 [10:34<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t1794\t221.05\t\t\u001b[32m63.55\t\t\u001b[32m0.195\u001b[0m\t\t0.580\t\t5.27m - 10.6m / 132.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 100%|██████████| 897/897 [15:55<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\t2691\t198.25\t\t\u001b[32m52.01\t\t\u001b[32m0.203\u001b[0m\t\t0.620\t\t5.35m - 15.9m / 131.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5:   0%|          | 0/897 [21:12<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\t3588\t184.28\t\t\u001b[32m49.84\t\t\u001b[37m0.201\u001b[0m\t\t0.618\t\t5.28m - 21.2m / 133.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6:   0%|          | 1/897 [26:30<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\t4485\t179.65\t\t\u001b[32m47.85\t\t\u001b[37m0.203\u001b[0m\t\t0.622\t\t5.29m - 26.5m / 132.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6: 100%|██████████| 897/897 [31:49<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\t5382\t174.39\t\t\u001b[32m43.34\t\t\u001b[32m0.207\u001b[0m\t\t0.637\t\t5.31m - 31.8m / 132.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7: 100%|██████████| 897/897 [37:09<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\t6279\t174.35\t\t\u001b[32m41.34\t\t\u001b[32m0.208\u001b[0m\t\t0.659\t\t5.34m - 37.2m / 132.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9:   0%|          | 0/897 [42:26<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\t7176\t165.61\t\t\u001b[32m47.72\t\t\u001b[37m0.203\u001b[0m\t\t0.627\t\t5.27m - 42.4m / 133.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9: 100%|██████████| 897/897 [47:44<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\t8073\t160.94\t\t\u001b[32m41.96\t\t\u001b[32m0.210\u001b[0m\t\t0.655\t\t5.31m - 47.7m / 132.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10: 100%|██████████| 897/897 [53:04<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\t8970\t155.20\t\t\u001b[32m34.41\t\t\u001b[32m0.220\u001b[0m\t\t0.704\t\t5.32m - 53.1m / 132.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11: 100%|██████████| 897/897 [58:19<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\t9867\t144.02\t\t\u001b[32m32.39\t\t\u001b[32m0.224\u001b[0m\t\t0.713\t\t5.25m - 58.3m / 132.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12: 100%|██████████| 897/897 [1:03:38<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\t10764\t130.93\t\t\u001b[32m29.92\t\t\u001b[32m0.231\u001b[0m\t\t0.743\t\t5.30m - 63.6m / 131.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13: 100%|██████████| 897/897 [1:08:55<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\t11661\t122.42\t\t\u001b[32m32.48\t\t\u001b[32m0.232\u001b[0m\t\t0.743\t\t5.29m - 68.9m / 132.5m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14: 100%|██████████| 897/897 [1:14:13<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\t12558\t116.28\t\t\u001b[32m26.92\t\t\u001b[32m0.241\u001b[0m\t\t0.769\t\t5.29m - 74.2m / 132.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16:   0%|          | 0/897 [1:19:32<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\t13455\t110.84\t\t\u001b[32m30.74\t\t\u001b[37m0.235\u001b[0m\t\t0.736\t\t5.32m - 79.5m / 132.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16: 100%|██████████| 897/897 [1:24:48<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\t14352\t101.11\t\t\u001b[32m26.70\t\t\u001b[32m0.252\u001b[0m\t\t0.803\t\t5.26m - 84.8m / 132.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18:   0%|          | 1/897 [1:30:10<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\t15249\t98.69\t\t\u001b[32m24.84\t\t\u001b[37m0.247\u001b[0m\t\t0.782\t\t5.37m - 90.2m / 132.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19:   0%|          | 0/897 [1:35:28<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\t16146\t96.74\t\t\u001b[32m25.84\t\t\u001b[37m0.248\u001b[0m\t\t0.771\t\t5.30m - 95.5m / 133.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20:   0%|          | 0/897 [1:40:49<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\t17043\t89.19\t\t\u001b[32m36.06\t\t\u001b[37m0.248\u001b[0m\t\t0.758\t\t5.35m - 100.8m / 132.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20: 100%|██████████| 897/897 [1:46:09<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\t17940\t89.05\t\t\u001b[32m22.81\t\t\u001b[32m0.259\u001b[0m\t\t0.828\t\t5.33m - 106.2m / 132.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22:   0%|          | 0/897 [1:51:28<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\t18837\t82.38\t\t\u001b[32m25.19\t\t\u001b[37m0.253\u001b[0m\t\t0.791\t\t5.31m - 111.5m / 132.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23:   0%|          | 0/897 [1:56:47<04:38,  3.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\t19734\t79.84\t\t\u001b[32m23.32\t\t\u001b[37m0.258\u001b[0m\t\t0.811\t\t5.32m - 116.8m / 132.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23: 100%|██████████| 897/897 [2:02:04<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\t20631\t75.29\t\t\u001b[32m22.77\t\t\u001b[32m0.261\u001b[0m\t\t0.817\t\t5.28m - 122.1m / 132.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24: 100%|██████████| 897/897 [2:07:26<00:00,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\t21528\t72.90\t\t\u001b[32m21.35\t\t\u001b[32m0.264\u001b[0m\t\t0.826\t\t5.35m - 127.4m / 132.7m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25: 100%|██████████| 897/897 [2:12:43<00:00,  3.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25\t22425\t72.20\t\t\u001b[32m22.71\t\t\u001b[37m0.261\u001b[0m\t\t0.812\t\t5.29m - 132.7m / 132.8m\n",
      "\n",
      "Valid Result: 0.3330782401219957\n",
      "\n",
      "\n",
      "\n",
      "###################################################\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------+\n",
      "Optim Iteration: 5\n",
      "\n",
      "\n",
      "+-----------------------------------------------+\n",
      "|                Hyperparameters                |\n",
      "+-------------------------+---------------------+\n",
      "|        Parameter        |        Value        |\n",
      "+-------------------------+---------------------+\n",
      "|        batch_size       |          36         |\n",
      "|        model_size       |         300         |\n",
      "|    learning_rate_type   |         noam        |\n",
      "|      learning_rate      | 0.00581362665130959 |\n",
      "|   learning_rate_warmup  |  1651.3951648615953 |\n",
      "|   learning_rate_factor  |  2.9588471664357647 |\n",
      "|     optim_adam_beta1    |  0.5512446936700348 |\n",
      "|     optim_adam_beta2    |  0.9681621628570081 |\n",
      "|      early_stopping     |          5          |\n",
      "|         use_cuda        |         True        |\n",
      "|       n_enc_blocks      |          10         |\n",
      "|         n_heads         |          12         |\n",
      "|           d_k           |          25         |\n",
      "|           d_v           |          25         |\n",
      "|       dropout_rate      |  0.6138635987489934 |\n",
      "|   pointwise_layer_size  |         3625        |\n",
      "| log_every_xth_iteration |          -1         |\n",
      "|        num_epochs       |          25         |\n",
      "|      embedding_type     |       fasttext      |\n",
      "|      embedding_name     |          6B         |\n",
      "|      embedding_dim      |         300         |\n",
      "|     clip_comments_to    |         229         |\n",
      "|         language        |          de         |\n",
      "|      use_stop_words     |         True        |\n",
      "|           seed          |         None        |\n",
      "+-------------------------+---------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                           \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - DEBUG - 20 initialized\n",
      "pre_training - DEBUG - Initilize parameters with nn.init.xavier_uniform_\n",
      "pre_training - DEBUG - Tagger initialized\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|██████████| 474/474 [15:33<00:00,  1.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t474\t616.78\t\t\u001b[32m48.46\t\t\u001b[32m0.210\u001b[0m\t\t0.713\t\t15.56m - 15.6m / 0.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 474/474 [31:05<00:00,  1.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t948\t400.15\t\t\u001b[32m30.74\t\t\u001b[32m0.217\u001b[0m\t\t0.732\t\t15.52m - 31.1m / 389.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3:  64%|██████▍   | 303/474 [40:17<04:13,  1.49s/it]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-198e2139da62>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muse_cuda\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_cuda\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperform_evaluation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'model'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/home/felix/ABSA-Transformer/models/transformer/train.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, use_cuda, perform_evaluation)\u001b[0m\n\u001b[1;32m    614\u001b[0m                                         \u001b[0msource_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_padding_masks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpadding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    615\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 616\u001b[0;31m                                         \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    617\u001b[0m                                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log_scalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_loss_history\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'loss'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    618\u001b[0m                                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log_scalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'lr'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'general'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/home/felix/ABSA-Transformer/models/transformer/train.py\u001b[0m in \u001b[0;36m_step\u001b[0;34m(self, input, target, source_mask)\u001b[0m\n\u001b[1;32m    266\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m                 \u001b[0;31m# preform training step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 268\u001b[0;31m                 \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    269\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    270\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/anaconda/envs/py35/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \"\"\"\n\u001b[0;32m--> 102\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/anaconda/envs/py35/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     88\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     89\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEYCAYAAAApuP8NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsnXd8FNXXh59DQmjSWwq9914UEASkCYJIBwuKvfcuFuxiF1/F8sNO70XAgh1pggiI0kmhSlEQkmzO+8dMwqYvkOxukvP4mQ87c+/c+e6YnJxzy7miqhiGYRRkCgVagGEYRqAxQ2gYRoHHDKFhGAUeM4SGYRR4zBAahlHgMUNoGEaBxwyhcdqISDERmSciR0Rk2lm0M0pEluSktkAgIotE5MpA6zDOHDOE+RgRGSkiq0TkXxGJc39hO+VA04OBykB5VR1ypo2o6qeq2jMH9KRCRC4QERWRmWmuN3evL/OxncdF5JPs6qlqH1X98AzlGkGAGcJ8iojcBbwKPINjtKoBbwEDcqD56sCfqpqYA23lFvuBDiJS3uvalcCfOfUAcbDfofyAqtqRzw6gNPAvMCSLOkVwDGWse7wKFHHLLgCigbuBfUAccJVb9gQQDyS4zxgDPA584tV2DUCBUPd8NLAN+AfYDozyuv6D130dgJXAEfffDl5ly4BxwI9uO0uACpl8t2T9bwM3u9dC3GtjgWVedV8DdgNHgdXA+e713mm+5zovHU+7Ov4D6rjXrnHL/w+Y7tX+88BXgAT658KOzA/7a5Y/OQ8oCszKos7DwLlAC6A50A54xKs8HMegRuEYuwkiUlZVH8PxMqeo6jmq+n5WQkSkBPA60EdVS+IYu7UZ1CsHLHDrlgdeBhak8ehGAlcBlYAw4J6sng18BFzhfu4FbMAx+t6sxHkH5YDPgGkiUlRVv0jzPZt73XM5cB1QEtiZpr27gWYiMlpEzsd5d1eqaxWN4MQMYf6kPHBAsw5dRwFPquo+Vd2P4+ld7lWe4JYnqOpCHK+o/hnqSQKaiEgxVY1T1Q0Z1OkL/KWqH6tqoqp+DvwBXOxV53+q+qeq/gdMxTFgmaKqPwHlRKQ+jkH8KIM6n6jqQfeZL+F4ytl9z0mqusG9JyFNe8eBy3AM+SfAraoanU17RoAxQ5g/OQhUEJHQLOpEktqb2eleS2kjjSE9DpxzukJU9RgwDLgBiBORBSLSwAc9yZqivM73nIGej4FbgK5k4CGLyN0isskdAT+M4wVXyKbN3VkVquoKnK4AwTHYRpBjhjB/8jNwArgkizqxOIMeyVQjfdjoK8eA4l7n4d6FqrpYVXsAEThe3rs+6EnWFHOGmpL5GLgJWOh6aym4oev9wFCgrKqWwemflGTpmbSZZZgrIjfjeJaxwH1nLt3wF2YI8yGqegRnUGCCiFwiIsVFpLCI9BGRF9xqnwOPiEhFEang1s92qkgmrAU6i0g1ESkNPJhcICKVRaS/21d4EifE9mTQxkKgnjvlJ1REhgGNgPlnqAkAVd0OdMHpE01LSSARZ4Q5VETGAqW8yvcCNU5nZFhE6gFP4YTHlwP3iUiWIbwReMwQ5lNU9WXgLpwBkP044dwtwGy3ylPAKuA3YD2wxr12Js9aCkxx21pNauNVCGcAIRb4G8co3ZRBGweBfm7dgzieVD9VPXAmmtK0/YOqZuTtLgYW4Uyp2YnjRXuHvcmTxQ+KyJrsnuN2RXwCPK+q61T1L+Ah4GMRKXI238HIXcQGswzDKOiYR2gYRoHHDKFhGAUeM4SGYRR4zBAahlHgyWrCrQFIaDGVsJKBlnHGtGhYLdASzoqTCUmBlnBWFC2cd32NnTt3cODAAcm+pm+ElKqumvhftvX0v/2LVbV3Tj3XF8wQZoOElaRI/aGBlnHG/PDzG4GWcFZs23cs0BLOijrhp70YJ2jo2L5Njranif/59Lt0Yu2E7Fb25DhmCA3D8A8iUCgk0CoyxAyhYRj+I0jTN5ohNAzDf0iOdTnmKGYIDcPwExYaG4ZR0BEsNDYMo6AjFhobhmFYaGwYRgFHLDQ2DKOAI1hobBhGQUegUHCanOBUZRhG/qRQcHqEwRmwG4aR/0iePpPdkV0zIr1FZLOIbBGRBzIof0VE1rrHn+7uhFliHqFhGH7i7CdUi0gIMAHoAUQDK0VkrqpuTK6jqnd61b8VaJldu+YRGobhP0SyP7KmHbBFVbepajwwGRiQRf0RODs2Zol5hIZh+A/fps9UEJFVXucTVXWi+zmK1DsNRgPtM3yUSHWgJvB1dg80jzCX6NGhIetmPcrvcx7jnqt6pCt/4e5LWT75AZZPfoDfZo8l7rsXUsqqhpdl3ls38+uMR1gz42GqRZTzp3QAliz+ghZNGtC0YV3Gv/hcuvKTJ09yxajhNG1Yly6dzmXnjh2pynfv2kWlciV59eXxflKcmh++WcrFXVpyUafmvDfhpXTlq5b/wNA+nWhRowxLFsxOub7ip+8Y3KtDytG6TgW++mKeP6UDzvtv1rg+jRvU4cUXMn7/l40cRuMGdTi/Q/uU979yxQrat25B+9YtaNeqOXNmz/Kz8ixITsOV3QEHVLWN1zHRu5UMWs5sK87hwHRVzWgf7VSYR5gLFCokvPrAUPre+CYxew/zw6f3Mv/b9fyxbU9Knftempny+cbhXWhev0rK+XvjruD59xbz9S9/UKJYGEl+3nLV4/Fw1+23MG/hEqKqVOH8Du3o268/DRs2Sqnz4f/ep0yZMqzf9BfTpk7m0Ycf4KNPJ6eU33/vXfTs1cevupPxeDw8/cjdTPxsDuERUQzv14WuPfpSu16DlDoRUVUZ9/LbfPjO66nubdehM9MX/wTAkUN/c9H5LejQpbvf9d9x280sWLSUqCpV6HRuW/r160/DRqfe/6QP3qdsmbJs+GMLU6dM5uGH7ueTz6bQuEkTfvxlFaGhocTFxdG+dXP69ruY0NAg+VU/+3mE0UBVr/MqOHtmZ8Rw4GZfGjWPMBdo26QGW3cfYEfMQRISPUxbvIZ+FzTLtP7Q3q2Z+sVqABrUCic0pBBf//IHAMf+i+e/Ewl+0Z3MqpUrqFW7DjVr1SIsLIzBQ4cxf96cVHXmz5vLqMuvBGDgpYNZ9s1XJO+RPW/ObGrUrJnqF9efrF+7imo1alG1ek0Kh4XRp/8gvlkyP1WdqKrVqd+wCZLFL+aShbPp1LUHxYoVz23JqVi5YgW1vd7/kGHDM3j/c1Le/6WDBrPsa+f9Fy9ePMXonTxxIsvv538kJ0aNVwJ1RaSmiIThGLu56Z4kUh8oC/zsizIzhLlAZKXSRO89lHIes/cQURVLZ1i3WkRZqkeWZ9nKzQDUrVaJw//8x+Tx1/Dz5/fzzB2XUMjPc69iY2OoUvWUhxoVVYW4mJj0dao4f5hDQ0MpVao0Bw8e5NixY7z80gs89MhjftXszb49cYRHRqWcV46IYu+euNNu54u5M7howOCclOYT3u8WnPcfk9H7r+r1/ks77x9gxS+/0Kp5Y9q0bMrrE94OIm8QX0PjTFHVROAWYDGwCZiqqhtE5EkR6e9VdQQwWdW3cKpAGEIROU9E3vXb8zLoxsjs/8aQXq2Z/dVakpKcGqGhhejYsjYPvDKLTpe9SM0qFbi8/7m5qDY9Gf3spPMsMqnz1JOPccttd3DOOYHbq8Mn/dmwf+8e/vpjAx26XJhTsnzGF/1Z1WnXvj1r1m3gh59X8uLzz3LixIncEXra5IhHiKouVNV6qlpbVZ92r41V1bledR5X1XRzDDOjQBhCoDfwhb8eFrPvMFUql005j6pcltj9RzKsO7hXa6Z+cWqALGbvYdZtjmZHzEE8niTmfrOOFg2qZnhvbhEVVYXo3dGnNMVEEx4ZmapOZFQVoqOdwbvExESOHj1CuXLlWLVyBY88dD8N69VkwhuvMf6FZ3n7rTf9qr9yRCR7Yk95UHvjYqhUOfy02lg8fybdel9M4cKFc1petkR5vVtw3n9kmvfv/D/yev9HnPfvTYOGDSlRogQbfv8990X7ytlPn8kV8o0hFJEaIrJJRN4VkQ0iskREirnF3YEv3Trfi8ga9+iQG1pWbdhJnWoVqR5ZnsKhIQzp1YoFy35LV69u9UqULVWc5eu2p7q3TKliVCjreFQXtK2fapDFH7Ru05atW/5ix/btxMfHM33qFPr265+qTt9+F/Ppxx8CMGvmdLpc0A0RYenX37Hpz+1s+nM7N996O/fc9yA33HSLX/U3ad6anTu2Er1rBwnx8SyaO4MLevQ9rTYWzZnGRQOG5JLCrGnTti1bvN7/tCmTM3j//VPe/8wZ0+nS1Xn/O7ZvJzExEYCdO3fy55+bqV6jhr+/QuacZWicWwRJ50GOURcYoarXishUYJCIfAEkqOoREUkAeqjqCRGpizPRMt2ehSJyHXAdAIVPP8TzeJK48/mpzHvrZkIKCR/OWc6mbXt49Ma+rNm4iwXfrgdgaO82TFu8OtW9SUnKgy/PZuHbtyIi/LppFx/M/PG0NZwNoaGhvPTqGwzo1xuPx8MVo6+iUaPGjHtiLK1ataHvxf258qoxXHPVFTRtWJey5crx4cfZzln1G6GhoTw0bjw3XHYJHk8SA4ddTp36DXlz/FM0btaSrj378vva1dx+7Uj+OXKYb79cxFsvP83sr1YCELN7J3tiY2hzbqeA6X/ltTe5uG8vPB4PV46+mkaNG/Pk42Np1boN/S7uz+irx3D16Mtp3KAOZcuW42N3xP6nH39g/IvPUTi0MIUKFeK1N96iQgW/746ZMRK8abjEx77EoEdEagBLVbWue34/UBjYBlRX1WdFpDTwJtAC8AD1VDXLIcFCxStpXt7X+OAvtq9xIMnr+xqvXr0qx2LVQmVraJGuj2Zb78Ssa1aras5uqpwN+c0jPOn12QMUA/oAL7vX7gT2As1xugWCpRfZMPI9AhQqFJweYXCqyjkEaAasdc9LA3GqmgRcDgRn3nDDyI+Ij0cAyO+GsDXwq9dcoreAK0VkOVAPyNtxl2HkKQSR7I9AkG9CY1XdATTxOh8vIkWBLV7X/sLxEJN50G8CDcMI2tA43xjCjFDVpwKtwTCMUwTXkr9T5GtDaBhGEBHAPsDsMENoGIZfEMRCY8MwDAuNDcMo8JghNAyjYCMgQbqdpxlCwzD8ghC4eYLZYYbQMAy/YYbQMIyCjYXGhmEYwesRBuekHsMw8iU5sdZYRHqLyGYR2SIiGabjF5GhIrLRTdL8WXZtmkdoGIZfEOSsQ2MRCQEmAD1wtvZcKSJzVXWjV526OHkEOqrqIRGplF275hEahuEfJEc8wnbAFlXdpqrxwGRgQJo61wITVPUQgKruy65RM4SGYfgNHw1hBRFZ5XVc59VEFLDb6zzaveZNPaCeiPwoIstFpHd2uiw0NgzDb/gYGh/IIlV/Rg2k3W8kFGf/oguAKsD3ItJEVQ9n9kDzCA3D8Bs5EBpHA97721YBYjOoM0dVE1R1O7AZxzBmihlCwzD8gi9G0AdDuBKoKyI1RSQMGA7MTVNnNtDVfWYFnFB5W1aNWmhsGIbfONs0XKqaKCK3AItx9hz6QFU3iMiTwCpVneuW9RSRjTibuN2rqgezatcMYTZUrR7Og2/dE2gZZ8ySP/y7OXxOU69CqUBLOCuWbtobaAlnzNETCTnfaA7Mp1bVhcDCNNfGen1W4C738AkzhIZh+AexPUsMwyjgCBCkK+zMEBqG4S8sDZdhGAaFLPuMYRgFGrHQ2DCMAo5gHqFhGIYZQsMwCjgWGhuGUdBxps8EpyU0Q2gYhp8QC40NwzDMIzQMo2BjfYSGYRR0bPqMYRgGFhobhmFYaGwYRsFGxELjAseG5d8y9dUnUE8SHS8eRq8rbkxV/t2sT/l2xscUCilEkWIlGHX/M0TUrMumFd8z6/9ewJOQQEjhwlx684M0aNPB7/rX/PgN7z//KElJSVw4cASDxtyaqnzOR+/w5azPCAkJpVTZ8tzyxMtUiqzC+hU/8sH4x1LqxWzfyt3Pv0X7bn38qv/7b5by3Nj78CR5GDTiSq695e5U5auW/8Bzj93Pn5t+58W3JtGr38CUstiY3Tx2z83siY0GEd7+eCZRVav7Vf/qH77mvecfxZPkoeeloxic5v3P/uhtls78lEIhoZQuW57bnnyFSpHOVh7746J54/G7ObAnFhEYO+FTKkdV86v+jLHsMwWKJI+HyePHcttrH1O2UjjPjRlAs/MvJKLmqf1j2vbsT+eBowBY9/1Spr/+FLe+8iHnlC7HTS+8R5mKlYnZupk37ryS5+Yu96t+j8fDxGce4vF3JlO+cgT3jbyIdhf0omrteil1ajVowvjPFlGkWHG+mPohH70yjntefIem7TryytQvAfjnyCFu6teRFud18bv+px++i3c/n0vliCiGXdSZrj0vok69hil1IqKq8vQr7zDp7dfS3f/Q7ddy3W330aFzN44d+9fvyUQ9Hg/vPPMgT06cSvnKEdw9ojftLuhJtdr1U+rUatCElz9fTJFixVk4ZRKTXhnHfS9OBOCVh29lyLV30PK8Lvx3/BiFgsj45IQUd3vO13BS9b+nqs+lKR8NvAjEuJfeVNX3smozONPF5nF2bFxHxSrVqRhVjdDCYbS58GLWfb80VZ1iJUqmfI7/77+Un5Cq9RtTpmJlACJr1SMx/iQJ8Sf9Jx746/dfiahag/Aq1SlcOIxOvQewYtniVHWatutIkWLFAajXtBUH98Wla+fnpQto1alrSj1/sf7XVVStUYuq1WsSFhbGRQMG883iBanqRFWtTv1GTZA0Rm7Ln5tITPTQoXM3AEqUOIdiftb/1++/ElGtZsr7P7/3JfzyTer336xdp5T3Wr9Zaw7sdd7/rq2b8Xg8tHT/+BQrXsLv7z9T3NA4uyPLJkRCgAlAH6ARMEJEGmVQdYqqtnCPLI0gmCHMFQ7v30PZyhEp52UrhnN4f/q9Q5bN+IhHB3dh1lvPMezOx9KV//rNIqrUa0zhsCK5qjctf+/bQ4XwyJTz8pUiOLg3vaFL5stZn9OqY7d017//Yg6del+SKxqzYu+eWCIiq6ScV46IYu+etDs+ZszObVsoVao0t18zgkE9OzB+3MN4PJ7ckpohB/fGUaHyqfdfoXJEhn9oklk66zNad3Lef+zObZQoWYpn7rya24deyP9eesLv+jMjeYndWe5i1w7YoqrbVDUemAwMOFtted4QikgNERl5hvf+m9N6ADTdftMZTxu4YNAVjJv+LZfcdD8LJ72Zqix225/Meut5Rt33dG5IzBJn75vUZPYDumz+DLZu/I1LRqfuA/17/152bdlEyw4X5IbErDkN/WlJTExk9YqfuOfRZ5iy8Dt279rO7Kmf5LTCLPH15wfgm/nT2bJhHZeOvgkAT2IiG9f8wtV3P8bLn33BnuhdfDVnSq7qPR18NIQVRGSV13GdVxNRwG6v82j3WloGichvIjJdRKpmUJ6KPG8IgRpAhoZQRALSB1q2YgSHvDyoQ/v3ULpC5Uzrt7nwYtZ9dyp0PrQvjncevJ7RY1+iYhX/dtIDlK8cwQEvD+rgvjjKVQpPV2/d8u+Y/t5rPPjapHRe649L5tG+Wx9CCxfOdb1pqRwRRVxsdMr53rgYKnl56FkRHhFFwybNqFq9JqGhoXTvdTEb16/NLakZUqFyJAf2nnr/B/bGUa5i+ve/dvl3THv3NR55/cOU91++ciS1GjQhvEp1QkJDObdbb7Zt+s1v2rPDx9D4gKq28TomejWR0V+EtH855gE1VLUZ8CXwYba6zvQLnS2uJ7dJRN4VkQ0iskREiolIbRH5QkRWi8j3ItLArT9JRAZ73Z/szT0HnC8ia0XkThEZLSLTRGQesEREzhGRr0RkjYisF5GzdqOzo3rDZuyL3sGB2N0kJsSz6st5NOt0Yao6+3ZvT/n8+09fU6lqDQCO/3OUCfdczYAb7qN2sza5LTVD6jZuQdyu7eyN3kVCQjw/fDGHtl16pqqzbdN6/m/c/Tz02iTKlK+Qro0fFs3m/ACExQBNWrRm1/atRO/aQXx8PAvnTKdrz4t8vvfI4cP8fXA/AL/8+C216zXITbnpqNu4BbE7t7EneicJCfF8/8Vs2l+Q+v1v3bSet568l0de/5Ay5SueurdJC/49eoQjfx8A4LcVP6Qa5Aoo7hK77I5siAa8PbwqQKp+D1U9qKrJHevvAq2zazTQo8Z1gRGqeq2ITAUGAVcBN6jqXyLSHngLSN8BdYoHgHtUtR+kjBidBzRT1b9dr3Cgqh51d71fLiJzNaP4z8V1xa8DKOfVV+MrIaGhDL/rCd648wqSPEl06DeEyFr1mPfuy1Rr0JTm5/dg2fSP+GPVj4SEhlK8ZGmufGQ8AMumf8j+6J0smvQGiya9AcCtr3xEqXLpjU1uERIayrUPPs0TN44kKclD90uGU61OfT6b8AJ1Gjen3QW9+PCVcZw4fowX73WilorhUTz0uvOHd1/Mbg7siaVxm/P8ptmb0NBQHn7qJa4beQlJSR4GDrucOvUb8caL42jcvBXdevZl/drV3D5mBEePHGbZ0kVMeOlp5n6zipCQEO4d+wxjhvVDVWnUtCWDR17lV/0hoaFc/9AzPH7jCJI8Hi68ZATV6jTg0wnPU6dRC9p37cWkl5/kv+PHeP6eawHn/T/yxkeEhIRw1d2P8ci1Q0CV2o2a0XPQZX7VnxmSM9NnVgJ1RaQmzqjwcNJEhCISoarJIVl/YFO22jKzByKS5c7aqnrUB9GZP1ikBrBUVeu65/cDhYGHgc1eVYuoakMRmQTMV9Xpbv1/VfUcEbmA9Iawi6pe5Z4XBl4BOgNJQH2gpqruSW4jK53VGzbTBz+YezZfNaCEl/DvQEtOk9c3eN96MFe6of3CXcN78teGdTk296ZUtYba/r7/ZVvvy1vPW62qmYZDInIR8CrO9JkPVPVpEXkSWKWqc0XkWRwDmAj8Ddyoqn9k9cysPMINOLG394tIPlcgJ2Zoes8L8QCVgcOq2iKDuom4obw4f1bCsmj3mNfnUUBFoLWqJojIDqDo2Yg2DOPMyIl5hKq6EFiY5tpYr88PAg+eTpuZGkJVzXakJRc4CmwXkSGqOs01eM1UdR2wAyfWn4ozXJ7cC/8PUDKjxlxKA/tcI9gV8P/og2EYbh9g8Ezu9sanwRIRGS4iD7mfq4hItp2PZ8EoYIyIrMPxSpMHN94FuojICqA9p7y+34BEEVknIndm0N6nQBsRWeW2naWLbBhG7hFSSLI9AkG2gyUi8iaO99UZeAY4DrwNtD2bB6vqDqCJ1/l4r+LeGdTfC5zrdelB93oC0D1N9Ule9x3AGTzJSEOW/YOGYeQsQeoQ+jRq3EFVW4nIrwDuSGxW/XOGYRjpEJyR42DEF0OYICKFcCctikh5nNFXwzAM35HAhb7Z4Usf4QRgBlBRRJ4AfgCez1VVhmHkS3JgQnWukK1HqKofichqIHlpxBBV/T13ZRmGkd8QCKqUYN74urIkBEjACY/zw/pkwzACQLBmqM7WqInIw8DnQCTOur7PROS0JisahmH4EhYHbWgMXIazKuM4gIg8DawGns1NYYZh5D/ycmi8M029UGBb7sgxDCM/k+cMoYi8gtMneBzYICKL3fOeOCPHhmEYPuMMlgRaRcZk5REmjwxvALw3fPDvTkKGYeQPfEvFHxCySrrwvj+FGIaR/wnWUWNf1hrXBp7G2TEqJX2VqgZJ2lvDMPICwRwa+zIncBLwP5zv0QcnDdbkXNRkGEY+JQd2scsVfDGExVV1MYCqblXVR4CuuSvLMIz8hgiEiGR7BAJfDOFJN0HqVhG5QUQuBirlsi7DMPIhOTGhWkR6i8hmEdkiIg9kUW+wiKiIZLsLmi/zCO8EzgFuw+krLA1c7cN9hmEYqTjb0FdEQnASwfTA2dFupbsZ28Y09Uri2KxffGnXl6QLyQ39A1x+OqINwzCSEXIkDVc7YIuqbgMQkck4Wew3pqk3DngBuMeXRrOaUD2L9Bsnp6Cql/ryAMMwDCBlX2MfqOBurZHMRK9N3qOA3V5l0Thbd5x6jEhLoKqqzheRszOEwJu+NJDf2XvoP16ame22qEHLlw/k7XGtTXv+CbSEs6JjrfKBlnDGlCiS89ue+xgaH8hiO8+MGkhx2Nwk0q8Ao09HV1YTqr86nYYMwzCyQiAnRoWjAe8dNqsAsV7nJXH2QlrmGt1wYK6I9FdVby8zFTlv8g3DMDIhByZUrwTqikhNIAYYDoxMLlTVI0CF5HMRWQbck5URBEuyahiGHykk2R9ZoaqJwC3AYmATMFVVN4jIkyLS/0x1+ewRikgRVT15pg8yDKNgI0KObN6kqguBhWmujc2k7gW+tOlLhup2IrIe+Ms9by4ib/jSuGEYhjfBmqHal9D4daAfcBBAVddhS+wMwzhNkjdvyu4IBL6ExoVUdWeaYW9PLukxDCMfExKk2Wd8MYS7RaQdoO7ylluBP3NXlmEY+Q0JoMeXHb4YwhtxwuNqwF7gS/eaYRjGaRGkdtCntcb7cObqGIZhnDEChAZpZlZfMlS/SwZrjlX1ulxRZBhGviXPeoQ4oXAyRYGBpF70bBiGkT0+TJgOFL6ExlO8z0XkY2BprikyDCNfkkNrjXOFM1lrXBOontNCDMPI/+RZj1BEDnGqj7AQ8DeQaXpswzCMzMhz+xoDuHuVNMfJ8gCQpKqZJms1DMPIDGetcaBVZEyWslyjN0tVPe5hRtBHzq9fgcX3ns+X93fmuq610pVf2iaKXx7rztw7OzL3zo4MaVcFgPa1y6Vcm3tnR35/picXNvb/XlnLvlpCt/bN6NK2MW+99mK68l9++oG+Xc+jduVzWDh3Zsr1DevXMbB3F3p0bEXvzm2ZN2uaP2WnsOqHr7mm33lc3acdU997PV35zA//j+v6d+LGgV14YMwg9saeGv/r2yycmwd15eZBXXn8lsDsTvHV0sW0a9mYNs0a8OpLL6QrP3nyJGOuGEmbZg3ocUEHdu3cAcC0KZ/R5bzWKUeFkmGs/22tn9VnTl5eYrdCRFqp6ppcV5NPKCTw+MDGjJ64gj1HTjDjtg58vWEfW/b9m6regnVxPDk79VYLv2z9m/6v/AjByCfkAAAgAElEQVRA6WKF+fKBzvzw5wG/aQfweDyMvf8OPpm+gPDIKPr36ESP3v2oW79hSp3IKlUZ/+ZE3p3waqp7ixUrzssT3qdm7TrsjYulX/eOdO7Wg9Kly/hV/4Sn7ueZd6dRITyS24f1pH3XXlSvXT+lTu2GTXl9yhKKFivO/Mn/44OXnuTBl94FIKxIUSbM+MZvetPi8Xi4767bmDF3EZFRVbiw87n0vqgfDRo2SqnzyYcfUKZMGVb99gczp03hiUcf4v2PPmPIsJEMGeak59v4+3ouGz6Ips1aBOqrpCJPbvAuIslGshOOMdwsImtE5FcRMaOYBc2qlWHngWPs/vs/EjzKgrVxdD8Dr653s3C+++MAJxKSckFl5qxds5LqNWtTrUZNwsLCuHjgEJYsmp+qTtVq1WnYuClSKPWPUK06dalZuw4AlSMiKV+xIn8f8K8h/3P9GiKr1SSiag0KFw6jS5+BLP/6i1R1mrfrRNFixQFo0LwNB/bGZtRUQFizagU1a9WmRs1ahIWFMXDwMBYtmJeqzqIF8xg+yvFW+w8cxHfLviZtwDZj+hQuHTzMb7qzJ/s9jQM1qpyVR7gCaAVc4ict+YbwUkWJO3wi5XzPkRM0r5beI+rVtDJta5Vlx/7jPD13E3uOnEhV3rdFBP/7bnuu603L3rhYIiOrpJxHREaxdvWK025n7ZqVJMTHU71m+q6B3OTAvj1UDI9KOa9QOYLN6zP/271k5qe0Ob97ynl8/EluG9qDQqGhDB1zKx26X5SretMSFxtLVJVT7z8yKorVK1ekqxNZxclYHxoaSqnSpfn74EHKV0hJzszsGdP4ZPIM/4j2ASFvTqgWAFXd6ictp4WIlAFGqupb7nkk8LqqDg6sMjLcXibtX+uvN+5j/q9xxHuSGHFuVV4Y3owr3jn1w16xZBHqh5fk+83+9aYgvVY4/dG+fXviuOvGMYyf8C6FCvm5hzyjruxM9H89bxp/bljHC5Nmp1z7aOmvlK8UTtzuHTwwZhA16jYkslrN3FKbDl/ef3Z1Vq38hWLFitGwcZOcF3im5NEJ1RVF5K7MClX15VzQczqUAW4C3gJQ1Vgg8EYQxwOMKFM05Ty8dFH2HU2d3Pvw8YSUz1N+2c29F9VPVX5R83CW/L6HxCT/j0+FR0YRGxudch4XG0Ol8Eif7//nn6NcNeJS7n7oMVq1aZ/9DTlMhcoR7N8Tk3J+YG8c5SuGp6v368/fMnniq7wwaTZhYUVSrpev5NSNqFqDZm07sPWP3/1qCCOjooiJPvX+Y2NiCI+ITFcnNno3UVFVSExM5OiRI5QtVy6lfNb0qVw6JLhSBAg5k6FaRHoDrwEhwHuq+lya8huAm3HSBf4LXJd2A/i0ZPWnOgQ4B2dXqIyO7MTWEJFNIvKuiGwQkSUiUkxEaovIFyKyWkS+F5EGbv3aIrJcRFa6+w/8614/R0S+cvsn14vIAPcRzwG1RWStiLzoPu93955fRKSxl5ZlItJaREqIyAfuM371aitHWb/7CDUqlKBK2WIUDhH6tojgq437UtWpWPLUL173xpXZuu9YqvJ+LSKZvzYuN+RlS/OWbdixbQu7d+4gPj6eebOm0aN3X5/ujY+P5/orhnHpsJH0HTAol5VmTL0mLYndtY090TtJSIjn20WzOLdrr1R1tmxaz+tP3MNjb35MmfIVU67/c+Qw8fHOH60jhw6y8dcVVKtdz6/6W7Zuy7atW9i5Yzvx8fHMmj6FPhf1S1Wn90X9mPzpxwDMnTWD87t0TfEIk5KSmDNrBpcOHupX3b5wtqPGbirACUAfoBEwQkQapan2mao2VdUWOJu8Z+u0ZeURxqnqk9k1kA11gRGqeq2ITAUGAVcBN6jqXyLSHsej64Zj4V9T1c9di57MCWCgqh4VkQrAchGZizOpu4n7ZRGRGl73TAaGAo+JSAQQqaqrReQZ4GtVvdoNrVeIyJeqmsoKich1wHUAoSVPf5DDk6Q8MXsjH1zblpBCwvQV0WzZ+y+396zL+ugjfL1xH1d0qk73RpVITFKOHE/g/im/pdwfVbYY4WWKsmLb36f97JwgNDSUJ597hSuGXIwnycPQkVdSr0EjXn72SZq2aEWPPv1Yt2YV1185jCNHDvPV4oW88vxTLP1xDQtmz2DFzz9w6NDfTJ/8CQDj35hI46bN/aY/JDSUGx96jkeuH4bH46HnwJFUr9OAj958jnqNW3Bu1968/9LjnDh+jGfuGgNAxYgqPP7mx+ze9idvPHkvIoKqMnTMbalGm/1BaGgoz7/0GkMu6YvH42Hk5aNp0Kgxz457nBatWtOn78VcduXV3HjNaNo0a0CZsmV5b9KnKff/9MP3REZFUcPPfbO+kAN9hO2ALaq6zWlPJgMDgBSPT1WPetUvQQZJY9LpymxqoIj8qqotz1Sta5iWqmpd9/x+oDDwMLDZq2oRVW0oIgeByqqaKCKlgFhVPUdECuNs2NwZSALq4yzzKwrMV9UmXs+br6pNRCTKfXYjEbkdqKSqD4vIKve+RPfZ5YBeqprpDu5Fw+tq1cvTz0PLK9gG74GlXY2ygZZwxnQ7vz1r16zOsV69mo2a6eMfLci23ui21VZntsG7iAwGeqvqNe755UB7Vb0lTb2bgbuAMKCbqv6V1TOz8gi7Z1HmK94dYx6gMnA42YvzkVFARaC1qiaIyA4cY5YpqhojIgdFpBkwDLjeLRJgkKpuzvxuwzByCx+tagXXaUlmoqpOzKKJjNIETgAmiMhI4BHgyqwemGkfoarmRlx2FNguIkPAWcInIskx03Kc0BlSJ4ItDexzjWBXTiV8+Ies+yonA/cBpVV1vXttMXCru3QQETljj9cwjNPjNDZvOqCqbbyOiV7NRANVvc6rAFlNAp2MD1MAA7HybxQwRkTWARtw4nuAO4C7RGQFEAEcca9/CrRx/0KMAv4AUNWDwI8i8ruIpF8DBtNxDOpUr2vjcMLz39yBlXE5+s0Mw8iSs93gHVgJ1BWRmiIShvM7Pte7gojU9Trti7sVcVacSRoun1DVHUATr/PxXsW9M7glBjhXVVVEhgOr3PsOAOdl8oyRaS55P28vab6fqv7HqTDZMAy/ImedfcYdQ7gFJ7oLAT5Q1Q0i8iSwSlXnAreIyIVAAnCIbMJiyEVDeAa0Bt50w9bDwNUB1mMYRg6SU4lZVXUhsDDNtbFen28/3TaDxhCq6vc4Kb8Mw8inBOnCkuAxhIZh5HMkjyZmNQzDyCny254lhmEYZ0RwmkEzhIZh+JEgdQjNEBqG4R8sNDYMw0CQIA2OzRAahuE3gtQhNENoGIZ/ELHQ2DAMwzxCwzAM6yM0DKNAY6PGhmEYWGhsGIZhobFhGAUbQSw0NgyjgCMWGhuGYQRpYGyGMFtEhNDQQGztkjP8eyIx+0pBTNmihQMt4azIZLfcAkkwjxrn3d9wwzDyHuLDkV0TIr1FZLOIbBGRBzIov0tENorIbyLylYhUz6gdb8wQGobhN8SH/7K8XyQEmAD0ARoBI0SkUZpqvwJtVLUZzm6WL2SnywyhYRh+Iwe282wHbFHVbaoaj7Nv8QDvCqr6jaoed0+X4+x9nLWu0/8qhmEYZ4hvoXEFEVnldVzn1UIUsNvrPNq9lhljgEXZybLBEsMw/IJj53waLDmgqm2yaCYtGQ5JichlQBugS3YPNENoGIZ/8C30zY5ooKrXeRUgNt2jnA3eHwa6qOrJ7Bq10NgwDP9x9qPGK4G6IlJTRMKA4cDcVI8QaQm8A/RX1X2+yDKP0DAMP3H2qfpVNVFEbgEWAyHAB6q6QUSeBFap6lzgReAcYJq7j/IuVe2fVbtmCA3D8AtCjoTGqOpCYGGaa2O9Pl94um2aITQMw38E58ISM4SGYfgPS8NlGEaBJydC49zADKFhGP7Bx7XEgcAMoWEYfsNCY8MwCjQ5NWqcG5ghNAzDfwSpIbSVJblEp3oVWHhXJ76453yu6VIzXfklrSL58eGuzLz1PGbeeh6D25xaNx5RuijvXd2a+Xd2ZN4dHYksU9Sf0gH44Zul9Ovckj4dm/Pemy+lK1+1/AeG9O5E8+plWDJ/dqqyuJjdXDtyABdf0Jr+XdsQs3unv2Sn8PO3XzK0R1sGd2vFR2+/kq78s/cnMLzXuYzq25FbLh9AXMyulLI7rhrMhS2rc/e1w/wpORVfLV1M+5aNadu8Aa+9lD6L1MmTJxlz5UjaNm9Az64d2LVzBwDTpnzGBR1apxwVS4Wx/re1flafOWebhiu3MI8wFygk8Gj/hox5fxV7j55g6s3n8c2mfWzddyxVvUXr9/DU3E3p7n9uaFPe+WYbP205SPGwEJL8nObY4/Hw1CN38+5ncwiPiGJY3y507dmX2vUapNSJiKrKUy+/zaR3Xk93/4O3X8d1t91Lh87dOH7sX6SQf//eejwexj9+L69/OItK4ZFcdWk3zu/eh5p1T+mv36gZk2Z/TdFixZnx6fu8+fzjPP36BwCMuvZWTvx3nNmTJ/lVt7f+++++jelzFhEZVYUeXc6ld99+1G9wKu3epx99QJkyZVi57g9mTp/CE2Mf4v0PP2PIsJEMGTYSgI0b1nP58EE0bdYiIN8jI4I1NDaPMBdoVrU0uw4eJ/rQfyR4lIXr4ujWsJJP99auVIKQQsJPWw4CcDzew4mEpNyUm471a1dRrUYtqlavSeGwMPoMGMTXS+anqhNVtTr1GzWhUJqf7K1//oHHk0iHzt0AKF7iHIoVK+437QAb162mSvVaRFWrQeGwMHr0vZTvvky1EIHW551PUVdXkxZt2bcnJqWsbYcuFC9R0q+avVmzagU1a9WmRs1ahIWFMXDQMBbNn5eqzqIF8xg+8nIA+l8yiO+XfY2m+YM5c9oULh0cOK82Q3IgQ3VuYIYwF6hUqih7jpxIOd979ASVS6cPb3s2rszs2zrw6sjmhLvlNSqU4J8TCbw+qgUzbj2Pe/rU8/tf0X1xcYRHnArVK4dHsS8uzqd7d2z7i5KlSnP7NSMZ3Ksj48c9jMfjyS2pGbJ/bxyVvPRXCo9k/97M9c+b9jHndenhD2k+ERcXS2TUqVyikVFRxMXFpK4TG0tUFScJS2hoKKVKl+bvgwdT1Zk9cxqXDgkeQ5ichisYQ+M8ZwhF5AYRucL9PFpEIr3K3ssgbbffyTBhWpq/1sv+2E/3F77lktd/4uctB3l2SBMAQgoJrWuU5YWFmxk6YTlVyxVnYOus8k7mPJpBejfxcdMdT6KHNSt+5p5Hn2bygm+J3rWD2VM/yWmJWZL2XQOZ7iO5aPYUNq1fy2XX3JrLqnwnI/1p3392dVav/IVixYrRsFGTnBd4pviQnTpQoXOeM4Sq+raqfuSejgYivcquUdWNARHmxd6jJ1I8PIDKpYqy72jqlGiHjyeQ4HF+mKetjKZxVCnn3iMn2BT7D9GH/sOTpHy1cS+NIkv5TzxQOSKSPV4eyN49MVQMD/f53gaNm1G1ek1CQ0Pp1qsfm35fl1tSM6RSeCT7vPTv2xNLxUrp9a/4cRmT/u9lXpz4GWFFivhTYpZERkYRGxOdch4bE0N4eGTqOlFRxEQ7iZoTExM5euQIZcuVSymfOWMqlw4e7h/Bp4OFxiAiNUTkDxH50N1harqIFBeR7iLyq4isF5EPRKSIW/85r92oxrvXHheRe0RkME722U9FZK2IFBORZSLSRkRuFJEXvJ47WkTecD9fJiIr3HvecTeDyVHWRx+leoXiRJUtRuEQ4aLmEXyzKXVatIolw1I+d2tYiW3uQMr66COUKlaYsiWcbSzb1yrP1n3/5rTELGnSvDW7tm8letcOEuLjWTRnBl179PXt3hatOXrkMH8f3A/Aip++pbbXIIU/aNisFbt3biV2904S4uNZumAm53fvk6rO5g2/8fwjd/LiO59RrnxFv+rLjpat27Jt6xZ27thOfHw8s2ZMoXfffqnq9L6oH5M/+xiAubNncH6XrikeYVJSEnNnzWDg4KF+1541vgTGBWfUuD4wRlV/FJEPgLuA64HuqvqniHwE3Oj+OxBooKoqImW8G1HV6W5esntUdRWkCg2mAz8D97nnw4CnRaSh+7mjqiaIyFvAKOAj77bdPRKuAwgt5dsghzeeJOWpuZt47+rWFBJh5qoYtuw7xq0X1uH3mCN8s2k/l3WoTreGlUhMUo4cT+DB6b8DkKTw4sLN/G9MW0RgQ8xRpq2MzuaJOUtoaCgPjRvP9aMuwZOUxMBhl1OnfkPefPEpGjdvSdeefVm/djV3XDOSo0cOs2zpIia8/DRzvl5JSEgI9zz6NGOGXQyqNGrWgsEjR/td/z2PvcDtVw0iyeOh35BR1KrXkImvPkODJi3ofOFFvPH8WI4fP8bDtzraKkdUYfzEzwG4fngfdm79i/+OH+Pijo15+NnXObdzd7/qf278awy5pC9JSR5GXj6aBg0b8+xTj9OiZWv69L2YUVdczU3XjqZt8waUKVuWd//3acr9P/34PZGRUdSoWctvmn0hmCdUS4b9Kbn1MJEawHeqWs097wY8CoSoamf3WnfgZmAosBpYBSwA5qtqvIg8DvyrquNFZBmpDWHKuYgsAcYCf+Fkta3ttvsQkOyeFQM+V9XHM9NcLKKe1hj9Rg69Af8z/ZaOgZZwVuT1DerrVD4n0BLOmO6d27N2zeocM13NWrTWuV/9mG29mhWKrc5iz5JcIRAeoU+W181E2w7ojpOO+xag22k8ZwqOMf0DmOV6lQJ8qKoPnqZmwzBygGBdaxyIwZJqInKe+3kE8CVQQ0TquNcuB74VkXOA0m422juAjGaF/gNkNuFrJnCJ+4wp7rWvgMEiUglARMqJSPWz/UKGYfhGTowai0hvEdksIltE5IEMyjuLyBoRSXTHErLXdfpf5azZBFwpIr8B5YBXgKtw9hdYDyQBb+MYuPluvW+BOzNoaxLwdvJgiXeBqh4CNgLVVXWFe20j8AiwxG13KRCR81/RMIx0iDOLKbsjyyacwc0JQB+gETAigylzu3BmlHzmq7RAhMZJqnpDmmtfAS3TXIvD2dU+Fd79eao6A5jhVXxBmrqph9qca1M45SEahuFXzjo0bgdsUdVtACIyGRiA4/QAoKo73DKfl2TZWmPDMPzCaYwaVxCRVV7nE1V1ovs5CtjtVRYNtD9bbX41hK6lDqKp7oZh+BMfFygdyGLUOMOFW2csyMU8QsMw/EYOjBpHA1W9zqsAsWfbaJ5bYmcYRt7lbAdLcOYE1xWRmiIShjO1bu7Z6jJDaBiGX/DFCGZnCFU1EWdO8WKcGShTVXWDiDwpIv2d50hbEYkGhgDviMiG7LRZaGwYht/IiQnV7tzihWmujfX6vBInZPYZM4SGYfgNHwdL/I4ZQsMw/IYZQsMwCjiBS7OVHWYIDcPwC4J5hIZhGGYIDcMwLDQ2DKNAIwHcnCk7zBAahuE/zBAahlHQsdDYMIwCj4XGhmEYZggNwyjoBGto7NftPPMiIrIf2JmLj6gAHMjF9nMb0x84clt7dVWtmFONicgXOJqz44Cq9s6p5/qCGcIAIyKr/L2Ha05i+gNHXtYebFg+QsMwCjxmCA3DKPCYIQw8E7OvEtSY/sCRl7UHFdZHaBhGgcc8QsMwCjxmCA3DKPCYITRyDREn+1zyv4YRrJghNHKTJgCqqmYMA4/9P8gcM4QBRERCAq0hN/D6hZssItMg/xnDvPBdvDzyKiISChQLsKSgxUaNA4SIhKpqoogUAl4D9gMbVHVGgKXlGCJSGPgF+F1Vr3CvieaxH7pkzSLSCCgBbFbVo4HW5Qsi0g+4E1gHHAPeUtW4wKoKPswjDAAiUt01giHAPGAvsB34XEQuD6y6s8PLCwlV1QSgPdBaRD6CvOkZupovAqYDQ4ENItIswLKyRUSaAuOAUTjeYBvg37z2/v2BGUI/IyLtgYdFpAwwAFgOvAxcAbwBTMyrxjCNt1fJNfgJQEugZV41hiJSDcer6gUsBv4BYrzKg/W7FAGmAY1x/h/crKr/AE1cb91wMUPof/YBB4ESqjoTxwi+CKxQ1buBd4EPReTcAGo8I5KNoIjcDXwATBWRu1Q1HmgFNBWRWd51gx23b20/8AUwAnga6KeqB0VkoIgUCbbvIiJNRORaHGM9APg/4BJV3SYifYCxwDmB1BhsmCH0E8leg6pux+mred09PwbsAXa7ofIh4HpVXR4oraeLt0ckItcB/d00Sr8DT4rIWK8wuZKIRAaxF5WCG/6OA5JwtF8FDHQNSju3rEEAJabDfa+NgQZuX+B04Cugn4h0B54DPlbVQwGUGXTYYIkfEJEQVfWISCFVTXKvPQXEqupbInIj0BZoDqxV1TFunZT6wYp3OCwi4UAUjgc1EOiM40F9Bbytqg8GTKgPpB3IEZEo4DvgGhzvagpOn24Y0Bd4SFXnBUJrRohIYVVNEJEawCwcQ70Y6I5jxOOARao6Ly8OWuUmZghzmWRj5o4Of4IzMPI1jpdRS1XfcOvVByqp6vfueZ76QRWRq3EGEi7F6ZuaBDyiqutF5H2cOYW9VPVw4FRmThqDXhhIdPsyBwMtVfVhEWmB88eqFPCrqv4QyP9PIlIVKOO+4/rA5cBnqrpRRLq55/er6j63fvJMhTz1s+UPLFV/LuMaQcEJSQ66xwU4He7tReQfVZ2kqpuBzZA3PEFvRKQjjhG8TFWPi0g8sAUY6v5ChgGDg9gIVgaeEJFbgDrAKzj9mz8BPwHXi0hDVV0LrPW+N8AGpRuwTkSKAlWBE8AMERkPJOJ45uE4/dKoaqL7rxnBNFgfoX/4P6CNqt6qqk8Bq4B4YBMwWkRaeVcOdiMoIqW9PjfFmZbRFCcES/6F+w7H6x0CPKequwMg1Vf+xhm0igK2Am8DlYHZQH2ckHKca3ACjld/84c420jMAE64P1s346TDvxi4B3jJ+x4jY8wj9A+LgVEicpWq/g+nr2kEUB3HM6wvIutU1RNIkb4gImFAVxGpjTPoEwF8jPOz1F1E/lbVpao6B5gjIs+r6vEASs6U5FDR7VfbDTwOdAT6qOocEdmIY8jLAufihMQnAiYYEJHiOF7rbyLSGVgP/AzcLyJJqvq1iHwDlAN2AwvAvMDssD5CP+HO8H8axzv63B0hbuIW71bVvwOn7vRw+6bm43hNbVV1t4jUAfoAjYAFqjo/kBqzw50WMwz4DWeTyQE4K3yeAFoAl6rqIREpDxQHaqvqsgDJBVL6Ls/BmW4VD/QDLlbVdSJyP9AFeBJYo6rxyX2B1ieYPRYa+wnXMDwAPCAiY1TVo6rr3CPPGEGXPcAGnP6z61zPagswEye07CoiJQIpMDvc8H0bsBTHqE92l809iNMPOFVEyqrqQVXdrarLAhleikglYLQ77WUpzkDIVFVdB6CqzwPf4vRFt/E2fmYEs8cMoR9R1UU4HketQGs5U8RZ9fKSqo4EbgVqAC+4xeWBHcA4d35ksLMdJ3yM59Q2kyeB+3AGrua5niMQcIMSDixzDeK/OKPzTUTkJhEp5+p7HpiKO+IdOKl5DwuNjSzJYG5dSZyJ0nNV9VZxEhE8ijNqWQQnpAzagRGvcLGwO8kbd7XFCzjTfeaISC2cvsASqvpXIPV644bGz+EY63E4AzmvAB+510YAg9RZyWOcBmYIDZ8QkbrAv6oa5xrD1cA3qnq9GwZfCXypqn8GVGgWeBnBATj9g0WBx1X1NxEZCjyLM/+xF3CDqv4eOLUOXpob43jbTXG8wWM4a9OrAXfgjHi/p6pTAqU1L2OG0MgSt1+sLo4nMhtYrKp7ReQcnKkbc1T16kBqPB1c728cMAjHkDQFrnL7AHvgJL/4RFUXB1BmKkSkP064fqeqrhRnHfownOWY7+JM0i/tDu7YwMgZYH2ERjq8BwXU4U+cX7ieQDcRiVDVf3EMSTcRqRzs89S89LUEbsQZGS6Jkxxisoj0UtWlwNWqujhYvo/rCT6FM1Cy0h3F/hMYD0QCNwBF3UGUQPdj5llsHqGRDq+lZrcAtXGmbDyKM81kCFDVnU9YA2ivqnsDJPV0qA/8oarPiEgEjod7var+KSIXAs+KyIpgMShenl1lnJUhlURkJNAJaIcziX0i8F+wztPMS5hHaGSIOIkgLsHx+toCD6jqQpxRScWZYDw+mI1gslfn9m+uEJE3AdTJyhKDs8SxI/AXcKMGQUYWL0+0vPvvNzgrkV7Dme4zFGcVTFtVXaOqm/yvMv9hfYQGkKpTPvnfx4AJOIMg3XA66JOAQqp60nvUNZhxJ7IPBWJx5t4tUNXrROQaHO+qM07C0kUBlJkKEekN3IUzX3MH8LK667TFSez7IU4I/1PAROYzzBAaaTOv1MPxPN7HWQK4ByeZQqIbKnuAd3C7DwOl2Rfc0ewFOPMe54lIWWAFME1VH3JX99QOppFut09wDk7arJI4IXAj4G6cuY5TgbuDfeVOXsNC4wJOGiN4C47heB5nsnFTYJlrBEcDN+FMkUkKdiMIKUlvt+N4g7ih7+3AbSLyjLu6J+BGMM3ATBFgqTrp2L7AGcz5BycB7HqcxLDzg2UwJ79ggyUFHC8j2B9oBvTGGR0uBczFWczfBGe0dXAwTTBOi1dYXx84jjO9ZAXwqYi0cgcVDuFMQu4pIue7BieguJo74qw4CgWGiMhcN1yPFpFEoLo6WYk2Jt8TOMX5DzOERnIm5jdxvL2tIvIBzjw7cLyp14CTqnokUBp9wTUofXA82uk4Ky2a4KSu/15EvsIZ9R6AM5k6oOnOvAz3uTip2n7D6YqIxsmPWBXH8HXAWT1i5BIWGhuoagzO6oTeIjJcVU8Ck3ESexYC4oPdCAKIkwHnMZxtArbgGLriqnoLcC9OjsReQGmgB06ewYDhGsF2OFmJrlXVy4C3cN79XpxBnquAx1T158Apzf+YR2gAoKozReQkznw6VHWyiEzCWW/7T4DlZUqalRSHgNXml+UAAATpSURBVE+B1jiGfYCq/iMiPYHlqnrUHYx4EbhSVbcFRnUqSuNkLO8O/ALswsnsUx4nzX7yHje2YiQXMUNopKCqC0QkCWdv5URVnY7TUR+0uF5VF6Ahzmj3nTg/17XVSbh6Lk76s2uBozhhZ19VPRgozd6o6lIRuRR4SUS2/3979xtaVR3Hcfz9yf4pW/mg/z2ZWUEgtRQpikpiDCoMi4KkglK0DKKIDCEDi0JBCPJBFEVIBEKhkhSyqAeiMkFYTgyd0Z89qWA9ERIlGN8efH+Tw9XNTZt3d+fzgsHduWfnd+5lfPn9fuec3ydyrcpjZHG8StJQebrHRXAS+fYZO0155vaXKdJjOqPK/Npd5JXVATL6YCb5vPB7ZG7HMnJhha+bdrLjIGkx2ZvdSV7o2epbZC4cF0JrWWV+7R3gjcgVZJ4l7328nrwN5RDwU+l1TfmhZblyv45c9OH9kVtkpvp5TwceGlsrmw10kRc+DgJbyAsMbcDRiPhgZMdWKCYRsUPSSeAzSb9HxLZmn1NduBBay4qI78r82npJf5T5tZH1+PqbeW7nqnym58nIA7tAPDS2lifpYXKNwU2REZdmE+JCaNNCmV/bQA6V/4opng1tU4sLoU0bkq6OiKFmn4e1HhdCM6s9P2JnZrXnQmhmtedCaGa150JoZrXnQminSBqWdEDSIUlfSZp1HsdaJOmb8vpRSWvG2He2pJfOoY11kl4f7/aGfTZLemICbXVIanrgu00OF0KrOhERnRExD/iXzMw9RWnC/zMRsSMiNoyxy2wyBsCsKVwIbTS7gZtLT+iwpA+BPjLTuFtSr6S+0nNsg0xfk3RE0h4y9Y6y/bmRKE1lGPx2Sf3l5x7yRui5pTe6sey3WtJ+SQclvV051puSBiR9T2YVj0nSinKcfklbG3q5XZJ2SzqqTLtD0gxJGyttv3C+X6RNfS6EdhpJFwMPkWFBkAXn84i4EzgOrAW6ImI+mbn7mqTLgU+AxcB9wHWjHH4TsCsi7gDmk4uQriGX/eqMiNVlIdVbyCDzTmCBpPslLQCeIvNTHifzls9mW0QsLO0dBpZX3usAHgAeAT4qn2E5cCwiFpbjr5A0ZxztWAvzogtWNVPSgfJ6NxnpeQMwGBH7yva7yXjJvWWVqEuBXjJl7beRcCdJXwArz9DGg+R6gUTEMHBMGbNZ1V1+fiy/t5GFsR3YXkKYkLRjHJ9pnqR3yeF3G9BTee/L8ijez5J+LZ+hG7i9Mn94ZWm76Wl3NnlcCK3qRER0VjeUYne8uomMm1zasF8n8H89piRgfUR83NDGq+fQxmZgSUT0KyNJF1XeazxWlLZfjohqwURSxwTbtRbiobFN1D7g3hKUhKRZylD4I8AcSXPLfktH+fsfgFXlb2dIuoKMA2iv7NMDLKvMPd4o6RoyfOkxSTMltZPD8LNpB/6UdAnwdMN7T0q6qJzzTeQq1z3AqrI/km5VBsXbNOYeoU1IRAyVntUWSZeVzWsj4qiklcC3kv4G9pBRmo1eITNRlgPDwKqI6JW0t9yesrPME94G9JYe6T/AMxHRV9YbPAAMksP3s3mLDEUaJOc8qwV3ANgFXAu8GBEnJX1Kzh32lRWih4Al4/t2rFV50QUzqz0Pjc2s9lwIzaz2XAjNrPZcCM2s9lwIzaz2XAjNrPZcCM2s9v4DJRONtZieSq8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset_logger = logging.getLogger('data_loader')\n",
    "best_f1 = 0.0\n",
    "best_model = None\n",
    "best_hp = None\n",
    "best_iteration = -1\n",
    "for optim_iteration in range(num_optim_iterations):\n",
    "        \n",
    "    print(f'\\n\\n+-------------------------------------------------+\\nOptim Iteration: {optim_iteration}\\n\\n')\n",
    "    logger.info(f'\\n\\n=================================\\nOptim Iteration: {optim_iteration}\\n=================================')\n",
    "    \n",
    "    # generate iteration hyper parameters\n",
    "    hp = randomize_params(default_hp, random_grid_search_ranges)\n",
    "        \n",
    "    logger.info('New Params:')\n",
    "    logger.info(hp)\n",
    "    print(hp)\n",
    "    \n",
    "    logger.debug('Load dataset')\n",
    "    dataset = load(hp, dataset_logger)\n",
    "    logger.debug('dataset loaded')\n",
    "    logger.debug('Load model')\n",
    "    trainer = load_model(dataset, hp, experiment_name)\n",
    "    logger.debug('model loaded')\n",
    "    \n",
    "    logger.debug('Begin training')\n",
    "    model = None\n",
    "    try:\n",
    "        result = trainer.train(use_cuda=hp.use_cuda, perform_evaluation=False)\n",
    "        model = result['model']\n",
    "    except Exception as err:\n",
    "        logger.exception(\"Could not complete iteration \" + str(optim_iteration))\n",
    "        print(f'Could not complete iteration {optim_iteration} because of {str(err)}')\n",
    "        continue\n",
    "        \n",
    "    # perform evaluation and log results\n",
    "    result = None\n",
    "    try:\n",
    "        result = trainer.perform_final_evaluation(use_test_set=False, verbose=False)\n",
    "    except Exception as err:\n",
    "        logger.exception(\"Could not complete iteration evaluation for it \" + str(optim_iteration))\n",
    "        print(f'Could not complete iteration {optim_iteration} evaluation because of {str(err)}')\n",
    "        continue\n",
    "    \n",
    "    it_f1 = result[1][1]\n",
    "    if best_f1 < it_f1:\n",
    "        best_f1 = it_f1\n",
    "        best_model = model\n",
    "        best_hp = copy.copy(hp)\n",
    "        best_iteration = optim_iteration\n",
    "        print('+-------------------------------------------------+')\n",
    "        print(f'Best Valid Result: {best_f1}')\n",
    "        print('+-------------------------------------------------+')\n",
    "    else:\n",
    "        print(f'\\nValid Result: {best_f1}\\n')    \n",
    "    print('\\n\\n###################################################\\n')\n",
    "    \n",
    "print('Best iteration: ' + str(best_iteration))\n",
    "print('Best f1: ' + str(best_f1))\n",
    "print('Best HP:')\n",
    "print(best_hp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}