notebooks/TDParse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "from multiprocessing.pool import Pool\n",
    "import json\n",
    "import math\n",
    "from typing import Callable, List, Union, Tuple, Dict, Any\n",
    "\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedKFold\n",
    "import matplotlib as mpl\n",
    "%matplotlib inline  \n",
    "import seaborn as sns\n",
    "\n",
    "# Models\n",
    "from bella.models.tdparse import TDParseMinus, TDParse, TDParsePlus\n",
    "from bella.models.base import SKLearnModel\n",
    "# Word Vector methods\n",
    "from bella.word_vectors import VoVectors, SSWE\n",
    "from bella.helper import read_config\n",
    "# Sentiment lexicons\n",
    "from bella import lexicons\n",
    "# Get the data\n",
    "from bella.parsers import dong, election\n",
    "# Tokenisers\n",
    "from bella.tokenisers import ark_twokenize, stanford\n",
    "# Dependency parsers\n",
    "from bella import dependency_parsers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def all_words(tokeniser: Callable[[str], List[str]], \n",
    "              *datasets) -> List[str]:\n",
    "    words = []\n",
    "    for dataset in datasets:\n",
    "        words.extend(dataset.word_list(tokeniser))\n",
    "    return list(set(words))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##\n",
    "#  ADD YOUR CONFIG FILE PATH HERE \n",
    "##\n",
    "CONFIG_FP = Path('..', 'config.yaml')\n",
    "\n",
    "# Load the training data\n",
    "dong_train = dong(read_config('dong_twit_train_data', CONFIG_FP))\n",
    "dong_test = dong(read_config('dong_twit_test_data', CONFIG_FP))\n",
    "X_dong_train, X_dong_test = dong_train.data(), dong_test.data()\n",
    "y_dong_train, y_dong_test = dong_train.sentiment_data(),\\\n",
    "                            dong_test.sentiment_data()\n",
    "election_train, election_test =\\\n",
    "election(read_config('election_folder_dir', CONFIG_FP))\n",
    "X_elec_train, X_elec_test = election_train.data(), election_test.data()\n",
    "y_elec_train, y_elec_test = election_train.sentiment_data(),\\\n",
    "                            election_test.sentiment_data()\n",
    "# Get word vectors\n",
    "# To reduce the number of vectors we load we only load\n",
    "# the words that are within the train and test set\n",
    "# which is still as fair in practice as loading the whole\n",
    "# word vectors\n",
    "dong_all_words = all_words(ark_twokenize, dong_train, dong_test)\n",
    "elec_all_words = all_words(ark_twokenize, election_train, election_test)\n",
    "dong_w2v = VoVectors(filter_words=dong_all_words)\n",
    "dong_sswe = SSWE(filter_words=dong_all_words)\n",
    "elec_w2v = VoVectors(filter_words=elec_all_words)\n",
    "elec_sswe = SSWE(filter_words=elec_all_words)\n",
    "\n",
    "# Getting the sentiment lexicons\n",
    "hu_liu_fp = Path(read_config('hu_liu_lexicon', CONFIG_FP))\n",
    "mpqa_fp = Path(read_config('mpqa_lexicon', CONFIG_FP))\n",
    "nrc_fp = Path(read_config('nrc_emotion_lexicon', CONFIG_FP))\n",
    "\n",
    "subset_cats = {'positive', 'negative'}\n",
    "mpqa_low = lexicons.Mpqa(mpqa_fp, subset_cats=subset_cats, lower=True)\n",
    "nrc_low = lexicons.NRC(nrc_fp, subset_cats=subset_cats, lower=True)\n",
    "hu_liu_low = lexicons.HuLiu(hu_liu_fp, subset_cats=subset_cats, lower=True)\n",
    "mpqa_huliu_low = lexicons.Lexicon.combine_lexicons(mpqa_low, hu_liu_low)\n",
    "all_three_low = lexicons.Lexicon.combine_lexicons(mpqa_huliu_low, nrc_low)\n",
    "###\n",
    "#    Set here the number of cpus to use for all of the\n",
    "#    grid searching\n",
    "###\n",
    "\n",
    "n_cpus = 7\n",
    "\n",
    "results_folder = Path(read_config('results_folder', CONFIG_FP))\n",
    "results_folder = results_folder.joinpath('TDParse')\n",
    "results_folder.mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TDParse models\n",
    "This notebook shows how to use the TDParse models and comparing the results of our implementation to the one in the original [paper](https://aclanthology.coli.uni-saarland.de/papers/E17-1046/e17-1046)\n",
    "\n",
    "The paper had 4 different models however we have implemented 3 of them as the fourth **TDParse+ (m)** we have incorporated the same target multi appearence solution into all of the impleneted models therefore **TDParse+ (m)** is now redudant as it equals **TDParse+** in our models. The reason we did this was that we did not know which target appearence they used e.g. if there was three apearance of the same target did they use the first, second and last appearence? this is the reason why the authors created the multi appearence solution, however they did not describe how they implmented the models they did not use the multi appearence solution.\n",
    "\n",
    "Models:\n",
    "1. **TDParse-** -- This only used the syntactic connected words of the target word as context.\n",
    "2. **TDParse** -- This used the syntactic connected words, left, right and target context.\n",
    "3. **TDParse+** -- This used the same as **TDParse** but had two more contexts left and right sentiment contexts which filtered out words that are not in the sentiment lexicons.\n",
    "\n",
    "The above models correspond to the following classes in our implementation:\n",
    "1. [TDParseMinus](../tdparse/models/tdparse.py), 2. [TDParse](../tdparse/models/tdparse.py), 3. [TDParsePlus](../tdparse/models/tdparse.py)\n",
    "\n",
    "The results reported are based on training and testing on the datasets of [Dong et al.](https://aclanthology.coli.uni-saarland.de/papers/P14-2009/p14-2009) and their own Election Twitter dataset which can be found [here](https://figshare.com/articles/EACL_2017_-_Multi-target_UK_election_Twitter_sentiment_corpus/4479563/1) as reported in the paper.\n",
    "\n",
    "The notebook is going to show the results of the original paper using our implementation. We are then going to look at the affects of lower casing and scaling with respect to those results as these parts of the implementation were not mentioned in the original paper but we show the importance of these implementation details.\n",
    "\n",
    "# Original Paper results\n",
    "For each of the datasets we require to find the C-value for the Support Vector Machine (SVM) as this was not reported in the paper but is an important hyper-parameter to tune for. Therefore the start of each dataset we are going to find the Best C value for each of the models keeping everything else constant. The defualt tokeniser is the [Ark tokeniser](http://ttic.uchicago.edu/~kgimpel/papers/gimpel+etal.acl11.pdf) of which we use this python port of the [tokeniser](https://github.com/Sentimentron/ark-twokenize-py). Also we default to lower casing all tokens as well. For the sentiment lexicon method **TDParse+** we use the following lexicons:\n",
    "1. [MPQA](http://mpqa.cs.pitt.edu/lexicons/subj_lexicon/), 2. [NRC](http://saifmohammad.com/WebPages/NRC-Emotion-Lexicon.htm), and 3. [Hu & Liu](https://www.cs.uic.edu/~liub/FBS/sentiment-analysis.html#lexicon)\n",
    "\n",
    "**NOTE** In the original paper they adopted approaches from *Vo and Zhang 2015* and have used the same pre-trained embeddings, pooling functions and tunned the C value over the same set of parameter values. However they did not state if they were going to use the same tokeniser *Ark tokeniser* or the same set of lexicons. However they did release their code which can be found [here](https://github.com/bluemonk482/tdparse) which shows that they do use the *Ark tokeniser* and the lexicons stated above which were the full set of lexicons in *Vo and Zhang 2015*.\n",
    "\n",
    "## Best C value Dong et al. Dataset\n",
    "\n",
    "We store the results of the C values as finding C values can take a long time > 3 hours using 5 processors and ~8GB of RAM. This is due to the fact we have to dependency parse each time we train and test as we do not cache pre-processing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Models\n",
    "models = [TDParseMinus, TDParse, TDParsePlus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TDParse Minus': '0.03125',\n",
       " 'TDParse': '0.0078125',\n",
       " 'TDParsePlus': '0.0078125'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Getting the best C Value for the Dong Twitter dataset\n",
    "c_values = [1] # include the default sklearn C Value\n",
    "for c in range(-15, 3, 2):\n",
    "    c_values.append(math.pow(2, c))\n",
    "    \n",
    "dong_best_c_fp = results_folder.joinpath('Dong Twitter Best C.json')\n",
    "dong_best_c = None\n",
    "# See if we have already got the result\n",
    "if dong_best_c_fp.is_file():\n",
    "    with dong_best_c_fp.open('r') as dong_best_c_file:\n",
    "        dong_best_c = json.load(dong_best_c_file)\n",
    "# If we have not we have to run each of the three models and then\n",
    "# save the result\n",
    "if dong_best_c is None:\n",
    "    model_kwargs = []\n",
    "    for model in models:\n",
    "        kwargs = {'word_vectors': [[dong_w2v, dong_sswe]],\n",
    "                  'parser': [dependency_parsers.tweebo]}\n",
    "        if model == TDParsePlus:\n",
    "            kwargs['senti_lexicon'] = [all_three_low]\n",
    "        model_kwargs.append((model, kwargs))\n",
    "    dong_best_c = SKLearnModel.models_best_parameter(model_kwargs,'C', c_values, \n",
    "                                                     X_dong_train, y_dong_train, \n",
    "                                                     n_cpus)\n",
    "    dong_best_c = {model.name(): c for model, c in dong_best_c.items()}\n",
    "    with dong_best_c_fp.open('w+') as dong_best_c_file:\n",
    "        json.dump(dong_best_c, dong_best_c_file)\n",
    "dong_best_c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best C value Elections dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TDParse Minus': '0.0078125',\n",
       " 'TDParse': '0.001953125',\n",
       " 'TDParsePlus': '0.001953125'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Getting the best C Value for the Election Twitter dataset\n",
    "c_values = [1] # include the default sklearn C Value\n",
    "for c in range(-15, 3, 2):\n",
    "    c_values.append(math.pow(2, c))\n",
    "    \n",
    "elec_best_c_fp = results_folder.joinpath('Election Twitter Best C.json')\n",
    "elec_best_c = None\n",
    "# See if we have already got the result\n",
    "if elec_best_c_fp.is_file():\n",
    "    with elec_best_c_fp.open('r') as elec_best_c_file:\n",
    "        elec_best_c = json.load(elec_best_c_file)\n",
    "# If we have not we have to run each of the three models and then\n",
    "# save the result\n",
    "if elec_best_c is None:\n",
    "    model_kwargs = []\n",
    "    for model in models:\n",
    "        kwargs = {'word_vectors': [[elec_w2v, elec_sswe]],\n",
    "                  'parser': [dependency_parsers.tweebo]}\n",
    "        if model == TDParsePlus:\n",
    "            kwargs['senti_lexicon'] = [all_three_low]\n",
    "        model_kwargs.append((model, kwargs))\n",
    "    elec_best_c = SKLearnModel.models_best_parameter(model_kwargs,'C', c_values, \n",
    "                                                     X_elec_train, y_elec_train, \n",
    "                                                     n_cpus)\n",
    "    elec_best_c = {model.name(): c for model, c in elec_best_c.items()}\n",
    "    with elec_best_c_fp.open('w+') as elec_best_c_file:\n",
    "        json.dump(elec_best_c, elec_best_c_file)\n",
    "elec_best_c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy and Macro F1 scores on the Datasets\n",
    "\n",
    "Below trains and tests on the two datasets and then displays the results comparing to the original results stated in the paper\n",
    "**NOTE** Takes around 23 minutes to run on 3 processors and no more than 3-5 GB of RAM to reduce the amount of RAM use fewer processors in the Pool Instances."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TDParse Minus, ({'TDParse Minus': '0.03125', 'TDParse': '0.0078125', 'TDParsePlus': '0.0078125'}, [w2v, sswe])\n",
      "0.03125\n",
      "TDParse, ({'TDParse Minus': '0.03125', 'TDParse': '0.0078125', 'TDParsePlus': '0.0078125'}, [w2v, sswe])\n",
      "0.0078125\n",
      "TDParsePlus, ({'TDParse Minus': '0.03125', 'TDParse': '0.0078125', 'TDParsePlus': '0.0078125'}, [w2v, sswe])\n",
      "0.0078125\n",
      "defaultdict(<function <lambda> at 0x7f6f43e3e488>, {'Dong Twitter': {'TDParse Minus': 0.58795142054996008}})\n",
      "defaultdict(<function <lambda> at 0x7f6f43e3e488>, {'Dong Twitter': {'TDParse Minus': 0.58795142054996008, 'TDParse': 0.66889543640346805}})\n",
      "defaultdict(<function <lambda> at 0x7f6f43e3e488>, {'Dong Twitter': {'TDParse Minus': 0.58795142054996008, 'TDParse': 0.66889543640346805, 'TDParsePlus': 0.68103783201584822}})\n",
      "TDParse Minus, ({'TDParse Minus': '0.0078125', 'TDParse': '0.001953125', 'TDParsePlus': '0.001953125'}, [w2v, sswe])\n",
      "0.0078125\n",
      "TDParse, ({'TDParse Minus': '0.0078125', 'TDParse': '0.001953125', 'TDParsePlus': '0.001953125'}, [w2v, sswe])\n",
      "0.001953125\n",
      "TDParsePlus, ({'TDParse Minus': '0.0078125', 'TDParse': '0.001953125', 'TDParsePlus': '0.001953125'}, [w2v, sswe])\n",
      "0.001953125\n",
      "defaultdict(<function <lambda> at 0x7f6f43e3e488>, {'Dong Twitter': {'TDParse Minus': 0.58795142054996008, 'TDParse': 0.66889543640346805, 'TDParsePlus': 0.68103783201584822}, 'Election Twitter': {'TDParse Minus': 0.37272079176784317}})\n",
      "defaultdict(<function <lambda> at 0x7f6f43e3e488>, {'Dong Twitter': {'TDParse Minus': 0.58795142054996008, 'TDParse': 0.66889543640346805, 'TDParsePlus': 0.68103783201584822}, 'Election Twitter': {'TDParse Minus': 0.37272079176784317, 'TDParse': 0.39269531150895132}})\n",
      "defaultdict(<function <lambda> at 0x7f6f43e3e488>, {'Dong Twitter': {'TDParse Minus': 0.58795142054996008, 'TDParse': 0.66889543640346805, 'TDParsePlus': 0.68103783201584822}, 'Election Twitter': {'TDParse Minus': 0.37272079176784317, 'TDParse': 0.39269531150895132, 'TDParsePlus': 0.41698121910543984}})\n",
      "1390.3772494792938\n"
     ]
    }
   ],
   "source": [
    "def model_eval(model, X_train, y_train, X_test, y_test):\n",
    "    model.fit(X_train, y_train)\n",
    "    predictions = model.predict(X_test)\n",
    "    acc = accuracy_score(y_test, predictions)\n",
    "    f1 = f1_score(y_test, predictions, average='macro')\n",
    "    return acc, f1\n",
    "\n",
    "\n",
    "dataset_model_acc = defaultdict(lambda: dict())\n",
    "dataset_model_f1 = defaultdict(lambda: dict())\n",
    "\n",
    "dong_params = (dong_best_c, [dong_w2v, dong_sswe])\n",
    "elec_params = (elec_best_c, [elec_w2v, elec_sswe])\n",
    "dataset_data_params = {'Dong Twitter': ((dong_train, dong_test), dong_params),\n",
    "                       'Election Twitter': ((election_train, election_test), elec_params)}\n",
    "for dataset_name, data_params in dataset_data_params.items():\n",
    "    data, params = data_params\n",
    "    train, test = data\n",
    "    X_train, y_train = train.data(), train.sentiment_data()\n",
    "    X_test, y_test = test.data(), test.sentiment_data()\n",
    "    \n",
    "    c_values, dataset_word_embeddings = params\n",
    "    model_instances_params = []\n",
    "    for model in models:\n",
    "        best_c = float(c_values[model.name()])\n",
    "        if model == TDParsePlus:\n",
    "            model_instance = model(dataset_word_embeddings,\n",
    "                                   dependency_parsers.tweebo,\n",
    "                                   C=best_c,\n",
    "                                   senti_lexicon = all_three_low)\n",
    "        else:\n",
    "            model_instance = model(dataset_word_embeddings, \n",
    "                                   dependency_parsers.tweebo,\n",
    "                                   C=best_c)\n",
    "        func_params = (X_train, y_train, X_test, y_test) \n",
    "        model_instances_params.append((model_instance, *func_params))\n",
    "    with Pool(3) as pool:\n",
    "        model_acc_f1 = pool.starmap(model_eval, model_instances_params)\n",
    "    for i, model in enumerate(models):\n",
    "        acc, f1 = model_acc_f1[i]\n",
    "        dataset_model_acc[dataset_name][model.name()] = acc\n",
    "        dataset_model_f1[dataset_name][model.name()] = f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are now going to add the scores from the paper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_acc = {'Election Twitter (Paper)': {'TDParse Minus': 0.5253,\n",
    "                                          'TDParse': 0.5645,\n",
    "                                          'TDParsePlus': 0},\n",
    "             'Dong Twitter (Paper)': {'TDParse Minus': 0.6170,\n",
    "                                      'TDParse': 0.7100,\n",
    "                                      'TDParsePlus': 0.7250}}\n",
    "paper_f1 = {'Election Twitter (Paper)': {'TDParse Minus': 0.4271,\n",
    "                                          'TDParse': 0.4609,\n",
    "                                          'TDParsePlus': 0},\n",
    "             'Dong Twitter (Paper)': {'TDParse Minus': 0.5700,\n",
    "                                      'TDParse': 0.6840,\n",
    "                                      'TDParsePlus': 0.7030}}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Macro F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dong Twitter</th>\n",
       "      <th>Dong Twitter (Paper)</th>\n",
       "      <th>Election Twitter</th>\n",
       "      <th>Election Twitter (Paper)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TDParse</th>\n",
       "      <td>66.89</td>\n",
       "      <td>68.4</td>\n",
       "      <td>39.27</td>\n",
       "      <td>46.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TDParse Minus</th>\n",
       "      <td>58.80</td>\n",
       "      <td>57.0</td>\n",
       "      <td>37.27</td>\n",
       "      <td>42.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TDParsePlus</th>\n",
       "      <td>68.10</td>\n",
       "      <td>70.3</td>\n",
       "      <td>41.70</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Dong Twitter  Dong Twitter (Paper)  Election Twitter  \\\n",
       "TDParse               66.89                  68.4             39.27   \n",
       "TDParse Minus         58.80                  57.0             37.27   \n",
       "TDParsePlus           68.10                  70.3             41.70   \n",
       "\n",
       "               Election Twitter (Paper)  \n",
       "TDParse                           46.09  \n",
       "TDParse Minus                     42.71  \n",
       "TDParsePlus                        0.00  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_f1_scores = {**dataset_model_f1, **paper_f1}\n",
    "all_f1_scores = (pd.DataFrame(all_f1_scores) * 100).round(2)\n",
    "all_f1_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dong Twitter</th>\n",
       "      <th>Dong Twitter (Paper)</th>\n",
       "      <th>Election Twitter</th>\n",
       "      <th>Election Twitter (Paper)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TDParse</th>\n",
       "      <td>69.65</td>\n",
       "      <td>71.0</td>\n",
       "      <td>55.02</td>\n",
       "      <td>56.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TDParse Minus</th>\n",
       "      <td>63.15</td>\n",
       "      <td>61.7</td>\n",
       "      <td>52.85</td>\n",
       "      <td>52.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TDParsePlus</th>\n",
       "      <td>70.52</td>\n",
       "      <td>72.5</td>\n",
       "      <td>55.06</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Dong Twitter  Dong Twitter (Paper)  Election Twitter  \\\n",
       "TDParse               69.65                  71.0             55.02   \n",
       "TDParse Minus         63.15                  61.7             52.85   \n",
       "TDParsePlus           70.52                  72.5             55.06   \n",
       "\n",
       "               Election Twitter (Paper)  \n",
       "TDParse                           56.45  \n",
       "TDParse Minus                     52.53  \n",
       "TDParsePlus                        0.00  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_acc_scores = {**dataset_model_acc, **paper_acc}\n",
    "all_acc_scores = (pd.DataFrame(all_acc_scores) * 100).round(2)\n",
    "all_acc_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From what we can see above we are very close to the scores from the original paper and further more we have the same method ordering as in **TDParse+** > **TDParse** > **TDParse-**. Therefore we have successfully reproduced this paper.\n",
    "\n",
    "The election dataset for the Macro F1 metric is by far the worse score differences. This could be due to the fact that we could not process the whole dataset and are therefore 688 examples/samples smaller than the dataset qouted in the original paper.\n",
    "\n",
    "Finally you can see that **TDParse+** was not run on the Election dataset from the original paper but we can show here how it is still better than the non-sentiment version **TDParse** showing that using a range of sentiment lexicons is still effective in the Political domain as well as the more general domain of the Dong et al. dataset.\n",
    "\n",
    "## The affect of scaling\n",
    "\n",
    "Above we have shown how we have successfully reproduced the methods described in the paper. We are now going to show the one parameter we found that causes a large difference in the result, of which the parameter was not mentioned in the original paper.\n",
    "\n",
    "We show below the affect of scaling the word vectors on the different methods. To scale we use [MinMax](http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html) scaling. They did use a different Support Vector Machine Library [LibLinear](https://www.csie.ntu.edu.tw/~cjlin/liblinear/) however we actually use the [Scikit-learn interface](http://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html) to this library. Even though in the [Practical guide to LibLinear](https://www.csie.ntu.edu.tw/~cjlin/papers/guide/guide.pdf) it states that you should scale and they also show this in their [README.md](https://github.com/bluemonk482/tdparse) of their code base but not in the paper. We show the importance of scaling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_eval(model, X_train, y_train, X_test, y_test):\n",
    "    model.fit(X_train, y_train)\n",
    "    predictions = model.predict(X_test)\n",
    "    acc = accuracy_score(y_test, predictions)\n",
    "    f1 = f1_score(y_test, predictions, average='macro')\n",
    "    return acc, f1\n",
    "\n",
    "\n",
    "scale_dataset_model_acc = defaultdict(lambda: dict())\n",
    "scale_dataset_model_f1 = defaultdict(lambda: dict())\n",
    "\n",
    "dong_params = (dong_best_c, [dong_w2v, dong_sswe])\n",
    "elec_params = (elec_best_c, [elec_w2v, elec_sswe])\n",
    "dataset_data_params = {'Dong Twitter': ((dong_train, dong_test), dong_params),\n",
    "                       'Election Twitter': ((election_train, election_test), elec_params)}\n",
    "for dataset_name, data_params in dataset_data_params.items():\n",
    "    data, params = data_params\n",
    "    train, test = data\n",
    "    X_train, y_train = train.data(), train.sentiment_data()\n",
    "    X_test, y_test = test.data(), test.sentiment_data()\n",
    "    \n",
    "    c_values, dataset_word_embeddings = params\n",
    "    model_instances_params = []\n",
    "    for model in models:\n",
    "        best_c = float(c_values[model.name()])\n",
    "        # The main difference to the code from 4 cells \n",
    "        # back is the `scale` parameter in the model instances\n",
    "        if model == TDParsePlus:\n",
    "            model_instance = model(dataset_word_embeddings,\n",
    "                                   dependency_parsers.tweebo,\n",
    "                                   C=best_c, scale=None,\n",
    "                                   senti_lexicon = all_three_low)\n",
    "        else:\n",
    "            model_instance = model(dataset_word_embeddings, \n",
    "                                   dependency_parsers.tweebo,\n",
    "                                   C=best_c, scale=None)\n",
    "        func_params = (X_train, y_train, X_test, y_test) \n",
    "        model_instances_params.append((model_instance, *func_params))\n",
    "    with Pool(3) as pool:\n",
    "        model_acc_f1 = pool.starmap(model_eval, model_instances_params)\n",
    "    for i, model in enumerate(models):\n",
    "        acc, f1 = model_acc_f1[i]\n",
    "        scale_dataset_name = f'{dataset_name} (not scaled)'\n",
    "        scale_dataset_model_acc[scale_dataset_name][model.name()] = acc\n",
    "        scale_dataset_model_f1[scale_dataset_name][model.name()] = f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Macro F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dong Twitter</th>\n",
       "      <th>Dong Twitter (Paper)</th>\n",
       "      <th>Dong Twitter (not scaled)</th>\n",
       "      <th>Election Twitter</th>\n",
       "      <th>Election Twitter (Paper)</th>\n",
       "      <th>Election Twitter (not scaled)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TDParse</th>\n",
       "      <td>66.89</td>\n",
       "      <td>68.4</td>\n",
       "      <td>51.33</td>\n",
       "      <td>39.27</td>\n",
       "      <td>46.09</td>\n",
       "      <td>38.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TDParse Minus</th>\n",
       "      <td>58.80</td>\n",
       "      <td>57.0</td>\n",
       "      <td>55.32</td>\n",
       "      <td>37.27</td>\n",
       "      <td>42.71</td>\n",
       "      <td>41.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TDParsePlus</th>\n",
       "      <td>68.10</td>\n",
       "      <td>70.3</td>\n",
       "      <td>40.90</td>\n",
       "      <td>41.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>33.39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Dong Twitter  Dong Twitter (Paper)  Dong Twitter (not scaled)  \\\n",
       "TDParse               66.89                  68.4                      51.33   \n",
       "TDParse Minus         58.80                  57.0                      55.32   \n",
       "TDParsePlus           68.10                  70.3                      40.90   \n",
       "\n",
       "               Election Twitter  Election Twitter (Paper)  \\\n",
       "TDParse                   39.27                     46.09   \n",
       "TDParse Minus             37.27                     42.71   \n",
       "TDParsePlus               41.70                      0.00   \n",
       "\n",
       "               Election Twitter (not scaled)  \n",
       "TDParse                                38.73  \n",
       "TDParse Minus                          41.75  \n",
       "TDParsePlus                            33.39  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_scale_f1_scores = {**dataset_model_f1, **scale_dataset_model_f1, \n",
    "                       **paper_f1}\n",
    "all_scale_f1_scores = (pd.DataFrame(all_scale_f1_scores) * 100).round(2)\n",
    "all_scale_f1_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dong Twitter</th>\n",
       "      <th>Dong Twitter (Paper)</th>\n",
       "      <th>Dong Twitter (not scaled)</th>\n",
       "      <th>Election Twitter</th>\n",
       "      <th>Election Twitter (Paper)</th>\n",
       "      <th>Election Twitter (not scaled)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TDParse</th>\n",
       "      <td>69.65</td>\n",
       "      <td>71.0</td>\n",
       "      <td>55.06</td>\n",
       "      <td>55.02</td>\n",
       "      <td>56.45</td>\n",
       "      <td>46.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TDParse Minus</th>\n",
       "      <td>63.15</td>\n",
       "      <td>61.7</td>\n",
       "      <td>58.82</td>\n",
       "      <td>52.85</td>\n",
       "      <td>52.53</td>\n",
       "      <td>52.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TDParsePlus</th>\n",
       "      <td>70.52</td>\n",
       "      <td>72.5</td>\n",
       "      <td>42.49</td>\n",
       "      <td>55.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>37.27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Dong Twitter  Dong Twitter (Paper)  Dong Twitter (not scaled)  \\\n",
       "TDParse               69.65                  71.0                      55.06   \n",
       "TDParse Minus         63.15                  61.7                      58.82   \n",
       "TDParsePlus           70.52                  72.5                      42.49   \n",
       "\n",
       "               Election Twitter  Election Twitter (Paper)  \\\n",
       "TDParse                   55.02                     56.45   \n",
       "TDParse Minus             52.85                     52.53   \n",
       "TDParsePlus               55.06                      0.00   \n",
       "\n",
       "               Election Twitter (not scaled)  \n",
       "TDParse                                46.56  \n",
       "TDParse Minus                          52.30  \n",
       "TDParsePlus                            37.27  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_scale_acc_scores = {**dataset_model_acc, **scale_dataset_model_acc,\n",
    "                        **paper_acc}\n",
    "all_scale_acc_scores = (pd.DataFrame(all_scale_acc_scores) * 100).round(2)\n",
    "all_scale_acc_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are now going to plot the data as it is shown in the paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seaborn_data(dataset_model_metric, metric_name):\n",
    "    seaborn_data = {'Model': [], 'Legend': [], f'{metric_name}': [],\n",
    "                    'Dataset': []}\n",
    "    for dataset_name, model_metric in dataset_model_metric.items():\n",
    "        num_models = len(model_metric)\n",
    "        if 'scaled' in dataset_name:\n",
    "            seaborn_data['Legend'].extend(['Not Scaled'] * num_models)\n",
    "        elif 'Paper' in dataset_name:\n",
    "            seaborn_data['Legend'].extend(['Original'] * num_models)\n",
    "        else:\n",
    "            seaborn_data['Legend'].extend(['Reproduction'] * num_models)\n",
    "        if 'Dong' in dataset_name:\n",
    "            seaborn_data['Dataset'].extend(['Dong Twitter'] * num_models)\n",
    "        else:\n",
    "            seaborn_data['Dataset'].extend(['Election Twitter'] * num_models)\n",
    "        for model, metric in model_metric.items():\n",
    "            seaborn_data['Model'].append(model)\n",
    "            seaborn_data[f'{metric_name}'].append(round(metric * 100, 2))\n",
    "    return seaborn_data\n",
    "\n",
    "\n",
    "# Need to transform the data into something that suits seaborn better\n",
    "acc_data = {**dataset_model_acc, **scale_dataset_model_acc, **paper_acc}\n",
    "acc_data = pd.DataFrame(seaborn_data(acc_data, 'Accuracy'))\n",
    "f1_data = {**dataset_model_f1, **scale_dataset_model_f1, **paper_f1}\n",
    "f1_data = pd.DataFrame(seaborn_data(f1_data, 'F1'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtQAAAGGCAYAAABbgxaRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3X28XeOZ+P/P1QTxECKkSoOkqIdGEnGoZyqTGm1Ka1TNKIKRUlQNrfrym6ZPM22jo6VMG0WVqKJFqx1DaQajLYkGIYziqEQQKkpT5OH6/bFX4uQ4kX3O2mftc04+79frvM5e93q4r733yZVr3/tea0VmIkmSJKlr3tHsACRJkqTezIJakiRJKsGCWpIkSSrBglqSJEkqwYJakiRJKsGCWpIkSSrBglpqgIhYKyJejYjNmh2LJDVSREyIiLu64bhHRMQtjT5uI5jT1VkW1GqKiGiNiL9FxCsRsSAi7o6IEyKi0r/JIqG/Wvz8LSKWtll+td7jZObrmbleZj5THPfqiDinXV/PRsRejX4OklRWm5z8apuf7zbw+MMiIiOi/7K2zJyamR9sVB9FP+Z0NYUFtZrpI5k5ENgS+DpwJnBJlQEUCX29zFwPOBB4Ztly0dYjRMQ7qv6wIWm185G2+S8zT252QJ1lTlez+Gaq6TLz5cz8OfAJ4OiIGAEQERtExI8iYn5EPBUR5yxLQMu+goyIcyPipYh4MiIOXHbMiBgeEXcUI+C/jogLI+LKzsYWESdGxLVtlp+OiCvaLD8fEdtFxIBi9GVoRHwG+Afg/ytGRa4tjvFO4Jai7TPF/ntHxO+LUfr7ImLPNsf+XUR8OSJ+DywE/OpRUtMVOe/WiPhzRDwaEYe1Wbd2RHyryNkvF3l6beCOYpMFRQ7cvf1UkojYIyLuLfa7NyL2aLNuWkR8JSL+t8jrt0TExl2I3ZyubmFBrR4jM+8B5gB7F00XABsA7wH2BY4Cjmmzy/uBR4GNgW8Cl0REFOuuAu4BNgImAUd2Maz/KfomIt4DLAb2KpZ3AJZm5iPtnsf5wE+BrxSjIh/PzI8DzwMfLNrOj4hhwA3A2cBg4BzghojYsM3hPlk874HAs118DpLUEBGxLnArtRz7TuBw4KIiHwKcC+wM7EEtr30eWArsU6wfVOTA37Y77mDgl8D51PL2fwC/jIiN2mz2T9T+D3gnsCZwRheegjld3cKCWj3NM8DgiOhHLVGflZmvZGYr8C1WLIyfysyLM3MJcDmwKbBJRGwB7AL8a2a+kZl3AT/vSjCZ+TAsT7T7AL8AXikS5768OerSFUcDP8vMX2fm0sz8FfAw0HZO4Q8y89HMXJSZi0v0JUmrckMxsrrs5/gOthkPtGbmZZm5ODP/QK3Y/HjxDeKxwKmZOTczl2Tm3Zn5eh19fxh4LDOvKI77Y+AR4CNttrksM/8vM/8GXAOM7uwTNKeru/Rf9SZSpd4N/JnaqPMawFNt1j1VrF9m+af7zFxYDE6vV+z758xc2Gbbp4HNuxjTHcB+QAu1ERSoJd59qY12dNWWwD9GxMfbtK3Bil8DPl3i+JLUGR/NzF+vYpstgfdHxII2bf2BK6jl3gHA413oezNWzPfwNjmf2pSJrs6JNqer4Syo1WNExC7UkuddwAvAImoJ6uFiky2AuXUcah61Ue512hTVXS2moZZg9wN2onbiJNRGU/YFvrqSfbKOtqepjVac8jZ9d3QcSWqWp4H/ycxx7VcUI9SvAVsB97dbvapc9gy1fN/WFsDNXYzz7ZjT1XBO+VDTRcT6ETEeuBq4MjMfLKZxXAN8LSIGRsSWwL8AqzyxMDOfAqYDkyJizYjYnRW/Nuys/wH+HngjM+cXy/9AbeThoZXs8xy1ud9v13Y5ta9Jx0ZEv+JknrER8a4SsUpSd7oJeG9EHBkRaxQ/u0TE9pm5FLgU+I+I2KzIa7tHxFrAfGpzqdvnxWV+VRz3nyKif0R8Atih6K/RzOlqOAtqNdMvIuIVap/qz6Z2Ekrbkw5PAf4KPEFt1Poqasm6HkcAuwMvUhtx+AlQzzy+jjxIbbT8DoDMfKGI+c7MXNlowxRgl2Ie4tVF29eofUBYEBEnZ+YT1JL4l6iNyD8FnIr/LiU1xy9ixetQX99+g8x8hdqc4MOpjSo/C3wDWKvY5AxqOfNeatP3vgG8o/i28GvA/xY5cLd2x32R2vzs06nl7c8D44t822jmdDVcrPxvR+o7IuInwCOZ+cVmxyJJkvoWPzWpTyq+gtwqahfP/3vgYGqXM5IkSWqoSk5KjIhtqX3lvsx7gH8FBgHHU5tbBfD/isvMSGW9C/gZteuZzgFOLC7vJEmS1FCVT/kori88l9pNOY4BXs3McysNQpIkSWqQZkz5GAs8XlyJQZIkSerVmlFQHw78uM3yyRHxQERc2u72nJIkSVKPV+mUj4hYk9pldt6Xmc9FxCbULi2TwFeATTPz2A72mwhMBFh33XV33m677SqLWZIaZcaMGS9k5pBmx1GljTfeOIcNG9bsMCSpS+rN21XfKfFA4L7MfA5g2W+AiLiYlVzAPTOnULsGJC0tLTl9+vQKQpWkxoqI1W6q27BhwzBnS+qt6s3bVU/5+EfaTPeIiE3brPsYMKvieCRJkqRSKhuhjoh1gXHAp9o0fzMiRlOb8tHabp0kSZLU41VWUGfmX6ldE7ht25FV9S9JkiR1h6rnUHeLRYsWMWfOHF577bVmh6I2BgwYwNChQ1ljjTWaHYokSVK36RMF9Zw5cxg4cCDDhg0jIpodjoDM5MUXX2TOnDkMHz682eFIkiR1m2Zch7rhXnvtNTbaaCOL6R4kIthoo4381kCSJPV5faKgBiymeyDfE0mStDroMwV1s/Xr14/Ro0czYsQIPvKRj7BgwYJK+29tbWXEiBFd2nfatGncfffdy5e/973v8aMf/ahRoUmSJPVpfWIOdXubnHJJQ4/33AXHrXKbtddem5kzZwJw9NFHc+GFF3L22WeX7nvJkiX069ev9HHezrRp01hvvfXYY489ADjhhBO6tT9JkqS+xBHqbrD77rszd+7c5cuTJ09ml112YeTIkXzxi18EaiPK2223HUcccQTbb789hx56KAsXLgRqdxY788wzGTNmDNdeey0zZ85kt912Y+TIkXzsYx/jpZdeAmDGjBmMGjWKUaNGceGFFy7v74c//CEnn3zy8uXx48czbdo0AG6++WbGjBnDqFGjGDt2LK2trXzve9/jvPPOY/To0dx5551MmjSJc889F2Clfe+3336ceeaZ7Lrrrrz3ve/lzjvv7L4XVJIkqQezoG6wJUuWcNttt3HQQQcBcMstt/DYY49xzz33MHPmTGbMmMEdd9wBwKOPPsqnP/1pZs+ezfrrr89FF120/DgbbbQR9913H4cffjhHHXUU3/jGN3jggQfYcccd+dKXvgTAMcccwwUXXMD9999fV2zz58/n+OOP56c//Sn3338/1157LcOGDeOEE07gtNNOY+bMmey9994r7LOyvgEWL17MPffcw7e//e0V2iVJklYnFtQN8re//Y3Ro0fzrne9i+eee45x48YBtYL6lltuYaeddmLMmDE88sgjPPbYYwBsvvnm7LnnngB88pOf5K677lp+vE984hMAvPzyyyxYsIB9990XqE0nueOOO1iwYAELFixgn332AeDII1d9j5zf/e537LPPPssvYzd48OC33X5lfS9zyCGHALDzzjvT2tq6yv4lSZL6IgvqBlk2h/qpp54iM5dPwchMzjrrLGbOnMnMmTP54x//yHHH1eZkt78KRtvlddddt8ux9O/fn6VLly5f7q5L16211lpA7YTMxYsXd0sfkiRJPZ0FdYOts846nH/++XzrW99i8eLFHHDAAVx66aW8+uqrAMydO5fnn38egD/96U/89re/BeCqq65ir732esvxNthgAzbccMPlc5SvuOIK9t13XwYNGsSgQYOWj2pPnTp1+T7Dhg1j5syZLF26lKeffpp77rkHgN1224077riDJ598EoA///nPAAwcOJBXXnml7r4lSZL0pj55lY9m22mnnRg5ciQ//vGPOfLII5k9eza77747AOuttx5XXnkl/fr1Y9ttt+XCCy/k2GOPZYcdduDEE0/s8HiXX345J5xwAgsXLuQ973kPl112GQCXXXYZxx57LBHBBz/4weXb77nnngwfPpwddtiB7bffnjFjxgAwZMgQpkyZwiGHHMLSpUt55zvfya233spHPvIRDj30UG688UYuuOCCuvqWJElSTWRms2PolJaWlpw+ffoKbbNnz2b77bdvUkRd09rayvjx45k1a1azQ+lWvfG9kbpLRMzIzJZmx1GljnK2JPUW9eZtp3xIkiRJJVhQN8mwYcP6/Oi0JEnS6sCCWpIkSSrBglqSJEkqwat8SJKk1cqtxw6trK9xl86prC81jwW1pD7J/zAlSVVxykeDRASnn3768uVzzz2XSZMmve0+N9xwAw8//HCH6x599FH2228/Ro8ezfbbb8/EiRO7FNeECRO47rrr6t6+tbWVESNGdKkvSZKk1VGfHKEeMvmkhh5v/ucuXOU2a621Fj/72c8466yz2Hjjjes67g033MD48ePZYYcd3rLuM5/5DKeddhoHH3wwAA8++GDngpYkSVIlHKFukP79+zNx4kTOO++8t6xrbW1l//33Z+TIkYwdO5Y//elP3H333fz85z/nc5/7HKNHj+bxxx9fYZ958+YxdOibX1nvuOOOACxZsoQzzjiDESNGMHLkyOV3Nvzyl7/MLrvswogRI5g4cSId3bBnxowZ7Lvvvuy8884ccMABzJs3b3n7qFGjGDVqFBdeuOoPD5IkSXqTBXUDnXTSSUydOpWXX355hfZTTjmFo48+mgceeIAjjjiCz3zmM+yxxx4cdNBBTJ48mZkzZ7LVVlutsM9pp53G/vvvz4EHHsh5553HggULAJgyZQqtra3MnDlz+fEATj75ZO69915mzZrF3/72N2666aYVjrdo0SJOOeUUrrvuOmbMmMGxxx7L2WefDcAxxxzDBRdcwP33399dL40kSVKfZUHdQOuvvz5HHXUU559//grtv/3tb/mnf/onAI488kjuuuuuVR7rmGOOYfbs2Xz84x9n2rRp7Lbbbrz++uv8+te/5lOf+hT9+9dm6wwePBiA3/zmN7z//e9nxx135Pbbb+ehhx5a4XiPPvoos2bNYty4cYwePZqvfvWrzJkzhwULFrBgwQL22Wef5fFJkiSpfn1yDnUzffazn2XMmDEcc8wxpY+12Wabceyxx3LssccyYsSIld5Z8bXXXuPTn/4006dPZ/PNN2fSpEm89tprK2yTmbzvfe/jt7/97Qrty0a+JUmS1DWOUDfY4MGDOeyww7jkkkuWt+2xxx5cffXVAEydOpW9994bgIEDB/LKK690eJybb76ZRYsWAfDss8/y4osv8u53v5tx48bx/e9/n8WLFwPw5z//eXnxvPHGG/Pqq692eFWPbbfdlvnz5y8vqBctWsRDDz3EoEGDGDRo0PJR86lTpzbiZZAkSVptWFB3g9NPP50XXnhh+fIFF1zAZZddxsiRI7niiiv4zne+A8Dhhx/O5MmT2Wmnnd5yUuItt9zCiBEjGDVqFAcccACTJ0/mXe96F//8z//MFltswciRIxk1ahRXXXUVgwYN4vjjj2fEiBEccMAB7LLLLm+Jac011+S6667jzDPPZNSoUYwePZq7774bgMsuu4yTTjqJ0aNHd3gyoyRJklYuelsB1dLSktOnT1+hbfbs2Wy//fZNikhvx/dGzdITb+wSETMys6Wbw+lROsrZUrP1xPygnqnevO0ItSRJklSCBbUkSZJUggW1JEmSVIIFtSRJklSC16GWGsATXCRJWn05Qi1JkiSVYEHdIHPmzOHggw9mm222YauttuLUU0/ljTfeeMt2zzzzDIceeugqj/ehD32oy3cxnDRpEueee26X9pUkgIhojYgHI2JmREwv2iZFxNyibWZEfKjZcUpST9Anp3w0+uv3VX3FnpkccsghnHjiidx4440sWbKEiRMncvbZZzN58uTl2y1evJjNNtuswzsZtverX/2qdNySVNIHMvOFdm3nZaaf2CWpjT5ZUFft9ttvZ8CAARxzzDEA9OvXj/POO4/hw4czfPhwbr75Zl599VWWLFnC5Zdfzvjx45k1axYLFy5kwoQJzJo1i2233ZZnnnmGCy+8kJaWFoYNG8b06dN59dVXOfDAA9lrr724++67efe7382NN97I2muvzcUXX8yUKVN444032HrrrbniiitYZ511mvxqqLfZ5JRLKuvruQuOq6wvSeqLzNk9kwV1Azz00EPsvPPOK7Stv/76bLHFFixevJj77ruPBx54gMGDB9Pa2rp8m4suuogNN9yQhx9+mFmzZjF69OgOj//YY4/x4x//mIsvvpjDDjuMn/70p3zyk5/kkEMO4fjjjwfgnHPO4ZJLLuGUU07ptucpabWSwC0RkcD3M3NK0X5yRBwFTAdOz8yX2u8YEROBiQBbbLFFVfGqG1i8SfWpZA51RGzbZs7dzIj4S0R8NiIGR8StEfFY8XvDKuKp2rhx4xg8ePBb2u+66y4OP/xwAEaMGMHIkSM73H/48OHLi+2dd955eVE+a9Ys9t57b3bccUemTp3KQw891D1PQNLqaK/MHAMcCJwUEfsA/wlsBYwG5gHf6mjHzJySmS2Z2TJkyJDKApakZqmkoM7MRzNzdGaOBnYGFgLXA18AbsvMbYDbiuVeZ4cddmDGjBkrtP3lL3/hT3/6E/3792fdddctdfy11lpr+eN+/fqxePFiACZMmMB3v/tdHnzwQb74xS/y2muvlepHkpbJzLnF7+ep5etdM/O5zFySmUuBi4FdmxmjJPUUzbjKx1jg8cx8CjgYuLxovxz4aBPiKW3s2LEsXLiQH/3oRwAsWbKE008/nQkTJrztnOY999yTa665BoCHH36YBx98sFP9vvLKK2y66aYsWrSIqVOndv0JSFIbEbFuRAxc9hj4IDArIjZts9nHgFnNiE+SeppmFNSHAz8uHm+SmfOKx88Cm3S0Q0RMjIjpETF9/vz5VcTYKRHB9ddfz7XXXss222zDe9/7XgYMGMC//du/ve1+n/70p5k/fz477LAD55xzDu973/vYYIMN6u73K1/5Cu9///vZc8892W677co+DUlaZhPgroi4H7gH+GVm3gx8s7iU3gPAB4DTmhmkJPUUlZ6UGBFrAgcBZ7Vfl5lZnPzyFsXJMFMAWlpaOtymrWbcSW7zzTfnF7/4xVvaJ0yYwIQJE5YvDxs2jFmzaoM6AwYM4Morr2TAgAE8/vjj/N3f/R1bbrklwPJ50htvvPHy7QHOOOOM5Y9PPPFETjzxxLf0OWnSpAY8I0mrq8x8AhjVQfuRTQhHknq8qq/ycSBwX2Y+Vyw/FxGbZua84qvE5yuOp6kWLlzIBz7wARYtWkRmctFFF7Hmmms2OyxJkiR1QtUF9T/y5nQPgJ8DRwNfL37fWHE8TTVw4ECmT5/e7DAq5SWYJElSX1PZHOrixJZxwM/aNH8dGBcRjwF/VyxLkiRJvUZlI9SZ+Vdgo3ZtL1K76kcjjk9ENOJQapDMVU53lyRJ6vWacZWPhhswYAAvvviiBVwPkpm8+OKLDBgwoNmhSJIkdas+cevxoUOHMmfOHHriJfVWZwMGDGDo0KHNDkOSJKlb9YmCeo011mD48OHNDkOSJEmroT4x5UOSJElqFgtqSZIkqQQLakmSJKkEC2pJkiSpBAtqSZIkqQQLakmSJKkEC2pJkiSpBAtqSZIkqQQLakmSJKkEC2pJkiSpBAtqSZIkqQQLakmSJKkEC2pJkiSpBAtqSZIkqQQLakmSJKkEC2pJkiSpBAtqSZIkqQQLakmSJKmE/s0OQJKkTU65pLK+nrvguMr6krR6cIRakiRJKsGCWpIkSSrBglqSJEkqwTnUkiR1k1uPHVpZX+MunVNZX5JW5Ai1JEmSVIIFtSRJklSCUz4kSauVIZNPqqyvqyrrSVIzOUItSZIklWBBLUmSJJVgQS1JkiSVYEEtSZIklWBBLUmSJJVgQS1JkiSVYEEtSZIklWBBLUmSJJVQWUEdEYMi4rqIeCQiZkfE7hExKSLmRsTM4udDVcUjSZIkNUKVd0r8DnBzZh4aEWsC6wAHAOdl5rkVxiFJkiQ1TCUFdURsAOwDTADIzDeANyKiiu4lSZKkblPVlI/hwHzgsoj4Q0T8ICLWLdadHBEPRMSlEbFhRfFIkiRJDVFVQd0fGAP8Z2buBPwV+ALwn8BWwGhgHvCtjnaOiIkRMT0ips+fP7+ikCVJkqRVq6qgngPMyczfF8vXAWMy87nMXJKZS4GLgV072jkzp2RmS2a2DBkypKKQJUmSpFWrpKDOzGeBpyNi26JpLPBwRGzaZrOPAbOqiEeSJElqlCqv8nEKMLW4wscTwDHA+RExGkigFfhUhfFIkiRJpVVWUGfmTKClXfORVfUvSZIkdQfvlChJkiSVYEEtSZIklWBBLUmSJJVgQS1JkiSVYEEtSZIklWBBLUmSJJVgQS1JkiSVYEEtSZIklWBBLUmSJJVgQS1JkiSVYEEtSZIklWBBLUmSJJXQv9kBdJdNTrmksr6eu+C4yvqSpCpERCvwCrAEWJyZLRExGPgJMAxoBQ7LzJeaFaMk9RSOUEuSVuYDmTk6M1uK5S8At2XmNsBtxbIkrfYsqCVJ9ToYuLx4fDnw0SbGIkk9hgW1JKkjCdwSETMiYmLRtklmzisePwts0tGOETExIqZHxPT58+dXEaskNVWfnUMtSSplr8ycGxHvBG6NiEfarszMjIjsaMfMnAJMAWhpaelwG0nqSxyhliS9RWbOLX4/D1wP7Ao8FxGbAhS/n29ehJLUc1hQS5JWEBHrRsTAZY+BDwKzgJ8DRxebHQ3c2JwIJalnccqHJKm9TYDrIwJq/09clZk3R8S9wDURcRzwFHBYE2OUpB7DglqStILMfAIY1UH7i8DY6iOSpJ7NKR+SJElSCRbUkiRJUgkW1JIkSVIJFtSSJElSCRbUkiRJUgle5aOXufXYoZX1Ne7SOZX1JUmS1Fs5Qi1JkiSVYEEtSZIklWBBLUmSJJVgQS1JkiSV0KmTEiOiP3AsMBJ4ApiSma92R2CSpHLM2ZJUjc6OUH8b2AV4ANgZuKbhEUmSGsWcLUkVeNsR6oj4LHB+Zi4tmnbMzH2LdZcB87s5PklSnczZktQcqxqhXhu4KyJ2K5bvioj/ioivArcC/9Wt0UmSOsOcLUlN8LYj1Jn57xFxNfCdiJgPfB7YFdgRuBD4WfeHKEmqhzlbkppjlSclZuaTwEER8VHg18B/ZuY3uz0ySVKnmbMlqXpvO+UjItaLiH+JiO8CmwJjga0j4s6IGF1JhJKkupizJak5VjVCfQ3wF+A3wAeAkZl5YkTsCFwQEQ9m5in1dBQRg4AfACOApHYpp0eBnwDDgFbgsMx8qQvPQ5LUwJwtSarfqgrq3YGNMnNpcYb4vQCZ+SCwX0Qc04m+vgPcnJmHRsSawDrA/wNuy8yvR8QXgC8AZ3b6WUiSoLE5W5JUp1UV1P8NXBER04D9gV+1XZmZl9XTSURsAOwDTCj2ewN4IyIOBvYrNrscmIYFtSR1VUNytiSpc1Z12bwjgBuADYErMvOsLvYznNr1Ty+LiD9ExA8iYl1gk8ycV2zzLLBJRztHxMSImB4R0+fP9zKqkrQSjcrZkqROWNVl85YA1zaonzHAKZn5+4j4DrXpHW37yojIlcQxBZgC0NLS0uE2krS6a2DOliR1QmdvPd5Vc4A5mfn7Yvk6agX2cxGxKUDx+/mK4pEkSZIaopKCOjOfBZ6OiG2LprHAw8DPgaOLtqOBG6uIR5IkSWqUVd7YpYFOAaYWV/h4AjiGWkF/TUQcBzwFHFZhPJIkSVJpdRXUEXEqMDUzX+hqR5k5E2jpYNXYrh5TkvRWjcjZkqT61TvlY3+gNSJuiohPRMRa3RmUJKkUc7YkVaiugjozDwa2BP4L+CzwbHHpu326MzhJUueZsyWpWnWflJiZL2bmhZm5O7AvsAvwm4hojYizI2K9botSktQp5mxJqk6nrvIREWOL29lOA54DjgKOBHaiNhIiSeohzNmSVI16T0o8FzgceBn4EXBOZs5ts/53wEvdEqEkqVPM2ZJUrXovmzcA+Fhm3tvRysxcFBEdXcFjtTBk8kmV9XVVZT1J6sXM2ZJUoXoL6n8HFrZtiIgNgbUz8xmAzHykwbFJkrrGnC1JFap3DvUNwNB2bUOB6xsbjiSpAczZklShegvqbTPzwbYNxfJ2jQ9JklSSOVuSKlRvQf18RGzdtqFYfrHxIUmSSjJnS1KF6i2oLwV+GhHjI2KHiPgIcB3wg+4LTZLUReZsSapQvSclfh1YBJwLbA48TS0x/0c3xSVJ6jpztiRVqK6COjOXApOLH0lSD2bOlqRq1TtCTUSsCWwLbAzEsvbMvL0b4pIklWDOlqTq1HunxL2Aa4G1gPWBvwADqX2N+J5ui06S1GnmbEmqVr0nJZ4HfDMzBwOvFL+/AlzUbZFJkrrKnC1JFaq3oH4v8J12bV8HTmtsOJKkBjBnS1KF6p1D/TK1rw0XAPMiYgdq1zNdr7sCkyR1mTlbvc6QySdV1tdVlfWk1UW9I9Q/Az5UPL4U+A0wg9p1TSVJPYs5W5IqVO9l8z7b5vG5EfE7aie4/Hd3BSZJ6hpztiRVa5UFdUT0A/4P2CEzXwfIzLu6OzBJUueZsyWpequc8pGZS4AlwIDuD0eSVIY5W5KqV+9Jid8GromIfwPmALlsRWY+0R2BSZK6zJwtSRWqt6D+bvF7XLv2BPo1LhxJUgOYsyWpQvWelFjv1UAkSU1mzpakapl0JUmSpBLqGqGOiDtpMwevrczcp6ERSZJKMWdLUrXqnUP9g3bL7wKOA65sbDiSpAYwZ0tSheqdQ315+7aI+ClwGfDlRgclSeo6c7YkVavMHOq5wMhGBSJJ6lbmbEnqJvXOoT62XdM6wCHA7xoekSSpFHO2JFWr3jnUR7Zb/itwN3BeY8ORJDWAOVuSKlTvHOoPdHf2S/WrAAATNElEQVQgkqTGMGdLUrXqmkMdEUdFxMh2baMiov0oiCSpyczZklStek9K/ArwdLu2p4GvNjYcSVIDmLMlqUL1FtTrA39p1/YyMKix4UiSGsCcLUkVqregfhj4h3ZtHwNmNzYcSVIDNCRnR0S/iPhDRNxULP8wIp6MiJnFz+gGxStJvVq9V/k4E/hVRHwCeBzYGhgLfKi7ApMkdVmjcvap1Irw9du0fS4zr2tIlJLUR9Q1Qp2ZdwEjgHuBdYF7gBGZ+b/1dhQRrRHxYDGqMb1omxQRc9uMdligS1JJDcrZQ4EP89bbmEuS2qn3xi5rAfMy8+tt2taIiLUy8/VO9PeBzHyhXdt5mXluJ44hSXobDcrZ3wY+Dwxs1/61iPhX4DbgCx0dLyImAhMBtthii648BUnqVeqd8nErtcTa9i5bOwNfB/ZrcExSQwyZfFJlfV1VWU9SXUrl7IgYDzyfmTMiou32ZwHPAmsCU6hNLfly+/0zc0qxnpaWluzSM5CkXqTekxJ3BH7fru0eYFQn+krgloiYUYxeLHNyRDwQEZdGxIYd7RgREyNiekRMnz9/fie6lKTVUtmcvSdwUES0AlcD+0fElZk5L2teBy4Ddm1UwJLUm9VbUL8MbNKubRNqt7Ot116ZOQY4EDgpIvYB/hPYChgNzAO+1dGOmTklM1sys2XIkCGd6FKSVkulcnZmnpWZQzNzGHA4cHtmfjIiNgWIiAA+CsxqXMiS1HvVW1D/FLgqIkZExDoRsSPwI+CaejvKzLnF7+eB64FdM/O5zFySmUuBi3G0Q5IaoXTOXompEfEg8CCwMd4oRpKA+gvqs6ldOuke4BVq8/IeBf5fPTtHxLoRMXDZY+CDwKxlox2Fj+FohyQ1Qqmc3VZmTsvM8cXj/TNzx8wckZmfzMxXGxm0JPVWdZ2UmJmvUZumcTK1UYkXMjMjot6CfBPg+tq3hPQHrsrMmyPiiuLGAAm0Ap/q7BOQJK2oATlbktQJ9V7lA4DMTGB+ROwYEUcBRwCb1bHfE3RwMkxmHtmZ/iVJ9etqzpYkdU7doxURMSQiTo2I+4CZ1OY7n9ptkUmSusycLUnVedsR6ohYAzgImAAcAPwR+DGwJfDx4gRDSVIPYM6WpOZY1Qj1c8D3qZ3Msltm7pCZXwHe6PbIJEmdZc6WpCZYVUH9ADAIeD+wy8puvCJJ6hHM2ZLUBG9bUGfmftRuvHILcAbwbET8AlgXWKPbo5Mk1c2cLUnNscqTEjPzqcz8SmZuA4yldkfDpcD9EfHN7g5QklQ/c7YkVa9T1yTNzLsycyLwLuAUYMduiUqSVJo5W5Kq0aWL/Gfma5n548w8sNEBSZIay5wtSd3Lu2ZJkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSX0r6qjiGgFXgGWAIszsyUiBgM/AYYBrcBhmflSVTFJkiRJZVU9Qv2BzBydmS3F8heA2zJzG+C2YlmSJEnqNZo95eNg4PLi8eXAR5sYiyRJktRpVRbUCdwSETMiYmLRtklmzisePwtsUmE8kiRJUmmVzaEG9srMuRHxTuDWiHik7crMzIjIjnYsCvCJAFtssUX3RypJkiTVqbIR6sycW/x+Hrge2BV4LiI2BSh+P7+SfadkZktmtgwZMqSqkCVJkqRVqqSgjoh1I2LgssfAB4FZwM+Bo4vNjgZurCIeSZIkqVGqmvKxCXB9RCzr86rMvDki7gWuiYjjgKeAwyqKR5IkSWqISgrqzHwCGNVB+4vA2CpikCRJkrpDsy+bJ0mSJPVqFtSSJElSCRbUkiRJUgkW1JIkSVIJFtSSJElSCRbUkiRJUgkW1JIkSVIJFtSSJElSCRbUkiRJUgkW1JIkSVIJFtSSJElSCRbUkiRJUgkW1JIkSVIJFtSSpA5FRL+I+ENE3FQsD4+I30fEHyPiJxGxZrNjlKSewIJakrQypwKz2yx/AzgvM7cGXgKOa0pUktTDWFBLkt4iIoYCHwZ+UCwHsD9wXbHJ5cBHmxOdJPUsFtSSpI58G/g8sLRY3ghYkJmLi+U5wLubEZgk9TQW1JKkFUTEeOD5zJzRxf0nRsT0iJg+f/78BkcnST2PBbUkqb09gYMiohW4mtpUj+8AgyKif7HNUGBuRztn5pTMbMnMliFDhlQRryQ1lQW1JGkFmXlWZg7NzGHA4cDtmXkE8Bvg0GKzo4EbmxSiJPUoFtSSpHqdCfxLRPyR2pzqS5ocjyT1CP1XvYkkaXWVmdOAacXjJ4BdmxmPJPVEjlBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVRaUEdEv4j4Q0TcVCz/MCKejIiZxc/oKuORJEmSyupfcX+nArOB9du0fS4zr6s4DkmSJKkhKhuhjoihwIeBH1TVpyRJktTdqpzy8W3g88DSdu1fi4gHIuK8iFirox0jYmJETI+I6fPnz+/2QCVJkqR6VVJQR8R44PnMnNFu1VnAdsAuwGDgzI72z8wpmdmSmS1Dhgzp3mAlSZKkTqhqhHpP4KCIaAWuBvaPiCszc17WvA5cBuxaUTySJElSQ1RSUGfmWZk5NDOHAYcDt2fmJyNiU4CICOCjwKwq4pEkSZIapeqrfLQ3NSKGAAHMBE5ocjySJElSp1ReUGfmNGBa8Xj/qvuXJEmSGsk7JUqSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSU0+9bjkiRJ6oGGTD6psr7mf+7CyvrqDo5QS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEklWFBLkiRJJVhQS5IkSSVYUEuSJEkleGMXSZWp8iYBV1XWkyRpdecItSRJklSCBbUkSZJUggW1JEmSVIIFtSRJklSCBbUkSZJUggW1JEmSVIIFtSRJklSCBbUkSZJUggW1JEmSVIIFtSRJklSCBbUkSZJUggW1JEmSVIIFtSRJklSCBbUkSZJUggW1JEmSVIIFtSRJklSCBbUkSZJUggW1JEmSVEKlBXVE9IuIP0TETcXy8Ij4fUT8MSJ+EhFrVhmPJOmtImJARNwTEfdHxEMR8aWi/YcR8WREzCx+Rjc7VknqCaoeoT4VmN1m+RvAeZm5NfAScFzF8UiS3up1YP/MHAWMBv4+InYr1n0uM0cXPzObF6Ik9RyVFdQRMRT4MPCDYjmA/YHrik0uBz5aVTySpI5lzavF4hrFTzYxJEnq0aocof428HlgabG8EbAgMxcXy3OAd1cYjyRpJYopejOB54FbM/P3xaqvRcQDEXFeRKzVxBAlqceIzO4fdIiI8cCHMvPTEbEfcAYwAfhdMd2DiNgc+K/MHNHB/hOBicXitsCj3R50NTYGXmh2EFqB70nP1Ffely0zc0izg+iMiBgEXA+cArwIPAusCUwBHs/ML3ewjzlbVfE96Zn60vtSV96uqqD+d+BIYDEwAFifWoI+AHhXZi6OiN2BSZl5QLcH1ENExPTMbGl2HHqT70nP5PvSXBHxr8DCzDy3Tdt+wBmZOb5pgVXMv8Oex/ekZ1od35dKpnxk5lmZOTQzhwGHA7dn5hHAb4BDi82OBm6sIh5J0spFxJBiZJqIWBsYBzwSEZsWbUHtnJdZzYtSknqO/k3u/0zg6oj4KvAH4JImxyNJgk2ByyOiH7WBl2sy86aIuD0ihgABzAROaGaQktRTVF5QZ+Y0YFrx+Alg16pj6EGmNDsAvYXvSc/k+1KhzHwA2KmD9v2bEE5P4t9hz+N70jOtdu9LJXOoJUmSpL7KW49LkiRJJaw2BXVEbNTmdrnPRsTcNstZ/H6ouNXu6RHxjmK//SLi5WL97Ij4YoUxTypi27pN22eLtpZi+VfLTh5a3fTG97Q36y2vd7vbY99XXEFoWfuhq9pfPUNv+XtrF7M5exV64/vaW/WW17qv5Oxmn5RYmcx8kdotdImIScCryy4BFRGvZuayde8ErqJ2ab9lf0R3Zub4iFgXmBkRv8jM+1bVZ0T0b3Pjmq56kNqVUb5aLH8ceKjN8/pQyeP3Wr34Pe2Vetnr/bnMvC4iPgh8HxjZhWOoiXrZ31tb5uy30Yvf116nl73WvT5nrzYj1PXKzOep3ZDg5IiIduv+CswAto6IYRFxZ/Fp6r6I2AOWf7K7MyJ+DjwcEetGxC+LT4CzIuITxXY7R8T/RMSMiPjvKC5H1YEbgIOLfbYCXqbNxdIjojUiNi7imR0RFxefOG+J2uWuiIhpbUZHNo6I1uLx+yLinuJT4QMRsU2jXseepAe+p31aD3u97wC2bt+47N9N8bglIqYVj/eNN0dw/hARAxv3yqg79LC/NzBnN0QPfF/7rB72WvfanG1B3YHi6iP9gHe2bY+IjYDdqI02PA+My8wxwCeA89tsOgY4NTPfC/w98ExmjiruAnlzRKwBXAAcmpk7A5cCX1tJOH8Bno6IEdRGPX7yNqFvA1yYme8DFgD/sIqnegLwneJTagu127/3ST3sPe3zetDr/RFqI4b1OgM4qfg3sTfwt07sqybpQX9vYM5umB72vvZpPei17rU5e7WZ8lHS3hHxB2Ap8PXMfCgiNgC+GxGjgSXAe9tsf09mPlk8fhD4VkR8A7gpM+8sEu0I4Nbiw2A/YN7b9H81tcR8ADAWOGYl2z2ZmTOLxzOAYat4Xr8Fzo6IocDPMvOxVWzflzT7PV3dVP16T46Ic4D5wHGdiPN/gf+IiKnU/k306YKlD2v2v29zdvdo9vu6OjFnd5IFdQci4j3U/lieB7anmEvUbrPTgOeAUdRG+l9rs+6vyx5k5v9FxBjgQ8BXI+I2arddfygzd68zpJuAycD0zPxLu29k2nq9zeMlwNrF48W8+W3EgDaxXRURvwc+DPwqIj6VmbfXGVOv0gPf0z6tB7zen8vM694mxJX9m/h6RPyy6Ot/I+KAzHzk7Z+tmq0H/L21Z85ugB74vvZZPeC17vU52ykf7UTtLmDfA76b+bYX6d4AmJeZS4EjqX3a6uh4mwELM/NKagl2DPAoMCTePJN1jYh438o6ysyF1O4q2dWvolqBnYvHy8+YLf4BPZGZ51O77XuvOwmgHj3xPe3Lesnr3cqb/yaWf80eEVtl5oOZ+Q3gXmC7ThxTTdAT/97M2eX1xPe1r+olr3UrPTxnO0Jds3ZEzATWoPYp6ArgP1axz0XATyPiKOBm2nw6a2dHal9lLAUWASdm5htRuxTM+cVXKP2Bb9PmTPD2MvPqzjyhds4FromIicAv27QfBhwZEYuAZ4F/K9FHT9Pj39M+pre93l8CLomIr1DcubXw2Yj4ALWvOR8C/qvO46laPf7vzZzdJT3+fe1Dettr3eNztndKlCRJkkpwyockSZJUggW1JEmSVIIFtSRJklSCBbUkSZJUggW1JEmSVIIFtbQSETEsIjIiVnl5yYiYEBF3VRGXJKlj5m01iwW1+oyIaI2INyJi43btfygS7LDmRCZJ6oh5W32FBbX6mieBf1y2EBE7Aus0LxxJ0iqYt9XrWVCrr7kCOKrN8tHAj5YtRMQGEfGjiJgfEU9FxDkR8Y5iXb+IODciXoiIJ4APtz1wse8lETEvIuZGxFcjosNbr0qS6mbeVq9nQa2+5nfA+hGxfZE0DweubLP+AmAD4D3AvtSS+DHFuuOB8cBOQAtwaLtj/5DaLVq3Lrb5IPDP3fIsJGn1Yd5Wr2dBrb5o2WjHOGA2MLdoX5aoz8rMVzKzFfgWcGSx/jDg25n5dGb+Gfj3ZQeMiE2ADwGfzcy/ZubzwHnF8SRJ5Zi31aut8ixYqRe6ArgDGE6brw2BjYE1gKfatD0FvLt4vBnwdLt1y2xZ7DsvIpa1vaPd9pKkrjFvq1ezoFafk5lPRcST1EYmjmuz6gVgEbUk+3DRtgVvjoTMAzZvs/0WbR4/DbwObJyZi7sjbklaXZm31ds55UN91XHA/pn51zZtS4BrgK9FxMCI2BL4F96cq3cN8JmIGBoRGwJfWLZjZs4DbgG+FRHrR8Q7ImKriNi3kmcjSX2feVu9lgW1+qTMfDwzp3ew6hTgr8ATwF3AVcClxbqLgf8G7gfuA37Wbt+jgDWpjZK8BFwHbNrw4CVpNWTeVm8WmdnsGCRJkqReyxFqSZIkqQQLakmSJKkEC2pJkiSpBAtqSZIkqQQLakmSJKkEC2pJkiSpBAtqSZIkqQQLakmSJKkEC2pJkiSphP8fSVo5GfWAYPIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6eef980c88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtQAAAGGCAYAAABbgxaRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl4XWW5///3bVsoMpUhIliwCCKF0oYSZhl7CooVFQFxYCpQmRVBkR9+Dzido7/iAalwPCggYhGhMh0O8gWBChwQaCEMpSCDQYpAA1qlFrDD/f1jr5Y0TZu0K9l7J3m/ritX9xqfO3snTz959rPXisxEkiRJ0qp5V60LkCRJknozA7UkSZJUgoFakiRJKsFALUmSJJVgoJYkSZJKMFBLkiRJJRiopW4QEatHxNyI2KTWtUhSd4qIoyLi3h447+cj4rbuPm93sE/XyjJQqyYioiUi3oyINyJiTkTcFxHHR0RVfyaLDn1u8fVmRCxqszy3q+fJzLczc63M/HNx3qsj4hvt2nolIj7c3d+DJJXVpk+e2+brR914/mERkRExcPG6zJycmft1VxtFO/bpqgkDtWrp45m5NvB+4HvAmcCl1Syg6NDXysy1gI8Cf168XKyrCxHxrmr/sSGp3/l42/4vM0+udUEryz5dteKLqZrLzL9l5k3AZ4AjI2IEQESsGxE/j4jWiHghIr6xuANa/BZkRJwXEX+NiD9GxEcXnzMiNo+Iu4sR8N9GxEUR8YuVrS0iToiIa9ssvxgRV7ZZnh0RW0fE4GL0ZWhEnAp8Gvg/xajItcU53gPcVqw7tTh+j4h4oBilfzgidm9z7t9HxLci4gFgHuBbj5Jqrujzbo+Iv0TE0xFxaJtta0TED4o++29FP70GcHexy5yiD9y1/VSSiNgtIh4qjnsoInZrs21qRHw7Iv636Ndvi4gNV6F2+3T1CAO16kZmPgjMAvYoVk0C1gU+AOwFHAEc3eaQnYGngQ2B/x+4NCKi2HYV8CCwAXAucPgqlvW7om0i4gPAAuDDxfI2wKLMfKrd93Eh8Gvg28WoyCGZeQgwG9ivWHdhRAwDbgDOBtYHvgHcEBHrtTndF4rve23glVX8HiSpW0TEmsDtVPrY9wCHARcX/SHAecAOwG5U+rWvAYuAPYvtQ4o+8P52510f+B/gQir99n8A/xMRG7TZ7XNU/g94D7AacMYqfAv26eoRBmrVmz8D60fEACod9VmZ+UZmtgA/YOlg/EJm/iQzFwJXABsDG0XEZsCOwL9m5j8z817gplUpJjOfhCUd7Z7AfwNvFB3nXrwz6rIqjgSuy8zfZuaizLwFeBJoO6fwp5n5dGbOz8wFJdqSpM7cUIysLv46roN9xgEtmXl5Zi7IzEeohM1DincQxwNfysyXMnNhZt6XmW93oe2PAc9k5pXFeX8JPAV8vM0+l2fmHzLzTeAaoHFlv0H7dPWUgZ3vIlXV+4C/UBl1HgS80GbbC8X2xZb8dZ+Z84rB6bWKY/+SmfPa7PsisOkq1nQ3sDfQRGUEBSod715URjtW1fuBz0bEIW3WDWLptwFfLHF+SVoZn8zM33ayz/uBnSNiTpt1A4ErqfS9g4HnVqHtTVi6v4cV9PlUpkys6pxo+3R1OwO16kZE7Eil87wXeA2YT6WDerLYZTPgpS6c6mUqo9zvbhOqVzVMQ6WD3RvYnsoHJ6EymrIX8J3lHJNdWPcildGKU1bQdkfnkaRaeRH4XWaObb+hGKF+C9gCeLTd5s76sj9T6e/b2gy4dRXrXBH7dHU7p3yo5iJinYgYB1wN/CIzHy+mcVwDfDci1o6I9wNfATr9YGFmvgBMA86NiNUiYleWfttwZf0O+Ajwz8xsLZY/TWXkYcZyjnmVytzvFa27gsrbpGMiYkDxYZ4xEfHeErVKUk+6GdgqIg6PiEHF144RMTwzFwGXAf8REZsU/dquEbE60EplLnX7fnGxW4rzfi4iBkbEZ4Btiva6m326up2BWrX03xHxBpW/6s+m8iGUth86PAX4B/A8lVHrq6h01l3xeWBX4HUqIw6/Aroyj68jj1MZLb8bIDNfK2q+JzOXN9pwCbBjMQ/x6mLdd6n8gTAnIk7OzOepdOLfpDIi/wLwJfy9lFQb/x1LX4f6+vY7ZOYbVOYEH0ZlVPkV4PvA6sUuZ1DpMx+iMn3v+8C7incLvwv8b9EH7tLuvK9TmZ99OpV++2vAuKK/7W726ep2sfyfHanviIhfAU9l5jm1rkWSJPUt/tWkPql4C3KLqFw8/yPAJ6hczkiSJKlbGajVV70XmArMpXJd0xOKyztJ6qJiHugjEXFzsTymuFlFc3HDji1rXaMk1QOnfEiSOhQRX6FyabF1MnNcRPwB+ERmzoyIE4GdMvOomhYpSXXAEWpJ0jIiYiiVS4n9tM3qBNYpHq9L5UNpktTveR1qSVJHLqBypYW126w7FrglIt4E/g7s0tGBktTf9LpAveGGG+awYcNqXYYkrbTp06e/lpkNta6jM8V14Wdn5vSI2LvNptOAAzLzgYj4KpVLXR7bwfETgAkAa6655g5bb711FaqWpO7X1X67182hbmpqymnTptW6DElaaRExPTObal1HZyLi34HDgQVUbiW9DnAXsHVmblHssxlwa2Zus6Jz2WdL6s262m87h1qStJTMPCszh2bmMCo38LiTyqUn142IrYrdxgIza1SiJNWVXjflQ5JUfZm5ICKOA34dEYuAvwLja1yWJNUFA7UkabkycyqVa7qTmdcDy9yOWpL6uz4RqOfPn8+sWbN46623al2K2hg8eDBDhw5l0KBBtS5FkiSpx/SJQD1r1izWXntthg0bRkTUuhwBmcnrr7/OrFmz2HzzzWtdjiRJUo/pEx9KfOutt9hggw0M03UkIthggw1810CSJPV5fSJQA4bpOuRrIkmS+oM+E6hrbcCAATQ2NjJixAg+/vGPM2fOnKq239LSwogRI1bp2KlTp3LfffctWf7xj3/Mz3/+8+4qTZIkqU/rE3Oo29volEu79XyvTjqm033WWGMNmpubATjyyCO56KKLOPvss0u3vXDhQgYMGFD6PCsydepU1lprLXbbbTcAjj/++B5tT5IkqS9xhLoH7Lrrrrz00ktLlidOnMiOO+7IyJEjOeecc4DKiPLWW2/N5z//eYYPH87BBx/MvHnzABg2bBhnnnkmo0eP5tprr6W5uZlddtmFkSNH8qlPfYq//vWvAEyfPp1Ro0YxatQoLrrooiXt/exnP+Pkk09esjxu3DimTp0KwK233sro0aMZNWoUY8aMoaWlhR//+Mecf/75NDY2cs8993Duuedy3nnnASy37b333pszzzyTnXbaia222op77rmn555QSZKkOmag7mYLFy7kjjvu4MADDwTgtttu45lnnuHBBx+kubmZ6dOnc/fddwPw9NNPc+KJJzJz5kzWWWcdLr744iXn2WCDDXj44Yc57LDDOOKII/j+97/PY489xnbbbcc3v/lNAI4++mgmTZrEo48+2qXaWltbOe644/j1r3/No48+yrXXXsuwYcM4/vjjOe2002hubmaPPfZY6pjltQ2wYMECHnzwQS644IKl1kuSJPUnBupu8uabb9LY2Mh73/teXn31VcaOHQtUAvVtt93G9ttvz+jRo3nqqad45plnANh0003ZfffdAfjCF77Avffeu+R8n/nMZwD429/+xpw5c9hrr72AynSSu+++mzlz5jBnzhz23HNPAA4//PBOa/z973/PnnvuueQyduuvv/4K919e24sddNBBAOywww60tLR02r4kSVJfZKDuJovnUL/wwgtk5pIpGJnJWWedRXNzM83NzTz77LMcc0xlTnb7q2C0XV5zzTVXuZaBAweyaNGiJcs9dem61VdfHah8IHPBggU90oYkSVK9M1B3s3e/+91ceOGF/OAHP2DBggXsv//+XHbZZcydOxeAl156idmzZwPwpz/9ifvvvx+Aq666ig9/+MPLnG/ddddlvfXWWzJH+corr2SvvfZiyJAhDBkyZMmo9uTJk5ccM2zYMJqbm1m0aBEvvvgiDz74IAC77LILd999N3/84x8B+Mtf/gLA2muvzRtvvNHltiVJkvSOPnmVj1rbfvvtGTlyJL/85S85/PDDmTlzJrvuuisAa621Fr/4xS8YMGAAH/rQh7jooosYP34822yzDSeccEKH57viiis4/vjjmTdvHh/4wAe4/PLLAbj88ssZP348EcF+++23ZP/dd9+dzTffnG222Ybhw4czevRoABoaGrjkkks46KCDWLRoEe95z3u4/fbb+fjHP87BBx/MjTfeyKRJk7rUtiRJkioiM3u+kYgPAb9qs+oDwL8CPy/WDwNagEMz868rOldTU1NOmzZtqXUzZ85k+PDh3Vhxz2tpaWHcuHE88cQTtS6lR/XG10Z9w+3jh1atrbGXzerSfhExPTObericutJRny1JvUVX++2qTPnIzKczszEzG4EdgHnA9cDXgTsy84PAHcWyJEmS1GvUYg71GOC5zHwB+ARwRbH+CuCTNainJoYNG9bnR6clSZL6g1oE6sOAXxaPN8rMl4vHrwAbdXRAREyIiGkRMa21tbUaNUqSJEldUtVAHRGrAQcC17bflpXJ3B1O6M7MSzKzKTObGhoaerhKSZIkqeuqPUL9UeDhzHy1WH41IjYGKP6dXeV6JEmSpFKqHag/yzvTPQBuAo4sHh8J3FjleiRJkqRSqhaoI2JNYCxwXZvV3wPGRsQzwL8Uy71SRHD66acvWT7vvPM499xzV3jMDTfcwJNPPtnhtqeffpq9996bxsZGhg8fzoQJE1aprqOOOoopU6Z0ef+WlhZGjBixSm1JkiT1R1W7sUtm/gPYoN2616lc9aNbNUw8qVvP1/rVizrdZ/XVV+e6667jrLPOYsMNN+zSeW+44QbGjRvHNttss8y2U089ldNOO41PfOITADz++OMrV7QkSZKqwluPd5OBAwcyYcIEzj///GW2tbS0sO+++zJy5EjGjBnDn/70J+677z5uuukmvvrVr9LY2Mhzzz231DEvv/wyQ4e+c2OK7bbbDoCFCxdyxhlnMGLECEaOHLnkzobf+ta32HHHHRkxYgQTJkygoxv2TJ8+nb322osddtiB/fffn5dffnnJ+lGjRjFq1CguuqjzPx4kSZL0DgN1NzrppJOYPHkyf/vb35Zaf8opp3DkkUfy2GOP8fnPf55TTz2V3XbbjQMPPJCJEyfS3NzMFltssdQxp512Gvvuuy8f/ehHOf/885kzZw4Al1xyCS0tLTQ3Ny85H8DJJ5/MQw89xBNPPMGbb77JzTffvNT55s+fzymnnMKUKVOYPn0648eP5+yzzwbg6KOPZtKkSTz66KM99dRIkiT1WQbqbrTOOutwxBFHcOGFFy61/v777+dzn/scAIcffjj33ntvp+c6+uijmTlzJocccghTp05ll1124e233+a3v/0tX/ziFxk4sDJbZ/311wfgrrvuYuedd2a77bbjzjvvZMaMGUud7+mnn+aJJ55g7NixNDY28p3vfIdZs2YxZ84c5syZw5577rmkPkmSJHVd1eZQ9xdf/vKXGT16NEcffXTpc22yySaMHz+e8ePHM2LEiOXeWfGtt97ixBNPZNq0aWy66aace+65vPXWW0vtk5lsu+223H///UutXzzyLUmSpFXjCHU3W3/99Tn00EO59NJLl6zbbbfduPrqqwGYPHkye+yxBwBrr702b7zxRofnufXWW5k/fz4Ar7zyCq+//jrve9/7GDt2LP/1X//FggULAPjLX/6yJDxvuOGGzJ07t8OrenzoQx+itbV1SaCeP38+M2bMYMiQIQwZMmTJqPnkyZO742mQJEnqNwzUPeD000/ntddeW7I8adIkLr/8ckaOHMmVV17JD3/4QwAOO+wwJk6cyPbbb7/MhxJvu+02RowYwahRo9h///2ZOHEi733vezn22GPZbLPNGDlyJKNGjeKqq65iyJAhHHfccYwYMYL999+fHXfccZmaVlttNaZMmcKZZ57JqFGjaGxs5L777gPg8ssv56STTqKxsbHDDzNKkiRp+aK3BaimpqacNm3aUutmzpzJ8OHDa1SRVsTXRrVy+/ihne/UTcZeNqtL+0XE9Mxs6uFyuk1EDACmAS9l5riICOA7wCHAQuA/M/PCFZ2joz5bknqLrvbbzqGWJC3Pl4CZwDrF8lHApsDWmbkoIt5Tq8IkqZ4YqCVJy4iIocDHgO8CXylWnwB8LjMXAWTm7BqVJ5VSj+9gqXdzDrUkqSMXAF8DFrVZtwXwmYiYFhG/iYgP1qY0SaovjlBL3cDRDvUlETEOmJ2Z0yNi7zabVgfeysymiDgIuAzYo4PjJwATADbbbLMqVCxJteUItSSpvd2BAyOiBbga2DcifgHMAq4r9rkeGNnRwZl5SWY2ZWZTQ0NDNeqVpJoyUEuSlpKZZ2Xm0MwcBhwG3JmZXwBuAPYpdtsL+EONSpSkuuKUj24ya9YsTjrpJJ588kkWLVrEuHHjmDhxIqutttpS+/35z3/m1FNP7fDmK20dcMABS64xvbLOPfdc1lprLc4444yVPlb9z0anXNr5Tt3k1UnHVK0t9YjvAZMj4jRgLnBsjeuRpLrQJwN1d89n7WzOamZy0EEHccIJJ3DjjTeycOFCJkyYwNlnn83EiROX7LdgwQI22WSTTsM0wC233FK6bkkqKzOnAlOLx3OoXPlDktSGUz66wZ133sngwYM5+uijARgwYADnn38+l112GRdffDEHHngg++67L2PGjKGlpYURI0YAMG/ePA499FC22WYbPvWpT7Hzzjuz+AYIw4YN47XXXqOlpYXhw4dz3HHHse2227Lffvvx5ptvAvCTn/yEHXfckVGjRvHpT3+aefPm1eYJkCRJ6scM1N1gxowZ7LDDDkutW2edddhss81YsGABDz/8MFOmTOF3v/vdUvtcfPHFrLfeejz55JN8+9vfZvr06R2e/5lnnuGkk05ixowZDBkyhF//+tcAHHTQQTz00EM8+uijDB8+nEsvrd5b95IkSaowUFfB2LFjWX/99ZdZf++993LYYYcBMGLECEaO7PAD82y++eY0NjYCsMMOO9DS0gLAE088wR577MF2223H5MmTmTFjRs98A5IkSVouA3U32GabbZYZXf773//On/70JwYOHMiaa65Z6vyrr776kscDBgxgwYIFABx11FH86Ec/4vHHH+ecc87hrbfeKtWOJEmSVp6BuhuMGTOGefPm8fOf/xyAhQsXcvrpp3PUUUfx7ne/e7nH7b777lxzzTUAPPnkkzz++OMr1e4bb7zBxhtvzPz585k8efKqfwOSJElaZQbqbhARXH/99Vx77bV88IMfZKuttmLw4MH827/92wqPO/HEE2ltbWWbbbbhG9/4Bttuuy3rrrtul9v99re/zc4778zuu+/O1ltvXfbbkCRJ0iqIzKx1DSulqakpF18JY7GZM2cyfPjwGlW06hYuXMj8+fMZPHgwzz33HP/yL//C008/vcy1q3uz3vrarKzefOvxvnod6np8TSJiemY29XA5daWjPluqtXrsH1Sfutpv98nrUPcW8+bNY5999mH+/PlkJhdffHGfCtOSJEn9gYG6htZee20cuZEkSerdnEMtSZIkldBnAnVvmwveH/iaSJKk/qBPTPkYPHgwr7/+OhtssAERUetyRCVMv/766wwePHip9X31A3CSJKn/6hOBeujQocyaNYvW1tZal6I2Bg8ezNCh1fsktSRJUi30iUA9aNAgNt9881qXIUmSpH6oz8yhliRJkmrBQC1JkiSVYKCWJEmSSjBQS5IkSSUYqCVJkqQSDNSSJElSCQZqSZIkqQQDtSRJklSCgVqSJEkqwUAtSZIklWCgliRJkkowUEuSJEklGKglSZKkEqoWqCNiSERMiYinImJmROwaEedGxEsR0Vx8HVCteiRJkqTuUM0R6h8Ct2bm1sAoYGax/vzMbCy+bqliPZKkFYiIARHxSETc3G79hRExt1Z1SVK9qUqgjoh1gT2BSwEy85+ZOacabUuSVtmXeGfwA4CIaALWq005klSfqjVCvTnQClxejHb8NCLWLLadHBGPRcRlEdFhJx0REyJiWkRMa21trVLJktR/RcRQ4GPAT9usGwBMBL5Wq7okqR5VK1APBEYD/5mZ2wP/AL4O/CewBdAIvAz8oKODM/OSzGzKzKaGhoYqlSxJ/doFVILzojbrTgZuysyXV3SggyCS+ptqBepZwKzMfKBYngKMzsxXM3NhZi4CfgLsVKV6JEnLERHjgNmZOb3Nuk2AQ4BJnR3vIIik/mZgNRrJzFci4sWI+FBmPg2MAZ6MiI3bjHR8CniiGvVIklZod+DA4spLg4F1gBnA28CzEQHw7oh4NjO3rF2ZklQfqhKoC6cAkyNiNeB54GjgwohoBBJoAb5YxXokSR3IzLOAswAiYm/gjMwc13afiJhrmJakiqoF6sxsBprarT68Wu1LkiRJPaGaI9SSpF4mM6cCUztYv1bVi5GkOuWtxyVJkqQSDNSSJElSCQZqSZIkqQQDtSRJklSCgVqSJEkqwUAtSZIklWCgliRJkkowUEuSJEklGKglSZKkEgzUkiRJUgkGakmSJKkEA7UkSZJUgoFakiRJKsFALUmSJJVgoJYkSZJKMFBLkiRJJRioJUmSpBIG1roASZL6qtvHD61aW2Mvm1W1tiQtzRFqSZIkqQQDtSRJklSCgVqSJEkqwUAtSZIklWCgliRJkkowUEuSJEklGKglSZKkEgzUkiRJUgkGakmSJKkEA7UkqUMRMSAiHomIm4vlyRHxdEQ8ERGXRcSgWtcoSfXAQC1JWp4vATPbLE8Gtga2A9YAjq1FUZJUbwzUkqRlRMRQ4GPATxevy8xbsgA8CAytVX2SVE8M1JKkjlwAfA1Y1H5DMdXjcODWjg6MiAkRMS0iprW2tvZslZJUBwzUkqSlRMQ4YHZmTl/OLhcDd2fmPR1tzMxLMrMpM5saGhp6rE5JqhcDa12AJKnu7A4cGBEHAIOBdSLiF5n5hYg4B2gAvljTCiWpjjhCLUlaSmaelZlDM3MYcBhwZxGmjwX2Bz6bmctMBZGk/spALUnqqh8DGwH3R0RzRPxrrQuSpHrglA9J0nJl5lRgavG4T/yf0TDxpKq1dVXVWpJUS32ic5QkSeoPNjrl0qq19eqkY6rWVm/nlA9JkiSpBAO1JEmSVIKBWpIkSSrBQC1JkiSVULVAHRFDImJKRDwVETMjYteIWD8ibo+IZ4p/16tWPZIkSVJ3qOYI9Q+BWzNza2AUMBP4OnBHZn4QuKNYliRJknqNqgTqiFgX2BO4FCAz/5mZc4BPAFcUu10BfLIa9UiSJEndpVoj1JsDrcDlEfFIRPw0ItYENsrMl4t9XqFyB65lRMSEiJgWEdNaW1urVLIkSZLUuWoF6oHAaOA/M3N74B+0m96RmQlkRwdn5iWZ2ZSZTQ0NDT1erCRJktRV1QrUs4BZmflAsTyFSsB+NSI2Bij+nV2leiRJkqRuUZVAnZmvAC9GxIeKVWOAJ4GbgCOLdUcCN1ajHkmSJKm7DKxiW6cAkyNiNeB54Ggqgf6aiDgGeAE4tIr1SJIkSaVVLVBnZjPQ1MGmMdWqQZIkSepu3ilRkiRJKsFALUmSJJVgoJYkSZJKMFBLkiRJJRioJUmSpBIM1JIkSVIJ1bwOdVVtdMqlVWvr1UnHVK0tSZIk1RdHqCVJkqQSDNSSJElSCQZqSZIkqQQDtSRJklSCgVqSJEkqwUAtSZIklWCgliR1KCIGRMQjEXFzsbx5RDwQEc9GxK8iYrVa1yhJ9cBALUlani8BM9ssfx84PzO3BP4KeBF+ScJALUnqQEQMBT4G/LRYDmBfYEqxyxXAJ2tTnSTVFwO1JKkjFwBfAxYVyxsAczJzQbE8C3hfLQqTpHpjoJYkLSUixgGzM3P6Kh4/ISKmRcS01tbWbq5OkuqPgVqS1N7uwIER0QJcTWWqxw+BIRExsNhnKPBSRwdn5iWZ2ZSZTQ0NDdWoV5JqykAtSVpKZp6VmUMzcxhwGHBnZn4euAs4uNjtSODGGpUoSXXFQC1J6qozga9ExLNU5lRfWuN6JKkuDOx8F0lSf5WZU4GpxePngZ1qWY8k1aOVGqGOiG0j4v6ImBsRj0XEnj1VmCSpPPttSep5KwzUxXVH2zoPOANYn8rllH7WM2VJklaF/bYkVV9nI9T3RkRTm+XVgRcy85/AC8AaPVaZJGlV2G9LUpV1Nof6COCHETELOAv4FnBfRLyLSqd8cg/Xp3ZuHz+0am2NvWxW1dqS1G3styWpylYYqDPzOWBcRBwM3AlcCAwDNgRez8yFPV6hJKnL7Lclqfq69KHEzJwC7AGMoNJBb2SnLEn1y35bkqqnsw8l7hURj0fEXOAO4ArgS8BFEXF+RKxVjSIlSV1jvy1J1dfZCPXlwNepXMD/O8APM/PRzNwTeBK4r4frkyStHPttSaqyzj6UOBh4KDPfjoiHi2UAMvMnEXFdj1YnSVpZ9tvqNhudUr2bYb466ZiqtSV1t84C9RnAQ8WnxTcATmi7MTNf76nCJEmrxH5bkqqss6t8XBURv6LSKbdmZlanLEnSqrDflqTq62yEmuJT4bOrUIskqRvYb0tSdXXpsnmSJEmSOmagliRJkkowUEuSJEklrHKgjoo9u7MYSVLPsd+WpJ5RZoR6NeCu7ipEktTj7LclqQes8CofEXHECjav1s21SJJKst+WpOrr7LJ5lwPTgbc72BbdX44kqST7bfVKDRNPqlpbV1WtJfUXnQXqZ4AzM3OZtwgjYjAwr6sNRUQL8AawEFiQmU0RcS5wHNBa7Pb/ZeYtXT2nJGkZ3dZvS5K6prNA/Ttgazqec7ew2L4y9snM19qtOz8zz1vJ89QV/6qWVEe6u9+WJHWis1uPf3EF2+YD+3R7RZKkVWa/LUnVt8KrfETEe7uxrQRui4jpETGhzfqTI+KxiLgsItbrxvYkqd/p5n5bktQFnV027w9tFyLiuhJtfTgzRwMfBU4qroX6n8AWQCPwMvCDjg6MiAkRMS0iprW2tna0iyR0UNQKAAATNUlEQVSpojv7bUlSF3QWqNt/InzvVW0oM18q/p0NXA/slJmvZubCzFwE/ATYaTnHXpKZTZnZ1NDQsKolSFJ/0G39tiSpazoL1NkdjUTEmhGx9uLHwH7AExGxcZvdPgU80R3tSVI/1i39tiSp6zq7ysfAiNiHd0Y82i+TmXd2oZ2NgOsjYnGbV2XmrRFxZUQ0UvkPoAVY7odpJEld0l39tiSpizoL1LOBy9osv95uOYEPdNZIZj4PjOpg/eFdqFGS1HWl++3ietV3A6tT+X9iSmaeExFjgIlU3t2cCxyVmc92Y+2S1Ct1dtm8YVWqQ5LUDbqp334b2Dcz50bEIODeiPgNlQ+SfyIzZ0bEicA3gKO6oT1J6tU6G6GWJPUzmZlURqABBhVfWXytU6xfF/hz9auTpPpjoJYkLSMiBgDTgS2BizLzgYg4FrglIt4E/g7sUssaJaledHaVD0lSP1Rc0rQRGArsFBEjgNOAAzJzKHA58B8dHeu9AyT1NwZqSdJyZeYc4C4qN+UalZkPFJt+Bey2nGO8d4CkfsVALUlaSkQ0RMSQ4vEawFhgJrBuRGxV7LZ4nST1e86hliS1tzFwRTGP+l3ANZl5c0QcB/w6IhYBfwXG17JISaoXBmpJ0lIy8zFg+w7WXw9cX/2KJKm+OeVDkiRJKsFALUmSJJVgoJYkSZJKMFBLkiRJJRioJUmSpBIM1JIkSVIJBmpJkiSpBAO1JEmSVII3dpEk1dxGp1xavcY2q15TkvoHR6glSZKkEgzUkiRJUgkGakmSJKkEA7UkSZJUgoFakiRJKsFALUmSJJVgoJYkSZJKMFBLkiRJJRioJUmSpBIM1JIkSVIJBmpJkiSpBAO1JEmSVIKBWpIkSSrBQC1JkiSVYKCWJEmSShhY6wKkntIw8aSqtXVV1VqSJEn1xhFqSZIkqQQDtSRJklSCgVqSJEkqwUAtSVpKRAyOiAcj4tGImBER3yzWR0R8NyL+EBEzI+LUWtcqSfXADyVKktp7G9g3M+dGxCDg3oj4DTAc2BTYOjMXRcR7alqlJNUJA7UkaSmZmcDcYnFQ8ZXACcDnMnNRsd/s2lQoSfXFKR+SpGVExICIaAZmA7dn5gPAFsBnImJaRPwmIj64nGMnFPtMa21trWbZklQTBmpJ0jIyc2FmNgJDgZ0iYgSwOvBWZjYBPwEuW86xl2RmU2Y2NTQ0VK9oSaoRA7Ukabkycw5wF/ARYBZwXbHpemBkreqSpHpStUAdES0R8XhENEfEtGLd+hFxe0Q8U/y7XrXqkSR1LCIaImJI8XgNYCzwFHADsE+x217AH2pToSTVl2qPUO+TmY3F24UAXwfuyMwPAncUy5Kk2toYuCsiHgMeojKH+mbge8CnI+Jx4N+BY2tYoyTVjVpf5eMTwN7F4yuAqcCZtSpGkgSZ+RiwfQfr5wAfq35FklTfqjlCncBtETE9IiYU6zbKzJeLx68AG1WxHkmSJKm0ao5QfzgzXypuBHB7RDzVdmNmZkRkRwcWAXwCwGabbdbzlUqSJEldVLUR6sx8qfh3NpVPh+8EvBoRGwMU/3Z4kwAvwSRJkqR6VZVAHRFrRsTaix8D+wFPADcBRxa7HQncWI16JEmSpO5SrSkfGwHXR8TiNq/KzFsj4iHgmog4BngBOLRK9UiSJEndoiqBOjOfB0Z1sP51YEw1apAkSZJ6gndKlCRJkkowUEuSJEklGKglSZKkEgzUkiRJUgkGakmSJKkEA7UkSZJUgoFakiRJKsFALUmSJJVgoJYkSZJKMFBLkiRJJRioJUmSpBIM1JIkSVIJBmpJkiSpBAO1JEmSVIKBWpIkSSrBQC1JkiSVYKCWJEmSSjBQS5IkSSUYqCVJkqQSDNSSJElSCQZqSZIkqQQDtSRpKRExOCIejIhHI2JGRHyz3fYLI2JureqTpHozsNYFSJLqztvAvpk5NyIGAfdGxG8y8/cR0QSsV+P6JKmuOEItSVpKViwegR5UfGVEDAAmAl+rWXGSVIcM1JKkZUTEgIhoBmYDt2fmA8DJwE2Z+XInx06IiGkRMa21tbUa5UpSTRmoJUnLyMyFmdkIDAV2iog9gUOASV049pLMbMrMpoaGhp4uVZJqzkAtSVquzJwD3AXsA2wJPBsRLcC7I+LZWtYmSfXCQC1JWkpENETEkOLxGsBYYHpmvjczh2XmMGBeZm5ZyzolqV54lQ9JUnsbA1cUH0J8F3BNZt5c45okqW4ZqCVJS8nMx4DtO9lnrSqVI0l1zykfkiRJUgkGakmSJKkEA7UkSZJUgoFakiRJKsFALUmSJJXgVT4kSZK0jIaJJ1WtrdavXlS1tnqCI9SSJElSCQZqSZIkqQQDtSRJklSCgVqSJEkqwUAtSZIklWCgliRJkkqoaqCOiAER8UhE3Fws/ywi/hgRzcVXYzXrkSRJksqq9nWovwTMBNZps+6rmTmlynVIkiRJ3aJqI9QRMRT4GPDTarUpSZIk9bRqTvm4APgasKjd+u9GxGMRcX5ErF7FeiRJkqTSqhKoI2IcMDszp7fbdBawNbAjsD5w5nKOnxAR0yJiWmtra88WK0mSJK2Eao1Q7w4cGBEtwNXAvhHxi8x8OSveBi4Hduro4My8JDObMrOpoaGhSiVLkiRJnatKoM7MszJzaGYOAw4D7szML0TExgAREcAngSeqUY8kSZLUXap9lY/2JkdEAxBAM3B8jeuRJEmSVkrVA3VmTgWmFo/3rXb7kiRJUnfyTomSJElSCQZqSZIkqQQDtSRJklSCgVqSJEkqwUAtSZIklWCgliRJkkowUEuSlhIRgyPiwYh4NCJmRMQ3i/WTI+LpiHgiIi6LiEG1rlWS6oGBWpLU3tvAvpk5CmgEPhIRuwCTga2B7YA1gGNrV6Ik1Y9a3ylRklRnMjOBucXioOIrM/OWxftExIPA0BqUJ0l1xxFqSdIyImJARDQDs4HbM/OBNtsGAYcDt9aqPkmqJwZqSdIyMnNhZjZSGYXeKSJGtNl8MXB3Zt7T0bERMSEipkXEtNbW1mqUK0k1ZaCWJC1XZs4B7gI+AhAR5wANwFdWcMwlmdmUmU0NDQ3VKVSSashALUlaSkQ0RMSQ4vEawFjgqYg4Ftgf+GxmLqpljZJUT/xQoiSpvY2BKyJiAJWBl2sy8+aIWAC8ANwfEQDXZea3alinJNUFA7UkaSmZ+RiwfQfr/T9DkjrglA9JkiSpBAO1JEmSVIKBWpIkSSrBQC1JkiSVYKCWJEmSSjBQS5IkSSUYqCVJkqQSDNSSJElSCQZqSZIkqQQDtSRJklSCgVqSJEkqwUAtSZIklWCgliRJkkowUEuSJEklGKglSZKkEgzUkiRJUgkGakmSJKkEA7UkSZJUgoFakiRJKsFALUmSJJVgoJYkSZJKMFBLkiRJJQysdQGS+o+GiSdVra2rqtaSJKm/c4RakiRJKsFALUmSJJVgoJYkSZJKMFBLkiRJJVQ1UEfEgIh4JCJuLpY3j4gHIuLZiPhVRKxWzXokScuKiMER8WBEPBoRMyLim8V6+2xJ6kC1R6i/BMxss/x94PzM3BL4K3BMleuRJC3rbWDfzBwFNAIfiYhdsM+WpA5VLVBHxFDgY8BPi+UA9gWmFLtcAXyyWvVIkjqWFXOLxUHFV2KfLUkdquYI9QXA14BFxfIGwJzMXFAszwLeV8V6JEnLUUzRawZmA7cDz2GfLUkdiszs+UYixgEHZOaJEbE3cAZwFPD74q1DImJT4DeZOaKD4ycAE4rFDwFP93jR1bEh8Fqti9BSfE3qU195Xd6fmQ21LmJlRMQQ4Hrg/wA/s8/uEz+HfYmvSX3qS69Ll/rtat0pcXfgwIg4ABgMrAP8EBgSEQOLEY+hwEsdHZyZlwCXVKnWqomIaZnZVOs69A5fk/rk61I7mTknIu4CdsU+25/DOuNrUp/64+tSlSkfmXlWZg7NzGHAYcCdmfl54C7g4GK3I4Ebq1GPJGn5IqKhGJkmItYAxlL5QLl9tiR1oNbXoT4T+EpEPEtlTvWlNa5HkgQbA3dFxGPAQ8DtmXkz9tmS1KFqTflYIjOnAlOLx88DO1W7hjrS594S7QN8TeqTr0sVZeZjwPYdrLfPVr3xNalP/e51qcqHEiVJkqS+qtZTPiRJkqRerd8E6ojYICKai69XIuKlNstZ/DujuNXu6RHxruK4vSPib8X2mRFxThVrPreobcs2675crGsqlm9Z/OGh/qY3vqa9WW95viPiZxHxx6K9hyNi1zbrD+7seNWH3vLz1q5m++xO9MbXtbfqLc91X+mzqz6HulYy83Uqt9AlIs4F5mbmecXy3MxcvO09wFVULu23+IfonswcFxFrAs0R8d+Z+XBnbcY7l5cq43EqV0b5TrF8CDCjzfd1QMnz91q9+DXtlXrZ8/3VzJwSEfsB/wWMXIVzqIZ62c9bW/bZK9CLX9dep5c9172+z+43I9RdlZmzqdyQ4OSIiHbb/gFMB7aMiGERcU/x19TDEbEbLPnL7p6IuAl4MiLWjIj/Kf4CfCIiPlPst0NE/C4ipkfE/42IjZdT0g3AJ4pjtgD+RpuLpUdES0RsWNQzMyJ+UvzFeVtULndFRExtMzqyYUS0FI+3jYgHi78KH4uID3bX81hP6vA17dPq7Pm+G9iy/crFvzfF46aImFo83iveGcF5JCLW7r5nRj2hzn7ewD67W9Th69pn1dlz3Wv7bAN1B4pPsg8A3tN2fURsAOxCZbRhNjA2M0cDnwEubLPraOBLmbkV8BHgz5k5qrij2K0RMQiYBBycmTsAlwHfXU45fwdejIgRVEY9frWC0j8IXJSZ2wJzgE938q0eD/yw+Cu1icqthPukOntN+7w6er4/TmXEsKvOAE4qfif2AN5ciWNVI3X08wb22d2mzl7XPq2Onute22f3mykfJe0REY8Ai4DvZeaMiFgX+FFENAILga3a7P9gZv6xePw48IOI+D5wc2beU3S0I4Dbiz8GBwAvr6D9q6l0zPsDY4Cjl7PfHzOzuXg8HRjWyfd1P3B2RAwFrsvMZzrZvy+p9Wva31T7+Z4YEd8AWoFjVqLO/wX+IyImU/md6NOBpQ+r9e+3fXbPqPXr2p/YZ68kA3UHIuIDVH5YZgPDKeYStdvtNOBVYBSVkf632mz7x+IHmfmHiBgNHAB8JyLuAK4HZmTmrl0s6WZgIjAtM//e7h2Ztt5u83ghsEbxeAHvvBsxuE1tV0XEA8DHgFsi4ouZeWcXa+pV6vA17dPq4Pn+amZOWUGJy/ud+F5E/E/R1v9GxP6Z+dSKv1vVWh38vLVnn90N6vB17bPq4Lnu9X22Uz7aiYgG4MfAjzJXeJHudYGXM3MRcDiVv7Y6Ot8mwLzM/AWVDnY08DTQEO98knVQRGy7vIYycx6VO5St6ltRLcAOxeMln5gtfoGez8wLqdxCuNd9CKAr6vE17ct6yfPdwju/E0veZo+ILTLz8cz8PpU7BG69EudUDdTjz5t9dnn1+Lr2Vb3kuW6hzvtsR6gr1oiIZmAQlb+CrgT+o5NjLgZ+HRFHALfS5q+zdraj8lbGImA+cEJm/jMql4K5sHgLZSBwAW0+Cd5eZl69Mt9QO+cB10TEBOB/2qw/FDg8IuYDrwD/VqKNelP3r2kf09ue728Cl0bEtynu3Fr4ckTsQ+VtzhnAb7p4PlVX3f+82Wevkrp/XfuQ3vZc132f7Z0SJUmSpBKc8iFJkiSVYKCWJEmSSjBQS5IkSSUYqCVJkqQSDNSSJElSCQZqaTkiYlhEZER0ennJiDgqIu6tRl2SpI7Zb6tWDNTqMyKiJSL+GREbtlv/SNHBDqtNZZKkjthvq68wUKuv+SPw2cULEbEd8O7alSNJ6oT9tno9A7X6miuBI9osHwn8fPFCRKwbET+PiNaIeCEivhER7yq2DYiI8yLitYh4HvhY2xMXx14aES9HxEsR8Z2I6PDWq5KkLrPfVq9noFZf83tgnYgYXnSahwG/aLN9ErAu8AFgLyqd+NHFtuOAccD2QBNwcLtz/4zKLVq3LPbZDzi2R74LSeo/7LfV6xmo1RctHu0YC8wEXirWL+6oz8rMNzKzBfgBcHix/VDggsx8MTP/Avz74hNGxEbAAcCXM/MfmTkbOL84nySpHPtt9WqdfgpW6oWuBO4GNqfN24bAhsAg4IU2614A3lc83gR4sd22xd5fHPtyRCxe9652+0uSVo39tno1A7X6nMx8ISL+SGVk4pg2m14D5lPpZJ8s1m3GOyMhLwObttl/szaPXwTeBjbMzAU9Ubck9Vf22+rtnPKhvuoYYN/M/EebdQuBa4DvRsTaEfF+4Cu8M1fvGuDUiBgaEesBX198YGa+DNwG/CAi1omId0XEFhGxV1W+G0nq++y31WsZqNUnZeZzmTmtg02nAP8AngfuBa4CLiu2/QT4v8CjwMPAde2OPQJYjcooyV+BKcDG3V68JPVD9tvqzSIza12DJEmS1Gs5Qi1JkiSVYKCWJEmSSjBQS5IkSSUYqCVJkqQSDNSSJElSCQZqSZIkqQQDtSRJklSCgVqSJEkqwUAtSZIklfD/AAcd4C/RcGhaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6ef04eccc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mpl.rcParams['axes.labelsize'] = 'large'\n",
    "\n",
    "for metric_name, metric_data in [('Accuracy', acc_data), ('F1', f1_data)]:\n",
    "    \n",
    "    fig, axes = plt.subplots(1, 2, figsize=(12,6))\n",
    "    for i, dataset_name in enumerate(['Dong Twitter', 'Election Twitter']):\n",
    "        data = metric_data[acc_data['Dataset'] == dataset_name]\n",
    "        ax = sns.barplot(x=\"Model\", y=f'{metric_name}', hue=\"Legend\", \n",
    "                         data=data, \n",
    "                         palette=sns.color_palette(\"colorblind\"),\n",
    "                         ax=axes[i])\n",
    "        ax.set_title(f'{dataset_name}')\n",
    "        leg = ax.legend()\n",
    "        if dataset_name == 'Election Twitter':\n",
    "            if metric_name == 'F1':\n",
    "                ax.set(ylim=(30, 48))\n",
    "            else:\n",
    "                ax.set(ylim=(35, 58))\n",
    "            leg.remove()\n",
    "        else:\n",
    "            if metric_name == 'F1':\n",
    "                ax.set(ylim=(40, 72))\n",
    "            else:\n",
    "                ax.set(ylim=(40, 75))\n",
    "        ax.set_ylabel(f'{metric_name} %')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see not scaling the data (in this can we used [MinMax](http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html) scaling) affects the results quite a lot, more so for the models that contain more parameters e.g. TDParse+.\n",
    "\n",
    "This scaling effect can also be seen in the Target Dependent models of which that notebook can be found [here](./target_model.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}