sentiment-classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.040317,
     "end_time": "2020-12-17T13:41:19.207082",
     "exception": false,
     "start_time": "2020-12-17T13:41:19.166765",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Sentiment Classification - Financial Statements\n",
    "### Import Libraries & Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:19.296138Z",
     "iopub.status.busy": "2020-12-17T13:41:19.295559Z",
     "iopub.status.idle": "2020-12-17T13:41:29.230357Z",
     "shell.execute_reply": "2020-12-17T13:41:29.229233Z"
    },
    "papermill": {
     "duration": 9.984013,
     "end_time": "2020-12-17T13:41:29.230488",
     "exception": false,
     "start_time": "2020-12-17T13:41:19.246475",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "\n",
    "import os\n",
    "import re\n",
    "import time\n",
    "import datetime\n",
    "import string\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "plt.style.use('ggplot')\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "import transformers\n",
    "from transformers import BertModel, BertTokenizer, DistilBertTokenizer, RobertaModel, RobertaTokenizer\n",
    "from transformers import AutoConfig, AutoModel, AdamW, get_linear_schedule_with_warmup\n",
    "\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import Dataset, random_split, DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:29.314933Z",
     "iopub.status.busy": "2020-12-17T13:41:29.314347Z",
     "iopub.status.idle": "2020-12-17T13:41:29.348795Z",
     "shell.execute_reply": "2020-12-17T13:41:29.347551Z"
    },
    "papermill": {
     "duration": 0.078084,
     "end_time": "2020-12-17T13:41:29.348896",
     "exception": false,
     "start_time": "2020-12-17T13:41:29.270812",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "statement_df = pd.read_csv('../input/sentiment-analysis-for-financial-news/all-data.csv', encoding='latin-1', header=None)\n",
    "statement_df.columns = ['sentiment', 'statement']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.039909,
     "end_time": "2020-12-17T13:41:29.428462",
     "exception": false,
     "start_time": "2020-12-17T13:41:29.388553",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Set Device and Apply Random Seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:29.515303Z",
     "iopub.status.busy": "2020-12-17T13:41:29.514711Z",
     "iopub.status.idle": "2020-12-17T13:41:29.522732Z",
     "shell.execute_reply": "2020-12-17T13:41:29.523142Z"
    },
    "papermill": {
     "duration": 0.054993,
     "end_time": "2020-12-17T13:41:29.523267",
     "exception": false,
     "start_time": "2020-12-17T13:41:29.468274",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f6ca6213530>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "RANDOM_SEED = 73\n",
    "np.random.seed(RANDOM_SEED)\n",
    "torch.manual_seed(RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.043691,
     "end_time": "2020-12-17T13:41:29.609769",
     "exception": false,
     "start_time": "2020-12-17T13:41:29.566078",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:29.708277Z",
     "iopub.status.busy": "2020-12-17T13:41:29.707620Z",
     "iopub.status.idle": "2020-12-17T13:41:29.715366Z",
     "shell.execute_reply": "2020-12-17T13:41:29.714916Z"
    },
    "papermill": {
     "duration": 0.062374,
     "end_time": "2020-12-17T13:41:29.715458",
     "exception": false,
     "start_time": "2020-12-17T13:41:29.653084",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>statement</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>neutral</td>\n",
       "      <td>According to Gran , the company has no plans t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>neutral</td>\n",
       "      <td>Technopolis plans to develop in stages an area...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>negative</td>\n",
       "      <td>The international electronic industry company ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>positive</td>\n",
       "      <td>With the new production plant the company woul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>positive</td>\n",
       "      <td>According to the company 's updated strategy f...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  sentiment                                          statement\n",
       "0   neutral  According to Gran , the company has no plans t...\n",
       "1   neutral  Technopolis plans to develop in stages an area...\n",
       "2  negative  The international electronic industry company ...\n",
       "3  positive  With the new production plant the company woul...\n",
       "4  positive  According to the company 's updated strategy f..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statement_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:29.807757Z",
     "iopub.status.busy": "2020-12-17T13:41:29.806447Z",
     "iopub.status.idle": "2020-12-17T13:41:29.809682Z",
     "shell.execute_reply": "2020-12-17T13:41:29.810080Z"
    },
    "papermill": {
     "duration": 0.051594,
     "end_time": "2020-12-17T13:41:29.810192",
     "exception": false,
     "start_time": "2020-12-17T13:41:29.758598",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4846, 2)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statement_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.041403,
     "end_time": "2020-12-17T13:41:29.892674",
     "exception": false,
     "start_time": "2020-12-17T13:41:29.851271",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Check for Missing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:29.982875Z",
     "iopub.status.busy": "2020-12-17T13:41:29.982094Z",
     "iopub.status.idle": "2020-12-17T13:41:29.985750Z",
     "shell.execute_reply": "2020-12-17T13:41:29.985346Z"
    },
    "papermill": {
     "duration": 0.05204,
     "end_time": "2020-12-17T13:41:29.985835",
     "exception": false,
     "start_time": "2020-12-17T13:41:29.933795",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentiment    0\n",
       "statement    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statement_df.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.041428,
     "end_time": "2020-12-17T13:41:30.068981",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.027553",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Remove Duplicate Entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:30.157363Z",
     "iopub.status.busy": "2020-12-17T13:41:30.156637Z",
     "iopub.status.idle": "2020-12-17T13:41:30.165387Z",
     "shell.execute_reply": "2020-12-17T13:41:30.164720Z"
    },
    "papermill": {
     "duration": 0.054477,
     "end_time": "2020-12-17T13:41:30.165510",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.111033",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Duplicate Entries:  6\n"
     ]
    }
   ],
   "source": [
    "print('Number of Duplicate Entries: ', len(statement_df[statement_df.duplicated()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:30.254573Z",
     "iopub.status.busy": "2020-12-17T13:41:30.253820Z",
     "iopub.status.idle": "2020-12-17T13:41:30.259915Z",
     "shell.execute_reply": "2020-12-17T13:41:30.259504Z"
    },
    "papermill": {
     "duration": 0.05188,
     "end_time": "2020-12-17T13:41:30.260002",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.208122",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "statement_df = statement_df.drop_duplicates()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.042448,
     "end_time": "2020-12-17T13:41:30.344928",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.302480",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Clean Statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:30.436589Z",
     "iopub.status.busy": "2020-12-17T13:41:30.435270Z",
     "iopub.status.idle": "2020-12-17T13:41:30.437645Z",
     "shell.execute_reply": "2020-12-17T13:41:30.438099Z"
    },
    "papermill": {
     "duration": 0.05132,
     "end_time": "2020-12-17T13:41:30.438215",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.386895",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def clean_statements(statement):\n",
    "    statement = re.sub(\" '\", \"'\", statement)\n",
    "    statement = re.sub(\" 's\", \"'s\", statement)\n",
    "    statement = re.sub('\\( ', '(', statement)\n",
    "    statement = re.sub(' \\)', ')', statement)\n",
    "    statement = re.sub('``', '\"', statement)\n",
    "    statement = re.sub(\"''\", '\"', statement)\n",
    "    statement = re.sub(r'\\s([?.,%:!\"](?:\\s|$))', r'\\1', statement)\n",
    "#     statement = statement.translate(str.maketrans('', '', string.punctuation))\n",
    "    return statement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:30.551298Z",
     "iopub.status.busy": "2020-12-17T13:41:30.541008Z",
     "iopub.status.idle": "2020-12-17T13:41:30.615811Z",
     "shell.execute_reply": "2020-12-17T13:41:30.616218Z"
    },
    "papermill": {
     "duration": 0.135815,
     "end_time": "2020-12-17T13:41:30.616325",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.480510",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "statement_df['statement'] = statement_df['statement'].apply(clean_statements)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.04208,
     "end_time": "2020-12-17T13:41:30.700717",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.658637",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Split Statements into Words and Characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:30.792562Z",
     "iopub.status.busy": "2020-12-17T13:41:30.791789Z",
     "iopub.status.idle": "2020-12-17T13:41:30.803921Z",
     "shell.execute_reply": "2020-12-17T13:41:30.804361Z"
    },
    "papermill": {
     "duration": 0.061783,
     "end_time": "2020-12-17T13:41:30.804462",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.742679",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "statement_df['num_char'] = statement_df['statement'].apply(len)\n",
    "statement_df['num_words'] = statement_df['statement'].apply(lambda x: len(x.split()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.041761,
     "end_time": "2020-12-17T13:41:30.889243",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.847482",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Number of Class Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:30.979123Z",
     "iopub.status.busy": "2020-12-17T13:41:30.978611Z",
     "iopub.status.idle": "2020-12-17T13:41:31.125238Z",
     "shell.execute_reply": "2020-12-17T13:41:31.125665Z"
    },
    "papermill": {
     "duration": 0.19399,
     "end_time": "2020-12-17T13:41:31.125777",
     "exception": false,
     "start_time": "2020-12-17T13:41:30.931787",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt0AAAHjCAYAAAD/r6OyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXhU9b3H8c9MAiQQCFmAEAiyJIDYYICwJKxKlBat5Yr1grJFqCBqe4UCKreAF1mUHYVrARW1cov3VrC2KhojBBO8RC1lK4TIoimJWSYGWUK23/2Dh7lGSBwwvwwJ79fz8DyZMzNnvmfCeXhzcnLGYYwxAgAAAGCN09sDAAAAAPUd0Q0AAABYRnQDAAAAlhHdAAAAgGVENwAAAGAZ0Q0AAABYRnQDuC7s27dPffr0kZ+fn9q3b+/tcazZvn27HA6HsrKyrL3G8ePH5XA49PHHH1t7DW+pz9sGwLuIbgDXlFatWik9PV2SNHDgQG3atKlG1jtz5kw1a9ZMhw4dcq//+zZu3CiHw3HZP59++mmNzIHqnThxQuPHj1dERIQaNWqksLAwJSQk6IMPPqjx14qMjNS8efMqLYuIiFB2drb69u1b4693NT7++GM5HA4dP37c26MA+JF8vT0AAFyUmZmpM2fOqEePHiopKdGnn36q/v3718i6jxw5ovHjx//gUW4fH5/LHiUODQ2tkTlQtdLSUiUkJCgiIkKbNm1Su3bt9PXXX2v79u0qKCiolRl8fHwUFhZWK68F4PrCkW4A14zU1FT17dtXvr6+Sk9PV0hIiG644YYffF52drZGjRql5s2by9/fX0OGDHEfmb54usAXX3yhOXPmyOFwXHJ08/vCwsIu+ePr6ytjjO644w717t1bpaWlkqSKigolJCSof//+KisrkyStWrVKMTExCggIUFhYmEaNGqXs7Gz3+i+eAvLOO+8oLi5O/v7+6tWrlw4cOKADBw5owIABaty4sfr06aODBw+6n7dx40b5+voqKSlJN910k/z8/NSnTx99/vnn1W5PZmamRo4cqebNmysoKEi333679u3b577/1KlTSkxMVFhYmBo1aqSIiAhNmzbtB9/3Y8eOaejQofL391eHDh30+uuvu+8bPHiwHnzwwUqPN8aoU6dOVb7/Bw4cUGZmplavXq2BAwfqhhtuUJ8+fTRz5kyNGjXK/biysjLNmzdPHTp0kJ+fn2666Sb9/ve/r7Quh8OhtWvXauzYsWratKkiIiL07LPPuu8fMmSIvvjiCz311FPun2YcP378ktNLLt7etGmThg0bpsaNG6tr167asWOH/vnPf2r48OFq0qSJunXrpp07d17R+37x+5mamqqePXuqcePG6t27tz777DP3aw8cOFCS1KFDBzkcDg0ZMuQHvy8ArlEGALwsMDDQBAYGGj8/P9OwYUMTGBhoGjdubHx8fNz3VaWiosL06dPH3HzzzWbnzp1m79695t577zXNmzc3eXl5pqyszGRnZ5u2bduaWbNmmezsbPPtt99edl0vv/yy8fHxqXbW3NxcEx4ebqZPn26MMebpp582QUFB5sSJE+7HrFy50nzwwQfm6NGjJi0tzcTFxZlBgwa57//oo4+MJBMTE2M+/PBDc+DAAdOvXz8THR1tBg4caJKSkszBgwdN//79TZ8+fSrN53A4TI8ePcz27dvN3//+d3PHHXeYsLAwc+bMmUrr/uqrr4wxxuTk5JhWrVqZKVOmmL1795pDhw6ZRx55xAQHB5vc3FxjjDGPPvqo6d69u/nkk0/MiRMnTGpqqlm3bl2V78GxY8eMJNO6dWvzhz/8wRw6dMjMnj3bOBwOk56ebowxZtOmTSYgIKDSe52UlGScTmel9+q7/vnPfxofHx8zd+5cc/78+Spff/z48SY6Otps27bNHD161Pzxj380gYGBZsOGDe7HSDItW7Y069atM5mZmWbVqlVGkklOTjbGGFNQUGDat29vpk+fbrKzs012drYpKytzb9vOnTsrbWvHjh3Nli1bzOHDh82IESNM69atzdChQ82bb75pDh8+bO6++27Ttm1bU1JS4vH7fvH7OXDgQJOSkmL+8Y9/mNtuu8107NjRlJaWmrKyMvPWW28ZSWb37t0mOzvbFBQUVPm+ALi2Ed0AvO7YsWPm2LFjplWrVuZPf/qTOXbsmOnbt69ZuXKl+76qJCUlGUnmwIED7mXFxcUmLCzMPPXUU+5lN9xwg5k/f361c7z88stGkmnSpEmlP9+P/uTkZOPj42PmzZtnfH19zZ/+9Kdq1/v5558bSSYrK8sY8/9hvGXLFvdj3njjDSPJ/M///I972ZtvvmkkucP14nxJSUnux7hcLtOkSROzfv36Suu+GN1z5841ffv2rTRPRUWF6dixo1mxYoUxxpi77rrLjB8/vtpt+K6LIfrv//7vlZbHxcWZ+++/3xhjzPnz501oaKh7LmOMGTVqlBk+fHi16/7P//xP06RJE+Pn52fi4+PNzJkz3SFvjDFHjx41DofD/OMf/6j0vKeeesrcfPPN7tuSzKOPPlrpMV26dDGPP/64+3anTp3M3LlzL7tt34/ui++VMcbs3r3bSDJLly51L7v4Pd63b58xxrP3/eL387PPPnM/ZteuXUaSOXTokDHGmJ07dxpJ1e4DAOoGzukG4HXt27fX3r17VVpaql/84hc6e/as/v73v+vPf/6zWrZsWe1zDxw4oJCQEHXr1s29rFGjRurbt68OHDhwxbP4+Phoz549lZY5HI5Kt2+55RZNnz5d8+bN05QpU3T33XdXun/79u1atGiRDh48qG+++UYVFRWSLvySYJs2bdyPu/nmm91fXzyPuHv37pcsy83NVUBAgHt5XFyc++ugoCDdeOONlU5D+a709HR99tlnlZ4vSefOndORI0ckSVOnTtXIkSP16aefaujQofrpT3+qYcOGyems/gzE784hSf3799eHH34oSWrYsKEmTJig9evXa9KkSSooKNCWLVv0xz/+sdp1TpkyRWPGjFFKSop2796t9957T0uWLNGiRYs0a9YsffrppzLGKDY2ttLzysrK5OPjU2lZTExMpdtt2rTR119/Xe3rV+VKvleSZ++7dOHv1nfXffHvx9dff60uXbpc1awArk1ENwCvuummm3TixAmVlZWptLRUgYGBqqioUHFxsTp27ChJOnjwoNq1a1flOr4fxdKF84cvt9wTkZGR1d5fXl6u1NRU+fj4KDMzs9Jrffnllxo+fLjGjh2rOXPmKDQ0VFlZWUpISFBJSUml9TRo0OCSbbjcsovRXhVjTJX3VVRUaOjQoXr++ecvuS8wMFCSNGzYMH355Zfatm2btm/frjFjxig6OloffvjhJSF7JXNMnjxZy5Yt0969e5WcnKzg4GDdeeedP7iegIAADR8+XMOHD9e8efM0adIkzZkzR4899pj7vUhLS1Pjxo0rPe/73++GDRtecv8PvZdVudLvlSfvuyQ5nc5K77Gn33MAdQ+/SAnAq9555x3t2bNHffr00fz587Vnzx7dc889mjhxovbs2aM9e/YoPDy8yuffdNNNys/Pr3Sk9/z589q9e7duuukmKzPPmzdPGRkZSk1N1WeffaZnnnnGfV96errOnTunlStXqn///urSpctVH12tyieffOL++ptvvtGhQ4d04403XvaxsbGxOnDggNq0aaPIyMhKf1q0aOF+XHBwsEaPHq3f//73+utf/6odO3ZUefT8cnNI0q5duyrNERkZqVtvvVXr16/Xhg0blJiYKF/fKz/Wc+ONN6qkpERFRUXq1auXpAv/ufn+9nTq1OmK1tuwYUOVl5df8Tye8PR992RGSdbmBFB7iG4AXnXDDTeoQ4cO2rt3r/7lX/5FkZGR2r9/v+688053pFQXarfeeqv69Omj++67T6mpqdq/f7/GjRun4uJiPfTQQ1c1U05OziV/zp8/L0nasWOHFi1apFdeeUV9+/bV+vXrNWfOHHeARkVFyeFwaNmyZTp27Ji2bt2q//iP/7iqOS7H4XBo5syZSklJ0b59+zRu3Dg1adJE991332Uf/8gjj6i8vFwjRozQzp07dfz4cX388ceaPXu20tLSJEmzZ8/Wm2++qcOHD+vIkSN6/fXXFRAQUO1PFyTpxRdf1KZNm5SRkaE5c+Zo165d+rd/+7dKj5k8ebLWrVungwcPatKkSdWu729/+5t+/vOf64033tD+/ft19OhRbd68Wc8++6z69++vFi1aKDIyUg888IB+9atf6bXXXlNmZqb+/ve/66WXXqr0nx9PdOjQQampqfryyy+Vn59fo0eXPXnfPXHDDTfI6XTqnXfeUW5uroqKimpsRgC1i+gG4HV/+9vf1LBhQ914440qKirSvn37NGjQII+e63A4tHXrVnXt2tV9Ob+cnBx98MEHV3Vt7fLycrVu3fqSP2+//bZcLpfGjBmj3/zmN/rZz34mSRo5cqQSExM1evRoFRUVqXv37nruuef0+9//Xt26ddPSpUu1cuXKK56jKk6nUwsXLtTkyZMVGxur7Oxs/fWvf1WTJk0u+/hWrVpp165dCg0N1d13360uXbro/vvv14kTJ9S6dWtJkp+fn+bMmaNevXopNjZWe/fu1bvvvlvpNIjLWbx4sdatW6fu3bvr1Vdf1SuvvKLevXtXesyIESMUGBio2267TR06dKh2fREREYqMjNTChQvVv39/RUdH63e/+53Gjx+vP//5z+7HrVu3To899pgWLFigbt26aejQoXrllVfcpyN56qmnnlJRUZG6dOmiFi1a6Msvv7yi51fHk/fd0/UsWrRIixcvVuvWrfWLX/yixmYEULscprqTAQEA14yNGzdq0qRJ7uuB1wUul0tt2rTRH/7wB40cOdLb4wCA1/CLlACAGldaWqqvv/5a8+fPV3h4uEaMGOHtkQDAq4huAECNS01N1S233KIOHTro1VdfvaKroABAfcTpJQAAAIBl/CIlAAAAYFmtnF5SUlKiuXPnqqysTOXl5erXr5/uvfdenT59WitWrFBeXp5atGihxx57zP3pXVu2bFFycrKcTqcSExPdnyx29OhRrVmzRiUlJerRo4cSExOv+gMwAAAAgNpQK6eXGGN0/vx5+fn5qaysTHPmzNGECRO0e/duBQQEaMSIEdq6datOnz6tMWPGKCsrS6tWrdLChQtVWFio+fPna9WqVXI6nXriiSeUmJioqKgoLVq0SD/72c/Uo0cP25sAAAAAXLVaOdLtcDjk5+cn6cI1cMvLy+VwOJSenq558+ZJkgYPHqx58+ZpzJgxSk9PV3x8vBo0aKCWLVsqLCxMmZmZatGihc6dO6fOnTtLkgYNGqT09HSPovvkyZPWtg8AAACo7hOUa+3qJRUVFZo1a5ZycnI0bNgwRUVFqaioSEFBQZKkoKAgnTp1StKF67pGRUW5nxscHCyXyyUfHx+FhIS4l4eEhMjlcl329ZKSkpSUlCTpwgc4XM2HZAAAAAA1odai2+l0asmSJTpz5oyWLl1a7Sd/VXXGy5WcCZOQkKCEhAT37fz8fM+HBQAAAK5QdUe6a/3qJU2aNFG3bt20Z88eBQYGqrCwUJJUWFioZs2aSbpwBLugoMD9HJfLpeDg4EuWFxQUKDg4uHY3AAAAALhCtRLdp06d0pkzZyRduJLJvn371KZNG8XGxmrHjh2SpB07dqh3796SpNjYWKWlpam0tFS5ubnKzs5WZGSkgoKC5O/vr4yMDBljlJKSotjY2NrYBAAAAOCq1crpJYWFhVqzZo0qKipkjFFcXJx69eqlzp07a8WKFUpOTlZoaKimTZsmSYqIiFBcXJymTZsmp9OpiRMnyum88P+DSZMmae3atSopKVFMTAxXLgEAAMA177r5REquXgIAAACbrqlzugEAAIDrDdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgma+3B6jLsmdM8vYIQI1ovWSDt0cAAKBe40g3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYJlvbbxIfn6+1qxZo2+++UYOh0MJCQkaPny43njjDX344Ydq1qyZJGn06NHq2bOnJGnLli1KTk6W0+lUYmKiYmJiJElHjx7VmjVrVFJSoh49eigxMVEOh6M2NgMAAAC4KrUS3T4+Pho7dqw6duyoc+fO6fHHH1f37t0lSXfccYfuuuuuSo/PyspSWlqali9frsLCQs2fP1+rVq2S0+nU+vXrNXnyZEVFRWnRokXas2ePevToURubAQAAAFyVWjm9JCgoSB07dpQk+fv7q02bNnK5XFU+Pj09XfHx8WrQoIFatmypsLAwZWZmqrCwUOfOnVPnzp3lcDg0aNAgpaen18YmAAAAAFetVo50f1dubq6OHTumyMhIHTp0SNu2bVNKSoo6duyocePGKSAgQC6XS1FRUe7nBAcHy+VyycfHRyEhIe7lISEhVcZ7UlKSkpKSJEmLFy9WaGhojW9Ldo2vEfAOG/sHAAD4f7Ua3cXFxVq2bJkmTJigxo0b6/bbb9c999wjSdq8ebNeffVVTZ06VcaYyz6/quWXk5CQoISEBPft/Pz8Hzc8UI+xfwAA8OOFh4dXeV+tXb2krKxMy5Yt08CBA9W3b19JUvPmzeV0OuV0OjV06FB98cUXki4cwS4oKHA/1+VyKTg4+JLlBQUFCg4Orq1NAAAAAK5KrUS3MUYvvPCC2rRpozvvvNO9vLCw0P317t27FRERIUmKjY1VWlqaSktLlZubq+zsbEVGRiooKEj+/v7KyMiQMUYpKSmKjY2tjU0AAAAArlqtnF5y+PBhpaSkqF27dpoxY4akC5cHTE1N1fHjx+VwONSiRQs9+OCDkqSIiAjFxcVp2rRpcjqdmjhxopzOC/8/mDRpktauXauSkhLFxMRw5RIAAABc8xzmSk6UrsNOnjxZ4+vMnjGpxtcJeEPrJRu8PQIAAHXeNXFONwAAAHC9IroBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDKiGwAAALCM6AYAAAAsI7oBAAAAy4huAAAAwDLf2niR/Px8rVmzRt98840cDocSEhI0fPhwnT59WitWrFBeXp5atGihxx57TAEBAZKkLVu2KDk5WU6nU4mJiYqJiZEkHT16VGvWrFFJSYl69OihxMREORyO2tgMAAAA4KrUypFuHx8fjR07VitWrNCCBQu0bds2ZWVlaevWrYqOjtbq1asVHR2trVu3SpKysrKUlpam5cuXa/bs2XrxxRdVUVEhSVq/fr0mT56s1atXKycnR3v27KmNTQAAAACuWq1Ed1BQkDp27ChJ8vf3V5s2beRyuZSenq7BgwdLkgYPHqz09HRJUnp6uuLj49WgQQO1bNlSYWFhyszMVGFhoc6dO6fOnTvL4XBo0KBB7ucAAAAA16paOb3ku3Jzc3Xs2DFFRkaqqKhIQUFBki6E+alTpyRJLpdLUVFR7ucEBwfL5XLJx8dHISEh7uUhISFyuVyXfZ2kpCQlJSVJkhYvXqzQ0NAa35bsGl8j4B029g8AAPD/ajW6i4uLtWzZMk2YMEGNGzeu8nHGmCtafjkJCQlKSEhw387Pz/d8UOA6w/4BAMCPFx4eXuV9tXb1krKyMi1btkwDBw5U3759JUmBgYEqLCyUJBUWFqpZs2aSLhzBLigocD/X5XIpODj4kuUFBQUKDg6urU0AAAAArkqtRLcxRi+88ILatGmjO++80708NjZWO3bskCTt2LFDvXv3di9PS0tTaWmpcnNzlZ2drcjISAUFBcnf318ZGRkyxiglJUWxsbG1sQkAAADAVauV00sOHz6slJQUtWvXTjNmzJAkjR49WiNGjNCKFSuUnJys0NBQTZs2TZIUERGhuLg4TZs2TU6nUxMnTpTTeeH/B5MmTdLatWtVUlKimJgY9ejRozY2AQAAALhqDnMlJ0rXYSdPnqzxdWbPmFTj6wS8ofWSDd4eAQCAOu+aOKcbAAAAuF4R3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABg2VVF99dff628vLyangUAAAColzyK7pUrV+rw4cOSpI8++kjTpk3TtGnTlJycbHU4AAAAoD7wKLr379+vTp06SZL+8pe/6He/+50WLlyorVu3Wh0OAAAAqA98PXlQWVmZfH195XK5dPr0aXXt2lWSVFRUZHU4AAAAoD7wKLrbt2+vLVu2KC8vTz179pQkuVwu+fv7Wx0OAAAAqA88Or1kypQp+vLLL1VSUqJRo0ZJkjIyMjRgwACrwwEAAAD1gcMYY7w9RG04efJkja8ze8akGl8n4A2tl2zw9ggAANR54eHhVd7n0eklxhh9+OGHSktL06lTp7R06VIdPHhQ33zzjeLj42tsUAAAAKA+8uj0ks2bN+ujjz7S0KFDlZ+fL0kKCQnRW2+9ZXU4AAAAoD7wKLp37NihWbNmqX///nI4HJKkli1bKjc31+pwAAAAQH3gUXRXVFTIz8+v0rLi4uJLlgEAAAC4lEfR3aNHD7366qsqLS2VdOEc782bN6tXr15WhwMAAADqA4+ie9y4cXK5XJowYYLOnj2rcePGKS8vT/fff7/t+QAAAIA6z6OrlzRu3FgzZ87UN998o/z8fIWGhqp58+a2ZwMAAADqhSqju6Ki4pJlzZo1U7NmzSrd73R6dLAcAAAAuG5VGd2jR4/2aAWbN2+usWEAAACA+qjK6H7++edrcw4AAACg3qoyulu0aHHJMmOMvv32WzVt2tR9vW4AAAAA1fPoFynPnDmjl156Sbt27VJ5ebl8fX3Vr18/JSYmKiAgwPaMAAAAQJ3m0W9Brl27ViUlJXr22Wf16quv6plnnlFpaanWrl1rez4AAACgzvMoug8cOKBHH31Ubdu2VaNGjdS2bVs9/PDDOnjwoO35AAAAgDrPo+gODw9Xbm5upWX5+fkKDw+3MhQAAABQn3h0TvdPfvITLViwQAMHDlRoaKjy8/O1c+dODRo0SMnJye7H3XrrrdYGBQAAAOoqj6L7yJEjCgsL05EjR3TkyBFJUlhYmDIyMpSRkeF+HNENAAAAXMqj6J47d67tOQAAAIB6y6Povujs2bMqLi6utCw4OLhGBwIAAADqG4+ie+/evVq3bp3y8vIuuY+PgQcAAACq51F0v/DCCxo5cqT69++vhg0b2p4JAAAAqFc8iu7S0lLdcsstcjo9usIgAAAAgO/wqKLvuOMOvfXWWzLG2J4HAAAAqHc8OtLdt29fLViwQFu3blXTpk0r3ff8889bGQwAAACoLzyK7uXLl6tr166Ki4vjnG4AAADgCnkU3bm5uXrmmWc4pxsAAAC4Ch5VdGxsrPbv3297FgAAAKBe8vjqJc8++6xuvPFGBQYGVrrvkUcesTIYAAAAUF94FN0RERGKiIiwPQsAAABQL3kU3b/85S9tzwEAAADUWx5FtySVlZXp5MmTOnXqVKXlP/nJT2p8KAAAAKA+8Si6Dx06pOXLl6u0tFTnzp2Tv7+/iouLFRISwnW6AQAAgB/g0dVLXnnlFd111116+eWX5e/vr5dfflkjR47U7bffbns+AAAAoM7z6Ej3yZMnNXz48ErLRowYoYcfflh33XXXDz5/7dq1+vzzzxUYGKhly5ZJkt544w19+OGHatasmSRp9OjR6tmzpyRpy5YtSk5OltPpVGJiomJiYiRJR48e1Zo1a1RSUqIePXooMTFRDofD860FAAAAvMCj6G7cuLHOnTunJk2aqHnz5srKylJAQICKi4s9epEhQ4bopz/9qdasWVNp+R133HFJtGdlZSktLU3Lly9XYWGh5s+fr1WrVsnpdGr9+vWaPHmyoqKitGjRIu3Zs0c9evTwcFMBAAAA7/Do9JK+ffvqb3/7myTp1ltv1VNPPaXHH39ccXFxHr1It27dFBAQ4NFj09PTFR8frwYNGqhly5YKCwtTZmamCgsLde7cOXXu3FkOh0ODBg1Senq6R+sEAAAAvMmjI90TJkxwfzp74xYAABc+SURBVP3zn/9cUVFROnfunG6++eYf9eLbtm1TSkqKOnbsqHHjxikgIEAul0tRUVHuxwQHB8vlcsnHx0chISHu5SEhIXK5XD/q9QEAAIDa4FF0f/vtt2ratKn7dteuXSVJOTk5CgsLu6oXvv3223XPPfdIkjZv3qxXX31VU6dOlTHmso+vanlVkpKSlJSUJElavHixQkNDr2rO6mTX+BoB77CxfwAAgP/nUXRPnz5dDz30UKXzp99//31t3rxZL7744lW9cPPmzd1fDx06VM8884ykC0ewCwoK3Pe5XC4FBwdfsrygoEDBwcFVrj8hIUEJCQnu2/n5+Vc1J3A9YP8AAODHCw8Pr/I+j87pfuihh/TCCy9ow4YNysnJ0cKFC/XBBx9ozpw5Vz1UYWGh++vdu3e7P2Y+NjZWaWlpKi0tVW5urrKzsxUZGamgoCD5+/srIyNDxhilpKQoNjb2ql8fAAAAqC0O4+F5G6dPn9a8efP01VdfqX///po6dap8fT37QMuVK1fq4MGD+vbbbxUYGKh7771XBw4c0PHjx+VwONSiRQs9+OCDCgoKkiS9+eab+uijj+R0OjVhwgT3EfYvvvhCa9euVUlJiWJiYvTAAw94fMnAkydPevS4K5E9Y1KNrxPwhtZLNnh7BAAA6rzqjnR7FN3FxcV66aWXtH//fvXr108fffSR7r///kqnb1zriG6gakQ3AAA/XnXR7dGh6hkzZqhz585aunSpGjdurEGDBum5555Tenq6nnjiiRobFAAAAKiPPDqne/To0Xr00UfVuHFjSVL79u21aNGiamseAAAAwAUeRXd8fPwlyxo2bKjx48fX+EAAAABAfVNtdM+cObPS7bVr11a6PWkS5zQDAAAAP6Ta6M7Jyal0+/sfu15SUlLzEwEAAAD1TLXR/UOX4/P0cn0AAADA9cyjc7oBAAAAXL1qLxlYWlqqzZs3u2+XlJRUul1WVmZvMgAAAKCeqDa6BwwYoIKCAvft/v37X3IbAAAAQPWqje6pU6fW1hwAAABAvcU53QAAAIBlRDcAAABgGdENAAAAWFZldL/22mvur/fv318rwwAAAAD1UZW/SJmUlKSxY8dKkpYsWaJXXnml1oYCAADXnm1/zvb2CECNGHZX61p/zSqju3379lq2bJnatm17yfW6v+tf//VfrQ0HAAAA1AdVRve0adOUlJSkvLw8GWMqXZ8bAAAAgOeqjO7AwECNHDlSklRRUcE1uwEAAICrVO2H41w0depUnT59Wp999plcLpeCg4PVq1cvBQQE2J4PAAAAqPM8umRgRkaGHn30UX3wwQc6ceKEkpKS9OijjyojI8P2fAAAAECd59GR7o0bN2rSpEnq37+/e1laWppefvllLVq0yNpwAAAAQH3g0ZHu7OxsxcXFVVrWr18/5eTkWBkKAAAAqE88iu6wsDClpaVVWrZr1y61atXKylAAAABAfeLR6SUTJkzQ4sWL9e677yo0NFR5eXnKzs7W448/bns+AAAAoM7zKLq7dOmi5557Tp9//rkKCwvVq1cv9ezZk6uXAAAAAB7wKLolKSAgQIMGDbI5CwAAAFAveXRONwAAAICrR3QDAAAAlhHdAAAAgGUeR3deXp7NOQAAAIB6y+PonjlzpiTpnXfesTYMAAAAUB9Ve/WSWbNmqWPHjurQoYMqKiokSf/93/+t4cOH18pwAAAAQH1Q7ZHu6dOn6+abb1ZeXp5KSko0a9YslZWVaf/+/Tp79mxtzQgAAADUadVGd0VFhfr166f7779ffn5+mjFjhowxeu+99zRjxgz9+te/rq05AQAAgDqr2tNLVq9erfz8fLVt21alpaU6c+aMGjRooN/+9reSpNOnT9fKkAAAAEBdVm10L1y4UOXl5fryyy81Z84cvfTSSyouLtb69evVoUMHdezYkY+CBwAAAH7AD169xMfHRx06dJCvr6+eeuopNWrUSDfddJNycnL0+uuv18aMAAAAQJ1W7ZHu7xo/frwkyeFwKD4+XvHx8daGAgAAAOoTj6/TPWTIEEnSc889Z2sWAAAAoF664o+B5xxuAAAA4MpccXQDAAAAuDJENwAAAGAZ0Q0AAABYRnQDAAAAlhHdAAAAgGVENwAAAGAZ0Q0AAABYRnQDAAAAlhHdAAAAgGVENwAAAGAZ0Q0AAABYRnQDAAAAlhHdAAAAgGVENwAAAGAZ0Q0AAABYRnQDAAAAlhHdAAAAgGVENwAAAGAZ0Q0AAABYRnQDAAAAlhHdAAAAgGVENwAAAGAZ0Q0AAABY5lsbL7J27Vp9/vnnCgwM1LJlyyRJp0+f1ooVK5SXl6cWLVroscceU0BAgCRpy5YtSk5OltPpVGJiomJiYiRJR48e1Zo1a1RSUqIePXooMTFRDoejNjYBAAAAuGq1cqR7yJAhevLJJyst27p1q6Kjo7V69WpFR0dr69atkqSsrCylpaVp+fLlmj17tl588UVVVFRIktavX6/Jkydr9erVysnJ0Z49e2pjfAAAAOBHqZXo7tatm/so9kXp6ekaPHiwJGnw4MFKT093L4+Pj1eDBg3UsmVLhYWFKTMzU4WFhTp37pw6d+4sh8OhQYMGuZ8DAAAAXMu8dk53UVGRgoKCJElBQUE6deqUJMnlcikkJMT9uODgYLlcrkuWh4SEyOVy1e7QAAAAwFWolXO6r4Qx5oqWVyUpKUlJSUmSpMWLFys0NPRHz/Z92TW+RsA7bOwfAOoj/uVD/eCNf/e8Ft2BgYEqLCxUUFCQCgsL1axZM0kXjmAXFBS4H+dyuRQcHHzJ8oKCAgUHB1e5/oSEBCUkJLhv5+fnW9gKoH5g/wAAXE9s/bsXHh5e5X1eO70kNjZWO3bskCTt2LFDvXv3di9PS0tTaWmpcnNzlZ2drcjISAUFBcnf318ZGRkyxiglJUWxsbHeGh8AAADwWK0c6V65cqUOHjyob7/9VlOmTNG9996rESNGaMWKFUpOTlZoaKimTZsmSYqIiFBcXJymTZsmp9OpiRMnyum88H+DSZMmae3atSopKVFMTIx69OhRG+MDAAAAP4rDXOnJ0nXUyZMna3yd2TMm1fg6AW9ovWSDt0cAUAds+zPndKN+GHZXayvrvSZPLwEAAACuF0Q3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACW+Xp7AAC4UhNe2eXtEYAasXF8nLdHAFBLONINAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJYR3QAAAIBlRDcAAABgGdENAAAAWEZ0AwAAAJZ5/WPgH374Yfn5+cnpdMrHx0eLFy/W6dOntWLFCuXl5alFixZ67LHHFBAQIEnasmWLkpOT5XQ6lZiYqJiYGC9vAQAAAFA9r0e3JM2dO1fNmjVz3966dauio6M1YsQIbd26VVu3btWYMWOUlZWltLQ0LV++XIWFhZo/f75WrVolp5MD9gAAALh2XZO1mp6ersGDB0uSBg8erPT0dPfy+Ph4NWjQQC1btlRYWJgyMzO9OSoAAADwg66JI90LFiyQJN12221KSEhQUVGRgoKCJElBQUE6deqUJMnlcikqKsr9vODgYLlcrtofGAAAALgCXo/u+fPnKzg4WEVFRXr66acVHh5e5WONMR6vNykpSUlJSZKkxYsXKzQ09EfP+n3ZNb5GwDts7B8Afljd2/f4lw/1gzf2Pa9Hd3BwsCQpMDBQvXv3VmZmpgIDA1VYWKigoCAVFha6z/cOCQlRQUGB+7kul8v9/O9LSEhQQkKC+3Z+fr7FrQDqNvYPwDvY9wDvsLXvVXfw2KvndBcXF+vcuXPur/fu3at27dopNjZWO3bskCTt2LFDvXv3liTFxsYqLS1NpaWlys3NVXZ2tiIjI702PwAAAOAJrx7pLioq0tKlSyVJ5eXlGjBggGJiYtSpUyetWLFCycnJCg0N1bRp0yRJERERiouL07Rp0+R0OjVx4kSuXAIAAIBrnleju1WrVlqyZMkly5s2bao5c+Zc9jl333237r77btujAQAAADWGw8QAAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFhGdAMAAACWEd0AAACAZUQ3AAAAYBnRDQAAAFjm6+0BrsaePXv08ssvq6KiQkOHDtWIESO8PRIAAABQpTp3pLuiokIvvviinnzySa1YsUKpqanKysry9lgAAABAlepcdGdmZiosLEytWrWSr6+v4uPjlZ6e7u2xAAAAgCrVueh2uVwKCQlx3w4JCZHL5fLiRAAAAED16tw53caYS5Y5HI5LliUlJSkpKUmStHjxYoWHh9f4LOGvv1Pj6wTww95/YqS3RwCuS4lTav7fUuB6UeeOdIeEhKigoMB9u6CgQEFBQZc8LiEhQYsXL9bixYtrczzUsMcff9zbIwDXLfY/wDvY9+qnOhfdnTp1UnZ2tnJzc1VWVqa0tDTFxsZ6eywAAACgSnXu9BIfHx898MADWrBggSoqKnTLLbcoIiLC22MBAAAAVapz0S1JPXv2VM+ePb09BmpBQkKCt0cArlvsf4B3sO/VTw5zud9MBAAAAFBj6tw53QAAAEBdQ3Tjmpebm6uPP/74qp47duzYGp4GuP6cOXNG27Ztc992uVxatmyZFycC6qf3339fO3bskCRt37690ueQvPDCC3wCdx1HdOOal5eXV2V0l5eX1/I0wPXnzJkzev/99923g4ODNX36dC9OBNRPt99+uwYPHizpQnQXFha675syZYratm3rrdFQA+rkL1KibsjNzdWiRYvUpUsXZWRkKDg4WDNnzpTL5dKLL76oU6dOqVGjRpo8ebLatGmjNWvWqFevXurXr5+kC0epX3vtNW3atElZWVmaMWOGBg8erICAAH3++ecqKSnR+fPnNWvWLD377LM6c+aMysrKNGrUKPXu3dvLWw/Univd13JycvTcc8+poqJCMTEx+stf/qLXXntNxcXFl92XNm3apJycHM2YMUPdu3fXsGHD9Mwzz2jZsmV68skn9dBDD7mvIjVv3jyNGzdO4eHheumll/TVV1+pvLxcv/zlL9kvUa/l5uZq4cKFioyM1PHjx9W6dWs98sgjysjI0Guvvaby8nJ16tRJv/rVr9SgQQO9/vrr+vTTT+Xj46Pu3btr3LhxeuONN+Tn56eWLVvqiy++0OrVq9WwYUMtWLBACxcu1NixY/XFF18oNzdXY8aMkXQhzo8ePaoHHnhAKSkpevfdd1VWVqaoqChNmjRJTifHV68VRDesys7O1m9+8xtNmTJFy5cv1yeffKLt27frV7/6lVq3bq0jR45ow4YNmjt3bpXruO+++/T222+7Pyxg+/btysjI0NKlSxUQEKDy8nL99re/VePGjXXq1CnNnj1bsbGxl/2kUqC+upJ9bePGjfrZz36mAQMGVDqC3aBBg8vuS/fdd5+++uorLVmyRNKFuLgoPj5eu3btUkREhAoLC1VYWKiOHTtq06ZN+slPfqKpU6fqzJkzevLJJxUdHS0/P79af2+A2nLy5ElNmTJFXbt21dq1a/WXv/xFSUlJ+t3vfqfw8HA9//zzev/99zV48GDt3r1bK1eulMPh0JkzZyqtp1+/fnrvvfc0duxYderU6ZL7Zs+e7Y7utLQ03X333crKylJaWprmz58vX19fbdiwQTt37nQfOYf3Ed2wqmXLlmrfvr0kqWPHjsrLy9Phw4e1fPly92PKysqueL3du3dXQECAJMkYo//6r//SP/7xDzkcDrlcLhUVFal58+Y1sg1AXXAl+1pGRoZmzJghSRowYIBee+01SVXvS9WJj4/X/Pnzde+992rXrl3un1Tt3btXn332md5++21JUklJifLz8/nxOOq1kJAQde3aVZI0aNAg/elPf1LLli0VHh4uSRo8eLC2bdumn/70p2rYsKFeeOEF9ezZU7169fL4NZo1a6ZWrVopIyNDrVu31smTJ9WlSxdt27ZNx44d0xNPPCHpwj7XrFmzmt9IXDWiG1Y1aNDA/bXT6VRRUZGaNGniPmL2XT4+PqqoqJB04R//6mK8UaNG7q8//vhjnTp1SosXL5avr68efvhhlZSU1OBWANe+K9nXqnI1+1JwcLCaNm2qEydOKC0tTQ8++KCkC/vw9OnT3bEBXA88/Qmrj4+PFi5cqH379iktLU3vvfdetT/x/b6LP2Fq06aN+vTpI4fDIWOMBg8erPvuu+9qx4dlnOiDWuXv76+WLVtq165dki78w3z8+HFJUosWLXT06FFJUnp6uvuXJP39/XXu3Lkq13n27FkFBgbK19dX+/fvV15ent2NAOqA6va1qKgo/e///q+kCz+avqiqfemH9sH4+Hi99dZbOnv2rNq1aydJuvnmm/Xuu+/q4kdBHDt2rMa3EbjW5OfnKyMjQ9KF/8RGR0crNzdXOTk5kqSUlBR169ZNxcXFOnv2rHr27KkJEya4983v8vPzq3K/69Onj9LT05Wamqr4+HhJUnR0tD755BP3T6dOnz7Nv4fXGI50o9b9+te/1vr16/Xmm2+qrKxM/fv3V/v27TV06FAtWbJETzzxhKKjo91Hs9u1aycfH59Kv0j5XQMGDNAzzzyjxx9/XO3bt1ebNm28sVnANaeqfW3ChAl67rnn9Pbbb6tnz55q3LixpKr3paZNm6pLly6aPn26YmJiNGzYsEqv069fP23cuFEjR450L7vnnnu0ceNG/fa3v5V04T/VF38vA6iv2rRpo+3bt2vdunUKCwtTYmKioqKitHz5cvcvUt522206ffq0nn32WZWWlsoYo/Hjx1+yriFDhmj9+vXuX6T8roCAALVt21ZZWVmKjIyUJLVt21ajRo3S008/LWOMfHx8NHHiRLVo0aJWth0/jE+kBIDrzPnz59WwYUM5HA6lpqYqNTVVM2fO9PZYQJ2Wm5vrvqoPcDkc6QaA68zRo0f10ksvyRijJk2a6KGHHvL2SABQ73GkGwAAALCMX6QEAAAALCO6AQAAAMuIbgAAAMAyohsAAACwjOgGAAAALCO6AQAAAMv+D0VE0PbJ5QoEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12,8))\n",
    "sns.countplot(x='sentiment', data=statement_df)\n",
    "plt.xlabel('')\n",
    "plt.ylabel('# of Examples')\n",
    "plt.title('# of Examples by Sentiment')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.042936,
     "end_time": "2020-12-17T13:41:31.212734",
     "exception": false,
     "start_time": "2020-12-17T13:41:31.169798",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Number of Characters by Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:31.314021Z",
     "iopub.status.busy": "2020-12-17T13:41:31.313173Z",
     "iopub.status.idle": "2020-12-17T13:41:31.448723Z",
     "shell.execute_reply": "2020-12-17T13:41:31.449143Z"
    },
    "papermill": {
     "duration": 0.192843,
     "end_time": "2020-12-17T13:41:31.449277",
     "exception": false,
     "start_time": "2020-12-17T13:41:31.256434",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtcAAAHjCAYAAADojTN7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzde1iUdeL//9cMBxWRM0gYHkk7eSgPKWlY0jk3s93K0rTVXGvtoJva1m5WrkWWhyzbPmbZZpuf3MvVsu1gZB4IWrE2zSxR84AfIcBBFBUF5v37w5/zjRVwtHvmZuD5uC4vuY/zmoH78uWb99zjMMYYAQAAAPjFnHYHAAAAABoLyjUAAABgEco1AAAAYBHKNQAAAGARyjUAAABgEco1AAAAYBHKNYCA9u2336pPnz5q3ry52rdv/4vO9eSTTyolJcWaYI1UY36NGvNzA+A/lGsAftG6dWvl5uZKkgYMGKB33nnHkvNOnjxZERER+uGHHzznr01VVZVeeukl9enTR61atVJkZKQuueQSTZ8+XaWlpZZk8bW9e/fK4XBo9erVdkfxiffff1/9+/dXTEyMWrZsqZSUFN111106ePCgpY+TlZUlh8OhXbt21Vj/yCOP6Msvv7T0sX6JMWPGaODAgXbHAHCGKNcAfG779u06fPiwLrnkEh0/flwbNmzQ5Zdfbsm5t23bprS0NLVv317x8fG17lNZWakbb7xRjz/+uG677TatWrVKGzdu1PTp0/Xll1/qb3/7myVZ6lJZWamG9nldDS3TqlWrNHToUF199dXKysrSpk2bNG/ePEVEROjYsWN+yRAeHq64uDi/PBaARswAgI+9+eab5qqrrjLGGJOVlWXatGnj1XH79u0zt99+u4mMjDTNmzc3aWlpJjc31xhjzM6dO42kGn+mTp1a63leeOEF43A4THZ2dq3bXS6XMcaYqVOnmk6dOpnly5ebLl26mLCwMDNw4ECzffv2GvveddddJjk52TRv3tx07tzZvPDCC8btdnv2GTlypBk0aJCZO3euadeunXE4HObQoUNm5cqVJi0tzURHR5uIiAhzxRVXmH//+981shw6dMg89NBD5txzzzWhoaGmXbt2Zvr06cYYc8rzbdeunee4lStXmtTUVNO8eXOTlJRkRo0aZUpKSk6bad26dSY1NdWEh4eb8PBw061bN/Pxxx/X+T05+Rr9/e9/Nx06dDDNmjUzgwYNMj/++KMxxpgdO3YYh8NhvvjiixrHrV692jgcDrNjx45az/vQQw+Znj171vm4J23bts0MHTrUREZGmqioKHP11VebTZs2ebYvXLjQBAUFmaysLHPJJZeYFi1amF69epkNGzYYY2r/uUlLS6vx3P77ub777rsmJSXFtGjRwtx8882mrKzMLF261HTu3NmEh4ebW2+91Rw4cKBGzsWLF5vu3bubZs2amXbt2pkJEyaY8vJyz/a0tDQzevRo8/TTT5vWrVub6OhoM3LkSM8+U6dOPSXnwoULT/v6ALAf5RqAz0RGRnqKcWhoqImMjDRhYWEmKCjIs60ubrfb9OnTx3Tv3t2sW7fObNq0ydx2220mKirKFBcXm6qqKlNQUGDOPfdcM2XKFFNQUGAOHTpU67m6d+9uBg0adNq8U6dONWFhYebaa681GzZsMN98843p0aOHueKKKzz7FBQUmIyMDPPVV1+ZH3/80SxatMi0bNnSvPHGG559Ro4caVq1amWGDBli/vOf/5hNmzaZyspK889//tMsWbLEbN261WzevNmMHj3aREdHe0qw2+02aWlppkOHDmbZsmVmx44dZs2aNWb+/PnGGGO+/vprI8ksXbrUFBQUmKKiImOMMZ999plp0aKFmTt3rsnLyzPr1683AwcONAMGDPCU/toyVVRUmOjoaDNhwgSTl5dn8vLyzD//+U+zdu3a075Gl19+uVm/fr1Zv3696dOnj+nWrZvnsa655hozatSoGscNHz7cpKen13nejIwMExkZecp/Nn6usLDQtG7d2owbN85s2rTJ/PDDD2b8+PEmJibG81osXLjQOBwOM2DAALN27Vrz/fffm6uvvtp07NjRVFZWmqqqKvPee+8ZSWb9+vWmoKDA7N+/3/Pc/rtch4WFmRtuuMFs3LjRrF692sTFxZmrr77aXH/99eabb74xa9euNQkJCWby5Mme4xYuXGiioqLMW2+95fkedu3a1QwfPtyzT1pamomMjDQPP/yw+f77781HH31kIiMjzRNPPGGMOfGfrDvvvNP069fPFBQUmIKCAnPkyJE6XxsADQflGoDP7Ny50+zcudO0bt3aLF261OzcudNcdtllZs6cOZ5tdcnMzDSSzHfffedZV1FRYRITE81TTz3lWdeuXTszbdq0enO0aNHCPPDAA6fNO3XqVBMUFOQpasacGIF0OBzm6NGjdR734IMP1iiOI0eONJGRkXWW/ZOqq6tNVFSUefvtt40x/+85nxyd/2/5+flGkvn8889rrE9LSzNTpkypsW737t1GkvnPf/5TZyaXy1Xr+epzckR127ZtnnVbt241ksynn35qjDFm6dKlJiwszDOaW1paalq0aGGWLFlS53kPHz5sBg8ebCSZxMREc/PNN5s5c+bUGH2fOnWqueyyy2oc53a7TceOHc3s2bONMSeKrSTz1VdfefbJyckxkswPP/xgjDFm3bp1RtIpP3+1leugoCBTXFzsWXf//fcbp9NZ42fkwQcfrDHq3q5dO/PXv/61xrnXrFljJHl+S5KWlma6du1aY5/f/e53pm/fvp7l0aNHe0bVAQQO5lwD8Jn27dvr4MGDqqys1M0336zY2Fht3LhRw4YNU/v27eu9u8d3332n2NhYXXjhhZ51zZo102WXXabvvvvujHIYY+RwOLzaNykpqcbc7TZt2sgYo6KiIkmS2+1WRkaGevToobi4OIWHh+vVV1/V7t27a5znggsuUHh4eI11O3fu1IgRI5SSkqKIiAhFRESorKzMc+xXX32l6Oho9erV64yeX25urubMmaPw8HDPn5Ov27Zt2+rMFB0drTFjxujaa6/V9ddfr4yMDG3duvW0jxcfH1/jrhqdO3dWXFyctmzZIkn61a9+pcjISM+bVt9++22Fh4fr5ptvrvOcYWFhev/997Vz5049++yzSkpK0rPPPqsuXbro+++/9zzPr776qsbzbNWqlXbt2lXjeTocDnXv3t2z3KZNG0nSTz/9dNrn9t/atGlTYx52YmKiEhMTa/yMJCYmen4+iouLtXv3bk2cOLFGzuuvv17SifcfnNSjR49THutsMgJoWILtDgCgcbrooou0e/duVVVVqbKyUpGRkXK73aqoqFDHjh0lSVu2bFHbtm3rPEdthfhMivJJXbp08bqQh4aG1prB7XZLkmbOnKlnn31Ws2bN0qWXXqpWrVpp9uzZ+te//lXjuJYtW55y7ptuuklxcXGaN2+ekpOTFRoaqv79++v48eOnPN6ZcLvdmjJlikaMGHHKtsTExHozvfbaa3rooYe0cuVKffrpp/rzn/+sl19+Wb/73e/OKIP52Zsjg4ODNXr0aL322mu67777tGDBAo0aNeqU17Y27du316hRozRq1ChNnz5dnTt31owZM7Rw4UK53W4NGjRIL7/88inHRUZGer52Op0KCgryLP/39/BMhISE1Fh2OBy1rjt57pN/v/jii7ryyitPOd+5557r+bq2n7WzyQigYaFcA/CJDz/8UJWVlfrtb3+r6667TrfddpuefvpphYaG6tFHH5V0YpS4LhdddJFKSkq0ZcsWzyjssWPHtH79et1///1nlGX48OGaPHmycnJy1K9fv1O2l5aWKjo62qtzrV27Vtddd51Gjx7tWffzUdO67N+/X1u2bNGHH36oa6+9VtKJW+udHPGUpJ49e8rlcmnDhg21jl6fLGPV1dU11vfq1UvffffdWd+j+eKLL9bFF1+siRMnaty4cZo/f3695bq4uFg7duxQp06dJEl5eXnav3+/LrjgAs8+9957r5555hm9+uqr2rhxo5YsWXLGuaKjo2uMCvfq1Utvvvmm2rRpoxYtWpzx+U6q63W0QuvWrZWcnKytW7fq3nvv/UXnCg0N9UlGAL7FtBAAPtGuXTt16NBBmzZt0i233KKUlBRt3rxZN910k1JSUpSSkqLg4Lr/f3/VVVepT58+uvPOO/XFF19o8+bNuvvuu1VRUaH77rvvjLI89NBDGjRokK699lq98MIL2rBhg3bv3q2PP/5YQ4YM0VtvveX1ubp06aLVq1fr888/V15env70pz/p3//+92mPi46OVnx8vF577TXl5eUpJydHw4YNq1ESr7rqKg0YMEC333673nvvPe3cuVNffPGFFixYIEmeaSgrV65UYWGh5/7cTz/9tN577z1NmDBB33zzjXbs2KGPP/5Yo0eP1tGjR+vMtH37dk2ZMkVZWVnavXu3cnJytG7duhpTcWoTFhame+65R1999ZU2bNigkSNHqmvXrkpPT/fs07ZtW1133XV66KGHNHDgQHXu3Lnecz755JN65JFH9Pnnn2vnzp369ttv9cgjj2jz5s265ZZbJEnjx49XdXW1hgwZonXr1mnXrl3KysrS448/ruzs7Pq/AT/Trl07OZ1OffjhhyoqKlJZWZnXx3pj+vTpmjt3rv7yl79o8+bN2rp1q5YvX37Gvw3o0KGDfvjhB3333XcqKSnx2y0JAfwylGsAPvOf//xHoaGhuuCCC1RWVqZvv/1WV1xxhVfHOhwOLV++XOeff75uvPFG9e7dW4WFhfr000/P+F7EISEh+uijjzRt2jT97//+r9LS0tS1a1f98Y9/VJ8+fTRy5Eivz/XnP/9ZaWlpuvnmm9WvXz+VlpbqwQcfPO1xTqdT//jHP7Rjxw5169ZNo0aN0sMPP6xzzjmnxnP+17/+pRtuuEHjxo1Tly5dNHz4cJWUlHjOMW/ePC1ZskTJycm65JJLJElXXnmlVq1apW+//VYDBgxQt27dNGHCBLVq1eqUKQw/17JlS23btk133HGHOnfurFtvvVWpqam1Trv4uXPOOUdjx47Vrbfeqssvv1wtWrTQsmXLTpnSMnbsWB0/flxjx4497euTlpam/Px83XPPPbrgggt05ZVXKicnR2+//bbGjBkj6cSocE5OjuLi4jR06FB16dJFd911l3bv3l3jdTyd1q1b69lnn1VGRobOOeeceueCn40RI0ZoyZIl+te//qU+ffqod+/eevLJJz1zv701evRo9e7dW6mpqYqPj9fixYstzQnANxzGNKBPEQAANBqvvPKKnnjiCf3f//2fmjVrZnccAPAL5lwDACxVXl6u7du364UXXtD48eMp1gCaFKaFAAAsNX78ePXp00cXXHCBpkyZYnccAPArpoUAAAAAFmHkGgAAALAI5RoAAACwCOUaAAAAsEijulvIvn377I4AAACARq6+Txhm5BoAAACwCOUaAAAAsAjlGgAAALAI5RoAAACwCOUaAAAAsAjlGgAAALAI5RoAAACwCOUaAAAAsAjlGgAAALAI5RoAAACwCOUaAAAAsAjlGgAAALAI5RoAAACwCOUaAAAAsAjlGrY7cOCAnnvuOZWVldkdBQAA4BcJ9seDHD9+XFOnTlVVVZWqq6vVt29f3XbbbSovL9fs2bNVXFys+Ph4TZgwQeHh4ZKkZcuWadWqVXI6nbrnnnvUo0cPf0SFDVasWKFt27ZpxYoVGj58uN1xAAAAzppfRq5DQkI0depUPf/885oxY4a++eYb5eXlafny5eratavmzp2rrl27avny5ZKkvXv3Kjs7W7NmzdLjjz+u119/XW632x9R4WcHDhxQVlaWjDFat24do9cAACCg+aVcOxwONW/eXJJUXV2t6upqORwO5ebmKi0tTZKUlpam3NxcSVJubq5SU1MVEhKihIQEJSYmavv27f6ICj9bsWKFqqqqJElVVVVasWKFzYkAAADOnt/mXLvdbk2aNEljxoxR165ddd5556msrEzR0dGSpOjoaB08eFCS5HK5FBsb6zk2JiZGLpfLX1HhRzk5OTWWs7OzbUoCAADwy/llzrUkOZ1OPf/88zp8+LBeeOEF7dmzp859jTFenTMzM1OZmZmSpIyMDMXFxVmSFf6TkJCg/Px8z3Lr1q35PgIAgIDlt3J9UsuWLXXhhRfqm2++UWRkpEpLSxUdHa3S0lJFRERIkmJjY7V//37PMS6XSzExMaecKz09Xenp6Z7lkpIS3z8BWKqoqKjG8k8//cT3EQAANGhJSUl1bvPLtJCDBw/q8OHDkk7cOeTbb79VmzZt1KtXL61Zs0aStGbNGvXu3VuS1KtXL2VnZ6uyslJFRUUqKChQSkqKP6LCz3r27FljuVevXjYlAQAA+OX8MnJdWlqqefPmye12yxijfv36qWfPnurcubNmz56tVatWKS4uThMnTpQkJScnq1+/fpo4caKcTqdGjx4tp5NbcjdGx44dq3cZAAAgkDiMtxOcA8C+ffvsjoAzNHbsWFVXV3uWg4KCNH/+fBsTAQAA1M/2aSEAAABAU0C5hq0uuuiiGstdu3a1KQkAAMAv5/e7hQA/99NPP9VYLiwstCkJACBQLV68uMZtXRu6k//2tW7d2uYkZyY5OVnDhg2zO0aDR7mGrSjXAICmhjfvN26Ua9gqPj5excXFNZYBADgTgTaaOmPGDEnS5MmTbU4CX2DONWz18zuFSJLb7bYpCdD0HDhwQM8995zKysrsjgIAjQblGrZyuVw1ln/+yZwAfGvFihXatm2bVqxYYXcUAGg0KNcA0AQdOHBAWVlZMsYoKyuL0WsAsAjlGrb674877927t01JgKZlxYoVnmlZVVVVjF4DgEUo17DVjTfeWO8yAN/IycnRyQ/oNcYoOzvb5kQA0DhQrmGrNWvW1LsMwDdiYmJqLMfGxtqUBAAaF8o1bJWVlVVjed26dTYlAZoW3kwMAL5BuYatqqqq6l0G4Bv9+vWrsZyammpTEgBoXCjXANAEDR48WMHBJz5HLDg4WIMHD7Y5EQA0DpRrAGiCoqKi1L9/fzkcDg0YMECRkZF2RwKARoGPPweAJmrw4MHat28fo9YAYCFGrmGrbt261Vju3r27TUmApicqKkpTpkxh1BoALES5hq0qKyvrXQYAAAgklGvY6vvvv6+xvGXLFpuSAAAA/HKUawAAAMAilGvYKigoqN5lAACAQEK5hq2cTme9ywAAAIGEJgNbxcXF1ViOj4+3KQkAAMAvR7mGrVwuV43l/fv325QEAADgl6Ncw1aXXnppjeWePXvalAQAAOCXo1zDVg6Ho95lAACAQMLHnzcyixcvVn5+vt0xvLZt27Yayzk5OSopKbEpzZlJTk7WsGHD7I4BAAAaEEauYatWrVrVuwwAABBIGLluZAJtJPXAgQN65JFHZIxRSEiIpk6dqsjISLtjAQAAnBVGrmGrqKgoRURESJL69+9PsQb86MCBA3ruuedUVlZmdxQAaDQo17BdXFycWrRoocGDB9sdBWhSVqxYoW3btmnFihV2RwGARoNyDdsFBwerbdu2jFoDfnTgwAF98cUXMsYoKyuL0WsAsAjlGgCaoBUrVsjtdkuS3G43o9cAYBHKNQA0QV9++aWqq6slSdXV1crJybE5EQA0DpRrAGiC+vbtq6CgIElSUFCQ+vXrZ3MiAGgcKNcA0AQNHjxYTueJfwKcTidvKAYAi1CuAaAJioqK0uWXXy6Hw8FtMAHAQnyIDAA0UYMHD9a+ffsYtQYAC1GuAaCJioqK0pQpU+yOAQCNCtNCAAAAAItQrgEAAACLUK4BoInas2ePxo8fr/z8fLujAECjQbkGgCZq/vz5Onr0qObPn293FABoNCjXANAE7dmzRwUFBZKkffv2MXoNABahXANAE/Tfo9WMXgOANSjXANAEnRy1Pmnfvn02JQGAxoVyDQBN0DnnnFNjOSkpyaYkANC4UK4BoAkaO3ZsvcsAgLNDuQaAJqht27ae0eukpCQlJyfbnAgAGgc+/hwALLB48eKAu+PGoUOHJEkhISGaMWOGzWm8l5ycrGHDhtkdAwBqxcg1ADRR1dXVatGihZo3b253FABoNBi5BgALBOJI6snR6smTJ9ucBAAaD0auAQAAAItQrgEAAACLUK4BAAAAi1CuAQAAAItQrgEAAACLUK4BAAAAi1CuAQAAAIv45T7XJSUlmjdvng4cOCCHw6H09HTdcMMNWrJkiT777DNFRERIOnGf2EsvvVSStGzZMq1atUpOp1P33HOPevTo4Y+oAAAAwFnzS7kOCgrSiBEj1LFjRx09elSPPvqounXrJkm68cYb9atf/arG/nv37lV2drZmzZql0tJSTZs2TS+++KKcTgbaAQAA0HD5pa1GR0erY8eOkqQWLVqoTZs2crlcde6fm5ur1NRUhYSEKCEhQYmJidq+fbs/ogIAAABnze9DwUVFRdq5c6dSUlIkSZ988okeeeQRvfLKKyovL5ckuVwuxcbGeo6JiYmpt4wDAAAADYFfpoWcVFFRoZkzZ2rUqFEKCwvTNddco1//+teSpHfffVdvvfWW7r//fhljvDpfZmamMjMzJUkZGRmKi4vzWXb4TkhIiCTx/QP8jGsPsAfXXuPmt3JdVVWlmTNnasCAAbrsssskSVFRUZ7tgwYN0nPPPSdJio2N1f79+z3bXC6XYmJiTjlnenq60tPTPcslJSW+ig8fqqyslMT3D/A3rj3AHlx7gS8pKanObX6ZFmKM0auvvqo2bdropptu8qwvLS31fL1+/XolJydLknr16qXs7GxVVlaqqKhIBQUFnmkkAAAAQEPll5HrrVu3au3atWrbtq0mTZok6cRt97744gvt2rVLDodD8fHxGjt2rCQpOTlZ/fr108SJE+V0OjV69GjuFAIAAIAGzy/l+vzzz9eSJUtOWX/ynta1GTp0qIYOHerLWAAAAIClGA4GAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALBLsjwcpKSnRvHnzdODAATkcDqWnp+uGG25QeXm5Zs+ereLiYsXHx2vChAkKDw+XJC1btkyrVq2S0+nUPffcox49evgjKgAAAHDW/FKug4KCNGLECHXs2FFHjx7Vo48+qm7dumn16tXq2rWrhgwZouXLl2v58uUaPny49u7dq+zsbM2aNUulpaWaNm2aXnzxRTmdDLQDAACg4fJLW42OjlbHjh0lSS1atFCbNm3kcrmUm5urtLQ0SVJaWppyc3MlSbm5uUpNTVVISIgSEhKUmJio7du3+yMqAAAAcNb8PhRcVFSknTt3KiUlRWVlZYqOjpZ0ooAfPHhQkuRyuRQbG+s5JiYmRi6Xy99RAQAAgDPil2khJ1VUVGjmzJkaNWqUwsLC6tzPGOPV+TIzM5WZmSlJysjIUFxcnCU54V8hISGSxPcP8DOuPcAeXHuNm9/KdVVVlWbOnKkBAwbosssukyRFRkaqtLRU0dHRKi0tVUREhCQpNjZW+/fv9xzrcrkUExNzyjnT09OVnp7uWS4pKfHxs4AvVFZWSuL7B/gb1x5gD669wJeUlFTnNr9MCzHG6NVXX1WbNm100003edb36tVLa9askSStWbNGvXv39qzPzs5WZWWlioqKVFBQoJSUFH9EBQAAAM6aX0aut27dqrVr16pt27aaNGmSJGnYsGEaMmSIZs+erVWrVikuLk4TJ06UJCUnJ6tfv36aOHGinE6nRo8ezZ1CAAAA0OD5pVyff/75WrJkSa3bnnjiiVrXDx06VEOHDvVlLAAAAMBSDAcDAAAAFqFcAwAAABahXAMAAAAWoVwDAAAAFqFcAwAAABahXAMAAAAWoVwDAAAAFqFcAwAAABahXAMAAAAWoVwDAAAAFvHq48/37t2r8PBwRUVFqaKiQu+//76cTqcGDx6sZs2a+TojAAAAEBC8Grl+8cUXdeTIEUnSW2+9pe+//155eXmaP3++T8MBAAAAgcSrkevi4mIlJSXJGKPc3FzNnDlToaGhGj9+vK/zAQAAAAHDq3IdEhKio0ePau/evYqNjVVERISqq6tVWVnp63wAAABAwPCqXF9++eV66qmnVFFRoeuuu06StHPnTiUkJPg0HAAAABBIvCrXo0aN0saNGxUUFKSLL75YkuRwODRy5EifhgMAAAACyWnLtdvt1kMPPaRZs2YpJCTEs75Tp04+DQYAAAAEmtPeLcTpdMrpdDK/GgAAADgNr6aF3HDDDZo9e7ZuueUWxcTEyOFweLa1bt3aZ+EAAACAQOJVuX7jjTckSZs2bTpl27vvvmttIgAAACBAeVWuKdAAAADA6Xn1CY0nlZSUKC8vz1dZAAAAgIDm1ch1SUmJXnzxRe3atUuStGjRIn355Zf65ptvNG7cOF/mAwAAAAKGVyPX8+fP1yWXXKK//e1vCg4+0ce7detW6xxsAAAAoKnyqlxv375dQ4YMkdP5/3YPCwvTkSNHfBYMAAAACDRelevIyEgVFhbWWLd3717FxcX5JBQAAAAQiLyacz148GA999xzGjJkiNxut7KysrRs2TINGTLE1/kAAACAgOFVub7qqqsUHh6uzz77TLGxsVq7dq1uv13r8h8AAB0sSURBVP129enTx9f5AAAAgIDhVbnetm2b+vTpc0qZ3r59u1JSUnwSDAAAAAg0Xs25/stf/lLr+unTp1saBgAAAAhk9Y5cu91uSZIxxvPnpJ9++klBQUG+TQcAAAAEkHrL9bBhwzxf33HHHTW2OZ1O3XLLLb5JBQAAAASgesv1yy+/LGOMnnzyST311FOe9Q6HQxEREQoNDfV5QAAA4D+LFy9Wfn6+3TEatT179kiSZsyYYXOSxi85ObnGYLE/1Fuu4+PjJUlz5syR0+n0fDqjJFVVVamyslIhISG+TQgAAPwmPz9fP+7YrajIRLujNF7mRJ9ylRyzOUjjdqCs8PQ7+YBXdwuZPn267rrrLnXu3Nmz7scff9Q777yjJ5980lfZAACADaIiE3Vl/zF2xwB+kc+zFtjyuF7dLWT37t0677zzaqxLSUnR7t27fRIKAAAACEReleuWLVuqrKysxrqysjI1a9bMJ6EAAACAQORVub7sssv04osvas+ePTp27Jj27Nmjl19+Wf369fN1PgAAACBgeDXn+o477tBbb72lxx57TJWVlQoNDdXAgQP9/u5LAAAAoCHzqlyHhoZqzJgxGj16tA4dOqRWrVrJ4XD4OhsAAAAQULwq1ydVVFSooqJCR48e9axr3bq15aEAAACAQORVud67d6/mzp1b691B3n33XctDAQAAAIHIqzc0LliwQBdddJHeeOMNhYWFaeHChbr66qv1+9//3tf5AAAAgIDh9X2u77rrLrVs2VLGGIWFhWn48OGMWgMAAAA/41W5DgkJUXV1tSSpVatWKikpkTFG5eXlPg0HAAAABBKv5lyff/75ysnJ0cCBA9W3b18988wzCgkJ0UUXXeTrfAAAAEDA8KpcT5w40fP1sGHDlJycrIqKCl1xxRU+CwYAAAAEmtNOC3G73XryySdVWVl54gCnU1dccYWuueYaNW/e3OcBAQAAgEBx2nLtdDpVVFQkY4w/8gAAAAABy6s3NP7617/Wa6+9puLiYrnd7hp/AAAAAJzg1Zzr//mf/5EkrV279pRt3I4PAAAAOMGrcv3yyy/7OkeDtXjxYuXn59sdo1Hbs2ePJGnGjBk2J2nckpOTNWzYMLtjAADQqHlVruPj432do8HKz8/X7rytSgyyO0njFXziFuo6tmOrvUEascJquxMAANA0eFWuJWnDhg3asmWLDh48WGP9+PHjLQ/V0CQGSWMiHHbHAM7agoO8IRkAAH/w6g2N//jHPzR//ny53W59+eWXCg8P18aNGxUWFubrfAAAAEDA8Grk+vPPP9ef/vQntW3bVqtXr9aoUaPUv39/LV261Nf5AAAAgIDhVbk+fPiw2rZte+KA4GBVVVUpJSVFW7Zs8Wk4AE0Xbyb2Pd5M7B+8mRhoWrwq14mJicrPz1dycrKSk5O1cuVKhYeHKzw83Nf5ADRR+fn5+mH7Tik8xu4ojVf1ifeS/FBYZnOQRqzcZXcCAH7mVbm+/fbbdejQIUnSnXfeqblz56qiokJjxozxaTgATVx4jNzdb7I7BXDWnBs/sDsCAD/zqlxfeumlnq/PO+88vfTSS2f0IK+88oq+/vprRUZGaubMmZKkJUuW6LPPPlNERIQkadiwYZ7HWbZsmVatWiWn06l77rlHPXr0OKPHAwAAAOzg9a34jhw5on379qmioqLG+osvvvi0xw4cOFDXXXed5s2bV2P9jTfeqF/96lc11u3du1fZ2dmaNWuWSktLNW3aNL344otyOr26sQkAAABgG6/K9erVq/X666+refPmCg0N9ax3OBxefXrjhRdeqKKiIq8C5ebmKjU1VSEhIUpISFBiYqK2b9+uzp07e3U8AAAAYBevyvXixYs1ceJEXXLJJZY++CeffKK1a9eqY8eOuvvuuxUeHi6Xy6XzzjvPs09MTIxcrtrfEJKZmanMzExJUkZGhuLi4izNJ0khISE6ZvlZAf8LCQnxyTXiKyEhIXZHACwRmNce//KhcbDj+vOqXLvdbnXv3t3SB77mmmv061//WpL07rvv6q233tL9998vY7z/JLn09HSlp6d7lktKSizNKEmVlZWWnxOwQ2VlpU+uEV/h2kNjwbUH2MdX119SUlKd27yayHzzzTdr6dKlcrvdloWKioqS0+mU0+nUoEGDtGPHDklSbGys9u/f79nP5XIpJoZbcQEAAKDhq3Pk+r777quxfODAAb3//vun3Nv6r3/961k9cGlpqaKjoyVJ69evV3JysiSpV69emjt3rm666SaVlpaqoKBAKSkpZ/UYAAAAgD/VWa4feOAByx5kzpw52rJliw4dOqRx48bptttu03fffaddu3bJ4XAoPj5eY8eOlXTik6z69euniRMnyul0avTo0dwpBAAAAAGhznJ94YUXWvYgDz/88Cnrrrrqqjr3Hzp0qIYOHWrZ4wMAAAD+UO+Q8OrVqzVnzpxat82ZM0dr1671SSgAAAAgENVbrj/99FPdfPPNtW4bMmSIPvnkE5+EAgAAAAJRveW6sLBQHTp0qHVb+/btVVhY6JNQAAAAQCCqt1y73W6Vl5fXuq28vNzSW/MBAAAAga7ect25c2etWrWq1m2ff/45H0kOAAAA/Ey9n9D4m9/8Rk8//bRKSkrUt29fRUVF6cCBA/ryyy+1Zs0aPfHEE/7KCQAAADR49ZbrlJQU/elPf9Lbb7+tlStXyhgjh8Ohzp076/HHH1enTp38lRMAAABo8Oot19KJqSFPP/20jh8/rvLycoWHhys0NNQf2QAAAICActpyfVJoaKhiYmJ8mQUAAAAIaHyuOAAAAGARyjUAAABgkTrL9aJFizxfb9682S9hAAAAgEBWZ7nOzMz0fP3888/7JQwAAAAQyOp8Q2P79u01c+ZMnXvuuaqsrNS7775b63633367z8IBAAAAgaTOcj1x4kRlZmaquLhYxhjt37/fn7kAAACAgFNnuY6MjNStt94qSXK73br//vv9FgoAAAAIRF7d5/r+++9XeXm5vvrqK7lcLsXExKhnz54KDw/3dT4AAAAgYHh1K768vDw98MAD+vTTT7V7925lZmbqgQceUF5enq/zAQAAAAHDq5HrN998U2PGjNHll1/uWZedna2FCxfq2Wef9Vk4AAAAIJB4NXJdUFCgfv361VjXt29fFRYW+iQUAAAAEIi8KteJiYnKzs6usS4nJ0etW7f2SSgAAAAgEHk1LWTUqFHKyMjQRx99pLi4OBUXF6ugoECPPvqor/MBAAAAAcOrct2lSxe99NJL+vrrr1VaWqqePXvq0ksv5W4hAAAAwM94Va4lKTw8XFdccYUvswAAAAABzas51wAAAABOj3INAAAAWIRyDQAAAFjE6znXxcXFio+P92UWAABgs6KiIpUfOqLPsxbYHQX4RQ6UFajKHeb3x/V65Hry5MmSpA8//NBnYQAAAIBAVu/I9ZQpU9SxY0d16NBBbrdbkvSPf/xDN9xwg1/CAQAA/0pISFCw85iu7D/G7ijAL/J51gLFxDXz++PWO3L9hz/8Qd27d1dxcbGOHz+uKVOmqKqqSps3b9aRI0f8lREAAAAICPWWa7fbrb59++quu+5S8+bNNWnSJBlj9PHHH2vSpEl68MEH/ZUTAAAAaPDqnRYyd+5clZSU6Nxzz1VlZaUOHz6skJAQPfLII5Kk8vJyv4QEAAAAAkG95fqZZ55RdXW19uzZoyeeeEJvvPGGKioq9Nprr6lDhw7q2LEjH4EOAAAA/P9Oe7eQoKAgdejQQcHBwXrqqafUrFkzXXTRRSosLNTf//53f2QEAAAAAoLX97keOXKkJMnhcCg1NVWpqak+CwUAAAAEIq/vcz1w4EBJ0ksvveSrLAAAAEBAO+OPP2eONQAAAFC7My7XAAAAAGpHuQYAAAAs4vUbGgHAn4qKiqTycjk3fmB3FODsle9XUdExu1MA8CNGrgEAAACLMHJ9GkVFRTpSJS04aOyOApy1gioprKjI7hhnJCEhQS53M7m732R3FOCsOTd+oISESLtjAPAjRq4BAAAAizByfRoJCQk6dqhUYyIcdkcBztqCg0bNEhLsjgEAQKPHyDUAAABgEco1AAAAYBHKNQAAAGARyjUAAABgEco1AAAAYBHKNQAAAGARyjUAAABgEco1AAAAYBHKNQAAAGARyjUAAABgEco1AAAAYBHKNQAAAGARyjUAAABgkWB/PMgrr7yir7/+WpGRkZo5c6Ykqby8XLNnz1ZxcbHi4+M1YcIEhYeHS5KWLVumVatWyel06p577lGPHj38ERMAAAD4Rfwycj1w4EA99thjNdYtX75cXbt21dy5c9W1a1ctX75ckrR3715lZ2dr1qxZevzxx/X666/L7Xb7IyYAAADwi/ilXF944YWeUemTcnNzlZaWJklKS0tTbm6uZ31qaqpCQkKUkJCgxMREbd++3R8xAQAAgF/EtjnXZWVlio6OliRFR0fr4MGDkiSXy6XY2FjPfjExMXK5XLZkBAAAAM6EX+ZcnwljjNf7ZmZmKjMzU5KUkZGhuLg4y/OEhITomOVnBfwvJCTEJ9eIr4SEhNgdAbBEYF57/MuHxsGO68+2ch0ZGanS0lJFR0ertLRUERERkqTY2Fjt37/fs5/L5VJMTEyt50hPT1d6erpnuaSkxPKclZWVlp8TsENlZaVPrhFf4dpDY8G1B9jHV9dfUlJSndtsmxbSq1cvrVmzRpK0Zs0a9e7d27M+OztblZWVKioqUkFBgVJSUuyKCQAAAHjNLyPXc+bM0ZYtW3To0CGNGzdOt912m4YMGaLZs2dr1apViouL08SJEyVJycnJ6tevnyZOnCin06nRo0fL6eR23AAAAGj4/FKuH3744VrXP/HEE7WuHzp0qIYOHerLSAAAAIDlGBIGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALGLbx58DAICG6UBZoT7PWmB3jEar/PB+SVJ4y1ibkzRuB8oKFRPXzu+PS7kGAAAeycnJdkdo9MqPVEmSYuKa2ZykcYuJa2fLzzPlGgAAeAwbNszuCI3ejBkzJEmTJ0+2OQl8gTnXAAAAgEUo1wAAAIBFKNcAAACARSjXAAAAgEUo1wAAAIBFuFsIgIar3CXnxg/sTtF4HT144u8WEfbmaMzKXZIi7U4BwI8o1wAaJO6163t79pRJktomUv58J5KfZaCJoVwDaJC4167vca9dALAec64BAAAAi1CuAQAAAItQrgEAAACLUK4BAAAAi/CGRi8UVksLDhq7YzRa+6tP/B0bZG+OxqywWmpndwgAAJoAyvVpcAsl36vas0eS1KxtW5uTNF7txM8yAAD+QLk+DW4H5nvcDgwAADQWzLkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALEK5BgAAACxCuQYAAAAsQrkGAAAALBJsd4Df//73at68uZxOp4KCgpSRkaHy8nLNnj1bxcXFio+P14QJExQeHm53VAAAAKBetpdrSZo6daoiIiI8y8uXL1fXrl01ZMgQLV++XMuXL9fw4cNtTAgAAACcXoOcFpKbm6u0tDRJUlpamnJzc21OBAAAAJxegxi5nj59uiTp6quvVnp6usrKyhQdHS1Jio6O1sGDB2s9LjMzU5mZmZKkjIwMxcXF+ScwLBUSEiJJfP8AP+PaA+zBtde42V6up02bppiYGJWVlekvf/mLkpKSvD42PT1d6enpnuWSkhJfRISPVVZWSuL7B/gb1x5gD669wFdfX7V9WkhMTIwkKTIyUr1799b27dsVGRmp0tJSSVJpaWmN+dgAAABAQ2Vrua6oqNDRo0c9X2/atElt27ZVr169tGbNGknSmjVr1Lt3bztjAgAAAF6xdVpIWVmZXnjhBUlSdXW1+vfvrx49eqhTp06aPXu2Vq1apbi4OE2cONHOmAAAAIBXbC3XrVu31vPPP3/K+latWumJJ56wIREAAABw9myfcw0AAAA0FpRrAAAAwCKUawAAAMAilGsAAADAIpRrAAAAwCKUawAAAMAilGsAAADAIpRrAAAAwCKUawAAAMAilGsAAADAIpRrAAAAwCKUawAAAMAiwXYHAIDGYPHixcrPz7c7xhnZs2ePJGnGjBk2JzkzycnJGjZsmN0xAKBWlGsAaKKaNWtmdwQAaHQo1wBgAUZSAQASc64BAAAAy1CuAQAAAItQrgEAAACLUK4BAAAAi1CuAQAAAItQrgEAAACLUK4BAAAAi1CuAQAAAItQrgEAAACLUK4BAAAAi1CuAQAAAItQrgEAAACLUK4BAAAAi1CuAQAAAItQrgEAAACLBNsdAAAA4JdYvHix8vPz7Y7htT179kiSZsyYYXOSM5OcnKxhw4bZHaPBo1wDAAD4UbNmzeyOAB+iXAMAgIDGaCoaEuZcAwAAABahXAMAAAAWoVwDAAAAFqFcAwAAABahXAMAAAAWoVwDAAAAFqFcAwAAABahXAMAAAAWoVwDAAAAFqFcAwAAABahXAMAAAAWoVwDAAAAFqFcAwAAABahXAMAAAAWoVwDAAAAFqFcAwAAABahXAMAAAAWcRhjjN0hrLJv3z67I9hu8eLFys/PtzvGGdmzZ48kqW3btjYnOTPJyckaNmyY3TEAAICfJSUl1bkt2I85gFo1a9bM7ggAAACWYOQaAAAAOAP1jVwz5xoAAACwCOUaAAAAsAjlGgAAALAI5RoAAACwCOUaAAAAsEiDvhXfN998o4ULF8rtdmvQoEEaMmSI3ZEAAACAOjXYkWu3263XX39djz32mGbPnq0vvvhCe/futTsWAAAAUKcGW663b9+uxMREtW7dWsHBwUpNTVVubq7dsQAAAIA6Ndhy7XK5FBsb61mOjY2Vy+WyMREAAABQvwY757q2D450OBw1ljMzM5WZmSlJysjIUFxcnF+yAQAAALVpsOU6NjZW+/fv9yzv379f0dHRNfZJT09Xenq6Z7mkpMRv+QAAANA0BeTHn3fq1EkFBQUqKipSVVWVsrOz1atXL7tjAQAAAHVqsCPXQUFB+u1vf6vp06fL7XbryiuvVHJyst2xAAAAgDo5TG2TmwPUvn377I4AAACARi4gp4UAAAAAgYZyDQAAAFiEcg0AAABYhHINAAAAWKRRvaERAAAAsBMj12gQHn30UbsjAE0S1x5gD669xotyDQAAAFiEcg0AAABYhHKNBiE9Pd3uCECTxLUH2INrr/HiDY0AAACARRi5BgAAACxCuUaDUVRUpKysrLM6dsSIERanAZqew4cP65NPPvEsu1wuzZw508ZEQOO0cuVKrVmzRpK0evVquVwuz7ZXX31Ve/futSsaLEC5RoNRXFxcZ7murq72cxqg6Tl8+LBWrlzpWY6JidEf/vAHGxMBjdM111yjtLQ0SSfKdWlpqWfbuHHjdO6559oVDRYItjsAAl9RUZGeffZZdenSRXl5eYqJidHkyZPlcrn0+uuv6+DBg2rWrJl+97vfqU2bNpo3b5569uypvn37Sjox6rxo0SK988472rt3ryZNmqS0tDSFh4fr66+/1vHjx3Xs2DFNmTJFM2bM0OHDh1VVVaU77rhDvXv3tvnZA/5zptdaYWGhXnrpJbndbvXo0UMffPCBFi1apIqKilqvpXfeeUeFhYWaNGmSunXrpmuvvVbPPfecZs6cqccee0z33XefkpOTJUlPPvmk7r77biUlJemNN95Qfn6+qqur9Zvf/IbrEo1aUVGRnnnmGaWkpGjXrl0655xzNH78eOXl5WnRokWqrq5Wp06ddO+99yokJER///vftWHDBgUFBalbt266++67tWTJEjVv3lwJCQnasWOH5s6dq9DQUE2fPl3PPPOMRowYoR07dqioqEjDhw+XdKKE//jjj/rtb3+rtWvX6qOPPlJVVZXOO+88jRkzRk4n46UNBeUaligoKNBDDz2kcePGadasWfryyy+1evVq3XvvvTrnnHO0bds2LViwQFOnTq3zHHfeeadWrFjhubH+6tWrlZeXpxdeeEHh4eGqrq7WI488orCwMB08eFCPP/64evXqJYfD4a+nCdjuTK61N998U9dff7369+9fY0Q6JCSk1mvpzjvvVH5+vp5//nlJJ0rESampqcrJyVFycrJKS0tVWlqqjh076p133tHFF1+s+++/X4cPH9Zjjz2mrl27qnnz5n5/bQB/2bdvn8aNG6fzzz9fr7zyij744ANlZmbqz3/+s5KSkvTyyy9r5cqVSktL0/r16zVnzhw5HA4dPny4xnn69u2rjz/+WCNGjFCnTp1O2fb44497ynV2draGDh2qvXv3Kjs7W9OmTVNwcLAWLFigdevWeUbCYT/KNSyRkJCg9u3bS5I6duyo4uJibd26VbNmzfLsU1VVdcbn7datm8LDwyVJxhgtXrxY33//vRwOh1wul8rKyhQVFWXJcwACwZlca3l5eZo0aZIkqX///lq0aJGkuq+l+qSmpmratGm67bbblJOT4/nN06ZNm/TVV19pxYoVkqTjx4+rpKSEX2ujUYuNjdX5558vSbriiiu0dOlSJSQkKCkpSZKUlpamTz75RNddd51CQ0P16quv6tJLL1XPnj29foyIiAi1bt1aeXl5Ouecc7Rv3z516dJFn3zyiXbu3Kk//vGPkk5ccxEREdY/SZw1yjUsERIS4vna6XSqrKxMLVu29IyA/VxQUJDcbrekE//I11e6mzVr5vk6KytLBw8eVEZGhoKDg/X73/9ex48ft/BZAA3fmVxrdTmbaykmJkatWrXS7t27lZ2drbFjx0o6cQ3/4Q9/8JQKoCnw9jemQUFBeuaZZ/Ttt98qOztbH3/8cb2/wf1vJ39j1KZNG/Xp00cOh0PGGKWlpenOO+882/jwMSbowCdatGihhIQE5eTkSDrxD/CuXbskSfHx8frxxx8lSbm5uZ43K7Zo0UJHjx6t85xHjhxRZGSkgoODtXnzZhUXF/v2SQABoL5r7bzzztO///1vSSd+pXxSXdfS6a7B1NRUvffeezpy5Ijatm0rSerevbs++ugjnfzIhJ07d1r+HIGGpqSkRHl5eZJO/Ge1a9euKioqUmFhoSRp7dq1uvDCC1VRUaEjR47o0ksv1ahRo/6/9u7Y5bQ4juP4R0ryMCpFMpBJySBhUJI/gMGGTBYLid2Cstgo+Qss/gHLKeV/MJnMFlHnGW735Ole3eXc67l6v8YznPP7Dd/O93xPn3Os2nzkdruf1l06ndbhcJBhGMpms5KkRCKh/X5vvW26XC7cD78ZJtf4azqdjpbLpTabje73u3K5nCKRiIrFoqbTqYbDoRKJhDWdDofDcjqdXwKNj/L5vMbjsQaDgSKRiILB4Cu2BXw7z2qt0WhoPp9ru90qlUrJ4/FIel5LPp9P8Xhc3W5XyWRS5XL5y3UymYzW67UqlYp1rFqtar1eq9frSfrx8PwzNwG8q2AwqN1up8VioUAgoGazqVgsptlsZgUaS6WSLpeLJpOJbrebTNNUvV7/5VyFQkHL5dIKND7yer0KhUI6nU6KRqOSpFAopFqtptFoJNM05XQ61Wq15Pf7/8ne8Wf8oREA3tT1epXL5ZLD4ZBhGDIMQ/1+/9XLAv5r5/PZ+ooO8DtMrgHgTR2PR61WK5mmqY+PD7Xb7VcvCQDeHpNrAAAAwCYEGgEAAACb0FwDAAAANqG5BgAAAGxCcw0AAADYhOYaAAAAsAnNNQAAAGCTTxFbraeVLsHdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12,8))\n",
    "sns.boxplot(x='sentiment', y='num_char', data=statement_df)\n",
    "plt.xlabel('')\n",
    "plt.ylabel('# of Characters')\n",
    "plt.title('# of Characters by Sentiment')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.043891,
     "end_time": "2020-12-17T13:41:31.538092",
     "exception": false,
     "start_time": "2020-12-17T13:41:31.494201",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Number of Words by Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:31.636383Z",
     "iopub.status.busy": "2020-12-17T13:41:31.635243Z",
     "iopub.status.idle": "2020-12-17T13:41:31.768160Z",
     "shell.execute_reply": "2020-12-17T13:41:31.767725Z"
    },
    "papermill": {
     "duration": 0.186044,
     "end_time": "2020-12-17T13:41:31.768270",
     "exception": false,
     "start_time": "2020-12-17T13:41:31.582226",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAHjCAYAAAAKUSgCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de1RVdcL/8c/hgEIiN0EMBe8VGlaKhpc8Wsx0cVwx3XGsnMnKaqYZK7XpYjU2pVjassuUWTnjTD5PPbaYx/JSZGLeCrPyHmoq+gACcgsvKLB/f7g8vxgF+SrsDYf3a62W7M05e38O+I2PX/b5bpdlWZYAAAAANIif0wEAAACAloQCDQAAABigQAMAAAAGKNAAAACAAQo0AAAAYIACDQAAABigQANoFTZv3qxBgwYpMDBQ3bp1czpOnZ599ln16tXrnJ8/f/58+fv7N2Ki5sOXXxuAloUCDaBZiY6OVlZWliTpqquu0vvvv98ox508ebJCQkK0Y8cO7/F/bvfu3XK5XFq6dGmt/Q8//HCd++Pi4holW0uxevVq/fKXv1RUVJQCAwPVtWtX3XLLLdq3b1+jnufAgQNyuVxauXJlrf233367/u///q9Rz3U+nn/++Wb9jzEATYcCDaDZ2LVrlw4fPqwrrrhCx48f14YNGzR06NBGOfbOnTvl8XjUrVs3RUVFnfb5nj17qlu3bvr8889r7V+xYoXi4uLOuD85OfmcsliWpRMnTpzTc52yfft2/eIXv1Dv3r2VkZGh7du3a/78+erWrZvKy8ttyRAUFKTo6GhbzgUA9aFAA2g21qxZoyuvvFL+/v7KyspShw4d1LVr17M+Ly8vT3fccYfCwsIUFBSkESNGaMOGDZKkvXv3yuVyaffu3Zo6dapcLpeeffbZMx7nmmuuqVWUCwoKtH37dj355JOn7d+6dau3QP/www8aNWqUgoODFRwcrNGjR2vXrl3ex5+69OCLL77QFVdcobZt22r58uWqrKzUAw88oNDQUIWHh+uBBx5QZWVlrUxbt27Vtddeq7CwMLVr107x8fFasGDBWb8mGRkZ6tu3rwIDAzVo0CBt3LhRkvTTTz+pffv2p83s7927V35+fqfN+p6yfPlyBQcH6/XXX9dll12m7t27a+TIkXrppZeUkJDgfdzBgwc1btw4RUVFqX379ho6dKhWrVrl/fzKlSvlcrn02Wefafjw4brgggvUp08fLV++3PuY2NhYSdLIkSPlcrm8s7z/eQnHz7+uCQkJCgoKksfjUW5urlatWqUrrrhC7dq1U3Jy8mkz15999pmGDh2qoKAgde7cWb/97W916NAh7+fHjRun5ORkzZ07V127dlVISIhuvPFGFRYWes/99NNPa9++fXK5XPX+vQLggywAcFhoaKgVGhpqBQYGWm3atLFCQ0OtCy64wHK73d7P1aWmpsYaNGiQddlll1lffvmltWnTJuu2226zwsLCrMLCQquqqsrKy8uzunTpYk2ZMsXKy8uzfvrppzMe6/3337dcLpdVWFhoWZZlLVy40Lr88sutoqIiy+1219ovycrLy7OOHDlixcXFWVdffbW1YcMGa8OGDdaIESOsnj17WpWVlZZlWdZ7771nuVwuKzEx0fr888+t3bt3WwUFBdaf/vQnKyoqykpPT7e2b99uPfroo1b79u2tnj17ejMlJCRYqamp1tatW63du3dbS5YssRYvXlzn1+PUua644gpr5cqV1vfff2+NGjXK6tSpk3X48GHLsizrvvvus0aMGFHreU899ZTVu3dvq6am5ozH/a//+i/L7XZbS5YsqfPcR44cseLj462bbrrJysrKsnbu3Gk9//zzVps2baxt27ZZlmVZX3zxhSXJ6tevn7V06VIrOzvbuvPOO63Q0FCrpKTEsizL2rhxoyXJWrRokZWXl2cVFBR4X5vb7T7ttXo8Hmv9+vXWN998Y/Xq1csaNmyY5fF4rHXr1lkbN260Lr74Yuu2227zPu/zzz+3goKCrDlz5ljZ2dnW119/bY0YMcK66qqrvK//7rvvtkJCQqw77rjD2rx5s7VmzRorLi7Ouuuuu7yvdcqUKVaXLl2svLy8ev9eAfA9FGgAjtuzZ4+1Z88eKzo62lq0aJG1Z88e68orr7ReeeUV7+fqkpGRYUmytm7d6t137Ngxq1OnTtZzzz3n3de1a1dr2rRp9ebIz8+3JFkffPCBZVmWde+991oTJ060LMuyLrvsslr7L730UsuyLGvevHlWUFCQt1yfOk5gYKD197//3bKsk0VPkrVq1SrvYyoqKqy2bdtac+fOrZVhwIABtQp0SEiI9d5779Wb++dOnSsjI8O7r7i42GrXrp319ttvW5ZlWd98840lycrOzrYsy7KqqqqsLl26WGlpaXUet7q62rrnnnssl8tlRUREWNdee601ffp0Kycnp9a5O3fubJ04caLWc0eOHGn98Y9/tCzr/xfoRYsWeT+fl5dnSbKWLVtmWZZl7d+/35JkffHFF6e9tv8s0JKsb7/91rsvLS3NkmRt2LDBu2/WrFlWhw4dvNsej8eaMmVKrWPv27ev1rHuvvtuKzIy0jp27Jj3MS+++KLVqVMn7/a0adOsrl271vk1A+C7uIQDgONOXUd74sQJ3XjjjerQoYO+//57paamqlu3bvW+UWvr1q3q0KGD+vTp493Xtm1bXXnlldq6datRjujoaF166aXKyMiQJH3++ee6+uqrJUlXX311rf2nLt/YunWr+vTpo8jIyFrHufjii087/8CBA70f7969W5WVlRoyZEitxwwbNqzW9mOPPabx48drxIgRevbZZ72XYpzN4MGDvR+Hh4crPj5e27ZtkyT1799fiYmJmjdvniRp6dKlOnjwoO6+++46j+fn56d58+YpNzdXr732mvr06aO33npL8fHx3ss+srKylJ+fr7CwMO/lLMHBwfryyy+1c+fOWse7/PLLvR936tRJbrdbBw8ebNBr+zmXy1XrEpJOnTpJkvr161dr36FDh1RdXe3N+corr9TKeOrvz89zxsfHq23btt7tzp07n1NGAL6HAg3AUX379lVwcLAGDRqk0tJShYaGKjo6WseOHVOPHj0UHBysnJyceo/hcrlO22dZ1hn3n01ycrI+//xz5eTkaN++fRo+fLikk9fjntr/448/6pprrjE6v9vtVmBgYK3P1/Xcn3v66aeVnZ2t2267TVu2bFFSUpKeeuop49d16nynTJgwQfPnz9eJEyc0b948paSkqGPHjmc9TqdOnZSamqpZs2Zpx44d6tq1q5577jlJUk1NjeLj4/Xdd9/V+m/79u16++23ax2nTZs2px27pqbG+HX5+fnJ7XZ7t099PQMCAk7bd+prUFNToylTppyWc+fOnbr++uvrzOhyuU77OgJonSjQABy1ZMkSfffddxo0aJCmTZum7777Trfccovuueceb7GJiYmp8/l9+/ZVUVGRd3ZVkiorK/X111+rb9++xnmuueYa7d69W++++64SExMVEhIiSRo+fLj27t2rd999V/7+/vJ4PN7zb926VUVFRd5jHDx4UNnZ2fWev1evXmrTpo3WrFlTa//atWtPe2yPHj304IMP6n/+53/0l7/8RX/729/O+jrWr1/v/bi0tFQ7duxQfHy8d98dd9yhY8eO6a233tInn3yie++996zH/E9t2rRRjx49VFBQIElKTEzUjz/+qJCQEPXq1avWf/V9D890XEneGePGlpiYqK1bt56WsVevXgoODjbK2VQZATRvFGgAjuratau6d++uTZs26de//rV69eqlLVu26Fe/+pW31NR384yrr75agwYN0pgxY7RmzRpt2bJFd911l44dO6YHHnjAOI/H45G/v79mzZrlvXxDkkJDQ9W/f3/NmjVLgwYNUvv27SVJY8aMUVRUlG6//XZt3LhR33zzje644w517txZt99+e53nadeunSZMmKCnnnpK//u//6sffvhBkydP1o4dO7yPqaio0EMPPaQVK1Zoz549+vbbb7Vs2bJal6ucicvl0uTJk7Vq1Spt3rxZd911l9q1a6cxY8bUOv/YsWP16KOPKi4u7qxL8r311lu6//77tXz5cu3atUvbt2/XjBkztHTpUv3617+WJP3mN79R9+7dNWrUKH366afau3evvvrqK7344otKT0+v9/g/FxkZqeDgYH366afKz89XSUlJg5/bEH/5y1/073//WxMnTtR3332n3bt3a9myZbrnnnt09OjRBh+ne/fuys/P17p161RUVKQjR440ak4AzRcFGoDjvv32W7Vp00bx8fEqKyvT5s2bvZdOnI3L5VJ6erouueQSjRo1SgMHDlR+fr4+++yzWtclN1T79u01cOBA/fTTT7UKtHTyMo6ffvqpVtkMCgrSp59+qrZt22r48OHyeDxq166dli1bdsbLFH5u+vTpSklJ0Z133um9hOWhhx7yft7f318lJSW65557FB8fr2uvvVbR0dFnvbmMn5+fXnjhBd1///1KTExUXl6ePvnkE7Vr167W4+677z4dP35c48ePP+ulJIMGDVJlZaUeeugh9evXT0OGDNEHH3ygV155RX/5y18kSYGBgcrMzFRiYqJ++9vf6qKLLtJNN92kr7/+ukHLEf48/+uvv64PPvhAsbGxuuKKKxr83IYYOXKkVqxYoc2bN+uqq65Sv379NHHiRLVv377WpR9nk5KSoltvvVWjRo1SVFSU0tLSGjUngObLZXFBFwC0SkuWLFFKSopycnK8b74DAJwdBRoAWpkjR44oJydH48aN0yWXXKL58+c7HQkAWhQu4QCAViYtLU2XXnqp/Pz8uOwAAM4BM9AAAACAAWagAQAAAAMUaAAAAMAABRoAAAAwUPfdCZqx3NxcpyMAAADAx9V1F1VmoAEAAAADFGgAAADAAAUaAAAAMECBBgAAAAxQoAEAAAADFGgAAADAAAUaAAAAMECBBgAAAAxQoAEAAAADFGgAAADAAAUaAAAAMECBBgAAAAxQoAEAAAADFGgAAADAAAUaAHxcaWmpZsyYobKyMqejAIBPoEADgI9bvHixdu7cqcWLFzsdBQB8AgUaAHxYaWmp1qxZI8uytHr1amahAaARUKABwIctXrxYNTU1kqSamhpmoQGgEVCgAcCHrV+/XtXV1ZKk6upqrVu3zuFEANDyUaABwIclJSXJ7XZLktxutwYPHuxwIgBo+SjQAODDRo8eLT+/k/+r9/Pz0+jRox1OBAAtHwUaAHxYWFiYhg4dKpfLpWHDhik0NNTpSADQ4vk7HQAA0LRGjx6t3NxcZp8BoJG4LMuynA5hKjc31+kIAAAA8HExMTFn3M8lHAAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIAB25axe+ihhxQYGCg/Pz+53W5Nnz5dFRUVmj17tgoLCxUVFaWJEycqODjYrkiwWWlpqd566y1NmDCBtWgBGzH2AGcw9nyXrTPQzzzzjGbOnKnp06dLktLT05WQkKA5c+YoISFB6enpdsaBzRYvXqydO3dq8eLFTkcBWhXGHuAMxp7vcvQSjqysLHk8HkmSx+NRVlaWk3HQhEpLS7VmzRpZlqXVq1errKzM6UhAq8DYA5zB2PNtthbov/71r5oyZYoyMjIkSWVlZQoPD5ckhYeHq7y83M44sNHixYtVU1MjSaqpqeFf44BNGHuAMxh7vs22a6CnTZumiIgIlZWV6fnnn6/zzi5nkpGR4S3d06dPV2RkZFPFRBP56quvVF1dLUmqrq7W+vXr9ac//cnhVIDvY+wBzmDs+TbbCnRERIQkKTQ0VAMHDtSuXbsUGhqqkpIShYeHq6SkRCEhIWd8bnJyspKTk73bRUVFtmRG47nyyiv15Zdfqrq6Wm63W0lJSXwfARsw9gBnMPZ8g6O38j527JiOHj3q/XjTpk2Ki4tTYmKiMjMzJUmZmZkaOHCgHXHggNGjR8vP7+RfNz8/P40ePdrhREDrwNgDnMHY8222zECXlZXppZdeknTy1xjDhg3T5Zdfrp49e2r27NlasWKFIiMj9cgjj9gRBw4ICwvT0KFDlZmZqWHDhrGcD2ATxh7gDMaeb7OlQEdHR2vmzJmn7W/fvr2mTp1qRwQ0A6NHj1Zubi7/CgdsxtgDnOHxePTVV195VxyD73BZlmU5HcJUbm6u0xEAAADqtWDBAmVmZmrEiBEaO3as03FwDhy9BhoAAKA1YR1o30aBBgAAaGSsA+3bKNAAAACNbP369bXWgV63bp3DidCYKNAAAACNLCkpSW63W5Lkdrs1ePBghxOhMVGgAQAAGhnrQPs2CjQAAEAjO7UOtMvlYh1oH0SBhm1ycnL0+9//Xvv373c6CtCqlJaWasaMGawCANhs9OjR6t27N7PPPogCDdvMnTtXR48e1dy5c52OArQqixcv1s6dO1kFALBZWFiYpkyZwuyzD6JAwxY5OTnKy8uTdPJGOMxCA/ZgLVoAaHwUaNjiP2edmYUG7MFatADQ+CjQsMWp2edTuB07YA/WogWAxkeBhi0uvPDCWtt13VseQONiLVoAaHwUaNjivvvuq3cbQNNgLVoAaHwUaNgiLi5OISEhkqTQ0FDFxsY6nAhoHcLCwjRw4EBJ0qBBg1gNAAAaAQUatikvL5ckVgEAbGZZVq0/AQDnhwINW3zxxRe1tjMzMx1KArQupaWl2rBhgyQpKyuLf8ACQCPwdzoAWod//vOftbb/8Y9/yOPxOJQGaD3OtIzd2LFjHU4FmFu4cGGLu4fAwYMHJUnR0dEOJzETGxur1NRUp2M0a8xAA4APYxk7wDmVlZWqrKx0OgaaADPQAODDkpKS9OWXX6q6uppl7NCitcQZ0bS0NEnS5MmTHU6CxsYMNGzxn78yvuuuuxxKArQuLGMHAI2PAg1bjBw5stY21z8D9ggLC9PQoUPlcrk0bNgwlrEDgEZAgYZt+vXrJ0nq37+/w0mA1uXUmGPsAUDjoEDDNps2bZIkbdy40eEkQOuycOFCWZalhQsXOh0FAHwCBRq2+OSTT2ptL1u2zKEkQOuSk5OjvLw8SVJubm6LWwYMAJojCjRs8dFHH9Xa/vDDDx1KArQuc+fOrXcbAGCOAg0APuzU7PMpubm5DiUBAN9BgQYAH3bhhRfW2o6JiXEoCQD4Dgo0bHHTTTfV2r711lsdSgK0Lvfdd1+92wAAcxRo2GLUqFG1tq+77jqHkgCtS1xcnHcWOiYmRrGxsQ4nAoCWjwIN23Tp0kWS1K1bN2eDAK3Mfffdp6CgIGafAaCR+DsdAK3HgQMHJEl79+51NgjQysTFxem1115zOgYA+AxmoGGLd955p9b23//+d4eSAAAAnB9moGGLtWvX1tpetWqV7r77bofSAOdu4cKFLe5mJAcPHpQkRUdHO5yk4WJjY5Wamup0DAA4Iwo0APi4yspKpyMAgE+hQAOAgZY4K5qWliZJmjx5ssNJAMA3cA00bDFkyJBa28OHD3coCQAAwPmhQMMW99xzT61trn8GAAAtFQUatmEdaAAA4Aso0LAN60ADAABfQIGGLT755JNa28uWLXMoCQAAwPmhQMMWH330Ua3tDz/80KEkAAAA54cCDQAAABigQAMAAAAGKNCwxU033VRr+9Zbb3UoCQAAwPmhQMMWo0aNqrV93XXXOZQEAADg/FCgYZtTs9DMPgMAgJbM3+kAaD1GjRp12kw0AABAS0OBbqEWLlyo/fv3Ox3DyMGDByVJ0dHRDidpuNjYWKWmpjodAwAANCMUaNimsrLS6QgAAADnjQLdQrXEWdG0tDRJ0uTJkx1OAgAAcO54EyEAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGPC382Q1NTV6/PHHFRERoccff1wVFRWaPXu2CgsLFRUVpYkTJyo4ONjOSAAAAIARW2eglyxZos6dO3u309PTlZCQoDlz5ighIUHp6el2xgEAAACM2VagDx06pI0bN+qaa67x7svKypLH45EkeTweZWVl2RUHAAAAOCe2Fej58+dr7Nixcrlc3n1lZWUKDw+XJIWHh6u8vNyuOAAAAMA5seUa6G+++UahoaHq0aOHtm7davz8jIwMZWRkSJKmT5+uyMjIxo4IGwQEBEgS3z/AZow9wBmMPd9lS4H+4YcftGHDBn377bc6fvy4jh49qjlz5ig0NFQlJSUKDw9XSUmJQkJCzvj85ORkJScne7eLiorsiI1GduLECUl8/wC7MfYAZzD2Wr6YmJgz7relQI8ZM0ZjxoyRJG3dulWLFy/Www8/rAULFigzM1MpKSnKzMzUwIED7YgDAAAAnDNH14FOSUnRpk2b9PDDD2vTpk1KSUlxMg4AAABwVrauAy1Jffv2Vd++fSVJ7du319SpU+2OAAAAAJwz7kQIAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABgwL8hD9qyZYs6duyojh07qqSkRP/617/k5+enMWPGKCwsrKkzAgAAAM1Ggwr0O++8oyeffFKS9I9//EOS5Ha79dZbb2nKlClnff7x48f1zDPPqKqqStXV1UpKStJtt92miooKzZ49W4WFhYqKitLEiRMVHBx8Hi8HAAAAaFoNKtDFxcWKjIxUdXW1vv/+e73xxhvy9/fX/fff36CTBAQE6JlnnlFgYKCqqqo0depUXX755fr666+VkJCglJQUpaenKz09XWPHjj2vFwQAAAA0pQZdAx0UFKTS0lJt27ZNXbp0UWBgoCSpqqqqQSdxuVze51RXV6u6uloul0tZWVnyeDySJI/Ho6ysrHN5DQAAAIBtGjQDfd111+nPf/6zqqqqNG7cOEnSjh071Llz5wafqKamRlOmTFF+fr6uvfZa9e7dW2VlZQoPD5ckhYeHq7y8/IzPzcjIUEZGhiRp+vTpioyMbPB50XwEBARIEt8/wGaMPcAZjD3f1aACnZKSokGDBsnPz0+dOnWSJEVERGjChAkNPpGfn59mzpypw4cP66WXXlJOTk6Dn5ucnKzk5GTvdlFRUYOfi+bjxIkTkvj+AXZj7AHOYOy1fDExMWfc36ACfaYD1HXAs2nXrp369Omj7777TqGhoSopKVF4eLhKSkoUEhJyTscEAAAA7FJngX7ggQcadIC//e1vZ31MeXm53G632rVrp+PHj2vz5s268cYblZiYqMzMTKWkpCgzM1MDBw5seHIAAADAAXUW6D/84Q/ej3ft2qXMzExdf/31ioqKUmFhoZYvX67hw4c36CQlJSV6/fXXVVNTI8uyNHjwYA0YMEAXXXSRZs+erRUrVigyMlKPPPLI+b8iAAAAoAnVWaD79Onj/fjUOtARERHefVdccYVeeOEFjR49+qwn6dq1q9LS0k7b3759e02dOtU0MwAAAOCYBi1jV1xc7F2G7pTAwEAVFxc3SSgAAACguWrQmwgTExM1Y8YM3XzzzYqIiNChQ4eUnp6uAQMGNHU+AAAAoFlpUIG+99579eGHH+rtt99WcXGxIiIilJSUpFtvvbWp8wEAAADNylkLdE1NjebOnav7779fv/nNb+zIBAAAADRbZ70G2s/PT5s2bZLL5bIjDwAAANCsNehNhKNGjdIHH3ygqqqqps4DAAAANGsNugZ62bJlKi0t1SeffHLa3QIbciMVAAAAwFc0qED//KYqAAAAQGvWoAL985uqAAAAAK1Zgwp0VVWVPvroI61atUolJSUKDw/X8OHDddNNN8nfv0GHAAAAzcjChQu1f/9+p2P4tJycHEk6492Y0XhiY2OVmppq6zkb1H7/+c9/avfu3br33nsVFRWlwsJCLVq0SEeOHNG4ceOaOCIAAGhs+/fv14+79ykstJPTUXyXdbJmFRdVOhzEd5WW5Tty3gYV6PXr12vmzJlq3769JCkmJkbdu3fXpEmTKNAAALRQYaGdNHLYeKdjAOfsi9XzHDlvg5axsyyrqXMAAAAALUKDZqAHDx6sGTNm6JZbblFkZKSKioq0aNEiDR48uKnzAQAAAM1Kgwr02LFjtWjRIr3zzjveNxEOHTpUN998c1PnAwAAAJqVegv0vHnzFB8fr/j4eN1+++26/fbb7coFAAAANEv1Fujq6mp9+OGHysvLU8eOHdWnTx/16dNH8fHx6tixo10ZAfgoltGyB0tp2cOJpbQAOKPeAn3//fdLksrKyrR9+3bt2LFDS5Ys0ZtvvqmwsDDFx8fr4YcftiUoAN+zf/9+7di1RwqOcDqKb6t2SZJ25Jc5HMSHVRQ7nQCAjRp0DXRoaKiSkpKUlJSknJwcbdy4UZ988onWrFlDgQZwfoIjVHPZr5xOAZwXv+8/djoCABvVW6Aty9KePXu0bds2bdu2Tbt27VKHDh0UHx+v8ePH65JLLrErJwAAANAs1Fugx40bp4iICA0ZMkTXX3+9evfurdlVFswAABW4SURBVMDAQLuyAQAAAM1OvTdSObXO86pVq7Rq1SqtXbtWubm5tgQDAAAAmqN6Z6AnTJggSSovL9e2bdu0Y8cOLV++XKWlpbrooosUHx+vG264wZagAAAAQHPQoDcRhoSEnPFNhF9//TUFGgAAAK1Kg99EeGoZu2PHjqlXr166+uqr1adPH7tyAgAAAM3CWd9EaFmWevfurfj4eI0aNUq9e/dWQECAXfkAAACAZqXeAv3kk0+qZ8+ecrvdduUBAAAAmrV6C/RFF11kVw4AAACgRah3GTsAAAAAtVGgAQAAAAN1FugFCxZ4P96yZYstYQAAAIDmrs4CnZGR4f145syZtoQBAAAAmrs630TYrVs3vfzyy+rSpYtOnDih//7v/z7j426//fYmC2eXhQsXav/+/U7H8Hk5OTmSpLS0NIeT+LbY2FilpqY6HQMAAJ9VZ4F+5JFHlJGRocLCQlmWpUOHDtmZy1b79+/Xvuwf1InV+pqUf/XJPyt3/+BsEB+WX+10AgAAfF+dBTo0NFQ333yzJKmmpkYPPvigbaGc0MktjQ9xOR0DOC/zyi2nIwAA4PPqXQf6lAcffFAVFRX65ptvVFxcrIiICA0YMEDBwcFNnQ8AAABoVhq0jF12drb+8Ic/6LPPPtO+ffuUkZGhP/zhD8rOzm7qfAAAAECz0qAZ6Pnz52v8+PEaOnSod9/atWv13nvv6cUXX2yycAAAAEBz06AZ6Ly8PA0ePLjWvqSkJOXn5zdJKAAAAKC5alCB7tSpk9auXVtr37p16xQdHd0koQAAAIDmqkGXcIwbN07Tp0/X0qVLFRkZqcLCQuXl5enxxx9v6nwAAABAs9KgAn3xxRfr1Vdf1caNG1VSUqIBAwaof//+rMIBAACAVqdBBVqSgoODNXz48KbMAgAAADR7DboGGgAAAMBJFGgAAADAAAUaAAAAMNDgAl1YWNiUOQAAAIAWocEFevLkyZKkJUuWNFkYAAAAoLmrdxWOKVOmqEePHurevbtqamokSR9++KFuuOEGW8IBAAAAzU29M9CPPvqoLrvsMhUWFur48eOaMmWKqqqqtGXLFh05csSujAAAAECzUW+BrqmpUVJSkn7zm98oMDBQkyZNkmVZWrZsmSZNmqSHH37YrpwAAABAs1DvJRxz5sxRUVGRunTpohMnTujw4cMKCAjQY489JkmqqKiwJSQAAADQXNRboF944QVVV1crJydHU6dO1bvvvqtjx47p7bffVvfu3dWjRw9u5w0AAIBW5ayrcLjdbnXv3l3+/v567rnn1LZtW/Xt21f5+fn617/+ZUdGAAAAoNmodwb65+6++25Jksvl0pAhQzRkyJAmCwUAAAA0Vw1eB3rEiBGSpFdffbWpsgAAAADNnvGtvLnmGQAAAK2ZcYEGAAAAWjMKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCgwTdSAQAAvqOgoEAVPx3RF6vnOR0FOGelZXmqqrnA9vMyAw0AAAAYYAYaAIBWqGPHjvL3q9TIYeOdjgKcsy9Wz1NEZFvbz8sMNAAAAGCAAg0AAAAYsOUSjqKiIr3++usqLS2Vy+VScnKybrjhBlVUVGj27NkqLCxUVFSUJk6cqODgYDsiAQAAAOfElgLtdrt15513qkePHjp69Kgef/xx9evXTytXrlRCQoJSUlKUnp6u9PR0jR071o5IAAAAwDmx5RKO8PBw9ejRQ5IUFBSkzp07q7i4WFlZWfJ4PJIkj8ejrKwsO+IAAAAA58z2VTgKCgq0Z88e9erVS2VlZQoPD5d0smSXl5fbHQeAgwoKCqSKCvl9/7HTUYDzU3FIBQWVTqcAYBNbC/SxY8f08ssva9y4cbrggoYvep2RkaGMjAxJ0vTp0xUZGdmouQICAsT/9uArAgICGn2MNBU/P97HDN/h5+fXYsaedPL/FeKnH3yAEz/3bCvQVVVVevnll3XVVVfpyiuvlCSFhoaqpKRE4eHhKikpUUhIyBmfm5ycrOTkZO92UVFRo2Y7ceJEox4PcNKJEycafYw0lcjISBVVBajmsl85HQU4L37ff6zIyNAWM/YkfvbBdzTlz72YmJgz7rdl+seyLL355pvq3LmzfvWr//+DMjExUZmZmZKkzMxMDRw40I44AAAAwDmzZQb6hx9+0KpVqxQXF6dJkyZJklJTU5WSkqLZs2drxYoVioyM1COPPGJHHAAAAOCc2VKgL7nkEn3wwQdn/NzUqVPtiAAAAAA0Ct7BAwAAABigQAMAAAAGKNAAAACAAQo0AAAAYIACDQAAABigQAMAAAAGKNAAAACAAQo0AAAAYIACDQAAABiw5U6EzV1BQYGOVEnzyi2nowDnJa9KuqCgwOkYAAD4NGagAQAAAAPMQEvq2LGjKn8q0fgQl9NRgPMyr9xS244dnY4BAIBPYwYaAAAAMECBBgAAAAxQoAEAAAADFGgAAADAAAUaAAAAMECBBgAAAAxQoAEAAAADFGgAAADAAAUaAAAAMECBBgAAAAxQoAEAAAADFGgAAADAAAUaAAAAMECBBgAAAAxQoAEAAAADFGgAAADAAAUaAAAAMECBBgAAAAxQoAEAAAADFGgAAADAAAUaAAAAMODvdAAArVxFsfy+/9jpFL7taPnJP4NCnM3hyyqKJYU6ncJYaVm+vlg9z+kYPqvi8CFJUnC7Dg4n8V2lZfmKiOxq+3kp0AAcExsb63SEViEnp0ySFNep5RW8liO0xf19bml5W6KKI1WSpIjItg4n8V0RkV0d+btMgQbgmNTUVKcjtAppaWmSpMmTJzucBM0J46/pMfZ8F9dAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGPB3OkBzkV8tzSu3nI7h0w5Vn/yzg9vZHL4sv1rq6nQIAAB8HAVaUmxsrNMRWoWqnBxJUtu4OIeT+K6u4u8zAABNjQItKTU11ekIrUJaWpokafLkyQ4nAQAAOHdcAw0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABvztOMkbb7yhjRs3KjQ0VC+//LIkqaKiQrNnz1ZhYaGioqI0ceJEBQcH2xEHAAAAOGe2zECPGDFCTzzxRK196enpSkhI0Jw5c5SQkKD09HQ7ogAAAADnxZYC3adPn9Nml7OysuTxeCRJHo9HWVlZdkQBAAAAzostl3CcSVlZmcLDwyVJ4eHhKi8vr/OxGRkZysjIkCRNnz5dkZGRtmRE4woICJAkvn+AzRh7gDMYe77LsQJtIjk5WcnJyd7toqIiB9PgXJ04cUIS3z/Abow9wBmMvZYvJibmjPsdW4UjNDRUJSUlkqSSkhKFhIQ4FQUAAABoMMcKdGJiojIzMyVJmZmZGjhwoFNRAAAAgAaz5RKOV155Rdu2bdNPP/2kCRMm6LbbblNKSopmz56tFStWKDIyUo888ogdUQAAAIDzYkuB/tOf/nTG/VOnTrXj9AAAAECj4U6EAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABijQAAAAgAEKNAAAAGCAAg0AAAAYoEADAAAABvydDgAALcnChQu1f/9+p2MYycnJkSSlpaU5nKThYmNjlZqa6nQMNCOMPfsw/s6OAg0APq5t27ZORwBaJcae76JAA4ABZmUAZzD20JxwDTQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAYo0AAAAIABl2VZltMhTOXm5jodwXELFy7U/v37nY5hJCcnR5IUFxfncJKGi42NVWpqqtMxAACAA2JiYs6439/mHGjF2rZt63QEAACA88YMNAAAAHAGdc1Acw00AAAAYIACDQAAABhw/Bro7777Tu+9955qamp0zTXXKCUlxelIAAAAQJ0cnYGuqanRO++8oyeeeEKzZ8/WmjVrdODAAScjAQAAAPVytEDv2rVLnTp1UnR0tPz9/TVkyBBlZWU5GQkAAACol6MFuri4WB06dPBud+jQQcXFxQ4mAgAAAOrn6DXQZ1pBz+VynbYvIyNDGRkZkqTp06crMjKyybMBAAAAZ+Joge7QoYMOHTrk3T506JDCw8NPe1xycrKSk5O920VFRbbkAwAAQOvVLNeB7tmzp/Ly8lRQUKCqqiqtXbtWiYmJTkYCAAAA6uXoDLTb7dbvfvc7/fWvf1VNTY1Gjhyp2NhYJyMBAAAA9eJW3gAAAMAZNMtLOAAAAICWhgINAAAAGKBAAwAAAAYo0AAAAIABCjQAAABggAINAAAAGKBAAwAAAAZa5DrQAAAAgFOYgYatHn/8cacjAK0SYw9wBmPPN1GgAQAAAAMUaAAAAMAABRq2Sk5OdjoC0Cox9gBnMPZ8E28iBAAAAAwwAw0AAAAYoEDDVgUFBVq9evU5PffOO+9s5DRA63T48GEtX77cu11cXKyXX37ZwUSA7/n000+VmZkpSVq5cqWKi4u9n3vzzTd14MABp6KhEVCgYavCwsI6C3R1dbXNaYDW6fDhw/r000+92xEREXr00UcdTAT4nl/+8pfyeDySThbokpIS7+cmTJigLl26OBUNjcDf6QBoGQoKCvTiiy/q4osvVnZ2tiIiIjR58mQVFxfrnXfeUXl5udq2bav7779fnTt31uuvv64BAwYoKSlJ0snZ4wULFuj999/XgQMHNGnSJHk8HgUHB2vjxo06fvy4KisrNWXKFKWlpenw4cOqqqrSHXfcoYEDBzr86gF7mY63/Px8vfrqq6qpqdHll1+ujz/+WAsWLNCxY8fOOJ7ef/995efna9KkSerXr5+uvfZazZgxQy+//LKeeOIJPfDAA4qNjZUkPfvss7rrrrsUExOjd999V/v371d1dbVuvfVWxiZ8VkFBgV544QX16tVLe/fu1YUXXqjf//73ys7O1oIFC1RdXa2ePXvq3nvvVUBAgP71r39pw4YNcrvd6tevn+666y598MEHCgwMVMeOHbV7927NmTNHbdq00V//+le98MILuvPOO7V7924VFBRo7Nixkk4W7R9//FG/+93vtGrVKi1dulRVVVXq3bu3xo8fLz8/5j2bCwo0GiwvL09//OMfNWHCBM2aNUvr16/XypUrde+99+rCCy/Uzp07NW/ePD3zzDN1HmPMmDFavHixd2H5lStXKjs7Wy+99JKCg4NVXV2txx57TBdccIHKy8v15JNPKjExUS6Xy66XCTQLJuNt/vz5uv766zVs2LBaM8sBAQFnHE9jxozR/v37NXPmTEkny8IpQ4YM0bp16xQbG6uSkhKVlJSoR48eev/993XppZfqwQcf1OHDh/XEE08oISFBgYGBtn9tADvk5uZqwoQJuuSSS/TGG2/o448/VkZGhp5++mnFxMTotdde06effiqPx6Ovv/5ar7zyilwulw4fPlzrOElJSVq2bJnuvPNO9ezZ87TPPfnkk94CvXbtWt100006cOCA1q5dq2nTpsnf31/z5s3Tl19+6Z3RhvMo0Giwjh07qlu3bpKkHj16qLCwUD/88INmzZrlfUxVVZXxcfv166fg4GBJkmVZWrhwobZv3y6Xy6Xi4mKVlZUpLCysUV4D0FKYjLfs7GxNmjRJkjRs2DAtWLBAUt3jqT5DhgzRtGnTdNttt2ndunXe3yJt2rRJ33zzjRYvXixJOn78uIqKivg1NHxWhw4ddMkll0iShg8frkWLFqljx46KiYmRJHk8Hi1fvlzXXXed2rRpozfffFP9+/fXgAEDGnyOkJAQRUdHKzs7WxdeeKFyc3N18cUXa/ny5dqzZ4/+/Oc/Szo53kJCQhr/ReKcUaDRYAEBAd6P/fz8VFZWpnbt2nlnsX7O7XarpqZG0skf4vUV67Zt23o/Xr16tcrLyzV9+nT5+/vroYce0vHjxxvxVQAtg8l4q8u5jKeIiAi1b99e+/bt09q1a3XfffdJOjmOH330UW95AHxdQ3/z6Xa79cILL2jz5s1au3atli1bVu9vYv/Tqd/6dO7cWYMGDZLL5ZJlWfJ4PBozZsy5xkcT42IanLOgoCB17NhR69atk3TyB+zevXslSVFRUfrxxx8lSVlZWd43CAYFBeno0aN1HvPIkSMKDQ2Vv7+/tmzZosLCwqZ9EUALUd946927t7766itJJ38FfEpd4+ls43DIkCH697//rSNHjiguLk6SdNlll2np0qU6deuAPXv2NPprBJqToqIiZWdnSzr5j9GEhAQVFBQoPz9fkrRq1Sr16dNHx44d05EjR9S/f3+NGzfOOy5/LjAwsM4xN2jQIGVlZWnNmjUaMmSIJCkhIUHr16/3/saooqKCn4fNDDPQOC8PP/yw3n77bX300UeqqqrS0KFD1a1bN11zzTWaOXOm/vznPyshIcE7yxwXFye3213rTYQ/N2zYMM2YMUOPP/64unXrps6dOzvxsoBmqa7xNm7cOL366qtavHix+vfvrwsuuEBS3eOpffv2uvjii/Xoo4/q8ssv17XXXlvrPElJSZo/f75uvvlm775bbrlF8+fP12OPPSbp5D+ST72XAfBFnTt31sqVKzV37lx16tRJv/3tb9W7d2/NmjXL+ybCX/ziF6qoqFBaWppOnDghy7J09913n3asESNG6O233/a+ifDngoOD1aVLFx04cEC9evWSJHXp0kV33HGHnn/+eVmWJbfbrXvuuUdRUVG2vHacHXciBIAWrrKyUm3atJHL5dKaNWu0Zs0aTZ482elYQItVUFDgXZkGOBNmoAGghfvxxx/17rvvyrIstWvXTg888IDTkQDApzEDDQAAABjgTYQAAACAAQo0AAAAYIACDQAAABigQAMAAAAGKNAAAACAAQo0AAAAYOD/AfQBXhcSEn50AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12,8))\n",
    "sns.boxplot(x='sentiment', y='num_words', data=statement_df)\n",
    "plt.xlabel('')\n",
    "plt.ylabel('# of Words')\n",
    "plt.title('# of Words by Sentiment')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.045433,
     "end_time": "2020-12-17T13:41:31.859139",
     "exception": false,
     "start_time": "2020-12-17T13:41:31.813706",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Identify Longest and Shortest Statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:31.954101Z",
     "iopub.status.busy": "2020-12-17T13:41:31.953264Z",
     "iopub.status.idle": "2020-12-17T13:41:31.957358Z",
     "shell.execute_reply": "2020-12-17T13:41:31.956899Z"
    },
    "papermill": {
     "duration": 0.053366,
     "end_time": "2020-12-17T13:41:31.957454",
     "exception": false,
     "start_time": "2020-12-17T13:41:31.904088",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Longest Statement:  52 words.\n"
     ]
    }
   ],
   "source": [
    "print('Longest Statement: ', statement_df['num_words'].max(), 'words.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:32.052385Z",
     "iopub.status.busy": "2020-12-17T13:41:32.051503Z",
     "iopub.status.idle": "2020-12-17T13:41:32.056452Z",
     "shell.execute_reply": "2020-12-17T13:41:32.055839Z"
    },
    "papermill": {
     "duration": 0.054016,
     "end_time": "2020-12-17T13:41:32.056571",
     "exception": false,
     "start_time": "2020-12-17T13:41:32.002555",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shortest Statement:  1 words.\n"
     ]
    }
   ],
   "source": [
    "print('Shortest Statement: ', statement_df['num_words'].min(), 'words.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:32.159369Z",
     "iopub.status.busy": "2020-12-17T13:41:32.154861Z",
     "iopub.status.idle": "2020-12-17T13:41:32.162219Z",
     "shell.execute_reply": "2020-12-17T13:41:32.161767Z"
    },
    "papermill": {
     "duration": 0.05948,
     "end_time": "2020-12-17T13:41:32.162309",
     "exception": false,
     "start_time": "2020-12-17T13:41:32.102829",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pos_statements = ' '.join(statement_df.loc[statement_df['sentiment'] == 'positive']['statement'].values)\n",
    "neg_statements = ' '.join(statement_df.loc[statement_df['sentiment'] == 'negative']['statement'].values)\n",
    "neutral_statements = ' '.join(statement_df.loc[statement_df['sentiment'] == 'neutral']['statement'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:32.314764Z",
     "iopub.status.busy": "2020-12-17T13:41:32.313127Z",
     "iopub.status.idle": "2020-12-17T13:41:32.315472Z",
     "shell.execute_reply": "2020-12-17T13:41:32.315883Z"
    },
    "papermill": {
     "duration": 0.107915,
     "end_time": "2020-12-17T13:41:32.315982",
     "exception": false,
     "start_time": "2020-12-17T13:41:32.208067",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pos_statements = re.sub('[^A-Za-z]+', ' ', pos_statements).strip().lower()\n",
    "neg_statements = re.sub('[^A-Za-z]+', ' ', neg_statements).strip().lower()\n",
    "neutral_statements = re.sub('[^A-Za-z]+', ' ', neutral_statements).strip().lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:32.413559Z",
     "iopub.status.busy": "2020-12-17T13:41:32.412935Z",
     "iopub.status.idle": "2020-12-17T13:41:33.143073Z",
     "shell.execute_reply": "2020-12-17T13:41:33.142623Z"
    },
    "papermill": {
     "duration": 0.780446,
     "end_time": "2020-12-17T13:41:33.143194",
     "exception": false,
     "start_time": "2020-12-17T13:41:32.362748",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pos_tokens = word_tokenize(pos_statements)\n",
    "neg_tokens = word_tokenize(neg_statements)\n",
    "neutral_tokens = word_tokenize(neutral_statements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:33.240152Z",
     "iopub.status.busy": "2020-12-17T13:41:33.239518Z",
     "iopub.status.idle": "2020-12-17T13:41:33.247268Z",
     "shell.execute_reply": "2020-12-17T13:41:33.246800Z"
    },
    "papermill": {
     "duration": 0.057514,
     "end_time": "2020-12-17T13:41:33.247363",
     "exception": false,
     "start_time": "2020-12-17T13:41:33.189849",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:33.379050Z",
     "iopub.status.busy": "2020-12-17T13:41:33.363639Z",
     "iopub.status.idle": "2020-12-17T13:41:33.528208Z",
     "shell.execute_reply": "2020-12-17T13:41:33.527742Z"
    },
    "papermill": {
     "duration": 0.234587,
     "end_time": "2020-12-17T13:41:33.528306",
     "exception": false,
     "start_time": "2020-12-17T13:41:33.293719",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pos_tokens_nostop = [w for w in pos_tokens if w not in stop_words]\n",
    "neg_tokens_nostop = [w for w in neg_tokens if w not in stop_words]\n",
    "neutral_tokens_nostop = [w for w in neutral_tokens if w not in stop_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:33.633161Z",
     "iopub.status.busy": "2020-12-17T13:41:33.632022Z",
     "iopub.status.idle": "2020-12-17T13:41:33.634875Z",
     "shell.execute_reply": "2020-12-17T13:41:33.634462Z"
    },
    "papermill": {
     "duration": 0.060131,
     "end_time": "2020-12-17T13:41:33.634964",
     "exception": false,
     "start_time": "2020-12-17T13:41:33.574833",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pos_freq = Counter(pos_tokens_nostop)\n",
    "neg_freq = Counter(neg_tokens_nostop)\n",
    "neutral_freq = Counter(neutral_tokens_nostop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:33.750151Z",
     "iopub.status.busy": "2020-12-17T13:41:33.748462Z",
     "iopub.status.idle": "2020-12-17T13:41:33.750835Z",
     "shell.execute_reply": "2020-12-17T13:41:33.751271Z"
    },
    "papermill": {
     "duration": 0.069942,
     "end_time": "2020-12-17T13:41:33.751380",
     "exception": false,
     "start_time": "2020-12-17T13:41:33.681438",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "top_10_pos = pos_freq.most_common()[:10]\n",
    "top_10_neg = neg_freq.most_common()[:10]\n",
    "top_10_neutral = neutral_freq.most_common()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:33.851170Z",
     "iopub.status.busy": "2020-12-17T13:41:33.849527Z",
     "iopub.status.idle": "2020-12-17T13:41:33.851919Z",
     "shell.execute_reply": "2020-12-17T13:41:33.852342Z"
    },
    "papermill": {
     "duration": 0.054413,
     "end_time": "2020-12-17T13:41:33.852445",
     "exception": false,
     "start_time": "2020-12-17T13:41:33.798032",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_ngrams(input, n):\n",
    "    return [tuple(input[i:i+n]) for i in range(len(input) - n + 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:33.986587Z",
     "iopub.status.busy": "2020-12-17T13:41:33.985911Z",
     "iopub.status.idle": "2020-12-17T13:41:33.989210Z",
     "shell.execute_reply": "2020-12-17T13:41:33.989625Z"
    },
    "papermill": {
     "duration": 0.08999,
     "end_time": "2020-12-17T13:41:33.989726",
     "exception": false,
     "start_time": "2020-12-17T13:41:33.899736",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('eur', 639),\n",
       " ('mn', 264),\n",
       " ('company', 241),\n",
       " ('said', 230),\n",
       " ('year', 212),\n",
       " ('finnish', 202),\n",
       " ('net', 196),\n",
       " ('sales', 192),\n",
       " ('profit', 192),\n",
       " ('million', 170)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_10_pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:34.088111Z",
     "iopub.status.busy": "2020-12-17T13:41:34.087588Z",
     "iopub.status.idle": "2020-12-17T13:41:34.093284Z",
     "shell.execute_reply": "2020-12-17T13:41:34.092858Z"
    },
    "papermill": {
     "duration": 0.056409,
     "end_time": "2020-12-17T13:41:34.093383",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.036974",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('eur', 417),\n",
       " ('mn', 237),\n",
       " ('profit', 156),\n",
       " ('net', 104),\n",
       " ('finnish', 102),\n",
       " ('company', 100),\n",
       " ('year', 100),\n",
       " ('sales', 98),\n",
       " ('operating', 97),\n",
       " ('period', 90)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_10_neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:34.194328Z",
     "iopub.status.busy": "2020-12-17T13:41:34.193563Z",
     "iopub.status.idle": "2020-12-17T13:41:34.197165Z",
     "shell.execute_reply": "2020-12-17T13:41:34.196732Z"
    },
    "papermill": {
     "duration": 0.055664,
     "end_time": "2020-12-17T13:41:34.197265",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.141601",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('company', 508),\n",
       " ('eur', 359),\n",
       " ('said', 238),\n",
       " ('finland', 227),\n",
       " ('finnish', 220),\n",
       " ('business', 195),\n",
       " ('million', 193),\n",
       " ('new', 189),\n",
       " ('group', 187),\n",
       " ('sales', 165)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_10_neutral"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.048143,
     "end_time": "2020-12-17T13:41:34.293760",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.245617",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Sentiment Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:34.398687Z",
     "iopub.status.busy": "2020-12-17T13:41:34.397851Z",
     "iopub.status.idle": "2020-12-17T13:41:34.400440Z",
     "shell.execute_reply": "2020-12-17T13:41:34.400003Z"
    },
    "papermill": {
     "duration": 0.0553,
     "end_time": "2020-12-17T13:41:34.400532",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.345232",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "MAX_LENGTH = 64\n",
    "BATCH_SIZE = 16\n",
    "NUM_CLASSES = 3\n",
    "EPOCHS = 5\n",
    "DROPOUT_PROB = 0.1\n",
    "WEIGHT_DECAY = 0.01\n",
    "NFOLDS = 10\n",
    "LEARNING_RATE = 2e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:34.503011Z",
     "iopub.status.busy": "2020-12-17T13:41:34.502329Z",
     "iopub.status.idle": "2020-12-17T13:41:34.504733Z",
     "shell.execute_reply": "2020-12-17T13:41:34.505122Z"
    },
    "papermill": {
     "duration": 0.056504,
     "end_time": "2020-12-17T13:41:34.505246",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.448742",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "statement_df['sentiment'] = le.fit_transform(statement_df['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:34.607017Z",
     "iopub.status.busy": "2020-12-17T13:41:34.606316Z",
     "iopub.status.idle": "2020-12-17T13:41:34.609167Z",
     "shell.execute_reply": "2020-12-17T13:41:34.608751Z"
    },
    "papermill": {
     "duration": 0.054688,
     "end_time": "2020-12-17T13:41:34.609271",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.554583",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def format_time(elapsed):\n",
    "    return str(datetime.timedelta(seconds=int(round((elapsed)))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:34.716146Z",
     "iopub.status.busy": "2020-12-17T13:41:34.715483Z",
     "iopub.status.idle": "2020-12-17T13:41:34.718004Z",
     "shell.execute_reply": "2020-12-17T13:41:34.718475Z"
    },
    "papermill": {
     "duration": 0.060027,
     "end_time": "2020-12-17T13:41:34.718575",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.658548",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class StatementDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, statements, labels, tokenizer, max_length):\n",
    "        self.statements = statements\n",
    "        self.labels = labels\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_length = max_length\n",
    "        \n",
    "    def __len__(self):\n",
    "        \n",
    "        return len(self.statements)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        statement = str(self.statements[idx])\n",
    "        label = self.labels[idx]\n",
    "        \n",
    "        encoding = self.tokenizer.encode_plus(\n",
    "            statement,\n",
    "            max_length=self.max_length,\n",
    "            padding='max_length',\n",
    "            add_special_tokens=True, \n",
    "            return_token_type_ids=False,\n",
    "            truncation=True,\n",
    "            return_attention_mask=True,\n",
    "            return_tensors='pt'  \n",
    "        ) \n",
    "        \n",
    "        return {\n",
    "            'statement_text': statement,\n",
    "            'input_ids': encoding['input_ids'].flatten(),\n",
    "            'attention_mask': encoding['attention_mask'].flatten(),\n",
    "            'labels': torch.tensor(label, dtype=torch.long)\n",
    "        }\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:34.821102Z",
     "iopub.status.busy": "2020-12-17T13:41:34.820556Z",
     "iopub.status.idle": "2020-12-17T13:41:34.835157Z",
     "shell.execute_reply": "2020-12-17T13:41:34.834755Z"
    },
    "papermill": {
     "duration": 0.068309,
     "end_time": "2020-12-17T13:41:34.835261",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.766952",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_train, df_test = train_test_split(statement_df, \n",
    "                                     test_size=0.2, \n",
    "                                     random_state=RANDOM_SEED, \n",
    "                                     stratify=statement_df['sentiment'].values)\n",
    "\n",
    "df_val, df_test = train_test_split(df_test, \n",
    "                                   test_size=0.5, \n",
    "                                   random_state=RANDOM_SEED, \n",
    "                                   stratify=df_test['sentiment'].values)\n",
    "\n",
    "df_train_full = pd.concat([df_train, df_val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:34.942881Z",
     "iopub.status.busy": "2020-12-17T13:41:34.942142Z",
     "iopub.status.idle": "2020-12-17T13:41:34.945086Z",
     "shell.execute_reply": "2020-12-17T13:41:34.944571Z"
    },
    "papermill": {
     "duration": 0.057,
     "end_time": "2020-12-17T13:41:34.945171",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.888171",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_dataset(df, tokenizer, max_length):\n",
    "    ds = StatementDataset(statements=df['statement'].to_numpy(),\n",
    "                          labels=df['sentiment'].to_numpy(),\n",
    "                          tokenizer=tokenizer,\n",
    "                          max_length=max_length)\n",
    "    return ds\n",
    "\n",
    "def create_dataloader(ds, batch_size):\n",
    "    return DataLoader(ds, batch_size, num_workers=4)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:35.050521Z",
     "iopub.status.busy": "2020-12-17T13:41:35.049815Z",
     "iopub.status.idle": "2020-12-17T13:41:35.052778Z",
     "shell.execute_reply": "2020-12-17T13:41:35.052336Z"
    },
    "papermill": {
     "duration": 0.058851,
     "end_time": "2020-12-17T13:41:35.052864",
     "exception": false,
     "start_time": "2020-12-17T13:41:34.994013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def cv_ensemble_performance(preds, labels):\n",
    "    preds = np.array(preds)\n",
    "    summed = np.sum(preds, axis=0)\n",
    "    preds = np.argmax(summed, axis=1)\n",
    "    print(confusion_matrix(y_true=labels, y_pred=preds))\n",
    "    print('')\n",
    "    print(classification_report(y_true=labels, y_pred=preds, digits=3, target_names=le.classes_))\n",
    "    \n",
    "def single_model_performance(preds, labels):\n",
    "    print(confusion_matrix(y_true=labels, y_pred=preds))\n",
    "    print('')\n",
    "    print(classification_report(y_true=labels, y_pred=preds, digits=3, target_names=le.classes_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:35.165470Z",
     "iopub.status.busy": "2020-12-17T13:41:35.164506Z",
     "iopub.status.idle": "2020-12-17T13:41:35.166922Z",
     "shell.execute_reply": "2020-12-17T13:41:35.167370Z"
    },
    "papermill": {
     "duration": 0.065024,
     "end_time": "2020-12-17T13:41:35.167501",
     "exception": false,
     "start_time": "2020-12-17T13:41:35.102477",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_model(model, device, data_loader, loss_function, \n",
    "                optimizer, scheduler, n_examples):\n",
    "    \n",
    "    model = model.train()\n",
    "    losses = []\n",
    "    correct_preds = 0\n",
    "    complete_preds = []\n",
    "    complete_labels = []\n",
    "    \n",
    "    for batch in data_loader:\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        \n",
    "        outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "            \n",
    "        _, preds = torch.max(outputs, dim=1)\n",
    "        loss = loss_function(outputs, labels)\n",
    "        complete_preds.append(preds.data.cpu().numpy().tolist())\n",
    "        complete_labels.append(labels.data.cpu().numpy().tolist())\n",
    "        correct_preds += torch.sum(preds == labels)\n",
    "        losses.append(loss.item())\n",
    "        \n",
    "        loss.backward()\n",
    "        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "        optimizer.zero_grad()\n",
    "    \n",
    "    complete_preds_flat = [x for y in complete_preds for x in y]\n",
    "    complete_labels_flat = [x for y in complete_labels for x in y]\n",
    "    acc_score = accuracy_score(y_true=complete_labels_flat, \n",
    "                             y_pred=complete_preds_flat)\n",
    "    return acc_score, np.mean(losses)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:35.279143Z",
     "iopub.status.busy": "2020-12-17T13:41:35.278456Z",
     "iopub.status.idle": "2020-12-17T13:41:35.281446Z",
     "shell.execute_reply": "2020-12-17T13:41:35.281018Z"
    },
    "papermill": {
     "duration": 0.064071,
     "end_time": "2020-12-17T13:41:35.281538",
     "exception": false,
     "start_time": "2020-12-17T13:41:35.217467",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def eval_model(model, device, data_loader, loss_function, n_examples):\n",
    "    model = model.eval()\n",
    "    \n",
    "    losses = []\n",
    "    correct_preds = 0\n",
    "    complete_preds = []\n",
    "    complete_labels = []\n",
    "    complete_outputs = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for item in data_loader:\n",
    "            input_ids = item['input_ids'].to(device)\n",
    "            attention_mask = item['attention_mask'].to(device)\n",
    "            labels = item['labels'].to(device)\n",
    "\n",
    "            outputs = model(input_ids=input_ids, \n",
    "                            attention_mask=attention_mask)\n",
    "            \n",
    "            _, preds = torch.max(outputs, dim=1)\n",
    "            loss = loss_function(outputs, labels)\n",
    "            \n",
    "            correct_preds += torch.sum(preds == labels)\n",
    "            complete_preds.append(preds.data.cpu().numpy().tolist())\n",
    "            complete_labels.append(labels.data.cpu().numpy().tolist())\n",
    "            complete_outputs.append(outputs.tolist())\n",
    "            losses.append(loss.item())\n",
    "        \n",
    "        accuracy = correct_preds.double() / n_examples\n",
    "        complete_preds_flat = [x for y in complete_preds for x in y]\n",
    "        complete_labels_flat = [x for y in complete_labels for x in y]\n",
    "        complete_outputs_flat = [x for y in complete_outputs for x in y]\n",
    "\n",
    "        acc_score = accuracy_score(y_true=complete_labels_flat, \n",
    "                             y_pred=complete_preds_flat)\n",
    "        \n",
    "        return_items = (acc_score, \n",
    "                        np.mean(losses), \n",
    "                        complete_preds_flat, \n",
    "                        complete_outputs_flat)\n",
    "        \n",
    "        return return_items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:35.394727Z",
     "iopub.status.busy": "2020-12-17T13:41:35.393950Z",
     "iopub.status.idle": "2020-12-17T13:41:35.396750Z",
     "shell.execute_reply": "2020-12-17T13:41:35.396241Z"
    },
    "papermill": {
     "duration": 0.065977,
     "end_time": "2020-12-17T13:41:35.396835",
     "exception": false,
     "start_time": "2020-12-17T13:41:35.330858",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_fold(epochs, model, device, train_dataloader, \n",
    "               val_dataloader, test_dataloader, loss_fn, optimizer, \n",
    "               scheduler, model_save_name, n_train, n_val, single_model=True):\n",
    "    \n",
    "    start_time = time.time()\n",
    "    history = defaultdict(list)\n",
    "    best_accuracy = 0\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        epoch_start_time = time.time()\n",
    "        print('Epoch ', epoch+1, '/', epochs)\n",
    "        print('-'*50)\n",
    "\n",
    "        training_output = train_model(model, \n",
    "                                      device, \n",
    "                                      train_dataloader, \n",
    "                                      loss_fn, \n",
    "                                      optimizer, \n",
    "                                      scheduler, \n",
    "                                      n_train)\n",
    "        \n",
    "        train_acc, train_loss = training_output\n",
    "        \n",
    "        val_output = eval_model(model, \n",
    "                                device, \n",
    "                                val_dataloader, \n",
    "                                loss_fn, \n",
    "                                n_val)\n",
    "\n",
    "        val_acc, val_loss, val_preds, val_outputs = val_output\n",
    "        \n",
    "        history['train_accuracy'].append(train_acc)\n",
    "        history['train_loss'].append(train_loss)\n",
    "        history['val_accuracy'].append(val_acc)\n",
    "        history['val_loss'].append(val_loss)\n",
    "        history['val_preds'].append(val_preds)\n",
    "\n",
    "        if val_acc > best_accuracy:\n",
    "            torch.save(model.state_dict(), model_save_name)\n",
    "            best_accuracy = val_acc\n",
    "            best_preds = val_preds\n",
    "            best_outputs = val_outputs\n",
    "\n",
    "        print('Train Loss: ', \n",
    "              train_loss, \n",
    "              ' | ', \n",
    "              'Train Accuracy: ', \n",
    "              train_acc)\n",
    "        print('Val Loss: ', \n",
    "              val_loss, \n",
    "              ' | ', \n",
    "              'Val Accuracy: ', \n",
    "              val_acc)\n",
    "        print('Epoch Train Time: ', \n",
    "              format_time(time.time() - epoch_start_time))\n",
    "        print('\\n')\n",
    "    \n",
    "    print('Finished Training.')   \n",
    "    print('Fold Train Time: ', format_time(time.time() - start_time))\n",
    "    print('\\n')\n",
    "    if single_model:\n",
    "        _, _, test_preds, test_outputs = eval_model(model, \n",
    "                                                    device, \n",
    "                                                    test_dataloader, \n",
    "                                                    loss_function, \n",
    "                                                    len(df_test))\n",
    "        \n",
    "        single_model_performance(test_preds, df_test['sentiment'].values)\n",
    "    return history, best_preds, best_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:35.515645Z",
     "iopub.status.busy": "2020-12-17T13:41:35.508342Z",
     "iopub.status.idle": "2020-12-17T13:41:35.518117Z",
     "shell.execute_reply": "2020-12-17T13:41:35.517715Z"
    },
    "papermill": {
     "duration": 0.072346,
     "end_time": "2020-12-17T13:41:35.518223",
     "exception": false,
     "start_time": "2020-12-17T13:41:35.445877",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_oof_and_test_preds(model_type, tokenizer, \n",
    "                           train_df, test_df, single_model=False):\n",
    "    oof_preds = []\n",
    "    oof_outputs = []\n",
    "    oof_preds_indices = []\n",
    "    test_preds_list = []\n",
    "    test_outputs_list = []\n",
    "    history_list = []\n",
    "    start_time = time.time()\n",
    "    \n",
    "    fold = 0\n",
    "    \n",
    "    x_train = train_df['statement']\n",
    "    y_train = train_df['sentiment']\n",
    "\n",
    "    for train_index, val_index in skf.split(x_train, y_train):\n",
    "        print('Fold: {}'.format(fold+1))\n",
    "        \n",
    "        x_tr = x_train.iloc[train_index]\n",
    "        y_tr = y_train.iloc[train_index]\n",
    "        x_va = x_train.iloc[val_index]\n",
    "        y_va = y_train.iloc[val_index]\n",
    "        \n",
    "        train = pd.DataFrame(list(zip(x_tr, y_tr)), \n",
    "                             columns=['statement', 'sentiment'])\n",
    "        val = pd.DataFrame(list(zip(x_va, y_va)), \n",
    "                           columns=['statement', 'sentiment'])\n",
    "\n",
    "        train_ds = create_dataset(train, tokenizer, MAX_LENGTH)\n",
    "        val_ds = create_dataset(val, tokenizer, MAX_LENGTH)\n",
    "        test_ds = create_dataset(test_df, tokenizer, MAX_LENGTH)\n",
    "        \n",
    "\n",
    "        if model_type == 'bert':\n",
    "            model = BERTSentimentClassifier(NUM_CLASSES)\n",
    "            model = model.to(device)\n",
    "        elif model_type == 'distilbert':\n",
    "            model = DistilBertForSequenceClassification(pretrained_model_name=DISTILBERT_MODEL_NAME, \n",
    "                                                        num_classes=NUM_CLASSES)\n",
    "            model = model.to(device)\n",
    "        elif model_type == 'roberta':\n",
    "            model = RobertaSentimentClassifier(n_classes=NUM_CLASSES)\n",
    "            model = model.to(device)\n",
    "        \n",
    "        train_loader = create_dataloader(train_ds, BATCH_SIZE)\n",
    "        val_loader = create_dataloader(val_ds, BATCH_SIZE)\n",
    "        test_loader = create_dataloader(test_ds, BATCH_SIZE)\n",
    "        \n",
    "        training_steps = len(train_loader.dataset) * EPOCHS\n",
    "        warmup_steps = int(0.1 * training_steps)\n",
    "        optimizer = AdamW(model.parameters(), \n",
    "                          lr=LEARNING_RATE, \n",
    "                          weight_decay=WEIGHT_DECAY, \n",
    "                          correct_bias=True)\n",
    "        scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                                    num_warmup_steps=warmup_steps, \n",
    "                                                    num_training_steps=training_steps)\n",
    "        \n",
    "        model_save_name = '{}_fold_{}.bin'.format(model_type, fold)\n",
    "        \n",
    "        history, preds, outputs = train_fold(epochs=EPOCHS,\n",
    "                                             model=model, \n",
    "                                             device=device, \n",
    "                                             train_dataloader=train_loader, \n",
    "                                             val_dataloader=val_loader,\n",
    "                                             test_dataloader=test_loader,\n",
    "                                             loss_fn=loss_function,\n",
    "                                             optimizer=optimizer,\n",
    "                                             scheduler=scheduler,\n",
    "                                             model_save_name=model_save_name,\n",
    "                                             n_train=len(train),\n",
    "                                             n_val=len(val),\n",
    "                                             single_model=False\n",
    "                                            )\n",
    "        \n",
    "        history_list.append(history)\n",
    "        oof_preds.append(preds)\n",
    "        oof_outputs.append(outputs)\n",
    "        oof_preds_indices.append(val_index)\n",
    "        _, _, test_preds, test_outputs = eval_model(model, \n",
    "                                                    device, \n",
    "                                                    test_loader, \n",
    "                                                    loss_function, \n",
    "                                                    len(test_df))\n",
    "        test_preds_list.append(test_preds)\n",
    "        test_outputs_list.append(test_outputs)\n",
    "        \n",
    "        fold += 1\n",
    "\n",
    "    print(str(NFOLDS), 'Fold CV Train Time: ', format_time(time.time() - start_time))\n",
    "\n",
    "    return history_list, test_outputs_list\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:35.621815Z",
     "iopub.status.busy": "2020-12-17T13:41:35.620594Z",
     "iopub.status.idle": "2020-12-17T13:41:35.623456Z",
     "shell.execute_reply": "2020-12-17T13:41:35.622992Z"
    },
    "papermill": {
     "duration": 0.056024,
     "end_time": "2020-12-17T13:41:35.623549",
     "exception": false,
     "start_time": "2020-12-17T13:41:35.567525",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "loss_function = nn.CrossEntropyLoss().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:35.726515Z",
     "iopub.status.busy": "2020-12-17T13:41:35.725830Z",
     "iopub.status.idle": "2020-12-17T13:41:35.728784Z",
     "shell.execute_reply": "2020-12-17T13:41:35.728359Z"
    },
    "papermill": {
     "duration": 0.055997,
     "end_time": "2020-12-17T13:41:35.728876",
     "exception": false,
     "start_time": "2020-12-17T13:41:35.672879",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.049457,
     "end_time": "2020-12-17T13:41:35.828128",
     "exception": false,
     "start_time": "2020-12-17T13:41:35.778671",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## BERT\n",
    "\n",
    "### Single Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:35.974383Z",
     "iopub.status.busy": "2020-12-17T13:41:35.973392Z",
     "iopub.status.idle": "2020-12-17T13:41:37.343897Z",
     "shell.execute_reply": "2020-12-17T13:41:37.343468Z"
    },
    "papermill": {
     "duration": 1.462351,
     "end_time": "2020-12-17T13:41:37.343996",
     "exception": false,
     "start_time": "2020-12-17T13:41:35.881645",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03d6d25387eb4b0883680a7a6cb1a5d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "BERT_MODEL_NAME = 'bert-base-cased'\n",
    "bert_tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:37.460024Z",
     "iopub.status.busy": "2020-12-17T13:41:37.459146Z",
     "iopub.status.idle": "2020-12-17T13:41:37.461637Z",
     "shell.execute_reply": "2020-12-17T13:41:37.462227Z"
    },
    "papermill": {
     "duration": 0.065213,
     "end_time": "2020-12-17T13:41:37.462344",
     "exception": false,
     "start_time": "2020-12-17T13:41:37.397131",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "bert_train_ds = create_dataset(df_train, bert_tokenizer, MAX_LENGTH)\n",
    "bert_test_ds = create_dataset(df_test, bert_tokenizer, MAX_LENGTH)\n",
    "bert_val_ds = create_dataset(df_val, bert_tokenizer, MAX_LENGTH)\n",
    "\n",
    "bert_train_dataloader = create_dataloader(bert_train_ds, BATCH_SIZE)\n",
    "bert_test_dataloader = create_dataloader(bert_test_ds, BATCH_SIZE)\n",
    "bert_val_dataloader = create_dataloader(bert_val_ds, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:37.578945Z",
     "iopub.status.busy": "2020-12-17T13:41:37.578145Z",
     "iopub.status.idle": "2020-12-17T13:41:37.580819Z",
     "shell.execute_reply": "2020-12-17T13:41:37.580406Z"
    },
    "papermill": {
     "duration": 0.064169,
     "end_time": "2020-12-17T13:41:37.580919",
     "exception": false,
     "start_time": "2020-12-17T13:41:37.516750",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class BERTSentimentClassifier(nn.Module):\n",
    "    def __init__(self, n_classes):\n",
    "        super(BERTSentimentClassifier, self).__init__()\n",
    "        self.model = BertModel.from_pretrained(BERT_MODEL_NAME)\n",
    "        self.drop = nn.Dropout(DROPOUT_PROB)\n",
    "        self.output = nn.Linear(self.model.config.hidden_size, n_classes)\n",
    "        \n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        _, pooled_output = self.model(\n",
    "            input_ids = input_ids,\n",
    "            attention_mask=attention_mask\n",
    "        )\n",
    "\n",
    "        output = self.drop(pooled_output)\n",
    "        \n",
    "        return self.output(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:41:37.694143Z",
     "iopub.status.busy": "2020-12-17T13:41:37.692010Z",
     "iopub.status.idle": "2020-12-17T13:42:04.467442Z",
     "shell.execute_reply": "2020-12-17T13:42:04.466500Z"
    },
    "papermill": {
     "duration": 26.833087,
     "end_time": "2020-12-17T13:42:04.467569",
     "exception": false,
     "start_time": "2020-12-17T13:41:37.634482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24030e20b08f4f2db2383b4150ea1582",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd712c7de4c74c2c832dbffc705a21d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435779157.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "bert_model = BERTSentimentClassifier(NUM_CLASSES)\n",
    "bert_model = bert_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:42:04.580161Z",
     "iopub.status.busy": "2020-12-17T13:42:04.579314Z",
     "iopub.status.idle": "2020-12-17T13:42:04.581920Z",
     "shell.execute_reply": "2020-12-17T13:42:04.581475Z"
    },
    "papermill": {
     "duration": 0.061488,
     "end_time": "2020-12-17T13:42:04.582007",
     "exception": false,
     "start_time": "2020-12-17T13:42:04.520519",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "training_steps = len(bert_train_dataloader.dataset) * EPOCHS\n",
    "\n",
    "bert_optimizer = AdamW(bert_model.parameters(), \n",
    "                       lr=LEARNING_RATE, \n",
    "                       weight_decay=WEIGHT_DECAY, \n",
    "                       correct_bias=True)\n",
    "\n",
    "warmup_steps = int(0.1 * training_steps)\n",
    "bert_scheduler = get_linear_schedule_with_warmup(bert_optimizer, \n",
    "                                                 num_warmup_steps=warmup_steps, \n",
    "                                                 num_training_steps=training_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:42:04.695545Z",
     "iopub.status.busy": "2020-12-17T13:42:04.694829Z",
     "iopub.status.idle": "2020-12-17T13:44:59.008009Z",
     "shell.execute_reply": "2020-12-17T13:44:59.008627Z"
    },
    "papermill": {
     "duration": 174.37433,
     "end_time": "2020-12-17T13:44:59.008793",
     "exception": false,
     "start_time": "2020-12-17T13:42:04.634463",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.9926277940923517  |  Train Accuracy:  0.4981921487603306\n",
      "Val Loss:  0.7699596381956532  |  Val Accuracy:  0.6735537190082644\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6985195621478656  |  Train Accuracy:  0.6934400826446281\n",
      "Val Loss:  0.5261421915023557  |  Val Accuracy:  0.7975206611570248\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.45097066311181083  |  Train Accuracy:  0.8316115702479339\n",
      "Val Loss:  0.3488022932602513  |  Val Accuracy:  0.8657024793388429\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2888139240474494  |  Train Accuracy:  0.8954028925619835\n",
      "Val Loss:  0.35467278236343014  |  Val Accuracy:  0.878099173553719\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.1969090861822517  |  Train Accuracy:  0.9331095041322314\n",
      "Val Loss:  0.4309984883894363  |  Val Accuracy:  0.8553719008264463\n",
      "Epoch Train Time:  0:00:33\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:53\n",
      "\n",
      "\n",
      "[[ 51   7   3]\n",
      " [  7 261  19]\n",
      " [  1  28 107]]\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative      0.864     0.836     0.850        61\n",
      "     neutral      0.882     0.909     0.895       287\n",
      "    positive      0.829     0.787     0.808       136\n",
      "\n",
      "    accuracy                          0.866       484\n",
      "   macro avg      0.859     0.844     0.851       484\n",
      "weighted avg      0.865     0.866     0.865       484\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bert_single_model_items = train_fold(epochs=EPOCHS, \n",
    "                                     model=bert_model, \n",
    "                                     device=device, \n",
    "                                     train_dataloader=bert_train_dataloader, \n",
    "                                     val_dataloader=bert_val_dataloader,\n",
    "                                     test_dataloader=bert_test_dataloader,\n",
    "                                     loss_fn=loss_function,\n",
    "                                     optimizer=bert_optimizer,\n",
    "                                     scheduler=bert_scheduler,\n",
    "                                     model_save_name='bert_best_model.bin',\n",
    "                                     n_train=len(df_train),\n",
    "                                     n_val=len(df_val),\n",
    "                                     single_model=True\n",
    "                                    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.055351,
     "end_time": "2020-12-17T13:44:59.119458",
     "exception": false,
     "start_time": "2020-12-17T13:44:59.064107",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### 10-Fold CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:44:59.246465Z",
     "iopub.status.busy": "2020-12-17T13:44:59.244978Z",
     "iopub.status.idle": "2020-12-17T14:14:50.821707Z",
     "shell.execute_reply": "2020-12-17T14:14:50.821249Z"
    },
    "papermill": {
     "duration": 1791.644878,
     "end_time": "2020-12-17T14:14:50.821816",
     "exception": false,
     "start_time": "2020-12-17T13:44:59.176938",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 1\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.9999292215522455  |  Train Accuracy:  0.5099489795918367\n",
      "Val Loss:  0.8584627445255008  |  Val Accuracy:  0.6192660550458715\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7108871262900683  |  Train Accuracy:  0.6910714285714286\n",
      "Val Loss:  0.46267955005168915  |  Val Accuracy:  0.8142201834862385\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.41067103086685647  |  Train Accuracy:  0.8441326530612245\n",
      "Val Loss:  0.36391209731144564  |  Val Accuracy:  0.8440366972477065\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2846046985108025  |  Train Accuracy:  0.8987244897959183\n",
      "Val Loss:  0.39975788683763575  |  Val Accuracy:  0.8463302752293578\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.19507535260968975  |  Train Accuracy:  0.9359693877551021\n",
      "Val Loss:  0.4937733236168112  |  Val Accuracy:  0.8440366972477065\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:55\n",
      "\n",
      "\n",
      "Fold: 2\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.9345976464602412  |  Train Accuracy:  0.5737244897959184\n",
      "Val Loss:  0.8188195207289287  |  Val Accuracy:  0.6261467889908257\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6669314816290018  |  Train Accuracy:  0.727295918367347\n",
      "Val Loss:  0.4623934823487486  |  Val Accuracy:  0.8279816513761468\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.40570712250714397  |  Train Accuracy:  0.85\n",
      "Val Loss:  0.45311605504580904  |  Val Accuracy:  0.8027522935779816\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2888744635165346  |  Train Accuracy:  0.8979591836734694\n",
      "Val Loss:  0.523078258282372  |  Val Accuracy:  0.8004587155963303\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.21095634112704773  |  Train Accuracy:  0.9311224489795918\n",
      "Val Loss:  0.6171338956004807  |  Val Accuracy:  0.8096330275229358\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:52\n",
      "\n",
      "\n",
      "Fold: 3\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.9213710804374851  |  Train Accuracy:  0.5989795918367347\n",
      "Val Loss:  0.7956806165831429  |  Val Accuracy:  0.6536697247706422\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6903735130417104  |  Train Accuracy:  0.7198979591836735\n",
      "Val Loss:  0.4809222476822989  |  Val Accuracy:  0.8302752293577982\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.416341229087236  |  Train Accuracy:  0.8530612244897959\n",
      "Val Loss:  0.39198250682758434  |  Val Accuracy:  0.8486238532110092\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.29192186214638  |  Train Accuracy:  0.8964285714285715\n",
      "Val Loss:  0.5029587112699768  |  Val Accuracy:  0.8371559633027523\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2087875788689268  |  Train Accuracy:  0.9349489795918368\n",
      "Val Loss:  0.5841947893204633  |  Val Accuracy:  0.8348623853211009\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:53\n",
      "\n",
      "\n",
      "Fold: 4\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.052338447862742  |  Train Accuracy:  0.42117346938775513\n",
      "Val Loss:  0.8271592600005013  |  Val Accuracy:  0.6674311926605505\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6977955611384645  |  Train Accuracy:  0.698469387755102\n",
      "Val Loss:  0.49657433160713743  |  Val Accuracy:  0.8165137614678899\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.4135621431530738  |  Train Accuracy:  0.851530612244898\n",
      "Val Loss:  0.356155415730817  |  Val Accuracy:  0.8532110091743119\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2924841744407099  |  Train Accuracy:  0.8954081632653061\n",
      "Val Loss:  0.4450040640575545  |  Val Accuracy:  0.8348623853211009\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.1971519097267669  |  Train Accuracy:  0.9380102040816326\n",
      "Val Loss:  0.47097229392134715  |  Val Accuracy:  0.8371559633027523\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:52\n",
      "\n",
      "\n",
      "Fold: 5\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0251260618774258  |  Train Accuracy:  0.4813775510204082\n",
      "Val Loss:  0.858852703656469  |  Val Accuracy:  0.6444954128440367\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7455800855646328  |  Train Accuracy:  0.6910714285714286\n",
      "Val Loss:  0.6181946476655347  |  Val Accuracy:  0.7201834862385321\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.4870189345612818  |  Train Accuracy:  0.8183673469387756\n",
      "Val Loss:  0.44175437678183826  |  Val Accuracy:  0.8302752293577982\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3162318636948357  |  Train Accuracy:  0.8895408163265306\n",
      "Val Loss:  0.4301317738635199  |  Val Accuracy:  0.8394495412844036\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.22683983464934387  |  Train Accuracy:  0.9255102040816326\n",
      "Val Loss:  0.494047970111881  |  Val Accuracy:  0.8463302752293578\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:55\n",
      "\n",
      "\n",
      "Fold: 6\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0997779293936125  |  Train Accuracy:  0.45357142857142857\n",
      "Val Loss:  0.89374150122915  |  Val Accuracy:  0.5871559633027523\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.829029139328976  |  Train Accuracy:  0.6459183673469387\n",
      "Val Loss:  0.6895696574023792  |  Val Accuracy:  0.7155963302752294\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6205985891575716  |  Train Accuracy:  0.7512755102040817\n",
      "Val Loss:  0.5030180471284049  |  Val Accuracy:  0.7912844036697247\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.378511843085289  |  Train Accuracy:  0.860204081632653\n",
      "Val Loss:  0.4754963205861194  |  Val Accuracy:  0.8119266055045872\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2547986825205842  |  Train Accuracy:  0.9102040816326531\n",
      "Val Loss:  0.636056660009282  |  Val Accuracy:  0.8027522935779816\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:55\n",
      "\n",
      "\n",
      "Fold: 7\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0170104491516827  |  Train Accuracy:  0.47691915327722517\n",
      "Val Loss:  0.778747840651444  |  Val Accuracy:  0.6988505747126437\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.636195521165685  |  Train Accuracy:  0.73578168834481\n",
      "Val Loss:  0.36247200199535917  |  Val Accuracy:  0.871264367816092\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.37801189293585175  |  Train Accuracy:  0.8553940321346595\n",
      "Val Loss:  0.29027324608926264  |  Val Accuracy:  0.8781609195402299\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2746573205113532  |  Train Accuracy:  0.8982402448355011\n",
      "Val Loss:  0.3181372472484197  |  Val Accuracy:  0.8735632183908046\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.20155315326973494  |  Train Accuracy:  0.9347105330272889\n",
      "Val Loss:  0.39540174127822475  |  Val Accuracy:  0.864367816091954\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:53\n",
      "\n",
      "\n",
      "Fold: 8\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.955704080864666  |  Train Accuracy:  0.5666921703647029\n",
      "Val Loss:  0.823146002633231  |  Val Accuracy:  0.6459770114942529\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7386045431703087  |  Train Accuracy:  0.677123182861515\n",
      "Val Loss:  0.6294610149094036  |  Val Accuracy:  0.7080459770114943\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.5247814634708854  |  Train Accuracy:  0.794440193828105\n",
      "Val Loss:  0.43618215886609896  |  Val Accuracy:  0.8413793103448276\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3304187456463895  |  Train Accuracy:  0.8788574343279776\n",
      "Val Loss:  0.4277239294190492  |  Val Accuracy:  0.8436781609195402\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.21426373596749898  |  Train Accuracy:  0.9293547564396838\n",
      "Val Loss:  0.5886189551757914  |  Val Accuracy:  0.8160919540229885\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:55\n",
      "\n",
      "\n",
      "Fold: 9\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.1189072236297577  |  Train Accuracy:  0.38791124713083397\n",
      "Val Loss:  0.874615518110139  |  Val Accuracy:  0.6160919540229886\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.8409783780817094  |  Train Accuracy:  0.6350420811017597\n",
      "Val Loss:  0.7235026976891926  |  Val Accuracy:  0.671264367816092\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.698390653462914  |  Train Accuracy:  0.6998214741137465\n",
      "Val Loss:  0.5696227316345487  |  Val Accuracy:  0.735632183908046\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.485260033783147  |  Train Accuracy:  0.8133129303749044\n",
      "Val Loss:  0.4793555744524513  |  Val Accuracy:  0.8160919540229885\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3275636572890529  |  Train Accuracy:  0.8890589135424637\n",
      "Val Loss:  0.4544459130348904  |  Val Accuracy:  0.8344827586206897\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:58\n",
      "\n",
      "\n",
      "Fold: 10\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.1023596711759645  |  Train Accuracy:  0.42132109155827596\n",
      "Val Loss:  0.8526156800133842  |  Val Accuracy:  0.6620689655172414\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7456814743639008  |  Train Accuracy:  0.6863045141545524\n",
      "Val Loss:  0.6300467806203025  |  Val Accuracy:  0.7471264367816092\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.5231424754228049  |  Train Accuracy:  0.7895944912012242\n",
      "Val Loss:  0.43163172687803  |  Val Accuracy:  0.8620689655172413\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.31764412206423476  |  Train Accuracy:  0.8854883958173935\n",
      "Val Loss:  0.4274303929081985  |  Val Accuracy:  0.8436781609195402\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.20287911434907738  |  Train Accuracy:  0.9301198673807702\n",
      "Val Loss:  0.493438404585634  |  Val Accuracy:  0.864367816091954\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:55\n",
      "\n",
      "\n",
      "10 Fold CV Train Time:  0:29:52\n"
     ]
    }
   ],
   "source": [
    "bert_history, bert_test_outputs = get_oof_and_test_preds(model_type='bert', \n",
    "                                                         tokenizer=bert_tokenizer, \n",
    "                                                         train_df=df_train_full, \n",
    "                                                         test_df=df_test,\n",
    "                                                         single_model=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:14:50.988312Z",
     "iopub.status.busy": "2020-12-17T14:14:50.980114Z",
     "iopub.status.idle": "2020-12-17T14:14:50.998733Z",
     "shell.execute_reply": "2020-12-17T14:14:50.999142Z"
    },
    "papermill": {
     "duration": 0.101973,
     "end_time": "2020-12-17T14:14:50.999278",
     "exception": false,
     "start_time": "2020-12-17T14:14:50.897305",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 54   5   2]\n",
      " [  6 259  22]\n",
      " [  1  17 118]]\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative      0.885     0.885     0.885        61\n",
      "     neutral      0.922     0.902     0.912       287\n",
      "    positive      0.831     0.868     0.849       136\n",
      "\n",
      "    accuracy                          0.890       484\n",
      "   macro avg      0.879     0.885     0.882       484\n",
      "weighted avg      0.892     0.890     0.891       484\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cv_ensemble_performance(bert_test_outputs, df_test['sentiment'].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.074562,
     "end_time": "2020-12-17T14:14:51.148408",
     "exception": false,
     "start_time": "2020-12-17T14:14:51.073846",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## DistilBERT\n",
    "### Single Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:14:51.303439Z",
     "iopub.status.busy": "2020-12-17T14:14:51.302852Z",
     "iopub.status.idle": "2020-12-17T14:14:52.657805Z",
     "shell.execute_reply": "2020-12-17T14:14:52.658402Z"
    },
    "papermill": {
     "duration": 1.435704,
     "end_time": "2020-12-17T14:14:52.658533",
     "exception": false,
     "start_time": "2020-12-17T14:14:51.222829",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b28940fb3f884e6ea4ca727ca7ad9684",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "DISTILBERT_MODEL_NAME = 'distilbert-base-uncased'\n",
    "distilbert_tokenizer = DistilBertTokenizer.from_pretrained(DISTILBERT_MODEL_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:14:54.250462Z",
     "iopub.status.busy": "2020-12-17T14:14:54.248622Z",
     "iopub.status.idle": "2020-12-17T14:14:54.251134Z",
     "shell.execute_reply": "2020-12-17T14:14:54.251561Z"
    },
    "papermill": {
     "duration": 1.512548,
     "end_time": "2020-12-17T14:14:54.251682",
     "exception": false,
     "start_time": "2020-12-17T14:14:52.739134",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "distilbert_train_ds = create_dataset(df_train, distilbert_tokenizer, MAX_LENGTH)\n",
    "distilbert_test_ds = create_dataset(df_test, distilbert_tokenizer, MAX_LENGTH)\n",
    "distilbert_val_ds = create_dataset(df_val, distilbert_tokenizer, MAX_LENGTH)\n",
    "\n",
    "distilbert_train_dataloader = create_dataloader(distilbert_train_ds, BATCH_SIZE)\n",
    "distilbert_test_dataloader = create_dataloader(distilbert_test_ds, BATCH_SIZE)\n",
    "distilbert_val_dataloader = create_dataloader(distilbert_val_ds, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:14:54.413651Z",
     "iopub.status.busy": "2020-12-17T14:14:54.412903Z",
     "iopub.status.idle": "2020-12-17T14:14:54.415600Z",
     "shell.execute_reply": "2020-12-17T14:14:54.415167Z"
    },
    "papermill": {
     "duration": 0.087701,
     "end_time": "2020-12-17T14:14:54.415708",
     "exception": false,
     "start_time": "2020-12-17T14:14:54.328007",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class DistilBertForSequenceClassification(nn.Module):\n",
    "\n",
    "    def __init__(self, pretrained_model_name, num_classes):\n",
    "\n",
    "        super().__init__()\n",
    "\n",
    "        config = AutoConfig.from_pretrained(pretrained_model_name, num_labels=num_classes)\n",
    "\n",
    "        self.distilbert = AutoModel.from_pretrained(pretrained_model_name, config=config)\n",
    "        self.pre_classifier = nn.Linear(config.dim, config.dim)\n",
    "        self.classifier = nn.Linear(config.dim, num_classes)\n",
    "        self.dropout = nn.Dropout(config.seq_classif_dropout)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask=None, head_mask=None):\n",
    "\n",
    "        assert attention_mask is not None, \"No Attention Mask\"\n",
    "        distilbert_output = self.distilbert(input_ids=input_ids,\n",
    "                                            attention_mask=attention_mask,\n",
    "                                            head_mask=head_mask)\n",
    "\n",
    "        hidden_state = distilbert_output[0]  \n",
    "        pooled_output = hidden_state[:, 0] \n",
    "        pooled_output = self.pre_classifier(pooled_output)  \n",
    "        pooled_output = nn.ReLU()(pooled_output)  \n",
    "        pooled_output = self.dropout(pooled_output)  \n",
    "        logits = self.classifier(pooled_output)  \n",
    "\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:14:54.575242Z",
     "iopub.status.busy": "2020-12-17T14:14:54.574720Z",
     "iopub.status.idle": "2020-12-17T14:15:04.373878Z",
     "shell.execute_reply": "2020-12-17T14:15:04.372819Z"
    },
    "papermill": {
     "duration": 9.881442,
     "end_time": "2020-12-17T14:15:04.374004",
     "exception": false,
     "start_time": "2020-12-17T14:14:54.492562",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab414fc0f7a1421e94c926b6895a6e01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d04ff423e9d743e4b30235f46ef8f465",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=267967963.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "distilbert_model = DistilBertForSequenceClassification(pretrained_model_name=DISTILBERT_MODEL_NAME, num_classes=NUM_CLASSES)\n",
    "distilbert_model = distilbert_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:15:04.547984Z",
     "iopub.status.busy": "2020-12-17T14:15:04.547052Z",
     "iopub.status.idle": "2020-12-17T14:15:04.553167Z",
     "shell.execute_reply": "2020-12-17T14:15:04.552393Z"
    },
    "papermill": {
     "duration": 0.101399,
     "end_time": "2020-12-17T14:15:04.553335",
     "exception": false,
     "start_time": "2020-12-17T14:15:04.451936",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "training_steps = len(distilbert_train_dataloader.dataset) * EPOCHS\n",
    "\n",
    "distilbert_optimizer = AdamW(distilbert_model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY, correct_bias=True)\n",
    "distilbert_scheduler = get_linear_schedule_with_warmup(distilbert_optimizer, num_warmup_steps=int(0.1 * training_steps), num_training_steps=training_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:15:04.832455Z",
     "iopub.status.busy": "2020-12-17T14:15:04.831422Z",
     "iopub.status.idle": "2020-12-17T14:16:41.268991Z",
     "shell.execute_reply": "2020-12-17T14:16:41.267819Z"
    },
    "papermill": {
     "duration": 96.580752,
     "end_time": "2020-12-17T14:16:41.269099",
     "exception": false,
     "start_time": "2020-12-17T14:15:04.688347",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0174256366146497  |  Train Accuracy:  0.5247933884297521\n",
      "Val Loss:  0.8844125674616906  |  Val Accuracy:  0.5950413223140496\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7230734325144902  |  Train Accuracy:  0.6867252066115702\n",
      "Val Loss:  0.5096909221141569  |  Val Accuracy:  0.8016528925619835\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.43774399130551284  |  Train Accuracy:  0.828254132231405\n",
      "Val Loss:  0.34935386671173957  |  Val Accuracy:  0.8615702479338843\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.32051020892998894  |  Train Accuracy:  0.8778409090909091\n",
      "Val Loss:  0.34033183741473383  |  Val Accuracy:  0.871900826446281\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.23231720472570538  |  Train Accuracy:  0.9201962809917356\n",
      "Val Loss:  0.39704285369765374  |  Val Accuracy:  0.859504132231405\n",
      "Epoch Train Time:  0:00:18\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:35\n",
      "\n",
      "\n",
      "[[ 51   7   3]\n",
      " [  6 269  12]\n",
      " [  0  31 105]]\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative      0.895     0.836     0.864        61\n",
      "     neutral      0.876     0.937     0.906       287\n",
      "    positive      0.875     0.772     0.820       136\n",
      "\n",
      "    accuracy                          0.878       484\n",
      "   macro avg      0.882     0.848     0.863       484\n",
      "weighted avg      0.878     0.878     0.877       484\n",
      "\n"
     ]
    }
   ],
   "source": [
    "distilbert_history, distilbert_preds, distilbert_outputs, = train_fold(epochs=EPOCHS,\n",
    "                                                                       model=distilbert_model,\n",
    "                                                                       device=device, \n",
    "                                                                       train_dataloader=distilbert_train_dataloader, \n",
    "                                                                       val_dataloader=distilbert_val_dataloader,\n",
    "                                                                       test_dataloader=distilbert_test_dataloader,\n",
    "                                                                       loss_fn=loss_function,\n",
    "                                                                       optimizer=distilbert_optimizer,\n",
    "                                                                       scheduler=distilbert_scheduler,\n",
    "                                                                       model_save_name='distilbest_best_model.bin',\n",
    "                                                                       n_train=len(df_train),\n",
    "                                                                       n_val=len(df_val),\n",
    "                                                                       single_model=True\n",
    "                                                                      )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.079387,
     "end_time": "2020-12-17T14:16:41.428947",
     "exception": false,
     "start_time": "2020-12-17T14:16:41.349560",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### 10-Fold CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:16:41.594844Z",
     "iopub.status.busy": "2020-12-17T14:16:41.594055Z",
     "iopub.status.idle": "2020-12-17T14:33:11.412140Z",
     "shell.execute_reply": "2020-12-17T14:33:11.412750Z"
    },
    "papermill": {
     "duration": 989.904244,
     "end_time": "2020-12-17T14:33:11.412936",
     "exception": false,
     "start_time": "2020-12-17T14:16:41.508692",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 1\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0293812131395146  |  Train Accuracy:  0.45663265306122447\n",
      "Val Loss:  0.8766298357929502  |  Val Accuracy:  0.5986238532110092\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7156587053318413  |  Train Accuracy:  0.689795918367347\n",
      "Val Loss:  0.5239637227995055  |  Val Accuracy:  0.8004587155963303\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.43323043420606727  |  Train Accuracy:  0.835969387755102\n",
      "Val Loss:  0.36729702406695913  |  Val Accuracy:  0.8623853211009175\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3136976310793234  |  Train Accuracy:  0.878061224489796\n",
      "Val Loss:  0.36658207167472157  |  Val Accuracy:  0.8577981651376146\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.23678905477709308  |  Train Accuracy:  0.9165816326530613\n",
      "Val Loss:  0.4569141762331128  |  Val Accuracy:  0.8509174311926605\n",
      "Epoch Train Time:  0:00:18\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:36\n",
      "\n",
      "\n",
      "Fold: 2\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0040917481694902  |  Train Accuracy:  0.5344387755102041\n",
      "Val Loss:  0.862993883235114  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.718045135541838  |  Train Accuracy:  0.6877551020408164\n",
      "Val Loss:  0.537940746971539  |  Val Accuracy:  0.7958715596330275\n",
      "Epoch Train Time:  0:00:21\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.4518455282157781  |  Train Accuracy:  0.8267857142857142\n",
      "Val Loss:  0.4448336788586208  |  Val Accuracy:  0.7958715596330275\n",
      "Epoch Train Time:  0:00:18\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.31883910407825394  |  Train Accuracy:  0.8795918367346939\n",
      "Val Loss:  0.4815028618488993  |  Val Accuracy:  0.7958715596330275\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.23146617644827586  |  Train Accuracy:  0.9198979591836735\n",
      "Val Loss:  0.5884375063968557  |  Val Accuracy:  0.7821100917431193\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:36\n",
      "\n",
      "\n",
      "Fold: 3\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.9980420582148494  |  Train Accuracy:  0.5788265306122449\n",
      "Val Loss:  0.8769117210592542  |  Val Accuracy:  0.6100917431192661\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7386035853502702  |  Train Accuracy:  0.6839285714285714\n",
      "Val Loss:  0.5589868607265609  |  Val Accuracy:  0.7568807339449541\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.4665455703224455  |  Train Accuracy:  0.8193877551020409\n",
      "Val Loss:  0.42007687288735596  |  Val Accuracy:  0.8165137614678899\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3283280591879572  |  Train Accuracy:  0.8755102040816326\n",
      "Val Loss:  0.4440562788929258  |  Val Accuracy:  0.823394495412844\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.24469524012992577  |  Train Accuracy:  0.914795918367347\n",
      "Val Loss:  0.5411684237354036  |  Val Accuracy:  0.8165137614678899\n",
      "Epoch Train Time:  0:00:18\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:36\n",
      "\n",
      "\n",
      "Fold: 4\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.9972177323029966  |  Train Accuracy:  0.5362244897959184\n",
      "Val Loss:  0.9044068732431957  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7544652235751249  |  Train Accuracy:  0.675765306122449\n",
      "Val Loss:  0.5556721846972194  |  Val Accuracy:  0.7981651376146789\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.46225061903194503  |  Train Accuracy:  0.8252551020408163\n",
      "Val Loss:  0.39283410779067446  |  Val Accuracy:  0.8302752293577982\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3267119692752556  |  Train Accuracy:  0.8778061224489796\n",
      "Val Loss:  0.37381999673587935  |  Val Accuracy:  0.8394495412844036\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2407040218005375  |  Train Accuracy:  0.9153061224489796\n",
      "Val Loss:  0.43456222941832884  |  Val Accuracy:  0.8394495412844036\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:36\n",
      "\n",
      "\n",
      "Fold: 5\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.975997833329804  |  Train Accuracy:  0.5931122448979592\n",
      "Val Loss:  0.8647110547338214  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6931697609473247  |  Train Accuracy:  0.7061224489795919\n",
      "Val Loss:  0.5114449177469526  |  Val Accuracy:  0.7912844036697247\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.4217374096415481  |  Train Accuracy:  0.8392857142857143\n",
      "Val Loss:  0.4026222489774227  |  Val Accuracy:  0.8188073394495413\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3055692795740098  |  Train Accuracy:  0.888265306122449\n",
      "Val Loss:  0.41513377321617945  |  Val Accuracy:  0.8188073394495413\n",
      "Epoch Train Time:  0:00:18\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2200353045662751  |  Train Accuracy:  0.9216836734693877\n",
      "Val Loss:  0.4931100271642208  |  Val Accuracy:  0.8119266055045872\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:35\n",
      "\n",
      "\n",
      "Fold: 6\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.9868518829345703  |  Train Accuracy:  0.5653061224489796\n",
      "Val Loss:  0.8533581644296646  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6946378626385513  |  Train Accuracy:  0.7045918367346938\n",
      "Val Loss:  0.5116132316844804  |  Val Accuracy:  0.7912844036697247\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.42297055411095524  |  Train Accuracy:  0.8346938775510204\n",
      "Val Loss:  0.4474038604114737  |  Val Accuracy:  0.805045871559633\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.30618045439525526  |  Train Accuracy:  0.8864795918367347\n",
      "Val Loss:  0.4855372489296964  |  Val Accuracy:  0.8096330275229358\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.22124191394493897  |  Train Accuracy:  0.9260204081632653\n",
      "Val Loss:  0.6180492703403745  |  Val Accuracy:  0.8027522935779816\n",
      "Epoch Train Time:  0:00:18\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:35\n",
      "\n",
      "\n",
      "Fold: 7\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.9943785730416212  |  Train Accuracy:  0.5850548329507779\n",
      "Val Loss:  0.8375866455691201  |  Val Accuracy:  0.5977011494252874\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6996367946872867  |  Train Accuracy:  0.7008416220351951\n",
      "Val Loss:  0.4921135444726263  |  Val Accuracy:  0.7954022988505747\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.43790449541274123  |  Train Accuracy:  0.8321856669217036\n",
      "Val Loss:  0.32669624633022715  |  Val Accuracy:  0.8735632183908046\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.32263673954014854  |  Train Accuracy:  0.8768171384850804\n",
      "Val Loss:  0.2931921311121966  |  Val Accuracy:  0.8827586206896552\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2396082692264723  |  Train Accuracy:  0.9166029074215761\n",
      "Val Loss:  0.36532153794541955  |  Val Accuracy:  0.8781609195402299\n",
      "Epoch Train Time:  0:00:18\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:36\n",
      "\n",
      "\n",
      "Fold: 8\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.012519367584368  |  Train Accuracy:  0.5363427697016068\n",
      "Val Loss:  0.8765045319284711  |  Val Accuracy:  0.6022988505747127\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6849062201695714  |  Train Accuracy:  0.7112981382300434\n",
      "Val Loss:  0.5439991972276143  |  Val Accuracy:  0.7632183908045977\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.4226718344763527  |  Train Accuracy:  0.8349910737056874\n",
      "Val Loss:  0.40820154015507015  |  Val Accuracy:  0.8505747126436781\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.311404083832735  |  Train Accuracy:  0.8811527671512369\n",
      "Val Loss:  0.44247271918824743  |  Val Accuracy:  0.8206896551724138\n",
      "Epoch Train Time:  0:00:18\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.23498759840092645  |  Train Accuracy:  0.9191532772251977\n",
      "Val Loss:  0.4960531508550048  |  Val Accuracy:  0.8160919540229885\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:35\n",
      "\n",
      "\n",
      "Fold: 9\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.040879245696029  |  Train Accuracy:  0.43075745983167557\n",
      "Val Loss:  0.8863021944250379  |  Val Accuracy:  0.593103448275862\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6996149222056071  |  Train Accuracy:  0.7003315480744708\n",
      "Val Loss:  0.4995812906750611  |  Val Accuracy:  0.8091954022988506\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.4317948301512051  |  Train Accuracy:  0.8321856669217036\n",
      "Val Loss:  0.3798490247822234  |  Val Accuracy:  0.8436781609195402\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3195829615451214  |  Train Accuracy:  0.8788574343279776\n",
      "Val Loss:  0.38376423310754554  |  Val Accuracy:  0.8505747126436781\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.22857608618896183  |  Train Accuracy:  0.9150726855394032\n",
      "Val Loss:  0.4269508024611111  |  Val Accuracy:  0.8574712643678161\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:37\n",
      "\n",
      "\n",
      "Fold: 10\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0370011104316246  |  Train Accuracy:  0.4659525631216526\n",
      "Val Loss:  0.8870962794337954  |  Val Accuracy:  0.593103448275862\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6971999965305251  |  Train Accuracy:  0.7069625095638867\n",
      "Val Loss:  0.522188703928675  |  Val Accuracy:  0.8022988505747126\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.4302978296953488  |  Train Accuracy:  0.8349910737056874\n",
      "Val Loss:  0.4134853035211563  |  Val Accuracy:  0.8229885057471265\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.31347209914791874  |  Train Accuracy:  0.8808977301708748\n",
      "Val Loss:  0.40618201478251387  |  Val Accuracy:  0.8367816091954023\n",
      "Epoch Train Time:  0:00:20\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.23091833658395258  |  Train Accuracy:  0.9188982402448355\n",
      "Val Loss:  0.42902854297842297  |  Val Accuracy:  0.8551724137931035\n",
      "Epoch Train Time:  0:00:19\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:01:37\n",
      "\n",
      "\n",
      "10 Fold CV Train Time:  0:16:30\n"
     ]
    }
   ],
   "source": [
    "distilbert_history, distilbert_test_outputs = get_oof_and_test_preds(model_type='distilbert', \n",
    "                                                                                                                                                                        tokenizer=distilbert_tokenizer, \n",
    "                                                                                                                                                                        train_df=df_train_full, \n",
    "                                                                                                                                                                        test_df=df_test,\n",
    "                                                                                                                                                                        single_model=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:33:11.624576Z",
     "iopub.status.busy": "2020-12-17T14:33:11.623814Z",
     "iopub.status.idle": "2020-12-17T14:33:11.637802Z",
     "shell.execute_reply": "2020-12-17T14:33:11.637216Z"
    },
    "papermill": {
     "duration": 0.123985,
     "end_time": "2020-12-17T14:33:11.637919",
     "exception": false,
     "start_time": "2020-12-17T14:33:11.513934",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 55   5   1]\n",
      " [  8 255  24]\n",
      " [  5  16 115]]\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative      0.809     0.902     0.853        61\n",
      "     neutral      0.924     0.889     0.906       287\n",
      "    positive      0.821     0.846     0.833       136\n",
      "\n",
      "    accuracy                          0.878       484\n",
      "   macro avg      0.851     0.879     0.864       484\n",
      "weighted avg      0.881     0.878     0.879       484\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cv_ensemble_performance(distilbert_test_outputs, df_test['sentiment'].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.100712,
     "end_time": "2020-12-17T14:33:11.839575",
     "exception": false,
     "start_time": "2020-12-17T14:33:11.738863",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## RoBERTa\n",
    "### Single Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:33:12.049801Z",
     "iopub.status.busy": "2020-12-17T14:33:12.049258Z",
     "iopub.status.idle": "2020-12-17T14:33:15.101648Z",
     "shell.execute_reply": "2020-12-17T14:33:15.101191Z"
    },
    "papermill": {
     "duration": 3.161617,
     "end_time": "2020-12-17T14:33:15.101743",
     "exception": false,
     "start_time": "2020-12-17T14:33:11.940126",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d22cce76f2b4d0a80a98884094ef2a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc48c50ab19742748ae6af80dcd2b0b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "ROBERTA_MODEL_NAME = 'roberta-base'\n",
    "roberta_tokenizer = RobertaTokenizer.from_pretrained(ROBERTA_MODEL_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:33:15.672924Z",
     "iopub.status.busy": "2020-12-17T14:33:15.671207Z",
     "iopub.status.idle": "2020-12-17T14:33:15.673517Z",
     "shell.execute_reply": "2020-12-17T14:33:15.673953Z"
    },
    "papermill": {
     "duration": 0.469129,
     "end_time": "2020-12-17T14:33:15.674063",
     "exception": false,
     "start_time": "2020-12-17T14:33:15.204934",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "roberta_train_ds = create_dataset(df_train, roberta_tokenizer, MAX_LENGTH)\n",
    "roberta_test_ds = create_dataset(df_test, roberta_tokenizer, MAX_LENGTH)\n",
    "roberta_val_ds = create_dataset(df_val, roberta_tokenizer, MAX_LENGTH)\n",
    "\n",
    "roberta_train_dataloader = create_dataloader(roberta_train_ds, BATCH_SIZE)\n",
    "roberta_test_dataloader = create_dataloader(roberta_test_ds, BATCH_SIZE)\n",
    "roberta_val_dataloader = create_dataloader(roberta_val_ds, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:33:15.890190Z",
     "iopub.status.busy": "2020-12-17T14:33:15.889427Z",
     "iopub.status.idle": "2020-12-17T14:33:15.892331Z",
     "shell.execute_reply": "2020-12-17T14:33:15.891887Z"
    },
    "papermill": {
     "duration": 0.115891,
     "end_time": "2020-12-17T14:33:15.892428",
     "exception": false,
     "start_time": "2020-12-17T14:33:15.776537",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class RobertaSentimentClassifier(nn.Module):\n",
    "    \n",
    "    def __init__(self, n_classes):\n",
    "        super(RobertaSentimentClassifier, self).__init__()\n",
    "        self.model = RobertaModel.from_pretrained(ROBERTA_MODEL_NAME)\n",
    "        self.drop = nn.Dropout(DROPOUT_PROB)\n",
    "        self.output = nn.Linear(self.model.config.hidden_size, n_classes)\n",
    "        \n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        _, pooled_output = self.model(\n",
    "            input_ids = input_ids,\n",
    "            attention_mask=attention_mask\n",
    "        )\n",
    "        \n",
    "        output = self.drop(pooled_output)\n",
    "        \n",
    "        return self.output(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:33:16.103282Z",
     "iopub.status.busy": "2020-12-17T14:33:16.102683Z",
     "iopub.status.idle": "2020-12-17T14:33:33.772354Z",
     "shell.execute_reply": "2020-12-17T14:33:33.771817Z"
    },
    "papermill": {
     "duration": 17.777472,
     "end_time": "2020-12-17T14:33:33.772459",
     "exception": false,
     "start_time": "2020-12-17T14:33:15.994987",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23e3cbf91d1143f28867e5a94edb6bcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=481.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a2c54544986416eaa3bbceb84b4d4d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=501200538.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "roberta_model = RobertaSentimentClassifier(n_classes=NUM_CLASSES)\n",
    "roberta_model = roberta_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:33:34.013608Z",
     "iopub.status.busy": "2020-12-17T14:33:34.012805Z",
     "iopub.status.idle": "2020-12-17T14:33:34.015830Z",
     "shell.execute_reply": "2020-12-17T14:33:34.015400Z"
    },
    "papermill": {
     "duration": 0.125232,
     "end_time": "2020-12-17T14:33:34.015945",
     "exception": false,
     "start_time": "2020-12-17T14:33:33.890713",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "training_steps = len(roberta_train_dataloader.dataset) * EPOCHS\n",
    "\n",
    "roberta_optimizer = AdamW(roberta_model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY, correct_bias=True)\n",
    "roberta_scheduler = get_linear_schedule_with_warmup(roberta_optimizer, num_warmup_steps=int(0.1 * training_steps), num_training_steps=training_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:33:34.244001Z",
     "iopub.status.busy": "2020-12-17T14:33:34.243129Z",
     "iopub.status.idle": "2020-12-17T14:36:31.040547Z",
     "shell.execute_reply": "2020-12-17T14:36:31.041045Z"
    },
    "papermill": {
     "duration": 176.912366,
     "end_time": "2020-12-17T14:36:31.041213",
     "exception": false,
     "start_time": "2020-12-17T14:33:34.128847",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0194741960399407  |  Train Accuracy:  0.4535123966942149\n",
      "Val Loss:  0.8739319328338869  |  Val Accuracy:  0.5950413223140496\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6309417169079308  |  Train Accuracy:  0.731404958677686\n",
      "Val Loss:  0.3191360494782848  |  Val Accuracy:  0.878099173553719\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3386210358032018  |  Train Accuracy:  0.8669938016528925\n",
      "Val Loss:  0.28189576254977333  |  Val Accuracy:  0.890495867768595\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2584299799016377  |  Train Accuracy:  0.9010847107438017\n",
      "Val Loss:  0.3123397754144765  |  Val Accuracy:  0.8657024793388429\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.20876398551939748  |  Train Accuracy:  0.9271694214876033\n",
      "Val Loss:  0.43192107340080604  |  Val Accuracy:  0.890495867768595\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:55\n",
      "\n",
      "\n",
      "[[ 54   7   0]\n",
      " [  9 256  22]\n",
      " [  0  16 120]]\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative      0.857     0.885     0.871        61\n",
      "     neutral      0.918     0.892     0.905       287\n",
      "    positive      0.845     0.882     0.863       136\n",
      "\n",
      "    accuracy                          0.888       484\n",
      "   macro avg      0.873     0.887     0.880       484\n",
      "weighted avg      0.890     0.888     0.889       484\n",
      "\n"
     ]
    }
   ],
   "source": [
    "roberta_single_model_items = train_fold(epochs=EPOCHS,\n",
    "                                                              model=roberta_model,\n",
    "                                                              device=device, \n",
    "                                                              train_dataloader=roberta_train_dataloader, \n",
    "                                                              val_dataloader=roberta_val_dataloader,\n",
    "                                                              test_dataloader=roberta_test_dataloader,\n",
    "                                                              loss_fn=loss_function,\n",
    "                                                              optimizer=roberta_optimizer,\n",
    "                                                              scheduler=roberta_scheduler,\n",
    "                                                              model_save_name='roberta_best_model.bin',\n",
    "                                                              n_train=len(df_train),\n",
    "                                                              n_val=len(df_val),\n",
    "                                                              single_model=True\n",
    "                                                             )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.104745,
     "end_time": "2020-12-17T14:36:31.251231",
     "exception": false,
     "start_time": "2020-12-17T14:36:31.146486",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### 10-Fold CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T14:36:31.468580Z",
     "iopub.status.busy": "2020-12-17T14:36:31.467889Z",
     "iopub.status.idle": "2020-12-17T15:07:07.591492Z",
     "shell.execute_reply": "2020-12-17T15:07:07.590692Z"
    },
    "papermill": {
     "duration": 1836.235128,
     "end_time": "2020-12-17T15:07:07.591678",
     "exception": false,
     "start_time": "2020-12-17T14:36:31.356550",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 1\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0048778176307678  |  Train Accuracy:  0.577295918367347\n",
      "Val Loss:  0.9016212629420417  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6932815229406162  |  Train Accuracy:  0.6979591836734694\n",
      "Val Loss:  0.3903901215110506  |  Val Accuracy:  0.823394495412844\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.36121326011358473  |  Train Accuracy:  0.8627551020408163\n",
      "Val Loss:  0.3799735640308687  |  Val Accuracy:  0.8509174311926605\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2776369364894166  |  Train Accuracy:  0.8992346938775511\n",
      "Val Loss:  0.3734742925236268  |  Val Accuracy:  0.8646788990825688\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.24362066188180934  |  Train Accuracy:  0.9176020408163266\n",
      "Val Loss:  0.5088410853807416  |  Val Accuracy:  0.8463302752293578\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:03:00\n",
      "\n",
      "\n",
      "Fold: 2\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0110045588746364  |  Train Accuracy:  0.548469387755102\n",
      "Val Loss:  0.8731415101460048  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6563506345967858  |  Train Accuracy:  0.7160714285714286\n",
      "Val Loss:  0.48333021253347397  |  Val Accuracy:  0.7752293577981652\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3454956888088158  |  Train Accuracy:  0.8716836734693878\n",
      "Val Loss:  0.48467548683817896  |  Val Accuracy:  0.8279816513761468\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2685910653551014  |  Train Accuracy:  0.8997448979591837\n",
      "Val Loss:  0.54341366414779  |  Val Accuracy:  0.8142201834862385\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.22778248577853855  |  Train Accuracy:  0.9239795918367347\n",
      "Val Loss:  0.594083219683463  |  Val Accuracy:  0.8348623853211009\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:59\n",
      "\n",
      "\n",
      "Fold: 3\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0002678146167678  |  Train Accuracy:  0.5880102040816326\n",
      "Val Loss:  0.8696758172341755  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6915066770753082  |  Train Accuracy:  0.6880102040816326\n",
      "Val Loss:  0.4451256913266012  |  Val Accuracy:  0.8165137614678899\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3545341274720065  |  Train Accuracy:  0.8607142857142858\n",
      "Val Loss:  0.38964588727269855  |  Val Accuracy:  0.8600917431192661\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.27674056046015144  |  Train Accuracy:  0.8982142857142857\n",
      "Val Loss:  0.5472503466797727  |  Val Accuracy:  0.8279816513761468\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.23607996979880394  |  Train Accuracy:  0.9229591836734694\n",
      "Val Loss:  0.6452471952153636  |  Val Accuracy:  0.8348623853211009\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:57\n",
      "\n",
      "\n",
      "Fold: 4\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0759361228164361  |  Train Accuracy:  0.4010204081632653\n",
      "Val Loss:  0.873147285410336  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6540692991748148  |  Train Accuracy:  0.7160714285714286\n",
      "Val Loss:  0.38743127403514727  |  Val Accuracy:  0.8532110091743119\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.33937310080747213  |  Train Accuracy:  0.8688775510204082\n",
      "Val Loss:  0.35483862246785847  |  Val Accuracy:  0.8509174311926605\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2721169648958104  |  Train Accuracy:  0.9015306122448979\n",
      "Val Loss:  0.4465199299156666  |  Val Accuracy:  0.841743119266055\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2221197583997736  |  Train Accuracy:  0.9275510204081633\n",
      "Val Loss:  0.5855461633098977  |  Val Accuracy:  0.8119266055045872\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:56\n",
      "\n",
      "\n",
      "Fold: 5\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.0700348204495955  |  Train Accuracy:  0.3579081632653061\n",
      "Val Loss:  0.8475352951458522  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.573830803225235  |  Train Accuracy:  0.7581632653061224\n",
      "Val Loss:  0.3827082932527576  |  Val Accuracy:  0.8371559633027523\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3271512582441982  |  Train Accuracy:  0.8691326530612244\n",
      "Val Loss:  0.442101956478187  |  Val Accuracy:  0.8279816513761468\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.24416703911764281  |  Train Accuracy:  0.9071428571428571\n",
      "Val Loss:  0.44536048686131835  |  Val Accuracy:  0.8784403669724771\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.22322287369823576  |  Train Accuracy:  0.9326530612244898\n",
      "Val Loss:  0.8838944461728845  |  Val Accuracy:  0.805045871559633\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:58\n",
      "\n",
      "\n",
      "Fold: 6\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.041339661637131  |  Train Accuracy:  0.4176020408163265\n",
      "Val Loss:  0.8371540776320866  |  Val Accuracy:  0.5940366972477065\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.712215600877392  |  Train Accuracy:  0.6823979591836735\n",
      "Val Loss:  0.46189738703625544  |  Val Accuracy:  0.7935779816513762\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3697370315388757  |  Train Accuracy:  0.8551020408163266\n",
      "Val Loss:  0.4620818372017571  |  Val Accuracy:  0.8073394495412844\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2762139951810241  |  Train Accuracy:  0.8956632653061225\n",
      "Val Loss:  0.46765528353197233  |  Val Accuracy:  0.8188073394495413\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.22634339888523125  |  Train Accuracy:  0.9221938775510204\n",
      "Val Loss:  0.6407660487374025  |  Val Accuracy:  0.805045871559633\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:03:00\n",
      "\n",
      "\n",
      "Fold: 7\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.021162067486988  |  Train Accuracy:  0.535067584799796\n",
      "Val Loss:  0.8677890832935061  |  Val Accuracy:  0.593103448275862\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.6201466300744352  |  Train Accuracy:  0.7268553940321346\n",
      "Val Loss:  0.31030877493321896  |  Val Accuracy:  0.8781609195402299\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3345829280683907  |  Train Accuracy:  0.8699311400153023\n",
      "Val Loss:  0.29113942850381136  |  Val Accuracy:  0.9011494252873563\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2749132318119329  |  Train Accuracy:  0.8982402448355011\n",
      "Val Loss:  0.34724530670791864  |  Val Accuracy:  0.8896551724137931\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.23670060516759087  |  Train Accuracy:  0.9209385360877327\n",
      "Val Loss:  0.615096904071314  |  Val Accuracy:  0.8505747126436781\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:59\n",
      "\n",
      "\n",
      "Fold: 8\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.067265416548504  |  Train Accuracy:  0.3942871716398878\n",
      "Val Loss:  0.9097906053066254  |  Val Accuracy:  0.593103448275862\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.7443801310609995  |  Train Accuracy:  0.6697271104310125\n",
      "Val Loss:  0.5249947926827839  |  Val Accuracy:  0.774712643678161\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.38595091415251176  |  Train Accuracy:  0.84952818158633\n",
      "Val Loss:  0.44231961680842297  |  Val Accuracy:  0.8275862068965517\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.2911485194879758  |  Train Accuracy:  0.8872736546799286\n",
      "Val Loss:  0.670014314353466  |  Val Accuracy:  0.7770114942528735\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.22941411100895848  |  Train Accuracy:  0.9234889058913542\n",
      "Val Loss:  0.8264851720471468  |  Val Accuracy:  0.7908045977011494\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:59\n",
      "\n",
      "\n",
      "Fold: 9\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.026524739779108  |  Train Accuracy:  0.5312420300943637\n",
      "Val Loss:  0.8535817627395902  |  Val Accuracy:  0.593103448275862\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.5689186091890664  |  Train Accuracy:  0.75592960979342\n",
      "Val Loss:  0.3439859871619514  |  Val Accuracy:  0.8528735632183908\n",
      "Epoch Train Time:  0:00:37\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.324448063416876  |  Train Accuracy:  0.8717163988778373\n",
      "Val Loss:  0.35394109965169  |  Val Accuracy:  0.8689655172413793\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.258808601715761  |  Train Accuracy:  0.8995154297373119\n",
      "Val Loss:  0.5051304969404425  |  Val Accuracy:  0.8413793103448276\n",
      "Epoch Train Time:  0:00:34\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.21784088354391173  |  Train Accuracy:  0.9255292017342515\n",
      "Val Loss:  0.5625377223893467  |  Val Accuracy:  0.8551724137931035\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:59\n",
      "\n",
      "\n",
      "Fold: 10\n",
      "Epoch  1 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  1.043645105953139  |  Train Accuracy:  0.41494516704922213\n",
      "Val Loss:  0.8627760538033077  |  Val Accuracy:  0.593103448275862\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  2 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.5802631466005875  |  Train Accuracy:  0.7472583524611068\n",
      "Val Loss:  0.3555028699338436  |  Val Accuracy:  0.8413793103448276\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  3 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.3264247131859505  |  Train Accuracy:  0.86967610303494\n",
      "Val Loss:  0.38771550224295687  |  Val Accuracy:  0.8367816091954023\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Epoch  4 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.26731671184695105  |  Train Accuracy:  0.8977301708747768\n",
      "Val Loss:  0.394182883735214  |  Val Accuracy:  0.8620689655172413\n",
      "Epoch Train Time:  0:00:36\n",
      "\n",
      "\n",
      "Epoch  5 / 5\n",
      "--------------------------------------------------\n",
      "Train Loss:  0.20602827134345303  |  Train Accuracy:  0.928079571537873\n",
      "Val Loss:  0.6835331328745399  |  Val Accuracy:  0.8206896551724138\n",
      "Epoch Train Time:  0:00:35\n",
      "\n",
      "\n",
      "Finished Training.\n",
      "Fold Train Time:  0:02:58\n",
      "\n",
      "\n",
      "10 Fold CV Train Time:  0:30:36\n"
     ]
    }
   ],
   "source": [
    "roberta_history, roberta_test_outputs = get_oof_and_test_preds(model_type='roberta', \n",
    "                                                               tokenizer=roberta_tokenizer,\n",
    "                                                               train_df=df_train_full, \n",
    "                                                               test_df=df_test,\n",
    "                                                               single_model=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T15:07:07.853569Z",
     "iopub.status.busy": "2020-12-17T15:07:07.852865Z",
     "iopub.status.idle": "2020-12-17T15:07:07.869888Z",
     "shell.execute_reply": "2020-12-17T15:07:07.870432Z"
    },
    "papermill": {
     "duration": 0.152588,
     "end_time": "2020-12-17T15:07:07.870583",
     "exception": false,
     "start_time": "2020-12-17T15:07:07.717995",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 55   6   0]\n",
      " [  6 242  39]\n",
      " [  0   8 128]]\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative      0.902     0.902     0.902        61\n",
      "     neutral      0.945     0.843     0.891       287\n",
      "    positive      0.766     0.941     0.845       136\n",
      "\n",
      "    accuracy                          0.878       484\n",
      "   macro avg      0.871     0.895     0.879       484\n",
      "weighted avg      0.890     0.878     0.880       484\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cv_ensemble_performance(roberta_test_outputs, df_test['sentiment'].values)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 5153.845303,
   "end_time": "2020-12-17T15:07:09.130299",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-12-17T13:41:15.284996",
   "version": "2.1.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "00decb22a75046d49d0c5291dd06482c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "01e50b1cd63148bc9db875af804b1f0c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "03d6d25387eb4b0883680a7a6cb1a5d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_dc98d58755374f87b4516c263be90f9b",
        "IPY_MODEL_33896ed3b2a3466fa8dbb3ab67d362fb"
       ],
       "layout": "IPY_MODEL_888df75bbce54584ad104edfd2e5e40d"
      }
     },
     "08773420ad6741caa0a65fb48484196b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "0c075b84562c4ba9af565048572a5b31": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_888f34c70e6648cb932a8731a1048a64",
       "placeholder": "​",
       "style": "IPY_MODEL_cbbb6c07a6074b6e987d91a6601df4e3",
       "value": " 433/433 [00:18&lt;00:00, 23.3B/s]"
      }
     },
     "0c88c2046f8b4e509309c87f020e5d63": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "0fcd7c611401462693b109d3cd34f5ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a4e7c2b3d22c4554aa12275809775cb8",
       "placeholder": "​",
       "style": "IPY_MODEL_00decb22a75046d49d0c5291dd06482c",
       "value": " 899k/899k [00:01&lt;00:00, 595kB/s]"
      }
     },
     "133ad0583afe4a49ac0410385444ac5e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "14a6ad083ddc43bdb77e7a0fa9441096": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "16a4bfdb98924bde83cdfccd8117f90e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1833c6d6839a4caa8be315c5569cd912": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1df53d8f70144c0d8fbbcc4991b76a35": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "22ca8215cc0f49a380cad6a5a28babee": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "23e3cbf91d1143f28867e5a94edb6bcf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_eb2d0cd0baaa49ca8b3d599129120ea5",
        "IPY_MODEL_2edd91e4e3a6486a8b2cfd82fa8af075"
       ],
       "layout": "IPY_MODEL_133ad0583afe4a49ac0410385444ac5e"
      }
     },
     "23e8ec172e8d41719928adfddf3934d1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "24030e20b08f4f2db2383b4150ea1582": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_aaa81d90221f4508b3dd9e1f669411c1",
        "IPY_MODEL_0c075b84562c4ba9af565048572a5b31"
       ],
       "layout": "IPY_MODEL_c6be51805a17469cabfff1333f26bbcc"
      }
     },
     "24184519e11b4ed4ab2bcffaf60d7904": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "26a91976ceb14f75b593baf9079dc791": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "2bd5fb74646c4624b571a8ffba930ef8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7891ec8d85b2456c8e59f292d5801017",
       "placeholder": "​",
       "style": "IPY_MODEL_e4d8b1a000fd4ebc8077467af72007a8",
       "value": " 442/442 [00:00&lt;00:00, 1.40kB/s]"
      }
     },
     "2c69e3f1400c4916ba5ee4a3335ea3a8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a8344490152045cc9a440b39480e3a07",
       "max": 267967963.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a4bf24ad2843485cba5d1a2e3d117a6d",
       "value": 267967963.0
      }
     },
     "2d5933e504c244dc85240a6ac3e07b46": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "2edd91e4e3a6486a8b2cfd82fa8af075": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a3af3dea2a3e4c2ba2b8b59341758788",
       "placeholder": "​",
       "style": "IPY_MODEL_ef5c4cefbf614d828153c1488b0ef8eb",
       "value": " 481/481 [00:00&lt;00:00, 1.18kB/s]"
      }
     },
     "30825165d82441bf993279d7c737cc1e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "32a08791e5204a51a82501c487953749": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "32f463af03244dd6992a18c6447f6530": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_613a3a8c50ed4eaba282cb6ff4798d70",
       "max": 435779157.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_0c88c2046f8b4e509309c87f020e5d63",
       "value": 435779157.0
      }
     },
     "33896ed3b2a3466fa8dbb3ab67d362fb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_32a08791e5204a51a82501c487953749",
       "placeholder": "​",
       "style": "IPY_MODEL_5fb700eb00434fd597bc2ceb15ce3a07",
       "value": " 213k/213k [00:00&lt;00:00, 571kB/s]"
      }
     },
     "38f47a754eeb4c4bb4867b2973ea9078": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "3f8aeb602e1c4c7089e1db7385d882c6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_cde62b0696ab48fbaf5aa9ed1ae718aa",
       "max": 231508.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_23e8ec172e8d41719928adfddf3934d1",
       "value": 231508.0
      }
     },
     "45362d85116141e98c01b54d80d7e5ca": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "49549192dddd41a4809d12d754e385a4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "594bb255127049aba77e7e830bdf361b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c30ea0b9311e400999fd6090f2f0a07c",
       "placeholder": "​",
       "style": "IPY_MODEL_ecc52b02e1c446879c4deb0e0094a34d",
       "value": " 268M/268M [00:07&lt;00:00, 35.2MB/s]"
      }
     },
     "59e80e0990b447a9abdb1c813f82752e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "5edf2c655f3243a5a93c8abc0870af2b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_49549192dddd41a4809d12d754e385a4",
       "max": 456318.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_1df53d8f70144c0d8fbbcc4991b76a35",
       "value": 456318.0
      }
     },
     "5fb700eb00434fd597bc2ceb15ce3a07": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "613a3a8c50ed4eaba282cb6ff4798d70": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6725a021f30245a89656ced5a09250c7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "687b165bfd6f418e8ec9c0aed2963c9e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_14a6ad083ddc43bdb77e7a0fa9441096",
       "max": 442.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_6725a021f30245a89656ced5a09250c7",
       "value": 442.0
      }
     },
     "69a768b5f883432a8305f7419a6e8327": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6d22cce76f2b4d0a80a98884094ef2a0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ac0f7dd8a881414db0954d5e89af21fe",
        "IPY_MODEL_0fcd7c611401462693b109d3cd34f5ea"
       ],
       "layout": "IPY_MODEL_30825165d82441bf993279d7c737cc1e"
      }
     },
     "717f12f595cf470fbdf33026a5527307": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "72a3ae49c1f14364b6e58475e5148d29": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "7891ec8d85b2456c8e59f292d5801017": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "78fc2cb2f10044a483e558a696678cac": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "7a2c54544986416eaa3bbceb84b4d4d7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_87f8c251f9a14c5aa0ae97a735830f80",
        "IPY_MODEL_d361afb874d245de9091d6d222e2c176"
       ],
       "layout": "IPY_MODEL_a8ed985ba7564eaca60627225552aaaa"
      }
     },
     "80b5f4fcddce4572b699ca0a5e2924e8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "87e247e0cc5a4e68a67536842884707a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "87f8c251f9a14c5aa0ae97a735830f80": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1833c6d6839a4caa8be315c5569cd912",
       "max": 501200538.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_9f9441d4c41048ed9f7689aabca77dbf",
       "value": 501200538.0
      }
     },
     "888df75bbce54584ad104edfd2e5e40d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "888f34c70e6648cb932a8731a1048a64": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "88bc380c16934646ba9e314af94a40a9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_80b5f4fcddce4572b699ca0a5e2924e8",
       "placeholder": "​",
       "style": "IPY_MODEL_22ca8215cc0f49a380cad6a5a28babee",
       "value": " 436M/436M [00:17&lt;00:00, 24.3MB/s]"
      }
     },
     "9f9441d4c41048ed9f7689aabca77dbf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "a38b32f203fb40d586bd5121394a04cd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "a3af3dea2a3e4c2ba2b8b59341758788": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a4bf24ad2843485cba5d1a2e3d117a6d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "a4e7c2b3d22c4554aa12275809775cb8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a8344490152045cc9a440b39480e3a07": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a883fb5c74d74242aa3f095651d61425": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_01e50b1cd63148bc9db875af804b1f0c",
       "placeholder": "​",
       "style": "IPY_MODEL_2d5933e504c244dc85240a6ac3e07b46",
       "value": " 232k/232k [00:00&lt;00:00, 615kB/s]"
      }
     },
     "a8ed985ba7564eaca60627225552aaaa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "aaa81d90221f4508b3dd9e1f669411c1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_87e247e0cc5a4e68a67536842884707a",
       "max": 433.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_38f47a754eeb4c4bb4867b2973ea9078",
       "value": 433.0
      }
     },
     "ab414fc0f7a1421e94c926b6895a6e01": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_687b165bfd6f418e8ec9c0aed2963c9e",
        "IPY_MODEL_2bd5fb74646c4624b571a8ffba930ef8"
       ],
       "layout": "IPY_MODEL_24184519e11b4ed4ab2bcffaf60d7904"
      }
     },
     "ac0f7dd8a881414db0954d5e89af21fe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_08773420ad6741caa0a65fb48484196b",
       "max": 898823.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_26a91976ceb14f75b593baf9079dc791",
       "value": 898823.0
      }
     },
     "afc75b0a706a4b048e96a33e4539d327": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_69a768b5f883432a8305f7419a6e8327",
       "placeholder": "​",
       "style": "IPY_MODEL_59e80e0990b447a9abdb1c813f82752e",
       "value": " 456k/456k [00:00&lt;00:00, 939kB/s]"
      }
     },
     "b28940fb3f884e6ea4ca727ca7ad9684": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_3f8aeb602e1c4c7089e1db7385d882c6",
        "IPY_MODEL_a883fb5c74d74242aa3f095651d61425"
       ],
       "layout": "IPY_MODEL_c4de6c404fda41869f22bae3be34b7ca"
      }
     },
     "c30ea0b9311e400999fd6090f2f0a07c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c4de6c404fda41869f22bae3be34b7ca": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c6be51805a17469cabfff1333f26bbcc": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cbbb6c07a6074b6e987d91a6601df4e3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "cc48c50ab19742748ae6af80dcd2b0b1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5edf2c655f3243a5a93c8abc0870af2b",
        "IPY_MODEL_afc75b0a706a4b048e96a33e4539d327"
       ],
       "layout": "IPY_MODEL_de454bb8ea4d4524a716c480ac717d0b"
      }
     },
     "cd712c7de4c74c2c832dbffc705a21d7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_32f463af03244dd6992a18c6447f6530",
        "IPY_MODEL_88bc380c16934646ba9e314af94a40a9"
       ],
       "layout": "IPY_MODEL_16a4bfdb98924bde83cdfccd8117f90e"
      }
     },
     "cde62b0696ab48fbaf5aa9ed1ae718aa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d04ff423e9d743e4b30235f46ef8f465": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_2c69e3f1400c4916ba5ee4a3335ea3a8",
        "IPY_MODEL_594bb255127049aba77e7e830bdf361b"
       ],
       "layout": "IPY_MODEL_717f12f595cf470fbdf33026a5527307"
      }
     },
     "d361afb874d245de9091d6d222e2c176": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_de79c243b4924bec97157273eed99e50",
       "placeholder": "​",
       "style": "IPY_MODEL_78fc2cb2f10044a483e558a696678cac",
       "value": " 501M/501M [00:14&lt;00:00, 35.4MB/s]"
      }
     },
     "dc98d58755374f87b4516c263be90f9b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f8248f37a6c049fd990977a2b811a680",
       "max": 213450.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a38b32f203fb40d586bd5121394a04cd",
       "value": 213450.0
      }
     },
     "de454bb8ea4d4524a716c480ac717d0b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "de79c243b4924bec97157273eed99e50": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e4d8b1a000fd4ebc8077467af72007a8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "eb2d0cd0baaa49ca8b3d599129120ea5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_45362d85116141e98c01b54d80d7e5ca",
       "max": 481.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_72a3ae49c1f14364b6e58475e5148d29",
       "value": 481.0
      }
     },
     "ecc52b02e1c446879c4deb0e0094a34d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ef5c4cefbf614d828153c1488b0ef8eb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f8248f37a6c049fd990977a2b811a680": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}