Notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bert1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/suman-patel0/Financial-Sentiment-Analysis-using-BERT/blob/main/Notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O8y-u51YlHZY"
      },
      "source": [
        "## Pre-trained model name"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MAoHop9oNf56"
      },
      "source": [
        "PRE_TRAINED_MODEL_NAME = 'bert-base-uncased'"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mzRcLEjMlRhz"
      },
      "source": [
        "## Importing the requirements"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iYT8E-aDNxYX"
      },
      "source": [
        "import transformers\n",
        "from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import confusion_matrix, classification_report\n",
        "\n",
        "from torch import nn, optim\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "from collections import defaultdict\n",
        "from textwrap import wrap\n",
        "\n",
        "import pickle\n",
        "\n",
        "from pydantic import BaseModel\n",
        "from typing import Dict\n",
        "\n",
        "from fastapi import FastAPI\n",
        "\n",
        "from colabcode import ColabCode\n",
        "\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "sns.set(style='whitegrid', palette='muted', font_scale=1.2)\n",
        "HAPPY_COLORS_PALETTE = [\"#01BEFE\", \"#FFDD00\", \"#FF7D00\", \"#FF006D\", \"#ADFF02\", \"#8F00FF\"]\n",
        "sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))\n",
        "\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "from warnings import filterwarnings\n",
        "filterwarnings('ignore')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "67R3H6BjN-J4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c0411d43-d465-4820-bf1b-e824785fb887"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/drive')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /drive; to attempt to forcibly remount, call drive.mount(\"/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dHcK6ZKVOIj_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "05e49199-7a93-4ecc-f51d-70301b2ac814"
      },
      "source": [
        "import os\n",
        "os.chdir (\"/drive/MyDrive/finance/\")\n",
        "!ls"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "all-data.csv  best_model_state.bin  model  model_gb.pkl  models.py  __pycache__\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aP9JrYfemsm_"
      },
      "source": [
        "## Have a look at the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0LFsh6RWOQM9"
      },
      "source": [
        "df = pd.read_csv(\"all-data.csv\", names = [\"label\", \"text\"], encoding='latin-1')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNkfc2N4OSwO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "outputId": "f2fcb03e-8d1e-4947-a468-bd031788b1ca"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>neutral</td>\n",
              "      <td>According to Gran , the company has no plans t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>neutral</td>\n",
              "      <td>Technopolis plans to develop in stages an area...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negative</td>\n",
              "      <td>The international electronic industry company ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "      <td>With the new production plant the company woul...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>positive</td>\n",
              "      <td>According to the company 's updated strategy f...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      label                                               text\n",
              "0   neutral  According to Gran , the company has no plans t...\n",
              "1   neutral  Technopolis plans to develop in stages an area...\n",
              "2  negative  The international electronic industry company ...\n",
              "3  positive  With the new production plant the company woul...\n",
              "4  positive  According to the company 's updated strategy f..."
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iroJT1oxOUGV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a55f2d05-c378-4091-89b9-6f6140fdee4a"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4846, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L6AFRGXQOWFb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "31648875-2527-4d03-cb6f-26827344c942"
      },
      "source": [
        "df.info()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 4846 entries, 0 to 4845\n",
            "Data columns (total 2 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   label   4846 non-null   object\n",
            " 1   text    4846 non-null   object\n",
            "dtypes: object(2)\n",
            "memory usage: 75.8+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fdunM8w7OZqz"
      },
      "source": [
        "no null values are present"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9r0c0xXROd-H",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 309
        },
        "outputId": "913c9f54-6898-4acf-bc97-bd14daef6c4c"
      },
      "source": [
        "sns.countplot(x = \"label\", data = df)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7fcf9405d110>"
            ]
          },
          "metadata": {},
          "execution_count": 9
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAETCAYAAAAPucBzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de1xUdf4/8NcMwsAyoGaFO2golow6XGYAUdcbpJiXWLzgAoXWA9pNrcTSUEpdyssWRYo+wjY0l8ivSXgFV90KddfEeykwZClqypKloTMCA8yc3x/8OOuIl0E5A+Lr+Xjw4DHn85lz3p85MK85lzlHJgiCACIiIonIW7sAIiJq3xg0REQkKQYNERFJikFDRESSYtAQEZGkGDRERCQpBg0REUnKbkHz4YcfYsSIEQgMDERISAji4+Oh1+vF9pKSEkRHR8Pf3x/Dhw9HVlaW1fNramqwYMEC9O/fHzqdDomJiaisrLTqk5eXh/DwcPj5+SEiIgL79++3y9iIiOjWZPb6wmZZWRkeeughdOzYEbW1tcjOzsaaNWuwd+9eVFVVITw8HLGxsXjhhReg1+vx5z//GW+99RaeeuopAMCCBQtQVFSEjIwMODs7Y86cOZDJZPjoo48AAEePHsVzzz2H5cuX4w9/+AO2bNmCxYsXY/v27VCpVDbVaLFYcO3aNTg6OkImk0n2WhARtSeCIKCurg6urq6Qy5tuv3SwVyE9e/a0eiyXy/HLL7/AYDDgq6++glwux/Tp0yGXyxEQEICoqCisW7cOTz31FGpqarB582asWLECHh4eAICkpCSMGTMG5eXlUKlU2LBhA8LCwhAaGgoAiIqKwoYNG7Bx40a89NJLNtV47do1nDx5smUHTkT0gOjduzfc3NyaTLdb0ADA7t27MXv2bBgMBshkMjz//PPo2LEjSktL0bdvX6sk1Gg0yMnJAQCcOXMGJpMJvr6+YnuvXr3g4uICvV4PlUqF0tJSjBs3zmp5Go0GpaWlNtfn6OgIoOHFcnJyupehEhE9MGpra3Hy5EnxPfRGdg2a4cOH4/Dhw6isrMTmzZvx+9//HgBgNBqbpKC7uzuMRqPYDqBJHzc3N6s+7u7uTeZRVlZmc32Nu8u4VUNE1Hy3OuRg16Bp1KlTJ0yZMgXBwcHw9vaGUqnEpUuXrPpcvXoVSqUSAMTfBoMBDz30kNjHYDBY9TEYDLecR3NoNBooFIpmP4+I6EFkMplQVFR0y/ZWO73ZYrGgvr4eZ8+ehVqtRklJCSwWi9heXFwMtVoNAOjRowcUCoXVQE6dOoXq6mqxj1qtbjLQ6+dBREStw25Bk5WVhV9++QUAcPnyZaSkpMDJyQkBAQEIDw+H2WxGRkYGamtrcfz4ceTk5CAmJgYA4OzsjMjISKSnp+PixYu4cuUKUlNTMWzYMHh6egIAJk+ejK+//hp79uxBXV0dcnNzcfLkSYwfP95eQyQiopuw2+nN06dPx3fffYdr165BqVTC19cXL730Evr16weg4Xs0KSkp0Ov16Ny5M+Lj4zFlyhTx+TU1NVi8eDF27NgBs9mMIUOGICUlBZ06dRL75OXlIT09HRUVFfDy8kJycjIGDhxoc42Nm3/cdUZEZLs7vXfaLWjuBwwaIqLmu9N7Jy9BQ0REkmLQEBGRpBg098hktty5E90TvsZE97dW+R5Ne6JwkMN7k+1fCqXmOz2+5507EVGbxS0aIiKSFIOGiIgkxaAhIiJJMWiIiEhSDBoiIpIUg4aIiCTFoCEiIkkxaIiISFIMGiIikhSDhoiIJMWgISIiSTFoiIhIUgwaIiKSFIOGiIgkxaAhIiJJMWiIiEhSDBoiIpIUg4aIiCTFoCEiIkkxaIiISFIMGiIikhSDhoiIJGW3oElNTcXYsWOh0+kwePBgJCcn47fffrPq4+PjAz8/P2i1WvHn+++/F9stFgvS0tIwaNAgaLVaxMfH48KFC1bz+OabbxAREQF/f3+MGjUK27dvt8v4iIjo5uwWNA4ODkhNTcWBAwewZcsWVFRUYN68eU36ffzxxzh27Jj44+PjI7ZlZmYiLy8P2dnZ+M9//gOVSoUXX3wRFosFAHD+/HlMmzYNcXFxOHToEObOnYt58+bhu+++s9cwiYjoBnYLmldffRV9+/aFo6MjunTpgri4OBw8eLBZ81i/fj0SEhLg7e0NV1dXzJkzB2VlZThy5AgAYNOmTejduzeioqLg5OSE0NBQhIaGYv369VIMiYiIbNChtRa8f/9+qNXqJtNfe+011NXVQaVSISYmBpMnTwYAGAwGXLhwARqNRuzr7u4OLy8v6PV6BAcHo7S01KodADQaDfLz85tVW1FRkc19AwMDmzVvujuNHyaI6P7TKkGzfft25OTkIDs722r62rVrodVqIZfLUVhYiNmzZ6O+vh6xsbEwGo0AGsLlem5ubmKb0WjE448/btXu7u4utttKo9FAoVA0d1gkIQY6UdtlMplu+wHd7med5efnY+HChcjIyEC/fv2s2gYOHAhnZ2c4OTlh6NCheO6557B161YAgFKpBNCwZXM9g8EgtimVyibtV69eFduJiMj+7Bo0OTk5SElJwapVqzBgwIA79pfL5RAEAUDDlounp6dVahoMBpw7dw59+vQBAKjV6iapWlxcfNNddEREZB92C5qsrCy89957WL169U13gxQXF+PEiROora1FfX099u3bh08++QRjx44V+0RHR2P16tUoKytDVVUVUlNT0aNHD3F+kZGR+P7775Gbm4u6ujrs2bMHBQUFiI6OttcwiYjoBnY7RrN48WJ06NABU6ZMsZqen58PlUqFn3/+GampqaioqICDgwNUKhUSExMRExMj9k1ISIDBYEBsbCyqq6sRGBiIjIwMyOUNedm9e3dkZGRg6dKlSElJQdeuXbFkyRL4+/vba5hERHQDmdC4b4rEA1rNPRnAe1OZhFXR6fE9W7sEIrqNO7138hI0REQkKQYNERFJikFDRESSYtAQEZGkGDRERCQpBg0REUmKQUNERJJi0BARkaQYNEREJCkGDRERSYpBQ0REkmLQEBGRpBg0REQkKQYNERFJikFDRESSYtAQEZGkGDRERCQpBg0REUmKQUNERJJi0BARkaQYNEREJCkGDRERSYpBQ0REkmLQEBGRpBg0REQkKQYNERFJym5Bk5qairFjx0Kn02Hw4MFITk7Gb7/9ZtWnpKQE0dHR8Pf3x/Dhw5GVlWXVXlNTgwULFqB///7Q6XRITExEZWWlVZ+8vDyEh4fDz88PERER2L9/v+RjIyKiW7Nb0Dg4OCA1NRUHDhzAli1bUFFRgXnz5ontRqMRCQkJGDx4MA4ePIhly5Zh5cqV2LFjh9hnyZIlKCoqwrZt21BQUICqqiokJSWJ7UePHkVycjLmzZuHw4cPIy4uDtOmTUN5ebm9hklERDewW9C8+uqr6Nu3LxwdHdGlSxfExcXh4MGDYvuuXbsgl8sxffp0KBQKBAQEICoqCuvWrQPQsDWzefNmzJw5Ex4eHujYsSOSkpKwe/duMUg2bNiAsLAwhIaGwsnJCVFRUXjiiSewceNGew2TiIhu0KG1Frx//36o1WrxcWlpKfr27Qu5/H/Zp9FokJOTAwA4c+YMTCYTfH19xfZevXrBxcUFer0eKpUKpaWlGDdunNVyNBoNSktLm1VbUVGRzX0DAwObNW+6O0eOHGntEojoLrVK0Gzfvh05OTnIzs4WpxmNRri5uVn1c3d3h9FoFNsBNOnj5uZm1cfd3b3JPMrKyppVn0ajgUKhaNZzSFoMdKK2y2Qy3fYDut3POsvPz8fChQuRkZGBfv36idOVSqUYGI2uXr0KpVIptgOAwWCw6mMwGKz63Nh+/TyIiMj+7Bo0OTk5SElJwapVqzBgwACrNrVajZKSElgsFnFacXGxuHutR48eUCgUVql56tQpVFdXi33UanWTVL1+HkREZH92C5qsrCy89957WL169U13g4SHh8NsNiMjIwO1tbU4fvw4cnJyEBMTAwBwdnZGZGQk0tPTcfHiRVy5cgWpqakYNmwYPD09AQCTJ0/G119/jT179qCurg65ubk4efIkxo8fb69hEhHRDWSCIAj2WJCPjw86dOgAJycnq+n5+flQqVQAGr5Hk5KSAr1ej86dOyM+Ph5TpkwR+9bU1GDx4sXYsWMHzGYzhgwZgpSUFHTq1Ensk5eXh/T0dFRUVMDLywvJyckYOHCgTTU27mds7jEa703NOwZEzXN6fM/WLoGIbuNO7512C5r7AYOmbWLQELVtd3rv5CVoiIhIUgwaIiKSFIOGiIgkxaAhIiJJMWiIiEhSDBoiIpIUg4aIiCTFoCEiIkkxaIiISFIMGiIikhSDhoiIJGVz0JSXl+Nml0UTBEG8lTIREdGNbA6aJ598EpcvX24yvbKyEk8++WSLFkVERO2HzUFzq4s819TUNLn0PxERUaMOd+qwcuVKAIBMJsPq1avxu9/9TmyzWCw4evQoevXqJV2FRER0X7tj0GzduhVAwxbNzp074eDgILY5OjqiW7dueOutt6SrkIiI7mt3DJpdu3YBAOLi4rBy5Up07NhR8qKIiKj9uGPQNPr000+lrIOIiNopm4MGAAoLC7F//378+uuvsFgsVm1Lly5t0cKIiKh9sDlo/v73vyMtLQ3e3t549NFHIZPJpKyLiIjaCZuD5rPPPsP8+fPxzDPPSFkPERG1MzZ/j8ZgMGDo0KFS1kJERO2QzUEzYsQIFBYWSlkLERG1QzbvOgsICMDy5cvxww8/QK1Ww9HR0ar96aefbvHiiIjo/mdz0DR+KTMrK6tJm0wmY9AQEdFN2Rw0paWlUtZBRETtlF3vR5Ofn4/Y2FjodDr4+Pg0affx8YGfnx+0Wq348/3334vtFosFaWlpGDRoELRaLeLj43HhwgWreXzzzTeIiIiAv78/Ro0ahe3bt0s+LiIiujWbt2gaL655Ky+99NId5+Hu7o7Y2FjU1NTgjTfeuGmfjz/+GCEhITdty8zMRF5eHrKzs+Hh4YG//e1vePHFF7FlyxbI5XKcP38e06ZNw5tvvok//vGP2LdvHxITE+Hp6Ql/f/87D5KIiFqczUHTeHHNRvX19fj555/h5OSERx991KagGTJkCADgwIEDzSyzwfr165GQkABvb28AwJw5czBo0CAcOXIEwcHB2LRpE3r37o2oqCgAQGhoKEJDQ7F+/XoGDRFRK7E5aBovrnm9S5cuISkpCdHR0S1W0GuvvYa6ujqoVCrExMRg8uTJABq+x3PhwgVoNBqxr7u7O7y8vKDX6xEcHIzS0lKrdgDQaDTIz89vVg1FRUU29w0MDGzWvOnuHDlypLVLIKK71Kxrnd2oS5cuSExMxGuvvYYRI0bcczFr166FVquFXC5HYWEhZs+ejfr6esTGxsJoNAJoCJfrubm5iW1GoxGPP/64Vbu7u7vYbiuNRgOFQnEPI6GWxkAnartMJtNtP6Df88kAHTp0wMWLF+91NgCAgQMHwtnZGU5OThg6dCiee+45cZedUqkE0LBlcz2DwSC2KZXKJu1Xr14V24mIyP5s3qI5evSo1WNBEHDx4kVkZmY22V3VUuRyuXgLaTc3N3h6eqKoqAi+vr4AGkLm3Llz6NOnDwBArVbj3//+t9U8iouLoVarJamPiIjuzOagiY2NhUwmE9/4G+l0OixatMimeZjNZtTX16Ourg5Aw+YW0HCnTr1eD4vFAh8fH8jlchw4cACffPIJZsyYIT4/Ojoaq1evxoABA+Dh4YHU1FT06NFD3K0SGRmJzMxM5ObmIiIiAt988w0KCgrwj3/8w9ZhEhFRC7M5aL766iurx3K5HA899FCzjmVs2bIF8+bNEx/7+fkBaLjawLVr15CamoqKigo4ODhApVIhMTERMTExYv+EhAQYDAbExsaiuroagYGByMjIgFzesAewe/fuyMjIwNKlS5GSkoKuXbtiyZIlPOOMiKgVyYQbN1EeYI0HtJp7MoD3pjIJq6LT43u2dglEdBt3eu9s1llnZWVlyMzMxI8//ggAeOKJJxAfH4+ePflGQEREN2fzWWf79u3D008/Db1eD39/f/j7+6OkpAQRERHYv3+/lDUSEdF9zOYtmrS0NMTExDS5dMyiRYvw/vvv44svvmjx4oiI6P5n8xbNyZMnrQ7MN4qNjcXJkydbtCgiojux1Jlau4R2r6VeY5u3aJRKJSoqKsTrjDUqLy/nFyKJyO7kjgqcWcjjw1LqkdIyJzrZvEUzcuRIzJ8/H//+979RXV2N6upq7N27FwsXLsTIkSNbpBgiImp/bN6iSUpKwrx58/DCCy9AJpOJ00eNGoU5c+ZIUhwREd3/bA4aV1dXpKen49y5c1anN3fv3l2y4oiI6P5nc9C88sor6NevH/7yl7/gscceE6f//e9/R0lJCZYtWyZJgUREdH+z+RjN4cOHMXTo0CbThw4disOHD7doUURE1H7YHDQGgwG/+93vmkx3dnbGlStXWrQoIiJqP2wOmsceewz79u1rMn3fvn3o1q1bixZFRETtR7NuE/Dee++htrYWgwYNAtAQMitWrMCsWbMkK5CIiO5vNgfNM888g0uXLuGDDz7AO++8AwBwcnLC888/j7i4OMkKJCKi+1uzrt78yiuv4IUXXsAPP/wAAHj88cdvetyGiIioUbOCBgBcXFzEG5YRERHdic0nAxAREd0NBg0REUmKQUNERJJi0BARkaQYNEREJCkGDRERSYpBQ0REkmLQEBGRpBg0REQkKQYNERFJyq5Bk5+fj9jYWOh0Ovj4+DRpLykpQXR0NPz9/TF8+HBkZWVZtdfU1GDBggXo378/dDodEhMTUVlZadUnLy8P4eHh8PPzQ0REBPbv3y/pmIiI6PbsGjTu7u6IjY1FcnJykzaj0YiEhAQMHjwYBw8exLJly7By5Urs2LFD7LNkyRIUFRVh27ZtKCgoQFVVFZKSksT2o0ePIjk5GfPmzcPhw4cRFxeHadOmoby83C7jIyKipuwaNEOGDMG4cePQvXv3Jm27du2CXC7H9OnToVAoEBAQgKioKKxbtw5Aw9bM5s2bMXPmTHh4eKBjx45ISkrC7t27xSDZsGEDwsLCEBoaCicnJ0RFReGJJ57Axo0b7TlMIiK6TrOv3iyV0tJS9O3bF3L5/7JPo9EgJycHAHDmzBmYTCb4+vqK7b169YKLiwv0ej1UKhVKS0sxbtw4q/lqNBqUlpY2q5aioiKb+wYGBjZr3nR3jhw50tolUBvD/z37aIn/vTYTNEajEW5ublbT3N3dYTQaxXYATfq4ublZ9XF3d28yj7KysmbVotFooFAomvUckhbfVIhahy3/eyaT6bYf0NvMWWdKpVIMjEZXr16FUqkU2wHAYDBY9TEYDFZ9bmy/fh5ERGR/bSZo1Go1SkpKYLFYxGnFxcVQq9UAgB49ekChUFil5qlTp1BdXS32UavVTVL1+nkQEZH92TVozGYzTCYT6urqADRsbplMJlgsFoSHh8NsNiMjIwO1tbU4fvw4cnJyEBMTAwBwdnZGZGQk0tPTcfHiRVy5cgWpqakYNmwYPD09AQCTJ0/G119/jT179qCurg65ubk4efIkxo8fb89hEhHRdewaNFu2bIGfnx/i4+MBAH5+fvDz88OhQ4egVCqRmZmJvXv3IigoCC+//DJmzJiB0aNHi89PTk5Gnz59MHbsWISGhkKhUODdd98V23U6HRYvXozFixcjMDAQa9euRUZGhhhERERkfzJBEITWLqKtaDyg1dyTAbw3Ne9kA2qe0+N7tnYJ1EadWci/DSn1SLHtve1O751t5hgNERG1TwwaIiKSFIOGiIgkxaAhIiJJMWiIiEhSDBoiIpIUg4aIiCTFoCEiIkkxaIiISFIMGiIikhSDhoiIJMWgISIiSTFoiIhIUgwaIiKSFIOGiIgkxaAhIiJJMWiIiEhSDBoiIpIUg4aIiCTFoCEiIkkxaIiISFIMGiIikhSDhh5YFouptUto9/gaEwB0aO0CiFqLXK7Amf09W7uMdq3HwLLWLoHaAG7REBGRpBg0REQkqTYVNCtWrECfPn2g1WrFn1dffVVsLykpQXR0NPz9/TF8+HBkZWVZPb+mpgYLFixA//79odPpkJiYiMrKSnsPg4iIrtOmggYAgoKCcOzYMfEnLS0NAGA0GpGQkIDBgwfj4MGDWLZsGVauXIkdO3aIz12yZAmKioqwbds2FBQUoKqqCklJSa01FCIiQhsMmlvZtWsX5HI5pk+fDoVCgYCAAERFRWHdunUAGrZmNm/ejJkzZ8LDwwMdO3ZEUlISdu/ejfLy8launojowdXmzjorKirCgAED4OLiIu7+6t69O0pLS9G3b1/I5f/LRo1Gg5ycHADAmTNnYDKZ4OvrK7b36tULLi4u0Ov1UKlUzarBVoGBgTb3pbt35MiRFp8n1519SLHuAK4/e2mJ9demgmbUqFGYMGECVCoVLl68iPfffx/PP/88tmzZAqPRCDc3N6v+7u7uMBqNACD+vrGPm5ub2GYrjUYDhUJxDyOhlsY3lfsX1939zZb1ZzKZbvsBvU3tOuvduzc8PT0hk8ng4eGBxYsX45dffsGxY8egVCqbBMbVq1ehVCoBQPxtMBis+hgMBrGNiIjsr00FzY1kMhlkMhkEQYBarUZJSQksFovYXlxcDLVaDQDo0aMHFAqFVaqeOnUK1dXVYh8iIrK/NhU027dvx+XLlwEAly5dwvz58/HQQw9Bq9UiPDwcZrMZGRkZqK2txfHjx5GTk4OYmBgAgLOzMyIjI5Geno6LFy/iypUrSE1NxbBhw+Dp6dmawyIieqC1qaDZunUrxowZA39/f0RGRsJkMuGTTz6BUqmEUqlEZmYm9u7di6CgILz88suYMWMGRo8eLT4/OTkZffr0wdixYxEaGgqFQoF33323FUdERERt6mSAVatW3ba9b9+++Pzzz2/Z7uzsjLfffhtvv/12S5dGRER3qU1t0RARUfvDoCEiIkkxaIiISFIMGiIikhSDhoiIJMWgISIiSTFoiIhIUgwaIiKSFIOGiIgkxaAhIiJJMWiIiEhSDBoiIpIUg4aIiCTFoCEiIkkxaIiISFIMGiIikhSDhoiIJMWgISIiSTFoiIhIUgwaIiKSFIOGiIgkxaAhIiJJMWiIiEhSDBoiIpIUg4aIiCTFoCEiIkm1u6CxWCxIS0vDoEGDoNVqER8fjwsXLrR2WURED6x2FzSZmZnIy8tDdnY2/vOf/0ClUuHFF1+ExWJp7dKIiB5IHVq7gJa2fv16JCQkwNvbGwAwZ84cDBo0CEeOHEFwcPBtnysIAgCgtra2Wct8xMF8d8WSTUwmk2TzNuMRyeZN0q47ADC7cP1Jydb11/ie2fgeeqN2FTQGgwEXLlyARqMRp7m7u8PLywt6vf6OQVNXVwcAOHnyZLOW+6Fn82sl2xUVXZFu5k4Z0s2bUFlUJO0CRnL9Sam566+urg7Ozs5NpreroDEajQAawuV6bm5uYtvtuLq6onfv3nB0dIRMJpOkRiKi9kYQBNTV1cHV1fWm7e0qaJRKJYCGLZvrGQwGse125HI53NzcJKmNiKg9u9mWTKN2dTKAm5sbPD09UXTd5p7BYMC5c+fQp0+fVqyMiOjB1a6CBgCio6OxevVqlJWVoaqqCqmpqejRowcCAwNbuzQiogdSu9p1BgAJCQkwGAyIjY1FdXU1AgMDkZGRAbm83WUqEdF9QSbc6nw0IiKiFsCP+UREJCkGDRERSYpBQ0REkmLQ0D2bO3cu5s6d29pl0P9XXl4OrVaL8vLy1i6FABw+fBharfae+9zPGDQPqBUrViAuLq61y6B7tHHjRoSFhVlNU6lUOHbsGFQqVStVRdcLCgrCsWPHxMc3+9+7sU97w6ChW6qvr7/lRfKIiGzFoGmj4uLisHTpUrz22mvQ6XQYNmwYPv/8c7H9u+++Q1xcHEJCQhAaGoply5ahvr4eAHD+/Hn4+Pjg/PnzYv8DBw7Ax8cHALB161Z89NFH4ua6VqtFSUmJ2Cc/Px8jR45EQEAAqqqq8Nlnn2Hs2LHQarUYMmQIUlJSUF1dbd8XpI26l/UEACdOnMCkSZOg1WoxYcIErF27VlxPQMN6+9Of/oT+/fsjJCQEL774In766ScADbtbFi5cKO4q02q1+PLLL63Wf2VlJXx9fZt8Wk5KSsLrr78uPt68eTMiIiIQGBiIsWPHIj8/X6qXrM2Ki4vDokWLMH36dGi1WoSHh2Pr1q1i+5dffonIyEgEBgZizJgxyMnJEduuXr2KWbNmISQkBDqdDqNGjcKOHTsANO9/DwB+/PFH9O3bFz///LNVfc8++yyWL18OADCbzVizZg1Gjx6NwMBATJgwAfv375f09bknArVJzz77rBAUFCQUFhYKZrNZyM/PF/r06SOcO3dOOHXqlBAQECBs375dqKurE86fPy9EREQIH374oSAIgvDTTz8JvXv3Fn766SdxfoWFhULv3r3Fx+np6cKzzz5rtczGPi+//LLw22+/CTU1NYLFYhF27twpnDlzRrBYLMKPP/4ojBw5UkhLSxOfl5SUJCQlJUn8irRN97Kerl69KoSEhAgrV64UTCaTcOrUKWHUqFFW6+nw4cPCsWPHhNraWuG3334Tpk2bJvzpT38S23Nzc4XQ0FCrmm5c/7NmzRLeeOMNsd1gMAj+/v7CwYMHreZx4sQJwWw2C4cOHRJ0Op1w6NAhyV63tujZZ58V/Pz8hIKCAqGurk4oKCgQ+vXrJ3z77bfCsWPHhH79+gn/+te/hPr6euHgwYOCTqcTdu7cKQiCIKSlpQkvvPCCYDAYBIvFIpw/f1744YcfBEFo3v9eo8mTJ4t/J4IgCGVlZYJarRbXaXp6uhAZGSmcPn1aMJvNwq5du4SAgADh7Nmzkr0+94JbNG3YU089hZCQEMjlcowZMwZubm4oKSnBunXrMGLECIwePRodOnSAp6cn/vKXv2Djxo0tstzZs2ejU6dOUCgUkMlkCA8Ph5eXF2QyGXr16oXY2Fh88803LbKs9uBu11NBQQEcHR0xbdo0ODk5wdvbG1OnTrWad2BgIAICAuDo6IhOnTrhpQfMKugAAAhXSURBVJdewrffftusLcpJkyZh+/btqKqqAgDk5eXBw8NDvG3GJ598gmnTpkGj0UAulyMoKAjjxo3Dpk2bWugVun+EhoZi+PDh6NChA4YPH44RI0YgNzcXGzduxJNPPokRI0bAwcEBwcHBmDx5srj16ujoiMrKSpw+fRqCIMDT0xOPP/74XdcxadIk5Obmiruuc3NzMWDAAHTr1g0AsHbtWrz++uvo2bMn5HI5Ro4cicDAQOTl5d37iyCBdncJmvbk0UcftXrs6uqKa9eu4cyZMzhw4AAKCgrENovF0mLHUxr/mBvt2LEDa9aswdmzZ1FfX4/6+np06dKlRZbVHtzteqqoqIBKpbK6PJKnp/XNjfR6PdLS0qDX68WgEAQBly9fbtL3VgYOHIjOnTvjn//8JyZOnIgvvvgCEydOFNvPnj2LJUuW4J133hGnmc1mBAUF2fgKtB83/u1369YN33//PQBY7dIEgMceewx79uwBAMTHx8NsNuPNN99ERUUFBg4ciNmzZ6N79+53VceYMWOwZMkSHDhwAMHBwdi0aROSk5MBAL/++iuMRiNmzJhh9bdTX19v89+EvTFo7kOPPPIIIiMj8fbbb9+0vfGeENd/6r148aJVn9vdb+f6P96KigrMmjULH3zwAcLCwuDk5IS1a9ciKyvrXobwQLjTeuratSvKy8shCIK4Pm48JTkxMRFhYWF4//334e7ujpKSEowfP14MK1uu4SeTyTBhwgTk5ubC19cXer0eGRn/u2HYww8/jFmzZuHpp5++26G2GxcuXGjyuGvXrpDJZFbHPAHg3Llz+P3vfw8AcHFxwcyZMzFz5kxUVlbirbfewrx585Cdnd1kGbbc68rV1RWjR49Gbm4uqqurUVdXhxEjRgBouN+WQqFAZmYmdDrd3Q7Vrrjr7D4UExODf/7zn9i5cydqa2thNptx9uxZ7N27FwDQuXNndOvWDRs2bEB9fT3OnTuHNWvWWM3jkUcewYULF+542+pr167BYrGgc+fOcHJyQmlpKT777DPJxtae3Gk9hYaGora2FqtWrUJtbS3OnDnTJMANBgNcXV2hVCrx66+/Ij093ar94YcfxuXLl3Hlyu3vQjpx4kR8++23eP/99zFs2DA88sj/boE8depUrFy5EidOnIDFYkFtbS2OHz9udbuNB0VBQQH27NkDs9mMPXv24F//+hcmTJiA8ePH48svv8TXX38Ns9mMw4cPIycnB1FRUQCAr776Cj/88APq6+vh4uIChUJxyw8Btv7vRUVFYdeuXVi7di0iIiLg5OQEAHByckJ0dDTeffddnDp1CoIgoKamBocOHUJZWVnLviAthEFzH/Lz88Pq1avx+eefY+jQoQgJCcErr7xi9Wn4nXfeQWFhIYKDg/H6669j0qRJVvMYM2YMvLy8MHjwYAQFBUGv1990Wb169UJiYiJmzZoFnU6Hd999F3/84x8lHV97caf15Obmho8//hhffvklQkJC8Oqrr2L8+PHiGwoALF68GNu2bYNOp8Pzzz+PkSNHWi1jwIABGDZsGMLDwxEUFISvvvrqprV07doVf/jDH7B79+4mfwtTp07FjBkzsHDhQvTv3x9DhgxBamrqA3lm4aRJk/D5558jKCgIixYtwqJFi8Szw9LS0rB8+XIEBwfjzTffxJw5c/DUU08BaDjTc8aMGQgODsbQoUNx6dIlLFq06KbLsPV/T6vVolu3bigsLGyyzpKSkjB69GjMnDkTQUFBCAsLw0cffWR1RmNbwqs3E7Uh2dnZ+PTTT7Fz587WLuWBExcXh/79++Pll19u7VLaHW7RELWiffv2oaKiAoIg4MSJE8jMzOSxEmp3eDIAUSs6deoUkpKSYDAY0KVLFzz99NP485//3NplEbUo7jojIiJJcdcZERFJikFDRESSYtAQEZGkGDREdjR37lw899xzNvcPCwvDhx9+eM/Lban5EN0NBg0REUmKQUNERJJi0BC1kuLiYiQkJGDgwIHQarWYOHGieB2065lMJrzxxhvQ6XQICQlBWloaLBaL2F5XV4cVK1YgLCwMvr6+GDt2LNavX2/PoRDdFr+wSdRKjEYjxowZg6SkJHTo0AGbN2/G9OnTsW3bNvTs2VPs9+mnn2Lq1Kn44osvcPz4cfz1r39Fly5dxHvXzJ8/H8XFxXjrrbfg5eWFEydOYMGCBXBwcBAv+kjUmhg0RK0kJCTE6vGsWbNQUFCAHTt2YNq0aeJ0tVqNmTNnAgC8vb1x+vRprFmzBlOnTsVPP/2EzZs3Iz8/H7169QIAdO/eHadPn0Z2djaDhtoEBg1RK7l8+TLS09NRWFiIX3/9FWazGSaTqck9aQICAqwe63Q6fPTRRzAajSgqKoIgCE2u7ltfXw8HBwfJx0BkCwYNUSuZO3cu/vvf/2LOnDno1q0bnJ2dMWvWLNTV1dk8j8YrSP3f//0fXFxcrNpsucEWkT0waIhayaFDhzBnzhw8+eSTAICqqiqcP38evXv3tur33XffWT0+duwYPDw8oFQq0a9fPwDAf//7X4SGhtqncKJmYtAQtZKePXti27ZtCAwMhMViwfLly2E2m5v00+v1WLFiBcaNG4eioiJkZWWJx2y8vLwwceJEzJ8/H7Nnz4ZWq0V1dTWKiopw+fJlXgma2gQGDVErWbp0KRYuXIioqCg8/PDDiI+PR01NTZN+cXFxKC8vx8SJE+Ho6IhnnnkGU6ZMEdvffvttrFmzBqtWrcL58+fh6uqKJ554As8884w9h0N0S7xNABERSYpf2CQiIkkxaIiISFIMGiIikhSDhoiIJMWgISIiSTFoiIhIUgwaIiKSFIOGiIgkxaAhIiJJ/T+6sd24zWyauQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dyCiiQKuOhHH"
      },
      "source": [
        "##Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "38uuUB3OqGJU"
      },
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "label_encoder = LabelEncoder()\n",
        "df[\"label\"] = label_encoder.fit_transform(df[\"label\"])"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "My9cvBvMmyE9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "outputId": "68ebe6c6-fa74-475a-e1e7-a1383378ce28"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>According to Gran , the company has no plans t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Technopolis plans to develop in stages an area...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>The international electronic industry company ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>With the new production plant the company woul...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>According to the company 's updated strategy f...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label                                               text\n",
              "0      1  According to Gran , the company has no plans t...\n",
              "1      1  Technopolis plans to develop in stages an area...\n",
              "2      0  The international electronic industry company ...\n",
              "3      2  With the new production plant the company woul...\n",
              "4      2  According to the company 's updated strategy f..."
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aKjQe1FLmu-t"
      },
      "source": [
        "class_names = ['negative', 'neutral', 'positive']"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KiKGnDbwOlFd"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GvPJZKutPa_y"
      },
      "source": [
        "### Choosing Sequence Length"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RuRaMmlJPdjp"
      },
      "source": [
        "token_lens = []\n",
        "\n",
        "for txt in df['text']:\n",
        "  tokens = tokenizer.encode(txt, max_length = 512, truncation = True)\n",
        "  token_lens.append(len(tokens))"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r9nzPp2rPtRJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 288
        },
        "outputId": "83a4ed80-d6d3-40f3-c432-57a66465e0ff"
      },
      "source": [
        "sns.distplot(token_lens)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7fcecfb05890>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAD+CAYAAAAK/2PhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfVxUdd74/9cMcqMwo1CmggqpKeIYIlpJrkntom7l2nVFIbt2bYlXpvndNvJR7W5dWVe1Ruu66S/WvTDL1EXJrEQz29Jc10xFyhDRahEU8Q5QhpsZmJnz++PIyDDcGswNvJ+PxzxGznnP57zPNM17zjmf8/loFEVREEIIIVxE6+4EhBBC9CxSeIQQQriUFB4hhBAuJYVHCCGES0nhEUII4VK93J2Ap7PZbFRXV+Pr64tGo3F3OkII4fEURaG+vp7AwEC0WufjGyk8baiurubEiRPuTkMIIbzOyJEj0el0Tsul8LTB19cXUN9APz+/Tms3Ly8Pg8HQae25mjfnL7m7hzfnDt6dv6tzr6ur48SJE/bvz6ak8LSh4fSan58f/v7+ndp2Z7fnat6cv+TuHt6cO3h3/u7IvaXLE9K5QAghhEtJ4RFCCOFSUniEEEK4lBQeIYQQLiWFRwghhEtJ4RFCCOFSUniEEEK4lNzHIzqsol6hPmQQRaarcwjqfSDYV4YUEkK0TQqP6LBKK2w9W0d4o/vRpoVAcPM3KQshhAM51SaEEMKlpPAIIYRwKSk8QgghXEoKjxBCCJeSwiOEEMKlpPAIIYRwKSk8QgghXMqlhcdms7Fs2TLi4uKIiYlh7ty5lJSUtBifn59PUlIS0dHRTJ06lbVr1zqsX7JkCfHx8YwfP55JkyaxaNEiTp8+bV9/+vRpRo0axbhx44iJibE/jEZjl+2jEEKI1rm08GRkZJCdnc26devYu3cvoaGhzJ8/H5vN5hRbVVVFSkoKkydP5sCBAyxfvpyVK1eyY8cOe0xycjJbt27l8OHDfPbZZ4SGhvLEE084tZWdnU1ubq790dwc4EIIIVzDpYUnMzOTlJQUhg0bRmBgIIsXL6awsJCcnByn2J07d6LValmwYAH+/v6MGzeOxMRENmzYYI+56aabCAoKsv+t1WopLCx0yb4IIYS4Ni4rPEajkZKSEgwGg32ZXq8nPDycY8eOOcUXFBQQFRWFVns1RYPBQEFBgUPchg0biI2NJSYmhrVr17Jo0SKntmbPns2tt95KUlISn376aSfulRBCiI5y2VhtVVVVgFpsGtPpdPZ1TeObnhLT6/VOscnJySQnJ3Pu3Dk2b95MVFSUfV1wcDCZmZmMGTMGm83Gp59+ypNPPsnKlSu54447OpR/Xl5eh+Lbo7kjPW9QHzIIgKKiIvuyUrMfF8tL3ZVSh3nrew+Suzt5c/6elLvLCk/DKbGmF/aNRqPD6bLG8WVlZQ7LKisrm40FGDBgAA8++CA//elP+eyzzwgJCSEwMJCYmBh7zL333sv+/fv56KOPOlx4DAYD/v7+bQe2U05ODrGxsZ3WnisVmRQ4W0R4eLh92aAQCL8x1I1ZtZ83v/eSu/t4c/6uzt1sNrf6Y91lp9p0Oh1hYWEOyRiNRoqLixk9erRTfGRkJPn5+Q4dD44ePUpkZGSL27BYLNTU1HDu3LkWY7RaLYqitLheCCFE13Jp54KkpCRWr15NYWEhNTU1pKWlERER0WwlTkhIwGq1kp6eTl1dHUeOHCErK4vZs2cDcPnyZd577z0qKioAKC0t5YUXXiAsLIzhw4cDcOjQIb7//nssFgt1dXVs376dDz/8kLvvvtt1Oy2EEMKBS+fjSUlJwWg0kpycTG1tLbGxsaSnp6PVajl06BDz5s1j27ZthIaGEhQUREZGBkuWLGHVqlUEBwezcOFCZsyYAYBGo2Hbtm2kpaVhNpvR6/VMnDiRNWvW4OfnB0BhYSGrVq3i4sWL+Pn5ERERwdKlS7nrrrtcudtCCCEacWnh0Wq1pKamkpqa6rRuwoQJ5ObmOiyLiopi48aNzbal1+tZs2ZNq9tLTEwkMTHx2hMWTk6bFF4ohJ3mgcRchHuuA61MPCqE6ACZgVS024U6hZ9+DUUm6IeG7WVwvg4eGeTuzIQQ3kTGahPtYrEp/OJbKDbDuih42K+Ue6+DQ0bIde4NL4QQLZLCI1pUUa9QZFIfrxbD/kr44zAYEwgaDcy4Dgb4wo4ykI6CQoj2ksIjWlRphU/K4b3zsLQIIvuAvxbqrvRw12ogIQROmeGfl92bqxDCe0jhEW3aXgYmGzxwg3qk09itfaFvL1h71j25CSG8jxQe0arLFth7GW7rC6HNDNzQSwPjg2DvJaixyvk2IUTbpPCIVn1aDlYFZoS0HBMdBGYF/lHhuryEEN5LCo9o0aV6+OISTNRDf7+W427qAzof+Oii63ITQngvKTyiRZnnoV5ROxC0xkcDd/SD7Itgk+5tQog2SOERzbLYFN49C6P6QFg7BuW+KxjO10OOzCouhGiDFB7RrA8uwpk6iO/XvvhJfdXnvdKtWgjRBik8ollvnIYh/nBz89MfObmuFwz1h53l2G86LTIpVNTLqTchhCMZq004OWxU2HsZfh/e/gFAq61qd+t9l9WRDBru95kWAsG+XZerEML7yBGPAJoMj1MEfbRw73Uda2NEbzBa4Vx91+QohOge5IhHAFeHx6mywIcX4fa+ENDBnyUjeqvPP9TAwFa6XwshejY54hEODhjBosBP+nb8tQP8IMgHvqvt/LyEEN2HFB7h4MvLaieBwQEdf61GA8MC4KSp8/MSQnQfUniE3WmTOtL0pGs42mkQHgDn6tRBRYUQojkuLTw2m41ly5YRFxdHTEwMc+fOpaSkpMX4/Px8kpKSiI6OZurUqaxdu9Zh/ZIlS4iPj2f8+PFMmjSJRYsWcfr0aYeYffv2MXPmTKKjo5k2bRrbt2/vkn3rDr6sVAf9nKi/9jaGBoACnJKjHiFEC1xaeDIyMsjOzmbdunXs3buX0NBQ5s+fj83m/PO4qqqKlJQUJk+ezIEDB1i+fDkrV65kx44d9pjk5GS2bt3K4cOH+eyzzwgNDeWJJ56wrz99+jSPPfYYc+bM4eDBgzzzzDM8++yzfPPNNy7ZX29iU+CwEaL6qNdprlX4lVN0RVJ4hBAtcGnhyczMJCUlhWHDhhEYGMjixYspLCwkJyfHKXbnzp1otVoWLFiAv78/48aNIzExkQ0bNthjbrrpJoKCrt7hqNVqKSwstP+9ZcsWRo4cSWJiIn5+fsTHxxMfH09mZmbX7qgX+qYKKiwwXvfj2tH3guBeUniEEC1zWXdqo9FISUkJBoPBvkyv1xMeHs6xY8eYOHGiQ3xBQQFRUVFotVdro8FgICsryyFuw4YN/OlPf6KqqopevXqxePFihzYab6+hjW3btnU4/7y8vA6/pi3NFVx32cRIfAgkuOIURZfU0QYMESEUFZU7xRoi1FFDi4qKHJY1xPa39eeHKl+Kis5QavbjYnmpC/agYzzpve8oyd19vDl/T8rdZYWnqqoKUItNYzqdzr6uabxO5/jzW6/XO8UmJyeTnJzMuXPn2Lx5M1FRUQ5tjBgxos022sNgMODv347RMtspJyeH2NjYTmvvx1AUhT37ICoQRg0eal8eFATh4c6HQOpBZjnh4eHNxkaWqVMk3DA4nEH9IfzG0K7ehQ7xpPe+oyR39/Hm/F2du9lsbvXHustOtTWcEjMaHYcvNhqNDqfLGsc3LRCVlZXNxgIMGDCABx98kEcffZTy8nJ7G02311obPVVulTogaMyPPM3WoOE6T7G5c9oTQnQvLis8Op2OsLAwhypoNBopLi5m9OjRTvGRkZHk5+c7dDw4evQokZGRLW7DYrFQU1PDuXPn7G00rbpttdET7ShTnw2BndPe0CsHhnKdRwjRHJd2LkhKSmL16tUUFhZSU1NDWloaERERzR4CJiQkYLVaSU9Pp66ujiNHjpCVlcXs2bMBuHz5Mu+99x4VFep8y6WlpbzwwguEhYUxfPhwAGbNmsXx48fZvHkz9fX1fPHFF+zatYukpCTX7bQX+KRcLTr6TjrxqusFIb2gWAqPEKIZLi08KSkpzJgxg+TkZOLi4igpKSE9PR2tVsuhQ4eIiYnhzJkzgHqaLCMjgz179jBhwgQWLVrEwoULmTFjBgAajYZt27Yxffp0xo0bx4MPPkifPn1Ys2YNfn7qQGFDhgwhPT2dt99+m9jYWF5++WVeeeUVoqOjXbnbHu2yRWFfJUxp57w77RUeIEc8QojmuXSQUK1WS2pqKqmpqU7rJkyYQG5ursOyqKgoNm7c2Gxber2eNWvWtLnNuLg4tm7dem0J9wCfVYBVUaeurrB0XrvhAeq1o8pObFMI0T3IkDk93I4y0PtATCf3txh6pYNBXnXntiuE8H5SeHq4f15WT7P5dvInoaHwHOl4z3UhRDcn8/H0MBX1CpVW9d8X6+F4Dcy6HmqtnbudIB+43he+lSMeIUQTUnh6mIYJ3wByr9ziZAPqumA06aH+8K0c8QghmpBTbT3Y97Xgq7l6w2dnCw9Qp1kor1e6ZgNCCK8khacH+74WIgLUqRC6QsN1nsPG1uOEED2LFJ4eymRT58wZ0bvrttFQeHKk8AghGpHC00MVmdRrO8O7sPAE+sAQfzniEUI4ksLTQzUMZ9NV13caGALliEcI4UgKTw9VbFInbNN1cb/GsUHwb5N0MBBCXCWFp4cqNl29BtOVGka8ltNtQogGUnh6oFornKu/On1BVxp7pfDI6TYhRAMpPD3QqSsTtLniiCfoSgeDf16GIpNCkUmhQk67CdGjSeHpgVzVsQCg2gr9feFApTpiwifl2IfsEUL0TFJ4eqBiE/Tr1XkTv7VlaIA6Lly1FBwhBFJ4eqQSMwx2wfWdBg2n9GRGUiEESOHpcSyK2rEgVAqPEMJNpPD0MEUmtfiE+rlum0E+cJ0vFJldt00hhOdyaeGx2WwsW7aMuLg4YmJimDt3LiUlJS3G5+fnk5SURHR0NFOnTmXt2rX2dXV1dTz//PMkJCQQExPD1KlT+eMf/4jJdPVn9enTpxk1ahTjxo0jJibG/jAae27f3hM16vMgFx7xAIT7q0VPCCFcWngyMjLIzs5m3bp17N27l9DQUObPn4/N5jwZTFVVFSkpKUyePJkDBw6wfPlyVq5cyY4dOwCwWCwEBweTnp7OoUOHePfdd9m/fz9paWlObWVnZ5Obm2t/6HS6Lt9XT/VdDWiAQS484gGI6A1l9VBpce12hRCex6WFJzMzk5SUFIYNG0ZgYCCLFy+msLCQnJwcp9idO3ei1WpZsGAB/v7+jBs3jsTERDZs2ABAnz59+O1vf8vw4cPx8fFhyJAh3H///Rw4cMCVu+R1TtSqp738XHySddiV6zyFctQjRI/nshlIjUYjJSUlGAwG+zK9Xk94eDjHjh1j4sSJDvEFBQVERUWh1V79hjQYDGRlZbW4jS+//JLIyEin5bNnz6auro4bb7yRuXPn8rOf/azD+efl5XX4NW1pruB2tfzasfSz1lNUdMFhuSEihKKi8jaXNSwHKCoqaldsUVE5KBq0DOHrs5WU+tZwsby0M3bnmrnjve8skrv7eHP+npS7ywpPVZU6B7Jer3dYrtPp7Ouaxjc9JabX65uNBfU03uHDh9m8ebN9WXBwMJmZmYwZMwabzcann37Kk08+ycqVK7njjjs6lL/BYMDfv/MujOTk5BAbG9tp7bVHnU3h1B64K9iX8P7hDuuCgiA8XNfmsoblUE54eHi7YhuWDymCMk1fBg3qS/iNoT9+h66RO977ziK5u4835+/q3M1mc6s/1l12wiVI/bZyurBvNBrt65rGNy0ylZWVzcauXr2aNWvW8M477xAaevULLTAwkJiYGPz8/AgICODee+9l5syZfPTRR52xS17nu1rX92hrbFjA1V51Qoiey2WFR6fTERYW5lAFjUYjxcXFjB492ik+MjKS/Px8h44HR48edTqVtmLFCt555x3effddRo4c2WYeWq0WRemZ33wF1erzQBf3aGtwY28wK1d71gkheiaXXmJOSkpi9erVFBYWUlNTQ1paGhEREc0eAiYkJGC1WklPT6euro4jR46QlZXF7Nmz7TFLly5ly5YtrF+/nmHDhjm1cejQIb7//nssFgt1dXVs376dDz/8kLvvvrtL99NTfVerPt/g657t33ilg0Fuz+3NLoTAhdd4AFJSUjAajSQnJ1NbW0tsbCzp6elotVoOHTrEvHnz2LZtG6GhoQQFBZGRkcGSJUtYtWoVwcHBLFy4kBkzZgBQUlLCW2+9ha+vLzNnznTYTm5uLgCFhYWsWrWKixcv4ufnR0REBEuXLuWuu+5y5W57jO+u9Gjr7eOe7V/vCzofyG3+Mp0QoofoUOH5sReotFotqamppKamOq2bMGGCvWA0iIqKYuPGjc22FRYWxvHjx1vdXmJiIomJidecb3fzfc3Vow530GjU021yxCNEz9ahU22/+tWvuOeee1i7di2XL1/uqpxEF/muFiLcWHhA7WAgU2EL0bN1qPB8+umn/PSnP2X16tVMmTKFp556ioMHD3ZVbqITGS0KZ+vUIw53atj+V5XuzUMI4T4dKjyDBw/miSeeYNeuXSxfvpyamhoefvhhpk+fzpo1aygvd76BUHiG7690LHD3EU94gPqh2y+FR4ge65p6tWm1WuLj41mxYgXPPvssJSUlLF26lKlTp/KHP/yBioqKzs5T/EjfeUjhCdDCqD7wlZypFaLHuqZebcXFxWRlZbFlyxbMZjOJiYk88MADnDt3jpUrV7Jw4UL7mGrCM3x35d6ZiAA45ebpCWJ0kH0RrIqCj0bj3mSEEC7XocKzbds2Nm3axIEDBxgzZgxPPPEEd999N717qyfuIyMjGT58ONOmTeuSZMW1+75WHbGgj5u6Ujd2iw42nINvqmB8zx0oXIgeq0OF57nnnuOee+7h6aefJioqqtmY66+/nscee6xTkhOd57sauKmPu7NQ3XpluL4vLknhEaIn6lDh+ec//0lgYGCrMQEBATz++OM/KinR+QpNMC3E3VmoBvrD8N6w5xL8doi7sxFCuFqHOhdMmDCBsrIyp+UVFRXNjrcmPEOtVaHUA7pSN/aTvvDPS2DroePmCdGTdajwtDS4Zn19PT4+HnDxQDSrYcppd45a0NQd/aDcAker3Z2JEMLV2nWq7YMPPgBAo9Hw8ccfO0xNYLVa+eqrrxzmZhGepdBDCw/A7ksw1nmmCyFEN9auwvP73//e/u9XX33VYZ2vry+DBw/mmWee6dzMRKdpKDwRvT1nLpyI3hqGBSh8VgGLBrs7GyGEK7Wr8Bw9ehSAO++8k/fee4+QEA+5Si3apbAW/LUwyM/99/AAWGxQZFK4VQ9by+D7GgVfLeh9INhX7usRorvrUK+2zz//vKvyEF3opAnC/UGr0QDuP+SptsL+SxDUC6qs8LczMKKP2usu2E1zBQkhXKfNwrN161amTZuGn58fW7dubTX23nvv7bTEROcprPWsHm0NIvuABiioUQuPEKJnaLPwLF68mLi4OK677joWL17cYpxGo5HC46EKTTBR7+4snAX6wNAAOFYD97g7GSGEy7RZeAoKCpr9t/AOly0KFRbP6tHW2Og+sLNcPf0mhOgZrml06mtls9lYtmwZcXFxxMTEMHfuXEpKSlqMz8/PJykpiejoaKZOncratWvt6+rq6nj++edJSEggJiaGqVOn8sc//hGTyeTQxr59+5g5cybR0dFMmzaN7du3d9n+eZqKeoV9V0aBDuqlXtCv9bAv+LFBYAPy5X4eIXqMDhWef/3rX+Tk5Nj/3rRpE//5n//JM888Q1VVVZuvz8jIIDs7m3Xr1rF3715CQ0OZP38+NpvNKbaqqoqUlBQmT57MgQMHWL58OStXrmTHjh0AWCwWgoODSU9P59ChQ7z77rvs37+ftLQ0exunT5/mscceY86cORw8eJBnnnmGZ599lm+++aYju+21Kq2w9aL679Mm+KQc6pzfare6MQCCfOBI2x8fIUQ30aHCk5aWZp9rp7CwkBdffBGDwUBeXh6vvfZam6/PzMwkJSWFYcOGERgYyOLFiyksLHQoZg127tyJVqtlwYIF+Pv7M27cOBITE+3TLfTp04ff/va3DB8+HB8fH4YMGcL999/PgQMH7G1s2bKFkSNHkpiYiJ+fH/Hx8cTHx5OZmdmR3fZq5Rb1OcRDe4tpNWAIVEcw8JR7jIQQXatDhae4uJiRI0cC6jTYkyZNYsmSJbz00kvs3r271dcajUZKSkowGAz2ZXq9nvDwcI4dO+YUX1BQQFRUFFrt1RQNBkOr15m+/PJLIiMjHdpovL32tNHdVNRDLw3oPHhEo7FBUGODHKO7MxFCuEKHJ4LTXJm46+DBg9x+++0ADBgwgEuXLrX6uoZTcXq9Y/cqnU7X7Gm6qqoqdDrHMfP1en2Lp/QyMjI4fPgwmzdvdmhjxIgR7W6jNXl5eR1+TVuaO9LrTPUhgzh9WY8OP4qLzwBgiAihqMh5ivLmlrcWC1BUVNSu2Lba1SkafBjCB6eqGX7qRAf28Np19XvflSR39/Hm/D0p9w4VnlGjRrFhwwbuvPNO9u/fz9NPPw1AaWlpm6MZNIzvZjQ6/qw1Go0OY781jm86EnZlZWWzsatXr2bNmjW88847hIaGOrTRdHsttdEWg8GAv79/h1/XkpycHGJjYzutveYUmRRMZ+EGLYQPUcfSCwqC8HDnSXCaW95aLJQ7jM/3Y9uNOg17TEGsjxtv/3HTVVzx3ncVyd19vDl/V+duNptb/bHeoVNtqampvP/++zz00EPMmjXLfjSxa9cuxo4d2+prdTodYWFhDskYjUaKi4ubnVIhMjKS/Px8h44HR48edTiVBrBixQreeecd3n33XftpwMZtNN355trozsotEHJNE5y7VqwOztTBV5XuzkQI0dU6PB/Pvn37+Oqrr3jppZfsyx944AGef/75Nl+flJTE6tWrKSwspKamhrS0NCIiIpqtxAkJCVitVtLT06mrq+PIkSNkZWUxe/Zse8zSpUvZsmUL69evZ9iwYU5tzJo1i+PHj7N582bq6+v54osv2LVrF0lJSR3Zba9Vb4NKi+d2LGgsOgj8NLDpvLszEUJ0tQ7/Fvbx8XG6TjN06NB2vTYlJQWj0UhycjK1tbXExsaSnp6OVqvl0KFDzJs3j23bthEaGkpQUBAZGRksWbKEVatWERwczMKFC5kxYwYAJSUlvPXWW/j6+jJz5kyH7eTm5gIwZMgQ0tPTefXVV1myZAkDBw7klVdeITo6uqO77ZXO1akjswV7wRFPbx/4ST947wK8PkK5Mq6cEKI76tBXks1mY8uWLezbt4+ysjKn+28a3+DZHK1WS2pqKqmpqU7rJkyYYC8YDaKioti4cWOzbYWFhXH8+PE2c46Li2tzjLnuqqROffaGIx6AX1wP/+872FUBd8kA6EJ0Wx0qPEuXLmX9+vVMmjSJsLCwLr8ILH6c0itTIHjDNR6AhBD16OytUik8QnRnHfpKys7O5vXXX2f69OldlY/oRGeuHPF4y1QD/lpIHgAZpepwPzI3jxDdU4c6F1gsFqKioroqF9HJzpghUKt+oXuLRwaB2Qbrz7k7EyFEV+nQV9LMmTP55JNPuioX0clKzd5zfadBjE5DTBCsKXV3JkKIrtKhU206nY6MjAxyc3MZPXo0vr6O32rz58/v1OTEj1NS5z2n2Rp7ZBAs+g5yjQoxOjndJkR306HC88EHHxAYGEhBQYHTeGcajUYKj4c5Y4bxzoMJeLzkAfDUD7C6FFZ6Yf5CiNZ1qPB8/vnnXZWH6GSVFgWj1Xt6tDUW7KvhP65X2HAO0oYr9PaRox4hupNrvuxcUVGBosg49p7qVENXai881QaQEgqXLDKSgRDdUYcKj9Vq5Y033mDixIncfvvtnD59GoDXX3+9R81x4w2Kr0zE6g2jFjRnaj+I7AN/PePuTIQQna1Dhef//u//+OCDD/j973/v0LEgKiqKLVu2dHpy4to1FB5vOuKx2NQRtYtMCsVmeOAGddDQw0Y5shaiO+lQ4dmyZQsvvPACs2bNcpigbeTIkZw8ebKzcxM/wikz+AB9veiIp9qqTs/d8OjXCwK0kF7i7syEEJ2pQ4WntLSU4cOHOy338fHBZDJ1WlLixztlggF+4M3X5fv4wMzr4e/n4LLMiy1Et9GhwhMWFtbstNH79u1rdloC4T7FZgjtvHnr3OZXA9RpsdeedXcmQojO0qHCk5yczMsvv8zevXsBOHnyJOvXr+fPf/4zv/zlL7skQXFtik0Q6ufuLH68sUEwUQd/LUF6UQrRTXToCsCcOXO4dOkSjz/+OCaTiXnz5uHv78+jjz7K/fff31U5ig6yKQqnzTCtm4zwPD8M5hbAvy7D5H7uzkYI8WN1+NLzokWLuP/++7l48SKKojBixAj69OnTFbmJa3SuDuoVGNQNTrWB2rvtN9/BmrNSeIToDtpdeMrLy3n99df59NNPqaqqAtSx2xISEnjyyScJCekmP6+7gYau1KF+YHVvKp0i0EfD/f0Vss7DGzcpBHpzjwkhRPsKT21tLcnJyVRUVPCLX/yCESNGoCgK3333HdnZ2Rw+fJj333+fgICArs5XtEPDqAWh/lf/7e0eHgRvn4XNF+Chge7ORgjxY7Sr8Kxfvx6z2cxHH33EgAEDHNY9+uijJCUlsWHDBh555JFW27HZbCxfvpz33nuP2tpaxo8fz4svvkhYWFiz8fn5+bz44oscO3aM4OBgHnnkER566CH7+rVr17J161ZOnDjBdddd5zSW3OnTp7nrrrvo3bu3w2ype/bsQafrvqNPNhzxhHWjwjO5L4zoDW+XSuERwtu1q1fb559/zqOPPupUdAAGDBjAvHnz+Oyzz9psJyMjg+zsbNatW8fevXsJDQ1l/vz52Gw2p9iqqipSUlKYPHkyBw4cYPny5axcuZIdO3bYY2644QZSUlLaHBU7Ozub3Nxc+6M7Fx1Qu1IH+YDex92ZdB6NRsN/DYTdl+DftdK7TQhv1q7C8+9//5vY2NgW112Eb/oAAB9ySURBVE+YMIEffvihzXYyMzNJSUlh2LBhBAYGsnjxYgoLC8nJyXGK3blzJ1qtlgULFuDv78+4ceNITExkw4YN9pjp06czbdq0ZgtiT3bKBEP9QdPNLoU8NBA0wDtyT48QXq1dhaeqqop+/VruTtSvXz97h4OWGI1GSkpKMBgM9mV6vZ7w8HCOHTvmFF9QUEBUVJTD0DwGg6HZG1jbMnv2bG699VaSkpL49NNPO/x6b1NshqHd8HLbkAANPwtWbya1yT09Qnitdl3jsVqt+Pi0fN5Gq9Vitbbef6qhMOn1eoflOp2u2aJVVVXldEpMr9e3WeAaCw4OJjMzkzFjxmCz2fj000958sknWblyJXfccUe72wHIy8vrUHx7NHek1xn+XTWWob0uUVpqoehsncM6Q0QIRUXlTq9pbnlrsQBFRUWd3m7T5aVmPy6WX50H+476YHaabuRvB04wsVf7PwtNddV77wqSu/t4c/6elHu7Co+iKPzmN79xmuq6QX19fZttBAUFAeqRT2NGo9G+rml8WVmZw7LKyspmY1sSGBhITEyM/e97772X/fv389FHH3W48BgMBvz9O+/GmJycnFZPX16rWqtCxR4YP7g/gwZCeJOUg4IgPNz5Gldzy1uLhXLCw8M7vd2myweFQPiNofa/o6wKS/8FX+lHMn/0tZ1L7Kr33hUkd/fx5vxdnbvZbG71x3q7Cs99993XZszgwYNbXa/T6QgLCyMvL4+xY8cCatEpLi5m9OjRTvGRkZF8/PHH2Gw2++m2o0ePEhkZ2Z6UW6TVarv10CsNvdi646k2gN4+GhJvUMg8DytuUgjq1c0uZAnRA7Sr8Lz66qudsrGkpCRWr17NbbfdxoABA0hLSyMiIqLZSpyQkMDrr79Oeno68+bNo6CggKysLF544QV7jMViwWq1YrFYUBQFs1n91vXz80Oj0XDo0CH69etHREQENpuNf/zjH3z44Yf8+c9/7pT98UQNXamHdpNRC5rzXwNhdSlsuQhzpGu1EF7HpbO1pKSkYDQaSU5Opra2ltjYWNLT09FqtRw6dIh58+axbds2QkNDCQoKIiMjgyVLlrBq1SqCg4NZuHAhM2bMsLeXnp7OypUr7X/ffPPNAHz22WcMHjyYwsJCVq1axcWLF/Hz8yMiIoKlS5dy1113uXK3Xaq4mx/xANzeFyIC4G9nYEq/q0eveh8I9pUjICE8nUsLj1arJTU1ldTUVKd1EyZMIDc312FZVFQUGzdubLG9RYsWsWjRohbXJyYmkpiYeO0Je6Fik9rlOMwfSuvaDPdKGo2GWdcr/OW0OldPwyyr00Ig2ItmXBWip+rQtAjC8xWbYJAf+Gm79y//+/qDAhyodHcmQoiOksLTzZzqpvfwNDU0QB1C58tK6MZ9RYTolqTwdDPFpp5ReAAm6dUpIApl1nUhvIoUnm5EURR11IJu3KOtsVg9BGjhi0vuzkQI0RFSeLqR8/VgtvWcI54ALdyqhxwjVFrcnY0Qor2k8HQj9nt4ekjhAZjaDywK7Lvs7kyEEO0lhacb6Qk3jzY1yB8i+8CuS+rRnhDC80nh6UZ6ws2jzZkeApctsPG8uzMRQrSHFJ5upNgEgT4Q7NLbgt1vVB+1a/VfS8Bsk77VQng6KTzdyClzwwRw3fvm0aY0Grj7OnWkhhWn3Z2NEKItUni6kZ50D09TowPhzmB48SSUmOWoRwhPJoWnGyk2wZAe1LGgqf+JgHoFnvzO3ZkIIVojhaebqLUqnK/vuUc8oO77H8Ih6wJsPCdHPUJ4Kik83cTpHtqjrTGLDZIGwLggmH8cvrqsUGRSqKiXIiSEJ5HC0030xHt4mqq2wmcV6sjVtTaYWwA7yqC8HopMiv1RHzJIipEQbtTDOt52X931Hh6LTS0aTdVaW37NAD/4z/7w9/PqOG6T9LC/0XhuRWfrGDRI5u4Rwl2k8HQTJ03q4evgbnbEU211LBoNbtO1/rop/eBINbx/AX4t02ML4VHkVFs3UVirFp3uPgFce2k0kDxAnSzu9VPuzkYI0ZhLC4/NZmPZsmXExcURExPD3LlzKSkpaTE+Pz+fpKQkoqOjmTp1KmvXrnVYv3btWhITE4mOjubOO+9sto19+/Yxc+ZMoqOjmTZtGtu3b+/UffIUhSYY1tvdWXiW63zh59fBPyogv9rd2QghGri08GRkZJCdnc26devYu3cvoaGhzJ8/H5vNeXTHqqoqUlJSmDx5MgcOHGD58uWsXLmSHTt22GNuuOEGUlJSmD9/frPbO336NI899hhz5szh4MGDPPPMMzz77LN88803XbaP7vLvWrhRCo+TnwZDqB98eFFmKhXCU7i08GRmZpKSksKwYcMIDAxk8eLFFBYWkpOT4xS7c+dOtFotCxYswN/fn3HjxpGYmMiGDRvsMdOnT2fatGkMGDCg2e1t2bKFkSNHkpiYiJ+fH/Hx8cTHx5OZmdll++gOtVaF0jq4sZt1LOgMvlqYNwiKTJAnRz1CeASXFR6j0UhJSQkGg8G+TK/XEx4ezrFjx5ziCwoKiIqKQqu9mqLBYKCgoKDd2ywoKHDY3rW04Q2+qVKf9T6O3YZb6/nVk9xznXraLbtMjnqE8AQu69VWVaV+O+r1eoflOp3Ovq5pvE7n2HVJr9c3G9vaNkeMGPGj2miQl5fX4de0pbkjvWvxr4AIIITDxaWcO11nX353RAhFReUOsYZmlrW0vLVYgKKiok5vt6tiJyh1fGK6jn0nzzJYC6WlpVwsL3WK9Qad9blxB2/OHbw7f0/K3WWFJygoCFCPfBozGo32dU3jy8rKHJZVVlY2G9vaNptur6NtNDAYDPj7d15f5ZycHGJjYzulrexC9We8Ycgg9I3+iwYFQXi4Y/Fubtm1xEI54eHhnd5uV8X+/EbY8wMc8x/I4PoiBg0aRPiNoU6xnq4zPzeu5s25g3fn7+rczWZzqz/WXXaqTafTERYW5pCM0WikuLiY0aNHO8VHRkaSn5/v0PHg6NGjREZGtnubkZGRTjvf0Ta8wWkz+GlA5+PuTDyXvxbi+sJhIxgVeaOEcCeXdi5ISkpi9erVFBYWUlNTQ1paGhEREc1W4oSEBKxWK+np6dTV1XHkyBGysrKYPXu2PcZisWA2m7FYLCiKgtlsxmw2o1w5kT9r1iyOHz/O5s2bqa+v54svvmDXrl0kJSW5bJ9dodgE1/uq966Ilk3tp97X842140e8QojO49LCk5KSwowZM0hOTiYuLo6SkhLS09PRarUcOnSImJgYzpw5A6inyTIyMtizZw8TJkxg0aJFLFy4kBkzZtjbS09P5+abb+a5557jzJkz3Hzzzdx88832e4OGDBlCeno6b7/9NrGxsbz88su88sorREdHu3K3u1yxWS08onX9/dTZSo9Yg5CJSoVwH5cOmaPVaklNTSU1NdVp3YQJE8jNzXVYFhUVxcaNG1tsb9GiRSxatKjVbcbFxbF169ZrS9gLKIrCaRPcom87VsDtfWF1TS/2V8p9T0K4iwyZ4+XO1kG1TR0YU7QtOgj8sZF13t2ZCNFzSeHxcidq1OcbpPC0i58Wonyq+bgMLlvkfJsQ7iCFx8udqFWf5Yin/aJ9qjArkHnO3ZkI0TNJ4fFyx2vAXwPBMsFFuw3U1DGqN7x91t2ZCNEzSeHxct/VQERvkNkQ2k+jgcQb4KtKyK+W021CuJoUHi93olYGB70Ws/pDLw2s8c5Rc4TwalJ4vJjFpvBDrczDcy2u84V7r4N3z0K93NQjhEtJ4fFiJ01gUeSI51o9PAjO18P2srZjhRCdRwqPF2vo0SY3Ql6b6SEw0E86GQjhalJ4vNjxK/fwyBHPteml1TBnoDpPT4lZTrcJ4SpSeLxYfrV6rUK6Ul+7R0PBpsD/d9rdmQjRc0jh8WJHq2FsoIxKfS0sNnW2Vh8NTAuBv56BgmqFino58hGiq0nh8VKKopBXDYZAd2finaqt8Em5+jAEwiULvHQSKmW6cCG6nBQeL1VkgiorGGRqmR9teG/1OtmOcjBJ4RGiy0nh8VJ51eqzHPH8eBoN3NcfKizwjvRwE6LLSeHxUg2FZ4wUnk4xso9axN8sgQt1cp1HiK4khcdLHa2Gof7Qt5f0LOgs/9EfTDb47+PYp08XQnQ+KTxe6tsqOc3W2UL9YfFQ+PAi/O2Mu7MRovtyaeGx2WwsW7aMuLg4YmJimDt3LiUlJS3G5+fnk5SURHR0NFOnTmXt2rUO600mE88//zy33HIL48eP54knnuDSpUv29V999RWjRo0iJibG/pgyZUqX7Z+r1NsUCmpgjHQs6HSPDFK7Vz/+HWy5IEc9QnQFlxaejIwMsrOzWbduHXv37iU0NJT58+djs9mcYquqqkhJSWHy5MkcOHCA5cuXs3LlSnbs2GGPeeWVV8jLy2Pr1q3s2rWLmpoann76aae2cnNz7Y89e/Z06T66wrfVUKfAeCk8nU6rgawxMFEHs4/CW6VSfITobC4tPJmZmaSkpDBs2DACAwNZvHgxhYWF5OTkOMXu3LkTrVbLggUL8Pf3Z9y4cSQmJrJhwwZAPdr54IMP+M1vfsOAAQPo27cvTz/9NLt37+bMme59nuSQUX2eqHdvHt1VUC8N226GKf0gpQDm5CuUypA6QnQalw22YjQaKSkpwWAw2Jfp9XrCw8M5duwYEydOdIgvKCggKioKrfZqbTQYDGRlZQFw8uRJzGYzY8eOta8fPnw4vXv35tixY4SGhtqXT506lbq6Om666SYWLlzILbfc0uH88/LyOvyatjRXcNvj49qh9NX0o/zoESo0UB8yiKKzdU5xhogQiorK21x2LbEARUVFnd6uq2KrqozNxpaa/bhYrk7S878K3Og3iLfPDWTLeYWH+lbxoL4a/ysfyd42C5ZLF5za6GrX+rnxBN6cO3h3/p6Uu8sKT1VVFaAWm8Z0Op19XdN4nU7nsEyv19tjG56bxjRub9iwYXz44YeMGDECk8nEe++9x9y5c9m0aROjR4/uUP4GgwF/f/8OvaY1OTk5xMbGXtNrTx5UuDUIJoxTX19kUghvJrWgIAgP17W57FpioZzw8PBOb9cVsUVFRQQF6ZqNHRQC4Tde/dFyC/DLSwq/+U7DXy/pyarWM+t6mKCD6ddB+PChTm10pR/zuXE3b84dvDt/V+duNptb/bHussITpH5bYTQaHZYbjUb7uqbxZWWOE6VUVlbaYxu3FxIS0mx7/fv3p3///vb4X//61+zevZuPP/64w4XHU9RY1aFy7nbt912P0TCGW2M3+MJjYXCsGt67AKtL4R8VDYOzOsbqfSDYV7q4C9Eal13j0el0hIWFOVRBo9FIcXFxs0UgMjKS/Px8h44HR48eJTIyEoCIiAj8/f0d2vvhhx+ora21xzRHq9V69T0aX1eBVVEvfovO13gMt4ZH3ZWP4OhA+H04PDQQLlsgKR8ePqZ2v26IlbHehGibSzsXJCUlsXr1agoLC6mpqSEtLY2IiIhmDwETEhKwWq2kp6dTV1fHkSNHyMrKYvbs2QAEBAQwa9Ys3njjDc6fP8/ly5dJS0vjjjvuICwsDIB//vOfnDp1CpvNRm1tLe+++y4HDx4kISHBlbvdqQ5Wqs/SscA9tBqI6wtLboTZN8DuS/C/J6/OjSSEaJtLC09KSgozZswgOTmZuLg4SkpKSE9PR6vVcujQIWJiYuw90oKCgsjIyGDPnj1MmDCBRYsWsXDhQmbMmGFv73e/+x2jR4/m7rvvJj4+Hn9/f1577TX7+iNHjjBnzhzGjx9PfHw8O3fu5G9/+5tDhwRvs+8yDPaHUH85neNO/lp4eiikDgENsPwU7CgDLz6YFsJlXDqFmFarJTU1ldTUVKd1EyZMIDc312FZVFQUGzdubLG9gIAAXnrpJV566aVm1y9cuJCFCxf+uKQ9iKIo7L6k3uAoPMNNfeAPEfDuWfjgojrkzsYxCjoZykiIFsmQOV7kWA1cqIepwe7ORDTmr4W5g+A/+8POcpiU49xBQQhxlRQeL7L9Sie/m3qrX2wNj1q5oO12Gg38LATWRkFJHcTlwNdGKT5CNEcKjxfZdaULb351872uhPvdqoNNY9R/T8mFdWfVHwcypbYQV0nh8RKKovBVJYzqo/66Fp6p2gqFJvh/g6Gvj9rd+n9PSjdrIRqTwuMlDldBuUUtPMLzBfvCU0NhWG94q1TtfCCEUEnh8RIfXFD/Y42VOXi8Rm8fWDQYxgbB84XwQqHi1TcvC9FZpPB4iQ8vqjeNBrm0A7z4sfy08Ggo3N8fXjwJi74DmxQf0cNJ4fEC39eo47MlyP07XslHA68NV282fbMEfpkPdTYpPqLnksLjBT68qD4nyP07XkujgbQRGpYOh43nYeYRqLJI8RE9kxQeD6coCmvPqsPwDw5wdzbiWjWMev3ADerRzz8qIP5rKJNu1qIHksLj4b6sVKe6nhfadqzwXI1Hvdb3gv8OhW+rYMphOC2jHIgeRgqPh/vbGdD5qCMhi+5jnA7eGQ2nzXD7YTgsoxyIHkQKjwe7WKew6Tz8cgAEyaCT3c5tfWF3jDqVXFwOvFki3a1FzyCFx4O9XKQOh/P/Brs7E9EVLDYI8YWPxsLtfeHxEzDzW7UXoxDdmdwV4qEKaxXeLIFfD4LIQDna6Y6qrbD/kvrvxBugXy+1B+O/LsOrwxRSQsFHxkcS3ZAc8XggRVF46nv1/o8XItydjXAFrQamXQe/j1CHRXrsBNyWA9vL5PSb6H6k8HigN0tgy0V1euXBAfKLtycJ84d3I+EvN0GpGe45AlFfwZunFWqtUoBE9yCFx8NsL1NI/R7uvk690130PDU2dXK530fArweC0QqPfwcD/wX/UxvO9jIFkxQh4cVcWnhsNhvLli0jLi6OmJgY5s6dS0lJSYvx+fn5JCUlER0dzdSpU1m7dq3DepPJxPPPP88tt9zC+PHjeeKJJ7h06ZJDTHZ2NgkJCdx8883MnDmTL7/8skv27ceyKQpvlij84lt1UMm/jIBTZpnwrSfz0ag9356LgPVR6nWgf9b35Z4jELwX7sxVeLFQYUeZQqlZTskJ7+HSwpORkUF2djbr1q1j7969hIaGMn/+fGw255nMqqqqSElJYfLkyRw4cIDly5ezcuVKduzYYY955ZVXyMvLY+vWrezatYuamhqefvpp+/rDhw/zu9/9jmeffZZDhw4xZ84cHnvsMc6cOeOS/W0Po0Vh3VmF23LUXk139YPPx4GP1nGyN5nwrefSaCCuL2REatih+5bsm2FBGFy2wJKT8PMjELZPPSJK+Foh9XuFv5YofFau/mCxSkESHsalvdoyMzNJSUlh2LBhACxevJi4uDhycnKYOHGiQ+zOnTvRarUsWLAArVbLuHHjSExMZMOGDUyfPh2TycQHH3zAihUrGDBgAABPP/00P//5zzlz5gyhoaFs2rSJO++8k/j4eAASExPZtGkT77//Po8//ni7cm74FVlXV9fh/d17SR3c06qARVGfzQqU18PJ2qEYD5j4vgZswGB/+PtN8IvrQWPVYK1T8LM0yaUep2UdXd5ZsX2w4mcxuzWHa43tgxWl3uw1+QKYTfDvOtD0u55IXzORN8DCG9SecSdq4Pvaq4/3S6CuUa3x1UB/X7jeDwb6gd5HPZUX4AMBGvXfADZF/Sw2jF9qQ/3cNjxbW3pcidWg/pL10aj/bnjWatTHZdMgri8w4aNR1/lw5XHlb61G3bbSJAflSg5Ko4c9rpntNs3BIb5hH2nyd6P1Dfn2upJjL43aZpn5er74wWTP1em/WzP1vbmS3+5l7fy9oKD+d2j8XlkbvWdWBS6YBxJSYLLva8M+2N9/mlmmgZgguLVvx68zN3xftnQU7rLCYzQaKSkpwWAw2Jfp9XrCw8M5duyYU+EpKCggKioKrfbqQZnBYCArKwuAkydPYjabGTt2rH398OHD6d27N8eOHSM0NJSCggLuueceh3YNBgMFBQXtzru+vh6AEydOtH9nr+gHTG5pZe8rz7pGy87B0XNX/xyLI8sF52UdXd5ZsdP9gAuX2hXbVTlca+xYP7AUXvKafAGMF9RnX6Di8gWHdRFXHj8F9VsjqJlGG9RfebhDAFBd6qaNdwJ/4NJFd2dxbfyB6muYjbAC8k5d+2br6+sJCHAeZNJlhaeqqgpQi01jOp3Ovq5pvE6nc1im1+vtsQ3PTWMat1dVVeW0Pb1eT2FhYbvzDgwMZOTIkfj6+qKReyqEEKJNiqJQX19PYGDzM1e6rPAEBak/w4xGo8Nyo9FoX9c0vqyszGFZZWWlPbZxeyEhVyeqadxeUFCQ0/Yat9EeWq3WqbgJIYRoXXNHOg1c1rlAp9MRFhZGXl6efZnRaKS4uJjRo0c7xUdGRpKfn+/Q8eDo0aNERkYCEBERgb+/v0N7P/zwA7W1tfaYyMhIh/VN2xBCCOF6Lu3VlpSUxOrVqyksLKSmpoa0tDQiIiKIjY11ik1ISMBqtZKenk5dXR1HjhwhKyuL2bNnA2o1nTVrFm+88Qbnz5/n8uXLpKWlcccddxAWFgbAAw88wOeff84XX3xBfX09mzdv5sSJE9x3332u3G0hhBCNaBQXdv632Wz8+c9/5r333qO2tpbY2FiWLFnC4MGDOXToEPPmzWPbtm2EhqqTz+Tn57NkyRKOHTtGcHAwc+fO5aGHHrK3ZzKZePnll9mxYwdWq5Wf/OQnLFmyhH79+tljsrOzeeONNzh79izh4eH87ne/Y9KkSa7aZSGEEE24tPAIIYQQMmSOEEIIl5LCI4QQwqWk8AghhHApKTxCCCFcSmYgdTGbzcby5cvtPfvGjx/Piy++aO8C7inS0tLYvXs3paWl9OnThylTprB48WKCg4PtMaNGjcLf3x8fHx/7sszMTEaNGuWOlB2sWLGCN9980+Emtvj4eJYtWwaoPSZffPFFe4/JRx55xKHHpDvdfffdDgPZ2mw2TCYTK1eu5Gc/+5lHve/btm1j/fr1FBQUUF1dzfHjxx3Wt/U+m0wmXnnlFXbs2IHFYmHKlCm88MILDj1T3ZH7119/zZtvvkleXh4mk4nw8HAee+wxEhIS7DFtfcbcmT+0/f+nW7+LFOFSq1atUuLj45UffvhBqaqqUv7whz8o99xzj2K1Wt2dmoM//elPytGjR5W6ujrl4sWLysMPP6w8+uijDjEjR45U9u/f76YMW/fGG28ov/rVr5pdZzQalUmTJikrVqxQTCaTkpubq0ycOFH5+OOPXZxl+7zzzjvKLbfcophMJkVRPOt937Nnj7J161YlKytLGTlypMO69rzPzz33nHLfffcpZ8+eVS5duqTMmzdP+e///m+35757925ly5YtSllZmWK1WpUdO3YoBoNB+eabb+wxrX3GXKG1/BWl7c+JO7+LpPC4WHx8vLJ+/Xr735cvX1bGjBmjHDhwwI1Zte3zzz9XYmJiHJZ50hdgU619KWzevFm5/fbbHf4He+2115Q5c+a4Kr0OmT59uvLaa6/Z//bE933//v1OX35tvc+1tbXK2LFjld27d9vXf//998rIkSOVkpIS1ySuNJ97c+677z7lrbfesv/t7sLToKX82/qcuPO7SK7xuFBbI3R7si+//LLZoYZSU1O59dZbue+++9i0aZMbMmtZXl4et912G/Hx8aSmpnLqlDrMbksjn3dk1HJX+fLLLzl58iRJSUkOyz35fW/Q1vvc1gjznuTcuXP8+9//dvp/oKXPmKdo6XPi7u8iucbjQh0dodtTbN++naysLNatW+ew/O233yYmJgatVsv+/ft56qmnsFgsJCcnuynTq6ZNm8Z//Md/EBoayvnz5/nTn/7Eww8/zIcfftjmyOee5O9//zs/+clPGDLk6jzonvy+N9YZI8x7gurqahYtWkR8fLzDqCetfcZaGpXZlVr7nLj7u0iOeFyooyN0e4Jt27bxP//zP6SnpzNmzBiHdZMmTSIgIAA/Pz+mTJnCr3/9az766CM3Zepo5MiRhIWFodFoGDBgAC+//DIXLlwgNzeXoKAgp/+5OjpquSucO3eOzz77zKmgePL73lhb77M3/P9gNBpJSUmhf//+LF261GFda58xT9Da58Td770UHhfq6Ajd7paVlcWSJUv461//ym233dZmvFarbXHGQXfTaDRoNBoURWlz5HNPsWnTJgYOHMiUKVNajfPU970zRph3p4qKCv7rv/6LQYMG8Ze//AU/P79W4xt/xjxR48+Ju7+LpPC4WEdG6HantWvX8vrrr7N69epmczt69CjffvstdXV1WCwW/vWvf7FmzRruvvtuN2TrbPv27ZSXlwNQVlbGc889R0hICDExMW2OfO4JLBYLmzZt4sEHH3S4RuJp77vVasVsNttn6jWbzZjNZmw2W6eMMO+u3C9cuMCcOXMYNWoUr7/+Or16OV+VaO0z5gqt5d+ez4k7v4tkkFAXa22Ebk8yatQoevXq5fQrr2H08M8//5y0tDTOnj2Lj48PoaGhzJ4922O+vOfPn8/XX39NbW0ter2eiRMn8pvf/Ibw8HCg7ZHP3e2TTz7hqaee4osvvnCY6NDT3vf333+fZ5991mn52rVrufXWWztlhHl35H7w4EFWrFhB7969HWYevvfee3nxxReBtj9j7sy/urq6zc+JO7+LpPAIIYRwKTnVJoQQwqWk8AghhHApKTxCCCFcSgqPEEIIl5LCI4QQwqWk8AghhHApKTxCCCFcSgqPEEIIl5LCI4QQwqX+f3F3veTkMK2xAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r_vc27VRPzWW"
      },
      "source": [
        "We can choose the maximum sequence length of 88, as length of all the sequences in our dataset are below 88\n",
        "\n",
        "(the larger the sequences, the slower the model is going to train)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovWZXRsbQA8V"
      },
      "source": [
        "MAX_LEN = 88\n",
        "BATCH_SIZE = 8\n",
        "EPOCHS = 3"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IR4AvHj1nl7H"
      },
      "source": [
        "### Building BERT Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XKRe1K4ERats"
      },
      "source": [
        "class DataSet(Dataset):\n",
        "\n",
        "  def __init__(self, text, label, tokenizer, max_len):\n",
        "    self.text = text\n",
        "    self.label = label\n",
        "    self.tokenizer = tokenizer\n",
        "    self.max_len = max_len\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.text)\n",
        "\n",
        "  def __getitem__(self, item):\n",
        "    text = str(self.text[item])\n",
        "    label = self.label[item]\n",
        "\n",
        "    encoding = self.tokenizer.encode_plus(\n",
        "      text,\n",
        "      add_special_tokens=True,\n",
        "      max_length=self.max_len,\n",
        "      return_token_type_ids=False,\n",
        "      pad_to_max_length=True,\n",
        "      return_attention_mask=True,\n",
        "      return_tensors='pt',\n",
        "    )\n",
        "\n",
        "    return {\n",
        "      'text': text,\n",
        "      'input_ids': encoding['input_ids'].flatten(),\n",
        "      'attention_mask': encoding['attention_mask'].flatten(),\n",
        "      'label': torch.tensor(label, dtype=torch.long)\n",
        "    }"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "acUaKusenLAN"
      },
      "source": [
        "### Splitting the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQVlp6fvm25m"
      },
      "source": [
        "df_train, df_test = train_test_split(\n",
        "  df,\n",
        "  test_size=0.2,\n",
        "  random_state=42\n",
        ")\n",
        "df_val, df_test = train_test_split(\n",
        "  df_test,\n",
        "  test_size=0.5,\n",
        "  random_state=42\n",
        ")"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b5oRdaGHnAzS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5b4d31db-576e-4b2f-d0b7-77fac3a14834"
      },
      "source": [
        "df_train.shape, df_val.shape, df_test.shape"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((3876, 2), (485, 2), (485, 2))"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZZJMNJfCnETE"
      },
      "source": [
        "def create_data_loader(df, tokenizer, max_len, batch_size):\n",
        "\n",
        "  ds = DataSet(\n",
        "    text = df.text.to_numpy(),\n",
        "    label = df.label.to_numpy(),\n",
        "    tokenizer = tokenizer,\n",
        "    max_len = max_len\n",
        "  )\n",
        "  return DataLoader(\n",
        "    ds,\n",
        "    batch_size = batch_size,\n",
        "    shuffle = True\n",
        "  )"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qDQln2Z0nWRS"
      },
      "source": [
        "train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN, BATCH_SIZE)\n",
        "val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)\n",
        "test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ne_TN3W5n0u5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2b75effa-414a-4ffa-d6d9-41ee5fce3be0"
      },
      "source": [
        "##example batch from training data loader\n",
        "\n",
        "data = next(iter(train_data_loader))\n",
        "data.keys()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['text', 'input_ids', 'attention_mask', 'label'])"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iMEjeYU6q8a2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d02ffcd5-2221-4bd0-a9a1-f25c3eebe1e8"
      },
      "source": [
        "print(data['input_ids'].shape)\n",
        "print(data['attention_mask'].shape)\n",
        "print(data['label'].shape)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([8, 88])\n",
            "torch.Size([8, 88])\n",
            "torch.Size([8])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_mf6mm4SnVrw"
      },
      "source": [
        "## Model Building"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hnHLqVEunZ8Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ecc761e6-f970-4f3e-de07-b26af0756590"
      },
      "source": [
        "bert_model = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jT2qkMnCn0e3"
      },
      "source": [
        "### Sentiment Classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jn4Es4gVnqpr"
      },
      "source": [
        "class SentimentClassifier(nn.Module):\n",
        "\n",
        "  def __init__(self, n_classes):\n",
        "    super(SentimentClassifier, self).__init__()\n",
        "    self.bert = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)\n",
        "    self.drop = nn.Dropout(p=0.3)\n",
        "    self.out = nn.Linear(self.bert.config.hidden_size, n_classes)\n",
        "\n",
        "  def forward(self, input_ids, attention_mask):\n",
        "    _, pooled_output = self.bert(\n",
        "      input_ids=input_ids,\n",
        "      attention_mask=attention_mask,\n",
        "      return_dict=False\n",
        "    )\n",
        "    output = self.drop(pooled_output)\n",
        "    return self.out(output)"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LL_6FF_jrV6B",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6d76a2b2-d91e-4d1f-d0c5-ecf9078c6590"
      },
      "source": [
        "model = SentimentClassifier(len(class_names))\n",
        "model = model.to(device)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "633vT0KTn5t9"
      },
      "source": [
        "### Optimizer and Loss Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DtxSfspeNuPB"
      },
      "source": [
        "optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)\n",
        "\n",
        "total_steps = len(train_data_loader) * EPOCHS\n",
        "\n",
        "scheduler = get_linear_schedule_with_warmup(\n",
        "  optimizer,\n",
        "  num_warmup_steps=0,\n",
        "  num_training_steps=total_steps\n",
        ")\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss().to(device)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x9IpZoskoBLO"
      },
      "source": [
        "### Model Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kddLew93OIJn"
      },
      "source": [
        "def train_epoch(\n",
        "  model,\n",
        "  data_loader,\n",
        "  loss_fn,\n",
        "  optimizer,\n",
        "  device,\n",
        "  scheduler,\n",
        "  n_examples\n",
        "):\n",
        "  model = model.train()\n",
        "\n",
        "  losses = []\n",
        "  correct_predictions = 0\n",
        "\n",
        "  for d in data_loader:\n",
        "    input_ids = d[\"input_ids\"].to(device)\n",
        "    attention_mask = d[\"attention_mask\"].to(device)\n",
        "    label = d[\"label\"].to(device)\n",
        "\n",
        "    outputs = model(\n",
        "      input_ids=input_ids,\n",
        "      attention_mask=attention_mask\n",
        "    )\n",
        "\n",
        "    _, preds = torch.max(outputs, dim=1)\n",
        "    loss = loss_fn(outputs, label)\n",
        "\n",
        "    correct_predictions += torch.sum(preds == label)\n",
        "    losses.append(loss.item())\n",
        "\n",
        "    loss.backward()\n",
        "    nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
        "    optimizer.step()\n",
        "    scheduler.step()\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "  return correct_predictions.double() / n_examples, np.mean(losses)"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JiHYXWiAoHDe"
      },
      "source": [
        "## Model Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y0p56TtRPqPb"
      },
      "source": [
        "def eval_model(model, data_loader, loss_fn, device, n_examples):\n",
        "  model = model.eval()\n",
        "\n",
        "  losses = []\n",
        "  correct_predictions = 0\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for d in data_loader:\n",
        "      input_ids = d[\"input_ids\"].to(device)\n",
        "      attention_mask = d[\"attention_mask\"].to(device)\n",
        "      label = d[\"label\"].to(device)\n",
        "\n",
        "      outputs = model(\n",
        "        input_ids=input_ids,\n",
        "        attention_mask=attention_mask\n",
        "      )\n",
        "\n",
        "      _, preds = torch.max(outputs, dim=1)\n",
        "\n",
        "      loss = loss_fn(outputs, label)\n",
        "      correct_predictions += torch.sum(preds == label)\n",
        "      losses.append(loss.item())\n",
        "\n",
        "  return correct_predictions.double() / n_examples, np.mean(losses)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yO4b3Ha8omM0"
      },
      "source": [
        "### Accuracy on training data and validation data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wvYo2WZ7QEFT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d07421d2-06a9-4188-ecd2-935136036c4c"
      },
      "source": [
        "%%time\n",
        "\n",
        "history = defaultdict(list)\n",
        "best_accuracy = 0\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "  print(f'Epoch {epoch + 1}/{EPOCHS}')\n",
        "  print('-' * 10)\n",
        "\n",
        "  train_acc, train_loss = train_epoch(\n",
        "    model,\n",
        "    train_data_loader,\n",
        "    loss_fn,\n",
        "    optimizer,\n",
        "    device,\n",
        "    scheduler,\n",
        "    len(df_train)\n",
        "  )\n",
        "\n",
        "  print(f'Train loss {train_loss} accuracy {train_acc}')\n",
        "\n",
        "  val_acc, val_loss = eval_model(\n",
        "    model,\n",
        "    val_data_loader,\n",
        "    loss_fn,\n",
        "    device,\n",
        "    len(df_val)\n",
        "  )\n",
        "\n",
        "  print(f'Val   loss {val_loss} accuracy {val_acc}')\n",
        "  print()\n",
        "\n",
        "  history['train_acc'].append(train_acc)\n",
        "  history['train_loss'].append(train_loss)\n",
        "  history['val_acc'].append(val_acc)\n",
        "  history['val_loss'].append(val_loss)\n",
        "\n",
        "  if val_acc > best_accuracy:\n",
        "    torch.save(model.state_dict(), 'best_model_state.bin')\n",
        "    best_accuracy = val_acc"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "----------\n",
            "Train loss 0.5506654015790249 accuracy 0.7889576883384932\n",
            "Val   loss 0.3622364265019776 accuracy 0.8494845360824742\n",
            "\n",
            "Epoch 2/3\n",
            "----------\n",
            "Train loss 0.2642201763745781 accuracy 0.9215686274509803\n",
            "Val   loss 0.5830367827162024 accuracy 0.8391752577319588\n",
            "\n",
            "Epoch 3/3\n",
            "----------\n",
            "Train loss 0.13416505673271364 accuracy 0.9625902992776058\n",
            "Val   loss 0.5966993015122096 accuracy 0.8556701030927836\n",
            "\n",
            "CPU times: user 4min 59s, sys: 16.2 s, total: 5min 15s\n",
            "Wall time: 5min 23s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-fhdfHGLQWGB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        },
        "outputId": "3b6b6a88-c428-4650-c80e-2db1dacfff2d"
      },
      "source": [
        "plt.plot(history['train_acc'], label='train accuracy')\n",
        "plt.plot(history['val_acc'], label='val accuracy')\n",
        "\n",
        "plt.title('Training history')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "\n",
        "plt.legend()\n",
        "plt.ylim([0, 1]);"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEhCAYAAACDefxEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhMZ/8G8Hsm+4pYKyHWxBJJJptIg4QWDUFpCF5UBWnpWxqxVbWqVRpr1doo2lBEEU14+aEtWiqJNRHVJUgGIZZ0ssgyc35/REZGtjMkk+3+XNdcNc/ZvvNc6bnnLHMeiSAIAoiIiESQVncBRERUezA0iIhINIYGERGJxtAgIiLRGBpERCQaQ4OIiERjaFC98/vvv8Pe3h537tzRajl7e3tERUVVUVVPpaamwt7eHnFxceXO16dPH6xbt67K6yEqTr+6CyAqi729fbnTra2tcfz4ca3XK5PJcOrUKTRu3Fir5U6dOgVLS0utt1dV9uzZA2NjY1HzxsXFYcyYMTh27BhsbGyquDKqyxgaVGOdOnVK/e/z58/j3Xffxb59+9C0aVMAgJ6ensb8eXl5MDQ0rHC9hoaG6nVo43mWqUpWVlbVsl2x/Ux1E09PUY3VtGlT9atBgwYACneURW09evTAt99+i5CQELi6umLWrFkAgJUrV+K1116Dk5MTevfujQULFkChUKjX++zpqaL3v/76K8aMGQMnJyf4+fnhl19+0ajn2dNT9vb22L59O0JDQyGTydCrVy9s3LhRY5mHDx/iv//9L5ydneHl5YVVq1Zh9uzZePPNNyv8/Hfv3sWUKVPg5OSEvn37Yu/evRrTnz09dfToUQwdOhROTk5wc3PDG2+8gStXriA1NRVjxowBAPTt2xf29vYYO3YsAEAQBGzevBl9+/aFg4MDXnnlFWzdurXEdlauXImPP/4Y3bt3x5gxYzBnzhy89dZbJWoeN24c5s2bV+Fno9qLoUG12tq1ayGTybBv3z5Mnz4dAGBkZIRFixYhJiYGS5YswdmzZ/Hpp59WuK6lS5diypQpiIqKgpOTE2bMmIGMjIwKt+/u7o6oqChMmTIFK1aswOnTp9XT586diz/++AMbNmzAtm3bkJaWhqNHj4r6bMuXL8eQIUNw4MABDBw4EPPnz0dycnKp8967dw/Tp0/HwIEDER0djV27dmH8+PHQ09PDSy+9pA6XyMhInDp1CmvWrAEA7NixA6tXr8bkyZMRHR2NiRMnYvny5YiMjNRY/3fffYfGjRtj586d+PzzzzFy5Ej89ttvSElJUc9z48YNnD17FiNHjhT1+aiWEohqgTNnzgh2dnbC7du31W12dnbC3LlzK1z2yJEjQteuXQWlUlnquoreHz58WL3MvXv3BDs7O+HEiRMa29u/f7/G+0WLFmlsa8CAAcKyZcsEQRCE5ORkwc7OTvjtt9/U0/Py8oRevXoJ48ePL7PelJQUwc7OTvjmm2/UbQUFBYKzs7Pw/fffq9t8fX2FtWvXCoIgCImJiYKdnZ2QkpJS6jpjY2NLnd6rVy9h6dKlGm2fffaZ0KdPH43tjBs3rsQ6Bw0aJKxYsUL9ftmyZYK/v3+Zn4vqBh5pUK3m6OhYou3IkSMYM2YMvL29IZPJMHPmTOTn5+PevXvlrqtz587qfzdp0gR6enq4f/9+uct06tRJ432zZs2Qnp4OAPjrr78AAE5OTurpBgYGcHBwKP9DlbJuPT09NG7cWL3uZ9nb28Pb2xv+/v6YOnUqtm3bhtu3b5e7/szMTNy5cwfu7u4a7R4eHpDL5cjJyVG3ldbPgYGB2Lt3L5RKJQoKCrBv3z6MGDFC1Gej2ouhQbWaiYmJxvuLFy/ivffeg5ubG9auXYu9e/di4cKFAID8/Pxy12VgYFCiTaVSabWMRCKB8MyDoyUSSbnreJF1F9HT00N4eDi2bduGbt264ciRI+jfvz9++umn59r2s57tZwAYMmQIMjMz8fPPP+Pnn3+GQqHA4MGDK2V7VHMxNKhOiY+PR6NGjTBjxgw4OTmhbdu2Wv8eo7J06NABAHDhwgV1W0FBARITE6tkexKJBI6OjggODsb27dvh7u6uvnhedLdT8RA0NzdHixYtEBsbq7Ges2fPwsbGptSgKM7c3Bx+fn6IjIzE7t27MWDAgBp1SzJVDYYG1Slt27bFgwcPEBkZiZSUFOzfvx87duyollratGkDX19fLFy4EGfPnsVff/2FBQsWIDMz87mPPspy7tw5rF27FhcvXsStW7dw+vRp/PHHH2jfvj0AoGXLlpBKpfjll19w//599d1kkydPRkREBHbv3o3r169j586d+P777zFlyhRR2x05ciROnDiBU6dO8dRUPcHfaVCd4uvri+DgYKxcuRLZ2dlwd3fHrFmzEBISUi31fP755/joo48wadIkmJqaIjAwEF5eXsjLy6vU7VhYWODChQvYsWMHMjIy0LRpU/j7++Odd94BUHiN5v3338emTZuwePFiuLm54bvvvsPo0aORk5ODDRs2YOHChWjRogVCQkIQEBAgaruOjo6ws7NDfn4+XF1dK/UzUc0kEco6SUpElU6pVOK1115Dnz59MGfOnOou54Xl5+ejT58+CAoKwvjx46u7HNIBHmkQVaHY2Fjcv38fXbp0QVZWFrZu3Qq5XI7XX3+9ukt7ISqVCg8fPsTOnTuRk5OD4cOHV3dJpCM6DY2YmBhs374dV69eRVZWFv74449y509JScHHH3+Mc+fOwcTEBAEBAZg+fXqlnw8mqipKpRLr16/HzZs3oa+vj44dO2Lbtm0VPlerprt16xb69u2Lpk2bYvHixTA3N6/ukkhHdHp66uTJk8jIyMDjx4/xwQcflBsaSqUSgwcPhouLC+bMmYO0tDQEBQVhzJgxmDhxoq5KJiKiYnR691TPnj0xaNAgtGrVqsJ54+LicOPGDYSGhsLMzAzt2rVDUFBQtd0JQ0RENfiaxtWrV2Fra6tx37eDgwNSU1ORmZkp6nBYpVIhKysLBgYGPKVFRCSSIAjIz8+HmZkZpFLNY4saGxqZmZmwsLDQaCsKELGhkZWVhWvXrlVJfUREdZ2dnV2J/XCNDQ1zc3NkZmZqtP3777/qaWIUPYbBzs7uuZ7/n5CQIPo5QcT+0hb7SzvsL+28SH/l5eXh2rVrpT5ap8aGRqdOnXDjxg0oFAp10iUmJsLGxkZ0aBSdkjI0NISRkdFz1fG8y9VX7C/tsL+0w/7Szov2V2mn9XV6IVypVCI3N1f94Ljc3Fzk5uaW+lA4Nzc3tG7dGmFhYcjOzkZycjLCw8MxatQoXZZMRETF6DQ0oqKi4OjoqL5l1tHREY6OjoiNjcWtW7cgk8kQFxcHoPCpnRs2bIBcLoeXlxdGjx6NQYMG8XZbIqJqpNPTU8OGDcOwYcPKnH7+/HmN961atcLmzZuruiwiIhKJT7klIiLRauyFcCIiKlSgEvCvEsgoKPZ69v2Ttn+f/Nsixwa7qqAWhgYRURUqUAll7uCfbSsRDE/my1JWvB1jKdBAH2igV/hfi4oXeS4MDSKiMuSrhHK/1Wvs8Mtozy5/xGAAgEnRDr/YTt/GCLB8pq3E60m7pT5gJNW8PTY+PhVA80rvE4YGEdVJeUU7/LJ2+sXa/y0jDHKec4ff6pkdfkODMnb8eoXzGUprz2OOGBpEVOPkFt/hP3mdy2+IS7eFEjv9f8sIgscidvimUs2deEN9wNa42A6/vG/6T9oMatEOvzIwNIioUj1Wvvg5/NxSd/jtgKtP35npae7MrQyAtiaAZTk7+OIvS736t8OvDAwNIgJQ+GTTx6oXO52TUQDkiRihx1xPc2fexABob/LkG34ZO335n0no0a2zeoevzx1+tWBoENUBgiAgR1XxN/vyTudkFAD5Inb4FnqaO/NmBkBHE+0u2uo9x1AF8Xo5aGvCoKhuDA2iaiYIArKf3eFreTonowAoELHDf/bUTQtDwF5f/Dl8i+fc4VPdwdAg0oEClYCr2cCFTOB8JnBRAVxVdMXjk4U/2qpohy9ByVM3LY2Azmbiz+Fb6AFS7vDpBTE0iCpZZoGAy1mF4XBBURgUl7OeXtw1lgLdzIBuello19yo3Dtzil7m3OFTDcHQIHoBd/MEnH8SDEWva9lA0YFDI31AZg5MtQaczQGZBWBvUngRNz7+OlztGldr/UTaYmgQiaASBPyTo3l66XwmcDvv6Ty2xoUBEdisMByczQt/5MXx6akuYWgQPSNXJeDKM6eXLmYCiifP/9GTAF1MgVcaAc5PwsHZHGhkwHCguo+hQfVaRoGgDoai15Wsp7eemukBTmbA2BZPTy91NQWM9RgQVD8xNKheEAQB8txip5cygfMKIPnx03maGxaeXhpg9fT0UgcTXoAmKo6hQXWOUhBwLVvz9NKFTCA9/+k8HU0ANwsgqOWTIwhzoIURw4GoIgwNqtWylQIuZ2oeQVzKfPp0UkMJ4GAGDG7yNBwczQELfQYE0fNgaFCtkZ4nlDi99Ec2UPRsuwZPbm+d3PLp6aXOpnwoHVFlYmhQjSMIAq4/Lnl6KTX36TytjApD4Y1mT48gbI15eytRVWNoULXKVwm4kl0YDkVHEBcyC5+lBABSAJ1Mgd4NAacn4eBkDjQxZDgQVQeGBunMvwUCLj05vXThyVFEYtbTR2mbSAsDofiP47qZASa8vZWoxmBoUJW4nSuUOL30V87T6U0MCo8a3mv19PRSR1M+QZWopmNo0AtRCQL+yim8KP2/xy1x52Lhxeq0Yo/XaGdceOQwrkVhODhbAC0Nef2BCAAgCAAKACHv6Qt5mu+F3FLaSpvvaVsjAz0ArpVeLkOjLKocGEmvA/kNAOgDEv0n/9Ur4720OqvVicdKAQlZms9fupgFZD15vIY+msEhD3jN6unjNZzMgQa8vZWqg6CqcMeqfZv2O29RbVWghbE9gA8qfb0MjbLcHQmHBj8CKWIXkKD0UNEHoKfl+/LCSez759nu0/f/KvVwLUcfV7P1cDVbH4nZ+riWo488QQ8Fgj6M9fTRyVQf776kh65m+uhmpo+cPxPQ3dUN9SVE6yVBAKAs3HlW1g5TZFsH83TglvHTbVe0LJRV1AkGgMTw6QuGgMRIs62oXWpeyrylzPdCbUaltBkg6dyFKjjOYGiUrfFq/HO3O9q1bY3C/0kKUHgIWVDK+wJAUL7g+yf/VT0uZzsi6qgklgDcnrxg9OTVqIyZHz95WQFILmosCtEXC6/qCs3KD+8KQlRQAch/jp1qJey8tV0OIoYI1JqkcOdXzs5ST5Jf+G+pGYBGz8xnVIU75eLvDYB6flqVoVEWg7Z4mD8AsKiKrK5CggplhVOBKh9/5yiRlFWAP7IL8GdWAf7MViJLVQB9FMBAUoC2xgWwM1Gio0kBOpoUoJ1xARrqq0SFV2rqddhYtyix3boQoi+uZIg6NVACyconO+KqqrW0b8Vl7Bilpi+wUy3t2642y+tV+En+iI+Hq10t+/+xDmJo1DUSKQBDZKkMnt7eWs7ocU6Nnt7e6mgOmL3A7a1pf8fDplEN+5+6nBAVH17ahpW4I9EHd9PQrHmrKjxNwW/FVPkYGnVARaPHWekXhkJpo8fVeU9CtHAnWrOk3IhHs8Y1LGSJKsDQqEU4ehwRVTeGRg3F0eOIqCZiaNQAHD2OiGoLnYaGSqXCqlWrsGfPHuTk5MDFxQWffPIJrK2tS53/wIED+PrrryGXy2Fubo7+/fsjNDQUhoY17/y0GIIg4FYeNK4/cPQ4IqpNdBoa4eHhiI6ORkREBJo3b44lS5YgODgYUVFRkEo172O/evUqZs+ejZUrV6J///64desWgoKCYGZmhunTp+uy7OdSNHpcUTBcfHIdgqPHEVFtptPQ2LlzJ4KCgtCuXTsAQGhoKLy8vBAfHw93d3eNeVNSUtCgQQMMGDAAAGBtbQ0fHx9cvXpVlyWLUnz0uKIXR48jorpIZ6GhUCggl8vh4OCgbrO0tIStrS2SkpJKhIa3tzdsbGwQExODAQMGQC6X4/jx45g8ebLW205ISHjuuuPj4zXeP1Lp4Q+VKa4pTfCH0hTXVCa4oTKGCoUBYIEC2OnlYKh+NuylObDTy0FbaQ70JQCynrzSgGvPXVHN9mx/UfnYX9phf2mnKvpLZ6GRmZkJoDAoirOwsFBPK87ExARvvPEGPvroI4SGhkKpVOL111/H0KFDtd62g4MDjIyMtFpGEAT8GJcAZRsHjdNLJUaPawD8x6L46HH6kEgsUfggjvolPj4erq783YFY7C/tsL+08yL9lZubW+aXbZ2Fhrm5OYDCI47iFAqFelpx+/btw4oVK7Bhwwa4uLggPT0dH374IWbPno1ly5ZVeb0TrwJbMx2ABI4eR0RURGehYWFhAWtrayQkJKBbt24ACgPj5s2b6Ny5c4n5ExIS0L17d7i5uQEAmjVrhhEjRmDmzJk6qfetl4CXHt3EkK6tOXocEdETOn1+dWBgIDZv3ozk5GRkZ2cjLCwMbdq0KfUQytXVFWfPnsX58+chCALu37+P3bt3a1wTqUreDSUYZpgOD0sJA4OI6Amd3j0VFBQEhUKB0aNHIycnB66urli/fj2kUini4uIwadIkxMTEoGXLlvDz88O9e/cwd+5cpKWlwcTEBB4eHvj44491WTIRERWj09CQSqUICQlBSEhIiWlubm44f/68Rtv48eMxfvx4XZVHREQV4PBqREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKIxNIiISDSGBhERicbQICIi0RgaREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKIxNIiISDSGBhERicbQICIi0RgaREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKIxNIiISDSGBhERicbQICIi0RgaREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKKJCo3+/ftj8+bNePDgQVXXQ0RENZio0PD398f27dvRu3dvvPfeezh9+nRV10VERDWQqNCYNm0ajh07hnXr1kEQBEyaNAmvvPIKNm3ahPT09KqukYiIagjR1zQkEgl69uyJL7/8EidOnMDw4cPx1VdfwcfHB++++y7i4+Orsk4iIqoBtL4Q/s8//+Drr7/Gtm3bYGpqijFjxkBPTw/jx4/H6tWry11WpVJhxYoV8PLygkwmw8SJEyGXy8uc//Hjx1iyZAl69eoFZ2dnvPrqq/jll1+0LZmIiCqJvpiZcnNzcejQIURGRuLcuXNwcXHBvHnzMGDAABgaGgIATp48ienTp+O9994rcz3h4eGIjo5GREQEmjdvjiVLliA4OBhRUVGQSjXzSxAETJ06FQCwfft2tGrVCnfu3EFBQcHzflYiInpBokLD29sbUqkU/v7+WLhwITp06FBiHicnJzRo0KDc9ezcuRNBQUFo164dACA0NBReXl6Ij4+Hu7u7xry//vorYmNj8fPPP8PKygoA0KJFC1EfioiIqoao0Jg3bx78/PxgZGRU5jyWlpY4fvx4mdMVCgXkcjkcHBw0lrG1tUVSUlKJ0Dhz5gxsbGywfv16HDx4EEZGRvD19cX7778PMzMzMWUTEVElExUar7zyCnJyckqExqNHj6Cvrw9zc/MK15GZmQmgMCiKs7CwUE8r7uHDh/j777/x8ssv4+jRo3j48CGmTZuGpUuX4pNPPhFTtlpCQoJW8xfHC/zaYX9ph/2lHfaXdqqiv0SFRkhICHr27ImxY8dqtMfExODkyZPYsGFDhesoChaFQqHRrlAoSg0dMzMz6OnpYebMmTAyMoKJiQkmTZqERYsWaR0aDg4O5R4llSU+Ph6urq5aL1dfsb+0w/7SDvtLOy/SX7m5uWV+2RZ199TFixfRvXv3Eu0eHh64cOGCqCIsLCxgbW2tUYhCocDNmzfRuXPnEvN36dIFQOGtvkWK/5uIiHRPVGjk5ORAT0+v5MJSKbKzs0VvLDAwEJs3b0ZycjKys7MRFhaGNm3alJqGr776Kho3boyVK1ciLy8PaWlpCA8PR//+/UVvj4iIKpeo0OjQoQP+7//+r0T74cOH1XdCiREUFITXXnsNo0ePhpeXF+RyOdavXw+pVIq4uDjIZDLcunULQOHpqW+++QYJCQno3r07AgIC4OLiglmzZoneHhERVS5R1zSCgoIwa9YspKen4+WXXwZQeEvszp07sWTJEtEbk0qlCAkJQUhISIlpbm5uOH/+vEZbx44d8d1334lePxERVS1RoeHn54ecnBx89dVXiIiIAFD4m4mPPvoIgwYNqtICiYio5hAVGgAwfPhwDB8+XP149KIf3BERUf0hOjSKMCyIiOov0aGxb98+REdHQy6XIz8/X2PasWPHKr0wIiKqeUTdPbVlyxYsWrQI7dq1g1wuR+/evWFra4uMjAwMGTKkqmskIqIaQtSRxq5du7Bw4UL4+/tjz549mDBhAlq1aoVVq1YhIyOjqmskIqIaQtSRxu3bt+Hi4gIAMDIyQlZWFgBg6NChiImJqbrqiIioRhEVGlZWVupnRrVo0QJJSUkAgLS0NI5vQURUj4g6PeXm5oaTJ0+iU6dO8PPzw+LFi3Hy5EmcPXsWPXv2rOoaiYiohhAVGh9++CHy8vIAAJMmTVI/9sPf3189uh4REdV9FYZGQUEBDh48CF9fXwCFT5oNCgpCUFBQlRdHREQ1S4XXNPT19fH555/z2gUREYm7EN61a1f8+eefVV0LERHVcKKuaUyZMgVLly6FQqFAt27dYGJiojG9efPmVVIcERHVLKJDAwBCQ0M1Rs8TBAESiUR9Cy4REdVtokLj22+/reo6iIioFhAVGh4eHlVdBxER1QKiQiM2Nrbc6e7u7pVSDBER1WyiQmPs2LGQSCQQBEHdVvzaBq9pEBHVD6JC45dfftF4n5+fj8TERKxbtw6zZs2qksKIiKjmERUapd1Sa2NjAxMTE6xduxYvv/xypRdGREQ1j6gf95XF1tYWV65cqaxaiIiohnvu0Hjw4AE2btwIa2vryqyHiIhqMFGnp7p27apx4RsAlEolTE1NsWLFiiopjIiIah5RobFo0SKN0JBIJGjcuDEcHR3RoEGDKiuOiIhqFlGhMWzYsKqug4iIagFR1zR++eUXnDhxokT7iRMnSm0nIqK6SVRorFixotTxNFQqFa9pEBHVI6JC48aNG+jYsWOJ9g4dOuDGjRuVXhQREdVMokLDyMgI6enpJdrv3r0LfX1Rl0WIiKgOEBUa3bt3x5o1a5Cbm6tue/z4Mb766it4enpWWXFERFSziDpMCA0NRWBgIPr27QsXFxcAwLlz5yAIAnbs2FGlBRIRUc0h6kijVatWiIqKwhtvvIHHjx/j8ePHCAgIwP79+2Fra1vVNRIRUQ0h+oJEkyZNMH369KqshYiIajhRRxp79+7FwYMHS7QfPHgQ+/fvF72xolt0vby8IJPJMHHiRMjl8gqXS0hIQNeuXTF27FjR2yIiosonKjS+/vprNGzYsER7o0aNsGnTJtEbCw8PR3R0NCIiInDq1Cm0bNkSwcHBUKlUZS6Tm5uLuXPncnRAIqIaQFRoyOVytG7dukR7q1atRB0pFNm5cyeCgoLQrl07mJmZITQ0FMnJyYiPjy9zmZUrV8LT0xOurq6it0NERFVD1DUNCwsLpKamwsbGRqM9JSUFpqamojakUCggl8vh4OCgbrO0tIStrS2SkpJKPZKIjY3FTz/9hP379yM8PFzUdkqTkJDw3MuWF2hUEvtLO+wv7bC/tFMV/SUqNHr16oUvvvgCGzZsQLNmzQAAaWlpCAsLQ+/evUVtKDMzE0BhUBRnYWGhnlZcVlYW5s2bh8WLF8PExETUNsri4OAAIyMjrZeLj4/nEY4W2F/aYX9ph/2lnRfpr9zc3DK/bIv+ncaYMWPw6quvon379gCAv/76Cy1btkRoaKioIszNzQEUHnEUp1Ao1NOKW7p0KXr37s1rGURENYio0LCyssL+/ftx4MAB9fCuY8aMgbOzMyIiIvDee+9VuA4LCwtYW1sjISEB3bp1A1AYGDdv3kTnzp1LzH/q1Cn8+++/+PHHHwEU/gK9oKAA3bt3x549e9CqVSvRH5KIiCqH6N9pGBkZISAgACqVCsePH8euXbuwYMECNGjQQFRoAEBgYCA2b94MT09PNG/eHGFhYWjTpk2ph1C7du2CUqlUv9+yZQsuXLiA1atXo2nTpmLLJiKiSiQ6NG7duoXIyEj88MMPuHfvHgYOHIhNmzZp9eypoKAgKBQKjB49Gjk5OXB1dcX69eshlUoRFxeHSZMmISYmBi1btiwRDObm5jA0NESLFi3EfzoiIqpU5YZG8aOK06dPo0ePHpg1axZmz56N4OBgdOjQQauNSaVShISEICQkpMQ0Nzc3nD9/vsxl3333Xa22RUREla/c0PDx8YGlpSWGDBmCzz77TH3n1OzZs3VSHBER1Szl/rjvwYMHaNeuHTp27IgmTZroqiYiIqqhyj3SOHbsGPbs2YNPPvkEeXl5GDx4MF5//XVIJBJd1UdERDVIuUcazZs3x9SpU3Hs2DF8+umnuH79OoYOHQqlUomYmBjcuXNHV3USEVENIOruKYlEAh8fH/j4+CAtLQ2RkZHYu3cvNm7cCCcnJ3z//fdVXScREdUAoh5YWFzz5s0xbdo0HDt2DOvWrUOjRo2qoi4iIqqBRP9O41nFjz6IiKh+0PpIg4iI6i+GBhERicbQICIi0RgaREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKIxNIiISDSGBhERicbQICIi0RgaREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKIxNIiISDSGBhERicbQICIi0RgaREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKLpV3cB1UWlUiE9PR2PHj2CUqksdR59fX0kJSXpuLLaqz70l7GxMWxsbGBgYFDdpRBVi3obGqmpqZBIJGjTpg0MDAwgkUhKzJOVlQUzM7NqqK52quv9JQgC7t+/j9TUVLRt27a6yyGqFjo9PaVSqbBixQp4eXlBJpNh4sSJkMvlpc574cIFTJ48GV5eXnBxccHrr7+OI0eOVFotWVlZsLa2hqGhYamBQfQsiUSCxrSijuAAABfpSURBVI0b4/Hjx9VdClG10WlohIeHIzo6GhERETh16hRatmyJ4OBgqFSqEvNmZGTAz88P0dHRiIuLQ3BwMEJCQnDp0qVKq0cq5SUd0g6/YFB9p9O95s6dOxEUFIR27drBzMwMoaGhSE5ORnx8fIl5e/fujaFDh8LKygpSqRT9+/dHx44dS52XiIh0Q2ehoVAoIJfL4eDgoG6ztLSEra2tqIunaWlp+Oeff9CpU6eqLLNeGDhwIA4cOFDdZRBRLaSzC+GZmZkACoOiOAsLC/W0smRlZeHdd9+Fr68vevToofW2ExISSrTp6+sjKyurwmXFzKMLkyZNgqurK4KDg194Xbt37wZQNZ+tpvRXVcrLy6u0I14eOWuH/aWdqugvnYWGubk5gMIjjuIUCoV6WmkUCgUmT56Mpk2bYunSpc+1bQcHBxgZGWm0JSUlVXinT026G0hPTw+Ghobl1pOXlwdDQ0MdVqWpuvorPz9fp7fAGhoawsnJ6YXXEx8fD1dX10qoqH5gf2nnRforNze31C/bgA5PT1lYWMDa2lqjEIVCgZs3b6Jz586lLvPw4UOMHz8eL730ElavXl2tO8TqtGDBAsTFxWHjxo2QyWR4+eWXAQBr1qzBf/7zHyxfvhze3t4YPXo0AGD+/Pnw8fGBTCZDv379EBERobG+Pn36YO/evQAKbz22t7dHVFQU/P39IZPJEBgYiL///rvMeq5du4Zx48ahe/fucHNzw9ixY0ucYoyPj8fYsWPRvXt3eHh44K233lJPe/jwIRYsWIA+ffpAJpNh0KBBiIuLAwDMmTMHc+bM0VjX2LFjsWbNGvV7e3t7bNu2DQEBAXBycsKpU6fw+++/Y+TIkfDw8ED37t0RHByMlJQUjfUcO3YMAQEBcHd3h6enJ+bOnQsAmDFjBj788EONeU+fPg2ZTFbhUTBRfaPT32kEBgZi8+bN8PT0RPPmzREWFoY2bdqUmob37t3DhAkT0K1bN3z22Wc6udPp2zsCttx++l6pNIKenlAl25rwEjCuhbg7cT755BMkJyfDw8MD7777rsa0c+fOwdvbG8ePH1f/SNHJyQnvv/8+GjZsiFOnTuGdd95B27Zt1WFTmgMHDmDLli2wsLBASEgIPvnkE2zbtq3M+d9++224uLhAqVRiyZIlmDp1qjqIrl27hjfffBMffPABvv76a0ilUsTGxgIovO36nXfeQYMGDbBjxw40b94cN27c0PqupF27dmHNmjVo164dcnNzkZiYiLlz56Jr167IysrCvHnzEBoaip07dwIATp48iffffx9hYWHw8fGBUqnExYsXAQCjRo3ClClTMGfOHPWR0q5du+Dv71/uUTBRfaTTu6eCgoLw2muvYfTo0fDy8oJcLsf69eshlUoRFxcHmUyGW7duASj8n/bPP//EoUOH4OrqCplMBplMhgULFuiy5BqvWbNmmDJlCgwNDWFiYgIACAgIUN911qtXL/Ts2RO//fZbueuZOnUqmjRpAiMjIwwfPrzcW5vt7OzQo0cPGBkZwdTUFCEhIZDL5epv9t9//z169uyJwMBAGBsbw9DQUB1YCQkJuHDhApYsWYIWLVqof2Bpa2ur1eeeMGEC2rdvD4lEAmNjY7i6usLZ2RkGBgZo2LAhpk2bhgsXLiAnJwcA8N1332HEiBHo16+fuq88PT0BAB4eHmjZsiWio6MBAA8ePMDRo0cRGBioVU1E9YFOjzSkUilCQkIQEhJSYpqbmxvOnz+vfj9t2jRMmzZNl+VhXAsJxrV4+j4rK7fGXNMoi7W1tca3dEEQsG7dOkRHR+Pu3buQSCR4/PgxGjZsWO56mjVrpv63qakpsrOzy5w3NTUVX3zxBS5evAiFQqE+Cnz48CEAQC6Xo2PHjqUuK5fL0ahRowrrqYiNjY3G+6SkJKxYsQJJSUnq2gVBwIMHD2BtbQ25XA4fH58y1xcYGIjdu3dj5MiR2LdvHzp16oQuXbq8UI1EdRF/3VZLlHX65tnTdtHR0di+fTtWrlyJ2NhYxMXFoWfPnhCEyjvNtmDBAhgYGGDfvn04d+4cjh07BgDqbVhbW+P69eulLmttbY2HDx8iIyOj1OlmZmYlAuvu3bsl5nv2c0+fPh0dOnTAwYMHce7cOfV1HDE1AcDQoUPx999/48qVK+rwIKKSGBq1RNOmTcvd6RVRKBTQ09NDo0aNIAgC/u///q/CU1PaUigUMDExgYWFBRQKBcLCwjSmjxo1CidOnMDu3buRm5uLvLw8dQ3dunWDs7Mz5s6di7S0NAiCgOvXr+PGjRsACu90O3PmDJKTk5Gfn4+tW7ciNTVVVE1mZmYwNzdHeno6vvzyS43p48aNw+7du3H06FHk5+cjJycHZ86cUU+3sLDAoEGDMH/+fKSnp2PgwIEv2k1EdRJDo5aYMGECrl27Bjc3N/Tq1avM+YYNGwZ3d3f4+fnh5ZdfxokTJ9C3b99KreWDDz7A5cuX4e7ujuHDh8PLy0tjup2dHbZs2YKoqCh4e3ujZ8+e2Lx5M4DCI6a1a9fCysoKI0aMgIuLC6ZNm4b09HQAgL+/PwYMGICRI0fCx8cHCoUCLi4uFdb02Wef4ccff4SLiwsmTJiAV199VWO6t7c3li1bhnXr1sHT0xO+vr4lfuAYGBiIxMREDB48GKampi/SRUR1lkSozPMWNUzRvcZl/U6jrFt9i9Sk32nUBrW9vx48eABvb2/s3bu33CcPiPnbEYO/O9AO+0s7lfE7jdL2nTzSIAKgVCqxceNGuLq68lE1ROWot+NpEBVJSkrCqFGj0KJFC3z11VfVXQ5RjcbQoHqvc+fOuHDhQnWXQVQr8PQUERGJxtAgIiLRGBpERCQaQ4OIiERjaBARkWgMDSIiEo2hUU88O5AREdHzYGgQEZFoDA2qcfLy8qq7BCIqA0OjFti+fTsGDBig0ZaZmQmZTIbTp08DAFatWoVXX30VMpkMvr6+WLVqFVQqlVbbGDhwIGQyGXr27ImFCxeqR70DgIKCAoSHh+O1115Tb2P79u3q6fHx8Zg8eXKpY4Lb29vj999/V78vGpe86JHne/fuRZ8+fbB161b4+PjA19dX1GfKycnB8uXL1fP069cPhw8fRkZGBpycnHDu3DmNzxgaGlpi/HEi0g4fI1Kc4ltA8Y36rbFSCWToVc22LN4CLMaJmtXf3x9Lly7VeGrloUOH0LhxY/WQpW3btsV3332H5s2b4/Lly5g0aRJatmyJESNGiNpG06ZNsW7dOrRu3Rr//PMP3n77bWzYsAEzZswAAKxevRpHjhzB8uXL0bVrVzx8+FC90y8aE3zmzJnYvHmzxpjgYt25cwfXr1/HwYMH1QNOVfSZPvjgA6SmpmLTpk1o27Ytbt++jYyMDDRo0AB+fn7YvXu3+rHqGRkZOHz4ML799lut6iIiTTzSqAUsLS3Rr18/7NmzR922Z88eDB8+XL2DHTJkiHrMbUdHR/j7+2s1+FK/fv1ga2sLiUSC9u3bY/To0erlBUFAREQEQkND4eDgAIlEAisrKzg6OgJ4Oib4G2+8UWJMcLGkUinmzp0LU1NT9Vjn5X2mBw8eICYmBgsXLkTbtm0BAC+99JL6CbWjRo3CoUOHoFAoAAD79+9HmzZt4OzsrFVdRKSJRxrFWYzT+Pb/uAaNDxEQEIDg4GDMnz8ft2/fxuXLlzVGp9uxYwd27dqFW7duQRAE5ObmarWD/N///odvvvkGN27cQEFBAQoKCtC4cWMAhWN/Z2dnq3fOzypvTHCxmjRpUuK5/eV9pqKjnLJqcnR0RPv27XHgwAGMGTMGkZGRGDVq1AvVSEQ80qg1PDw80LRpUxw8eBA//PADevbsiebNmwMAzp07h8WLF2P+/Pk4ffo04uLitBrj+s6dO5gxYwbeeustnDx5EvHx8erTUgDQqFEjmJqaIjk5udTlKxp/29TUVOP6iJgxvyv6TDY2NgBQ7nZHjRqF3bt349y5c5DL5RgyZEiZ8xKROAyNWkIikWD48OHYtWsXoqKiEBAQoJ5WNC64lZUV9PT0EBcXhx9//FH0urOysqBSqdCoUSMYGhri6tWrGhe5JRIJxo4di2XLluHKlSsQBAEPHjzApUuXADwdE3zv3r0lxgQHCsf9LpqWnp6OtWvXVlhTRZ/JysoKgwYNwscff6wOjjt37uDq1avqeQYOHAi5XI5PP/0Ufn5+MDc3F90nRFQ6hkYtMnToUFy5cgUSiQQ+Pj7q9qLrCaNGjYKHhwe+/fZb+Pv7i15v+/btMX36dMyYMQMuLi744osvSnwr/+9//4thw4ap5xk+fDgSEhIAPB0TPCYmpsSY4ADw0UcfIS0tDZ6enpgwYQIGDx5cYU1iPtOiRYvg4uKCiRMnQiaTYdy4cbh586Z6uqmpKYYMGYLExEStjryIqGwcI7wctX3Ma12rif21detW7N+/H/v376+0dXKM8OrB/tIOxwgn0tKDBw8QERGB8ePHV3cpRHUGQ4PqpC+++AJ9+vSBk5OTqNNhRCQOb7mlOmnWrFmYNWtWdZdBVOfwSIOIiESr16FRh+8BoCrCvxmq7+ptaBgYGGj84IxIjPz8fOjr86wu1V/1NjSaNWsGuVyO7OxsfnskUVQqFdLS0tCgQYPqLoWo2tTbr0yWlpYAgFu3biE/P7/UefLy8mBoaKjLsmq1+tBfZmZmaNKkSXWXQVRt6m1oAIXBURQepYmPj4eTk5MOK6rd2F9EdV+9PT1FRETa02loqFQqrFixAl5eXpDJZJg4cSLkcnmZ81+5cgWBgYFwcnKCj48PB9AhIqpmOg2N8PBwREdHIyIiAqdOnULLli0RHBxc6rCkmZmZCAoKgre3N86ePYtVq1bhq6++wv/+9z9dlkxERMXo9JrGzp07ERQUhHbt2gEoHLPZy8sL8fHxcHd315j3yJEjkEqleOeddyCVSuHs7IyAgADs2LGjxHjZZSm6KyovL++5a87NzX3uZesj9pd22F/aYX9p53n7q2ifWdqdpToLDYVCAblcDgcHB3WbpaUlbG1tkZSUVCI0rl69ii5dumgMzuPg4IDIyEjR2yy6K+ratWvPXXfR479JHPaXdthf2mF/aedF+ys/Px/GxsYabToLjczMTAAocbeShYWFetqz81tYWGi0WVpaljpvWczMzGBnZwcDAwP1WNpERFQ+QRCQn59f6lAHOguNolHTFAqFRrtCoSh1RDVzc3Pcv39fo+3ff//VavQ1qVRaIniIiKhizx5hFNHZhXALCwtYW1trHC4pFArcvHmz1AFtOnXqhCtXrmhcJE9MTESnTp10Ui8REZWk07unAgMDsXnzZiQnJyM7OxthYWFo06ZNqaNL9evXD0qlEuvXr0deXh4uXbqEyMhIjBo1SpclExFRMTod7lWlUmHlypXYs2cPcnJy4OrqioULF8LGxgZxcXGYNGkSYmJi0LJlSwCFv9NYuHAhkpKS0KhRI0ycOBHjxo3TVblERPSMOj1GOBERVS4+RoSIiERjaBARkWgMDSIiEo2hQUREotXb0OATd7WjbX/Z29vD0dERMplM/frjjz90WHH1iomJwejRo+Hi4gJ7e/sK509JScHEiRMhk8ng5eWFlStX1qsRJbXtrz59+qBbt24af18//fSTDiqtGcLCwjBw4EC4uLjA29sb8+bNw8OHD8tdptL+xoR6auPGjYKvr6/w999/C5mZmcL8+fOFQYMGCUqlssS8CoVC6NGjh7BmzRrh8ePHwvnz5wV3d3fh0KFD1VB59dCmvwRBEOzs7IQzZ87ouMqa48SJE8KPP/4oREZGCnZ2duXOW1BQIPj5+Qnz588XMjMzhb///lvw9fUVwsPDdVRt9dOmvwRBEHx9fYUffvhBB5XVTMuXLxcSExOFvLw8IT09XZgwYYIwZcqUMuevzL+xehsavr6+wvbt29XvMzIyhK5duwpnz54tMe8PP/wgvPzyyxo7yC+++EIYO3asTmqtCbTpL0FgaBQ5c+ZMhTvBM2fOCF27dhUyMjLUbdu3bxf69OlT1eXVOGL6SxAYGs86fvy4IJPJypxemX9j9fL0VEVP3H1WWU/cvXr1qk7qrW7a9leRkJAQdO/eHa+//jp2796ti1JrpatXr8LW1lbjYZ4ODg5ITU3V6gGd9U1YWBg8PDwwaNAgfP311+qnWtdHp0+fLvcRS5X5N1Yvxwivjifu1mba9hcAbN26FTKZDFKpFGfOnMHMmTNRUFCA0aNHV3m9tU1Zf19F07R5SGd9sWTJEnTp0gXGxsa4dOkSQkND8ejRI4SGhlZ3aTp38OBBREZGIiIiosx5KvNvrF4eaTzPE3ef3Tlq+8Td2kzb/gKAHj16wNjYGIaGhujVqxfefPNNHDhwoMprrY3K+vsqmkYleXh4wNzcHPr6+nBxccF///tfREVFVXdZOhcTE4OPPvoI69evR9euXcucrzL/xuplaPCJu9rRtr9KI5VK69XdQNro1KkTbty4oRHKiYmJsLGxYWiIVPzUcX0RGRmJhQsXYsOGDfD09Cx33sr8G6t/Pf0En7irHW36KzExEZcvX0ZeXh4KCgrw66+/YsuWLRg4cGA1VF49lEolcnNz1efZc3NzkZubq/HFo4ibmxtat26NsLAwZGdnIzk5GeHh4fXq70ub/rp+/TpiY2PV0y9duoQvv/yyXv19ffvtt1i2bBk2b95c6v+Dz6rUv7HnuVJfFyiVSmHZsmWCp6en4OTkJLz11ltCSkqKIAiCEBsbKzg7OwtyuVw9f2JiojBixAihW7duQq9evYRt27ZVV+nVQpv+OnbsmDBgwADB2dlZcHV1Ffz9/YUdO3ZUZ/k698MPPwh2dnYlXmfOnBHkcrng7OwsxMbGque/efOm8NZbbwlOTk6Cp6ensGLFCkGlUlXjJ9Atbfrr4sWLgr+/v+Ds7CzIZDJhwIABwvr164W8vLxq/hS6Y2dnJ3Tp0kVwdnbWeBX9P1iVf2N8yi0REYlWb09PERGR9hgaREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKIxNIhqkdTUVNjb2yMuLq66S6F6ql4+sJDoecyZMwf79u0r0W5qaorz589XQ0VEusfQINKCm5sbVq1apdFWH597RPUXQ4NICwYGBmjatGmp08aOHQsbGxs0btwYkZGRyM/Px8CBAzF//nwYGRkBAPLz87F69WpERUXh4cOHaN26Nd5++234+/ur15OVlYVVq1bhyJEjuH//Ppo1a4YRI0YgODhYPc/du3cxZcoUnDlzBk2aNMHUqVMxbNiwqv3wROA1DaJKdfjwYTx69Ag7duzAsmXLcPToUSxfvlw9fcWKFYiMjMS8efPw448/YvDgwQgNDcXp06cBAIIgIDg4GMePH8eHH36IQ4cOYenSpbCystLYzvLlyzFkyBAcOHBAHUzJyck6/axUT1XO47OI6r7Zs2cLnTt3LvGQuKKxmf/zn/8Ivr6+QkFBgXqZnTt3Cg4ODkJWVpaQnZ0tdO3aVYiIiNBY7zvvvKMeOvi3334T7OzshEuXLpVaQ0pKimBnZyd888036raCggLB2dlZ+P777yv7IxOVwNNTRFpwdHTE0qVLNdqMjY3V/+7WrRv09PTU711cXJCXl4ebN28CKDw95e7urrG8u7s7Nm3aBABISEhAgwYN0K1bt3LrKD6Wi56eHho3boz09PTn+1BEWmBoEGnB2NgYtra21V0GDAwMNN5LJBIOckU6wWsaRJXo8uXLUCqV6vfnz5+HoaEhWrduDVtbWxgaGiI2NlZjmdjYWHTs2BEA4ODggIyMDFy+fFmndROJxSMNIi3k5+fj3r17JdqbNGkCAHj06BEWLlyI8ePHIyUlBatXr8bIkSNhamoKoPAOqy+//BJWVlbo1KkTDh8+jGPHjmHLli0AAE9PT7i5uWHGjBmYM2cO7O3tcffuXfzzzz8ICAjQ3QclKgNDg0gLcXFx8Pb2LtFedPdT//79YWZmhtGjRyMvLw9+fn6YOXOmer4ZM2ZAKpVi8eLF6ltuw8LC0KNHDwCFp5k2btyIlStX4uOPP8ajR4/QrFkzBAYG6uYDElWAI/cRVZKxY8eidevW+Oyzz6q7FKIqw2saREQkGkODiIhE4+kpIiISjUcaREQkGkODiIhEY2gQEZFoDA0iIhKNoUFERKIxNIiISLT/B5c8EqIHmDrqAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lO0oRdmDou4G"
      },
      "source": [
        "### Accuracy on test data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KHLydXRXasNe",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6e495f9b-53b9-4df4-9ad3-2c0e8a27ba3f"
      },
      "source": [
        "test_acc, _ = eval_model(\n",
        "  model,\n",
        "  test_data_loader,\n",
        "  loss_fn,\n",
        "  device,\n",
        "  len(df_test)\n",
        ")\n",
        "\n",
        "test_acc.item()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8597938144329897"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A1Ag6Y1Hoz1b"
      },
      "source": [
        "### Get Predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dWR_JQJUa1-1"
      },
      "source": [
        "def get_predictions(model, data_loader):\n",
        "  model = model.eval()\n",
        "\n",
        "  texts = []\n",
        "  predictions = []\n",
        "  prediction_probs = []\n",
        "  real_values = []\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for d in data_loader:\n",
        "\n",
        "      texts = d[\"text\"]\n",
        "      input_ids = d[\"input_ids\"].to(device)\n",
        "      attention_mask = d[\"attention_mask\"].to(device)\n",
        "      label = d[\"label\"].to(device)\n",
        "\n",
        "      outputs = model(\n",
        "        input_ids=input_ids,\n",
        "        attention_mask=attention_mask\n",
        "      )\n",
        "\n",
        "      _, preds = torch.max(outputs, dim=1)\n",
        "\n",
        "      texts.extend(texts)\n",
        "      predictions.extend(preds)\n",
        "      prediction_probs.extend(outputs)\n",
        "      real_values.extend(label)\n",
        "\n",
        "  predictions = torch.stack(predictions).cpu()\n",
        "  prediction_probs = torch.stack(prediction_probs).cpu()\n",
        "  real_values = torch.stack(real_values).cpu()\n",
        "\n",
        "  return texts, predictions, prediction_probs, real_values"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4TM894yDbQ2O"
      },
      "source": [
        "y_texts, y_pred, y_pred_probs, y_test = get_predictions(\n",
        "  model,\n",
        "  test_data_loader\n",
        ")"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5mfQsPgCo4fL"
      },
      "source": [
        "### Classification Report"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tbhW5hXIbT3p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a8c4e734-6187-44d3-9d6e-3eb1cede2445"
      },
      "source": [
        "print(classification_report(y_test, y_pred, target_names=class_names))"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "    negative       0.86      0.88      0.87        56\n",
            "     neutral       0.90      0.87      0.89       285\n",
            "    positive       0.78      0.83      0.80       144\n",
            "\n",
            "    accuracy                           0.86       485\n",
            "   macro avg       0.85      0.86      0.85       485\n",
            "weighted avg       0.86      0.86      0.86       485\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hgMSA_gso7_7"
      },
      "source": [
        "### Confusion Matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yj9W4T8Rbe4H",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "outputId": "0962b89a-51f7-45e6-8997-4212ff7d72d5"
      },
      "source": [
        "def show_confusion_matrix(confusion_matrix):\n",
        "  hmap = sns.heatmap(confusion_matrix, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
        "\n",
        "  hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')\n",
        "  hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')\n",
        "  \n",
        "  plt.ylabel('True sentiment')\n",
        "  plt.xlabel('Predicted sentiment');\n",
        "\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "df_cm = pd.DataFrame(cm, index=class_names, columns=class_names)\n",
        "show_confusion_matrix(df_cm)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAAErCAYAAACGiGfxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd1xV9f/A8de97CEOBBQUZ2JuQcWRuU1c4RaStCxNxXJklpXl1gx3grlnGk5yZri+mqKo5QBMJRyguGXIvvf3Bz9vXXFck8vlXt7PHvcR93POPfd9jnDf9zOPQq1WqxFCCCGMhNLQAQghhBAvQxKXEEIIoyKJSwghhFGRxCWEEMKoSOISQghhVMwNHUBRdPZ6iqFDMHlVXOwMHYLJU6AwdAhFgo1FPhyjfqDO+6adXvDqb6hnUuMSQghhVKTGJYQQpk5hWnUUSVxCCGHqlGaGjiBfSeISQghTpzCt/khJXEIIYeqkqVAIIYRRkRqXEEIIoyI1LiGEEEZFBmcIIYQwKtJUKIQQwqhIU6EQQgijIjUuIYQQRkVqXEIIIYyKJC4hhBBGxUxGFQohhDAm0sclhBDCqEhToRBCCKMiNS4hhBBGRWpcQgghjIrUuIQQQhgVWatQCCGEUZGmQiGEEEZFmgqFEEIYFalxCSGEMCqSuIQQQhgVGZwhhBDCqEgflxBCCKMiTYVCCCGMitS4hBBCGBOFJC4hhBDGxNQSl2k1fIp8cfb0cXq3a0hgwNuaspycbLauX8nH/bvj16EJge/6smvrzwaM0vQcjziGV50adPVpb+hQTMr2sK349e5O86YN8faqQ7cuPqxeuRy1Wm3o0AqMQqnQ+WEMinSNKyEhgU6dOrFjxw5cXV0NHU6hcP/eHRbM+JY6Xo25GX9VU75hxSJ+27GZwaO+pGLlalyIOsOi2VMwtzCnXafuBozYNNy5c5vx4z6ncdNmXLtyxdDhmJRSjo58OHgoFStWwtLSklOnIpk6eQJKpZJ3AvobOrwCYWo1riKTuDZv3syCBQvYt2+fpszV1ZXTp08bMKrCRaVSMW/a13R4uxeZmZlaievAr9vp0qsf3m+0BsDFtRyXYs6zee0ySVyvSKVS8eXnn9Hbz5/MjAxJXPmsabPmWs/LlS/P/vDfiDxxXBKXkZKmQqGxcc0SFCjw7Tsgz7aszAwsLK20yiytrLideIPbiTcKKELTtDhkIQoUvDfwQ0OHYvLUajVnz57hjz9O0bCRt6HDKTAKhULnhzEosMQVEBDAtGnTGD16NJ6enrRo0YINGzZotv/5558EBATg7e1Nq1atmDNnDtnZ2ZrtZ8+epWfPntSvX5/u3buzYsUKPDw8NNsjIiLo06cPjRo1wtvbm48++ohr164BEBkZyTfffENCQgL169enfv36/Pbbb1y/fh0PDw+uX7/OgwcPqF27dp4a2NixY/nss880z7du3UrXrl3x8vLSNDOagnOnT/DrLxsZ/sXEp/7y1m/UjJ1b1nMl9iJqtZq/os+yb3cYAPfu3C7ocE3GiePH2PjzBiZPn2E0HxrGKDk5mSYN69Owfm0G9OuLn38//Pu9a+iwCo7iJR5GoEBrXJs3b6Z3795ERkYyduxYJkyYwLVr14iNjWXAgAH4+/tz5MgR1qxZw/79+1m8eDGQ+0v34Ycf0qpVKyIiIvj+++9Zv3691rHNzc354osvOHLkCHv27EGpVDJmzBgAGjRowIQJEzRNg6dPn6Zt27Zary9RogTt2rVj06ZNmrKUlBT27NlDr169NPHPmzePqVOncuLECSZMmMD48eOJjIzU52XTu6SH95k37WuGjfmWkqVKP3Wf94Z9SpVqrzNmsD992nsza+JY2vjkDt4wlg7dwub+/ft8+flnfDt5KqVLOxk6HJNmZ2fHhk1bWbdhI5+P+5rVq1awZVOoocMqMFLjegUdOnTA29sbpVJJx44dKVasGFFRUaxbt462bdvi4+ODubk5bm5uDB48mM2bNwOwf/9+LCwsGDJkCJaWllSuXJn+/bXbpr28vKhXrx4WFhaUKFGCwMBA/vjjD9LS0nSOr2fPnuzcuZNHjx4BsH37dlxcXGjYsCEAy5cvZ8iQIdSqVQulUkmDBg3o3LkzW7ZsyacrZBhX/77Mvbu3mfblCHq3a0Tvdo3YuHoxNxOu07tdI/4XvotiDsUZPX4Ga3f+TvC6X1i4djuOTi4AuJQtZ+AzME6XL/7F7Vu3+GTYRzSoW5MGdWvyY8hCrl27SoO6Ndm14xdDh2gylEol7u4VqOZRnV59/Bjw3gcsmDfH0GEVGKVSqfPDGBTo4AxnZ2et53Z2dqSmphIXF0dERAT79+/XbFOpVJrhqjdv3sTV1VXrorq5uWkdKzo6mlmzZhEdHa1JPGq1mnv37uXZ91maNGlCyZIl2bVrFz169GDjxo306NFDs/3KlStMnTqVGTNmaMpycnJo0KCBjlegcKrqUZNZSzZole0JC+Xksf8xbuo8TYICsLCw0Dw/vG8PNep4UrxEyQKN11TUrFWb0C1hWmU/r/+J/x08wPzgRbiUKWugyEyfSqUiIzPD0GEUGGOpSemqUIwqdHJywtfXl0mTJj11e5kyZUhISECtVmv+ARISErT2GTFiBK1btyYoKAgHBweioqLo1q2bJvnp8k1CoVDQvXt3Nm3aRO3atYmOjiY4OFizvXTp0owcOZIuXbr811MtlKxtbHCvVFWrzKFEKczNLTTll2LOcysxgcqvVefh/fv8ErqGuMt/MXnuUkOEbBJsbG2p+lo1rbJSpUphYWGRp1z8dwsXzMPTqwHlypUnOzuLkycjWbFsMV19i9BoWNPKW4VjVKGfnx+7du1iz549ZGZmkpOTw5UrVzh06BAArVq1IjMzk5CQEDIzM4mLi2PVqlVax0hOTsbOzg57e3vu3LnDvHnztLaXLl2ae/fu8fDhw+fG0qNHD/744w+CgoJo0aIFTk7/9D3079+fBQsWcPbsWVQqFZmZmZw5c4Zz587l05UovLKyMgldtZhRA/sw5YtAsrIymTJvGRWryAesKNxSU1OYMvEbevh2IuCdPmz8eT3DR4zm08++MHRoBUYffVwzZ86kU6dOeHp68sYbbzBu3Dju37+vtU9UVBR9+/albt26tGzZMs/ndnp6OuPHj6dRo0Z4enoyYsQIHjx48OLzURfQ9PGAgAAaNWrE8OHDNWWtW7cmMDCQ7t278+effzJ37lyioqLIzs7Gzc0NPz8/+vbtC8CZM2eYMGECsbGxVKpUCR8fH+bNm8fZs2eB3H6w6dOnk5iYSPny5RkwYADjxo0jPDyccuXKkZ2dzejRozl27Bg5OTnMmDEDDw8P2rRpo9nnsQ8//JBDhw4RHBxM69attc4jLCyMFStWcPXqVczMzKhWrRoff/yxph9MF2evp7zKpRQ6qOJiZ+gQTJ7C1L7GF1I2Fq9+DKf3Nrx4p/93e3kfnfabNWsWHTp04LXXXiMpKYkxY8ZgaWlJSEgIkDu4rX379vj7+/Phhx8SHR3NoEGDmDhxIh06dABg/PjxnDt3juDgYKytrRkzZgwKhYJFixY9970LLHHltzVr1rB69Wr27Nlj6FBemiQu/ZPEpX+SuApGfiQu54G6L892aXYHkpKS8pQ7ODjg4ODwzNft37+f0aNHc+rUKSB3FPasWbM4dOiQpqtm5syZnD17llWrVpGenk6jRo2YP38+LVq0AODy5ct07NiR/fv3P3c1o0LRx6WLI0eOUKVKFVxcXDh37hxLliyhZ8+ehg5LCCEKvZdpAly5ciULFizIUx4YGKjVYvako0ePUr16dc3zmJgYatSooTW+oFatWoSG5k5DiIuLIyMjg9q1a2u2V6lSBRsbG6Kjo00jcV2+fJmxY8eSnJyMo6MjXbp0YdCgQYYOSwghCr2XSVz9+/enW7duecqfV9vauXMnoaGhrFmzRlOWkpJCsWLF8hwjJSVFsx3Is0+xYsU0257FaBLXu+++y7vvFqGZ7kIIkU9eJnG9qEnwSTt27ODbb78lODiYmjVrasrt7e25e/eu1r5JSUnY29trtkPuwLpSpUpp9klOTtZse5ZCMapQCCGE/uhr5YzQ0FAmTJhASEgIjRs31tpWvXp1oqKiUKlUmrLz589rmhMrVqyIlZWV1qjsy5cvk5aWptXk+DSSuIQQwtTpYa3CVatW8f3337N06VK8vLzybG/fvj05OTkEBwdrpg6Fhobi5+cHgLW1Nb6+vsybN49bt27x8OFDZs6cSYsWLV64aITRjio0ZjKqUP9kVKH+yajCgpEfowrLDd2q877XF/rqtJ+Hhwfm5uZYWlpqlf/7/oZRUVFMmDCB6OhoSpYsycCBA7W6fNLT05kyZQq7d+8mJyeH5s2bM2HCBEqUKPHc95bEZQCSuPRPEpf+SeIqGPmRuMoP26bzvtd+ePvFOxmY0QzOEEII8R+Z2HcMSVxCCGHiZJFdIYQQRkUSlxBCCKMiiUsIIYRRMbW7lEviEkIIEyc1LiGEEEbF1BKXTitnvPvuu09d5j4lJUXWDxRCiEJOodD9YQx0qnEdP36crKysPOWZmZmcPHky34MSQgiRf0ytxvXcxJWQkKD5+ebNm2RkZGie5+TkcOjQIUqXLq2/6IQQQrwyZVEanNG6dWvNisFPu2mjUqlk1KhRegtOCCHEqzOxCtfzE9fatWtRq9X069ePhQsXUrx4cc02CwsL3NzccHR01HuQQggh/rsiVeN6vFR9eHg4ZcuW1boFsxBCCONQpGpcj7m5uaFSqYiLi+POnTs8uaB8w4YN9RKcEEKIV1ekBmc8dvbsWUaOHEl8fHyepKVQKIiOjtZLcEIIIV6dieUt3RLXN998Q6VKlZg1axbOzs4ml72FEMKUmVo3j06JKzY2lqCgICpVqqTveIQQQuQzU6tr6JSGK1euzP379/UdixBCCD14PK1Jl4cx0ClxffXVV8yePZuYmJg8fVxCCCEKtyK55FNAQAAqlYpu3bqhUCjytJeeO3dOL8EJIYR4dcZSk9KVTolr8uTJ+o5DCCGEnhSpCciPdevWTd9xCCGE0BMTq3Dpfj+u+/fv88svv3DlyhUCAwMpWbIkf/zxB05OTri5uekzRpNT2dnO0CGYPMdGww0dgsmL+S3I0CEUCRUcrV75GKbWVKjT4Iy//voLHx8fVq9ezfr160lJSQFg//79zJ07V68BCiGEeDWmNjhDp8Q1Y8YMOnfuzK+//oqlpaWmvHnz5kRGRuotOCGEEK+uSA6HP3v2LP369ctzUmXLluXOnTt6CUwIIUT+MLUal059XGq1muzs7DzlN27cwN7ePt+DEkIIkX9MbVShTjWuJk2asG7dOq2yzMxMgoODadasmV4CE0IIkT9MralQpxrXp59+ir+/P1FRUWRlZTFx4kQuXbpETk4OGzZs0HeMQgghXoGxJCRd6ZS43N3d2bZtG+vWraN48eKoVCp8fX3p16+f3AFZCCEKORPLW7rP43J0dGT4cJkbI4QQxqZI1rgAcnJy+Pvvv7l79y4qlUprW5MmTfI9MCGEEPnDxPKWbokrMjKSMWPGcPPmTbkDshBCGBlTG1Wo8x2Qa9euzcKFCyldurTJVTuFEMKUKU3sM1unxJWQkMDChQupUKGCvuMRQgiRz/SVt3bs2MHatWuJiYkhNTWVCxcuaG338PDAysoKMzMzTdn69evx8PAAQKVSMWfOHDZu3EhaWhqenp5MnDjxhevf6jSPq2HDhly8ePFlz0kIIUQhoK95XA4ODvj7+zNu3Lhn7rN48WJOnz6teTxOWgBLlixh+/btrFmzhsOHD+Pq6spHH32UZxzFk3SqcU2aNIkvvviC69ev4+Hhgbm59ssaNmyoy2GEEEIYgL66uJo3bw5ARETEf3r9+vXr+eCDD6hcuTIAY8aMoWnTppw8efK5eUWnxHX16lX++usvfv/99zzbZHCGEEIUbi8zOCMpKYmkpKQ85Q4ODjg4OLz0e48ePZqsrCxcXV3x8/Ojd+/eACQnJxMfH0+tWrW03qNChQpER0e/euIaP3489erVY/DgwTg5OcngDCGEMCIKdP/MXrlyJQsWLMhTHhgY+NJzeVesWEH9+vVRKpUcO3aMTz/9lOzsbPz9/TW3x3oyGRYrVkyz7Vl0Slw3btwgJCREBmcIIYQRepmmwv79+z/1rvf/pbb17zm+b775JgMGDCAsLAx/f3/NAu3Jyclar0lOTn7h4u06JS4vLy8uX74siUsIIYzQy7SS/dcmQV0olUrNXOBixYrh5ubGuXPnqF27NpCbtK5evcrrr7/+3OPolLi6devG9OnTiY+Pf+rgDE9Pz/9yDkIIIQqAvnp3cnJyyM7OJisrC4CMjAwALCwsiI6ORqVS4eHhgVKpJCIiguXLlzNs2DDN6/v27cvSpUtp3LgxLi4uzJw5k4oVK+Ll5fXc99V5dXiAKVOm5NkmgzOEEKJw09cE5G3btvHFF19ontepUweAVatWkZqaysyZM7l58yZmZma4uroyYsQI/Pz8NPt/8MEHJCcn4+/vT1paGl5eXgQHB6NUPn+mlkL95BpOTxEfH//c7S+aLCa0pWa+8JKLV1TaWxaE1reY34IMHUKRUMHR6pWP0XP5KZ333fhe4W9B06nGJYlJCCGMl6kNBH9m4jp16hR169bFzMyMU6een62lj0sIIQqvIrNWob+/P0eOHMHR0RF/f38UCkWeleFB+riEEKKwM6209ZzEFR4eTqlSpTQ/CyGEME6mtmjEMxPXv/u1FAoFZcuWferJJyQk6CcyIYQQ+cLEbsel2+rwbdq04d69e3nK79+/T5s2bfI9KCGEEPlHqVTo/DAGOo0qfNaI+fT0dCwtLfM1ICGEEPmryDQVApqFFhUKBUuXLsXW1lazTaVScerUKapUqaLfCIUQQrwSI6lI6ey5iSssLAzIrXHt2bNH6y6WFhYWlCtXjokTJ+o3QiGEEK+kSNW4fv31VwACAgJYsGABxYsXL5CghBBC5B/TSls69nGtXr1a33GIQuj+/fsEL5jLgf3hPHzwACdnZ97/YDDde/Y2dGiF3sh32/B2m3pUq+iCQgFRl24wfclu9v7+9DmP1SuX4fCaMVhZmFOs4Sda2wZ0a8Iwv1ZULleauw9SWLntGFN/3PXMvuei7Lddv7D55zXcTLhOZkYmLmVd8enagx59AzS1jujzZ1g0dyYX/4qmWDEH2nV8mwGDArValEyNmYm1FeqUuACOHTvG0aNHuXPnDiqVSmvbtGnT8j0wY/H5558DMH36dANHkr8ePUrlgwH9cHZ2YdqMIMq6unL79u08//bi6Vo0qsbKrUc5ef4Kj9KzeK9bEzbP/Yj2H8zl6J+xWvvaWFuwZsb7HDj+F281q6G17b1uTQn6rCfDp27gyKlL1KzqyoKv/LAwN+PbH34pyFMyCiVKluKdAYMpX6EiFhaWnP3zJPO/n4KZUkm3Pv24lXiTLz4ZzBst2zLi82+Iv3aVWVPHg1rNwKEjDB2+3hSppsLHfvzxR2bNmkXlypVxdnY2+oswf/58jh8/LjXJ51i5fCnp6WnM/SFEM3LU1a2cgaMyHr6BwVrPv5y7jXZNa/B2m7p5Etecz/vw+x+xHD/zd57E1a+LN2u2H2ftLxEAxMXfpXK50nwzrDPfLd3Do/RM/Z6IkWnQuJnW87Ju5fj90H7+PB1Jtz792L5lA7Z2dowaNwGlUknFylW5e+cWi3+Yhf97g7CxsX3GkY2bkX9k56FT4lq7di1ff/0177zzjr7jKTSys7MxMzMz+iT9X+37bS/16nsR9N009of/hr29Pc1btuKjocOxsbExdHhGR6FQ4GBvTWqadqLx79wIr5ruvNFvJj3b513z08rKgoyMLK2ytIws7Gys8KzpzuGTl/QatzFTq9VciD7H+TOn8R8wCIDzZ/7Aq1FTrdtmNPBuxoKgqVz+K4ZadU1z3VVTW6tQpwnIycnJvPnmm/qORSMgIIBp06YxevRoPD09adGiBRs2bNBs//PPPwkICMDb25tWrVoxZ84csrOzAbh+/ToeHh5cv35ds39ERAQeHh5A7kjJRYsWERkZSf369alfvz5RUVGafXbs2EG7du2oV68ejx49Yu3atXTq1In69evTvHlzJkyYQFpaWoFdC0O5fu0q4Xv3kJKSwpwFC/lk1Bj27t7F5G+/NnRoRumzge0pXsyGZZuOaMo8KrkwfVQ33v18OelPJKfHfj1yHv/OjWhSt7LmNcPfaQWAq5MMlnqa1JRkurbxplMLL0YMfpe3e/nTrXful+57d+9Q0tFRa//Hz+/dvVPgsRYUhUL3hzHQqcbVtm1bjh07Rvny5fUdj8bmzZtZsGABM2fOZPfu3Xz66ac0bdqUrKwsBgwYwNSpU2nXrh2JiYkMHToUKysrhgwZ8sLjdu3alStXruRpKoyIyG2K2bNnD6GhodjY2GBpaYmTkxMLFy7E3d2d2NhYhgwZQkhICCNHjtTbuRcGKpWK4iVK8O2kqVhYWEBNyMrK5LPRI/hs3FcUL17C0CEajUG9mvPZ+2/Rc+Qi4m89AMDSwpy13w1kwg/bibp845mvnb54D6VLFmP34o9RKhQ8SE7jh58O8M3QzqhUMjjjaWxs7QheGUpGejpRZ/9gWchcHEs74dOlu6FDMxhTaznSKXHVq1ePuXPncvHiRapXr577QfYvXbp0yffAOnTogLe3NwAdO3ZkwoQJREVFceLECdq2bYuPjw+Qu6bi4MGDmT17tk6J60U+/fRTSpT450O5ffv2mp+rVKmCv78/O3bsMPnEVdrJCVdXN61/68pVXwPgRkKCJC4djQhow1dDOtJz5CL2R1zQlJcp7UDNqq7M+bw3cz7PHaWpUCgwM1OSfGIuE4N3MHPZr2RmZfPxlPWMnP4zZRwdSLyXRJvG1QGIvW66NYRXoVQqcSvnDkDlqtVITk5ixaL5+HTpTinH0ty/e1dr/wf/v5xdKcfSBR5rQTErionr8STjVatW5dmmUCj0kricnZ21ntvZ2ZGamkpcXBwRERHs379fs02lUuXb0OBy5bQHIOzevZtly5Zx5coVsrOzyc7OxvGJpgZTVN+zAZHHI8jOzsbcPPfX5MrffwPg6io3FtXF10M6MfydVvgOD87TF5Vw+wFePadolXVuWYevP+qId9/p3LqXrLUtJ0elqa316dCAv6/f4XT0Nf2egIlQq1RkZub2LdasU4/fdm9HpVJp+rlOHDuMlbU1VapVN2SYemVio+F1S1wxMTH6jkNnTk5O+Pr6MmnSpKdut7OzA9Dqh7p165bWPs+rNv+70/bmzZuMHDmS2bNn07p1aywtLVmxYsVTE7ipeXfA++zds4tpkyfQ7933uHPnFrODvqNz17dxkInoLzTz0x4M7NGM/l+s4GJcIi6OxYDcgRVJKelkZ6vyNBF61sitJfy7vHL50jSpV4WIP/+mmJ0V/X2b0LO9Fz1GhMg8rqdYteQHatX1oqxrObKzszj7x0l+XrOc9p3eBqBztz6EbVzP7OkT6NE3gBvx11i5+Ad8e/qb7IhCKKKJqzDx8/Pj/fff54033qBVq1aYmZlx/fp1rly5wptvvknJkiUpV64cP//8M2PHjiUhIYFly5ZpHcPJyYn4+HgyMzOfu0hwamoqKpWKkiVLYmlpSUxMDGvXrtX3KRYK1TyqM2/hj8yfE4RfL18cS5embfsOfDR0uKFDMwqB/z+A4ufZg7TKV4cdY9A3a3Q+jlKhZEjfFsz9ojdqNZw8fwWfj+Zx5NTlfI3XVKSmpjLvu0ncuX0LSytLyriW4/0hn9DJtxcAzi5lmDYnhEXzvmfY+32xty9Gx7d7MmBQoIEj168i2ccFsGnTJlavXs3Vq1fZtm0b5cuXZ/Hixbi7u/PWW2/pM0YtderUYenSpcydO5dvvvmG7Oxs3Nzc8PPz0+wzY8YMJkyYQMOGDfHw8KBnz55aayp27NiR3bt388Ybb6BSqZ45n6tKlSqMGDGCkSNHkp6eTr169Xj77bfZvHmz3s+zMPBu3ATv9RsNHYZRsqn/8h+Ea36JYM3/z9d67NLVW7zxznf5FZbJG/LJZwz55LPn7vN6rbrM+bFozeE0tRqXQq1De8OGDRsICgqif//+LFq0iB07dlC+fHk2bdrE1q1bZSLvS0rNlCYefSvtLTVDfYv5LcjQIRQJFRytXvkYn+248OKd/t93nTxe+f30Tad5XGvWrGHChAkMGzZMaz2vmjVrcumSTIAUQojCzFyh0PlhDHRqKrxy5Qp16tTJU25ra0tKSkq+ByWEECL/GEk+0plONS4nJyeuXr2ap/z06dMFOilZCCHEy1MqFDo/jIFOicvX15fp06fz999/o1AoSE9P5+DBg3z//ff06NFD3zEKIYR4BUVyyachQ4YQHx9Px44dUavVdO3aFYDu3bvz/vvv6zVAIYQQr8bURhXqlLjMzc2ZPn06gYGBnDt3DpVKRa1atXB3d9d3fEIIIV5Rkb2RJOQuh/R4SaTU1FS9BCSEECJ/mVje0q2Pa9myZezcuVPz/LPPPqNBgwa0bt2ay5dlBr8QQhRmipf4zxjolLh++uknSpfOXTk5MjKSvXv3EhQURO3atZk5c6ZeAxRCCPFqlArdH8ZAp6bCxMRETRPhgQMHeOutt+jYsSOvvfYaAQEBeg1QCCHEqzGWhKQrnWpctra2JCfn3mbh+PHjmvtkWVtbk56err/ohBBCvDKFQqHzwxjoVONq1KgRM2bMwNPTk6ioKJo3bw5AbGwsZcqU0WuAQgghXo2ZTlUU46HT6Xz55ZdYWlqyd+9eJk6cqOnvOnToEE2bNtVrgEIIIV6NvlbO2LFjB/7+/nh6euLhkXdx3qioKPr27UvdunVp2bJlnnsZpqenM378eBo1aoSnpycjRozgwYMHL3xfnWpcLi4uhISE5Cn/+uuvdXm5EEIIA9JXH5eDgwP+/v6kp6fz5Zdfam1LSUnhgw8+wN/fn5UrVxIdHc2gQYNwdnamQ4cOAEydOpVz587xyy+/YG1tzZgxYxg7diyLFi16/vno53SEEEIUFvpa8ql58+Z07tz5qWvW/vrrryiVSoYOHYqVlRX16v5nwqkAACAASURBVNWjV69erFu3DsitbW3dupVPPvkEFxcXihcvztixYzlw4AAJCQnPfV+juwOyEEKIl6N8iflZSUlJJCUl5Sl3cHDAwcFB5+PExMRQo0YNlMp/6ke1atUiNDQUgLi4ODIyMqhdu7Zme5UqVbCxsSE6OhpXV9dnHlsSlxBCmLiXqUmtXLmSBQsW5CkPDAxk+HDdb9CakpJCsWLFtMocHBw0t8J6/P8n9ylWrNgLb5cliUsIIUyc+Ut0cvXr359u3brlKX+Z2haAvb09d+/e1SpLSkrC3t5esx0gOTmZUqVKafZJTk7WbHsWSVxCCGHiXqbG9bJNgs9SvXp1du3ahUql0jQXnj9/nurVqwNQsWJFrKysOHfuHG+++SYAly9fJi0tTbPPs+g8OOPYsWMEBgbSpUsXbt68CUBoaCgRERH/6aSEEEIUDH0Nh8/JySEjI4OsrCwAMjIyyMjIQKVS0b59e3JycggODiYzM5MzZ84QGhqKn58fkLuAha+vL/PmzePWrVs8fPiQmTNn0qJFC9zc3J5/ProEt3fvXj788EMcHByIi4vTCnLJkiUvdaJCCCEKlr5GFW7bto06deowcOBAAOrUqUOdOnU4ceIE9vb2LFmyhEOHDtGgQQOGDx/OsGHD8PHx0bx+3LhxvP7663Tq1IlWrVphZWXFd9999+LzUavV6hft1L17d/z8/OjVqxf169cnLCyM8uXLc/78eQYPHszhw4df7myLuNTMF15y8YpKe+veiSz+m5jfggwdQpFQwdHqlY+x4sRVnfcd0LDw32dRpz6u2NhYmjRpkqfcwcGBhw8f5ntQQggh8s/LNgEWdjo1FTo4OHD79u085RcuXMDFxSXfgxJCCJF/9NXHZSg6Ja727dsze/ZsrbH1ly5d4vvvv6djx456C04IIcSrU7zEwxjolLhGjRqFWq2mWbNmpKen07NnT7p06YKbmxuBgYH6jlEIIcQr0NfgDEPRqY/L1taW1atXExERwblz51CpVNSqVeup/V5CCCEKF2O5z5auXmoCsre3t+YmkkIIIYyDqa2mrlPietq6Vf8mzYVCCFF4GcugC13plLjCwsK0nmdnZ5OYmIilpSXOzs6SuF6Smb5ujiM0/j4w29AhmLxZh2MNHUKR8F2nvDdofFlFsqnw119/zVN29+5dxo4dS9++ffM9KCGEEPnH1JoK//P5ODo6MmLECGbOnJmf8QghhMhnCoVC54cxeKXV4c3Nzbl161Z+xSKEEEIPjCMd6U6nxHXq1Cmt52q1mlu3brFkyRJq1aqll8CEEELkDzMjqUnpSqfE5e/vj0Kh4Mn1eD09PZk8ebJeAhNCCJE/TCxv6Za4wsPDtZ4rlUpKlSqFldWrr1oshBBCvxQm1lj4wsEZWVlZBAUFkZOTg5ubG25ubpQtW1aSlhBCGAlTW/LphYnLwsKCQ4cOGc1oEyGEENqUKHR+GAOdhsO3aNGCQ4cO6TsWIYQQemBqNS6d+rjq1avH/PnziYmJoXbt2tjY2Ght79Kli16CE0II8epMbcknhfrJoYL/0qZNGzZu3PjcVeAVCgXR0dF6Cc5UpWcbOgLT9yA1y9AhmDxZ8qlg5MeST+Exd3Tet0310q/8fvr23BpXfHw8KpWKmJiYgopHCCFEPjO1UYWvtHKGEEKIws/EWgpfnLhu375Ndvbz27ZcXFzyLSAhhBD5q8jVuLp16/bMbWq1Wvq4hBCikDO1Oym9MHHNmzeP4sWLF0QsQggh9MDURhW+MHF5enri6OhYELEIIYTQA9NKWy9IXLJahhBCGL8iVeN6zhQvIYQQRsK00tYLEpfM3xJCCBNgYplL5nEJIYSJK1JNhUIIIYyfaaUtSVxCCGH6TCxzSeISQggTV+RWzhBCCGHcTKyLSxKXEEKYOhPLW7rdAVkIIYTxUigUOj90NX/+fF5//XXq16+veYwaNUqzPSoqir59+1K3bl1atmzJqlWr8u18pMYlhBAmTl9NhQ0aNGD16tV5ylNSUvjggw/w9/dn5cqVREdHM2jQIJydnenQocMrv6/UuIQQwsQpXuKRH3799VeUSiVDhw7FysqKevXq0atXL9atW5cvx5calxBCmLqXyEhJSUkkJSXlKXdwcMDBwUGr7Ny5czRu3BgbGxs8PT0ZMWIE5cuXJyYmhho1aqBU/lM3qlWrFqGhof/5FP5NEpd4qhXLlhD+217+/jsW1GqqVH2NQYOH0Kz5m4YOzWj9tHoZh/aHc/XK36BWU6lKVQLeH4x3kzc0+xw/eoTlixcSf+0KaWmPKO3kTJv2Hen/wRAsLCwMGH3hdOfyOS4f2MrDhL9Ju3+b6j7v4NGuj2Z70s2rXNizjofxf5N69ybujdpSv8/wPMe5cuxXLh/6hUd3b2Bp54B7o7Z4tO+LQmkajVIvMxx+5coVLFiwIE95YGAgw4f/c+3eeustunfvjqurK7du3SIoKIj33nuPbdu2kZKSQrFixbRe7+DgQEpKyn8/iX8x6cQVGRnJhx9+yOnTp19pn6LoeMQxfLv1oGat2ljbWLNlYyjDh33E0hWrqe/pZejwjNKpyON07NqN6jVqYW1lzfZtm/hi1DDmhiyndl1PAGzt7OjZ9x0qVXkNW1s7Ll6I5vtpE0hPSyNw1FgDn0Hhk5ORTjGX8pTzbMHZrUvybs/MwKaEE2VqenP54NanHiPu6B7ObllM3V5DcKxck6QbV/gz9AdUqhxqdAzQ9ykUiJfp4+rfv/9TbyD8ZG2rWrVqmp9dXFyYMmUKDRo04PTp09jb23P37l2t/ZOSkrC3t3+5wJ/BpBPX44v42Pz58zl+/LhWZ+KT+4hcCxdpfwiM/PQzjhz5H+G/7ZXE9R/NnBui9XzIx6M5fuwwh/aHaxJXrTr1qFWnnmafMmVd+fNUJKdPnSjQWI2FS40GuNRoAMD57SvzbC/p/hol3V8D4ErE3qce41rkPtwbtsa9YRsA7BzL8OjuTaJ3raFam16YW1nrKfqC8zKJ62lNgrq9R+6oRLVaTfXq1dm1axcqlUrTXHj+/HmqV6/+0sd9GtOoBwu9U6lUpKakYmNjY+hQTIZKpeJRairWz7mmV+JiiTh6mPpejQowsqIlJysLpbl2M6zSwpKczAweXLtkoKjyl+Il/tPVzp07uXfvHgB3797l66+/plSpUtSvX5/27duTk5NDcHAwmZmZnDlzhtDQUPz8/PLlfApF4goICGDy5MkMHTpUc9JhYWGa7b/99hu+vr54eXnRsWNHrQ6+pKQkRo4cibe3N56enrz11lvs3r0bgIiICDw8PAAICwtj0aJFREZGauYcREVFae1z6dIlatSoQWJiolZ8/fr1Y+7cuQDk5OSwbNkyfHx88PLyonv37hw9elSv16cwWPJjCMnJSfTs1efFOwudrFm+mJTkZLr49syzrWfnNrRtVp93e3elnldDhn7yqQEiLBpcXvfkWuR+7v4dhVqtJjnxGpcP5n7+pCfdfcGrjYNCoftDV2FhYXTs2JG6devi6+tLRkYGy5cvx97eHnt7e5YsWcKhQ4do0KABw4cPZ9iwYfj4+OTL+RSapsLQ0FDmzp3LvHnzOHz4MIGBgVSoUAG1Ws2IESOYM2cOrVq14tSpU3z00UcUL16c9u3bs3TpUlJTUwkPD8fOzo6EhATS0tLyHL9r165cuXIlT1NhRESE5ueqVatSu3ZtNm/ezJAhQwCIi4vj5MmTTJ8+HYCFCxeyb98+Fi5cSIUKFQgPD2fo0KFs27YNd3d3PV8lw9jw01qWLl7E3AXBuJQpY+hwTMKWjetZs2IxU4Pm4+yS95rOX7SS9PR0Lv4VzaIFsylZqhTvDwo0QKSmr1q7PmSmJHFk4VegVmFhY0fl5l2I2b0OhaJQfLd/ZfqYxhUSEvLc7TVq1GDDhg16eOdClLhatWpFy5YtAWjZsiVt27Zl06ZNALRp04a2bdsC0LBhQ3r37s2GDRto3749FhYWPHjwgNjYWGrVqoWbm9srxdGzZ08WLVrERx99hEKhYNOmTTRu3Jhy5coBsGJF7oibSpUqAdCuXTu8vLzYvn07Q4cOfaX3LoxWLl/Kwh/mM3dBMI2bNDV0OCZh/ZrlLP9xIVOD5tOgUZOn7lPWLff3rVKVqiiVZkz55nP8At7Hxsa2IEMtEszMLajbayi1uw8mI/k+VsVKcPvCHwDYljaRL2omtuZToUlcjxPDv59fuHABQNOU95i7uzsHDx4EYODAgeTk5PDVV19x8+ZNmjRpwqeffkr58uX/UxwdO3Zk6tSpRERE0LBhQ7Zs2cK4ceMAuHPnDikpKQwbNkxrfkJ2dvYrJ8zC6If5c1mzagU/BP9Ig4bSx5Ifli5aQOhPq5gxZyH1PBvq9Bq1WoVKpSI7Kwuki1FvlGZm2JQoDcD1UwexLeVCCbcqBo4qf8iNJPUkPj4+z/MyZcqgUCi4fv261rarV69StmxZAGxsbPjkk0/45JNPePDgARMnTuSLL75gzZo1ed5Dl3W47Ozs8PHxYdOmTaSlpZGVlaWp7Tk4OGBlZcWSJUvw9PT8r6dqFL6bNoWNoRuYPnMWFStW4s7t2wBYWVvnmZ8hdDN/1nTCtoQyftJ3lHevxN07dwCwsrbC3j73mm5YuwL3CpUo714RFAouRJ8jZP4smr3ZimIOxQ0XfCGVnZFG6p0bAKhysslIus/D+FjMLK2xd3JFlZ1FcuK1/983naxHyTyMj0VhZo5Dmdym/ZTbCdyLi6ZUhepkZ6RxJWIv8X8cxnvgVyY0j8u0FJrEtX//fg4ePMgbb7zB4cOH2bt3r6YvKiAggH379tGiRQtOnz5NaGgokydPBiA8PBx3d3cqVaqEjY0NVlZWWrWhf3NyciI+Pp7MzEwsLS2fGUuvXr0YMGAAt27domvXrpp9LS0t6du3L9999x1TpkyhcuXKZGRkcPbsWUqXLq1pPjQFa9fkLog58uNhWuVd3+7GpKnTDRGS0du4PvfL1FeffaJV3qHT23zxzRQgt/YeMn8WN28koFAqKFPGFd+efenlZxrzifLbg2uXOLLwS83zv4/s5O8jO3GsUos3hk0lPekeB4JGaLY/vH6JG2ePYVPSmfZf//+UD7Wa2P/t4MymEEBByfKv0XTIJEpXqVXAZ6NHJpa5FGq1Wm3oIAICAvDw8CAhIYGjR49SunRphg0bhq+vL5C77tUPP/zAtWvXcHZ25r333qNPn9zRbStXrmTt2rXcvn0bS0tL6taty1dffYW7uzsRERG8++67mibH5ORkPv74Y86fP49KpWL16tUkJSVp7fNYp06duHTpEmFhYVpNlTk5OaxZs4bQ0FBu3LiBlZUVNWrUYOzYsbz22ms6nW96dn5cNfE8D1KzDB2CyZt1ONbQIRQJ33XyePFOL3AxMe+AtWd5zaXwt0cXmsTVqFEjreVETJkkLv2TxKV/krgKRn4krku3dE9cVZ0Lf+IqNE2FQggh9MPEWgolcQkhhKl7mRtEGoNCkbiediMyIYQQ+cPE8lbhSFxCCCH0x8TyliQuIYQweSaWuSRxCSGEiXuZVd+NgSQuIYQwcdLHJYQQwqgoJXEJIYQwLqaVuSRxCSGEiZOmQiGEEEbFxPKWJC4hhDB1UuMSQghhVGTJJyGEEEbFtNKWJC4hhDB5JlbhksQlhBCmTlbOEEIIYVxMK29J4hJCCFNnYnlLEpcQQpg6pYl1ckniEkIIE2dieQuloQMQQgghXobUuIQQwsSZWo1LEpcQQpg4GQ4vhBDCqMj9uIQQQhgXSVxCCCGMiTQVCiGEMCoyOEMIIYRRMbG8JYlLCCFMnollLklcQghh4kxtySeFWq1WGzoIIYQQQley5JMQQgijIolLCCGEUZHEJYQQwqhI4hJCCGFUJHEJIYQwKpK4hBBCGBVJXEIIIYyKJC4hhBBGRRKXEEIIoyKJSwghhFGRxCVEAXi8spqssCbEq5PEJYSe7dy5k2XLlgGgMLHFToUwBElcQuiRSqXi0KFDHD58mPPnzwNS6xLiVUniKqLUajU5OTmGDsOkqVQqlEolffv2xcLCgu3btwNS68pvKpVKvgwUMZK4iiC1Wo1CocDMzIwbN25w8OBBLl26ZOiwTIparUapzP3zqlevHvXq1SM6OpoDBw5ototX9/jLgUKhICYmhujoaO7cuaPZLtfZNEniKoIef+OfPn06Xbp0YcOGDfj7+/Pjjz9y69YtQP7g/6vHtdgna1WdO3emWLFi7Nmzh5SUFKl15ROlUsnDhw8ZNmwYAwcOZObMmfTp04dDhw5pvqAJ02P27bfffmvoIIT+PflHvHPnTn7//Xd+/PFH+vbtS+XKlVm2bBkpKSk0bdpU/uD/A5VKhZmZGQARERFERkZia2uLra0tpUqVIikpiVOnTqFUKqlZs6aBozVOKpVK63czOzubuXPnolQqWb58Od27d+fGjRts2LCBcuXKUaFCBQNGK/RFalxFhEKhIC4ujpSUFAB27dpFgwYNcHV1JTIyknnz5mFra4uvr6+BIzU+j2unSqWSq1ev0q9fPz7//HNWrVrFJ598wvLlywHw8fHBzc2NQ4cOce3aNa3XCt08bn6NjY0F4OHDhxw/fpz3338fS0tL9uzZwy+//EKVKlWoV6+eIUMVeiSJq4g4c+YMo0eP5sKFC6SmppKVlYWNjQ3Dhw/n448/5u2332bTpk1UqlSJmJgYQ4drVBQKBWq1msTERKZOnUrNmjXZv38/mzdvxsfHh59++okLFy5QvHhx2rRpw6NHjwgLC9O8VjydSqV6avmKFSuYOnUqOTk5XLt2jYyMDK5fv06vXr34/vvvGT9+PLNnzyY7O1vT9C1MiyQuE7FgwQIePnz4zO1ubm7ExsZibW2NnZ0dTk5OTJ06lZIlS/L777/z/vvvAzBx4kROnjxJZmZmQYVulP5dU5o3bx6jR4+mRIkS+Pj48MUXXwCwevVqli1bRlZWFkFBQQC0bdsWNzc3jh49ytWrVw0Se2GWlpbGjz/+CORN6o/7D52cnDh79ixmZmbUq1cPtVrN4MGD6dKlC3v37uWtt97ixo0bTJ8+Xa6xiZLEZQKysrK4dOkSWVlZmm+p+/btIy4uTvMB6+joSP369dm/fz8Ao0ePxtnZGUtLS8LDwwkNDeXNN9/k5s2btG/fHktLS4OdT2HyZFPevwdfnDt3jpMnTxIZGUn37t2xsrKiQ4cO3L17l379+rF582aCg4MZMmQIJ06cYM+ePQC8++67TJs2DXd39wI/n8Lu/PnzREdHk5OTo/ld3rVrF4Cm/7Bq1aq4u7sTGRkJwKhRo7CyssLGxoaYmBjmz5+Pr68vxYsXp27duoY5EaFXCrU0shudx0OA1Wq11rBrgMzMTCwtLenevTvW1tb4+PgQEBBAeno6Y8aMoXbt2gwaNAiAyMhI9u/fz71790hMTCQgIIBWrVoBeQdzFGX379/n5MmTtG3bFsj9onDv3j3eeustSpQowYQJE2jRooXmmm3atIljx44xc+ZMAObPn8+ePXtwcnLS9HeJXI9/l5/l+vXrtG3blp49e9KtWze8vLy4cOECw4YNY968edSoUQOAtWvX8ueff5KWlkZ6ejqjR4+mevXqBXUaoqCphVFKT0/PU7Z69Wq1n5+fWq1Wq+/du6detWqVul69eurVq1er1Wq1es6cOZrt/5aRkaH1PDs7Ww8RG6f09HT1rFmz1J07d1bHxcWpx48fr540aZI6NTVVvWLFCnXNmjXVly9fVqvV/1y36dOnqz09PdVxcXHqSZMmqfv166c+e/asIU+jUHv48KHW88zMTPXHH3+sXrRokVqtVqtPnjypHjt2rLply5bqc+fOqdVqtbpnz57qxYsX5znWgwcPND/n5OSoc3Jy9Bi5MBQZDm+Erl27Rv/+/SlZsiSvvfYaP/30E5aWlpQpU4Y5c+ZQvXp1atasSd26dXFycmLjxo0cO3aMunXrcvHiRd544w1sbW21JiJDbjOYUql87jfgouJxM5W5uTkODg78+uuvLFu2DFtbW0aPHk2pUqVwdXXlwIED3Lp1i1atWmlqD5UrV+bMmTNs3ryZtLQ0Zs6cSeXKlQ18RoXH49+znJwc/ve///HVV1/h4+ODlZUVW7ZsoUKFCsTGxrJs2TIGDhyIm5sbzZo14+LFi4SFhXHz5k2cnZ3JysrC29tbMzhGoVBgbW2teQ8zMzNpNTBR8gllhEqVKkXLli2ZNWsWrVu3Zv369Tg4OFCjRg3eeecdZs6cqRlc0aNHD7788ktu377N559/zo0bNzR/zE/+UT9OYALNagwAV69eJSUlBUdHR8aMGYOrqyvZ2dk4OzszbNgwfv75Z+Li4jA3Nyc7OxtXV1dCQkJYsmQJK1euxMXFxcBnU3hcvXqVxo0bc/36dczMzHB0dKRUqVIMGzYMT09PwsPDsba2pl+/fpQtW5aJEycCYGNjw6RJk/D19SUsLIxly5bx4MEDzZcs+V0uYgxd5RO6UalUmp+zs7PVQ4YMUXt4eKiHDx+utd/du3fVjRo1ytOMcufOHfW3336r3rt3b4HEa+xycnLUX3/9tXrnzp1qtVqtjo2NVY8bN04dGBio1fyUnJysHjx4sLpPnz5qtVr730nklZaWpv7rr780z+Pj49UtWrRQ16xZU9Ok/diOHTvUHh4e6r///lur/Pfff1cPGjRI02woih5pKizkHjer/PsbpUqlwtnZmerVqxMeHk6bNm1wcHAgOzsbOzs7rK2tCQkJwdfXFzs7O1QqFXZ2drRs2VKarHSUnZ3NgQMHWLlyJQMHDqRkyZLcv3+fEydOYG5uzuuvvw6ApaUlzs7OBAcH06FDBxwdHQ0ceeFmbm6Oo6MjiYmJxMbGUrlyZRwcHChWrBjx8fG0b99e87tetmxZoqOjCQ8Pp1u3bprmwPLly9OlSxecnZ1lEFERJU2FhdzjJo+ff/6ZyZMns23bNh49eoS3tzfNmzenXLlyzJo1C/inueSdd97B2tqa6dOnA2j1WT1rUqeAEydOaIa/W1hYMHjwYCwtLZk9ezYAzZo1o3bt2mzYsIH09HS2b9/OihUrqFu3LseOHaNq1aqGDL/QetpdCJYuXUqfPn1QKpX06NGDhg0bcvXqVTZv3gzkjmq1t7dn6NChREREsG/fvqfO65KkVTRJjauQO3PmDB988AHnzp3DwcGBnTt3cuTIEd5++20cHBywsLBgy5Yt1KpVCzc3N86cOYNSqaRVq1Z4eXnh5OSkdTz5Q8/1eM27x9/Y79+/T6dOnbTm/jg4OGBtbc2cOXPw9/fHyckJS0tLoqKimDdvHocOHaJ79+5UrVpVMyhA/EP9r6WwAMLDw0lMTKR8+fJUqlSJ3377jXv37tGkSRNKlChBfHw8ERERdOnSBaVSyfnz56lSpQqVKlXC29sbW1tbrePLIKKiS+ZxFSJPNnvcu3ePkJAQHBwcCAwMBODo0aO89957BAUF0alTJxITE5k9ezbHjx+nUqVKnD9/nhUrVmjmsEhTirbHo80gtznw3yPPFi5cyLp169i+fTslSpQAcudw9e3bFy8vL80yQxkZGZw9exZvb2+DnYcxOX36NN988w2ZmZmMGjWKJk2aUKxYMdatW8eUKVM4fPgwJUuW5ODBg/zwww8UL16cmzdvYmtry9KlS7G3twfkd1n8QxJXIfDvD1OAvXv30q5dOx49ekR0dDReXl4ATJo0iR07duDq6sqdO3fYtWsXdnZ2pKSksG7dOpRKJR988IGhTsOo/Pjjjxw+fJgyZcrQpEkTfH19USgUtG7dmjZt2vDll18CkJycjL+/PxcvXmT9+vWycOtL+vPPP/n888/p168f77zzjta2hw8fMnDgQMqWLcv8+fPJyMggNjaW9evXU6NGDfr06WOgqEVhJ02FhcDjJo+jR48SERHBmDFj8PHxwcXFhdKlS5OcnMzgwYNJTk4mJCSEZs2asWzZMtRqNU2bNsXS0hIvLy88PT2B3JqENKM8XUpKCsOHD+fcuXN8/PHHPHr0iL1792rmt7m4uBAUFETNmjWpWLEiERERWFpa0rJlS2rXrk3x4sUNfQqF0uNBRE/6448/SEhIYMSIEVhaWrJmzRpiYmJISkqiatWqlC1bljlz5tC4cWPc3d1xcnKiVatW1KpV67nHFUWbuaEDEHD58mVGjRqFmZkZtWvXBnJrVytWrMDS0pKjR49ibW1NUFAQ9vb27N+/n/bt23PixAnNEk+PqdVqzM3lnxVyE/iT1+LxyvcbN24Echds3bp1Kw8ePCAjI4MOHTqwb98+pk2bxuTJk8nMzGTatGk0a9aswOM3Fmq1WtNicPToUXJycqhcuTKurq5YWFhw//59Ro8ezdWrVylWrBiurq7Mnj2bH374gebNm9OiRQt2795NgwYNNMd8PJlb5mOJp5FPuAL2tHb6zZs3U716dWbMmEF2djZNmzZl9OjR7Ny5k44dO5KYmMjBgwcJDQ0lIiKCuLg4Fi9eTPny5fMcX/oA/rnGjycEnzhxAldXVypUqMBff/1FWloaKpWKb7/9lt27d+Pn58fIkSN59OgRAFOnTuXixYvExcXh4+Nj4LMp/BQKBdeuXWPMmDHcv38fZ2dnbt26Rb9+/QgICKBSpUr89ddflC5dmgYNGhAXF8e9e/c0NalZs2ZhZ2endUypZYnnkcRVQJ61BE1aWhq//PILn376KZA7/L1ly5b06dOHadOm0bFjR3r37k18fDwnTpzA2dmZefPmaWpZT6tVFHWPr/Evv/zC+PHjqVixIgkJCcyePZucnBzS0tJo0KABLVu2JCwsjDJlypCSkkJoaChdu3bF0dGR119/XTNXS2h7sk9WpVKxevVq6tSpw7hx40hLS+Onn35i/fr1ODo60rFjR9zd3TE3NyclJYUFCxaQk5OjWR3/cdJ68rhCPIt8rSkA/25KOXDgeyf4tgAAD3ZJREFUABs2bNC089vY2FClShX++OMPIPdDwMrKigoVKnD79m3NvYkCAwOZM2cOEydOxNLSUjM3RpJWXpcuXWLjxo0cOXKEJUuWsGXLFt58801CQkKwt7fHzMyM1q1bM2vWLMqUKcPvv/+On58f169fx8rKytDhF1qPx3E9/l1+vKzYgwcP2Lp1K40bNwZyl2fq0qUL3t7emluSnD59mmnTptG6dWtUKhWLFi2iVKlSWseXpCV0JZ96BUChUJCQkMDYsWNJTEykWrVqbNy4EQ8PDyZPnkyLFi3YsmUL58+fp2bNmprXtW3blkWLFtG3b18cHBwANLcykT/yZ9u3bx8LFy7Ex8dHMyJz2rRpdO7cmdu3b9OhQweOHTtGly5dKFu2LBcuXGDo0KEyiu05/n37kfDwcJYuXYqrqysNGzakfv36NGzYkBs3bgC5v6NOTk5YW1trklvZsmWpUKECq1evxsPDA5AalvjvJHHpwdP+ILdv3061atVYvXo1AAcPHmTw4MG0a9eOt956iwsXLtC/f38GDRpEVFQUMTExfPPNNyQmJrJnzx569eoF5CZB6cd6vgEDBrBv3z7UajXp6elYW1tjbm7O+++/z5o1axgzZgzdu3fn9u3bxMXF0aZNG0OHXOgplUqSk5MJCwtjw4YN9O3blzt37rBmzRpOnTpFdnY2MTExXL58mSpVqgC5rQFubm4AlCtXDn9/f+Cf1VskaYn/ShJXPlI/cZuQR48eYWtrS2pqKqtWrWL+/PkABAcHs3r1at5++22aNGmCpaUl06ZNo2LFisTHx2NjY8OWLVuwsLDg7t27lCxZ0pCnZXQsLS0ZNmwYISEhHDlyRJOYevfuzbp161i/fj116tShSpUqmg9Z8XyJiYl88sknZGVlMXHiRM18Nnd3d/bt24eZmRmPHj1i2LBhfPbZZ+zfv58DBw4QFBQE/PO3oX7ixqdC/BeSuPLJk00pISEhlC1blrZt29K6dWu8vLzYunUr48aNw9bWluDgYOrWrcvVq1d58OABderUYfDgwVrH3Lp1K46OjlSqVMkQp2TU3njjDdavX8/BgwepVauW5tYiU6ZMIScnR+ZjvSQXFxdq1qzJxo0bNaMvAdq0acORI0eoVq0aXbp0Yfny5f/X3t0HRVW9cQD/AoE0RMomREURg+0K7S6gWItNNYJMpA0o2RgiirumFuAgSLyIDKGGzUDrLDipozjFMKAhYhA2izJDgEAxVIIvBIS4IJSCzcY6LAt7fn/w2zusgG8htPp8/oF7ztlzz7kLPHvPvdwHp0+fBmOM+/kFMGkqHUIeBAWuKWJubo6BgQGUlZXh+PHjeO+993Dt2jWkpaVh1qxZsLCwgFKpxObNmxEeHg4AuHTpErKysvDOO+9AJBLBzMwMvb29aGxsxNGjR9HT04PU1FQ6K3gAZmZm2L59O2JjY6FUKhEWFgYARtcQyf2JiIhAQ0MDmpqasHjxYgCAra0t1Go1VCoVnJ2dkZqaavS/hXQdizwMFLimiGEpxdzcHHv27OEuQPf09KC8vBxeXl7Q6XSoqKiAo6MjqqqqoFQqER4ejqCgIK6fuXPnQqPRYNmyZdiwYcNMTeeR4OLiggULFsDW1namh/JI4PF4CA4OhlKphIODA1auXInu7m6o1WoEBwdz7QxBS6/XU9AiDwU9q3AK7dq1C6Wlpdi/fz/3FID29nZ89NFH2LRpExYtWoSSkhJotVrcunULW7du5ZZSDG/D2CeWk39v7BIu+feGhoYQEhKC33//HUFBQaipqcGSJUuQkpIy00MjjxEKXFOov78fGzduRGBgIMLCwrhPm5mZmTh79izS09O5lBkGEyWKJOS/rLa2FgqFAgEBAQgODubOaOkDF5ku9FF0CvF4PAQFBeHcuXP47bffuPKIiAjY2dmN++dWw1IK/bITUyKRSGBnZ4f29nbodDoAo09woZ9jMl0ocE2xkJAQ3Lp1C5WVlfjnn38AANbW1sjLy+NyZBnQEhYxRWZmZoiLi0NzczNKS0sB0BNcyPSiv5xTzMrKClKpFJWVlbhy5YpR3UQpzAkxRYYbXwxPdCFkOtE1roeAMYb29nbMmzdvpodCyENDN76QmUKBixBCiEmhj0uEEEJMCgUuQgghJoUCFyGEEJNCgYsQQohJocBFCCHEpFDgIo+0sLAw7NixY9Lt6dTV1QWBQICGhoYZ2f/d1NfXQyAQoLe3d6aHQsgdUeAi0yohIQECgQACgQDu7u7cA1pv3rw5LfvPyspCYmLiPbf39/fnEoA+Stzd3VFUVGRU5uXlherqajg4OMzQqIydOnWKy7JAyFj0nBYy7by9vbFv3z6MjIygubkZycnJ6O3txaFDh8a1ZYxheHgYlpaWU7LvOXPmTEk/jyIrKyvY29vP9DAIuSs64yLTztLSEvb29nB0dMTSpUuxfv16VFVVYXBwEEVFRXB3d0ddXR1WrFgBkUiEc+fOQafTISsrC76+vhCJRFi+fDkKCgqM+u3u7oZMJoNYLMbbb7+N3NzccfueaKkwLy8Py5Ytg1AohI+PD6Kiori2V69eRXZ2NneW2NXVBQDo7OxEVFQUvL29sWjRIkilUrS0tBj1W1ZWBn9/f4hEInz44Yfj6ifS2toKmUwGb29veHp64t1330VxcTFXr9FosHv3brz55pvw8PDAihUroFQquXrDcmRZWRk2b94MDw8P+Pn5GZ1d+fr6YmRkBImJidy8gPFLhYbtyspKrF69GmKxGMHBwWhtbUVraytCQkLg4eGBVatWoa2tzWgezc3NkEql8PLygkQiQWRkJLq7u7n6rKws+Pv748yZMwgICICnpyfCwsK4x6TV19fj008/BQBujAkJCXc9fuTxQIGLzDhra2vo9XoMDw8DGH2UUEZGBhISEnD69GmIRCLs3LkTSqUSaWlpKCsrQ0REBDIyMvDtt98CGD0zi4yMxN9//43c3FwcOHAAFRUVuHDhwh33rVAokJGRgTVr1qCkpASHDx+Gu7s7gNE/ri+88AKkUimqq6tRXV2N5557Djdu3MCaNWvA4/GQl5eHY8eOwcXFBevWrUN/fz8A4OLFi4iNjUVAQABOnToFqVSKPXv23PVYxMTEYM6cOSgoKEBJSQkSEhIwe/Zsbo5btmxBS0sL5HI5SktLERISgpiYGNTW1hr1k5mZiaCgIHz33XdYvnw5kpOT0dHRAQAoLCyEhYUFkpKSuHndiVwuR3R0NIqKimBpaYmYmBikpqZi69atXNnY5de2tjaEhYXB09MThYWF+Prrr2Fubg6pVAqtVsu1u379OvLz85GRkYGCggJoNBokJSUBGF22NOT4Moxxpq5Nkv8gRsg0io+PZ+vXr+e2W1tbmZ+fH/vggw8YY4ydOHGC8fl89vPPP3Ntrl69ygQCAWtrazPqKysriwUGBjLGGKupqWF8Pp/98ccfXH1fXx8TiUQsKSmJK1u7di23rdFomEgkYocPH550vEuXLmUKhcKoTKFQcOM10Ov1zM/Pjx09epQxxlhsbCxbvXq1UZvc3Nxxc7vdggUL2IkTJyasq6urY0KhkKnVaqPyhIQE9vHHHzPGGFOpVIzP57OcnByufnh4mHl6erL8/HyuzM3Nbdx+6urqGJ/PZz09PUbb5eXlXJuysjLG5/PZDz/8wJUplUrG5/PZwMAAY2z0PY6OjjbqW6vVMrFYzPWlUCiYm5sb6+vr49p8//33TCAQsMHBQcYYY8XFxYzP5096rMjji65xkWn3008/wcvLCyMjIxgaGoKPjw/S0tKM2ohEIu775uZmMMawatUqozbDw8Ncss62tjbY2dnBxcWFq+fxeEbbt2tra4NWq8Ubb7xxX+NvamrChQsX4OXlZVQ+ODiIzs5OAKOZryUSiVH9woUL79q3VCpFcnIyTp48iddeew2+vr549dVXuf3qdDq89dZbRq/R6XRwdnY2KhubQsfCwgLPPPMMbty4ce+THGPsDRKGa2Bjy+bOnQtgNJGqjY0Nmpqa0NnZOe74aLVao4wJDg4O4PF4RtuMMfT19eH5559/oLGSxwMFLjLtxGIxvvjiC1hYWMDBwQFWVlZG9RYWFkZJN9n/nwOdn5+PJ5980qjtTCQv1Ov1kEgkE6arN2QDflAREREIDAzEjz/+iPr6ehw8eBAymQzbtm2DXq+Hra0tCgsLx73u9ptXbt82MzPjjuP9mujGmLH5twzvgV6v574GBQVh06ZN41439uaYyW64MfRDyGQocJFpZ21tPe4M4U4MZxw9PT1YsmTJhG3mzZuHmzdv4sqVK3j55ZcBjJ4BdHR0QCgUTvgaV1dXzJo1CzU1NeOSfBpYWlqOy6MmFApx8uRJODo6jstqPbbvX375xaissbFx0jmO9eKLLyI0NBShoaE4dOgQjhw5gm3btkEkEkGtVkOr1YLP599TX5OZaF5TRSgUoqWlBS+99NK/+mBhCGwjIyPcmTUhAN2cQUyAs7Mz3n//fezcuRPFxcXo7OzE5cuXUVhYyN1C7+Pjg/nz5yMuLg7nz5/HpUuXEBcXd8fMvDY2NtiwYQOys7ORl5eHjo4OXL58GQcPHuTaODk5obGxEdeuXUN/fz/0ej3Wrl2LkZERfPLJJ2hoaEBXVxcaGhogl8u54BQeHo5ff/0VcrkcHR0dKC8vR05Ozh3nqdFo8Nlnn6G2thYqlQoXL15EVVUVXF1dAQASiQSLFy9GVFQUzpw5A5VKhebmZuTm5uL48eP3dUydnJxQX1+PP//8k7uhZKps2bIF7e3t2L59O86fPw+VSoW6ujrs3r0bKpXqvsYIABUVFejv74dGo5nScRLTRWdcxCTs2rULOTk5OHDgALq6umBjY4NXXnkFoaGhAEaXq/bv34+UlBSEhobCzs4OMpkMQ0NDd+w3OjoaPB4P33zzDdLT0/H000/D29ubq4+KikJKSgoCAgKg1Wpx9uxZODk54dixY/jyyy8RGRmJgYEB2NvbY+HChdw1IKFQiMzMTMjlchw5cgRubm5ITExERETEpGN54oknoFarsWPHDly/fh1PPfUUXn/9dcTHx3Nz/Oqrr5CdnY3PP/8cf/31F2bPno358+dj48aN93U84+PjkZ6eDj8/P+h0unu6Vf9eubq6oqCgAPv27YNMJoNWq8Wzzz4LiURyX0upYrEY69atQ0pKCvr7+7Fy5Urs3bt3ysZJTBclkiSEEGJSaKmQEEKISaHARQghxKRQ4CKEEGJSKHARQggxKRS4CCGEmBQKXIQQQkwKBS5CCCEmhQIXIYQQk/I/0zN51V8fl7IAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bdoNH910pMaR"
      },
      "source": [
        "### Visualization of sentiment, confidence and probability"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dsFM2SnIcCL4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8af33a59-2e7a-44ac-bd06-86df7ba1d38f"
      },
      "source": [
        "idx = 6\n",
        "\n",
        "text = y_texts[idx]\n",
        "true_sentiment = y_test[idx]\n",
        "\n",
        "print(\"\\n\".join(wrap(text)))\n",
        "print()\n",
        "print(f'True sentiment: {class_names[true_sentiment]}')"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The total value of the project is valued at SEK 30bn ( EUR 2.83 bn USD\n",
            "3.81 bn ) .\n",
            "\n",
            "True sentiment: neutral\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "20khXCqaOwoy"
      },
      "source": [
        "pred_df = pd.DataFrame({\n",
        "  'class_names': class_names,\n",
        "  'values': y_pred_probs[idx]\n",
        "})"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QqzBhdUWM6ko",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "outputId": "13ea355b-edc6-4f1a-e7c7-da5ee360525d"
      },
      "source": [
        "sns.barplot(x='values', y='class_names', data=pred_df, orient='h')\n",
        "plt.ylabel('sentiment')\n",
        "plt.xlabel('probability')\n",
        "plt.xlim([0, 1]);"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb0AAAEQCAYAAAAkrKp0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhM9+IG8HdCEhGhSlRJo4ubBBNZGSkVCbHE0iBiuUKrvUVxQ2mt5SouQe1LPSU3VfSSm9hraUIpbSIhKtMkPxVLFrsI2Udmvr8/PIZpLDPMkjjv53nyPJ3zPTnz5hudN+fMmXNkQggBIiIiCbCydAAiIiJzYekREZFksPSIiEgyWHpERCQZLD0iIpKMmpYOIDUajQbFxcWwtraGTCazdBwiompBCIF79+7B3t4eVlbPv7/G0jOz4uJinD171tIxiIiqJRcXFzg4ODz397P0zMza2hrA/V+cjY2NhdNYnlKphFwut3SMKoFz8RDn4iHOxX0qlQpnz57VvoY+L5aemT04pGljYwNbW1sLp6kaOA8PcS4e4lw8xLl46EXfFuKJLEREJBksPSIikgyWHhERSQZLj4iIJIOlR0REksHSIyIiyWDpERGRZLD0iIhIMlh6REQkGSw9IiKSDJYeERFJBkuPiIgkg6VHRESSwdIjIiLJYOkREZFksPSIiEgyWHpERCQZLD0iIpIMlh4REUkGS4+IiCSDpUdERJLB0iMiIslg6RERkWSw9IiISDJYekREJBksPSIikgyWHhERSQZLj4iIJEPSpXf58mV4eXnh8uXLlo5CRERmIJnSi4uLQ2BgoM6yJk2aIDU1FU2aNLFQKiIiMifJlB4REZHZSi88PBzz58/HxIkT4e3tDX9/f2zdulU7/vvvvyM8PBwKhQIBAQFYtmwZKioqtONpaWkIDQ2Fl5cX+vXrh+joaLi6umrHk5KSMHDgQLRt2xYKhQKjRo1CTk4OACAlJQWzZs3SHs708vJCfHw8cnNz4erqitzcXBQUFMDd3R2pqak6uSdPnowvvvhC+3jHjh3o06cPfHx80LNnT+zdu9dUU0ZERMYmzGTo0KHC19dXJCYmCrVaLfbu3StatGghsrOzRVZWlvD09BQ//vijuHfvnsjNzRV9+vQRa9asEUIIcffuXaFQKMSqVatEeXm5yMrKEt26dRMuLi7a7aekpIjU1FShUqnE7du3xejRo8XAgQO147GxsSIgIEAnU05OjnBxcRE5OTlCCCEmTJggpk+frh0vLCwUHh4e4sSJEzrbSEtLE2q1WiQnJwtvb2+RnJys9zyUlZWJlJQUUVZWZvgkvoRSUlIsHaHK4Fw8xLl4iHNxn7FeO816eLN79+5QKBSwsrJCcHAwHBwckJ6eji1btqBLly7o0aMHatasiaZNm2LkyJGIi4sDABw+fBjW1tYYPXo0bGxs8Pbbb2P48OE62/bx8YGnpyesra3xyiuvYOzYsTh9+jRKS0v1zhcaGooff/wRJSUlAIA9e/bgtddeQ5s2bQAA//nPfzB69GjI5XJYWVnB19cXvXr1wvbt2400Q0REZEo1zflkjRo10nlsb2+P4uJiXLx4EUlJSTh8+LB2TKPRQAgBALh69SqaNGkCK6uHHd20aVOdbWVkZGDJkiXIyMjQlpYQAvn5+ZXWfRI/Pz/Ur18f+/btQ//+/fG///0P/fv3145funQJ//73vxEZGaldplar4evrq+cMEBGRJZm19J7E0dERISEhmDNnzmPHGzdujMuXL0MIAZlMBgCVPmYwfvx4BAYG4uuvv0bdunWRnp6Ovn37aovz0cJ8EplMhn79+iE2Nhbu7u7IyMjA2rVrteMNGzbEhAkT0Lt37+f9UYmIyIKqxNmbgwcPxr59+3DgwAGoVCqo1WpcunQJR48eBQAEBARApVLhm2++gUqlwsWLF7Fx40adbRQWFsLe3h516tTBzZs3sWLFCp3xhg0bIj8/H3fu3Hlqlv79++P06dP4+uuv4e/vD0dHR+3Y8OHDsWrVKqSlpUGj0UClUuHMmTNQKpVGmgkiIjKlKlF6rVu3xoYNG7B161Z07NgRCoUC//znP7V7cw4ODvj2228RHx8PhUKBzz77DH379oWNjY12G/PmzcPu3bvh7e2NDz/8EEFBQTrP0a5dO/j7+6Nr167w9fVFQkLCY7M0btwY7du3x88//4zQ0FCdseHDh2PMmDGYNWsW2rZti/feew+LFi0y6H1DIiKyHJl4cPyvmtm0aRO+//57HDhwwNJRDFJeXg6lUgm5XA5bW1tLx7G4kydPwsfHx9IxqgTOxUOci4c4F/cZ67WzSuzp6eP48eO4evUqhBBIS0vD+vXr+d4aEREZpEqcyKKPrKwsTJ48GYWFhWjQoAF69+6NTz75xNKxiIioGqk2pTds2DAMGzbM0jGIiKgaqzaHN4mIiF4US4+IiCSDpUdERJLB0iMiIslg6RERkWSw9IiISDJYekREJBksPSIikgyWHhERSQZLj4iIJIOlR0REksHSIyIiyWDpERGRZLD0iIhIMlh6REQkGSw9IiKSDJYeERFJBkuPiIgkg6VHRESSwdIjIiLJYOkREZFksPSIiEgyWHpERCQZLD0iIpIMlh4REUkGS4+IiCSDpUdERJLB0iMiIslg6RERkWToXXqrVq1CaWlppeVlZWVYtWqVUUMRERGZgt6lt3r1apSUlFRaXlpaitWrVxs1FBERkSnoXXpCCMhkskrLL126hLp16xo1FBERkSnUfNYKgYGBkMlkkMlk6N+/P6ysHvakRqPBjRs30L17d5OGJCIiMoZnlt6AAQMghMCKFSvQq1cv1K5dWztmbW0NJycndOnSxaQhiYiIjOGZpTd69GgAwOuvv47g4GDY2tqaPBQREZEpyIQQwtIhpKS8vBxKpRJyuZx/QBAR6clYr53P3NN7ID8/H5GRkfj1119x69Yt/LUrMzIynjuEFOWeeg81cMPSMYiIqgU1HAGbtS+8Hb1Lb/r06Th79iyGDRuGRo0aPfZMTiIioqpM79I7ceIEoqKi4OHhYco8REREJqP35/Tq1asHe3t7U2YhIiIyKb1Lb9SoUVi7di0qKipMmYeIiMhk9D68uW/fPqSlpcHf3x9vv/02rK2tdcajoqKMHo6IiMiY9C69xo0bo3HjxqbMQkREZFJ6l978+fNNmYOIiMjkDL6fXmZmJg4cOKC9zZBKpYJGozF6MCIiImPTe0+voKAA48aNQ3JyMmQyGQ4ePIg33ngDs2fPRp06dTB16lRT5iQiInpheu/pRUZGwsrKCocOHUKtWrW0y7t164Zjx46ZJBwREZEx6b2nd/z4caxZswZNmjTRWf7WW2/h8uXLRg9GRERkbHrv6d25cwf16tWrtLy4uFjnHntERERVld5t1bJlSxw9erTS8h07dvDSZEREVC3ofXhz7NixGDNmDK5duwaNRoOdO3fizz//REJCAr777jtTZiQiIjIKvff02rdvj7Vr1yI1NRVWVlb49ttvcfPmTWzYsAE+Pj6mzEhERGQUeu/pAYCfnx/8/PxMlYWIiMikDCq9B4QQlW4iy5NZiIioqtO79K5fv44FCxbgt99+Q0FBQaVx3jmdiIiqOr1Lb/Lkybh69SpGjhwJR0dH3jmdiIiqHb1L7/Tp0/jhhx/g5uZmyjxEREQmo/cbcW+++SZUKpUpsxAREZmU3qU3bdo0LF68GBkZGVCr1abMREREZBJ6H95s1qwZ1Go1+vXr99hxqZ7IMmXKFADAggULLJyEiIieRe/SmzhxIm7evIlJkyahYcOG1fpElpUrV+LEiRP4/vvvLR2FiIjMSO/SO3PmDP773/+iRYsWpsxTZVRUVKBGjRrVutyJiEiX3u/pOTs7o6KiwpRZtMLDwzF//nxMnDgR3t7e8Pf3x9atW7Xjv//+O8LDw6FQKBAQEIBly5Zps+Xm5sLV1RW5ubna9ZOSkuDq6goA2LVrF9atW4eUlBR4eXnBy8sL6enp2nX27t2LoKAgeHp6oqSkBJs3b0bPnj3h5eWF9957D7Nnz9beNZ6IiKoXvUtvxowZWLRoEf7v//6v0tVYTCEuLg5hYWFISUnB5MmTMXv2bOTk5OD8+fP44IMPMGTIEBw/fhybNm3C4cOH8e233+q13T59+mDkyJHw9fVFamoqUlNT0bJlS+34gQMHEBMTg5MnT6J27dpwdHTEmjVrcOrUKURHR+P48eP45ptvTPVjExGRCeldeiNGjEBycjJCQkLQqlUryOVynS9j6969OxQKBaysrBAcHAwHBwekp6djy5Yt6NKlC3r06IGaNWuiadOmGDlyJOLi4ozyvJMmTcIrr7wCW1tbyGQydO3aFc2aNYNMJsM777yDIUOG4NdffzXKcxERkXnp/Z7e3LlzTZmjkkaNGuk8tre3R3FxMS5evIikpCQcPnxYO6bRaIy29+nk5KTzeP/+/YiKisKlS5dQUVGBiooKNGjQwCjPRURE5qV36fXt29eUOfTm6OiIkJAQzJkz57Hj9vb2AKDzvtv169d11nnaySmPXjj76tWrmDBhApYuXYrAwEDY2NggOjoaGzdufJEfgYiILKTa3Rph8ODB2LdvHw4cOACVSgW1Wo1Lly5p7+pev359ODk5Ydu2baioqEB2djaioqJ0tuHo6Ii8vLxnXmGmuLgYGo0G9evXh42NDTIzM7F582aT/WxERGRaT93Tk8vlOHr0KF599VW0atXqqXtISqXS6OEep3Xr1tiwYQOWL1+OWbNmoaKiAk2bNsXgwYO160RGRmL27Nlo06YNXF1dERoaiq+++ko7HhwcjP3796NDhw7QaDRP/LzeO++8g/Hjx2PChAkoKyuDp6cn3n//faO9f0hEROYlE095M2z79u3o2bMnbGxsEBcX99TSqyqHP6u68vJyKJVKvKIajRq4Yek4RETVghqOKLBZC7lcDltb2+fezlP39B4tsiddfoyIiKi60Ps9vc6dO+P27duVlt+9exedO3c2aigiIiJT0Lv08vLyoNFoKi1XqVS4du2aUUMRERGZwjM/spCcnKz979TUVNSrV0/7WK1W4/jx42jcuLFp0hERERnRM0svPDwcMpkMMpkMY8eOrTReu3ZtzJo1yyThiIiIjOmZpXfkyBEIIdCpUyds374dr776qnbM2toa9evX550IiIioWnhm6b322msAgMzMTJOHISIiMiW9L0MGAIWFhThz5gxu3rxZ6VqXISEhRg1GRERkbHqX3tGjR/HZZ5+hqKgINWrU0BmTyWQsPSIiqvL0Lr3IyEgEBQVh0qRJvMsAERFVSwZ9Tm/06NEsPCIiqrb0Lj25XI6cnBxTZiEiIjIpvQ9vfvrpp1i4cCFKS0vh5uYGa2trnfEHZ3kSERFVVXqX3ogRIwAAY8eO1flcnhACMpkMGRkZxk9HRERkRHqXHu8WTkRE1Z3epde2bVtT5iAiIjI5vU9kAYDz589j/vz5GDVqFG7cuH8D1ISEBF6thYiIqgW9Sy8lJQUhISHIzMzEsWPHUFZWBgDIysrC6tWrTRaQiIjIWPQuvSVLlmDMmDH47rvvdM7cbNeuHc6cOWOScERERMakd+llZmYiODi40vIGDRogPz/fqKGIiIhMQe/Ss7W1RWFhYaXlFy9e1LndEBERUVWld+n5+/tj3bp10Gg02mX5+flYtmwZAgMDTRKOiIjImPQuvc8//xznzp1DYGAgVCoVxowZg86dO6OsrAzjx483ZUYiIiKjkIm/3hjvKcrLy7F3714olUpoNBq4u7ujd+/esLGxMWXGl0p5eTmUSiXkcjlsbW0tHYeIqFow1mun3nt6x48fh1KpRL9+/TBz5ky0aNECW7ZswcyZM1FUVPTcAUjaTp48aekIVQbn4iHOxUOcC+PSu/QWLVqE27dvAwAuXLiAOXPmQC6XQ6lUYuHChSYLSEREZCx6X4YsOzsbLi4uAICffvoJfn5+mD17NlJTUxEREWGygERERMZi0GXIHtxdITk5Ge3btwdw/5ZCBQUFxk9GRERkZHqXnqurK7Zs2YLk5GQkJiaiQ4cOAIArV67wc3pERFQt6F16EydORFxcHIYNG4aQkBA0b94cAHD48GG4u7ubLCAREZGx6P2enq+vL3799VcUFxejbt262uVhYWGws7MzSTgiIiJj0rv0AKBGjRo6hQcAzs7ORg1ERERkKgadyEJERFSdsfSIiEgyWHpERCQZLD0iIpIMlh4REUkGS4+IiCSDpUdERJLB0iMiIslg6RERkWSw9IiISDJYekREJBksPSIikgyWHhERSQZLj4iIJIOlR0REksHSIyIiyWDpERGRZLD0iIhIMlh6REQkGSw9IiKSDJYeERFJBkuPiIgkg6VHRESSwdIjIiLJYOkREZFksPSIiEgyWHpERCQZLD0iIpIMlh4REUnGS116KSkp8PLyeuF1iIjo5fBSl56vry9SU1O1j1euXInw8PCnrkNERC+vl7r0iIiIHlUlSi88PBxz587Fp59+Ci8vL3Tt2hW7du3SjsfHxyMkJAQ+Pj4IDg5GTEyMduzu3buYMGECFAoFvL290a1bN+zfvx8AkJSUBFdXVwDArl27sG7dOu3hTC8vL6Snp+usc+7cObRs2RLXrl3TyTd06FAsX74cAKBWqxEVFYUePXrAx8cH/fr1w2+//WbS+SEiIuOoaekAD8TExGD58uVYsWIFjh07hrFjx6JZs2YQQmD8+PFYtmwZAgICcOrUKYwaNQr16tVD165dsWHDBhQXFyMhIQH29va4fPkySktLK22/T58+uHTpEk6cOIHvv/9euzwpKUn7382bN4e7uzvi4uIwevRoAMDFixdx8uRJLFiwAACwZs0aHDp0CGvWrEGzZs2QkJCATz/9FDt37oSzs7OJZ4mIiF5EldjTA4CAgAB06tQJNWvWRKdOndClSxfExsYiLi4OnTt3RpcuXVCjRg20adMGYWFh2Lp1KwDA2toaBQUFOH/+PIQQaNq0KZo3b/7cOUJDQxEbGwshBAAgNjYW7dq1g5OTEwAgOjoaX3zxBd566y1YWVkhKCgIPj4+2LNnz4tPAhERmVSVKb0HpfLo4ytXruDKlSt44403dMacnZ1x5coVAMBHH32E9u3bY8aMGWjXrh0iIiKQk5Pz3DmCg4Nx69YtJCUlQa1WY/v27RgwYAAA4ObNmygqKsKYMWPg6+ur/UpJSal0SJSIiKqeKnN4My8vr9Ljxo0bQyaTITc3V2csOzsbr7/+OgDAzs4OERERiIiIQEFBAb766itMnToVmzZtqvQcMpnsmTns7e3Ro0cPxMbGorS0FPfu3UOXLl0AAHXr1oWtrS3Wr18Pb2/v5/1RiYjIQqrMnt7hw4dx5MgRqNVqHDlyBD/99BP69euHvn37Ij4+HocOHYJarUZKSgpiYmK0e18JCQn4888/UVFRATs7O9ja2sLK6vE/lqOjI/Ly8qBSqZ6aZcCAATh48CCio6PRp08f2NjYAABsbGwwaNAgLFy4EFlZWRBCoKysDMnJybhw4YJxJ4SIiIyuyuzphYaGYuvWrRg/fjwaNmyIuXPnaj80vmTJEixfvhyTJk1Co0aN8Pnnn6N79+4AgNzcXERGRuLGjRuwsbGBh4cH5s6d+9jnCA4Oxv79+9GhQwdoNBqdE1oe5eXlBScnJyQmJmLatGk6Y5MnT8amTZsQERGBK1euwNbWFi1btsTkyZONOBtERGQKMvHgjA0LCg8PR9u2bTFu3DhLRzG58vJyKJVKyOVy2NraWjqOxZ08eRI+Pj6WjlElcC4e4lw8xLm4z1ivnVXm8CYREZGpsfSIiEgyqsR7ek96b42IiMiYuKdHRESSwdIjIiLJYOkREZFksPSIiEgyWHpERCQZLD0iIpIMlh4REUkGS4+IiCSDpUdERJLB0iMiIslg6RERkWSw9IiISDJYekREJBksPSIikgyWHhERSQZLj4iIJIOlR0REksHSIyIiyWDpERGRZLD0iIhIMlh6REQkGSw9IiKSDJYeERFJBkuPiIgkg6VHRESSwdIjIiLJqGnpAFIjhAAAqFQqCyepOsrLyy0docrgXDzEuXiIc/HwNfPBa+jzkokX3QIZpLCwEGfPnrV0DCKiasnFxQUODg7P/f0sPTPTaDQoLi6GtbU1ZDKZpeMQEVULQgjcu3cP9vb2sLJ6/nfmWHpERCQZPJGFiIgkg6VHRESSwdIjIiLJYOkREZFksPSIiEgyWHpERCQZLD0iIpIMlh4REUkGS88ENBoNlixZgnfffRdeXl746KOPkJeX98T109PTMWjQIHh4eKBTp07YuHGjGdOaliFzcfr0aXzyySd499134e3tjb59++LgwYNmTmw6hv67eECpVKJVq1YIDw83Q0rzMHQuysrKsGDBAnTs2BGenp4ICgrCkSNHzJjYNAydh127dqF3797w9vZGx44dMW/evJfmOr579+7FkCFD4O3tDVdX12eun5OTg48++gheXl549913sXTpUv2uyynI6NatWycCAgJEVlaWKCoqEjNmzBC9evUSarW60rqFhYXCz89PrFy5UpSVlYnU1FTRpk0bsW/fPgskNz5D5uLnn38W27dvF7du3RJqtVrs379fyOVy8fvvv1sgufEZMhcPlJWViV69eonhw4eLoUOHmjGtaRkyFxqNRowYMUKMGDFCZGdnCyGEuHLlisjJyTF3bKMzZB4yMjKEm5ub2Ldvn9BoNCI3N1d0795dLF261ALJje/o0aNi9+7dIiYmRri4uDx13YqKChEcHCxmzJghioqKRFZWlggICBDr169/5vOw9EwgICBAbN68Wfv4zp07olWrVuLEiROV1o2NjRXt27fX+Ue+cOFCER4ebpaspmbIXDxO3759RVRUlKnimdXzzMX8+fPF3LlzxYoVK16q0jNkLn755Rfh7u4ubt26Zc6IZmHIPBw8eFAoFAqdZQsWLBAjR440eU5zSkxMfGbpJSYmilatWok7d+5ol23evFkEBgY+c/s8vGlkhYWFyMvLg1wu1y6rW7cumjVrhoyMjErrZ2ZmomXLljoXUJXL5cjMzDRLXlMydC7+6tq1azh//jzc3NxMGdMsnmcukpOTcfjwYXz22WfmimkWhs5FYmIinJycsHbtWrRv3x6BgYGYM2cOiouLzRnb6Aydhw4dOsDJyQl79+6FWq1GdnY2Dh06hKCgIHPGrhIyMzPRrFkz1K1bV7tMLpcjNzcXRUVFT/1elp6RPZjwR38ZAODg4PDYX0ZRUVGl22TUrVv3mb+46sDQuXhUcXExxo0bh4CAAPj5+Zkso7kYOhfFxcWYNm0a5s6dCzs7O7NkNBdD5+L27dvIysoCAMTHx2PTpk1ITU1FZGSk6cOakKHzYGdnh9DQUMyaNQvu7u4ICgqCl5cXQkJCzJK3KnnS6+aDsadh6RlZnTp1ANz/K+5RhYWF2rG/rv/XX9Ldu3cfu251Y+hcPDr+8ccfw9HRsdq/sD1g6FxERkbC398fbdq0MUs+czJ0Luzt7VGjRg1MmjQJdnZ2aNKkCf7xj38gPj7eLHlNxdB52L59O5YsWYJvvvkGSqUSv/zyC27fvo3JkyebJW9V8qTXzQdjT8PSMzIHBwc0bdoUSqVSu6ywsBDZ2dlo0aJFpfXd3NyQnp4OjUajXfbHH3+8FIf0DJ0L4P5f9cOHD8frr7+O5cuXw8bGxlxxTcrQuTh27Bh27NgBhUIBhUKB9evX49SpU1AoFMjJyTFndKMzdC5atmwJADr3n3wZ7kVp6DwolUooFAr4+vrCysoKjRo1QlhYGBISEswZu0pwc3PDpUuXdP5g+OOPP+Dk5MTSs4RBgwZhw4YNuHDhAkpKSrBo0SK8+eab8PHxqbRu165doVarsXbtWqhUKpw5cwYxMTEYPHiwBZIbnyFzcePGDYSHh8PV1RWLFy9GzZo1LZDYdAyZi61bt2LPnj3YuXMndu7ciUGDBkEul2Pnzp1o0qSJBdIblyFzERQUhAYNGmDp0qVQqVS4du0a1q9fj27dulkguXEZMg8+Pj44ceIEUlNTIYTArVu3sG3bNp33BKsztVqN8vJy3Lt3DwBQXl6O8vJynR2CB3x9feHs7IxFixahpKQEFy5cwPr16/V73XzRM22oMrVaLRYvXizatWsnPDw8xIgRI7SnVycnJwtPT0+Rl5enXf+PP/4QYWFhwt3dXXTs2FF89913lopudIbMxcqVK4WLi4vw8PAQnp6e2q8vv/zSkj+C0Rj67+JRL9vZm4bOxdmzZ8XQoUOFp6eneO+998S8efNESUmJpeIbjaHzEB0dLbp16yY8PT2Fn5+fiIiIEJcvX7ZUfKOKjY0VLi4ulb4SExNFXl6e8PT0FMnJydr1s7OzxYgRI4SHh4do166dWLJkidBoNM98Ht45nYiIJIOHN4mISDJYekREJBksPSIikgyWHhERSQZLj4iIJIOlR0REksHSI3oJrVy50igXIg4PD8f06dOfus6UKVPwwQcfPPG54+LitFdVIbI0lh4RvZDp06dj+fLlTxwPDg7G0aNHtY937typ101CiUzh5brOE9FLTqVSVbnrkf71avd/VatWLdSqVctMaYiejnt6RBYUHh6OqVOnYvHixVAoFPD29saXX36J8vJy7fi0adOwbNkydOjQAQEBAQCA06dP4+9//ztat26NNm3aYOLEibh161al7e/evRudO3eGu7s7PvzwQ+Tm5mrHcnJyMHbsWHTo0AEeHh7o3bs3duzYUWkbGo3mifmAyoc3/+rRw5tJSUn44osvAACurq5wdXXFlClTEBcXB19fX5SWlup876pVq9C1a1fwwlFkLCw9Igs7cOAACgoKsGXLFixevBjx8fH4+uuvteP79u1Dfn4+oqOjERUVhRs3bmDEiBFo3LgxYmJisHbtWpw9exb//Oc/dbZ748YNbNmyBcuWLcPmzZtRVFSEcePGaQukpKQE7dq1w/r167F7926EhYVh2rRpSExMNCifIby8vDBz5kwA9+8kcezYMUyfPh3BwcGQyWTYv3+/dl2NRoO4uDiEhoa+FHdVoKqBpUdkYa+88gpmz56Nd955B4GBgRg/fjx++OEHlCIjQhAAAAN3SURBVJSUAAAaNWqEf/3rX2jevDlcXV2xefNm1KlTB/Pnz4erqyt8fX2xaNEipKSkIDk5Wbvd0tJSLFiwAO7u7mjdujUWLlyI9PR0bam5urpi6NChcHNzg7OzM8LDw+Hv7489e/YYlM8QNjY22lu/ODo6wtHREQ4ODqhVqxbef/99bNu2Tbvu8ePHcf36dfTv39/g5yF6EpYekYW5u7ujRo0a2sfe3t5QqVTIzs4GALRq1QpWVg//Vz137hw8PT113ttzc3ODg4MD/vzzT+2yV199Fc2aNdM+fuutt1C/fn3tOqWlpVi8eDF69uyJtm3bwsvLC0ePHsXly5cNymcsAwcOxKlTp7R3SY+JiUFgYCAaNGhg1OchaWPpEVVxdnZ2JtnuwoULsWvXLowZMwYbN27Ejh070LFjR+39zMztb3/7G3x8fLBt2zbcunULhw4dQlhYmEWy0MuLpUdkYWlpaVCr1drHqampsLGxgbOz82PXb968OU6fPg2VSqVdlpmZicLCQri4uGiX5efn6+yNXbhwAbdv30bz5s0BACkpKejduzeCg4Ph5uaGN954AxcvXnzhfM9ibW0NADrbfGDgwIHYsWMHtm7dikaNGqF9+/bP9RxET8LSI7KwgoICzJ49G1lZWfj555+xfPlyDBw4ELVr137s+kOHDkVRURGmTp2Ks2fPIiUlBZ9//jl8fX3h6+urXc/Ozg5Tp05FWloa0tLSMGXKFLRo0QJ+fn4A7h/uTEhIwJkzZ3Du3Dl8+eWXuH79+gvnexYnJycAwKFDh5Cfn4/i4mLtWPfu3QEAa9aswYABA3gCCxkdS4/Iwrp16wZ7e3sMGTIEEyZMQKdOnTBp0qQnrt+wYUNERUXh6tWrCA0NxahRo+Di4oIVK1borOfo6IiwsDBERERgyJAhqFWrFlauXKktkqlTp6Jp06YYNmwYPvjgA7z22mvo1q3bC+d7ltatW2PYsGGYOXMm/Pz8MGfOHO2Yra0t3n//fQgheAILmQTvnE5kQeHh4XB2dsa8efMsHaXKiIiIQEVFBVavXm3pKPQS4hVZiKhKuHPnDs6cOYP4+HhER0dbOg69pFh6RFQl9O3bF7dv38bHH3+MNm3aWDoOvaR4eJOIiCSDJ7IQEZFksPSIiEgyWHpERCQZLD0iIpIMlh4REUkGS4+IiCTj/wEH8Dbx4mE1kQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iQpQv-fLN6pX"
      },
      "source": [
        "### Predicting on Raw Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OysrAbVWPXWW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "outputId": "919362af-7c18-4a50-c7a2-3e51fb404911"
      },
      "source": [
        "df"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>According to Gran , the company has no plans t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Technopolis plans to develop in stages an area...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>The international electronic industry company ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>With the new production plant the company woul...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>According to the company 's updated strategy f...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4841</th>\n",
              "      <td>0</td>\n",
              "      <td>LONDON MarketWatch -- Share prices ended lower...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4842</th>\n",
              "      <td>1</td>\n",
              "      <td>Rinkuskiai 's beer sales fell by 6.5 per cent ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4843</th>\n",
              "      <td>0</td>\n",
              "      <td>Operating profit fell to EUR 35.4 mn from EUR ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4844</th>\n",
              "      <td>0</td>\n",
              "      <td>Net sales of the Paper segment decreased to EU...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4845</th>\n",
              "      <td>0</td>\n",
              "      <td>Sales in Finland decreased by 10.5 % in Januar...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4846 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      label                                               text\n",
              "0         1  According to Gran , the company has no plans t...\n",
              "1         1  Technopolis plans to develop in stages an area...\n",
              "2         0  The international electronic industry company ...\n",
              "3         2  With the new production plant the company woul...\n",
              "4         2  According to the company 's updated strategy f...\n",
              "...     ...                                                ...\n",
              "4841      0  LONDON MarketWatch -- Share prices ended lower...\n",
              "4842      1  Rinkuskiai 's beer sales fell by 6.5 per cent ...\n",
              "4843      0  Operating profit fell to EUR 35.4 mn from EUR ...\n",
              "4844      0  Net sales of the Paper segment decreased to EU...\n",
              "4845      0  Sales in Finland decreased by 10.5 % in Januar...\n",
              "\n",
              "[4846 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8LDCEIDaPZaR"
      },
      "source": [
        "text = df['text'][4841]\n",
        "\n",
        "encoded_review = tokenizer.encode_plus(\n",
        "  text,\n",
        "  max_length=MAX_LEN,\n",
        "  add_special_tokens=True,\n",
        "  return_token_type_ids=False,\n",
        "  pad_to_max_length=True,\n",
        "  return_attention_mask=True,\n",
        "  return_tensors='pt',\n",
        ")"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EBQ2VSanPkY-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a6623f9-2c60-47ca-fba4-3b5e75dc18fe"
      },
      "source": [
        "input_ids = encoded_review['input_ids'].to(device)\n",
        "attention_mask = encoded_review['attention_mask'].to(device)\n",
        "\n",
        "output = model(input_ids, attention_mask)\n",
        "_, prediction = torch.max(output, dim=1)\n",
        "\n",
        "print(f'Review text: {text}')\n",
        "print(f'Sentiment  : {class_names[prediction]}')"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Review text: LONDON MarketWatch -- Share prices ended lower in London Monday as a rebound in bank stocks failed to offset broader weakness for the FTSE 100 .\n",
            "Sentiment  : negative\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GL8wcrLspZgf"
      },
      "source": [
        "##Saving the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cLhq9hWlFY8l"
      },
      "source": [
        "# torch.save(model, '/drive/MyDrive/finance/model')\n",
        "\n",
        "# saved_model = torch.load('/drive/MyDrive/finance/model')"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BIsUqpTFfzz3"
      },
      "source": [
        "# pickle.dump(model, open(\"model_gb.pkl\", \"wb\"))"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vxfxnwy5pd5N"
      },
      "source": [
        "## API Building"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uwGZYVdJE_EF",
        "outputId": "367b26cc-7822-40b7-98c5-cc910c0c1e7d"
      },
      "source": [
        "class Model:\n",
        "    def __init__(self):\n",
        "        self.device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "        self.tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "        classifier = SentimentClassifier(3)\n",
        "        classifier.load_state_dict(\n",
        "            torch.load(\"/drive/MyDrive/finance/best_model_state.bin\", map_location=self.device)\n",
        "        )\n",
        "        classifier = classifier.eval()\n",
        "        self.classifier = classifier.to(self.device)\n",
        "    def predict(self, text):\n",
        "        encoded_text = self.tokenizer.encode_plus(\n",
        "            text,\n",
        "            max_length=MAX_LEN,\n",
        "            add_special_tokens=True,\n",
        "            return_token_type_ids=False,\n",
        "            pad_to_max_length=True,\n",
        "            return_attention_mask=True,\n",
        "            return_tensors=\"pt\",\n",
        "        )\n",
        "        input_ids = encoded_text[\"input_ids\"].to(self.device)\n",
        "        attention_mask = encoded_text[\"attention_mask\"].to(self.device)\n",
        "        with torch.no_grad():\n",
        "            probabilities = F.softmax(self.classifier(input_ids, attention_mask), dim=1)\n",
        "        confidence, predicted_class = torch.max(probabilities, dim=1)\n",
        "        predicted_class = predicted_class.cpu().item()\n",
        "        probabilities = probabilities.flatten().cpu().numpy().tolist()\n",
        "        return (\n",
        "            [\"negative\", \"neutral\", \"positive\"][predicted_class],\n",
        "            confidence,\n",
        "            dict(zip([\"negative\", \"neutral\", \"positive\"], probabilities)),\n",
        "        )\n",
        "yomodel = Model()\n",
        "def get_model():\n",
        "    return yomodel"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cb_PLjn8QekS"
      },
      "source": [
        "class SentimentRequest(BaseModel):\n",
        "  text: str\n",
        "\n",
        "class SentimentResponse(BaseModel):\n",
        "  probabilities:Dict[str, float]\n",
        "  sentiment: str\n",
        "  confidence: float"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZOy6oT3URBVO"
      },
      "source": [
        "app = FastAPI()\n",
        "\n",
        "@app.on_event(\"startup\")\n",
        "def load_model():\n",
        "  global loaded_model\n",
        "  loaded_model = get_model()\n",
        "\n",
        "@app.get('/')\n",
        "def index():\n",
        "  return {'message': 'This is the homepage of the API'}\n",
        "\n",
        "@app.post('/predict', response_model=SentimentResponse)\n",
        "def predict(data: SentimentRequest):\n",
        "  received = data.dict()\n",
        "  text = received['text']\n",
        "  sentiment, confidence, probabilities = loaded_model.predict(text)\n",
        "  return SentimentResponse(\n",
        "        sentiment=sentiment, confidence=confidence, probabilities=probabilities\n",
        "    )"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AZYbxEHGYP_9"
      },
      "source": [
        "server = ColabCode(port=10000, code = False)"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S50jSHELZVWG",
        "outputId": "f0d4f33d-2324-4279-d3c5-ccdca3a525be"
      },
      "source": [
        "server.run_app(app = app)"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Public URL: NgrokTunnel: \"https://e860-35-192-220-243.ngrok.io\" -> \"http://localhost:10000\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lmu2VA8HZX3v"
      },
      "source": [
        ""
      ],
      "execution_count": 48,
      "outputs": []
    }
  ]
}