blog_text_augment_02.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "blog_text_augment_02.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPkRnMVRoZxBX9/obsDvZWc",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sujitpal/deeplearning-ai-examples/blob/master/blog_text_augment_02.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OaRXoIydsCwY"
      },
      "source": [
        "## Augmenting to balance unbalanced text dataset (Part 2)\n",
        "\n",
        "This notebook uses the data prepared from the previous notebook, and tries out 5 + 1 baseline strategies against a 3 layer fully connected network.\n",
        "\n",
        "Because we investigate the case where we augment before and after embedding, the embedding is done separately in the previous notebook using Google Universal Sentence Encoder (GUSE), and input to the network is always a (512,) vector."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FRc-yyHYDqR5",
        "outputId": "40523634-9d67-4133-c2a7-fa4b9b3d9034"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import operator\n",
        "import os\n",
        "import pandas as pd\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "\n",
        "from google.colab import drive\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
        "from sklearn.neighbors import base\n",
        "\n",
        "from imblearn.over_sampling import SMOTE\n",
        "\n",
        "%matplotlib inline"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n",
            "  \"(https://pypi.org/project/six/).\", FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TyrO7tlKD8ic",
        "outputId": "ae121b19-c381-4fc8-c4dc-cff70f89a1af"
      },
      "source": [
        "# Mount Google Drive\n",
        "\n",
        "ROOT = \"/content/drive\"     # default location for the drive\n",
        "print(ROOT)                 # print content of ROOT (Optional)\n",
        "\n",
        "drive.mount(ROOT)           # we mount the google drive at /content/drive"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive\n",
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1HSFewHPDxYs"
      },
      "source": [
        "DATA_DIR = \"/content/drive/MyDrive/nlp-deeplearning-ai-data\"\n",
        "\n",
        "# pre-split data splits prepared in previous (01) notebook\n",
        "SPAM_TRAIN_FILE = os.path.join(DATA_DIR, \"spam_train_emb.tsv\")\n",
        "SPAM_VALID_FILE = os.path.join(DATA_DIR, \"spam_valid_emb.tsv\")\n",
        "SPAM_TEST_FILE = os.path.join(DATA_DIR, \"spam_test_emb.tsv\")\n",
        "\n",
        "# only used for TextAttack experiment\n",
        "SPAM_TA_TRAIN_FILE = os.path.join(DATA_DIR, \"spam_train_ta_emb.tsv\")\n",
        "\n",
        "CLF_LAYER_DIMS = [512, 128, 32, 2]\n",
        "\n",
        "BATCH_SIZE = 128\n",
        "LEARNING_RATE = 1e-3\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "ACCURACY_STATS = []"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hzut3LbFttj9",
        "outputId": "a27aacdd-c34c-4f71-ab12-f50a9e19f839"
      },
      "source": [
        "spam_train_df = pd.read_csv(SPAM_TRAIN_FILE, sep='\\t').dropna().reset_index()\n",
        "spam_valid_df = pd.read_csv(SPAM_VALID_FILE, sep='\\t').dropna().reset_index()\n",
        "spam_test_df = pd.read_csv(SPAM_TEST_FILE, sep='\\t').dropna().reset_index()\n",
        "\n",
        "len(spam_train_df), len(spam_valid_df), len(spam_test_df)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3900, 557, 1115)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "8gIg756-M3b7",
        "outputId": "be7fe5b8-1fb6-43db-d856-1c7d2fbf9d66"
      },
      "source": [
        "spam_train_df.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "      <th>embedding</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Ugh I don't wanna get out of bed. It's so warm.</td>\n",
              "      <td>ham</td>\n",
              "      <td>5.3949911e-02,5.8220369e-03,-3.1541418e-02,3.3...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Dear i have reache room</td>\n",
              "      <td>ham</td>\n",
              "      <td>4.7420025e-02,2.3147971e-03,1.6905151e-02,-1.1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>I always chat with you. In fact i need money c...</td>\n",
              "      <td>ham</td>\n",
              "      <td>3.6402624e-02,-1.0346117e-01,5.0689392e-02,2.1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Yes just finished watching days of our lives. ...</td>\n",
              "      <td>ham</td>\n",
              "      <td>1.6084902e-02,-1.7980557e-03,-4.4794872e-02,3....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>Nice.nice.how is it working?</td>\n",
              "      <td>ham</td>\n",
              "      <td>-4.0968325e-02,-2.6070992e-02,-1.6136026e-02,-...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index  ...                                          embedding\n",
              "0      0  ...  5.3949911e-02,5.8220369e-03,-3.1541418e-02,3.3...\n",
              "1      1  ...  4.7420025e-02,2.3147971e-03,1.6905151e-02,-1.1...\n",
              "2      2  ...  3.6402624e-02,-1.0346117e-01,5.0689392e-02,2.1...\n",
              "3      3  ...  1.6084902e-02,-1.7980557e-03,-4.4794872e-02,3....\n",
              "4      4  ...  -4.0968325e-02,-2.6070992e-02,-1.6136026e-02,-...\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vYaJo6w6wHBs",
        "outputId": "a2357ebb-c2ee-4826-8fe9-ed3cc0a3365a"
      },
      "source": [
        "spam_train_df[\"label\"].value_counts()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ham     3355\n",
              "spam     545\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        },
        "id": "L9Fg-3EVwKzW",
        "outputId": "f48a9f22-7d48-431b-e229-d3c87edcc22a"
      },
      "source": [
        "spam_train_df[\"label\"].value_counts().plot(kind=\"bar\")"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f6865c248d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEMCAYAAAAoB2Y1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASSElEQVR4nO3db5Bd9X3f8ffHEthOnQRRNlSV5Igk6mSw48h0I+gkD1w8BgEPRKatCw9ijcuM0g50ktbTRs4kJbbDDJ429ow7hKkcVAvXMWUSe1BsOUQlblNPi9HKJoDADFv+jKQKWEcYm3qiRvTbB/e3ya3Y1a6k3XvV/b1fM3fuud/zO+d+z3D53KPfPfduqgpJUh/eNO4GJEmjY+hLUkcMfUnqiKEvSR0x9CWpI4a+JHVkwdBP8pYkjyT50ySHknyk1T+T5Lkkj7bb5lZPkk8lmU7yWJIrhva1Pckz7bZ9+Q5LkjSX1YsYcwK4uqpeS3IB8LUkX2nr/kVV/d4p468DNrXblcDdwJVJLgZuByaBAg4m2VtVryzFgUiSFrZg6Nfg21uvtYcXtNvpvtG1Dbi3bfdwkouSrAXeA+yvquMASfYDW4HPz7ejSy65pDZu3LiIw5AkzTp48OC3q2pirnWLOdMnySrgIPATwF1V9fUk/wS4I8m/Ah4CdlbVCWAdcHho8yOtNl99Xhs3bmRqamoxLUqSmiQvzLduUR/kVtXrVbUZWA9sSfJO4MPATwI/A1wM/MoS9EqSHUmmkkzNzMwsxS4lSc0ZXb1TVd8BvgpsrapjNXAC+PfAljbsKLBhaLP1rTZf/dTn2FVVk1U1OTEx579OJElnaTFX70wkuagtvxV4H/CtNk9PkgA3Ak+0TfYCH2hX8VwFvFpVx4AHgWuSrEmyBrim1SRJI7KYOf21wJ42r/8m4P6q+lKSP04yAQR4FPjHbfw+4HpgGvg+8EGAqjqe5GPAgTbuo7Mf6kqSRiPn808rT05Olh/kStKZSXKwqibnWuc3ciWpI4a+JHXE0Jekjizqy1k6vY07vzzuFlaU5++8YdwtSCuWZ/qS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjqyYOgneUuSR5L8aZJDST7S6pcl+XqS6ST/McmFrf7m9ni6rd84tK8Pt/rTSa5droOSJM1tMWf6J4Crq+qngc3A1iRXAR8HPllVPwG8AtzSxt8CvNLqn2zjSHI5cBPwDmAr8NtJVi3lwUiSTm/B0K+B19rDC9qtgKuB32v1PcCNbXlbe0xb/94kafX7qupEVT0HTANbluQoJEmLsqg5/SSrkjwKvAzsB/4H8J2qOtmGHAHWteV1wGGAtv5V4K8P1+fYZvi5diSZSjI1MzNz5kckSZrXokK/ql6vqs3AegZn5z+5XA1V1a6qmqyqyYmJieV6Gknq0hldvVNV3wG+Cvwd4KIkq9uq9cDRtnwU2ADQ1v8w8GfD9Tm2kSSNwGKu3plIclFbfivwPuApBuH/99uw7cADbXlve0xb/8dVVa1+U7u65zJgE/DIUh2IJGlhqxcewlpgT7vS5k3A/VX1pSRPAvcl+U3gm8A9bfw9wGeTTAPHGVyxQ1UdSnI/8CRwEri1ql5f2sORJJ3OgqFfVY8B756j/ixzXH1TVX8O/IN59nUHcMeZtylJWgp+I1eSOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHVkwdBPsiHJV5M8meRQkl9q9d9IcjTJo+12/dA2H04yneTpJNcO1be22nSSnctzSJKk+axexJiTwIeq6htJfhA4mGR/W/fJqvo3w4OTXA7cBLwD+JvAf0ryt9rqu4D3AUeAA0n2VtWTS3EgkqSFLRj6VXUMONaWv5fkKWDdaTbZBtxXVSeA55JMA1vauumqehYgyX1trKEvSSNyRnP6STYC7wa+3kq3JXksye4ka1ptHXB4aLMjrTZf/dTn2JFkKsnUzMzMmbQnSVrAokM/yduA3wd+uaq+C9wN/DiwmcG/BH5rKRqqql1VNVlVkxMTE0uxS0lSs5g5fZJcwCDwP1dVXwCoqpeG1n8a+FJ7eBTYMLT5+lbjNHVJ0ggs5uqdAPcAT1XVJ4bqa4eG/TzwRFveC9yU5M1JLgM2AY8AB4BNSS5LciGDD3v3Ls1hSJIWYzFn+j8L/ALweJJHW+1XgZuTbAYKeB74RYCqOpTkfgYf0J4Ebq2q1wGS3AY8CKwCdlfVoSU8FknSAhZz9c7XgMyxat9ptrkDuGOO+r7TbSdJWl5+I1eSOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHVkwdBPsiHJV5M8meRQkl9q9YuT7E/yTLtf0+pJ8qkk00keS3LF0L62t/HPJNm+fIclSZrLYs70TwIfqqrLgauAW5NcDuwEHqqqTcBD7THAdcCmdtsB3A2DNwngduBKYAtw++wbhSRpNBYM/ao6VlXfaMvfA54C1gHbgD1t2B7gxra8Dbi3Bh4GLkqyFrgW2F9Vx6vqFWA/sHVJj0aSdFpnNKefZCPwbuDrwKVVdaytehG4tC2vAw4PbXak1earn/ocO5JMJZmamZk5k/YkSQtYdOgneRvw+8AvV9V3h9dVVQG1FA1V1a6qmqyqyYmJiaXYpSSpWVToJ7mAQeB/rqq+0MovtWkb2v3LrX4U2DC0+fpWm68uSRqRxVy9E+Ae4Kmq+sTQqr3A7BU424EHhuofaFfxXAW82qaBHgSuSbKmfYB7TatJkkZk9SLG/CzwC8DjSR5ttV8F7gTuT3IL8ALw/rZuH3A9MA18H/ggQFUdT/Ix4EAb99GqOr4kRyFJWpQFQ7+qvgZkntXvnWN8AbfOs6/dwO4zaVCStHT8Rq4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SerIgqGfZHeSl5M8MVT7jSRHkzzabtcPrftwkukkTye5dqi+tdWmk+xc+kORJC1kMWf6nwG2zlH/ZFVtbrd9AEkuB24C3tG2+e0kq5KsAu4CrgMuB25uYyVJI7R6oQFV9SdJNi5yf9uA+6rqBPBckmlgS1s3XVXPAiS5r4198ow7liSdtXOZ078tyWNt+mdNq60DDg+NOdJq89XfIMmOJFNJpmZmZs6hPUnSqc429O8GfhzYDBwDfmupGqqqXVU1WVWTExMTS7VbSRKLmN6ZS1W9NLuc5NPAl9rDo8CGoaHrW43T1CVJI3JWZ/pJ1g49/Hlg9sqevcBNSd6c5DJgE/AIcADYlOSyJBcy+LB379m3LUk6Gwue6Sf5PPAe4JIkR4Dbgfck2QwU8DzwiwBVdSjJ/Qw+oD0J3FpVr7f93AY8CKwCdlfVoSU/GknSaS3m6p2b5yjfc5rxdwB3zFHfB+w7o+4kSUvKb+RKUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOLBj6SXYneTnJE0O1i5PsT/JMu1/T6knyqSTTSR5LcsXQNtvb+GeSbF+ew5Eknc5izvQ/A2w9pbYTeKiqNgEPtccA1wGb2m0HcDcM3iSA24ErgS3A7bNvFJKk0Vkw9KvqT4Djp5S3AXva8h7gxqH6vTXwMHBRkrXAtcD+qjpeVa8A+3njG4kkaZmd7Zz+pVV1rC2/CFzaltcBh4fGHWm1+epvkGRHkqkkUzMzM2fZniRpLuf8QW5VFVBL0Mvs/nZV1WRVTU5MTCzVbiVJnH3ov9SmbWj3L7f6UWDD0Lj1rTZfXZI0Qmcb+nuB2StwtgMPDNU/0K7iuQp4tU0DPQhck2RN+wD3mlaTJI3Q6oUGJPk88B7gkiRHGFyFcydwf5JbgBeA97fh+4DrgWng+8AHAarqeJKPAQfauI9W1akfDkuSltmCoV9VN8+z6r1zjC3g1nn2sxvYfUbdSZKWlN/IlaSOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SerIOYV+kueTPJ7k0SRTrXZxkv1Jnmn3a1o9ST6VZDrJY0muWIoDkCQt3uol2MffrapvDz3eCTxUVXcm2dke/wpwHbCp3a4E7m73kpbRxp1fHncLK8bzd94w7hbO2XJM72wD9rTlPcCNQ/V7a+Bh4KIka5fh+SVJ8zjX0C/gj5IcTLKj1S6tqmNt+UXg0ra8Djg8tO2RVpMkjci5Tu/8XFUdTfIjwP4k3xpeWVWVpM5kh+3NYwfA29/+9nNsT5I07JzO9KvqaLt/GfgisAV4aXbapt2/3IYfBTYMbb6+1U7d566qmqyqyYmJiXNpT5J0irMO/SR/LckPzi4D1wBPAHuB7W3YduCBtrwX+EC7iucq4NWhaSBJ0gicy/TOpcAXk8zu53er6g+THADuT3IL8ALw/jZ+H3A9MA18H/jgOTy3JOksnHXoV9WzwE/PUf8z4L1z1Au49WyfT5J07vxGriR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6sjIQz/J1iRPJ5lOsnPUzy9JPRtp6CdZBdwFXAdcDtyc5PJR9iBJPRv1mf4WYLqqnq2q/w3cB2wbcQ+S1K1Rh/464PDQ4yOtJkkagdXjbuBUSXYAO9rD15I8Pc5+VphLgG+Pu4mF5OPj7kBjct6/Pv8/em3+6HwrRh36R4ENQ4/Xt9pfqqpdwK5RNtWLJFNVNTnuPqS5+PocjVFP7xwANiW5LMmFwE3A3hH3IEndGumZflWdTHIb8CCwCthdVYdG2YMk9Wzkc/pVtQ/YN+rnFeC0mc5vvj5HIFU17h4kSSPizzBIUkcMfUnqiKEvSR05776cpaWX5F3ARob+e1fVF8bWkMRf/hbXDbzxtfmJcfXUA0N/hUuyG3gXcAj4P61cgKGvcfsD4M+Bx/mr16aWmaG/8l1VVf6Sqc5H66vqXeNuojfO6a98/92fr9Z56itJrhl3E73xTH/lu5dB8L8InAAClGdYOg88DHwxyZuAv+CvXps/NN62Vja/nLXCJZkG/jmnzJtW1Qtja0oCkjzH4O9pPF4G0ch4pr/yzVSVP2qn89Fh4AkDf7QM/ZXvm0l+l8GVEidmi16yqfPAs8B/TvIV/t/XppdsLiNDf+V7K4P/oYY/MPOSTZ0Pnmu3C9tNI+CcviR1xDP9FS7JW4BbgHcAb5mtV9U/GltTEpBkAviXvPG1efXYmuqA1+mvfJ8F/gZwLfBfGPyJyu+NtSNp4HPAt4DLgI8AzzP463paRk7vrHBJvllV707yWFW9K8kFwH+tqqvG3Zv6luRgVf3t2ddmqx2oqp8Zd28rmdM7K99ftPvvJHkn8CLwI2PsR5o1+9o8luQG4H8CF4+xny4Y+ivfriRrgF9j8Efo3wb8+nhbkgD4zSQ/DHwI+LfADwH/bLwtrXxO76xwSd4M/D0GP197QStXVX10bE1JGhs/yF35HmDwVfeTwGvt9r/G2pEEJPmxJH+Q5NtJXk7yQJIfG3dfK51n+itckieq6p3j7kM6VZKHgbuAz7fSTcA/raorx9fVyueZ/sr335L81LibkObwA1X12ao62W7/gaHr9bU8PNNfoZI8zuDnFlYDmxj8zok/razzRpKPA68A9zF4rf5DYA3wrwGq6vj4ulu5DP0VKsmPnm69P62scWs/rTxrNogy+7iqnN9fBoa+pLFI8n7gD6vqu0l+HbgC+FhVfWPMra1ozulLGpdfa4H/c8DVwO8Ad4+5pxXP0Jc0Lq+3+xuAT1fVl/EnlpedoS9pXI4m+XcMPsDd175IaCYtM+f0JY1Fkh8AtjL4G7nPJFkL/FRV/dGYW1vRDH1J6oj/lJKkjhj6ktQRQ1+SOmLoS1JHDH1J6sj/BbBz/uS/XRpSAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A-0m6b0cf0rE"
      },
      "source": [
        "## Baseline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBsPxE1bPFgQ"
      },
      "source": [
        "def create_dataset(input_df):\n",
        "  emb_vecs = []\n",
        "  for emb in input_df[\"embedding\"].values:\n",
        "    vec = np.array([float(x) for x in str(emb).split(\",\")])\n",
        "    emb_vecs.append(vec)\n",
        "  labels = [0 if label == \"ham\" else 1 for label in input_df[\"label\"].values]\n",
        "  return TensorDataset(\n",
        "      torch.tensor(np.array(emb_vecs), dtype=torch.float32),\n",
        "      torch.tensor(np.array(labels), dtype=torch.long)\n",
        "  )\n",
        "\n",
        "\n",
        "train_ds = create_dataset(spam_train_df)\n",
        "val_ds = create_dataset(spam_valid_df)\n",
        "test_ds = create_dataset(spam_test_df)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rSRVZ4wDXHMn"
      },
      "source": [
        "train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\n",
        "val_dl = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False)\n",
        "test_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2G6Y-QyVD4aJ",
        "outputId": "24e0d744-9a25-4ed5-8074-75a2083f8e2e"
      },
      "source": [
        "class SpamClassifier(nn.Module):\n",
        "  def __init__(self, dimensions):\n",
        "    super(SpamClassifier, self).__init__()\n",
        "    self.input_layer = nn.Linear(dimensions[0], dimensions[1])\n",
        "    self.hidden_layers = nn.ModuleList()\n",
        "    for i in range(1, len(dimensions) - 2):\n",
        "      hidden_layer = nn.Linear(dimensions[i], dimensions[i + 1])\n",
        "      self.hidden_layers.append(hidden_layer)\n",
        "    self.output_layer = nn.Linear(dimensions[-2], dimensions[-1])\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = F.relu(self.input_layer(x))\n",
        "    for hidden_layer in self.hidden_layers:\n",
        "      x = F.relu(hidden_layer(x))\n",
        "    x = F.softmax(self.output_layer(x), dim=1)\n",
        "    return x\n",
        "\n",
        "clf = SpamClassifier(CLF_LAYER_DIMS)\n",
        "\n",
        "# self-test\n",
        "input = torch.rand(32, 512)\n",
        "output = clf(input)\n",
        "print(input.size(), output.size())"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([32, 512]) torch.Size([32, 2])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tFnLjo3varCS"
      },
      "source": [
        "dev = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "clf = clf.to(dev)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ARIEvAPsKt-f"
      },
      "source": [
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.AdamW(clf.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Emo_75xMG_5w"
      },
      "source": [
        "def train(train_dl, model, loss_fn, optimizer):\n",
        "  model.train()\n",
        "  train_loss, train_acc, num_batches = 0, 0, 0\n",
        "  for data, labels in train_dl:\n",
        "    data, labels = data.to(dev), labels.to(dev)\n",
        "\n",
        "    logits = model(data)\n",
        "    loss = loss_fn(logits, labels)\n",
        "\n",
        "    train_loss += loss.item()\n",
        "    _, preds = torch.max(logits.data, dim=1)\n",
        "    train_acc += (preds == labels).sum() / labels.size()[0]\n",
        "    num_batches += 1\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "  return train_loss / num_batches, train_acc / num_batches\n",
        "\n",
        "def validate(val_dl, model, loss_fn):\n",
        "  model.eval()\n",
        "  val_loss, val_acc, num_batches = 0, 0, 0\n",
        "  for data, labels in val_dl:\n",
        "    data, labels = data.to(dev), labels.to(dev)\n",
        "    with torch.no_grad():\n",
        "      logits = model(data)\n",
        "      loss = loss_fn(logits, labels)\n",
        "\n",
        "      val_loss += loss.item()\n",
        "      _, preds = torch.max(logits.data, dim=1)\n",
        "      val_acc += (preds == labels).sum() / labels.size()[0]\n",
        "      num_batches += 1\n",
        "\n",
        "  return val_loss / num_batches, val_acc / num_batches"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VKKv9i0AMfVC",
        "outputId": "e536ebef-987a-4d06-dd22-d6f76420cc82"
      },
      "source": [
        "train_losses, train_accs, val_losses, val_accs = [], [], [], []\n",
        "for epoch in range(1, NUM_EPOCHS + 1):\n",
        "  train_loss, train_acc = train(train_dl, clf, loss_fn, optimizer)\n",
        "  val_loss, val_acc = validate(val_dl, clf, loss_fn) \n",
        "  print(\"EPOCH {:2d} | train_loss: {:.3f}, acc: {:.3f} | val_loss: {:.3f}, acc: {:.3f}\"\n",
        "    .format(epoch, train_loss, train_acc, val_loss, val_acc))\n",
        "  train_losses.append(train_loss)\n",
        "  train_accs.append(train_acc)\n",
        "  val_losses.append(val_loss)\n",
        "  val_accs.append(val_acc)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH  1 | train_loss: 0.672, acc: 0.537 | val_loss: 0.531, acc: 0.894\n",
            "EPOCH  2 | train_loss: 0.438, acc: 0.917 | val_loss: 0.371, acc: 0.969\n",
            "EPOCH  3 | train_loss: 0.359, acc: 0.976 | val_loss: 0.342, acc: 0.975\n",
            "EPOCH  4 | train_loss: 0.336, acc: 0.983 | val_loss: 0.335, acc: 0.975\n",
            "EPOCH  5 | train_loss: 0.329, acc: 0.987 | val_loss: 0.333, acc: 0.980\n",
            "EPOCH  6 | train_loss: 0.326, acc: 0.991 | val_loss: 0.333, acc: 0.981\n",
            "EPOCH  7 | train_loss: 0.324, acc: 0.991 | val_loss: 0.332, acc: 0.980\n",
            "EPOCH  8 | train_loss: 0.322, acc: 0.993 | val_loss: 0.332, acc: 0.980\n",
            "EPOCH  9 | train_loss: 0.321, acc: 0.994 | val_loss: 0.332, acc: 0.980\n",
            "EPOCH 10 | train_loss: 0.321, acc: 0.994 | val_loss: 0.332, acc: 0.978\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        },
        "id": "nzchdxbVYjKC",
        "outputId": "b141bbc0-f8b0-4e76-bfa7-c924203f99d9"
      },
      "source": [
        "def training_lossplots(title, train_losses, train_accs, val_losses, val_accs):\n",
        "  plt.subplot(2, 1, 1)\n",
        "  epochs = np.arange(1, NUM_EPOCHS + 1)\n",
        "  plt.plot(epochs, train_losses, label=\"train\")\n",
        "  plt.plot(epochs, val_losses, label=\"valid\")\n",
        "  plt.xlabel(\"epoch\")\n",
        "  plt.ylabel(\"loss\")\n",
        "\n",
        "  plt.subplot(2, 1, 2)\n",
        "  plt.plot(epochs, train_accs, label=\"train\")\n",
        "  plt.plot(epochs, val_accs, label=\"valid\")\n",
        "  plt.xlabel(\"epoch\")\n",
        "  plt.ylabel(\"accuracy\")\n",
        "\n",
        "  plt.suptitle(title)\n",
        "  _ = plt.show()\n",
        "\n",
        "\n",
        "training_lossplots(\"Baseline\", train_losses, train_accs, val_losses, val_accs)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEjCAYAAAA/ugbCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xU9b3/8ddnZrZQliZF+iJSNYoKWIjGlkTFiF5NxMR6E01X703TxJvkpt3k3ugvuTdqgiTGqMEWkmBJbLFhFFlUUEAE6SxIL8uyZWY+vz/O2WVYFxjYmT27zPv5eMzjtO8557OjnPecbu6OiIgUrljUBYiISLQUBCIiBU5BICJS4BQEIiIFTkEgIlLgFAQiIgVOQSCSA2Z2tZnNzBiuMrMjoqxJJFsKAjkkmdlyM9sVbpC3mNnjZjawtdbv7p3dfWlrrU+kJRQEcij7hLt3BvoC7wP/F3E9Im2SgkAOee5eAzwCjAYws4lm9oaZbTezVWb2/Ya2ZlZqZveZ2SYz22pms82sTzitq5n91szWmtkaM/uRmcWbW6eZuZkdGfb/3sxuD/dKdpjZLDMbmtF2pJk9bWabzWyRmX0qj1+HyAcoCOSQZ2YdgUuBV8NRO4ErgW7AROCLZnZhOO0qoCswEDgM+AKwK5z2eyAJHAkcB3wM+FyWZUwG/hPoDiwBfhzW1gl4Gvgj0Dtsd4eZjT7wv1Tk4CgI5FD2FzPbCmwDPgr8D4C7P+/ub7l72t3nAdOAj4Tz1BMEwJHunnL3Oe6+PdwrOA+40d13uvt64P8RbLiz8Wd3f83dk8D9wJhw/PnAcne/292T7v4G8Cfgky3+60WylIi6AJE8utDdnwkP30wCXgh/aQ8GfgocDRQDJcDD4Tz3EuwNPGBm3YD7gO+E8xQBa82sYfkxYFWWtazL6K8GOof9g4ETw8BqkAjrEGkV2iOQQ174y346kAI+THAYZgYw0N27Ar8GLGxb7+7/6e6jgVMIfrFfSbDBrwV6unu38NPF3Y9qYXmrgBcyltktvOLoiy1crkjWFARyyLPAJILj8wuBMmCzu9eY2Xjg0xltzzCzD4V7EdsJDhWl3X0t8BRwq5l1MbOYmQ01s498cI0H5DFguJldYWZF4WecmY1q4XJFsqYgkEPZo2ZWRbBB/zFwlbvPB74E/MDMdgDfBR7KmOdwgiuMthOExgvsPkxzJcGhpAXAlrBd35YU6O47CE46TwYqCQ4h/YzgcJVIqzC9mEZEpLBpj0BEpMApCERECpyCQESkwCkIREQKnIJARKTAKQhERAqcgkBEpMApCERECpyCQESkwCkIREQKnIJARKTAKQhERAqcgkBEpMApCERECpyCQESkwCkIREQKnIJARKTAJaIu4ED17NnTy8vLoy5DRKRdmTNnzkZ379XctHYXBOXl5VRUVERdhohIu2JmK/Y2TYeGREQKXEEFQU19KuoSRETanIIJgntfWc4ZP3+eqtpk1KWIiLQpBRMER/fvytptNdz5/JKoSxERaVMKJgiOG9SdSWP6cddLy1i9pTrqckRE2oyCCQKAb54zEgN+9vdFUZciItJmFFQQ9O/WgetOO4JH51YyZ8WWqMsREWkTCioIAL7wkaH0KivhR48vwN2jLkdEJHIFFwSdShJ842MjeGPlVmbMrYy6HBGRyBVcEABcfMIARvftws/+9o7uLRCRgleQQRCPGbecP4rKbTX8duayqMsREYlUQQYBwClDe/LR0X2447klrN9RE3U5IiKRKdggAPj2eaOoS6W59cl3oy5FRCQyBR0EQ3p24sqTy3lozioWVG6PuhwRkUgUdBAAXH/mMLp2KNLlpCJSsAo+CLp2LOLGs4bxz/c28czC9VGXIyLS6go+CAA+c9JghvbqxE+eWEhdMh11OSIirUpBABTFY3xn4iiWbdzJfa/u9SU+IiKHpLwGgZmdY2aLzGyJmd20lzafMrMFZjbfzP6Yz3r25YwRvfnwkT355bOL2VpdF1UZIiKtLm9BYGZx4HbgXGA0cJmZjW7SZhhwMzDB3Y8CbsxXPftjFtxktqOmnl88sziqMkREWl0+9wjGA0vcfam71wEPAJOatLkWuN3dtwC4e6Rna0ce3oVLxw3ivldX8N6GqihLERFpNfkMgv7Aqozh1eG4TMOB4Wb2spm9ambnNLcgM7vOzCrMrGLDhg15Kjfw7x8dTmlRnP96YmFe1yMi0lZEfbI4AQwDTgcuA+4ys25NG7n7FHcf6+5je/XqldeCepWV8KUzhvLMwvW8vGRjXtclItIW5DMI1gADM4YHhOMyrQZmuHu9uy8D3iUIhkj964QhDOjegR8+toBUWjeZicihLZ9BMBsYZmZDzKwYmAzMaNLmLwR7A5hZT4JDRUvzWFNWSovi3HTuSN5Zt4OHK1btfwYRkXYsb0Hg7kngK8CTwELgIXefb2Y/MLMLwmZPApvMbAHwHPANd9+Ur5oOxMQP9eWEwd35+VPvUlWbjLocEZG8sfb2fJ2xY8d6RUVFq6zrzVVbufD2l/nS6UP55jkjW2WdIiL5YGZz3H1sc9OiPlncpo0Z2I0Lx/Rj6sxlrN5SHXU5IiJ5oSDYj2+eMxIDfvb3RVGXIiKSFwqC/ejXrQPXnXYEj86tZM6KLVGXIyKScwqCLHzhI0PpXVbCDx9bQFqXk4rIIUZBkIVOJQm+/vERvLlqK4/Oq4y6HBGRnFIQZOni4wcwum8Xfva3d6ipT0VdjohIzigIshSPGf9x/mgqt9Uw9aXI73kTEckZBcEBOHnoYXxsdB/ueP491u+oibocEZGcUBAcoJvPG0V9Ks2tT74bdSkiIjmhIDhAQ3p24sqTy3lozirmV26LuhwRkRYrrCBI5+bF9NefOYxuHYr40WMLaW+P6BARaapwguDt6TD1TKjZ3uJFde1YxI1nD+eVpZt4ZmGkL1UTEWmxwgmCjj1g7TyYfi2kW37556dPHMTQXp34yRMLqUvmZk9DRCQKhRMER5wO5/4M3v07PPufLV5cUTzGdyaOYtnGndz76ooWL09EJCpZBYGZ3WBmXSzwWzN73cw+lu/icm78tTDuc/DyL+HNaS1e3BkjenPqsJ7877OL2Vpdl4MCRURaX7Z7BP/q7tuBjwHdgSuAn+atqnw656cw5DR49HpY9VqLFmVmfGfiKHbU1POLZxbnqEARkdaVbRBY2D0PuNfd52eM2/tMZueY2SIzW2JmNzUz/Woz22Bmb4afz2Vf+kGKF8En74Eu/eGBT8PWlr2KcuThXbh03CDue3UF722oylGRIiKtJ9sgmGNmTxEEwZNmVgbs8wypmcWB24FzgdHAZWY2upmmD7r7mPAz9QBqP3gde8CnH4RkLUy7DGpbtgH/948Op7Qozk8eX5ijAkVEWk+2QfBZ4CZgnLtXA0XANfuZZzywxN2Xunsd8AAw6aArzbVeI+CSu2H9fPjLF1p0j0GvshK+fMaRPPvOemYu3pjDIkVE8i/bIDgZWOTuW83scuAWYH+31fYHMo+7rA7HNXWxmc0zs0fMbGCW9eTGsLPhYz+GhY/C8z9p0aKumVDOgO4d+NHjC0jpnQUi0o5kGwR3AtVmdizwNeA94A85WP+jQLm7HwM8DdzTXCMzu87MKsysYsOGDTlYbYaTvgjHXQEv/g+89chBL6a0KM5N547knXU7eKiiZecdRERaU7ZBkPTgWQqTgF+5++1A2X7mWQNk/sIfEI5r5O6b3L02HJwKnNDcgtx9iruPdfexvXr1yrLkLJnBxNtg0Cnw1y/DmjkHvaiJH+rL2MHdufWpRVTVJnNYpIhI/mQbBDvM7GaCy0YfN7MYwXmCfZkNDDOzIWZWDEwGZmQ2MLO+GYMXANGcbU0Uw6X3QufeMO3TsP3g3kJmZtxy/mg2VtVxx3NLclykiEh+ZBsElwK1BPcTrCP4df8/+5rB3ZPAV4AnCTbwD7n7fDP7gZldEDa73szmm9lc4Hrg6oP4G3KjU0+47EGoqwouK62rPqjFjBnYjQvH9GPqzGWs2nxwyxARaU2W7dMzzawPMC4cfM3dI3na2tixY72ioiJ/K1j0t+CS0qMugkt+Fxw6OkCVW3dx5q3Pc/aoPvzq08fnoUgRkQNjZnPcfWxz07J9xMSngNeATwKfAmaZ2SW5K7ENGXEunP19mD89OIF8EPp168B1px7BY/PWMmfFlpyWJyKSa9keGvoOwT0EV7n7lQT3CPxH/sqK2IQb4NjL4Lkfw4K/HtQiPv+RofQuK+GHjy0grctJRaQNyzYIYk0OBW06gHnbHzM4/xcwYDz8+Quwdu4BL6JTSYKvf3wEb67ayqPzDu7ks4hIa8h2Y/53M3syfDbQ1cDjwBP5K6sNKCqFS++DDj2CcwY73j/gRVxy/ACO6teFn/3tHWrqW/4OBBGRfMgqCNz9G8AU4JjwM8Xdv5XPwtqEsj5w2TTYtQUe/AzU1xzQ7LGYccvE0VRuq2HqS0vzVKSISMtkfXjH3f/k7v8efv6cz6LalL7HwEW/gdWz4dEb4ADfUXzy0MP42Og+3PH8e6zffmBBIiLSGvYZBGa2w8y2N/PZYWYtf/lvezH6AjjjFpj3ALz8iwOe/dvnjaI+lebnTy3KQ3EiIi2zzyBw9zJ379LMp8zdu7RWkW3CaV+Hoy+GZ/4T3jmw0yPlPTtx1cnlPDxnNfMr9/esPhGR1nXoXvmTa2Yw6XboNwamXwvvzz+g2b961jC6dSjiR48tJNub+EREWoOC4EAUdYDJ06CkDKZNhp3Zv3uga4cibjx7OK8s3cTTCw78CiQRkXxREByoLn1h8v1QtR4evByS2b+0/tMnDmJor07cNP0t/jRntfYMRKRNUBAcjP4nwIV3wMpX4PF/y/pKoqJ4jN9ccQKDenTkaw/P5ZO/foUFlYVzzl1E2iYFwcE6+mI47Zvwxn3w6h1Zz3Zk7zKmf/EU/vviY1i6cSfn/99LfH/GfLbtqs9jsSIie6cgaInTb4ZRF8BTt8Dip7OeLRYzPjVuIM997XQuP2kwf3hlOWfd+jwPV6zSc4lEpNUpCFoiFoOLfg19joJH/hU2HNh9Al07FvGDSUcz4ysfZlCPjnzjkXlc8ut/8vYaXWIqIq1HQdBSxZ2CK4kSpfDHS6F68wEv4uj+XXnkC6fwP5ccw4pN1Vzwq5l8969vs61ah4tEJP8UBLnQbWBwJdH2NfDQlZA68A14LGZ8cuxA/vH107ny5HLue3UFZ976PA/N1uEiEcmvvAaBmZ1jZovMbImZ3bSPdhebmZtZs2/PaRcGjocL/g+WvwR/O/jn8XXtUMT3LziKR7/6YYb07MQ3/zSPi3W4SETyKG9BYGZx4HbgXGA0cJmZjW6mXRlwAzArX7W0mmMnw4QboeK38NpdLVrUUf268vAXTubWTx7Lqs3VfOJXM7nlL2+xtTr7+xZERLKRzz2C8cASd1/q7nXAA8CkZtr9EPgZcGg8mvOs78Lwc4O9gveea9GizIyLTxjAs187natOLuePs1Zy5q0v8ODslTpcJCI5k88g6A+syhheHY5rZGbHAwPd/fF9LcjMrjOzCjOr2LBhQ+4rzaVYHC6+C3qNgIevgk3vtXiRDYeLHr/+VIb26sS3/vQW/3LnP3lrtQ4XiUjLRXay2MxiwG3A1/bX1t2nuPtYdx/bq1ev/BfXUiVlwQttYongSqJdW3Oy2FF9u/DQ50/mtk8dy+otu7jg9pl8+89vsWWnDheJyMHLZxCsAQZmDA8IxzUoA44Gnjez5cBJwIx2fcI4U/fy4FWXW5bDI9dAKpmTxZoZ/3L8AP7x9Y9wzSlDeHD2Ks689XmmvabDRSJycPIZBLOBYWY2xMyKgcnAjIaJ7r7N3Xu6e7m7lwOvAhe4e0Uea2pdg0+B82+D9/4R3H2cQ11Ki/juJ0bz+PUfZlifMm6e/hYX3fEyc1flZu9DRApH3oLA3ZPAV4AngYXAQ+4+38x+YGYX5Gu9bc7xV8JJX4ZZd8Kc3+d88SMP78KD153ELy4dQ+W2Gi6842Vunq7DRSKSPWtvj0IeO3asV1S0s52GVBKmXQpLn4cr/wrlH87LanbU1PPLZxZz9z+XU1aa4BsfH8HkcYOIxywv6xOR9sPM5rh7s4fedWdxa4gn4JLfQY8j4MErYPOyvKymrLSIW84fzRPXn8qIPmV8589vc9EdL/OmDheJyD5oj6A1bXoP7joTPA0DxsHAE2HgOOg/Fkpz+wpod2fG3Ep+/PhCNlTVcunYgXzznJH06FSc0/WISPuwrz0CBUFrWzsXKu6GVa/B+gWAAwa9RwehMGB8EBCHDQ3ek9xCVbVJ/vfZxfxu5jI6lQSHiy4br8NFIoVGQdBW1WyHNRWwajasfi3o1oY3iXXoETy/aMC4oNvveCjpfNCrWvz+Dr771/m8snQTR/fvwg8mHc3xg7rn6A8RkbZOQdBepNOw8d0wFGYFwbAxfMeBxYP3HgwM9xgGjAvuVTiAvQZ357F5a/nR4wt4f3stJw7pwai+XRjep4wRh5cxvE9nykqL8vO3iUikFATtWfVmWDMnOJS0albQX1cVTOvUa3coDDwR+o2Bog77XeTO2iS/fuE9Xlq8kXff30F1XapxWv9uHcJQKGPE4Z0Z0acLQ3t3oiQRz9dfKCKtQEFwKEmnYP3CIBRWzw66m5cG02IJOPyY3SehB4yHrgP2udeQTjtrtu5i0bodLHp/B+++v4NF63bw3oYq6lPB/xvxmFF+WEdGHt6lMSCG9ylj8GGddK5BpJ1QEBzqdm7cHQqrZkPl61BfHUwr67fnSei+x0CiZL+LrE+lWb5xJ4vCYFi0LgiJFZurafhfpiQRY1ifIBRGNu5FlHF4l1IsBye6RSR3FASFJlUP77+dcRJ6FmxdGUyLl0DfY4NzDT2OCB6QV9wJijsHn5LO4XAnKC4L7oHIsKsuxeL1u4PhnbD7/vbaxjZdShMZh5fKGBF2u3XUpasiUVEQCOxYF5xnaLg6qfINSNXuf754SUY4hKHRzPAuK+X9miIqq2Os2Blj2TZj0RZnQ22CKjpQ7aV07NyFwYf3ZMThXRgeBsSwPp3pWJzYfx0i0iIKAvmgZB3s2hKceK6rgtoqqNsJdTuCbrPDDZ9mpqeye7ZRGqPaS6milJ1eyk5KSceKwBLBuxxiCSyewOJFxOKJ8FNEPJEgniginigikSiiqKiIRFExRUVFFBUVU1xURCxe1LiM3Z+mw82NC4fNAAOLZfRndC0W9tPMuKbt97aMfS03c9re7GXawcyTzXwN0xtqy+xv2j2QaYfSocPMbWhjv+dnOF4CiYPbs95XEOinWKFKFENZH6BPbpaXrGsmKDKHg8CI1VXRsbYK37EV376N+M5tpJJ1eCqJp5OQroFkEuqSmKewdJKYp0iQJm5BN0aaBCniYTdGipil9l+jtEFZBEiz9vEDdp8/bg9mvr1slKMw8TYY99mcL1ZBILmRKIZED+jYY79NYwQvoyjLctGptLOzLklVTZKttUl21CSpqg2Gq2rrdw/vqmNXTS07a2vZVVNLTW0wXFtXx67aWmpra4mRJk4YLKQaA6WIFGaOhf/IY6QxIIZTFDcSMaMoBkUxSMQIx0FRrKEbjE/EdrdNxDwcDsdb0CYeY3e/GYk4JMyJW9DODGJmxM2wWNAfs+B7i4XT42bEzDAzYuZB27BdPGMZQVsap8XM9rl5beQOeJMuzYzb3zQOcr7MjW4r7wXtcx7bz/C+2uxvehbDA8fvo/aDpyCQNi8eM7qUFtGlhTe7uTu76lNU1STZ0Rgku4OlPpWmPpWmLpmmPuW7h1Np6pN7DteknPpkmmQ6TV3Y3zi9fnfb+mQ4PZXO+ER7ODYeC0ImHoZWrEk33vBpCJKYhWEUjI/FwsCxoD/osnt6Y9swxBraGI39QVgFbRpCrGG9DUHX2CZs19imaV3NLL+5ehpCMhaO+2C75v6GYHzDVXAWHhm0MFCD4d3TGttkBG5Dm4Z5d+eCfWB5me0bj6I1jjdKEjHyccungkAKhpnRsThBx+IEvSOsw933CJo9QieZJuVOKh18kmknHXZT6T3HNw67k0qnSaactO+nbeZy3Ummgnkb1plMeWN/2oP7TFJh23RDXR78DZnLq0sFXfeG+TPauONOY9vdy2KP5abDdaa8oT8Ylt1+dOHRXH7S4JwvV0Eg0srMjOKEUZzQU+D3x8MwaAyLcHh3cHwwaDwMk8xgSqebLiMjiMKwagi+5tblYS1BTeAE62k4gtUw3cOBsC9su+c8De1x3z2tYdl7zLM7BRvmP25Qt7x8zwoCEWmzgkNDEMco0lNO8kY/SURECpyCQESkwLW7G8rMbAOwIuo6WqgnsDHqItoQfR+76bvYk76PPbXk+xjs7r2am9DuguBQYGYVe7vDrxDp+9hN38We9H3sKV/fhw4NiYgUOAWBiEiBUxBEY0rUBbQx+j5203exJ30fe8rL96FzBCIiBU57BCIiBU5BICJS4BQErcjMBprZc2a2wMzmm9kNUdcUNTOLm9kbZvZY1LVEzcy6mdkjZvaOmS00s5OjrilKZvZv4b+Tt81smpmVRl1TazGz35nZejN7O2NcDzN72swWh93uuVqfgqB1JYGvufto4CTgy2Y2OuKaonYDsDDqItqIXwJ/d/eRwLEU8PdiZv2B64Gx7n40EAcmR1tVq/o9cE6TcTcBz7r7MODZcDgnFAStyN3XuvvrYf8Ogn/o/aOtKjpmNgCYCEyNupaomVlX4DTgtwDuXufuW6OtKnIJoIOZJYCOQGXE9bQad38R2Nxk9CTgnrD/HuDCXK1PQRARMysHjgNmRVtJpH4BfBNIR11IGzAE2ADcHR4qm2pmnaIuKiruvgb4ObASWAtsc/enoq0qcn3cfW3Yv46cvWdWQRAJM+sM/Am40d23R11PFMzsfGC9u8+JupY2IgEcD9zp7scBO8nhrn97Ex7/nkQQkP2ATmZ2ebRVtR3uDe/0zA0FQSszsyKCELjf3adHXU+EJgAXmNly4AHgTDO7L9qSIrUaWO3uDXuIjxAEQ6E6G1jm7hvcvR6YDpwScU1Re9/M+gKE3fW5WrCCoBVZ8OLT3wIL3f22qOuJkrvf7O4D3L2c4CTgP9y9YH/xufs6YJWZjQhHnQUsiLCkqK0ETjKzjuG/m7Mo4JPnoRnAVWH/VcBfc7VgBUHrmgBcQfDr983wc17URUmb8VXgfjObB4wBfhJxPZEJ94weAV4H3iLYVhXM4ybMbBrwCjDCzFab2WeBnwIfNbPFBHtMP83Z+vSICRGRwqY9AhGRAqcgEBEpcAoCEZECl8jXgs3sd0DDteJHNzPdCG6pPw+oBq5uuOt2X3r27Onl5eU5rlZE5NA2Z86cjXt7Z3HegoDgWRm/Av6wl+nnAsPCz4nAnWF3n8rLy6moqMhRiSIihcHMVuxtWt4ODe3lWRmZJgF/8MCrQLeGmyVERKT15HOPYH/6A6syhleH49Y2bWhm1wHXAQwaNKhVihOR/HJ3Umkn5Y47u/vTkHInHX7c+UDXHRwn7cFy0g40DrPHvNm09YxpmW0b5s0Vw1o0//DDy+jfrUOOqtktyiDImrtPIbyZZOzYsbrxQXImnXbqUmnqU2mSKac+HXT36M/o1ofTPjAu7KbSTjK157hkykml09Q3mZZKezh997RUsJUi8/aezP/hG+772XPc3tvRTLugbTPryehv2Dimwo21e7CRTqeDaal0w4aa3f3pcLixv6Fd8/PIgfvRhUdz+UmDc77cKINgDTAwY3hAOE4OcXXJNLvqUlTXJ9lVl6I2maYumaYu1aSbMb4+HJfZtr5J29pmxjW7zIx29anW2yAVxY1ELEYibhTFYyRiFnzi4bhYjHjMsPBHo2X8eMz8Jdk4PXPh4Uj74Kg9xlvGyObaNq7HIB4zimJGzIJPvLE/mBYLh+NGRr8Ri9GkvREPxwXtCNvtZbkZ/YTLNIOYBfWZBX9HOJlYWHyssV0wDRra7G5rFvyFmW2NzDYfbNvQbalc/J+Wj70BiDYIZgBfMbMHCE4Sb8t4xKpEKJ12dtWnqK5LNW6wG/vrUuyqT7GrLhi3t/EN8wfTk3ssL5mDX4OJGHRKOJ3iqeCTCLodYym6xpN0jKXoEEvRIZaktCRJaSxJB0tSYklKYklKSFJsSUqop5gkRSSJkyZuTsx8dz9OjDRxgvExTwVdnJinG6ebBcNGGiNoZ3hjP57G3MHTEA7jaUind/d7KvhZbrHgE4uHW6Rw2OJNpsXC6c2N39+0Jp8PTDPIyeZvHxoePp7K72pyovG/Q9Pvc2/j9/cdZ8z7gfEN7ZsZxxHk8OnTjfJ5+eg04HSgp5mtBr4HFAG4+6+BJwguHV1CcPnoNfmq5ZDjDql6SO6C+vCTrIH6aqivCcfXhOMb+qshWUOqrpq1GzezZsMWqmvqgkMZ6TTJdMMhDM9qt92ATuHHDBIxIx7+ok3ELRwOfgHH40aiyEiUNYwL2ga/iCFBioTXk/A64ul64mE3lq4LPqk6LF2HpeqwVC0kwy4EP7OS4eegGSRKgn9w+/3H3WTjmu1GOZbYz4Y3YwPsvjsUGgMj9cHA8DSkmo5v6Pdm5tlb+KSbnyYZmnyfOTxvcEAm3gbjPpvzxeYtCNz9sv1Md+DL+Vp/m7V2Hqyp2L0Bb9yI72p2w/3B6eG0g/yHmiJBmRczxEqIxRK7d4ENrKhhtxtiDbvgZEzP7DcjRsMhhSx2ndPs/fUz8QTESyBRHHZLIN4x7BaH3czpTbuZ7fbVfi/tGjb+ItnaZ0jvL4gzQ77p+HSTYG6yjJ7D8/LntIuTxe1eKgnvPAazfg0rX2ky0aCoIxSVQqIDFHXYs79Dd0iUhuM7hONLm/R3/ECbdLyEBZvqefrd7TzxzlZWV0G8uANnjurLJ47tx2nDe1KSiEfydYi0ew17f8QhXhR1NS2mIMin6s3w+j3w2lTYvhq6DYKP/RhGT4KSsmCjHS/O2a9Rd+ftNdt5dF4lj82tpHJbDcWJGGeOKOfGY/tx5sjedCjWxl9E9qQgyIf3FwS//uc9FBzqKT8VzvtvGH5OcBgixxat28Gjcyt5dF4lKzZVUxQ3Th3Wi2+cM4KzR/WhrH8b3BsAABGuSURBVLT9/2IRkfxREORKOgXvPgmz7oRlLwaHao75FJz4BehzVM5Xt3RDFY/NW8ujcytZvL6KmMEpQ3vypdOH8vGjDqdbx+Kcr1NEDk0Kgpaq2QZv3AevTYEty6FLfzjre3D8VdDpsJyuatXmah5/K9j4z6/cjhmMK+/BDycdxTlH96VXWUlO1ycihUFBcLA2LoZZv4E3/wj1O2HgSXD292Hk+Tk9ebRuWw2Pv7WWx+ZV8sbKrQCMGdiNWyaOYuIxfenbNT83mIhI4VAQHIh0Gt77R3D4Z8kzwYneoy+GEz8P/Y7L2Wo2VdXyxNvreGxuJa8t34w7jO7bhW+eM4JPHNOPgT065mxdIiIKgmzUVsHcacEewKbF0LkPnP5tGHsNdO6dk1Vsq67nyfnreHReJf98bxOptDO0VyduOGsY5x/TjyN7d87JekREmlIQ7MvmZTB7Krx+L9RuC371XzQFjroouEmphapqkzyz4H0enVvJi4s3UJ9yBvXoyOdPO4JPHNuPkYeX7fFsGBGRfFAQNOUeXPUz6zew6Ingcs/Rk4KrfwaMa/E1/7XJFM8uXM+jcyv5xzvrqU2m6du1lKtPKef8Y/pxzICu2viLSKtSEDSoq4a3Hg4CYP186HgYnPq14LkeXfrlbDXfemQef3mzkp6di5k8biDnH9uPEwZ1JxbTxl9EoqEg2LY6OPwz5/ewawv0ORou+BV86JLgzt8cWrmpmhlzK7n6lHJumTiKRDxvL4gTEclaYQaBO6yaBa/eCQsfBRxGTgwO/wyekLcHkP3u5WXEY8YXTx+qEBCRNqOwgiBZC29PDy7/XDsXSrvCyV+CcddC99y/9SfT1uo6Hpy9iklj+tOnS2le1yUiciAKJwjmPQRPfht2boCeI4Lneh87GYo7tcrq75+1kl31Ka499YhWWZ+ISLYKJwhKu0K/4+GkL8ARZ7Tq8+drkynufnk5HxneixGHl7XaekVEslE4QTD848EnAn99o5KNVbVcd5r2BkSk7dEZyzxLp50pLy1ldN8unDI0tw+hExHJBQVBnr3w7gaWrK/iutOO0I1iItImKQjybMqLS+nbtZSJx/SNuhQRkWYpCPLo7TXbeGXpJq6ZUE6R7hsQkTYqq62TmU03s4lmpq3ZAbjrpaV0LkkwefygqEsREdmrbDfsdwCfBhab2U/NbEQeazokrNm6i8fmreWy8QPponcGi0gbllUQuPsz7v4Z4HhgOfCMmf3TzK4xM23lmnH3zGUYcM2EIVGXIiKyT1kf6jGzw4Crgc8BbwC/JAiGp/NSWTu2bVc9015byfnH9KVfN71KUkTatqxuKDOzPwMjgHuBT7j72nDSg2ZWka/i2qsHXlvJzroUn9PjJESkHcj2zuL/dffnmpvg7mNzWE+7V5dMc/fLy5lw5GEc3b9r1OWIiOxXtoeGRptZt4YBM+tuZl/KU03t2mPzKlm3vUZ7AyLSbmQbBNe6+9aGAXffAlybn5LaL3dnyotLGda7M6cP7xV1OSIiWck2COKW8XwEM4sDLX97+yFm5pKNvLNuB9fqcRIi0o5kGwR/JzgxfJaZnQVMC8ftk5mdY2aLzGyJmd3UzPRBZvacmb1hZvPM7LwDK79tueulZfQqK2HSmNy941hEJN+yPVn8LeDzwBfD4aeBqfuaIdxruB34KLAamG1mM9x9QUazW4CH3P1OMxsNPAGUZ19+27Fw7XZefHcD3/j4CEoS8ajLERHJWlZB4O5p4M7wk63xwBJ3XwpgZg8Ak4DMIHCgS9jfFag8gOW3KVNfWkbH4jifOVGPkxCR9iXb+wiGAf8FjAYaX7jr7vu6NKY/sCpjeDVwYpM23weeMrOvAp2As7Opp61Zt62GGXPX8JkTB9Oto06diEj7ku05grsJ9gaSwBnAH4D7crD+y4Dfu/sA4Dzg3uYebGdm15lZhZlVbNiwIQerza3f/3M5qbTz2Q/rcRIi0v5kGwQd3P1ZwNx9hbt/H5i4n3nWAAMzhgeE4zJ9FngIwN1fIdjb6Nl0Qe4+xd3HuvvYXr3a1mWZVbVJ7p+1gnM/1JeBPTpGXY6IyAHLNghqw1/qi83sK2Z2EdB5P/PMBoaZ2RAzKwYmAzOatFkJnAVgZqMIgqDt/eTfhwdnr2JHTZJrdQOZiLRT2QbBDUBH4HrgBOBy4Kp9zeDuSeArwJPAQoKrg+ab2Q/M7IKw2deAa81sLsElqVe7ux/4nxGNZCrN72YuY3x5D8YM7Lb/GURE2qD9niwOLwO91N2/DlQB12S7cHd/guCS0Mxx383oXwBMyLraNuaJt9exZusuvn/BUVGXIiJy0Pa7R+DuKeDDrVBLu+Lu3PXiUo7o2YmzRvaOuhwRkYOW7Q1lb5jZDOBhYGfDSHefnpeq2oFZyzbz1ppt/OSiDxGL6XESItJ+ZRsEpcAm4MyMcQ4UbBDc9eJSDutUzL8c3z/qUkREWiTbO4uzPi9QCJas38Gz76zn384eTmmRHichIu1btncW302wB7AHd//XnFfUDkx9aRkliRhXnDw46lJERFos20NDj2X0lwIX0Y6fC9QS63fUMP31NXxy7AB6dNLjJESk/cv20NCfMofNbBowMy8VtXH3vrKC+nRaj5MQkUNGtjeUNTUMKLhrJqvrktz76go+OqoPR/Ta343VIiLtQ7bnCHaw5zmCdQTvKCgoj8xZzdbqeq47TY+TEJFDR7aHhsryXUhbl0o7U19axnGDunHC4O5RlyMikjNZHRoys4vMrGvGcDczuzB/ZbU9Ty9Yx8rN1Vx3qt5HLCKHlmzPEXzP3bc1DLj7VuB7+SmpbZry4lIG9ejIx446POpSRERyKtsgaK5dtpeetntzVmzm9ZVb+dypQ4jrcRIicojJNggqzOw2Mxsafm4D5uSzsLZkyotL6daxiEtOGBB1KSIiOZdtEHwVqAMeBB4AaoAv56uotmTZxp08teB9Lj9xMB2LC2YnSEQKSLZXDe0EbspzLW3Sb2cupSgW48pT9DgJETk0ZXvV0NNm1i1juLuZPZm/stqGTVW1PFyxmouO60/vstKoyxERyYtsDw31DK8UAsDdt1AAdxbf9+pKapNpPneqHichIoeubIMgbWaDGgbMrJxmnkZ6KKmpT/GHV5Zz5sjeDOtT8PfTicghLNuzn98BZprZC4ABpwLX5a2qNmD662vYtLOOa0/V4yRE5NCW7cniv5vZWIKN/xvAX4Bd+SwsSum0M3XmUj7UvysnHdEj6nJERPIq24fOfQ64ARgAvAmcBLzCnq+uPGT84531LN2wk/+97Dg9TkJEDnnZniO4ARgHrHD3M4DjgK37nqX9mvLSUvp368B5R+txEiJy6Ms2CGrcvQbAzErc/R1gRP7Kis6bq7by2rLNXDOhnET8YF/XICLSfmR7snh1eB/BX4CnzWwLsCJ/ZUXnrpeWUlaaYPL4QftvLCJyCMj2ZPFFYe/3zew5oCvw97xVFZFVm6v521trufa0I+hcosdJiEhhOOCtnbu/kI9C2oLfzlxGzIxrTtENZCJSOHQQPLS1uo6HKlZxwZh+HN5Vj5MQkcKhIAjdP2sl1XUp3UAmIgVHQQDUJlPc88/lnDqsJ6P6dom6HBGRVqUgAGa8Wcn6HbVcd5r2BkSk8OQ1CMzsHDNbZGZLzKzZ9xmY2afMbIGZzTezP+aznua4O3e9tJSRh5fx4SN7tvbqRUQil7drJM0sDtwOfBRYDcw2sxnuviCjzTDgZmCCu28xs1Z/tPUL727g3feruPWTx+pxEiJSkPK5RzAeWOLuS929juAVl5OatLkWuD18vwHuvj6P9TTrrpeW0qdLCZ84tl9rr1pEpE3IZxD0B1ZlDK8Ox2UaDgw3s5fN7FUzO6e5BZnZdWZWYWYVGzZsyFmBb6/ZxstLNnHNhCEUJ3S6REQKU9RbvwQwDDgduAy4K/OVmA3cfYq7j3X3sb169crZyqe+tJROxXEu0+MkRKSA5TMI1gADM4YHhOMyrQZmuHu9uy8D3iUIhryr3LqLR+etZfL4QXTtUNQaqxQRaZPyGQSzgWFmNsTMioHJwIwmbf5CsDeAmfUkOFS0NI81Nbr75WUAXDOhvDVWJyLSZuUtCNw9CXwFeBJYCDzk7vPN7AdmdkHY7Elgk5ktAJ4DvuHum/JVU4PtNfVMe20VEz/UlwHdO+Z7dSIibVpeH7Hp7k8ATzQZ992Mfgf+Pfy0mgdfW0VVbVKPkxARIfqTxa2uPpXmdy8v46QjevChAV2jLkdEJHIFFwSPz1vL2m01epyEiEiooILA3Zny4lKO7N2Z04e3+k3MIiJtUkEFwT/f28SCtdu59tQhxGJ6nISICBRYEEx5cSk9O5cwaUzTG5xFRApXwQTBonU7eOHdDVx9ymBKi+JRlyMi0mYUTBA8NX8dHYrifObEwVGXIiLSpuT1PoK25KtnDeOi4/vTvVNx1KWIiLQpBbNHAOguYhGRZhRUEIiIyAcpCERECpwFj/tpP8xsA7Ai6jpaqCewMeoi2hB9H7vpu9iTvo89teT7GOzuzb7Qpd0FwaHAzCrcfWzUdbQV+j5203exJ30fe8rX96FDQyIiBU5BICJS4BQE0ZgSdQFtjL6P3fRd7Enfx57y8n3oHIGISIHTHoGISIFTELQiMxtoZs+Z2QIzm29mN0RdU9TMLG5mb5jZY1HXEjUz62Zmj5jZO2a20MxOjrqmKJnZv4X/Tt42s2lmVhp1Ta3FzH5nZuvN7O2McT3M7GkzWxx2u+dqfQqC1pUEvubuo4GTgC+b2eiIa4raDcDCqItoI34J/N3dRwLHUsDfi5n1B64Hxrr70UAcmBxtVa3q98A5TcbdBDzr7sOAZ8PhnFAQtCJ3X+vur4f9Owj+oRfsyxHMbAAwEZgadS1RM7OuwGnAbwHcvc7dt0ZbVeQSQAczSwAdgcqI62k17v4isLnJ6EnAPWH/PcCFuVqfgiAiZlYOHAfMiraSSP0C+CaQjrqQNmAIsAG4OzxUNtXMOkVdVFTcfQ3wc2AlsBbY5u5PRVtV5Pq4+9qwfx3QJ1cLVhBEwMw6A38CbnT37VHXEwUzOx9Y7+5zoq6ljUgAxwN3uvtxwE5yuOvf3oTHvycRBGQ/oJOZXR5tVW2HB5d75uySTwVBKzOzIoIQuN/dp0ddT4QmABeY2XLgAeBMM7sv2pIitRpY7e4Ne4iPEARDoTobWObuG9y9HpgOnBJxTVF738z6AoTd9blasIKgFZmZERwDXujut0VdT5Tc/WZ3H+Du5QQnAf/h7gX7i8/d1wGrzGxEOOosYEGEJUVtJXCSmXUM/92cRQGfPA/NAK4K+68C/pqrBSsIWtcE4AqCX79vhp/zoi5K2oyvAveb2TxgDPCTiOuJTLhn9AjwOvAWwbaqYO4yNrNpwCvACDNbbWafBX4KfNTMFhPsMf00Z+vTncUiIoVNewQiIgVOQSAiUuAUBCIiBU5BICJS4BQEIiIFTkEg0orM7HQ9aVXaGgWBiEiBUxCINMPMLjez18Kb/n4Tvjehysz+X/iM/GfNrFfYdoyZvWpm88zszw3PiTezI83sGTOba2avm9nQcPGdM947cH9456xIZBQEIk2Y2SjgUmCCu48BUsBngE5AhbsfBbwAfC+c5Q/At9z9GIK7YBvG3w/c7u7HEjwnp+HJkccBNwKjgSMI7jgXiUwi6gJE2qCzgBOA2eGP9Q4ED/hKAw+Gbe4DpofvEejm7i+E4+8BHjazMqC/u/8ZwN1rAMLlvebuq8PhN4FyYGb+/yyR5ikIRD7IgHvc/eY9Rpr9R5N2B/t8ltqM/hT6dygR06EhkQ96FrjEzHpD47tiBxP8e7kkbPNpYKa7bwO2mNmp4fgrgBfCN9CtNrMLw2WUmFnHVv0rRLKkXyIiTbj7AjO7BXjKzGJAPfBlgpfFjA+nrSc4jwDBI4F/HW7olwLXhOOvAH5jZj8Il/HJVvwzRLKmp4+KZMnMqty9c9R1iOSaDg2JiBQ47RGIiBQ47RGIiBQ4BYGISIFTEIiIFDgFgYhIgVMQiIgUOAWBiEiB+/+grm5Jnmb6GQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N9uwGr-bZa-g"
      },
      "source": [
        "def predict(test_dl, model):\n",
        "  model.eval()\n",
        "  predictions, labels = [], []\n",
        "  for data, labels_b in val_dl:\n",
        "    data, labels_b = data.to(dev), labels_b.to(dev)\n",
        "    with torch.no_grad():\n",
        "      logits = model(data)\n",
        "      _, preds = torch.max(logits.data, dim=1)\n",
        "      predictions.extend([p.item() for p in preds])\n",
        "      labels.extend([l.item() for l in labels_b])\n",
        "\n",
        "  return predictions, labels\n",
        "\n",
        "predictions, labels = predict(test_dl, clf)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rMJGq8oIaNtp",
        "outputId": "ae75bd67-95f4-459e-8485-152a98419116"
      },
      "source": [
        "accuracy = accuracy_score(predictions, labels)\n",
        "ACCURACY_STATS.append((1, \"baseline\", accuracy))\n",
        "\n",
        "print(\"accuracy: {:.3f}\".format(accuracy))\n",
        "print(\"confusion matrix\")\n",
        "print(confusion_matrix(predictions, labels))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy: 0.975\n",
            "confusion matrix\n",
            "[[480   6]\n",
            " [  8  63]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ExvOLSvWlzx_"
      },
      "source": [
        "## Class Weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ENaqpYNlmPXe",
        "outputId": "33f2f858-57f1-4408-aa99-94e6b41a0153"
      },
      "source": [
        "num_major = len(spam_train_df[spam_train_df[\"label\"] == \"ham\"])\n",
        "num_minor = len(spam_train_df[spam_train_df[\"label\"] == \"spam\"])\n",
        "\n",
        "weight_major = 1 / num_major\n",
        "weight_minor = 1 / num_minor\n",
        "total_weight = weight_major + weight_minor\n",
        "class_weights = torch.tensor([weight_major / total_weight, weight_minor / total_weight])\n",
        "class_weights = class_weights.to(dev)\n",
        "\n",
        "class_weights"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0.1397, 0.8603], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "olXM9d-Tl16w"
      },
      "source": [
        "clf = SpamClassifier(CLF_LAYER_DIMS)\n",
        "clf = clf.to(dev)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss(weight=class_weights)\n",
        "optimizer = torch.optim.AdamW(clf.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cWpN9OSgnBC6",
        "outputId": "d3a8b450-153b-45ce-897f-050b904fce16"
      },
      "source": [
        "train_losses, train_accs, val_losses, val_accs = [], [], [], []\n",
        "for epoch in range(1, NUM_EPOCHS + 1):\n",
        "  train_loss, train_acc = train(train_dl, clf, loss_fn, optimizer)\n",
        "  val_loss, val_acc = validate(val_dl, clf, loss_fn) \n",
        "  print(\"EPOCH {:2d} | train_loss: {:.3f}, acc: {:.3f} | val_loss: {:.3f}, acc: {:.3f}\"\n",
        "    .format(epoch, train_loss, train_acc, val_loss, val_acc))\n",
        "  train_losses.append(train_loss)\n",
        "  train_accs.append(train_acc)\n",
        "  val_losses.append(val_loss)\n",
        "  val_accs.append(val_acc)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH  1 | train_loss: 0.623, acc: 0.920 | val_loss: 0.505, acc: 0.961\n",
            "EPOCH  2 | train_loss: 0.408, acc: 0.966 | val_loss: 0.353, acc: 0.981\n",
            "EPOCH  3 | train_loss: 0.350, acc: 0.977 | val_loss: 0.341, acc: 0.984\n",
            "EPOCH  4 | train_loss: 0.341, acc: 0.981 | val_loss: 0.337, acc: 0.981\n",
            "EPOCH  5 | train_loss: 0.337, acc: 0.987 | val_loss: 0.336, acc: 0.978\n",
            "EPOCH  6 | train_loss: 0.335, acc: 0.988 | val_loss: 0.337, acc: 0.980\n",
            "EPOCH  7 | train_loss: 0.330, acc: 0.989 | val_loss: 0.337, acc: 0.978\n",
            "EPOCH  8 | train_loss: 0.328, acc: 0.991 | val_loss: 0.338, acc: 0.983\n",
            "EPOCH  9 | train_loss: 0.328, acc: 0.992 | val_loss: 0.340, acc: 0.980\n",
            "EPOCH 10 | train_loss: 0.326, acc: 0.993 | val_loss: 0.340, acc: 0.980\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        },
        "id": "60MWswl4nOHi",
        "outputId": "1993dd00-a284-488c-de7a-2ab2027e9297"
      },
      "source": [
        "training_lossplots(\"Class Weights\", train_losses, train_accs, val_losses, val_accs)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEjCAYAAAAlhuZMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gc5bX48e/Zpm7ZluTeK9h0RAkkgDG9mZaEGkISSCMQCCSQSsgl4QYCubmBXLgQQighoQQMN2CM4+DADwwy3TYuuMpykau6tp3fHzOSVmvZ3rV3NSvpfJ5nn515d8rZsTVn3/edeUdUFWOMMSZVPq8DMMYY07NY4jDGGJMWSxzGGGPSYonDGGNMWixxGGOMSYslDmOMMWmxxGH6BBG5VUQe8zqOdInID0XkwRSX7ZHf0fQ8ljhMryEil4hIlYg0iMh6EXlJRD7rQRz3i8gfEuaDItK4i7Kjd7ctVf2lqn4tQ3GtEpGTMrEt07dZ4jC9gojcAPwW+CUwGBgF3AfM8CCcecBxCfOVwBrgc0llAAu6KyhjMsUSh+nxRKQUuA34tqo+q6qNqhpR1RdU9aZdrPOUiGwQkR0iMk9EpiZ8doaILBKRehFZJyI3uuXlIvKiiGwXka0i8m8R6epvaB6wv4iUu/OfA54EipLK3lTViIgME5FnRKRWRFaKyLUJsXRqfhKRL4nIahHZIiI/6aIWERKRP7uxLxSRSne9R3GS6Qtujez7IpIvIo+529ouIu+IyOD0jr7piyxxmN7gM0A+8Pc01nkJmAgMAt4FHk/47CHg66paAhwA/NMt/x5QDVTg1Gp+COw0Zo+qrgVW01HDOA74N/D/ksrmuYnnBeADYDgwHfiuiJyavF0RmYJTi7oUGAqUuuskOgcnSfUHZgK/d2O6HKfWc7aqFqvqr4Er3G2MBMqAbwDNuzhexrSzxGF6gzJgs6pGU11BVf+oqvWq2grcChzs1lwAIsAUEemnqttU9d2E8qHAaLdG82/d9WBvrwHHuYnhSOAtnOTRVnasu8wRQIWq3qaqYVVdAfwvcFEX27wQeEFVX1fVMPBTdk5cr6vqP1Q1BjwKHLybwxDBOXYTVDWmqgtUtW43yxsDWOIwvcMWoFxEAqksLCJ+EblDRD4VkTpglftRWzPSBcAZwGoReU1EPuOW3wksB14RkRUicvNudtPWz3EgsEJVm4DXE8oKgPnAaGCY21S0XUS249RkumoyGgasbZtxt7klaZkNCdNNQP5ujsujwCzgSRGpEZFfi0hwN9/JGMASh+kd3gRagXNTXP4SnE7zk3Caasa45QKgqu+o6gycZqzngL+55fWq+j1VHYfTJHSDiEzfxT7m4fzaPxOnpgGwEKdZ6EzgHVVtwUkEK1W1f8KrRFXP6GKb64ERbTMiUoBTY0hVp9qJW2v6uapOAY4BzgK+lMb2TB9licP0eKq6A6fZ5l4ROVdECt3LXU8XkV93sUoJTqLZAhTiXIkFgIiERORSESlV1QhQB8Tdz84SkQkiIsAOINb2WRcxLQc2AtfhJg63WWu+WzbPXfRtoF5EfiAiBW5t6AAROaKLzT4NnC0ix4hICKeJTVI+UE484xK+6zQROVBE/O73jOzq+xiTyBKH6RVU9TfADcCPgVqcX/LX4NQYkv0Zp/N6HbAIp/8h0eXAKrcZ6xs4ndHgdKa/CjTg1HLuU9W5uwlrHk5H+hsJZf/GqcnMc+OO4fzSPwRYCWwGHsSpCSV/x4XAd3A6v9e7cWzCSYKp+BXwY7dJ7EZgCE4yqgMW4/S5PJritkwfJvYgJ2N6JhEpBrYDE1V1pdfxmL7DahzG9CAicrbbFFcE3AV8REfnvjHdwhKHMT3LDKDGfU0ELtrNJcHGZIU1VRljjEmL1TiMMcakxRKHMcaYtFjiMMYYkxZLHMYYY9JiicMYY0xaLHEYY4xJiyUOY4wxabHEYYwxJi2WOIwxxqTFEocxxpi0WOIwxhiTFkscxhhj0mKJwxhjTFoscRhjjElLwOsAukN5ebmOGTPG6zCMMaZHWbBgwWZVrUgu7xOJY8yYMVRVVXkdhjHG9CgisrqrcmuqMsYYkxZLHHvQEol5HYIxxuQUSxy78d0n3+Objy3wOgxjjMkpljh2Y9KQEuYuqaVq1VavQzHGmJxhiWM3vnzMGMqL87hz1hJU1etwjDEmJ1ji2I3CUIBrpo1n/sqtvL58s9fhGGNMTsi5xCEip4nIEhFZLiI372KZL4jIIhFZKCJPZDOei48axfD+BVbrMMYYV04lDhHxA/cCpwNTgItFZErSMhOBW4BjVXUq8N1sxpQX8HPd9Il8WL2DVxZtzOaujDGmR8ipxAEcCSxX1RWqGgaeBGYkLXMVcK+qbgNQ1U3ZDur8w4YzrryI37yyhFjcah3GmL4t1xLHcGBtwny1W5ZoEjBJRN4QkbdE5LRsBxXw+7j+5Eks3djACx/UZHt3xhiT03ItcaQiAEwETgAuBv5XRPonLyQiV4tIlYhU1dbW7vNOzzxwKPsP7cfds5cSicX3eXvGGNNT5VriWAeMTJgf4ZYlqgZmqmpEVVcCS3ESSSeq+oCqVqpqZUXFTmN0pc3nE248ZRJrtjbxVFX1Pm/PGGN6qlxLHO8AE0VkrIiEgIuAmUnLPIdT20BEynGarlZ0R3An7jeIw0b153dzltlQJMaYPiunEoeqRoFrgFnAYuBvqrpQRG4TkXPcxWYBW0RkETAXuElVt3RHfCLCjadOZkNdC4+91eWgkcYY0+tJX7g3obKyUjM5rPplD85n0fo65n1/GsV5fWJkemNMHyQiC1S1Mrk8p2ocPcWNp05ma2OYh19f6XUoxhjT7Sxx7IVDRvbn5CmDeWDeCrY3hb0OxxhjupUljr30vVMm0RCOcv+8bumXN8aYnGGJYy/tN6Qf5xw8jIffWMmm+havwzHGmG5jiWMfXH/SJCIx5b65n3odijHGdBtLHPtgTHkRX6gcwePzV1O9rcnrcIwxpltY4thH3zlxIoLwuznLvA7FGGO6hSWOfTSsfwGXHT2apxdU82ltg9fhGGNM1lniyIBvTRtPftDPPbOXeh2KMcZknSWODCgvzuMrx47lxQ/Xs7Bmh9fhGGNMVlniyJCrjhtHv/wAd79itQ5jTO9miSNDSguCfP348cz5ZBMLVm/zOhxjjMkaSxwZdOWxYygvDnHnrE/oC4NHGmP6JkscGVQYCvDtaRN4a8VW3ljeLSO9G2NMt7PEkWGXHDWKYaX53PnKEqt1GGN6JUscGZYX8HPdSRP5YO12Zi/a6HU4xhiTcZY4suCCw0YwtryI37yylHjcah3GmN7FEkcWBPw+rj95Eks21vPChzVeh2OMMRlliSNLzjpwKPsNKeGe2UuJxOJeh2OMMRljiSNLfD7hxlMms2pLE08vqPY6HGOMyRhLHFk0ff9BHDKyP7+bs4yWSMzrcIwxJiMscWSRiPD9UyezfkcLj89f43U4xhiTETmXOETkNBFZIiLLReTmLj7/sojUisj77utrWQumbj3UvLdPmzhmQjnHjC/jvrnLaWyNZigwY4zxTk4lDhHxA/cCpwNTgItFZEoXi/5VVQ9xXw9mJRhVeOoK+Msl0Lhvd4HfeOpktjSGefiNlRkKzhhjvJNTiQM4EliuqitUNQw8CczwJBIROONOaNoMz3/LSSR76bBRAzhp/0HcP28FO5oiGQzSGGO6X64ljuHA2oT5arcs2QUi8qGIPC0iI7vakIhcLSJVIlJVW1u7d9EMPRhO+Q9Y+jK8dd/ebcP1vVMmU98S5f55n+7TdowxxmtZSxwicp2I9BPHQyLyroickoFNvwCMUdWDgNnAI10tpKoPqGqlqlZWVFTs/d6OvBomnwmzfwbr3t3rzew/tB9nHzyMh99YRW19697HY4wxHstmjeMrqloHnAIMAC4H7tjDOuuAxBrECLesnapuUdW2M++DwOGZCXcXRGDG76F4MDz9FWip2+tNXX/SRMKxOPfOXZ7BAI0xpntlM3GI+34G8KiqLkwo25V3gIkiMlZEQsBFwMxOGxUZmjB7DrA4Q/HuWuFAuPAh2L4GXvzuXvd3jKso5sLDRvDE/DWs296c4SCNMaZ7ZDNxLBCRV3ASxywRKQF2O/aGqkaBa4BZOAnhb6q6UERuE5Fz3MWuFZGFIvIBcC3w5ax9g0SjjoZpP4SPn4H3Ht3rzVx70kQAfvfqskxFZowx3Uqy9cwIEfEBhwArVHW7iAwERqjqh1nZ4W5UVlZqVVXVvm8oHoNHz4O1b8PVc2HQ/nu1mVtnLuTRt1Yz+/rjGFdRvO9xGWNMFojIAlWtTC7PZo3jM8ASN2lcBvwY2JHF/WWfzw/n/y/kFcNTV0K4aa828+1pEwj5fdxjtQ5jTA+UzcTxB6BJRA4Gvgd8Cvw5i/vrHiWD4bz7oXYxvLzTje0pqSjJ48pjx/DCBzUsqtn7znZjjPFCNhNHVJ12sBnA71X1XqAki/vrPhOmw7HfhXcfcfo89sLXjxtPSX6Au2cvyXBwxhiTXdlMHPUicgvOZbj/5/Z5BLO4v+514o9hxJHwwndha/pDiZQWBvn6ceN4dfEm3l2zLQsBGmNMdmQzcXwRaMW5n2MDzj0Zd2Zxf93LH3Qu0RWBp6+EaDjtTVx57FjKikLcNctqHcaYniNricNNFo8DpSJyFtCiqj2/jyNR/1Fwzu+dEXTn/Dzt1YvyAnxr2gT+36dbeGP55iwEaIwxmZfNIUe+ALwNfB74AjBfRC7M1v48M+UcOOJr8ObvYemstFe/9KhRDC3N585ZS8jWpdHGGJNJ2Wyq+hFwhKpeoapfwhn59idZ3J93TrkdBh8If/8G1NWktWp+0M+10yfy/trtzFm8KUsBGmNM5mQzcfhUNfFMuCXL+/NOMB8+/zBEW+GZq5wbBdNw4eEjGFNWyF2vLCEet1qHMSa3ZfNE/rKIzHKf2Pdl4P+Af2Rxf94qnwhn3gWrX4d56V0DEPT7uP7kSXyyoZ4XP1qfpQCNMSYzstk5fhPwAHCQ+3pAVX+Qrf3lhEMugYMugtf+E1a9ntaqZx80jP2GlHDP7KVEY7sd0ssYYzyV1aYjVX1GVW9wX3/P5r5yxpm/gQFj4ZmvQWPqV0r5fMINJ09i5eZGnnm3OosBGmPMvsl44hCRehGp6+JVLyK9f3yNvGL4/J+gaQs89820hmA/ecpgDh7Zn/96dRmt0fT6SYwxprtkPHGoaomq9uviVaKq/TK9v5w09CDnSqtlr8Cb96a8mohw0ymTqdnRwhPz12QxQGOM2Xu98yqnXHDkVbDfWfDqrbBuQcqrHTuhjM+MK+PeuctpCkezF58xxuwlSxzZ0vbI2ZIh7iNnUxtRXkS48dTJbG4I8/Abq7IbozHG7AVLHNlUMAAueAi2r3UGQ0yxv+Pw0QOYvt8g7n/tU3Y0R7IcpDHGpMcSR7aNOgpO/BEsfNYZhj1FN5wyibqWKP87b0UWgzPGmPRZ4ugOx14P406Al34AGxeltMrUYaWcddBQ/vjGSjY3tGY1PGOMSYclju7g88F5D0BeiTMEe4qPnL3+5Em0RGLcN/fTLAdojDGps8TRXUoGw/kPQO0SeDm1G+jHVxRz4eEjeOyt1dRsb85ygMYYk5qcSxwicpqILBGR5SKyy4d6i8gFIqIiUtmd8e2T8SfCZ6+Hd/8MHz2d0irXTp+Iolz58DvM/WSTDb1ujPFcTiUOEfED9wKnA1OAi0VkShfLlQDXAfO7N8IMmPZDGHmU+8jZPXd8jxhQyL2XHEZTJMqVf3qHz//Pm8xfsaUbAjXGmK7lVOLAeWbHclVdoaph4ElgRhfL/QL4T6ClO4PLCH8QLnjQ6fd4+ispPXL2lKlDmHPDCfzHuQewZmsTX3zgLb70x7f5qDq1e0OMMSaTci1xDAfWJsxXu2XtROQwYKSq/t/uNiQiV4tIlYhU1dbWZj7SfdF/FMy4z3nk7Ku3prRKKODjsqNH89pN07jl9P34sHo7Z//+db752AKWb6rPbrzGGJMg1xLHbomID7gb+N6ellXVB1S1UlUrKyoqsh9cuvY/C468Gt66F5a8nPJqBSE/Xz9+PPO+P41rp09k3tJaTrlnHjc+9QFrt6Z2tZYxxuyLXEsc64CRCfMj3LI2JcABwL9EZBVwNDCzR3WQJzr5FzDkQGcU3R3r9rx8gn75QW44eRLzvj+Nrxw7lpkf1HDib/7Fz57/mE31Pa8FzxjTc0guXaUjIgFgKTAdJ2G8A1yiqgt3sfy/gBtVtWp3262srNSqqt0u4p3Ny+H+42DYIfClmeAP7NVm1u9o5ndzlvO3qrWE/D6+fOwYvnHceEoLgxkO2BjTV4jIAlXd6Yd5TtU4VDUKXAPMAhYDf1PVhSJym4ic4210WVI+Ac66G1a/AfN+vdebGVpawK/OP5BXbzieU6YO5n9e+5TP/vqf3Dt3OY2tNsquMSZzcqrGkS05XeNo8/dvwAdPwhUzYexx+7y5xevr+M0rS3h18SbKi0N8e9oELjlqFHkBfwaCNcb0BbuqcVjiyBWtDfDACdBaD998A4rKM7LZBau3ceesT3hrxVaG9y/guukTOf+w4QT8OVXZNMbkoB7RVNWn5RXD5x+G5m1O7SMez8hmDx89gL9cdTSPffUoyotDfP+ZDznlnnm8+GEN8Xjv/9FgjMk8Sxy5ZMiBcOrtsHy2c5luhogIn51YznPfPpb7Lz+cgF+45on3OPv3rzN3iQ1jYoxJjyWOXHPE1zoeOVud+iNnUyEinDp1CC9ddxz3fPFg6loiXPnwO3zh/jd5e+XWjO7LGNN7WR9HLmreBv9znPP42W/8G/JLs7KbcDTO36rW8rs5y9hU38rxkyq46dTJHDA8O/szxvQs1sfRkxQMgAsfgh3VMPPalB85m67kYUw+qN7OWf/9Ot96fAHLNzVkZZ/GmJ7PEkeuGnkknPhjWPQcLPhTVneVPIzJa0tqOeWe17jpqQ+o3mbDmBhjOrOmqlwWj8PjF8Dq/wcz7oWyCdBvGBSWO6PrZsmWhlb+8K9P+fNbq1FVLj1qNN+aNp5BJflZ26cxJvfYfRw9MXEANGyC+4+H+pqOMl8QSoZAyVDoNxRKhiW9D3USTLBgn3bd1TAmx4wvY0BhiIFFzis/aDcUGtNbWeLoqYkDnJsCa5c6yaNufRfv6yHcRZ9Efn8ngewywQyDwrI91l5WbW7knleXMvODmp26WwpDfgYUhigrDnVKKAOLkueDDCzKo7QgiN8nGTw4xphsscTRkxNHKlrqnARSV5P0npBgGjYCSf/evmBCYhmakGiGdS4PFlCzvZl125vZ2hhmW2OYLe771qbwTmWN4ViXYYpA/4Jgp+SSnHQGFIUYmDBfGPIjYsnGmO62q8Sxd0OxmtyT3895VUze9TKxqJM8ukwwNbDxY1j2CkS66BAvGMCwkqEMyyuBQL7TDBbIg0ABFORDSQEE8535YD4RCdEYD9IQC1AfDbI96mNH2M+2iJ8tra1safGzqUXYVCssXiNsbIJwvOvkEAr4KEuowfQvDFIUCpAf9JEf8lMQ9JMfbHv3JUz7KQh1XZ4f9FvNp7eLx0FjEI9C3H3XuDvfVVms47PkMo0lfJ5KWdIPp04/fGT35bv6kZTONhLLp5wLBf13f6zSZImjL/EHoHS489oVVWjZ0XWtpX6D0yQWbXHuNYm2QKQFos3uewvEIwAEgf7uKyUhUF8QDeQT8+cR9eUTlhBhQrQQolmDNLUEaWwK0BALEI77aI0L4bgQiQsx/ITx0YyPmPuK4yOqfuIJZYmf4Qvg8/vx+wP4A0H3PYDfHyAQCBAMBvH7gwSCAYKBIMH2d+cVCgYJ+BS/QEAg4AO/KAFxy3yKXwS/TwmA8y7gE9xltH3aJ4Jf4vhQ599A45A43aksvouyxM929epqmeSyPW0nlf24r3gsYT7Wse5O5W3Lx9MsT9hmW3nbSTy5dt1XjTrGEofJMhHnP1lBfxi0f/rrx6JOAom2QKQ5YToxwSQkmoTlJNKMRFvwRVsIRloo2Gn5po5tJfyS1E6//Dp+ZUqqJ46Y++pjVHyAgPjaX5IwjXT+bOeXJE37O+Z9icsllrvTAX/X5cmvTuX+jn3uVJ4Qjy/gfN72Lv7OZe3LpFrm7/gsuUz8nffVXuYeW+dIJxz0XfyfbC/f1bL7UF6U+SegWuIwmeUPgL/YGbSxm+yywUm162aFxF+lbeWJTRO6cxJqK49GI4QjzisSiREHYnEhqhBT9z0OMYRYHKIKUe2YjsWd5SLudFSFaFyJxsV5qTo1KFUisbbtKuGYu5wK0RhE3OWicQhHlcZonKaw0hRRGiNxGiNKa1SJIzj1GGmfjiNdHjWfQEHQT0EoQKHbxFcQ8u80XRgKkB9sm/a3T4cCPgI+we9re+947Tzv26k8kPCZz0enZXzWrJhTLHGY3kvETWSZ+28ecF+FGdti9sTiSnMkRlM4Sks4TlMkSlM4Rks4RlM4RlMkRnM4SnP7dKzTdFM4SnMkTnM4yo7mCM3t5TGaIzHC0cyM4JyqQJdJqCNJBfyCX6S9aV9EnBTZNo906g7Y6XNxlumY7vhQEst2sW0SPveJE0/AJwT8PkJ+nzvvI+gXgu580O/Mt5UH/L5OZQG/7HbdgK9tG7tetiDoz3jitcRhTC/l9wnFeQGK87LzZx6NxTslk0gsTjSuxNxXtP09TjwO0Xg8qVyJt893XrfTcjElpgnLxDo+i2nivPO5gtPtg7a33mjyPG0tO5rwOe0jRXd83rm8y211se1oPE5L1Ik9Eou3H5tO8zElEnfeo1l8xMGrNxzHhEElGd2mJQ5jzF4J+H2U+H2U5Ntz7feVqhKJOUk24iaX5KTTucxZNhpTwrG25JO4bscy5cV5GY/XEocxxnhMRAgFhFAPGT6wZ0RpjDEmZ1jiMMYYk5Y+MeSIiNQCq72OYx+VA5u9DiJH2LHozI5HZ3Y8OuzrsRitqjvdCNInEkdvICJVXY0Z0xfZsejMjkdndjw6ZOtYWFOVMcaYtFjiMMYYkxZLHD3HA14HkEPsWHRmx6MzOx4dsnIsrI/DGGNMWqzGYYwxJi2WOIwxxqTFEkcOE5GRIjJXRBaJyEIRuc7rmHKBiPhF5D0RedHrWLwmIv1F5GkR+UREFovIZ7yOySsicr37d/KxiPxFRPK9jqk7icgfRWSTiHycUDZQRGaLyDL3fUAm9mWJI7dFge+p6hTgaODbIjLF45hywXXAYq+DyBH/BbysqvsBB9NHj4uIDAeuBSpV9QDAD1zkbVTd7k/AaUllNwNzVHUiMMed32eWOHKYqq5X1Xfd6Xqck8Junvva+4nICOBM4EGvY/GaiJQCxwEPAahqWFW3exuVpwJAgYi0PTKlxuN4upWqzgO2JhXPAB5xpx8Bzs3Evixx9BAiMgY4FJjvbSSe+y3wfaB7nyKUm8YCtcDDbtPdgyJS5HVQXlDVdcBdwBpgPbBDVV/xNqqcMFhV17vTG4DBmdioJY4eQESKgWeA76pqndfxeEVEzgI2qeoCr2PJEQHgMOAPqnoo0EiGmiJ6GrftfgZOMh0GFInIZd5GlVvUufciI/dfWOLIcSISxEkaj6vqs17H47FjgXNEZBXwJHCiiDzmbUieqgaqVbWtFvo0TiLpi04CVqpqrapGgGeBYzyOKRdsFJGhAO77pkxs1BJHDhMRwWm/Xqyqd3sdj9dU9RZVHaGqY3A6Pv+pqn32V6WqbgDWishkt2g6sMjDkLy0BjhaRArdv5vp9NELBZLMBK5wp68Ans/ERi1x5LZjgctxflm/777O8Dook1O+AzwuIh8ChwC/9DgeT7i1rqeBd4GPcM5tfWroERH5C/AmMFlEqkXkq8AdwMkisgynVnZHRvZlQ44YY4xJh9U4jDHGpMUShzHGmLRY4jDGGJOWgNcBdIfy8nIdM2aM12EYY0yPsmDBgs1dPXO8TySOMWPGUFVV5XUYxhjTo4jI6q7KranKGGNMWvpEjcMYY3qzSCxOU2uMxnCUpnCUxrbp1hjHTCijMJTZU70lDmOM6SaqSksk3n5STzzRt79HYjS1RmkMJ7wnLuOWN4U7kkM4tusxP1+94XgmDCrO6PewxGGM6RPicSUcixOOxYlEnfdw1H0lTEdiSjgWIxyN09o2H40Tjsbcz9rKE9Z351sTyprbTuzhGI0JJ/p07rkuCvkpzAs476EARXl++hcEGVaa3z5fGAp0Xi5p+REDCjJ+LC1xGGN6BFWlrjlKbUMLm+pb2dwQpra+lc0NrdTWt7ZPN4djHSf2hBN5NJ7ZUTJCfh9BvxAK+AgFfAT9znvIfS8M+RnSL3+nE/kuT/hJn+cH/Ph8ktGYM8UShzHGM6pKQ2t0l0mgtr6V2oZWNruJoqsmmaBfqCjOo7wkj8H98inKCxD0C3nuSbz9hB7ofGLf6bOE6aC/83KJ6wf9QsjvwxlLsW+yxGGMybjmcIzNDa1sSkoCnZKBO90S2TkZ+ATKivOoKM6joiSPiYNKqChxpsuLQ1SU5DGoJI/y4jxKC4J9+iTuBUscxvRRqkprNE5LJEZLJE5zJEZLJNb+3ppQlvh5a/sy8YTl4+xoDrfXHBpao13uc2BRyK0dhDh81AA3EeQlJAXnfUBhCH+ONtMYSxzG5KS2k3pix2pTQkdrcyTWfpVNc7jj5N3snthbojGaw+7J3Z1uSxKJyWBv5Qd95Af9FAT95Af95AV8lBYEmTqsX6dEUJEwPbAoRNBvt471BpY4jNkHbSf4xCtompIuo2w/2YdjncraLrFMvvqmbT6dvly/T9yTuHNCz3enC4J+ivMClBXlURDykx9wT/judF7Cyb9t+fygn7yE6Y4E4WtPEtY01LdZ4jAmBZFYnGUbG/h43Q4+WreDD9ftYGVtA43hGLE0zvAhv4/CPD9FoQAFIX/71TSJV9845QEK8/wUBtuuuglQGPJTGPJTlBfotExB0G+/5E23ssRhTJJoLM6yTQ18tG4HH6/bwYfVO1i8vo7WqNO0U5wXYOqwfpx76HBK8gMUuif19pN926WVbcnBvcSyMGQneNM7WOIwe6YKLduhfgPk9YOSIeDzex1VRkRjcZbXNvBRtU42hKQAABp9SURBVFOT+GidkyTa2v+LQn6mDi/lsqNHc9CIUg4YXsrYsqKcvb7emO5gicNAaz3sWAd11e77uo75uhpnOtLYsbz4oWQolA6HfsPd9xEJ8yOgsBx8ufXrOhqL82lto5Mgqrfz0bodLOoiSVx61GgOHO4kiXHlliSMSWaJo7eLNCclhZqdE0TrjqSVBIoHO4mgYj+YcJKTEEqGQMuOhMSyDta/D5/8H8RaO2/CH4J+w5z12pOLm1Ta3gsGQJY6WWNx5dPaBj6s3tHeL7GwZkd7kigM+TlgWCmXHDmaA0f048Dh/S1J9GSq0LAJtiyHYL7zQ6aoIud+vPQWljh6smgY6msSkkB159rCjnXQvHXn9QrLnRP5gLEw5rMJJ3T3RF8yFAKh1ONQhaYtXe+/bh2secuJM550bX+gYOeE0m9YQnIZDvmle9x9LK6scJNEW7/Ewpo6miMx5+uG/Ewd1o+LjxzFQSNKOXB4KWPLi+0+gZ4oHocda6B2KWxeArVLYPNS571le+dlfcHO/5+S/2/1GwGFA7P246U3E01nxK0eqrKyUnv0g5zWzIe183dODo2bdl42v3/SH0pSM1K/4c4vsu4Wjzm/CNu/Q83O36dhA2jSvQWhkk5NYtHi4Wzxl7OsuYQP6wqYX5vHOxvjNIWd9QqCfg4Y3o8DhjsJ4sDhpYyrsCTR40TDsHWFmxwSk8QyiDZ3LFdYDhWToXyS81420an91tXs/EOmbj3EI533Eyjo+u8kzR8vvZWILFDVyp3KLXHksGgrzP4ZzP+DMx8q3kWzj/ufvt8wyMvs8MndKhaB+g3Etlezbf1KdmxcSeuWtUjdOvKb11Ma3sRAkpvVICJ5tBYOxtdvKPllI/H1G+rUmkqGOsekZIhbi8rz4EuZ3Qo3ujWGpBrE1hWda6ilIzuSQ/v7ZCgqS31f8bjzY2un/jw3wdTVQP36Pf542emHWOlwCBVl5njkmF0lDmuqylW1S+GZr8CGj+Cob8DxP8hqn0B3i8eVDXUtrNrcyIrNjaza3MiqLc702q1NRGKlwCHAIRTnBRhTXsjY8mImDAiwX3EDU4oaGR7Yga9+PcH69QTrapyrvtZVweL1O/e5ABSWJSSUoVDiJpV+wzrKC8typ108HncuSgi3vRqgtaFjuq1cY86JK1TsvrdNJ837Pfxzb9rqJoXEGsRSp9mpjfhh4DgnKex3VkeSKJ+UmR9EPp/7I2IIcHjXy8SiTs13V8llw0d7rumXDnf+b6XT3JtNh17uNMllkCWOXKMK7z0GL30fggVw8V9h8mleR7VXVJXahlZWbW7aKUGs2tLYaciL/KCPMWVFTBpUwqlThzC2rIgx5UWMLS+ivDiU3p3KqtC8zfn1WLfeea9f3/GLsn49rP8AGmuBpBq3L+gmkSFdJ5e2GkzyL8x4HCJNCSf0hBN7a33nk3+X741uUkiYT7ySLRMC+UmJpav3Isgr2X0CChU5J/JgUedkpOoc29pPOieHzUvcY50QR/lEGHkkHHZ5Rw1i4HjvT7b+gJMASkcAR3W9TNRtCuuqubWuGqrf6bpv0SuTTs944rCmqlzSvB1evB4WPgtjj4PzHnBOXjlue1O4Iym0JYgtjaza3NRpsLugXxg5sJBx5UWMKStibEVRe4IY0i+/+69oikWgYaObXNwaS1tyaavB1K93TubJ8kqhoL9z1Vq6J3lfYOcTcl5XJ+ik+bwuTvKhYhCfk7SSE0+4i/muaizJy0VbUv8uicmoaRuE6zsfo4rJUDHJaVZqq0H0H9Vr7gPapWjrzheDeCVQsNe1aGuqynVr34Znvur8apn+Mzj2upz741q2sZ7FG+p3ShDbmzo6HH0CIwYUMqa8iMrRAxlT5kyPKy9mWP98Arl057Q/mPDrcjda6twkUtO5BtO8DYKFSb/Qi5w28a5+obdN+0OZb3Is6J+5bcUiCUml0UkG4T01mTU4N4cm9kEUD+41TatpC+QBvbdPLauJQ0ROA/4L8AMPquodSZ+PBv4IVABbgctUtVpEpgH3JCy6H3CRqj4nIn8Cjof2XtIvq+r72fweWRWPwet3w9xfOSewr8yCkUd4HVW7xtYoMz+o4Yn5a/hoXUfH9NDSfMaWF3HGgUMZW+Y0KY0pL2LUwEJCgRxKDpmQ3895VUzyOpLu4Q86iSiTycj0KllLHCLiB+4FTgaqgXdEZKaqLkpY7C7gz6r6iIicCPwKuFxV5+L0jCIiA4HlwCsJ692kqk9nK/ZuU1cDz14Nq/4NB1wIZ92dM5f+Laqp44m3V/PcezU0tEaZPLiEW8+ewtHjyxg9sIiCUG7Vhowx3SebNY4jgeWqugJARJ4EZgCJiWMKcIM7PRd4rovtXAi8pKpNWYy1+y15CZ77ltMWOuM+OOQSz6v1zeEYL35YwxNvr+G9NdsJBXycddBQLj1qFIeNGmBDaRtjgOwmjuHA2oT5ana+TOED4Hyc5qzzgBIRKVPVLQnLXATcnbTe7SLyU2AOcLOq7nTtpYhcDVwNMGrUqH35HpkVaYHZP4G3H4AhB8GFD0P5BE9DWrqxnifmr+HZd6upa4kyvqKIn5w1hQsOG07/why5pNAYkzO87hy/Efi9iHwZmAesA2JtH4rIUOBAYFbCOrcAG4AQ8ADwA+C25A2r6gPu51RWVubGpWObPoGnvwKbFsJnroHpP/XsprSWSIyXPl7PE/PX8M6qbYT8Pk47YAiXHjWKI8cOtNqFMWaXspk41gEjE+ZHuGXtVLUGp8aBiBQDF6hq4oAzXwD+rqqRhHXWu5OtIvIwTvLJbarw7iPw0s3OVTWXPg0TT/YklE9rG/jL/DU8/W4125sijCkr5JbT9+PCw0dQVtx7rwIxxmROSolDRJ4FHsLpa0j1QcXvABNFZCxOwrgIuCRpu+XAVnebt+BcYZXoYrc8cZ2hqrpenJ/E5wIfpxiPN5q3wQvXwaLnYdw0OO9+KBncrSGEo3FmLdzAE/PX8OaKLQR8wqlTh3DJUaP4zLgyGxHWGJOWVGsc9wFXAr8TkaeAh1V1ye5WUNWoiFyD08zkB/6oqgtF5DagSlVnAicAvxIRxWmq+nbb+iIyBqfG8lrSph8XkQpAgPeBb6T4Hbrfmrfgma851/yffBt85jvdOpzF6i2N/OXttTxVtZYtjWFGDCjgplMn8/nKEQwq8WCgQ2NMr5DWneMiUopTC/gRTsf3/wKPJTYl5aJuv3M8HoN5d8Frd0D/0XDhQzB8F2PjZFgkFmfO4o08Pn8N/162Gb9PmL7fIC45ahTHTayw2oUxJmX7fOe4iJQBlwGXA+8BjwOfBa7AqTkYcMatefZqWP0GHPRFOPM3zp3FWVa9rYkn317LX6vWUlvfyrDSfK4/aRJfPGIkQ0qtdmGMyZxU+zj+DkwGHgXOTuig/quI9IBBoLrJ4hfg+WucMWrOux8Oviiru4vG4sxdUssT81fzr6XOIHLTJg/i0qNGccLkQfYMCmNMVqRa4/idezf3TrqqxvQ5kWaY9SOoegiGHQoXPARl47O2u/U7mvnrO2v56ztrWb+jhUEleXxn2gS+eOQohvcvyNp+jTEGUk8cU0TkvbZLZUVkAHCxqt6XvdB6iI2LnHszahfDMdfCiT/JytDQsbgyb1ktT8xfw5zFG1HgcxMr+NnZU5m+/yCCuTR4oDGmV0s1cVylqve2zajqNhG5Cudqq75J1alhzPqRMyroZc/ChOlZ2dWTb6/hv/+5nHXbmykvDvH148dz8RGjGFVWmJX9GWPM7qSaOPwiIupeguUOYNh3x6Jo2gozvwOfvAgTToJz/wDFg7Kyq/fWbOOWv3/EISP788Mz9ufkKYN73+izxpgeJdXE8TJOR/j97vzX3bK+Z9Ub8OxV0LAJTrkdjv5W1u7NiMWVnzz/MYNK8nj0q0dRnOf1CDHGGJN64vgBTrL4pjs/G3gwKxHlqlgU5v0a5t0JA8bC12Y7HeFZ9MTba/h4XR2/u/hQSxrGmJyR0tnIHRLkD+6r79m+Bp65Cta+BQdfAmfc6TzRLYu2NLRy58ufcMz4Ms4+KPcfH2uM6TtSvY9jIs5DlqYA7XeTqeq4LMWVOxY+By9cC/E4nP8gHPT5btntHS99QlM4xm0zptpItcaYnJJq+8fDwM9wHuc6DWfcqt7dQ6sK/7gR3nnQGS7kgodg4Nhu2fWC1Vt5akE1Xz9+HBMGZf+uc2OMSUeqJ/8CVZ2DM7bValW9FTgze2HlABHoNww+e73zHPBuShrRWJyfPLeQoaX5XHvixG7ZpzHGpCPVGkeriPiAZe6It+uA7Dby54LPfa/bd/n4/DUsWl/HfZceRpF1iBtjclCqNY7rgELgWuBwnMEOr8hWUH1VbX0rd72yhM9NLOf0A4Z4HY4xxnRpjz9p3Zv9vqiqNwINOP0bJgt+9dJiWiIxbj3HOsSNMblrjzUOVY3hDJ9usujtlVt59t11XH3cOMZX9P5WQGNMz5VqI/p7IjITeApobCtU1WezElUfE43F+enzHzO8fwHfnjbB63CMMWa3Uk0c+cAW4MSEMgUscWTAI2+u5pMN9fzPZYdTGLIOcWNMbkv1znHr18iSTXUt3DN7KcdPquDUqYO9DscYY/Yo1TvHH8apYXSiql/JeER9zC//sZhwNM7PrUPcGNNDpNou8mLCdD5wHlCT+XD6lrdWbOG592u49sQJjCkv8jocY4xJSapNVc8kzovIX4DX97SeiJwG/BfgBx5U1TuSPh8N/BGoALYCl6lqtftZDPjIXXSNqp7jlo8FngTKgAXA5aoaTuV75JKI2yE+YkAB3zzBOsSNMT3H3o43NRHY7ZOL3Ps/7gVOxxkc8WIRmZK02F3An1X1IOA2nIEU2zSr6iHu65yE8v8E7lHVCcA24Kt7+R089ac3VrF0YwO3nj2VgpDf63CMMSZlKSUOEakXkbq2F/ACzjM6dudIYLmqrnBrBE8CM5KWmQL8052e28XnyXEIzpVdT7tFjwDnpvIdcsmGHS389tWlTN9vECdNsQ5xY0zPklLiUNUSVe2X8JqU3HzVheHA2oT5arcs0QfA+e70eUCJiJS58/kiUiUib4lIW3IoA7aranQ328x5t/9jMZG48rOzp3odijHGpC3VGsd5IlKaMN8/4WS+L24EjheR94DjcQZPjLmfjVbVSuAS4LciMj6dDYvI1W7iqaqtrc1AqJnxxvLNvPBBDd86YTyjygq9DscYY9KWah/Hz1R1R9uMqm7HeT7H7qwDRibMj3DL2qlqjaqer6qHAj9K2Daqus59XwH8CzgU5ybE/iIS2NU2E7b9gKpWqmplRUVFSl8y28JRp0N81MBCvnF8WnnQGGNyRqqJo6vl9nRF1jvARBEZKyIh4CJgZuICIlLuDtcOcAvOFVaIyAARyWtbBjgWWKSqitMXcqG7zhXA8yl+B8/98Y2VfFrbyK3nTCE/aB3ixpieKdXEUSUid4vIePd1N86lsLvk9kNcA8wCFgN/U9WFInKbiLRdJXUCsERElgKDgdvd8v3dfX6AkyjuUNVF7mc/AG4QkeU4fR4PpfgdPFWzvZnfzVnGyVMGc+J+1iFujOm5xPkRv4eFRIqAnwAn4dxBPhu4XVUbd7tijqisrNSqqipPY/jW4wv45yebmH398YwcaH0bxpjcJyIL3L7mTlK9AbARuDnjUfUR85bW8o+PNnDjKZMsaRhjerxUr6qaLSL9E+YHiMis7IXVe7RGY9w6cyFjygq56rhxXodjjDH7LNU+jvK2q50AVHUbe7hz3Dge/PdKVmxu5OczDiAvYB3ixpieL9XEEReRUW0zIjKGLkbLNZ1Vb2viv/+5jNMPGMLxk3LjkmBjjNlXqY6O+yPgdRF5DRDgc8DVWYuql/jFi4sQhB+flTxElzHG9FypDjnyMlAJLAH+AnwPaM5iXD3e3CWbmLVwI9+ZPoHh/Qu8DscYYzIm1Qc5fQ24DudO7feBo4E36fwoWeNqiTgd4uMqivjaZ61D3BjTu6Tax3EdcASwWlWn4Qz/sX33q/RdD8xbweotTdx2zgGEAns7cr0xxuSmVM9qLaraAiAiear6CTA5e2H1XGu3NnHv3OWcedBQPjux3OtwjDEm41LtHK927+N4DpgtItuA1dkLq+f6+QuL8PuEH5+5v9ehGGNMVqR65/h57uStIjIXKAVezlpUPdScxRt5dfFGfnjGfgwttQ5xY0zvlGqNo52qvpaNQHq6lkiMW19YyMRBxVx57FivwzHGmKxJO3GYrv3hX5+ydmszT1x1FEG/dYgbY3ovO8NlwOotjfzhtU855+BhHDPeOsSNMb2bJY59pKrcOnMhIb+PH1mHuDGmD7DEsY9mL9rI3CW1fPekiQzul+91OMYYk3WWOPZBczjGz19YxOTBJVxxzBivwzHGmG5hneP74L5/LWfd9mb+evXR1iFujOkz7Gy3l1ZubuT+11Zw/qHDOWpcmdfhGGNMt7HEsRdUlZ8+/zF5AR83n7Gf1+EYY0y3ssSxF2Yt3MC/l23mhlMmMajEOsSNMX2LJY40NYWj3PbCIvYf2o/Ljx7tdTjGGNPtspo4ROQ0EVkiIstF5OYuPh8tInNE5EMR+ZeIjHDLDxGRN0VkofvZFxPW+ZOIrBSR993XIdn8Dsn++5/LqdnRwi9mTCVgHeLGmD4oa2c+EfED9wKnA1OAi0Uk+RmqdwF/VtWDgNuAX7nlTcCXVHUqcBrwW3d03jY3qeoh7uv9bH2HZMs3NfDgv1dw4eEjqBwzsLt2a4wxOSWbP5mPBJar6gpVDQNPAjOSlpkC/NOdntv2uaouVdVl7nQNsAmoyGKse9R2h3hB0M/Np1uHuDGm78pm4hgOrE2Yr3bLEn0AnO9OnweUiEina1tF5EggBHyaUHy724R1j4jkdbVzEblaRKpEpKq2tnZfvgcA//hoA68v38xNp06mvLjLXRpjTJ/gdSP9jcDxIvIecDywDoi1fSgiQ4FHgStVNe4W3wLsh/Mo24HAD7rasKo+oKqVqlpZUbFvlZWG1ii/eHERU4f145KjrEPcGNO3ZfPO8XXAyIT5EW5ZO7cZ6nwAESkGLlDV7e58P+D/gB+p6lsJ66x3J1tF5GGc5JNV/z1nGRvqWrjvssPw+yTbuzPGmJyWzRrHO8BEERkrIiHgImBm4gIiUi4ibTHcAvzRLQ8Bf8fpOH86aZ2h7rsA5wIfZ/E7sGxjPQ+9vpKLjhjJYaMGZHNXxhjTI2QtcahqFLgGmAUsBv6mqgtF5DYROcdd7ARgiYgsBQYDt7vlXwCOA77cxWW3j4vIR8BHQDnwH1n8Dvz0+YUU5QX4/mnWIW6MMQCiql7HkHWVlZVaVVWV9nrPv7+O6558n9vPO4BLrW/DGNPHiMgCVa1MLve6czyn/a1qLQeNKOWiI0Z5HYoxxuQMG1Z9Nx7+8pFsaWy1DnFjjElgNY7dCAV8DC0t8DoMY4zJKZY4jDHGpMUShzHGmLT0iauqRKQWWO11HPuoHNjsdRA5wo5FZ3Y8OrPj0WFfj8VoVd1p6I0+kTh6AxGp6uqyuL7IjkVndjw6s+PRIVvHwpqqjDHGpMUShzHGmLRY4ug5HvA6gBxix6IzOx6d2fHokJVjYX0cxhhj0mI1DmOMMWmxxJHDRGSkiMwVkUUislBErvM6plwgIn4ReU9EXvQ6Fq+JSH8ReVpEPhGRxSLyGa9j8oqIXO/+nXwsIn8RkXyvY+pOIvJHEdkkIh8nlA0Ukdkissx9z8izISxx5LYo8D1VnQIcDXxbRKZ4HFMuuA5nqH4D/wW8rKr7AQfTR4+LiAwHrgUqVfUAwI/zDKC+5E/AaUllNwNzVHUiMMed32eWOHKYqq5X1Xfd6Xqck0Lyc9v7FBEZAZwJPOh1LF4TkVKc59Y8BKCq4bYnaPZRAaBARAJAIVDjcTzdSlXnAVuTimcAj7jTj+A8/G6fWeLoIURkDHAoMN/bSDz3W+D7QHxPC/YBY4Fa4GG36e5BESnyOigvqOo64C5gDbAe2KGqr3gbVU4YnPC47Q04D8zbZ5Y4egD3eezPAN9V1Tqv4/GKiJwFbFLVBV7HkiMCwGHAH1T1UKCRDDVF9DRu2/0MnGQ6DCgSkcu8jSq3qHMJbUYuo7XEkeNEJIiTNB5X1We9jsdjxwLniMgq4EngRBF5zNuQPFUNVKtqWy30aZxE0hedBKxU1VpVjQDPAsd4HFMu2CgiQwHc902Z2KgljhwmIoLTfr1YVe/2Oh6vqeotqjpCVcfgdHz+U1X77K9KVd0ArBWRyW7RdGCRhyF5aQ1wtIgUun830+mjFwokmQlc4U5fATyfiY1a4shtxwKX4/yyft99neF1UCanfAd4XEQ+BA4BfulxPJ5wa11PA+8CH+Gc2/rUHeQi8hfgTWCyiFSLyFeBO4CTRWQZTq3sjozsy+4cN8YYkw6rcRhjjEmLJQ5jjDFpscRhjDEmLZY4jDHGpMUShzHGmLRY4jAmx4nICTYSsMklljiMMcakxRKHMRkiIpeJyNvujZr3u88NaRCRe9znRMwRkQp32UNE5C0R+VBE/t72nAQRmSAir4rIByLyroiMdzdfnPDcjcfdu6ON8YQlDmMyQET2B74IHKuqhwAx4FKgCKhS1anAa8DP3FX+DPxAVQ/CudO5rfxx4F5VPRhnrKW2kU0PBb4LTAHG4YwqYIwnAl4HYEwvMR04HHjHrQwU4AwoFwf+6i7zGPCs+xyN/qr6mlv+CPCUiJQAw1X17wCq2gLgbu9tVa12598HxgCvZ/9rGbMzSxzGZIYAj6jqLZ0KRX6StNzejvHTmjAdw/52jYesqcqYzJgDXCgig6D9Wc+jcf7GLnSXuQR4XVV3ANtE5HNu+eXAa+5THqtF5Fx3G3kiUtit38KYFNivFmMyQFUXiciPgVdExAdEgG/jPFzpSPezTTj9IOAMcf0/bmJYAVzpll8O3C8it7nb+Hw3fg1jUmKj4xqTRSLSoKrFXsdhTCZZU5Uxxpi0WI3DGGNMWqzGYYwxJi2WOIwxxqTFEocxxpi0WOIwxhiTFkscxhhj0mKJwxhjTFr+P/qgkX/FJURpAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PXAgHyHInSiK",
        "outputId": "5070517b-3dfb-4084-ab40-c79fbd8d4d48"
      },
      "source": [
        "predictions, labels = predict(test_dl, clf)\n",
        "\n",
        "accuracy = accuracy_score(predictions, labels)\n",
        "ACCURACY_STATS.append((2, \"class_weights\", accuracy))\n",
        "\n",
        "print(\"accuracy: {:.3f}\".format(accuracy))\n",
        "print(\"confusion matrix\")\n",
        "print(confusion_matrix(predictions, labels))"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy: 0.977\n",
            "confusion matrix\n",
            "[[479   4]\n",
            " [  9  65]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EmQuCG9ifjS7"
      },
      "source": [
        "## Undersampling Majority Class\n",
        "\n",
        "Sampling (especially oversampling) must be restricted to the training split to protect against data leakage."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-QwB41jTaUxg",
        "outputId": "7cd7f4b0-1aab-4856-edde-93982d71941f"
      },
      "source": [
        "minor_df = spam_train_df[spam_train_df[\"label\"] == \"spam\"]\n",
        "num_minor = len(minor_df)\n",
        "num_major = len(spam_train_df) - num_minor\n",
        "\n",
        "sample_major_df = (\n",
        "  spam_train_df[spam_train_df[\"label\"] == \"ham\"]\n",
        "  .sample(num_minor, replace=False, axis=0)\n",
        ")\n",
        "\n",
        "under_df = pd.concat([minor_df, sample_major_df], axis=0)\n",
        "len(under_df)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1090"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "afvjYxEJhocp",
        "outputId": "b51cba05-dddd-446f-f53b-e157e65fdcfc"
      },
      "source": [
        "under_df[\"label\"].value_counts()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "spam    545\n",
              "ham     545\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "rBUrosI5h0vz",
        "outputId": "28a94b2f-5fb8-411a-fe0e-bbb985ff5e64"
      },
      "source": [
        "under_df[\"label\"].value_counts().plot(kind=\"bar\")"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f685c79b7b8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEKCAYAAADpfBXhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOMUlEQVR4nO3df6zddX3H8edLCv6YU0CuHWmrxdhkYf5kHeuif2yQOZBlJZkyzDIa16T/sMVNE8cWzeLmH5IlsrkYsk7MilOROE2ropOg7kc2HEUcP0TDHULaDmxVQJ3Rib73x/10Hkrbe9vee0993+cjOTnf7+f7Pfd8bu7Js99+7/ecm6pCktTLU6Y9AUnS4jPuktSQcZekhoy7JDVk3CWpIeMuSQ2tmvYEAM4666xav379tKchST9Rbr/99q9X1czhtp0UcV+/fj27d++e9jQk6SdKkgePtM3TMpLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjop3sT0k2L9VZ+Y9hRaeeAdl0x7Cm342lxcHV6bHrlLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1tKC4J3kgyV1Jvphk9xg7M8nNSe4b92eM8SR5V5LZJHcmOW8pvwFJ0pMdy5H7r1TVy6pq41i/CrilqjYAt4x1gIuBDeO2Dbh2sSYrSVqYEzktsxnYMZZ3AJdOjF9fc24FTk9y9gk8jyTpGC007gV8OsntSbaNsdVV9dBYfhhYPZbXAHsmHrt3jEmSlslC/xLTK6tqX5LnAjcn+fLkxqqqJHUsTzz+kdgG8LznPe9YHipJmseCjtyrat+43w98FDgf+NrB0y3jfv/YfR+wbuLha8fYoV9ze1VtrKqNMzMzx/8dSJKeZN64J/mpJD99cBl4FXA3sAvYMnbbAuwcy7uAK8ZVM5uAxyZO30iSlsFCTsusBj6a5OD+H6iqTyW5DbgxyVbgQeCysf9NwKuBWeC7wOsXfdaSpKOaN+5VdT/w0sOMfwO48DDjBVy5KLOTJB0X36EqSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1NCC457klCR3JPn4WD8nyeeTzCb5UJLTxvhTx/rs2L5+aaYuSTqSYzlyfwNw78T61cA1VfVC4BFg6xjfCjwyxq8Z+0mSltGC4p5kLXAJ8J6xHuAC4MNjlx3ApWN581hnbL9w7C9JWiYLPXL/S+DNwI/G+nOAR6vq8bG+F1gzltcAewDG9sfG/pKkZTJv3JP8OrC/qm5fzCdOsi3J7iS7Dxw4sJhfWpJWvIUcub8C+I0kDwA3MHc65q+A05OsGvusBfaN5X3AOoCx/dnANw79olW1vao2VtXGmZmZE/omJElPNG/cq+qPq2ptVa0HLgc+U1W/DXwWeM3YbQuwcyzvGuuM7Z+pqlrUWUuSjupErnP/I+CNSWaZO6d+3Ri/DnjOGH8jcNWJTVGSdKxWzb/Lj1XV54DPjeX7gfMPs8/3gNcuwtwkScfJd6hKUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNTRv3JM8Lcl/JPnPJPckedsYPyfJ55PMJvlQktPG+FPH+uzYvn5pvwVJ0qEWcuT+feCCqnop8DLgoiSbgKuBa6rqhcAjwNax/1bgkTF+zdhPkrSM5o17zfnOWD113Aq4APjwGN8BXDqWN491xvYLk2TRZixJmteCzrknOSXJF4H9wM3AfwGPVtXjY5e9wJqxvAbYAzC2PwY8ZzEnLUk6ugXFvap+WFUvA9YC5wM/e6JPnGRbkt1Jdh84cOBEv5wkacIxXS1TVY8CnwV+CTg9yaqxaS2wbyzvA9YBjO3PBr5xmK+1vao2VtXGmZmZ45y+JOlwFnK1zEyS08fy04FfBe5lLvKvGbttAXaO5V1jnbH9M1VVizlpSdLRrZp/F84GdiQ5hbl/DG6sqo8n+RJwQ5K3A3cA1439rwPel2QW+CZw+RLMW5J0FPPGvaruBF5+mPH7mTv/fuj494DXLsrsJEnHxXeoSlJDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDU0b9yTrEvy2SRfSnJPkjeM8TOT3JzkvnF/xhhPknclmU1yZ5LzlvqbkCQ90UKO3B8H3lRV5wKbgCuTnAtcBdxSVRuAW8Y6wMXAhnHbBly76LOWJB3VvHGvqoeq6gtj+dvAvcAaYDOwY+y2A7h0LG8Grq85twKnJzl70WcuSTqiYzrnnmQ98HLg88DqqnpobHoYWD2W1wB7Jh62d4xJkpbJguOe5JnAPwB/UFXfmtxWVQXUsTxxkm1JdifZfeDAgWN5qCRpHguKe5JTmQv7+6vqI2P4awdPt4z7/WN8H7Bu4uFrx9gTVNX2qtpYVRtnZmaOd/6SpMNYyNUyAa4D7q2qd05s2gVsGctbgJ0T41eMq2Y2AY9NnL6RJC2DVQvY5xXA7wB3JfniGPsT4B3AjUm2Ag8Cl41tNwGvBmaB7wKvX9QZS5LmNW/cq+pfgRxh84WH2b+AK09wXpKkE+A7VCWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIamjfuSd6bZH+SuyfGzkxyc5L7xv0ZYzxJ3pVkNsmdSc5byslLkg5vIUfufwdcdMjYVcAtVbUBuGWsA1wMbBi3bcC1izNNSdKxmDfuVfXPwDcPGd4M7BjLO4BLJ8avrzm3AqcnOXuxJitJWpjjPee+uqoeGssPA6vH8hpgz8R+e8eYJGkZnfAvVKuqgDrWxyXZlmR3kt0HDhw40WlIkiYcb9y/dvB0y7jfP8b3Aesm9ls7xp6kqrZX1caq2jgzM3Oc05AkHc7xxn0XsGUsbwF2ToxfMa6a2QQ8NnH6RpK0TFbNt0OSDwK/DJyVZC/wp8A7gBuTbAUeBC4bu98EvBqYBb4LvH4J5ixJmse8ca+q1x1h04WH2beAK090UpKkE+M7VCWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaWpK4J7koyVeSzCa5aimeQ5J0ZIse9ySnAO8GLgbOBV6X5NzFfh5J0pEtxZH7+cBsVd1fVf8L3ABsXoLnkSQdwVLEfQ2wZ2J97xiTJC2TVdN64iTbgG1j9TtJvjKtuTR0FvD1aU9iPrl62jPQFPjaXFzPP9KGpYj7PmDdxPraMfYEVbUd2L4Ez7/iJdldVRunPQ/pUL42l89SnJa5DdiQ5JwkpwGXA7uW4HkkSUew6EfuVfV4kt8D/hE4BXhvVd2z2M8jSTqyJTnnXlU3ATctxdfWgni6SycrX5vLJFU17TlIkhaZHz8gSQ0Zd0lqyLhLUkNTexOTFs/4PJ9LgPVM/Eyr6p3TmpM0KclLePLr8yNTm9AKYNx7+BjwPeAu4EdTnov0BEneC7wEuIcfvz4LMO5LyLj3sLaqXjLtSUhHsKmq/GTYZeY59x4+meRV056EdAT/7sd+Lz+P3Hu4FfhokqcAPwACVFU9a7rTkgC4nrnAPwx8nx+/Pv3f5hLyTUwNJPkqc5+Zf1f5A9VJJsks8EYO+Z1QVT04tUmtAB6597AHuNuw6yR1oKr88MBlZtx7uB/4XJJPMvffXsBLIXXSuCPJB5i7qmvy9enVMkvIuPfw1XE7bdykk8nTmYv65C/9vRRyiXnOXZIa8si9gSQzwJuBnwOednC8qi6Y2qSkIcnTgK08+fX5u1Ob1Argde49vB/4MnAO8DbgAeb+IpZ0Mngf8DPArwH/xNyf3vz2VGe0AnhapoEkt1fVzye58+C1w0luq6pfmPbcpCR3VNXLD74+k5wK/EtVbZr23DrztEwPPxj3DyW5BPhv4MwpzkeadPD1+WiSFwEPA8+d4nxWBOPew9uTPBt4E/DXwLOAP5zulKT/tz3JGcBbgF3AM4G3TndK/XlaRtKSSvJU4DeZ+8jfU8dwVdWfTW1SK4C/UG0gyQuSfCzJ15PsT7IzyQumPS9p2Mncx2M8Dnxn3P5nqjNaATxybyDJrcC7gQ+OocuB36+qX5zerKQ5Se6uqhdNex4rjUfuPTyjqt5XVY+P298zcT2xNGX/luTF057ESuORewNJrgYeAW5g7m3dvwWcAfwFQFV9c3qz00qV5C7mXo+rgA3MfQaSH/m7TIx7A+Mjfw86+APNwfWq8vy7ll2S5x9tux/5u7SMewNJLgM+VVXfSvJW4Dzgz6vqC1OemqQp8Zx7D28ZYX8lcAHwHuDaKc9J0hQZ9x5+OO4vAf62qj6BH/0rrWjGvYd9Sf6GuV+k3jTeNOLPVlrBPOfeQJJnABcx9zdU70tyNvDiqvr0lKcmaUqMuyQ15H/dJakh4y5JDRl3SWrIuEtSQ8Zdkhr6P28GaJk/1SD/AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZcD2DB8hiHiY"
      },
      "source": [
        "train_ds = create_dataset(under_df)\n",
        "\n",
        "train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\n",
        "val_dl = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False)\n",
        "test_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False)"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DxgPiIUj58L"
      },
      "source": [
        "clf = SpamClassifier(CLF_LAYER_DIMS)\n",
        "clf = clf.to(dev)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.AdamW(clf.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TkFSeHH-ju5J",
        "outputId": "cca12b1a-5a64-4973-eb69-57fd3fb3f299"
      },
      "source": [
        "train_losses, train_accs, val_losses, val_accs = [], [], [], []\n",
        "for epoch in range(1, NUM_EPOCHS + 1):\n",
        "  train_loss, train_acc = train(train_dl, clf, loss_fn, optimizer)\n",
        "  val_loss, val_acc = validate(val_dl, clf, loss_fn) \n",
        "  print(\"EPOCH {:2d} | train_loss: {:.3f}, acc: {:.3f} | val_loss: {:.3f}, acc: {:.3f}\"\n",
        "    .format(epoch, train_loss, train_acc, val_loss, val_acc))\n",
        "  train_losses.append(train_loss)\n",
        "  train_accs.append(train_acc)\n",
        "  val_losses.append(val_loss)\n",
        "  val_accs.append(val_acc)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH  1 | train_loss: 0.686, acc: 0.544 | val_loss: 0.660, acc: 0.950\n",
            "EPOCH  2 | train_loss: 0.657, acc: 0.921 | val_loss: 0.627, acc: 0.978\n",
            "EPOCH  3 | train_loss: 0.605, acc: 0.966 | val_loss: 0.573, acc: 0.975\n",
            "EPOCH  4 | train_loss: 0.528, acc: 0.971 | val_loss: 0.498, acc: 0.972\n",
            "EPOCH  5 | train_loss: 0.447, acc: 0.972 | val_loss: 0.424, acc: 0.974\n",
            "EPOCH  6 | train_loss: 0.389, acc: 0.974 | val_loss: 0.380, acc: 0.977\n",
            "EPOCH  7 | train_loss: 0.361, acc: 0.975 | val_loss: 0.362, acc: 0.977\n",
            "EPOCH  8 | train_loss: 0.349, acc: 0.975 | val_loss: 0.352, acc: 0.975\n",
            "EPOCH  9 | train_loss: 0.344, acc: 0.978 | val_loss: 0.349, acc: 0.974\n",
            "EPOCH 10 | train_loss: 0.339, acc: 0.981 | val_loss: 0.349, acc: 0.969\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        },
        "id": "CH7nXpFykVX0",
        "outputId": "ab3449d0-1e5b-4d91-a225-f604205e7e86"
      },
      "source": [
        "training_lossplots(\"Undersampling\", train_losses, train_accs, val_losses, val_accs)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEjCAYAAAA/ugbCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV5Z348c/3LllIQhJIWENIZAeVHWVxxYXiXq0DKnWt85up1unYmaltp7V22nE682ttZ2yn/SmI1UFbay1ax30FF1ZF2ZGQEPYtZCHLXb6/P84JXEICF8jNucn9vl+v87r3nPOcc7/3Qs73Ps9zz/OIqmKMMSZ1+bwOwBhjjLcsERhjTIqzRGCMMSnOEoExxqQ4SwTGGJPiLBEYY0yKs0RguhwRuU1EFnkdx+lo+R5EpFZEzvAyJtN1WSIwnhMRFZHBLbY9KCJPeRVTslHVbFXd7HUcpmuyRGBMDBHxex2DMR3NEoFJeiJyoYhUisj9IrJbRHaIyO0x+3uKyEIRqRaRJcCgFscPF5HXRWS/iKwXkRtj9j0hIr8WkZdFpA64SERmisgaEakRkW0i8i23bL6IvCQie0TkgPu8KOZc74jIv4jIB25TzotubE+7sS0VkZKY8ioi3xCRzSKyV0T+XURa/ZuMrTW5MT8qIn9xY/xYRAbFlL3MfZ8HReRXIvKuiNx1uv8OpuuyRGA6iz5ALtAfuBN4VETy3X2PAg1AX+AOdwFARLKA14H/AXoBs4BficjImHPfBPwYyAEWAY8Df62qOcCZwFtuOR8wDxgIFAP1wH+1iHMWMMeNcxDwoXtMD2At8IMW5a8DJgDjgGtiYz+BWcAPgXxgkxs/IlIAPAc8APQE1gNT4jynSVGWCExnEQIeUtWQqr4M1ALD3Kac64Hvq2qdqn4OzI857kpgi6rOU9Wwqq4E/gh8JabMn1V1sapGVbXBfa2RItJdVQ+o6goAVd2nqn9U1UOqWoNz8b2gRZzzVPULVT0I/C/whaq+oaph4A/A2Bbl/01V96tqBfAIMDvOz+NPqrrEPe/TwBh3+0xgtao+7+77JbAzznOaFGWJwCSDCBBssS2Ic0Futs+9sDU7BGQDhUAA2Bqzrzzm+UDgHBGpal6Am3FqGM1ijwUnscwEyt1mlckAItJNRH4jIuUiUg28B+S16FfYFfO8vpX17Bav1TLufsQn9uLe/FngHn/4nOqMKlkZ5zlNirJEYJJBBVDSYlspR1/Q27IHCAMDYrYVxzzfCryrqnkxS7aq/k1MmaOG4FXVpap6DU5T0gvA791d9wPDgHNUtTtwvrtd4oizLS3j3n4a5wLYAcT2W0jsujGtsURgksGzwPdEpEhEfCJyCXAVTlv3calqBHgeeND9xj4SuDWmyEvAUBGZIyJBd5koIiNaO5+IpInIzSKSq6ohoBqIurtzcL7VV4lID45t7z8V/+B2Qg8A7sP5LE7HX4CzRORaEQkAX+fo2o8xx7BEYJLBQ8AHOB21B4CfAje77f3xuAenaWQn8ARO5ywAblv+ZTidq9vdMv8GpB/nfHOALW7zz//BaUoCpw0/E9gLfAS8Emd8x/NnYDnwCc5F/PHTOZmq7sXp//gpsA8YCSwDGk8vTNOViU1MY4w3RESBIaq6KYGv4cPpI7hZVd9O1OuYzs1qBMZ0MSJyuYjkiUg68B2cPoyPPA7LJDFLBMZ0PZOBL3CasK4CrlXVem9DMsnMmoaMMSbFWY3AGGNSnCUCY4xJcZYIjDEmxVkiMMaYFGeJwBhjUpwlAmOMSXGWCIwxJsVZIjDGmBRnicAYY1KcJQJjjElxlgiMMSbFWSIwxpgUl9BEICIzRGS9iGwSkW+3sv/nIvKJu2xw55M1xhjTgRI2+qg7ofcG4FKciTGWArNVdU0b5e8FxqrqHQkJyBhjTKsCCTz3JGCTqm4GEJFngGuAVhMBMJs45oAtKCjQkpKS9orRGGNSwvLly/eqamFr+xKZCPoDW2PWK4FzWisoIgOBUuCtNvbfDdwNUFxczLJly9o3UmOM6eJEpLytfcnSWTwLeE5VI63tVNXfquoEVZ1QWNhqQjPGGHOKEpkItgEDYtaL3G2tmQUsSGAsRKM2E5sxxrQmkU1DS4EhIlKKkwBmATe1LCQiw4F84MMExsIzS7fyq3c2MX5gPuOK8xk/MJ/hfXII+JOlUmSMMd5IWCJQ1bCI3AO8CviBuaq6WkQeApap6kK36CzgGU3w5MlF+ZmMLsrjo837+PMn2wHIDPoZPSD3cHIYV5xPflZaIsMwxpik0+kmr58wYYKeTmexqrL9YAMryg+wvPwAKyoOsGZ7NWG36eiMgizGDcw/nByG9MrG55P2Ct8YYzwhIstVdUJr+xLZNJSURIT+eZn0z8vkqtH9AKhvirCqsooVFVUsLz/AW+t289zySgByMgKMGZB3ODGMKc6je0bQy7dgjDHtKuUSQWsy0/ycc0ZPzjmjJ+DUGsr3HWJ5+QGWVxxgRfkBfvHmRlRBBIb1zmFscXOtIY/SgixErNZgjOmcUq5p6FTVNIT4ZGsVK8qrWF5xgJUVB6hpCAPQIyuNccV5h5PD6KI8MtP8HR6jMca0xZqGAOqrAIXM/FM6PCcjyHlDCjlviHMfQzSqbNpT6/QzuDWHN9buBiDgE0b07c74gfmMLXaalfrnZVqtwRiTlFKnRvDho/Dqd6HPmVByHgycCgOnQLce7RbbgbomVm51O6HLq/hkaxX1IeceuV456Yx3O6GnDi5gRN/u7fa6xhhzIserEaROIti1Bta9BFsWwdaPIdwACPQ+E0qmQsk0Jzm0Y2IIR6Ks21nDigonOSwvP0DlgXoAzi7KZfakYq4a3Y/s9NSpmBljvGGJoKVwI2xb4SSF8kVQ8TGEnQs0vUY5SaE5MWT1PP2gY+yubuDlz3awYMlW1u+qISvNz9Vj+jF7UjFn9c+15iNjTEJYIjiRcBNsdxNDc40hdMjZ12vkkaRQMg2yCtrlJVWVlVurWPBxBS+u2k5DKMqoft2ZPamYa8b0I8d+omqMaUeWCE5WuAl2fAJb3ncSQ8XHEKpz9hWOiGlKmgbZpz8IXnVDiD+v3Mb/LNnK2h3VZAb9XDW6L7MnFTNmQJ7VEowxp80SwemKhGC7mxjKF0PFR9BU6+wrGHakKalkGmT3OuWXUVVWVR5kwZIKFn66nUNNEYb3yWH2pGKuHduf3EyrJRhjTo0lgvYWCcGOT480JVV8BE01zr6CoUeakUqmQU6fU3qJ2sYwCz/ZzoIlFXy27SAZQR9XnNWPm84ZwLjifKslGGNOiiWCRIuEYWdMYij/8Ehi6DnYTQruT1a79z3p03++zakl/PmT7dQ2hhnaO5tZE4v58rj+5HWzQfKMMSdmiaCjRcKwc5WbFBZD+QfQWO3s6zEIhlwK42+DXiNO6rR1jWFeWrWd/1mylU+3VpEW8HHFWU5fwsQSqyUYY9pmicBr0YibGBY7/QxfvAWRJqeWMPFOGH4l+E+u/X/N9mqeWVrBn1Zso6YxzKDCLGZPKubL44roYUNpG2NasESQbOr2wsqnYNnjUFUB2b2dGsK4WyG3/0md6lBTmL+s2sGCJRWsqKgize9jxpl9mDVpAJPP6Gm1BGMMYIkgeUUjsOlNWPoYbHwNxAfDZ8LEu6D0Ameo05OwfmcNC5ZU8PyKSqobwpQWZDFr4gCuH19EQXZ6gt6EMaYzsETQGRzYAsvmwYonoX4/9BziNBuNng2ZeSd1qoZQhJc/28EzS7ayZMt+gn7hspF9mD2pmCmDetpEO8akIM8SgYjMAH6BM1XlY6r6cCtlbgQeBBT4VFWPmdc4VpdNBM1CDbDmz04toXIJBDLh7K84tYS+o0/6dJt217BgyVb+uKKSqkMhint0Y9akAdwwvoheORkJeAPGmGTkSSIQET+wAbgUqMSZzH62qq6JKTME+D1wsaoeEJFeqrr7eOft8okg1o5PYenj8NkfnCEviiY6CWHktRA8uYt4QyjCq6t3smBJBR9t3k/AJ1wzpj//NGMYvbpbQjCmq/MqEUwGHlTVy931BwBU9V9jyvwU2KCqj8V73pRKBM3qq+DTBU4tYd8myOwB4+bAhDsgv+SkT7d5Ty1PfVTBUx+VE/QL904fwu1TS0gP2GQ6xnRVXiWCG4AZqnqXuz4HOEdV74kp8wJOrWEqTvPRg6r6Sivnuhu4G6C4uHh8eXl5QmJOeqpQ9q6TENa9DBp17kmYeBcMvgR8J3ch37K3jn/5yxreWLub0oIsvn/lSC4afupDZBhjklcyJ4KXgBBwI1AEvAecpapVbZ03JWsErTm4DVbMh+VPQO0uyCt2aghj55z0CKlvr9/Nj15cw+a9dVw8vBf/fOVISguyEhO3McYTx0sEvgS+7jZgQMx6kbstViWwUFVDqlqGUzsYksCYuo7c/nDRd+Cbq+ErT0DeQHjjQfjZCHj+r2HrUqcGEYeLhvXilb87n+/MHM6Ssv1c9vN3efh/11HbGE7oWzDGJIdE1ggCOBf26TgJYClwk6qujikzA6cD+VYRKQBWAmNUdV9b57UawXHsXufcpPbJAmesoz5nO81GZ90AafF9w99d08BPX1nPc8sr6ZWTzgMzh3PtmP52Y5oxnZyXPx+dCTyC0/4/V1V/LCIPActUdaE4V5f/C8wAIsCPVfWZ453TEkEcGmvhs9/Dksdg92pIz4UxNzn3JRTEV+FaWXGABxeu5tPKg4wfmM+DV43irKLcBAdujEkUu6EsVak6s60tfQxWvwDRkHPH8sS7YNhM8B9/ruRoVHluRSU/fWUd++qamDVxAN+6bBg97S5lYzodSwQGanfDyt85dy8f3Ao5/WDy38KkuyFw/At7dUOIX76xkSc+2EJmmp9vXjKUOZMHEvQnsovJGNOeLBGYI6IRZ1yjj/8bNr8DPc6Ay/7FqSGcoB9g0+4afvjiGt7fuJchvbJ58OpRTB3cPnM4G2MSyxKBad2mN+CV78De9U6T0Yx/hd6jjnuIqvL6ml386C9r2Lq/nhmj+vDdK0YwoEe3DgraGHMqLBGYtkVCTnPROz+BhoPOcNgXffeE9yI0hCI89v5mHn37C6Kq/PUFg/ibCwaRmWZ3JxuTjCwRmBM7tB/eedjpWE7Lhgv/CSZ+DQLHn+Rmx8F6/vXldSz8dDv9cjP47hUjmXlWH/u5qTFJxhKBid+e9fDqd5xmo56D4bIfw9DLT9h/8PHmfTz44hrW7qjm3DN68ODVoxjep3sHBW2MORFLBObkbXjNSQj7NsKgi+Hyf4Vew497SCSqLFhSwX+8tp7q+hBzzh3INy8dSl43mzrTGK9ZIjCnJhKCJf8P3n3YuUlt4p1w4QPQrcdxD6s61MTPXt/AUx+Vk5sZ5FuXD2PWxGL8NiGOMZ6xRGBOT90+pzN52VxI7+4kg4l3gj943MPW7qjmwYWr+bhsPyP7dueH14xiYsnxk4gxJjEsEZj2sWsNvPqAc/9BwTC4/Ccw5JLjHqKq/OWzHfz4L2vZcbCBq0f344GZw+mbm9kxMRtjgHYYfVRE7hOR7uJ4XERWiMhl7RumSXq9R8KcF2DWAme4iqevh6dugD0b2jxERLjy7H68ef8FfOPiwbyyeicX/8e7PPr2JhpCkQ4M3hjTlnjHCLhDVauBy4B8YA5wzPzDJgWIwPCZ8LcfO3ckb/0Yfj0Z/vfbUH+gzcO6pQX4+8uG8ebfX8D5Qwv491fXc9nP3+P1NbvobLVSY7qaeBNBcy/fTOB37lDS1vOXygJpMOVeuHcFjL0FlvwGfjnW6VyOtD2PwYAe3fjNnAn87s5JpAV8fO3JZdzxxFK27j/UgcEbY2LF1UcgIvOA/kApMBpnWOl3VHV8YsM7lvURJKmdn8ErD8CW96FwBMz4ifOz0+MIRaLM/2ALP3t9A1FV7ps+lLvOK7XB7IxJgNPuLBYRHzAG2KyqVSLSAyhS1VXtG+qJWSJIYqqw7iV47XtwYAsM/ZLTfFQw+LiHbauq54cLV/Paml0M653DT758JuMH2q+LjGlP7TFV5WRgvZsEbgG+BxxsrwBNFyECI66Cry+BSx50age/Ohde/S7UtzkNNf3zMvntVyfw/746gZqGENf/+kMeeH4VVYeaOix0Y1JZvDWCVThNQmcDTwCPATeq6gUJja4VViPoRGp2wVs/gpVPOTehXfw9GHcr+NoemK6uMcwjb2xg7uIt5GUG+d6VI2yqTGPaQXvUCMLqZIxrgP9S1UeBnDheeIaIrBeRTSLy7Vb23yYie0TkE3e5K854TGeQ0xuu+S+4+x3nvoOXvgn/fR5sfrfNQ7LSA3z3ipEsvGcqA3p045vPfsotj3/M5j21HRa2Makm3kRQIyIP4Pxs9C9un8FxbysVET/wKPAlYCQwW0RGtlL0WVUd4y6PnUTsprPoNwZufxm+8gQ01sCTV8MzN8O+L9o8ZFS/XP74N1P40bVnsqryIDMeeZ9H3thAY9juPTCmvcWbCP4KaMS5n2AnUAT8+wmOmQRsUtXNqtoEPINTozCpSARGXQf3LIWL/xm+eNvpP3jtn6GhutVD/D5hzrkDefP+C7j8zD488sZGvvTI+3ywaW8HB29M1xZXInAv/k8DuSJyJdCgqk+e4LD+wNaY9Up3W0vXi8gqEXlORAa0diIRuVtElonIsj179sQTsklWwQw4/1tw73I48wb44Jfwn+Ng+XxnGs1W9MrJ4D9nj2X+HZMIR5WbHvuYbz77CXtrGzs4eGO6pniHmLgRWAJ8BbgR+FhEbmiH138RKFHVs4HXgfmtFVLV36rqBFWdUFhY2A4vazzXvS9c92v42tvQYxC8+A347QVQ9n6bh1wwtJDXvnk+91w0mJdWbWf6/32XZ5ZUEI3ancnGnI54m4a+C0xU1VtV9as4zT7/fIJjtgGx3/CL3G2Hqeo+VW3+WvcY0OE3qBmP9R8Hd7wCN8x1fmI6/0p49hbYX9Zq8Yygn29dPoz/ve88hvXJ4dvPf8aNv/mQDbtqOjhwY7qOeBOBT1V3x6zvi+PYpcAQESkVkTRgFrAwtoCI9I1ZvRpYG2c8pisRgTOvd/oPLvoebHoTHp0Er/+gzf6Dwb1yePbuc/npDWfzxZ5aZv7iff7tlXXUN1lnsjEnK95E8IqIvOr+3PM24C/Ay8c7QFXDwD3AqzgX+N+r6moReUhErnaLfUNEVovIp8A3gNtO5U2YLiKYCRf8g9t/cD0sfgT+czyseLLV/gMR4cYJA3jz/gu5bmx/fv3OF1z683d5e93uVk5ujGlL3PMRiMj1wFR39X1V/VPCojoOu6EshWxb7oxftPVj6HM2zHgYSqa2Wfyjzfv47p8+44s9dcw8qw8/uGoUvbtndGDAxiQvm5jGdF6q8PkfnWai6koYeQ1c+hDkl7RavDEc4f+9t5n/fGsTQb+Pb102lDmTS2yaTJPyTjkRiEgN0FoBAVRVu7dPiPGzRJCimg7Bh/8Fi34O0TBM/jqcdz+kt36De/m+Or73wue8v3EvZxfl8pPrzuLM/rkdHLQxycNqBKbrOLgN3nwIVj0DWb1g+vdhzM3gO7a7S1V5cdUOHnpxDfvrGrl1Sgn3XzaM7PSAB4Eb4y1LBKbrqVwOr3wbKpc4/Qdf+jcYOKXVogfrQ/z7q+t4+uMKeudk8ODVI7l8VB8byM6klPYYdM6Y5FI0Hu58Da5/HA7tg3lfgt/f6syD0EJuZpB/ufYsnv+bKeRnpfF/nlrBXfOXUXnAZkUzBqxGYLqCpkPwwX86/QcadfsP/r7V/oNwJMq8xc6saAB/d8kQ7phms6KZrs+ahkxqOLgN3vwhrHoWsnvD9B/A6Nmt9h9sq6rnB39ezRtrdzG8Tw4/vu4sxg/M9yBoYzqGJQKTWrYudfoPti2DvmOc+w8GTm616Kurd/LgwtXsONjA9OG9uPO8Uiaf0dP6D0yXY4nApJ5oFD5/zrn/oGa7MwT2JT+E/IHHFK1rDPPb9zbzu4/K2V/XxMi+3blzWilXje5HWsCajEzXYInApK6mOlj8S1j8C6f/YMq9MO2bkJ59TNGGUIQXVm7j8UVlbNxdS2FOOrdOHshN5wykR1aaB8Eb034sERhzsBLe+CF89nvI7gOX/ADOntXm/QfvbdzL44vKeG/DHtIDPr48rog7p5UwuNcJZ2g1JilZIjCm2dal8Mo/OeMY9Rvr9B8Un9tm8Q27api7qIznV26jKRzlwmGF3DmtlGmDC6wfwXQqlgiMiRWNwmd/gDcedPsPvgyX/hDyits8ZG9tI09/VMHvPtrC3tomhvfJ4Y5ppVw9uh8ZQX/HxW7MKbJEYExrmuqcvoPFv3AGtxs+0+lUHnwppHVr9ZDGcISFn2zn8UVlrNtZQ0F2GrecO5Bbzh1IQXZ6B78BY+JnicCY46na6sx9sPoFOLQXglkw9HIYdW2bSUFV+eCLfTz2/mbeXr+HtICP68b0587zShna2/oRTPKxRGBMPCJhKF8Ma16ANQuPJIVhM2DktTDkUmfynBY27a5l3uIy/riikoZQlPOGFHDntFIuGFpo/QgmaVgiMOZkRcJQvsipJaxd6Ixn1JwURl0Hgy85JikcqGvif5ZUMP+DLeyuaWRIr2zumFbKdWP7Wz+C8ZxniUBEZgC/APzAY6r6cBvlrgeeAyaq6nGv8pYITIeLhGHL+0dqCvX7IS0bhsYmhSMzoTWFo7y0yulHWL29mh5ZadxyTjG3TB5IrxybMc14w5NEICJ+YANwKVCJM5n9bFVd06JcDs4cyGnAPZYITFJrTgqr/wRrXzySFIZ9yWk+ikkKqspHm/fz+KIy3ly3i6DPx9Vj+nHntFJG9O3wOZ1MivMqEUwGHlTVy931BwBU9V9blHsEeB34B+BblghMpxEJxSSFl9ykkOMkhVHXwqDph5NC2d465i0u4w/LKqkPRZg6uCd3TivlwqG98Nk0mqYDeJUIbgBmqOpd7voc4BxVvSemzDjgu6p6vYi8QxuJQETuBu4GKC4uHl9eXp6QmI05ZZEQlL3nJIV1L0H9gZikcB0MuhiCGRw8FDrcj7CzuoEzCrO4Y2op148rIjPN+hFM4iRlIhARH/AWcJuqbjleIohlNQKT9CIhKHvX7Wh+ERqqIL37UUkhJEFe/mwHjy8qY1XlQfK6Bbn5nGK+OrmE3t2tH8G0v6RsGhKRXOALoNY9pA+wH7j6eMnAEoHpVCIh2PwurHGbjw4nhZkw6lr0jItYtu0Qj79fxqtrdgIwqDCbsQPyGFOcx9gB+QztnU3AJs4xp8mrRBDA6SyeDmzD6Sy+SVVXt1H+HaxGYLqycFNM89GL0HDQSQrDr4CR11KRfw4LP9/LyooqVm6tYn9dEwCZQT9nF+W6iSGPscX5VmswJ+14iSCQqBdV1bCI3AO8ivPz0bmqulpEHgKWqerCRL22MUkpkAZDLnGW8M/d5iO3T+HTBRSn53LPkEvhjBHohEHsDPRjRU0eS3eEWbm1irmLyghFnC9ufXMzGFucxxg3MZzZL9f6GMwpsxvKjPFauAk2v+MkhS/egtqdR+/PKoQeZxDJK2VXsB8bmgpZWp3PW3uyWHvAufj7fcKIvjlOYhiQz5jiPEp7ZtkvksxhdmexMZ1JYy0cKIP9m2MWd71621FFoxn51HQrZpuvL2sbC1hancf6pkK2aG+iGT0YXZx/uL9hTFEe+TbBTsqyRGBMVxGqhwNbWiSJzbBvMxzcChz5e673ZbNV+rC+qYAt2ody7U1j91LyioYxpLSUMcX5DO/T3abjTBGe9BEYYxIgmAm9RjhLS+FGOFB+ODlk7t/M0P2bGbxvM3JwKaIRqAc2Qu2GDMq1N2/Rh0PZA0krHESP4uGUDjubPv0GIj7rb0gllgiM6SoC6VA41Fli+MD5GWtVBewvQ/d/ATs2ULBjA32qttD90HKC5WEoB96HBtLYH+hNY1oekfQ8yMzH3y2ftJyeZOb2Iju/gLTsnpCZ7y49ID0HbKTVTssSgTGpwB+EnoOg5yCES8gGspv3RcI07d9K5Refs7t8LfW7NhKs2UZ6fTXdaivIlbXkUUu2NLR5+ih+GoPdCaflEc3Mx5/Vg2B2D9KyeyLdesQkjZilWw/n57OWQDxnicCYVOcPkFZYyhmFpZxx7lVH7YpElf11TZTXNLC3qoaD+/dQW7WHhuq9NNXsI1K3H2nYj7/xIFn11eQ11JFbXUu+bCJPasmjjmypb/OlVfxEM3KRzB74urm1i9hkkZ4N/jSntuNPd36Ce9Rjesz+mHL+4JHnPusDORFLBMaYNvl9QmFOOoU56dAvFyhqs2xdY5jdNY3sqWlkS00De2oa2V3TyL7qOuoO7iVUs49I3T6k4QC5Wkee1JIrteSHasmrraXAf4ievi/Ik1pytIbMaF37vAlf4CSSSBtJR06QTOL60c0JysRzjpHXwIBJcbzWybFEYIxpF1npAUrTA5QWZB23XDgSZV9dk5sonISxpbqRpbWN7K5uZE+ts31/dR3+cD1phEkjRJqECBIhjRDphEgTdzsxjxIiU8Jk+SN080fI9Efo5guTKREyJEyGhskIh0mPhEkLNR/bRJA6ghoi4C7+aAi/NuGPNuGLNOFcxAUBiGnJOrpRK44mrhM2g51gf8FQSwTGmM4v4PfRu3uGO0xGbpvlVJWmSJSGpigN4Qj1TRHqQ87S4C71TdEW6837oxyI2dZ8fEPoyP76UIQGt3w4emo/ow/4hIygn4ygj/SAn/Sgj4yAs54R9JMe8Ln7WytzZFtz+dj1dLfMkXP4yAz6E3LRtkRgjElKIuJcOAN+cgkm9LVCkeiRJNFK4ml0k0dj2HlscNcbwhEa3ceGkPM8tkx1Qyhmv7OtMRylKRw9pTh/dM0o5kwuad83jyUCY4wh6PcR9PvIyUhswmkWjSqNYTehHJVMjiSL5lpObHIZP7BHQuKxRGCMMR3M5xMy0/xJM1Cg/a7KGGNSnCUCY4xJcZ1u0DkR2YNzM3xnVgDs9TqIJGKfxxH2WRzNPo+jnc7nMVBVC1vb0ekSQVcgIsvaGgUwFdnncYR9Fkezz+Noifo8rGnIGGNSnCUCY4xJcZYIvPFbrwNIMvZ5HGGfxdHs8zhaQt79Y8QAABkkSURBVD4P6yMwxpgUZzUCY4xJcZYIjDEmxVki6EAiMkBE3haRNSKyWkTu8zomr4mIX0RWishLXsfiNRHJE5HnRGSdiKwVkclex+QlEfmm+3fyuYgsEJEMr2PqKCIyV0R2i8jnMdt6iMjrIrLRfcxvr9ezRNCxwsD9qjoSOBf4uoiM9Dgmr90HrPU6iCTxC+AVVR0OjCaFPxcR6Q98A5igqmcCfmCWt1F1qCeAGS22fRt4U1WHAG+66+3CEkEHUtUdqrrCfV6D84fe39uovCMiRcAVwGNex+I1EckFzgceB1DVJlWt8jYqzwWATBEJAN2A7R7H02FU9T1gf4vN1wDz3efzgWvb6/UsEXhEREqAscDH3kbiqUeAfwRObXD2rqUU2APMc5vKHhOR40/11YWp6jbgP4AKYAdwUFVf8zYqz/VW1R3u851A7/Y6sSUCD4hINvBH4O9UtdrreLwgIlcCu1V1udexJIkAMA74taqOBepox6p/Z+O2f1+DkyD7AVkicou3USUPdX73326//bdE0MFEJIiTBJ5W1ee9jsdDU4GrRWQL8AxwsYg85W1InqoEKlW1uYb4HE5iSFWXAGWqukdVQ8DzwBSPY/LaLhHpC+A+7m6vE1si6EAiIjhtwGtV9Wdex+MlVX1AVYtUtQSnE/AtVU3Zb3yquhPYKiLD3E3TgTUehuS1CuBcEenm/t1MJ4U7z10LgVvd57cCf26vE1si6FhTgTk4334/cZeZXgdlksa9wNMisgoYA/zE43g849aMngNWAJ/hXKtSZrgJEVkAfAgME5FKEbkTeBi4VEQ24tSYHm6317MhJowxJrUlrEbQ2g0RLfaLiPxSRDaJyCoRSeX2UGOM8Uwim4ae4NgbImJ9CRjiLncDv05gLMYYY9qQsETQxg0Rsa4BnlTHR0Bec4+4McaYjhPw8LX7A1tj1ivdbTtaL+4oKCjQkpKSBIZljDFdz/Lly/e2NWexl4kgbiJyN07zEcXFxSxbtszjiIwxpnMRkfK29nn589FtwICY9SJ32zFU9beqOkFVJxQWtprQjDHGnCIvawQLgXtE5BngHJyxRI7bLGSMMZ2dqhKKKOFolFBYCUWjhCNKKBIlFIkSjjY/V8Luo7M9ytDeORTld2v3mBKWCNwbIi4ECkSkEvgBEARQ1f8GXgZmApuAQ8DtiYrFU5Ew7N8Mu9fA7rWwZy3U7gF/AHxB8AfBF3AWf9DddhL7DpdpuS9m3Rdoe19rr+Hzg4jXn1zyikYhGoJIyH0Mx6yHnaXVfc3r4bb3aaTD3oaqHh6sRtUZuOao+4rEB+J3Hpv/T7jr6nO3H7XElo3Z7q6rNJ/jxGWb90dUCOMjohBGnPWoj7CKs019hKMQwkckCuGouz3qXGjDESUSVcJR5zEUiR61Ho46F9vY9Yh73JFtznpTpHl7lKZwy+3OOZrC0cPnPHIBV0Lh6OELfjh66vdu/cu1Z3LLuQNP69+9NQlLBKo6+wT7Ffh6ol6/w0WjcLDCudg3X/R3r4W9GyDS5BYS6HEG5PR1/vibDp3chaT9xpg6sbgSkT/uxKO+AFFfgAh+Z5EAYfcxqlE0GgWNopEIqlGIOo8ajTjb3W3EbCMaBW0u7xyPuvs0ikQjwJF9cnifHn4uGgWcfb5oBJ+Gj1n8GsankcOPvi4yWKq4S1cTVSGKEMGHuo9RfESRYx6b96k2Pxei4kNxlqgIio+IBAgRJCxBwr50whIk4gsSlnQiviBRXxpRXxoRfzrRYJBoejpRfzr401B/GhpIA38G6k9DAukQSEeCGUggHQmmI/4MfMF0/MF0JC2DYCBI0C8EfD6Cfp/z3O9jQH5mQj6zTtFZnFRUoWaHe7Ffd+TCv2c9hOqOlMsthl7DYfB06DUSeo2AgqEQPI1/yGjkON82T/BNNBo5/FwjIXYcqKG6tp5IuAmNhIiGQ0QjocPPcZ83lycaQZrPFw7ji4QRDSHRMD5tRKKHYi6azqOfMH6N4FzyowQIEyBCkAjpRMiWY7/9RrT5D9h35A/z8OOx+xQhqrHlfIcvAEf+0FvZp7EXiiAq6UQIEBEfzRFHJHA4aTUvUZ+bvOTI9qi7OPsDqPgPb3PWA0TFSYYRcZLkkW1+N1EGifoDiPicBUDAJ+JcsAVE5Miju+3IfncfzrpPjt7mnMvd38Y5fG6PoQiIivMJavTIY+zzo7Y5ifXwozqfauvHtCh7zDGx251/tYBP8RPFL86jDyXgAx9R/Ljb3H1+FBHFTwQfOP/iovjdOAOH//c4/0N82pwqNCaWI186aH4eDUG4CSKNEK6H8EH3ubs0NTmP0dCp/33HEj+4CQN/OgTSnMcLvw1n3dA+rxHDEsHx1O07ukmn+aLfcPBImezezkV+3Fedx14joXAYZHRv/3h8fmcJpJ/0oXtqGlm0aQ/vb9jLexv3src2vqHug34hze8jLeB8M0kLuEua+xizrXl/eivb0lo+Bnyk+YQ0f5SAz4f4Avj9PnwCPp/gF8Engs+H89znrPt94pRxn8euB3zNxzQf38q5msvEnEusGcy0l2j06AQRaYxJIA0xz5v3uwkk3BDzvLHtc2S22+yUR7FEANBQDXvWxTTpuN/262JGec3Icy7yZ97gXvBHQOEIyOrpXdzH0RCKsLz8AO9t2MN7G/eydocz7UGPrDTOG1LAeUMKGVSY5Vy4Az7S/H6CATn6wu7z4fPZRdKYuPl84Ms8vZq/B1IrETQdgr3r3SadmHb86sojZYJZTpPO0MuONOkUjoCcPkndgaqqbNxdy3sb9vD+xr18XLaPhlCUoF+YMLAH/zhjGOcPKWRk3+52cTfGHCV1EsHiX8Lr3+dwh6s/3WmzHzjlSJNOrxGQO+BIY2mS21/XxKJNe3nfvfjvrG4AYFBhFrMmFnP+0ALOKe1JVnrq/DMbY05e6lwhiiY6HS3NF/38UueXLZ1IUzjKiooDh7/1f779IKqQmxlk2uACzh9awLQhhfTP61zVUmOMtzrXlfB0DJzsLJ2IqrJ5b93hb/wfbt7HoaYIAZ8wrjifv79kKOcNLeSs/rn4rbnHGHOKUicRdBIHD4VY/MVe3t+4h/c27GVbVT0AJT27cf24Is4bUsDkQT3JyQh6HKkxpquwROCxUCTKJ1ureN/9dc+qyiqiCjnpAaYM7snfXjSI8wYXUtyz/W8rN8YYsETgifJ9dby30enk/fCLfdQ0hvEJjBmQx70XD+H8oQWMLsoj4O8cndbGmM7NEkEH++kr6/jVO18A0D8vkytH9+P8IQVMGVRAbjdr7jHGdDxLBB3oQF0Tjy8q49KRvfnOzBGU9Oxmd7UaYzxniaADLVhaQWM4yrcuG0ZpQXxDPBhjTKJZI3QHCUWiPPlBOdMGFzCsT47X4RhjzGGWCDrIK5/vZGd1A3dMK/E6FGOMOYolgg4yd3EZpQVZXDi0l9ehGGPMUSwRdICVFQdYWVHFbVNKbMA3Y0zSsUTQAeYt3kJORoAbxhd5HYoxxhzDEkGC7ThYz8uf7eCvJgywUUCNMUnJEkGC/e7DcqKq3DqlxOtQjDGmVZYIEqi+KcKCJRVcNrIPA3rYWEHGmORkiSCBXvhkGwcOhbh9aonXoRhjTJssESSIqjJvcRmj+nVnUmkPr8Mxxpg2WSJIkMWb9rFhVy23Ty218YSMMUktrkQgIs+LyBUiYokjTnMXl1GQncZVo/t6HYoxxhxXvBf2XwE3ARtF5GERGZbAmDq9zXtqeWvdbm45dyDpAb/X4RhjzHHFlQhU9Q1VvRkYB2wB3hCRD0TkdhGxQfRbmP/BFtL8Pm4+Z6DXoRhjzAnF3dQjIj2B24C7gJXAL3ASw+sJiayTOlgf4g/LK7lqdD8Kc9K9DscYY04orltdReRPwDDgd8BVqrrD3fWsiCxLVHCd0R+WbeVQU8R+MmqM6TTiHfPgl6r6dms7VHVCO8bTqYUjUeYt3sKk0h6c2T/X63CMMSYu8TYNjRSRvOYVEckXkb9NUEyd1htrd7Gtqp47ppZ6HYoxxsQt3kTwNVWtal5R1QPA1xITUuc1d/EWivIzuXRkb69DMcaYuMWbCPwSc1eUiPiBtBMdJCIzRGS9iGwSkW+3sr9YRN4WkZUiskpEZsYfenL5fNtBlpTt57YpJfhtzgFjTCcSbyJ4BadjeLqITAcWuNva5CaLR4EvASOB2SIyskWx7wG/V9WxwCyc+xU6pbmLy8hK83PjxAFeh2KMMScl3s7ifwL+Gvgbd/114LETHDMJ2KSqmwFE5BngGmBNTBkFurvPc4HtccaTVHbXNPDSpzuYPWkA3TPstgpjTOcSVyJQ1Sjwa3eJV39ga8x6JXBOizIPAq+JyL1AFnDJSZw/aTz9UQVNkSi3WSexMaYTinesoSEi8pyIrBGRzc1LO7z+bOAJVS0CZgK/a208IxG5W0SWiciyPXv2tMPLtp/GcISnPy5n+vBelBZkeR2OMcactHj7CObh1AbCwEXAk8BTJzhmGxDbYF7kbot1J/B7AFX9EMgAClqeSFV/q6oTVHVCYWFhnCF3jBc/3cHe2iZut9qAMaaTijcRZKrqm4CoarmqPghccYJjlgJDRKRURNJwOoMXtihTAUwHEJEROIkgub7yH4eqMndRGUN7ZzN1cE+vwzHGmFMSbyJodJtsNorIPSJyHZB9vANUNQzcA7wKrMX5ddBqEXlIRK52i90PfE1EPsX5JdJtqqqn9E48sKRsP2t2VHOHzTlgjOnE4v3V0H1AN+AbwI9wmoduPdFBqvoy8HKLbd+Peb4GmBpvsMlm7uIy8rsFuXZsf69DMcaYU3bCRODeD/BXqvotoBa4PeFRdQJb9x/itTW7+NsLB5ERtDkHjDGd1wmbhlQ1AkzrgFg6lfkfbMEvwpxzS7wOxRhjTku8TUMrRWQh8Aegrnmjqj6fkKiSXG1jmGeXbmXmWX3pk5vhdTjGGHNa4k0EGcA+4OKYbQqkZCJ4btlWahrD3DHNfjJqjOn84r2z2PoFXNGo8sQHWxhbnMeYAXknPsAYY5JcvDOUzcOpARxFVe9o94iS3Nvrd7Nl3yHuv2yY16EYY0y7iLdp6KWY5xnAdXTSAeJO19zFZfTNzWDGmX28DsUYY9pFvE1Df4xdF5EFwKKERJTE1u+sYfGmffzjjGEE/fHei2eMMcntVK9mQ4Be7RlIZzBvcRkZQR+zJxZ7HYoxxrSbePsIaji6j2AnzhwFKWN/XRN/WrmN68cXkZ91wsnZjDGm04i3aSgn0YEkuwVLKmgMR7l9SonXoRhjTLuKdz6C60QkN2Y9T0SuTVxYySUUifLkh1s4b0gBQ3qnfE40xnQx8fYR/EBVDzavqGoV8IPEhJR8Xv5sB7uqG+0GMmNMlxRvImitXLw/Pe3UmuccOKMgiwuGJNekOMYY0x7iTQTLRORnIjLIXX4GLE9kYMliRUUVn1Ye5PapJfh8NueAMabriTcR3As0Ac8CzwANwNcTFVQymbe4jJyMAF8eV+R1KMYYkxDx/mqoDvh2gmNJOtur6vnfz3dy57RSstJToiXMGJOC4v3V0Osikhezni8iryYurOTw5IflqCpfnTzQ61CMMSZh4m0aKnB/KQSAqh6gi99ZXN8UYcGSCi4f1Yei/G5eh2OMMQkTbyKIisjhcRVEpIRWRiPtSp5fWcnB+pD9ZNQY0+XF2/D9XWCRiLwLCHAecHfCovKYqjJv8RbO6p/LhIH5XodjjDEJFVeNQFVfASYA64EFwP1AfQLj8tT7G/eyaXctt08tQcR+MmqM6driHXTuLuA+oAj4BDgX+JCjp67sMuYuLqMwJ50rzu7rdSjGGJNw8fYR3AdMBMpV9SJgLFB1/EM6p027a3ln/R7mnDuQ9IDf63CMMSbh4k0EDaraACAi6aq6DuiSczXO/2ALaX4fN51jcw4YY1JDvJ3Fle59BC8Ar4vIAaA8cWF54+ChEM8tr+SaMf0oyE73OhxjjOkQ8d5ZfJ379EEReRvIBV5JWFQeeXZZBfWhCLdPtZ+MGmNSx0mPm6Cq7yYiEK+FI1Hmf1DOuWf0YGS/7l6HY4wxHcZmYHe9tmYX26rqucNqA8aYFGOJwDVvcRkDemQyfURvr0MxxpgOZYkAWFVZxdItB7htSil+m3PAGJNiLBEA8xZvITs9wI0TbM4BY0zqSWgiEJEZIrJeRDaJSKvzGYjIjSKyRkRWi8j/JDKe1uyubuClVdu5YXwRORnBjn55Y4zxXMJmWxERP/AocClQCSwVkYWquiamzBDgAWCqqh4QkQ4f2vqpj8oJR5XbppR09EsbY0xSSGSNYBKwSVU3q2oTzhSX17Qo8zXgUXd+A1R1dwLjOUZDKMLTH1cwfXhvSgqyOvKljTEmaSQyEfQHtsasV7rbYg0FhorIYhH5SERmJDCeYyz8dDv76pq4Y2pJR76sMcYkFa8n4g0AQ4ALcUY2fU9EzoqdDQ1ARO7Gnf+guLh9xgBSVeYuKmN4nxwmD+rZLuc0xpjOKJE1gm3AgJj1IndbrEpgoaqGVLUM2ICTGI6iqr9V1QmqOqGwsLBdgvto837W7ayxOQeMMSkvkYlgKTBEREpFJA2YBSxsUeYFnNoAIlKA01S0OYExHTZ3cRk9stK4ZkzL1ipjjEktCUsEqhoG7gFeBdYCv1fV1SLykIhc7RZ7FdgnImuAt4F/UNV9iYqpWfm+Ot5Yu4ubzykmI2hzDhhjUltC+whU9WXg5Rbbvh/zXIG/d5cOM/+Dcvwi3HLuwI58WWOMSUopd2dxTUOI3y/bypVn96V39wyvwzHGGM+lXCL4w7JKahvD3DHNRhk1xhhIsUQQiSrzP9zC+IH5nF2U53U4xhiTFFIqEby1bjfl+w7ZnAPGGBMjpRLB3EVl9MvN4PJRNueAMcY0S5lEsHZHNR9u3sdXp5QQ8KfM2zbGmBNKmSviG2t2kRn0M2vigBMXNsaYFOL1WEMd5t7pQ7huXH/yuqV5HYoxxiSVlKkRABTld/M6BGOMSToplQiMMcYcyxKBMcakOHGG++k8RGQPUO51HKepANjrdRBJxD6PI+yzOJp9Hkc7nc9joKq2Oo5/p0sEXYGILFPVCV7HkSzs8zjCPouj2edxtER9HtY0ZIwxKc4SgTHGpDhLBN74rdcBJBn7PI6wz+Jo9nkcLSGfh/URGGNMirMagTHGpDhLBB1IRAaIyNsiskZEVovIfV7H5DUR8YvIShF5yetYvCYieSLynIisE5G1IjLZ65i8JCLfdP9OPheRBSKSMlMKishcEdktIp/HbOshIq+LyEb3Mb+9Xs8SQccKA/er6kjgXODrIjLS45i8dh+w1usgksQvgFdUdTgwmhT+XESkP/ANYIKqngn4gVneRtWhngBmtNj2beBNVR0CvOmutwtLBB1IVXeo6gr3eQ3OH3p/b6PyjogUAVcAj3kdi9dEJBc4H3gcQFWbVLXK26g8FwAyRSQAdAO2exxPh1HV94D9LTZfA8x3n88Hrm2v17NE4BERKQHGAh97G4mnHgH+EYh6HUgSKAX2APPcprLHRCTL66C8oqrbgP8AKoAdwEFVfc3bqDzXW1V3uM93Au02w5YlAg+ISDbwR+DvVLXa63i8ICJXArtVdbnXsSSJADAO+LWqjgXqaMeqf2fjtn9fg5Mg+wFZInKLt1ElD3V+7tluP/m0RNDBRCSIkwSeVtXnvY7HQ1OBq0VkC/AMcLGIPOVtSJ6qBCpVtbmG+BxOYkhVlwBlqrpHVUPA88AUj2Py2i4R6QvgPu5urxNbIuhAIiI4bcBrVfVnXsfjJVV9QFWLVLUEpxPwLVVN2W98qroT2Coiw9xN04E1HobktQrgXBHp5v7dTCeFO89dC4Fb3ee3An9urxNbIuhYU4E5ON9+P3GXmV4HZZLGvcDTIrIKGAP8xON4POPWjJ4DVgCf4VyrUuYuYxFZAHwIDBORShG5E3gYuFRENuLUmB5ut9ezO4uNMSa1WY3AGGNSnCUCY4xJcZYIjDEmxVkiMMaYFGeJwBhjUpwlAmM6kIhcaCOtmmRjicAYY1KcJQJjWiEit4jIEvemv9+48ybUisjP3THy3xSRQrfsGBH5SERWicifmseJF5HBIvKGiHwqIitEZJB7+uyYeQeedu+cNcYzlgiMaUFERgB/BUxV1TFABLgZyAKWqeoo4F3gB+4hTwL/pKpn49wF27z9aeBRVR2NM05O88iRY4G/A0YCZ+DccW6MZwJeB2BMEpoOjAeWul/WM3EG+IoCz7plngKed+cRyFPVd93t84E/iEgO0F9V/wSgqg0A7vmWqGqlu/4JUAIsSvzbMqZ1lgiMOZYA81X1gaM2ivxzi3KnOj5LY8zzCPZ3aDxmTUPGHOtN4AYR6QWH54odiPP3coNb5iZgkaoeBA6IyHnu9jnAu+4MdJUicq17jnQR6dah78KYONk3EWNaUNU1IvI94DUR8QEh4Os4k8VMcvftxulHAGdI4P92L/Sbgdvd7XOA34jIQ+45vtKBb8OYuNnoo8bESURqVTXb6ziMaW/WNGSMMSnOagTGGJPirEZgjDEpzhKBMcakOEsExhiT4iwRGGNMirNEYIwxKc4SgTHGpLj/Dy+Lqg7vhIYdAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "etD-Ncl7kEYd",
        "outputId": "bcf8e252-53ac-4ae5-d88a-33cc42385396"
      },
      "source": [
        "predictions, labels = predict(test_dl, clf)\n",
        "\n",
        "accuracy = accuracy_score(predictions, labels)\n",
        "ACCURACY_STATS.append((3, \"under_sampling\", accuracy))\n",
        "\n",
        "print(\"accuracy: {:.3f}\".format(accuracy))\n",
        "print(\"confusion matrix\")\n",
        "print(confusion_matrix(predictions, labels))"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy: 0.968\n",
            "confusion matrix\n",
            "[[471   1]\n",
            " [ 17  68]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BWfqS5Mjkht9"
      },
      "source": [
        "## Oversampling Minority Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D1My4SG9kdHK",
        "outputId": "fad7de78-9548-4d58-a581-22b8a46ee2dc"
      },
      "source": [
        "major_df = spam_train_df[spam_train_df[\"label\"] == \"ham\"]\n",
        "num_major = len(major_df)\n",
        "num_minor = len(spam_train_df) - num_major\n",
        "\n",
        "sample_minor_df = spam_train_df[spam_train_df[\"label\"] == \"spam\"].sample(\n",
        "    num_major, replace=True, axis=0)\n",
        "\n",
        "over_df = pd.concat([major_df, sample_minor_df], axis=0)\n",
        "len(over_df)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6710"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RZ25RX3Gk8bi",
        "outputId": "824391ef-c6b4-450b-e847-c2d0948e0ecf"
      },
      "source": [
        "over_df[\"label\"].value_counts()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ham     3355\n",
              "spam    3355\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        },
        "id": "SvDjFYSQlDgo",
        "outputId": "b653801e-7f45-4ef2-a207-b27683a24768"
      },
      "source": [
        "over_df[\"label\"].value_counts().plot(kind=\"bar\")"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f685c691dd8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEMCAYAAAAoB2Y1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASP0lEQVR4nO3df4xd513n8fendtIWCsQhQ/DaLg7gFUpLccPgZAV/dFM1cdI/HLS73eQPapVIhlWy4kfFkiLY0JZIrYBW6ipEuBtvnVIaImgVU1yCN3QXqt00HrchiZNGGfJDttdJpjhNm60wOHz3j/sM3HVmPGN75l4zz/slHd1zvuc55z5Hufnc4+eccydVhSSpD68ZdwckSaNj6EtSRwx9SeqIoS9JHTH0Jakjhr4kdWTB0E/yuiQPJvmrJAeTvL/VP5Hk6SQPtWlzqyfJx5JMJ3k4yWVD+9qe5Mk2bV++w5IkzWX1ItocB66sqpeTnAd8Mcnn27pfqqo/PKn9NcCmNl0O3AFcnuRC4FZgEijgQJI9VfXiUhyIJGlhC4Z+DZ7eerktntemUz3RtQ24q233QJILkqwF3gbsq6pjAEn2AVuBT8+3o4suuqg2bty4iMOQJM06cODA16pqYq51iznTJ8kq4ADwg8DtVfWlJP8BuC3JfwbuB26pquPAOuDQ0OaHW22++rw2btzI1NTUYrooSWqSPDvfukVdyK2qV6pqM7Ae2JLkzcD7gB8Cfgy4EPjlJegrSXYkmUoyNTMzsxS7lCQ1p3X3TlV9HfgCsLWqjtbAceC/AVtasyPAhqHN1rfafPWT32NnVU1W1eTExJz/OpEknaHF3L0zkeSCNv964B3AV9s4PUkCXAc82jbZA7y73cVzBfBSVR0F7gOuSrImyRrgqlaTJI3IYsb01wK727j+a4B7qupzSf48yQQQ4CHgZ1v7vcC1wDTwLeA9AFV1LMkHgf2t3QdmL+pKkkYj5/JPK09OTpYXciXp9CQ5UFWTc63ziVxJ6oihL0kdMfQlqSOLejhLp7bxlj8ZdxdWlGc+9M5xd2FF8fO5dFbCZ9MzfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHVkw9JO8LsmDSf4qycEk72/1S5J8Kcl0kj9Icn6rv7YtT7f1G4f29b5WfyLJ1ct1UJKkuS3mTP84cGVV/QiwGdia5Argw8BHq+oHgReBG1v7G4EXW/2jrR1JLgWuB94EbAV+J8mqpTwYSdKpLRj6NfByWzyvTQVcCfxhq+8Grmvz29oybf3bk6TV766q41X1NDANbFmSo5AkLcqixvSTrEryEPACsA/4a+DrVXWiNTkMrGvz64BDAG39S8B3D9fn2Gb4vXYkmUoyNTMzc/pHJEma16JCv6peqarNwHoGZ+c/tFwdqqqdVTVZVZMTExPL9TaS1KXTununqr4OfAH4V8AFSVa3VeuBI23+CLABoK3/LuBvhutzbCNJGoHF3L0zkeSCNv964B3A4wzC/9+2ZtuBe9v8nrZMW//nVVWtfn27u+cSYBPw4FIdiCRpYasXbsJaYHe70+Y1wD1V9bkkjwF3J/kN4CvAna39ncAnk0wDxxjcsUNVHUxyD/AYcAK4qapeWdrDkSSdyoKhX1UPA2+do/4Uc9x9U1V/C/y7efZ1G3Db6XdTkrQUfCJXkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1ZMHQT7IhyReSPJbkYJKfa/VfT3IkyUNtunZom/clmU7yRJKrh+pbW206yS3Lc0iSpPmsXkSbE8B7q+rLSb4DOJBkX1v30ar6reHGSS4FrgfeBPwL4L8n+Zdt9e3AO4DDwP4ke6rqsaU4EEnSwhYM/ao6Chxt899M8jiw7hSbbAPurqrjwNNJpoEtbd10VT0FkOTu1tbQl6QROa0x/SQbgbcCX2qlm5M8nGRXkjWttg44NLTZ4Vabr37ye+xIMpVkamZm5nS6J0lawKJDP8kbgD8Cfr6qvgHcAfwAsJnBvwR+eyk6VFU7q2qyqiYnJiaWYpeSpGYxY/okOY9B4H+qqj4DUFXPD63/OPC5tngE2DC0+fpW4xR1SdIILObunQB3Ao9X1UeG6muHmv0k8Gib3wNcn+S1SS4BNgEPAvuBTUkuSXI+g4u9e5bmMCRJi7GYM/0fB34KeCTJQ632K8ANSTYDBTwD/AxAVR1Mcg+DC7QngJuq6hWAJDcD9wGrgF1VdXAJj0WStIDF3L3zRSBzrNp7im1uA26bo773VNtJkpaXT+RKUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOLBj6STYk+UKSx5IcTPJzrX5hkn1Jnmyva1o9ST6WZDrJw0kuG9rX9tb+ySTbl++wJElzWcyZ/gngvVV1KXAFcFOSS4FbgPurahNwf1sGuAbY1KYdwB0w+JIAbgUuB7YAt85+UUiSRmPB0K+qo1X15Tb/TeBxYB2wDdjdmu0Grmvz24C7auAB4IIka4GrgX1VdayqXgT2AVuX9GgkSad0WmP6STYCbwW+BFxcVUfbqueAi9v8OuDQ0GaHW22++snvsSPJVJKpmZmZ0+meJGkBiw79JG8A/gj4+ar6xvC6qiqglqJDVbWzqiaranJiYmIpdilJahYV+knOYxD4n6qqz7Ty823Yhvb6QqsfATYMbb6+1earS5JGZDF37wS4E3i8qj4ytGoPMHsHznbg3qH6u9tdPFcAL7VhoPuAq5KsaRdwr2o1SdKIrF5Emx8Hfgp4JMlDrfYrwIeAe5LcCDwLvKut2wtcC0wD3wLeA1BVx5J8ENjf2n2gqo4tyVFIkhZlwdCvqi8CmWf12+doX8BN8+xrF7DrdDooSVo6PpErSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6smDoJ9mV5IUkjw7Vfj3JkSQPtenaoXXvSzKd5IkkVw/Vt7badJJblv5QJEkLWcyZ/ieArXPUP1pVm9u0FyDJpcD1wJvaNr+TZFWSVcDtwDXApcANra0kaYRWL9Sgqv4iycZF7m8bcHdVHQeeTjINbGnrpqvqKYAkd7e2j512jyVJZ+xsxvRvTvJwG/5Z02rrgENDbQ632nz1V0myI8lUkqmZmZmz6J4k6WRnGvp3AD8AbAaOAr+9VB2qqp1VNVlVkxMTE0u1W0kSixjemUtVPT87n+TjwOfa4hFgw1DT9a3GKeqSpBE5ozP9JGuHFn8SmL2zZw9wfZLXJrkE2AQ8COwHNiW5JMn5DC727jnzbkuSzsSCZ/pJPg28DbgoyWHgVuBtSTYDBTwD/AxAVR1Mcg+DC7QngJuq6pW2n5uB+4BVwK6qOrjkRyNJOqXF3L1zwxzlO0/R/jbgtjnqe4G9p9U7SdKS8olcSeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUkQVDP8muJC8keXSodmGSfUmebK9rWj1JPpZkOsnDSS4b2mZ7a/9kku3LcziSpFNZzJn+J4CtJ9VuAe6vqk3A/W0Z4BpgU5t2AHfA4EsCuBW4HNgC3Dr7RSFJGp0FQ7+q/gI4dlJ5G7C7ze8Grhuq31UDDwAXJFkLXA3sq6pjVfUisI9Xf5FIkpbZmY7pX1xVR9v8c8DFbX4dcGio3eFWm6/+Kkl2JJlKMjUzM3OG3ZMkzeWsL+RWVQG1BH2Z3d/OqpqsqsmJiYml2q0kiTMP/efbsA3t9YVWPwJsGGq3vtXmq0uSRuhMQ38PMHsHznbg3qH6u9tdPFcAL7VhoPuAq5KsaRdwr2o1SdIIrV6oQZJPA28DLkpymMFdOB8C7klyI/As8K7WfC9wLTANfAt4D0BVHUvyQWB/a/eBqjr54rAkaZktGPpVdcM8q94+R9sCbppnP7uAXafVO0nSkvKJXEnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOnFXoJ3kmySNJHkoy1WoXJtmX5Mn2uqbVk+RjSaaTPJzksqU4AEnS4i3Fmf6/rqrNVTXZlm8B7q+qTcD9bRngGmBTm3YAdyzBe0uSTsNyDO9sA3a3+d3AdUP1u2rgAeCCJGuX4f0lSfM429Av4M+SHEiyo9Uurqqjbf454OI2vw44NLTt4VaTJI3I6rPc/ieq6kiS7wH2Jfnq8MqqqiR1OjtsXx47AN74xjeeZfckScPO6ky/qo601xeAzwJbgOdnh23a6wut+RFgw9Dm61vt5H3urKrJqpqcmJg4m+5Jkk5yxqGf5NuTfMfsPHAV8CiwB9jemm0H7m3ze4B3t7t4rgBeGhoGkiSNwNkM71wMfDbJ7H5+v6r+NMl+4J4kNwLPAu9q7fcC1wLTwLeA95zFe0uSzsAZh35VPQX8yBz1vwHePke9gJvO9P0kSWfPJ3IlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHRh76SbYmeSLJdJJbRv3+ktSzkYZ+klXA7cA1wKXADUkuHWUfJKlnoz7T3wJMV9VTVfV3wN3AthH3QZK6NerQXwccGlo+3GqSpBFYPe4OnCzJDmBHW3w5yRPj7M8KcxHwtXF3YiH58Lh7oDE55z+f/4w+m98334pRh/4RYMPQ8vpW+0dVtRPYOcpO9SLJVFVNjrsf0lz8fI7GqId39gObklyS5HzgemDPiPsgSd0a6Zl+VZ1IcjNwH7AK2FVVB0fZB0nq2cjH9KtqL7B31O8rwGEzndv8fI5AqmrcfZAkjYg/wyBJHTH0Jakjhr4kdeScezhLSy/JW4CNDP33rqrPjK1DEv/4W1zv5NWfzY+Mq089MPRXuCS7gLcAB4F/aOUCDH2N2x8Dfws8wj99NrXMDP2V74qq8pdMdS5aX1VvGXcneuOY/sr3v/35ap2jPp/kqnF3ojee6a98dzEI/ueA40CA8gxL54AHgM8meQ3w9/zTZ/M7x9utlc2Hs1a4JNPAL3LSuGlVPTu2TklAkqcZ/D2NR8ogGhnP9Fe+maryR+10LjoEPGrgj5ahv/J9JcnvM7hT4vhs0Vs2dQ54CvgfST7P///Z9JbNZWTor3yvZ/A/1PAFM2/Z1Lng6Tad3yaNgGP6ktQRz/RXuCSvA24E3gS8brZeVT89tk5JQJIJ4D/x6s/mlWPrVAe8T3/l+yTwvcDVwP9k8CcqvznWHkkDnwK+ClwCvB94hsFf19MycnhnhUvylap6a5KHq+otSc4D/rKqrhh339S3JAeq6kdnP5uttr+qfmzcfVvJHN5Z+f6+vX49yZuB54DvGWN/pFmzn82jSd4J/B/gwjH2pwuG/sq3M8ka4FcZ/BH6NwC/Nt4uSQD8RpLvAt4L/BfgO4FfGG+XVj6Hd1a4JK8F/g2Dn689r5Wrqj4wtk5JGhsv5K589zJ41P0E8HKb/u9YeyQBSb4/yR8n+VqSF5Lcm+T7x92vlc4z/RUuyaNV9eZx90M6WZIHgNuBT7fS9cB/rKrLx9erlc8z/ZXvfyX54XF3QprDt1XVJ6vqRJt+j6H79bU8PNNfoZI8wuDnFlYDmxj8zok/raxzRpIPAy8CdzP4rP57YA3wmwBVdWx8vVu5DP0VKsn3nWq9P62scWs/rTxrNogyu1xVju8vA0Nf0lgkeRfwp1X1jSS/BlwGfLCqvjzmrq1ojulLGpdfbYH/E8CVwH8F7hhzn1Y8Q1/SuLzSXt8JfLyq/gR/YnnZGfqSxuVIkt9lcAF3b3uQ0ExaZo7pSxqLJN8GbGXwN3KfTLIW+OGq+rMxd21FM/QlqSP+U0qSOmLoS1JHDH1J6oihL0kdMfQlqSP/D7ZfAMbE2ZrOAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DXwG4OQolGv5"
      },
      "source": [
        "train_ds = create_dataset(over_df)\n",
        "\n",
        "train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\n",
        "val_dl = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False)\n",
        "test_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False)"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "06qrjKeylOQw"
      },
      "source": [
        "clf = SpamClassifier(CLF_LAYER_DIMS)\n",
        "clf = clf.to(dev)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.AdamW(clf.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u2gYBcbClTro",
        "outputId": "497ba021-62ad-460b-9540-a41c437e99de"
      },
      "source": [
        "train_losses, train_accs, val_losses, val_accs = [], [], [], []\n",
        "for epoch in range(1, NUM_EPOCHS + 1):\n",
        "  train_loss, train_acc = train(train_dl, clf, loss_fn, optimizer)\n",
        "  val_loss, val_acc = validate(val_dl, clf, loss_fn) \n",
        "  print(\"EPOCH {:2d} | train_loss: {:.3f}, acc: {:.3f} | val_loss: {:.3f}, acc: {:.3f}\"\n",
        "    .format(epoch, train_loss, train_acc, val_loss, val_acc))\n",
        "  train_losses.append(train_loss)\n",
        "  train_accs.append(train_acc)\n",
        "  val_losses.append(val_loss)\n",
        "  val_accs.append(val_acc)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH  1 | train_loss: 0.560, acc: 0.892 | val_loss: 0.376, acc: 0.978\n",
            "EPOCH  2 | train_loss: 0.352, acc: 0.973 | val_loss: 0.338, acc: 0.983\n",
            "EPOCH  3 | train_loss: 0.336, acc: 0.982 | val_loss: 0.333, acc: 0.981\n",
            "EPOCH  4 | train_loss: 0.330, acc: 0.987 | val_loss: 0.332, acc: 0.981\n",
            "EPOCH  5 | train_loss: 0.327, acc: 0.989 | val_loss: 0.331, acc: 0.981\n",
            "EPOCH  6 | train_loss: 0.325, acc: 0.990 | val_loss: 0.331, acc: 0.981\n",
            "EPOCH  7 | train_loss: 0.325, acc: 0.990 | val_loss: 0.332, acc: 0.981\n",
            "EPOCH  8 | train_loss: 0.324, acc: 0.990 | val_loss: 0.331, acc: 0.983\n",
            "EPOCH  9 | train_loss: 0.323, acc: 0.991 | val_loss: 0.332, acc: 0.981\n",
            "EPOCH 10 | train_loss: 0.324, acc: 0.990 | val_loss: 0.332, acc: 0.981\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        },
        "id": "hJ3ByNQxlaIj",
        "outputId": "4b13b894-dd25-4e04-d33d-9d857eea9bd5"
      },
      "source": [
        "training_lossplots(\"Oversampling\", train_losses, train_accs, val_losses, val_accs)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEjCAYAAAAomJYLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxcdZnv8c9T1dVLkl6ykT0pQCCENRiSkAygogyCCjqoiDAJiuCMC87VO+r1XvUyd2aYuY6jc/UiEYWgCEgAzZ1xQEQ2IYkkQBAIYckesnTW7qTTW9Vz/zinuqs71elKurpPVff3/XrV65zzO79z6ulK6nzrLHXK3B0REZHuYlEXICIixUkBISIiOSkgREQkJwWEiIjkpIAQEZGcFBAiIpKTAkKkyJnZE2Z2fTj+STP7bdQ1ydCggJCSY2YLzexPZtZkZtvN7FYzq4u6roHg7ne7+8VR1yFDgwJCSoqZfRn4J+C/ArXAXGAa8KiZlRfwecoKtS6RUqWAkJJhZjXA/wS+4O4Pu3ubu28APgYkga+Y2SEzG5W1zEwz22VmiXD6U2a2xsz2mtkjZjYtq6+b2efM7A3gDQv8q5ntNLOGcK/l9LDvZWb2Qti+2cy+nbWeZLiu68J5e83ss2Z2rpm9ZGb7zOwHWf0XmtkzZvYDM9tvZq+Z2UU9vAYLzewP3Wr+rJm9Ea73h2Zm4by4mf1L+PevN7PPh/0VfpIXBYSUknlAJfBgdqO7HwB+A5wBLAP+Imv21cASd28zs8uB/wZ8BBgLPA3c0+05rgDmADOAi4ELgJMJ9lY+BuwO+x0E/hKoAy4D/srMrui2rjnAScDHge8B3wDeC5wGfMzMLuzW9y1gDPAt4MHsoOvFB4BzgTPDGv88bP8M8H7gbOCc8G8TyZsCQkrJGGCXu7fnmLctnP8L4BMA4Sfpq8I2gM8C/+jua8J1/ANwdvZeRDh/j7sfAtqAamA6YOFy2wDc/Ql3/5O7p939JYKgyd7gA/yduze7+28JAuUed9/p7lsJwmlmVt+dwPfCvaL7gLUEwZOPW9x9n7tvAh4nCAQIwuL77r7F3fcCt+S5PhFAASGlZRcwpodDJBPC+Q8A55nZBIJP/2mCjTEE5yq+Hx6K2QfsAQyYlLWezZkRd/898APgh8BOM1sUHubCzOaY2eNmVm9m+wnCZ0y3mnZkjR/KMT0ia3qrd71z5kZgYg+vQ3fbs8abstY7Mfvv6TYu0isFhJSSZUALwSGiDmY2guBQymPhJ+XfEhzWuRq4N2vDuxm40d3rsh5V7v5s1uq63N7Y3f/N3d9JcMjpZIKT4xDslSwFprh7LfAjgrA5VpMy5w5CU4G3+7A+CPaqJmdNT+nj+mSIUUBIyXD3/QQnqf+PmV1iZgkzSwK/BLYAPwu7/oLg/MCVdB5egmAj/nUzOw3AzGrN7KM9PV94UnlOeIL7INBMsEcCwaGnPe7ebGazCcKoL44Dvhj+TR8FTiU4r9IXvwRuMrNJ4WXAX+3j+mSIUUBISXH3fyY40fwdoAFYQbBncJG7t4TdlhKcHN7u7quzln2I4BLZe82sAXiZYM+jJzXAj4G9BId8dgP/O5z318DNZtYIfJNgY9wXK8KadwF/D1zp7ruPvEivfkywN/US8AJB4LQDqT6uV4YI0w8GiUTLzBYC17v7n/Xz87wf+JG7T+u1swjagxAZtMysyswuNbMyM5tEcPnsQ1HXJaVDASEyeBnBOZu9BIeY1hAcDhPJiw4xiYhITtqDEBGRnBQQIiKSkwJCRERyUkCIiEhOCggREclJASEiIjkpIEREJCcFhIiI5KSAEBGRnBQQIiKSkwJCRERyUkCIiEhOCggREclJASEiIjmVRV1AoYwZM8aTyWTUZYiIlJRVq1btcvexueYNmoBIJpOsXLky6jJEREqKmW3saZ4OMYmISE4KCKA9lSaV1i/riYhkG/IBsWl3Exf88+M8/PL2qEsRESkqQz4gJo2sIh437nx2fdSliIgUlSEfEPGYseC8JM9t2MvLW/dHXY6ISNEY8gEB8NFZU6hKxFn87IaoSxERKRoKCKC2KsFHzpnEr1e/zZ6DrVGXIyJSFBQQoQXzkrS2p7nnj5uiLkVEpCgoIEInj6tm/jtG8/PlG2lPpaMuR0QkcgqILAvOS7JtfzO/fXVH1KWIiEROAZHlolPHMXlkFXfqZLWIiAIiWzxm/OV50/jj+j28+nZD1OWIiERKAdHNx2dN1SWvIiIoIA5TOyzBh8+ZxK9e3MpeXfIqIkOYAiKHBeclaWlPc+9zm6MuRUQkMgqIHE4ZX828E0fzs2UbdMmriAxZCogeLJiX5O39zfxujS55FZGhSQHRg/eeOo5JdVXc8cyGqEsREYmEAqIHmUteV6zfw5ptuuRVRIYeBcQRfPzcKVQmYrrkVUSGJAXEEdQNK+fDM3XJq4gMTQqIXiyYl6S5Lc19K3XJq4gMLZEEhJldYmZrzexNM/tajvkLzazezF4MH9dHUSfA9PE1zD1hFD9bpru8isjQMuABYWZx4IfA+4EZwCfMbEaOrve5+9nh4/YBLbKbhfOOZ+u+Q/xuzc4oyxARGVBR7EHMBt5093Xu3grcC1weQR15e++pxzGprkonq0VkSIkiICYB2Qf0t4Rt3f2Fmb1kZkvMbMrAlJZbWTzGtedNY9m63by2XZe8isjQUKwnqf8fkHT3M4FHgcW5OpnZDWa20sxW1tfX92tBH581hYqyGIuf3divzyMiUiyiCIitQPYeweSwrYO773b3lnDyduCduVbk7ovcfZa7zxo7dmy/FJsxcnhwyetDL2xhX5MueRWRwS+KgHgOOMnMjjezcuAqYGl2BzObkDX5IWDNANbXo8wlr7/UJa8iMgQMeEC4ezvweeARgg3/L939FTO72cw+FHb7opm9YmargS8CCwe6zlxOnVDDnONHcdeyjaTSHnU5IiL9KpJzEO7+G3c/2d1PdPe/D9u+6e5Lw/Gvu/tp7n6Wu7/b3V+Los5cFs5LsmXvIR7TXV5FZJAr1pPURet9M8YxsbaSO3XJq4gMcgqIo1QWj3HNedN49q3dvL6jMepyRET6jQLiGFx17lQqymLaixCRQU0BcQxGDS/n8rMn8tDzW9nf1BZ1OSIi/UIBcYwWzEtyqC2lS15FZNBSQByj0ybWMjs5iruWb9AlryIyKCkg+mDh/CSb9xzi96/pLq8iMvgoIPrg4hnjmFBbqbu8isigpIDog7J4jGvmTuMPb+7iDV3yKiKDjAKijz4xeyrlZTEWL9sQdSkiIgXV54Aws5vMrMYCPzGz583s4kIUVwpGDS/n8rMm8sCqrew/pEteRWTwKMQexKfcvQG4GBgJXAvcUoD1lozMJa/365JXERlEChEQFg4vBX7m7q9ktQ0Jp0+q5dzkSN3lVUQGlUIExCoz+y1BQDxiZtVAugDrLSkL5iXZtKeJJ9bqklcRGRwKERCfBr4GnOvuTUACuK4A6y0pf37aeMbX6C6vIjJ4FCIgzgPWuvs+M7sG+O/A/gKst6Qk4jGumTuVp9/YxZs7dcmriJS+QgTErUCTmZ0FfBl4C7irAOstOR2XvD67MepSRET6rBAB0e7uDlwO/MDdfwhUF2C9JWf0iAo+eOZEHnh+Cw3NuuRVREpbIQKi0cy+TnB563+YWYzgPMSQtHBekqbWFPev3BJ1KSIifVKIgPg40ELwfYjtwGTgfxdgvSXpjMm1vHPaSO5atoG0LnkVkRLW54AIQ+FuoNbMPgA0u/uQPAeRsXBeko27m3jidV3yKiKlqxC32vgY8Efgo8DHgBVmdmVf11vKLjl9PONqKrhTJ6tFpISVFWAd3yD4DsROADMbC/wOWFKAdZekRDzGNXOm8S+Pvs5b9Qc4ceyIqEsSETlqhTgHEcuEQ2h3gdZb0j4xZyrl8Rh36YtzIlKiCrEhf9jMHjGzhWa2EPgP4DcFWG9JGzOigg+cNYElq7bQqEteRaQEFeIk9X8FFgFnho9F7v7Vvq53MFg4L8nB1hRLVumSVxEpPYU4B4G7PwA8UIh1DSZnTq7jnKl1LH52AwvOSxKLDamb3IpIiTvmPQgzazSzhhyPRjNrKGSRpWzBvCQbdjfx5Bv1UZciInJUjjkg3L3a3WtyPKrdvaaQRZay958+geOqK7jzmQ1RlyIiclQiudrIzC4xs7Vm9qaZfe0I/f7CzNzMZg1kfYVUXhbjk3Om8eTr9ayrPxB1OSIieRvwgDCzOPBD4P3ADOATZjYjR79q4CZgxcBWWHhXz5lKIm7ctUxfnBOR0hHFHsRs4E13X+furcC9BHeC7e7vgH8CmgeyuP4wtrqCD5w5UZe8ikhJiSIgJgGbs6a3hG0dzOwcYIq7/8dAFtafFs5LcqClnQd0yauIlIii+8ZzeLvw7xL8+FBvfW8ws5VmtrK+vrivEjprSh1nT6njrmUbdZdXESkJUQTEVmBK1vTksC2jGjgdeMLMNgBzgaW5TlS7+yJ3n+Xus8aOHduPJRfGdfOTrNt1kKd0yauIlIAoAuI54CQzO97MyoGrgKWZme6+393HuHvS3ZPAcuBD7r4ygloL6v2nT2BsdQWLdX8mESkBAx4Q7t4OfB54BFgD/NLdXzGzm83sQwNdz0AKLnmdyuNr61m/62DU5YiIHFEk5yDc/TfufrK7n+jufx+2fdPdl+bo+67BsPeQ0XnJ64aoSxEROaKiO0k92B1XXcllZ0zg/pVbONDSHnU5IiI9UkBEYEF4yeuDz+uSVxEpXgoIgPVPQ2rgvsA2c+pIzppSx53PbtAlryJStBQQe9bB4g/C986Ep/8FmvYMyNMunDeNdfUH+cObuwbk+UREjpYCoi4JV98HY0+Bx26G754KS78IO9f069NeesYExoyo4E5d8ioiRUoBEYvByX8Of/kr+KtlcObH4aX74P/OhbuugNcfgXS64E9bURbn6jlTeXztTjboklcRKUIKiGzjZsCH/g3+5lW46JtQ/xr84mPwg1nwxx9DS2Fv133NnKnETXd5FZHipIDIZfhoOP/L8KU/wV/8BKrq4Ddfge/OgEe+AXsLs0E/rqaSS8+YwP0rN3NQl7yKSJFRQBxJPAFnXAmf+T18+nfwjotg+a3wb2fDfdfCxmXgfbsKaeH8JI265FVEipACIl9TzoWP3gFfegnm3wTrn4I7LoFFF8Lqe6G95ZhWO3NKHWdOruXOZzfgfQwbEZFCUkAcrdrJ8N5vw395FT7wr9DWDA/dCN87A574JzhwdHdqNTMWzkvyli55FZEio4A4VuXDYdan4HMr4JoHYfyZ8MQ/wL+eBr/6HGz/U96ruuzMCYwZUa67vIpIUSmLuoCSZxacm3jHRVD/Oqz4Eay+B178OSTPh7l/BSdfArF4j6uoKItz9eyp/J/H3+Sv717F9PE1TB9fzakTaphUV0UsZgP4B4mIBGywHPeeNWuWr1xZJDd9PbQXnr8LViyChi0wMglzPgtnfxIqa3Iusq+plW8vfYUXN+9j456mjnPfIyrKOGV8NdPHVzN9Qg2njq/mlPHVVFcmBu7vEZFBy8xWufthP8gGCoj+lWqH1/4fLP8RbF4O5dUw8xqYcwOMOqHHxQ62tPP6jkZe297Ia9saWBMOG5o7L4WdPLKK6eNrOHVCdbDHMaGa5OjhxLW3ISJHQQFRDLauCoLilQchnYJTLoW5nw0OQ1nvG3V3Z9v+Zl7b3sCabZ3hsW7XQVLhDf8qymKdexthaJw6voaRw8v7+68TkRKlgCgmDdtg5U9g5U+haTeMOz04T3H6lZCoPOrVNbeleHPngY7AeG17I2u2NbD7YGtHn3E1FV0CY/qEak4YM4LyMl2jIDLUKSCKUdsh+NP9wRfvdr4Kw8YEh59GnQAV1VBREwwrw2FFNZSPOOLJ7mz1jS28tr2B17Y1sibc63hzZyNtqeDfOxE3Thw7glMn1HQ5vzG2ugLLY49GRAYHBUQxcw++dLf8Vnj9YaCXf4/yEZ2B0eVR023Yvb2atsQINjTGWLPHeXXHoY4A2d7Q3LH6UcPLecfYEdQNS1BblfUIp2uqurVXJUjEtSciUqoUEKWipRGa94fDhmDYkhlmP3K1Zc3rLWQAyqo69lDaEyM4SBX7UpXsaqtgV2sZze3QnHIOtUNLCtLESIUPx0h5MJ4mRlk8TiKRoLw8QUWijIpEgoryBBXl5VSVl1FZXk5lRYKqigTDKsqpqihnWGU5wyrKKYuXBXtFFgsesThYPGs8Fpyjsaw+hz2s2/K99LFYjvVZXueCeuUe3n7FwdNZ4/m0ZZZPd5ufaSOrnc5/50y/jvHu/chjmSMt321dEL5W1vm6ZsYPG9L5+vbU51iX93RwPs/TnY8u0+F4Op2jrdtyXdqOZrlcr0vHRA/tPfU/0jK9tFeNhOPPz/0cvThSQOh7EMUk86m/L9JpaDuYR6B0tpW1NFLb0khtSz3TvAE4CLEUWBpiKTzzBgnfHJYrgFLAofBRsrqHSBg4h23Qe9iI5xPMIv1h0iz4zGMFX60CYrCJxQoTNFkO++zjHoZF9qeqTIB4lzBpbWvjQHMLBw610NjcyoFDLRw81MqB5lYONrdysLmFpuZWmlraaGpppbm5labWNg61tNLe3o7h4X5KMAymPcd0mrh1nV8edyriRnkMEnEoj3U+EnEnEYNEOF0W65wui0HCnETMKbM0sVgMsxgxMyxmxGLxcDxGLGbBvLAtFjNisVjHw8yId0zHg/kWtOf+9BzL+uTcva1be8c4WevgCP26L3OMy+cKxu57QfkGal+W79hbzOxx5tg7zLVH2eNy2W3ZfayH5Y5wPrDLnkWuPblu7ceyTHZ7oqrnWvpAASFHzwziZeTz36ccGBU+jlY67bS0p2luS3UMm9tTtLR1b0vTkjXMXuZQW4q92X3b0rS0B8PmthStLYevKzUAvxMejxllmUc8Fg6NslgsHHYbj8eIm4FBzAgCqmNoxIJZHdOW1S/Wsdzh/WJhLvS4HJ3LxWIWLBsz4llt8TAY49nj4byYdWuPda47nj0/bM/0tRztXYcAQb0dsWZBfUGbHZaD2fPDpTuXD5fp6Evu/h1HxbKeE+i40WbHQbnuR/TCkY4M7OiXeznHuy2fe35mOhGPMTbH/7O+UkBI0YrFjKryOFXl+V25VSjtqTTNHYGSoj3ltKfTtKWcVNppS6XDYdDennbaU06qW5/2tIfzOvtn+nRdLuyfCvt3zOv6POk0pN1JO6TSadIeTAenPYJ2xzv6eWY67Id3Lp9zuexp73m5zhp0SK1YnD2ljl99bn7B16uAEOmmLB5jRDzGiAq9PXqTTjspD0IunRmGAZVy7zo/TdDWQ3tmHZl5Hf3TwbzMeNqD9uwLbDJhmLlGIPsTukNwVCqcH052XSZrRR3z3Lt8cnfI+ZzZex7ZMnsXnXsbdJu2nO30uFzu/mYwanhF7n+gPtI7QESOWSxmxDASA7uTJwNEF7CLiEhOCggREclp0HxRzszqgY1R19FHYwD9rFwnvR5d6fXopNeiq768HtPcPedFUIMmIAYDM1vZ0zcahyK9Hl3p9eik16Kr/no9dIhJRERyUkCIiEhOCojisijqAoqMXo+u9Hp00mvRVb+8HjoHISIiOWkPQkREclJAiIhITgqIImBmU8zscTN71cxeMbOboq4pamYWN7MXzOzfo64lamZWZ2ZLzOw1M1tjZudFXVOUzOxvwvfJy2Z2j5kd/Y+5lzAz+6mZ7TSzl7PaRpnZo2b2RjgcWYjnUkAUh3bgy+4+A5gLfM7MZkRcU9RuAtZEXUSR+D7wsLtPB85iCL8uZjYJ+CIwy91PB+LAVdFWNeDuBC7p1vY14DF3Pwl4LJzuMwVEEXD3be7+fDjeSLABmBRtVdExs8nAZcDtUdcSNTOrBS4AfgLg7q3uvi/aqiJXBlSZWRkwDHg74noGlLs/Bezp1nw5sDgcXwxcUYjnUkAUGTNLAjOBFdFWEqnvAX8LpHvrOAQcD9QDd4SH3G43s+FRFxUVd98KfAfYBGwD9rv7b6OtqiiMc/dt4fh2YFwhVqqAKCJmNgJ4APiSuzdEXU8UzOwDwE53XxV1LUWiDDgHuNXdZwIHKdDhg1IUHlu/nCA4JwLDzeyaaKsqLu6F+4F0BUSRMLMEQTjc7e4PRl1PhOYDHzKzDcC9wHvM7OfRlhSpLcAWd8/sUS4hCIyh6r3Aenevd/c24EFgXsQ1FYMdZjYBIBzuLMRKFRBFwIKflvoJsMbdvxt1PVFy96+7+2R3TxKcfPy9uw/ZT4juvh3YbGanhE0XAa9GWFLUNgFzzWxY+L65iCF80j7LUmBBOL4A+HUhVqqAKA7zgWsJPi2/GD4ujbooKRpfAO42s5eAs4F/iLieyIR7UkuA54E/EWzDhtRtN8zsHmAZcIqZbTGzTwO3AO8zszcI9rJuKchz6VYbIiKSi/YgREQkJwWEiIjkpIAQEZGcyqIuoFDGjBnjyWQy6jJERErKqlWrdvX0m9SDJiCSySQrV66MugwRkZJiZht7mqdDTCIiktOg2YMQkdLi7qQdUmkn7d4xTKchlT2dGQ/bgz6OA+7geDDMGqf7vPD5MsvQpb3rPCeYcdj6s9ZBph8QMyNmRjzWOYzHwMyIZ7XHYhA3IxbrbDcj6B+2xywzTudyHesI1jmQFBAiBZROOyl32lNOezpNKu20p71z2FN7Ok17ynO3Z6bD+ZmNZ+aR9s5l0t3ne/CcqXCj2p61MW7v6E84L00qTcf60ulu68l6vq5Dsjbi4XM5XWpJh/0ydaS8c0Mu+YtZGEix7PCBs6bU8bNPzyn48ykgZMhIpZ3G5jYam9vZfygYNoTTDVnTDd3mNbelum240902+J3t6SLZ6JnRsQHJfEKNxzs/qZZ1+bRrXT7FlsUyGyAoi8WIxSARixGPxYiHn3i7fDoO+2ZvuGLhhive7Xmy6+r8dEy3T9XW8TxdP1WHfcPlzMDIDIO/mS7T1tGe6UuueVnLcNg6j7x+oGOPJhOk6TBkU1ntXUI0O1izQtQzIZzV3rluwlDNfh7vCPRJdVX98v9IASElwd1paU+HG/DcG/bGI85r50BLe6/PM7w8TnVlgpqqMqorE4weUc6w8jjxWIyycCPXdRijLN5De2Y6bpR1m87Zr8v8ru2xmJEIN9ZdNvqxHBv8cGMq0lcKCImUu1N/oIWtew+xdd+hLsP6Ay1dNvStqSP/PEQ8ZtRUlnVu4CsSJMcMC6azNvrZfWqy5o2oKKMsrus2RDIUENKv2lNptjc0Hx4AWeMt7V03/NWVZUyqq+K4mkqmjR5OdWWwIa+uLKOmKtjAd93gB/OGlccH/CSeyGCmgJA+aW5LHfbJP3u4vaGZVLcD82NGlDOprorpE6q56NTjmFRXxaSRw8JhFbVViYj+GhHJpoAoBu0tsH8LHNgJ8XJIVEH5MEgMD4ZlVRCL5tBHQ3NbsLHv9sl/SzjcdaClS/+YwYTaKibVVTH7+FEdG/3sYWUiHsnfIiUonYK2JmhtgraD0HYoGE+1BO+L8mGQCB8Rv1cGIwVEf3OH5n2wb3MQAvs3w75NneP7t8CBHb2vpywrNLoHSKIqazzrzdJlPFffcF2JKlpSaVZv3s/ydbt5act+tuxtYuu+QzQ2dz2xW14WCzb2dVVcNP04Jo2sYnJWAIyvqdRx/KEknYb2Q4dvwNsOhsOmHBv4g1ltueZnLZ9q6b2G7hK9vCfKM//vM+Nh/8x4rvmZtrLKzOVMQ4ICoq/SKWjcFmzo922G/ZuyxsMQaD3QdZmySqidDLVT4KSLoW5qMD1iXPiJqYc3V2tT8AbKnn+wvnO8NXyDtR86uj8Bo93LOZ4KJlDB5fFgI5+oNMqGxUjELZiOx4LLEAFagB3hQ4YOT0Nbc+fGvK3p6NfR0wZ8xLj8N+DxRFjHUQTQgR19DyCLddZbTIEx7jS48qcFX60CojetB7M++W/u+sl/32Zo2Aqe6rpM1SiomwKjT4QT3hVs/OumhKEwFYaP6d//VOl05xsjfHO0HGpk3dadvLG1nvXbdrJj1x4S6WaqrIWpI+D4GmPyiDTjqpxyP4ZPbTJEWOfGu7dP3bn2ZIvtEFBPh7Dy3dtpb476LwjUTeuX1Sog2lthx8tdN/r7N3dON+3u2t/iUDMp2NhPO69zT6B2SmcIlA+P5m/JiMVojlXx4s4Wlq9rZfm6/Ty/aR+t7THMxjFjwknMmT2auSeMYvbxo6gbVh5tvSJRicWhojp4yGEUEIf2wo/f3TmdGB5u6KfAxHPC8amdewEjxkO8+F625rYUL2zax/J1u1mxfncYCGnMYMaEGq6dO425J4xmdnIUtcN0lZCI9K74tnQDbfhYuOoX4V7AZKgaWRzHFHuRHQjL1+3mhc2dgXDaxBr+cu405igQRKQPFBCxGEy/LOoqetXcluL5TXtZvm4Py9ft5sVN+2hNdQ2EuSeM5tzjR+l7BCJSEAqIItXcluL5jXtZvr5rIMQMTptYy4J5QSDMSioQRKR/KCCKyHMb9vD06/UsX7eHFzd3BsLpkxQIIjLwFBBF4uGXt/PZn6/qCISF85PMPWEUs5KjqKlUIIjIwFNAFAF359Yn3iQ5ehhLv/BnCgQRKQpF9I2VoWvF+j2s3rKf688/QeEgIkVDAVEEbnvyLUYPL+fKd06OuhQRkQ4KiIi9vqORx9fWs2BeUnc5FZGiooCI2KKn1lGViHPt3P65l4qIyLFSQERo+/5mfv3iVj42azIjh+t+SCJSXBQQEbrjmfWk0s71558QdSkiIodRQESkobmNX6zYxKVnTGDKqGFRlyMichgFRETuWbGJxpZ2brzgxKhLERHJKa+AMLMHzewyM1OgFEBre5o7ntnAvBNHc8bk2qjLERHJKd8N/v8FrgbeMLNbzOyUfqxp0Fu6+m22NzRzwwU69yAixSuvgHD337n7J4FzgA3A78zsWTO7zsz01d+j4O4seuotpo+v5sKTx0ZdjohIj/I+ZGRmo4GFwPXAC8D3CQLj0X6pbJB6Ym09r+84wA0XnICVwA8TicjQldfN+szsIeAU4GfAB919WzjrPjNb2V/FDUa3PfUWE2or+eBZE6MuRUTkiJmU3D4AAA4uSURBVPK9m+u/ufvjuWa4+6wC1jOord68j+Xr9vCNS08lEdf5fhEpbvlupWaYWV1mwsxGmtlf91NNg9aip9ZRXVnGVbOnRF2KiEiv8g2Iz7j7vsyEu+8FPtM/JQ1OG3cf5D9f3sYn50yjWrf0FpESkG9AxC3rjKqZxYFebx5kZpeY2Voze9PMvpZj/jQze8zMXjKzJ8xscta8lJm9GD6W5lln0br96fXEY8Z185NRlyIikpd8z0E8THBC+rZw+sawrUdhiPwQeB+wBXjOzJa6+6tZ3b4D3OXui83sPcA/AteG8w65+9l51lfU9hxs5f5Vm/nwzEmMq6mMuhwRkbzkuwfxVeBx4K/Cx2PA3/ayzGzgTXdf5+6twL3A5d36zAB+H44/nmP+oHDXsg00t6X1xTgRKSn5flEu7e63uvuV4eM2d0/1stgkYHPW9JawLdtq4CPh+IeB6vD7FgCVZrbSzJab2RW5nsDMbgj7rKyvr8/nTxlwh1pTLH52AxdNP453HFcddTkiInnL915MJ5nZEjN71czWZR4FeP6vABea2QvAhcBWIBM808JLaK8Gvmdmh93Vzt0Xufssd581dmxxfit5yarN7G1q48YLdVM+ESkt+R5iugO4FWgH3g3cBfy8l2W2AtnXc04O2zq4+9vu/hF3nwl8I2zbFw63hsN1wBPAzDxrLRqptPPjp9dz9pQ6zk2OjLocEZGjkm9AVLn7Y4C5+0Z3/zZwWS/LPAecZGbHm1k5cBXQ5WokMxuTdYfYrwM/DdtHmllFpg8wH8g+uV0SHn55O5v2NHGjbqshIiUo36uYWsIN+Rtm9nmCPYERR1rA3dvDvo8AceCn7v6Kmd0MrHT3pcC7gH80MweeAj4XLn4qcJuZpQlC7JZuVz8VvcxN+ZKjh3HxaeOjLkdE5KjlGxA3AcOALwJ/R3CYaUFvC7n7b4DfdGv7Ztb4EmBJjuWeBc7Is7aitGL9HlZv2c//uuJ04jHtPYhI6ek1IMLvM3zc3b8CHACu6/eqBoHbnnyL0cPLufKdk3vvLCJShHo9BxFezvpnA1DLoPH6jkYeX1vPgnlJKhPxqMsRETkm+R5ieiG83cX9wMFMo7s/2C9VlbhFT62jKhHn2rnToi5FROSY5RsQlcBu4D1ZbQ4oILrZvr+ZX7+4latnT2Xk8F5vVyUiUrTyCgh313mHPN3xzHpSaef683VbDREpbfn+otwdBHsMXbj7pwpeUQlraG7jFys2cekZE5gyaljU5YiI9Em+h5j+PWu8kuC+SW8XvpzSds+KTTS2tHPjBbqthoiUvnwPMT2QPW1m9wB/6JeKSlRre5o7ntnAvBNHc8bk2qjLERHps2P9YeSTgOMKWUipW7r6bbY3NOuW3iIyaOR7DqKRrucgthP8RoTQeVuN6eOrufDk4ryrrIjI0cr3EJN+yOAInlhbz+s7DvDdj52lm/KJyKCR7+9BfNjMarOm63r6EZ+h6Lan3mJCbSUfPGti1KWIiBRMvucgvuXu+zMT4W82fKt/SiotqzfvY/m6PXxq/vEk4sd6SkdEpPjku0XL1S/fS2QHtUVPraO6soyrZk/pvbOISAnJNyBWmtl3zezE8PFdYFV/FlYKNu4+yH++vI1PzplGdWUi6nJERAoq34D4AtAK3AfcCzTT+eM+Q9btT68nHjOum5+MuhQRkYLL9yqmg8DX+rmWkrLnYCv3r9rMh2dOYlxNZdTliIgUXL5XMT1qZnVZ0yPN7JH+K6v43bVsA81taX0xTkQGrXwPMY0Jr1wCwN33MoS/SX2oNcXiZzdw0fTjeMdx+oqIiAxO+QZE2symZibMLEmOu7sOFUtWbWZvUxs3Xqib8onI4JXvparfAP5gZk8CBpwP3NBvVRWxVNr58dPrOXtKHecmR0ZdjohIv8lrD8LdHwZmAWuBe4AvA4f6sa6i9fDL29m0p4kbLzhBt9UQkUEt35v1XQ/cBEwGXgTmAsvo+hOkg17mpnzJ0cO4+LTxUZcjItKv8j0HcRNwLrDR3d8NzAT2HXmRwWfF+j2s3rKf688/gXhMew8iMrjlGxDN7t4MYGYV7v4acEr/lVWcbnvyLUYPL+fKd06OuhQRkX6X70nqLeH3IH4FPGpme4GN/VdW8Xl9RyOPr63nv7zvZCoT8ajLERHpd/l+k/rD4ei3zexxoBZ4uN+qKkKLnlpHVSLOtXOnRV2KiMiAOOo7srr7k/1RSDHbvr+ZX7+4latnT2Xk8PKoyxERGRD6AYM83PHMelJp5/rzdVsNERk6FBC9aGhu4xcrNnHpGROYMmpY1OWIiAwYBUQv7lmxicaWdm68QLfVEJGhRQFxBK3tae54ZgPzThzNGZNre19ARGQQUUAcwdLVb7O9oVm39BaRIUkB0YPMbTWmj6/mwpPHRl2OiMiAU0D04Im19by+4wA36KZ8IjJEKSB6cNtTbzGhtpIPnjUx6lJERCLRrwFhZpeY2Voze9PMDvtNazObZmaPmdlLZvaEmU3OmrfAzN4IHwv6s87uVm/ex/J1e/jU/ONJxJWhIjI09dvWz8ziwA+B9wMzgE+Y2Yxu3b4D3OXuZwI3A/8YLjsK+BYwB5gNfMvMBuzXeRY9tY7qyjKumj1loJ5SRKTo9OfH49nAm+6+zt1bgXuBy7v1mQH8Phx/PGv+nwOPuvue8PevHwUu6cdaO2zcfZD/fHkbn5wzjerKxEA8pYhIUerPgJgEbM6a3hK2ZVsNfCQc/zBQbWaj81y2X9z+9HrKYjGum58ciKcTESlaUR9g/wpwoZm9AFwIbAVS+S5sZjeY2UozW1lfX9/nYvYcbOX+VZu5YuZExtVU9nl9IiKlrD8DYiuQfRB/ctjWwd3fdvePuPtM4Bth2758lg37LnL3We4+a+zYvn9X4a5lG2huS+uLcSIi9G9APAecZGbHm1k5cBWwNLuDmY0xs0wNXwd+Go4/AlxsZiPDk9MXh2395lBrisXPbuC9px7HO46r7s+nEhEpCf0WEO7eDnyeYMO+Bvilu79iZjeb2YfCbu8C1prZ68A44O/DZfcAf0cQMs8BN4dt/WbJqs3sbWrjBt2UT0QEAHP3qGsoiFmzZvnKlSuPadlU2nn3d55g1PByHvrrefrmtIgMGWa2yt1n5ZoX9UnqovDwy9vZtKeJz16o22qIiGQM+YDI3JQvOXoY75sxPupyRESKxpAPiI27m1izvZHrzz+BeEx7DyIiGWVRFxC15JjhPPPV91BdOeRfChGRLrRVBMZWV0RdgohI0Rnyh5hERCQ3BYSIiOQ0aL4HYWb1wMao6+ijMcCuqIsoIno9utLr0UmvRVd9eT2muXvOexUNmoAYDMxsZU9fWBmK9Hp0pdejk16Lrvrr9dAhJhERyUkBISIiOSkgisuiqAsoMno9utLr0UmvRVf98nroHISIiOSkPQgREclJAVEEzGyKmT1uZq+a2StmdlPUNUXNzOJm9oKZ/XvUtUTNzOrMbImZvWZma8zsvKhripKZ/U34PnnZzO4xsyH1+8Bm9lMz22lmL2e1jTKzR83sjXA4shDPpYAoDu3Al919BjAX+JyZzYi4pqjdRPBDUwLfBx529+nAWQzh18XMJgFfBGa5++lAnODXKoeSO4FLurV9DXjM3U8CHgun+0wBUQTcfZu7Px+ONxJsACZFW1V0zGwycBlwe9S1RM3MaoELgJ8AuHtr+LvtQ1kZUGVmZcAw4O2I6xlQ7v4U0P0XNi8HFofji4ErCvFcCogiY2ZJYCawItpKIvU94G+BdNSFFIHjgXrgjvCQ2+1mNjzqoqLi7luB7wCbgG3Afnf/bbRVFYVx7r4tHN9O8BPOfaaAKCJmNgJ4APiSuzdEXU8UzOwDwE53XxV1LUWiDDgHuNXdZwIHKdDhg1IUHlu/nCA4JwLDzeyaaKsqLh5cmlqQy1MVEEXCzBIE4XC3uz8YdT0Rmg98yMw2APcC7zGzn0dbUqS2AFvcPbNHuYQgMIaq9wLr3b3e3duAB4F5EddUDHaY2QSAcLizECtVQBQBC34I+yfAGnf/btT1RMndv+7uk909SXDy8ffuPmQ/Ibr7dmCzmZ0SNl0EvBphSVHbBMw1s2Hh++YihvBJ+yxLgQXh+ALg14VYqQKiOMwHriX4tPxi+Lg06qKkaHwBuNvMXgLOBv4h4noiE+5JLQGeB/5EsA0bUt+qNrN7gGXAKWa2xcw+DdwCvM/M3iDYy7qlIM+lb1KLiEgu2oMQEZGcFBAiIpKTAkJERHJSQIiISE4KCBERyUkBIVIEzOxdunOtFBsFhIiI5KSAEDkKZnaNmf0x/DLjbeHvVhwws38Nf6PgMTMbG/Y928yWm9lLZvZQ5h79ZvYOM/udma02s+fN7MRw9SOyfvfh7vCbwiKRUUCI5MnMTgU+Dsx397OBFPBJYDiw0t1PA54EvhUuchfwVXc/k+Bbv5n2u4EfuvtZBPcRytyFcybwJWAGcALBN+xFIlMWdQEiJeQi4J3Ac+GH+yqCm6KlgfvCPj8HHgx/x6HO3Z8M2xcD95tZNTDJ3R8CcPdmgHB9f3T3LeH0i0AS+EP//1kiuSkgRPJnwGJ3/3qXRrP/0a3fsd6/piVrPIXenxIxHWISyd9jwJVmdhx0/A7wNIL30ZVhn6uBP7j7fmCvmZ0ftl8LPBn+YuAWM7siXEeFmQ0b0L9CJE/6hCKSJ3d/1cz+O/BbM4sBbcDnCH7EZ3Y4byfBeQoIbrv8ozAA1gHXhe3XAreZ2c3hOj46gH+GSN50N1eRPjKzA+4+Iuo6RApNh5hERCQn7UGIiEhO2oMQEZGcFBAiIpKTAkJERHJSQIiISE4KCBERyUkBISIiOf1/TUUq/Zs57MAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Jj5FBbClgM1",
        "outputId": "7397024b-dc8f-4236-808c-8ad568e71ef7"
      },
      "source": [
        "predictions, labels = predict(test_dl, clf)\n",
        "\n",
        "accuracy = accuracy_score(predictions, labels)\n",
        "ACCURACY_STATS.append((4, \"over_sampling\", accuracy))\n",
        "\n",
        "print(\"accuracy: {:.3f}\".format(accuracy))\n",
        "print(\"confusion matrix\")\n",
        "print(confusion_matrix(predictions, labels))"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy: 0.978\n",
            "confusion matrix\n",
            "[[481   5]\n",
            " [  7  64]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "II5RFfa6lwXF"
      },
      "source": [
        "## SMOTE Oversampling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gih6TY-esZgB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "809ce30a-07ba-4f2f-8cf9-fa204dce4d78"
      },
      "source": [
        "def create_xy_from_dataframe(input_df):\n",
        "  emb_vecs = []\n",
        "  for emb in input_df[\"embedding\"].values:\n",
        "    vec = np.array([float(x) for x in str(emb).split(\",\")])\n",
        "    emb_vecs.append(vec)\n",
        "  labels = [0 if label == \"ham\" else 1 for label in input_df[\"label\"].values]\n",
        "  return np.array(emb_vecs), np.array(labels)\n",
        "\n",
        "\n",
        "def create_dataset_from_xy(xs, ys):\n",
        "  return TensorDataset(\n",
        "      torch.tensor(xs, dtype=torch.float32),\n",
        "      torch.tensor(ys, dtype=torch.long)\n",
        "  )\n",
        "\n",
        "\n",
        "train_x, train_y = create_xy_from_dataframe(spam_train_df)\n",
        "oversampler = SMOTE()\n",
        "train_x, train_y = oversampler.fit_resample(train_x, train_y)\n",
        "uniq, counts = np.unique(train_y, return_counts=True)\n",
        "counts"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([3355, 3355])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Db0VO9GluBN7"
      },
      "source": [
        "train_ds = create_dataset_from_xy(train_x, train_y)\n",
        "\n",
        "train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\n",
        "val_dl = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False)\n",
        "test_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False)"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S8ivghknu2Gx"
      },
      "source": [
        "clf = SpamClassifier(CLF_LAYER_DIMS)\n",
        "clf = clf.to(dev)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.AdamW(clf.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-0hQsr_wu6M_",
        "outputId": "ce0b67ed-7a00-4398-9375-d01463c001d4"
      },
      "source": [
        "train_losses, train_accs, val_losses, val_accs = [], [], [], []\n",
        "for epoch in range(1, NUM_EPOCHS + 1):\n",
        "  train_loss, train_acc = train(train_dl, clf, loss_fn, optimizer)\n",
        "  val_loss, val_acc = validate(val_dl, clf, loss_fn) \n",
        "  print(\"EPOCH {:2d} | train_loss: {:.3f}, acc: {:.3f} | val_loss: {:.3f}, acc: {:.3f}\"\n",
        "    .format(epoch, train_loss, train_acc, val_loss, val_acc))\n",
        "  train_losses.append(train_loss)\n",
        "  train_accs.append(train_acc)\n",
        "  val_losses.append(val_loss)\n",
        "  val_accs.append(val_acc)"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH  1 | train_loss: 0.546, acc: 0.864 | val_loss: 0.368, acc: 0.977\n",
            "EPOCH  2 | train_loss: 0.343, acc: 0.979 | val_loss: 0.337, acc: 0.981\n",
            "EPOCH  3 | train_loss: 0.331, acc: 0.986 | val_loss: 0.333, acc: 0.981\n",
            "EPOCH  4 | train_loss: 0.327, acc: 0.989 | val_loss: 0.332, acc: 0.983\n",
            "EPOCH  5 | train_loss: 0.324, acc: 0.992 | val_loss: 0.332, acc: 0.983\n",
            "EPOCH  6 | train_loss: 0.322, acc: 0.993 | val_loss: 0.332, acc: 0.983\n",
            "EPOCH  7 | train_loss: 0.320, acc: 0.994 | val_loss: 0.334, acc: 0.981\n",
            "EPOCH  8 | train_loss: 0.319, acc: 0.995 | val_loss: 0.334, acc: 0.981\n",
            "EPOCH  9 | train_loss: 0.319, acc: 0.996 | val_loss: 0.333, acc: 0.983\n",
            "EPOCH 10 | train_loss: 0.318, acc: 0.996 | val_loss: 0.332, acc: 0.981\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        },
        "id": "g64560awu9qG",
        "outputId": "569456ed-e3d8-49ca-cd38-7a5fd356c2f2"
      },
      "source": [
        "training_lossplots(\"SMOTE\", train_losses, train_accs, val_losses, val_accs)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEjCAYAAAAomJYLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhc9X3v8fd3Fq2WVwnHq+SwGwIGBNgmCwGykAUSSFgSwKYL93lKEtKbtElul7T0tknbNG3S5qalJNgOBEiAJG4gEEKANtgsMvsSMMGSd1u28aZ9Zr73j3PGGomRNbZndEbS5/U85znn/M7vnPOdo9HvO2c3d0dERGSwWNQBiIhIeVKCEBGRvJQgREQkLyUIERHJSwlCRETyUoIQEZG8lCBERCQvJQgZ98zsnWa2ysz2mNkuM3vMzM40s6Vm5mb2z4PqXxyWL8spqzSzr5nZejPrMrO1ZvYnZmbh9JfMbH/Ypc2sO2f8/4TrSueUZbuZI7w5RA5QgpBxzcwmAj8H/hWYCswC/hroCav8DrjMzBI5sy0BXhu0qB8D5wMfAuqAq4HrgG8BuPtJ7j7B3ScA/wN8Jjvu7n8XLmN1Tlm221zszyxSKCUIGe+OA3D329097e5d7v5Ld38+nL4VeAH4AICZTQUWAyuzCzCz84H3A5e6+4vunnL3x4GrgOvN7JgR/DwiRaMEIePda0DazJab2YVmNiVPnRXANeHwFcDP6N/DAHgf8IS7b8idyd2fADYS7FmIjDpKEDKuufte4J2AA/8JtJvZSjObnlPtJ8C5ZjaJIFGsGLSYemDLEKvYEk4vxEIz253T/a7gDyJSAkoQMu65+yvuvtTdZwMnAzOBf8mZ3gXcC/w5MM3dHxu0iB3AjCEWPyOcXojH3X1yTnf0IX0QkSJTghDJ4e6/BZYRJIpcK4AvALfmme1XwNlmNie30MzOBuYAvy5+pCKlpwQh45qZnWBmXzCz2eH4HOBK4PFBVR8lONfwr4OX4e6/Ah4C7jazk8wsbmYLCZLJd919bUk/hEiJKEHIeLcPOBt4wsw6CBLDiwR7Cwd44CF33zXEci4FHgbuB/YTJIfvAZ89hFgW5bkP4sxD/DwiRWN6YZCIiOSjPQgREclLCUJERPJSghARkbyUIEREJC8lCBERyUsJQkRE8lKCEBGRvJQgREQkLyUIERHJSwlCRETyUoIQEZG8lCBERCQvJQgREclLCUJERPJSghARkbyUIEREJC8lCBERySsRdQDFUl9f701NTVGHISIyqqxZs2aHuzfkmzZmEkRTUxMtLS1RhyEiMqqYWdtQ03SISURE8lKCAFLpDOmMRx2GiEhZGfcJYv3OTt79Dw9z/4tbow5FRKSsjPsEMWtKNfG4sXx1a9ShiIiUlXGfIOIx4+qFjTy5bhevbNkbdTgiImVj3CcIgMua51CZiLFi9ZAn80VExh0lCGByTQUfWzCLnz6ziT2dfVGHIyJSFpQgQtcsbqSrL82P12yIOhQRkbKgBBE6aeYkzmyaworVbWR0yauIiBJErmsWNbF+VyePvtYedSgiIpFTgsjxgZPexlF1lbrkVUQEJYgBKhIxPnX2XB55tZ3WHR1RhyMiEikliEE+dfZcknHTJa8iMu4pQQxyVF0VF548gx+v2UBHTyrqcEREIqMEkceSxY3s607x02c3RR2KiEhklCDyOH3uFE6aOZEVq9pw1yWvIjI+KUHkYWYsWdTEq9v28cS6XVGHIyISCSWIIVy0YCaTa5IsX9UadSgiIpGIJEGY2QfN7FUze93Mvpxn+lIzazezZ8PuD0Y6xqpknMvPnMMvX97G5t1dI716EZHIjXiCMLM48B3gQmA+cKWZzc9T9U53XxB2N49okKGrzm4k484Pn1gfxepFRCIVxR7EWcDr7v6Gu/cCdwAXRxDHsOZMreH8E6Zz+5Pr6Umlow5HRGRERZEgZgG5j0zdGJYNdqmZPW9md5nZnHwLMrPrzKzFzFra20vz/KQlixvZ2dHLfS9sKcnyRUTKVbmepP4voMndTwEeBJbnq+TuN7l7s7s3NzQ0lCSQdx5Tz9sbalm2SndWi8j4EkWC2ATk7hHMDssOcPed7t4Tjt4MnDFCsb1F9pLX5zbs5tkNu6MKQ0RkxEWRIJ4CjjWzeWZWAVwBrMytYGYzckYvAl4Zwfje4pLTZ1FbEWfF6tYowxARGVEjniDcPQV8BniAoOH/kbu/ZGY3mtlFYbXPmdlLZvYc8Dlg6UjHmauuKsmlZ8zm589tYef+nuFnEBEZA2ysPEqiubnZW1paSrb817fv54JvPsqffOB4rn/vMSVbj4jISDKzNe7enG9auZ6kLjvHHDWBdx5Tz62Pt5FKZ6IOR0Sk5JQgDsE1ixrZsqebX72yLepQRERKTgniEJx/4nRmTa5muS55FZFxQAniEMRjxlULG1n9xk5e27Yv6nBEREpKCeIQXXHmHCoTMT3lVUTGPCWIQzSltoKLTp3JPU9vYk9XX9ThiIiUjBLEYViyuImuvjR3r9kYdSgiIiWjBHEYTp41idPnTuYHj7eRyYyN+0hERAZTgjhMSxY3sW5HB//z+o6oQxERKQkliMN04ckzqJ9QqZPVIjJmKUEcpopEjE+dPZeHX91O286OqMMRESk6JYgj8Omz5xI349bHdeOciIw9ShBHYPrEKj5w8tu486kNdPXqlaQiMrYoQRyhpYub2Nud4mfPbhq+sojIKHLECcLMbjCziRb4npk9bWbvL0Zwo0Fz4xROnDGRZataGSuPThcRgeLsQfyeu+8F3g9MAa4Gvl6E5Y4KwStJG/nt1n081fpm1OGIiBRNMRKEhf0PAT9w95dyysaFixfMYmJVguWrW6MORUSkaIqRINaY2S8JEsQDZlYHjKs36lRXxLn8zDk88OJWtu7pjjocEZGiKEaC+H3gy8CZ7t4JJIFri7DcUeXqhU2k3fnhk+ujDkVEpCiKkSAWAa+6+24zuwr4c2BPEZY7qsydVsN5xx/FD59YT29qXO1AicgYVYwE8V2g08xOBb4A/A5YUYTljjrXLG5ix/4efvHilqhDERE5YsVIECkPru+8GPg3d/8OUFeE5Y467zqmnnn1tXo+k4iMCcVIEPvM7CsEl7fea2YxgvMQ404sZly9sJGn1+/mhY3j7iibiIwxxUgQlwM9BPdDbAVmA/9YhOWOSp9onk1NRZwVq1ujDkVE5IgccYIIk8JtwCQz+wjQ7e7j8hwEwMSqJJecPoufPbeZXR29UYcjInLYivGojcuAJ4FPApcBT5jZJ450uaPZNYua6E1luPOpDVGHIiJy2IpxiOnPCO6BWOLu1wBnAX9RhOWOWsdNr2PR26dx6+NtpPVKUhEZpYqRIGLuvj1nfGeRljuqLVncyKbdXTz0yraoQxEROSzFaMjvN7MHzGypmS0F7gXuK8JyR7ULTpzOzElVrFitlwmJyOhUjJPUfwLcBJwSdje5+5eOdLmjXSIe49MLG/nN6zt4ffu+qMMRETlkRTkU5O53u/v/DrufDFffzD5oZq+a2etm9uWD1LvUzNzMmosR50i74sw5VMRj2osQkVHpsBOEme0zs715un1mtvcg88WB7wAXAvOBK81sfp56dcANwBOHG2PUpk2o5COnzuDuNRvZ190XdTgiIofksBOEu9e5+8Q8XZ27TzzIrGcBr7v7G+7eC9xB8JiOwf4G+HtgVD8/e+niJjp609zztF5JKiKjSxRXG80Ccm8Q2BiWHWBmpwNz3P3egy3IzK4zsxYza2lvby9+pEVwyuzJLJgzmeWr9UpSERldyu5y1PBZTt8keDLsQbn7Te7e7O7NDQ0NpQ/uMC1Z3Mgb7R385vUdUYciIlKwKBLEJmBOzvjssCyrDjgZeMTMWoGFwMrReqIa4EPvmMG02gqWr9LJahEZPaJIEE8Bx5rZPDOrAK4AVmYnuvsed6939yZ3bwIeBy5y95YIYi2KykScK8+ay0O/3caGXZ1RhyMiUpARTxDungI+AzwAvAL8yN1fMrMbzeyikY5npHx64VxiZtz6hPYiRGR0SESxUne/j0F3W7v7Xw5R99yRiKnUZkyq5gMnTefOpzbwxxccR1UyHnVIIiIHVXYnqceyaxY1sbuzj5XPbo46FBGRYSlBjKCz503l+Ol1LFulS15FpPwpQYwgM+OaxY28vGUvT69/M+pwREQOSglihH38tFnUVSV0yauIlD0liBFWU5HgsuY53PfCFrbvHdVPERGRMU4JIgJXL2wklXF++OT6qEMRERmSEkQ6BT+8Ap66GTp2jsgqm+prOff4Bm57Yj29qcyIrFNE5FApQezbDG+2wr1fgH86Dn54ObxwF/SW9o7nJYuaaN/XwwMvbS3pekREDlckN8qVlclz4Y9Ww7YX4fk74YW74bX7oWICnPhROOUymPceiBX3xrb3HNdA47QaVqxu5aOnzizqskVEikEJAsAM3vaOoLvgr6HtsSBZvLwSnrsdJkyHky8NksWMBUH9IxSLGVcvbOT/3vsKL23ew0kzJxXhg4iIFI8OMQ0Wi8O8d8PF34EvroXLVsDsM4NzFDedC/92Jjz6D7Br3RGv6pNnzKE6GWeFLnkVkTKkBHEwySqYfzFccRt88TX46LeCvYmH/xa+vQBufh88+Z/QcXjveZhUk+Rjp83ip89uYndnb5GDFxE5MkoQhaqeAmcshWvvhc+/CBf8FfR2wH1fhH86Hm677LBObi9Z3EhPKsOPWjYMX1lEZATZWHkmUHNzs7e0RPDKiK0vwgs/CpLD3k2QrA1Pbn8S5p0L8eFP81z+H6vZvKeLR774XuKxIz+/ISJSKDNb4+55X8imPYgj9baT4X03BnsVS++Fd1wKr/4Cbr0Uvnki/OLLsGkNHCQRL1ncxIZdXTzy6vYRDFxE5OC0B1EKfd2w9pfBnsVrD0C6F6YdA++4LNizmPr2gdXTGd719w9z7PQJ/OD3z44oaBEZjw62B6EEUWpdu+GVlfD8j6D1N4DDrGY45XI4+RKorQfgXx9ayz89+Bpnz5vKvPpamupraZpWS1N9DU3TavWCIREpCSWIcrFnE7x4Fzz/Y9j2Algcjj4PTrmcfU3v4x8f3sjLm/fSurODHfsHXtU0Y1JVmDBqmRcmjab6WuZOrVHyEJHDpgRRjra93H9ye8+G4OT2ce+HibOgopbuWBU7e5Js64mzpTPOhv0x2vbDG3tgc1ecTq+igyp6rIKZk2oO7GnMq6+lcVqQROZMraEyoeQhIkNTgihnmQysXx0ki7UPQvce6N1f8OyO0ROropMq9mcq2ZeppIMqOr2KTirxigkkq+uoqqmjZsJEJk6czOQpU5g6eQqJqjqoqA27Cf3DyRqIJSCmaxjGDXfIpAZ16fzj7oD3zzdgmPzjB53G0NOGXAdgseDG1ty+xcPhePD9fUtZtl5sUFl2OePvO3+wBKFHbUQtFoOmc4IuK5OBVFdwn0Xv/uDeigPDHQOGrbeDqt4Oqvo6mNrbQV/XPro79tLXtY9MTzuxvjYSHZ1U7u+mcnvfIQZnYaLIdvHDGD+cecJhi/U3DgX1OcT6+foMHC+aIly+7OnhG+/cLl1AY5/tPH3k8Y0Vb0ka8YEJZcCwhY/eOZw+hzlfbp+g33ACfOgfi74plCDKUSzW/2ueow5p1mTYDebu7NrXyfqt7WzevoNtO3bSvmsXb+7exd49e4j1dVBtPdTSTTU9VMSc6rhTZU5lLNtlqIxlqIg5FbEMFRZ0STIkPUMikyGZyZCwNHEyxOkhTpq4Z4iTJuZBZ15oo5V9FHqU/4SH+LfLpyh5xoOGKZ4cOrnGk5CsHjT9IPXfMh6WHWwd2V/fkPNMMhs4PHjagfHDmUb/+IBpHnw/MpkguXkm+P54Ouxn8pSlw/qZQWXp4AfBW8ryLSOTs5zs+Ej+iMmZP5PpL0/1FPxNOhRKEOOEmTF1Yi1TJ9ay4LimAdPcnR37e2nb2cG6HR1s3t1NZ2+Knb1pOnvTdPWl6MwO96bp7E3R1ZWmsy8oO9R3WiTjRnUyTk1FgpqKONWVcWorElRXxIPxbD8ZpyrsKhOxA8NVyRhViZzhsF85qCwZH3+HC0SKSQlCMDMa6ippqKukuWnqIc+fSmfo6ssmj6GSSphYeoPEkk00udN3d/WxZU/XgLLuVPpg9xgeVCJmg5JH/iRTmS1PDEw42bKKRIxE3EjEYlQkgn4ibiTjMZLxGIlYMJyIGxXx/rrJuJGIB/1kLEZMd8nLKKMEIUcsEY9RF49RV5Xv4NaRcXd60xm6+zL09KXp7svQnUrTnR3uC4dTwXBPbnkqt04wX+70PV19A6b3hPP0pUtz4UbMgm01OIlkk0tyQOIZnIjCZBPrTzq50xOxfNOHrpvMk8hyy7OxZIeTif7pMQt+VMjYpwQhZc3MqEzEg8t1q4ufgPJJZ3xA4ulNZUilM/Slnb50hlQmGE6F40FZdtiDuhmnL5WnbiZDKqzTG/b75w2m5c7b1efBMlI+YN6+TDhvOqc8M3JXJMYM4jEjZkEiisWMeMyIWzhs4XjYDagfH1gv20/Eg+nZevEY4fwx4saA5R6YN0xW2XXEwuVlh83668VihhnheDgcrmvwcg7MF3vrcswGfh7Lrjcczh3P1u8vI+88/csdWMeGmoewLIwvu7dcbEoQIoPEY0ZtZYLaytH17+EeJIkBSSMnmfSlg2STOpDo/EBSyk1k2eFgWv982elpdzKZ/n4q46QzTsYH9lOZbD2C/qB5svWCuhl6Uv110hmGXUfGOTDueYazdcaDBXMm89Przxm+4iEaXf8BIjIkMwsPWUE1ukESgqTZnzAGJo4gmeSUZ/Inndx5gnr95emM4+F6MmEidILpg9fLoHEfMJx/nv743zpPxvs/X0NdZUm2nxKEiIxZBw7vFOV65fFH1wGKiEheShAiIpLXmHkWk5m1A21Rx3GE6oHDe8H12KTtMZC2Rz9ti4GOZHs0untDvgljJkGMBWbWMtRDs8YjbY+BtD36aVsMVKrtoUNMIiKSlxKEiIjkpQRRXm6KOoAyo+0xkLZHP22LgUqyPXQOQkRE8tIehIiI5KUEISIieSlBlAEzm2NmD5vZy2b2kpndEHVMUTOzuJk9Y2Y/jzqWqJnZZDO7y8x+a2avmNmiqGOKkpn9cfh/8qKZ3W5mVVHHNJLM7Ptmtt3MXswpm2pmD5rZ2rA/pRjrUoIoDyngC+4+H1gIXG9m8yOOKWo3AK9EHUSZ+BZwv7ufAJzKON4uZjYL+BzQ7O4nA3HgimijGnHLgA8OKvsy8JC7Hws8FI4fMSWIMuDuW9z96XB4H0EDMCvaqKJjZrOBDwM3Rx1L1MxsEvBu4HsA7t7r7rujjSpyCaDazBJADbA54nhGlLv/N7BrUPHFwPJweDnwsWKsSwmizJhZE3Aa8ES0kUTqX4A/BQ7tZddj0zygHbglPOR2s5nVRh1UVNx9E/ANYD2wBdjj7r+MNqqyMN3dt4TDW4HpxVioEkQZMbMJwN3A5919b9TxRMHMPgJsd/c1UcdSJhLA6cB33f00oIMiHT4YjcJj6xcTJM6ZQK2ZXRVtVOXFg3sXinL/ghJEmTCzJEFyuM3d74k6ngidA1xkZq3AHcB5ZnZrtCFFaiOw0d2ze5R3ESSM8eoCYJ27t7t7H3APsDjimMrBNjObARD2txdjoUoQZcCCN8B/D3jF3b8ZdTxRcvevuPtsd28iOPn4a3cft78Q3X0rsMHMjg+LzgdejjCkqK0HFppZTfh/cz7j+KR9jpXAknB4CfCzYixUCaI8nANcTfBr+dmw+1DUQUnZ+Cxwm5k9DywA/i7ieCIT7kndBTwNvEDQho2rx26Y2e3AauB4M9toZr8PfB14n5mtJdjL+npR1qVHbYiISD7agxARkbxKliDy3e03aLqZ2bfN7HUze97MTs+ZtiS8I3CtmS3JN7+IiJRWKfcglvHWu/1yXQgcG3bXAd+F4JZx4KvA2cBZwFeLddu4iIgULlGqBbv7f4c3fQ3lYmBFeM3u4+HzZmYA5wIPuvsuADN7kCDR3H6w9dXX13tT08FWJyIig61Zs2bHUO+kLlmCKMAsYEPO+MawbKjyg2pqaqKlpaWoAYqIjHVm1jbUtFF9ktrMrjOzFjNraW9vjzocEZExJco9iE3AnJzx2WHZJoLDTLnlj+RbgLvfRHgNdHNzs67XFZERl8k4GXcyTtjPGc4MKs8MMTxUnXzLGbxMd+qqkpzZNLXony3KBLES+IyZ3UFwQnqPu28xsweAv8s5Mf1+4CtRBSkiQ3N3elIZ+tIZelMZerP9IYb70hl68pRl6/XkKQvqedhP05vKkMr0N6YexuFho+kEfQ40ouAEdQmn5dZ9y7yZbHkw7cD84ROOctdRLreRLZgzmZ9ef07Rl1uyBBHe7XcuUG9mGwmuTEoCuPu/A/cBHwJeBzqBa8Npu8zsb4CnwkXdmD1hLTJeZDJBg5jKOKl00Eim0k5fOtt45gwfaFRzpw8aT2foGzRPX05ZTzpDX9gw96W9f3pOnbc06GHdYqpIxKiMx0gmYlTEY1Qkwi4sq4zHqKlIkIgbcTPMwMwwIBaOx8wg7AflYZ0BZTnzWlgHG1C3fzzfMrN1IBYLYonF+tcRz1lfPNa/3GB84HCwrKBOPHc4ZgM+Uzw29DInVMaL+nfIKuVVTFcOM92B64eY9n3g+6WISyQfd6ejN83erj72dPX197tTdPWlSaX7G89U2kll+hvtoAEP+qlMfyOcSjt9GacvlSGV6W+wg/LcBj9YXl8qQ1+YEDIl+mUajxnJuJGMh41uPEYyMWg8blQkYkyoTOTUCcqTsZxGO7cRz9OYV8RjVObUza6jIpFTnlM3GQ8aRykfUR5iEimq3lSGvd1vbeD7h4PyvV2pA+PZunu7U6QPoVU2I2g4Y0Yip2FNhI1oMh4jEQ+mVcSNRCxGVdKoyClPxiysFzvQaGfnT4TjyXDebKOdjPd3FQkbOD6ose+v318vHlMDLIVTgpCykMk4nX1pOntS7O9J0dmbDhv0VE7D3t/oD0wCQb2uvvRB11ERjzGxOsmk6gQTq5NMra2gaVotE6sTTKpOMrEqGfSrkwfGJ1YnqK6IB41+IkYibNTV0Mp4oAQhhyWVztDRk6ajN0VHTqMe9FPs70nT0ZMKG/x0WBbU7egNp4X1s8PDMYO6ysSABvzohgkDG/ia3IY+MaDRr0qW5jityFilBCF09aZp29VB645O2nZ2sGVP94GGO9vwBw17KkgKPSl6UoW/DbS2Ik5tZYIJlQlqKuPUViSYPrGK2srEgWkDh4M6E3KTQXWSusoEMf1yFxkxShDjxP6eFG07O2jb2Unrzg5ad3TQujNICNv29gyoW1eVoC7baIcN9rTamgGN91CNeu48tRUJqpNxNeoio5QSxBiyt7uPth1BAmjb2cG6cI+gdWcnO/YPTAL1EyppmlbDO49pYF59DY3TammaVsvcaTVMqk6WLshMGlI9kOoeot/VP55JlS6O0SiWhEQVJCqDfrJq4Hi2H6+E2Kh+SEK0MhlIh9/Bvu6DfFe7wdP5/wb5ymKJ4DjpKKIEEbVUL3S0Q8d26NzFcO8a39+TYtveHrbt7Q67YHj7vh72dfcNqDu5JsnpE6u4cFYVR02qZHpdFUdNrGL6xCqCHNAZdqFugvvYD/yDZP8ZBv1j9HUN08gfpJ8ZGKOUSLxiiIar0P4QjVyisjwaOfdB36/D+C4O1U/3liZmixWw7auH+ZsMMa22HmY3Fz1kJYhS6O0MGvz9YcO/fzt07MgZbg/726F7zyEtekLYHZ1vYsWg8RSwK+yKabgGpWZaYV/q4f4RRuEvrpJxD5JrsRrEnn3B9zDV89Zfyeme4eMpd8N9v6qnHGHCzCbL2KAfU0P1D+HvMuSPq4PsUc8+E/7gV8XfjEVf4ljkDj17BzX47TkNfU6D37EDevfnX07lJJjQALVHwfT5UHsu1DbAhAZ22WT+edVO1u3oprO3/4tgBg11lcyYVM3bJlUxY1IVb5tYzYxJVRw1sZLKRAmuzLHY0P8o8Qo12mNdJhP8ih6qERtmL3dkWP7vaLJ67H5H06mh9+zjg38dFocSRF83bHhiUIOf8+u/Y0dQlvdXlQW/lsNGnlnNMOGoYLy2oX84209UDhnGt1e+xB1b27jyrLk0Taulqb6Gpmm1zJ5SQ0VCx5NlBMViEAvPcUj5iCeCrqJ2xFapBNGzD1Zc1D8eSwxs4BtOyGnkjwqO9WWHa6YFf7AjtK+7j7vWbOQjp8zkxotPPuLliYgUgxJEzTRY8vP+X/lVk0f8CpC71mxkf0+KpYubRnS9IiIHowQRi8G8d0W2+kzGWb6qldPnTubUOZMji0NEZDAd3I7YI69tp3VnJ0vPmRd1KCIiAyhBROyWx1qZPrGSC09+W9ShiIgMoAQRobXb9vE/a3dw9cJGknH9KUSkvKhVitCyVa1UJGJcedbcqEMREXkLJYiI7Ons456nN3HxqTOZNmHo+yNERKKiBBGRO1vW09WXZuk5TVGHIiKSlxJEBNIZZ/mqNs6aN5WTZk6KOhwRkbyUICLw4Mvb2LS7i9/T3oOIlDEliAgsW7WOWZOrueDE6VGHIiIyJCWIEfbKlr08/sYurlnUSEKXtopIGSuohTKze8zsw2amFu0ILXuslapkjMvPnBN1KCIiB1Vog///gE8Ba83s62Z2fAljGrN2dfTy02c3ccnps5lcU5rnt4uIFEtBCcLdf+XunwZOB1qBX5nZKjO71syGfIGxmX3QzF41s9fN7Mt5pjea2UNm9ryZPWJms3Ompc3s2bBbeegfrfzc/uR6elIZPbVVREaFgg8Zmdk0YCnwB8AzwLcIEsaDQ9SPA98BLgTmA1ea2fxB1b4BrHD3U4Abga/lTOty9wVhdxGjXF86ww9Wt/HOY+o5bnpd1OGIiAyr0HMQPwH+B6gBPuruF7n7ne7+WYJXJOdzFvC6u7/h7r3AHcDFg+rMB34dDj+cZ/qY8cBLW9m6t1t7DyIyahS6B/Ftd5/v7l9z9y25E9y9eYh5ZgEbcsY3hmW5ngMuCYc/DtSFeyoAVWbWYmaPm9nHCoyzbN3yWHQ9OWQAAA2eSURBVCuN02o474Sjog5FRKQghSaI+WZ24G02ZjbFzP6oCOv/IvAeM3sGeA+wCUiH0xrD5PMp4F/M7OjBM5vZdWESaWlvby9COKXx/MbdrGl7k2sWNRGLjcGXqYvImFRogvhDd9+dHXH3N4E/HGaeTUDutZyzw7ID3H2zu1/i7qcBfxaW7Q77m8L+G8AjwGmDV+DuN7l7s7s3NzQ0FPhRRt6yx1qprYjzyebZw1cWESkThSaIuJkd+OkbnoAe7jrNp4BjzWyemVUAVwADrkYys/qceyu+Anw/LJ9iZpXZOsA5wMsFxlpWtu/r5r+e38wnzpjNxKohL/gSESk7hSaI+4E7zex8MzsfuD0sG5K7p4DPAA8ArwA/cveXzOxGM8telXQu8KqZvQZMB/42LD8RaDGz5whOXn/d3UdlgvjhE+vpSztLdHJaREYZc/fhKwW/8v8XcH5Y9CBws7unh55rZDU3N3tLS0vUYQzQm8qw+Ou/5uRZE1l27VlRhyMi8hZmtmaoi40ShSzA3TPAd8NOCnTfC1vYsb+Ha8+ZF3UoIiKHrKAEYWbHEtzENh+oypa7+9tLFNeo5+7c8tg63t5Qy7uOqY86HBGRQ1boOYhbCPYeUsB7gRXAraUKaix4ZsNuntu4h2sX69JWERmdCk0Q1e7+EME5izZ3/yvgw6ULa/S75bFW6qoSXHK6Lm0VkdGpoENMQE94onqtmX2G4H6GoR6xMe5t3dPNL17YwtLFTdRWFrqJRUTKS6F7EDcQPIfpc8AZwFXAklIFNdrd+ngbaXeuWdQUdSgiIodt2J+34U1xl7v7F4H9wLUlj2oU6+5L88Mn13PBidOZO60m6nBERA7bsHsQ4b0O7xyBWMaElc9tZldHL9fqxjgRGeUKPUD+TPjSnh8DHdlCd7+nJFGNUu7OssdaOX56HYuOnjb8DCIiZazQBFEF7ATOyylzQAkix5PrdvHylr187ZJ3kPPoKhGRUanQO6l13qEAy1a1MrkmyccWDH7thYjI6FPondS3EOwxDODuv1f0iEapjW928sBLW7nu3UdTXRGPOhwRkSNW6CGmn+cMVxG8/W1z8cMZvX7weBtmxtWLGqMORUSkKAo9xHR37riZ3Q78piQRjUKdvSnueHIDHzhpOrMmV0cdjohIURR6o9xgxwJ6uXLop89sZk9Xn57aKiJjSqHnIPYx8BzEVuBLJYlolHF3lq1ax0kzJ9LcOCXqcEREiqbQQ0x1pQ5ktFr1u528tm0/3/jkqbq0VUTGlIIOMZnZx81sUs74ZDP7WOnCGj1ueWwd02or+MgpM6IORUSkqAo9B/FVd9+THXH33cBXSxPS6NG2s4OHfrudT589l6qkLm0VkbGl0ASRr964f4718lVtxM349EJd2ioiY0+hCaLFzL5pZkeH3TeBNaUMrNzt70nx45YNfPiUGUyfWDX8DCIio0yhCeKzQC9wJ3AH0A1cX6qgRoO712xkX0+KpXpqq4iMUYVexdQBfLnEsYwamYyzfFUrC+ZM5rS5urRVRMamQq9ietDMJueMTzGzB0oXVnl7dG07b+zo4NpzmqIORUSkZAo9xFQfXrkEgLu/yTi+k3rZY60cVVfJhSfr0lYRGbsKTRAZM5ubHTGzJvI83XUwM/ugmb1qZq+b2VsOUZlZo5k9ZGbPm9kjZjY7Z9oSM1sbdmXz/uvXt+/n0dfauWphIxWJw31SiYhI+Sv0UtU/A35jZo8CBrwLuO5gM4Tvsv4O8D5gI/CUma1095dzqn0DWOHuy83sPOBrwNVmNpXgPotmgkS0Jpz3zUP4bCWxYnUrFfEYnzp77rB1RURGs4J+Arv7/QSN9avA7cAXgK5hZjsLeN3d33D3XoKrny4eVGc+8Otw+OGc6R8AHnT3XWFSeBD4YCGxltKerj7uWrORixbMpH5CZdThiIiUVKEP6/sD4AZgNvAssBBYzcBXkA42C9iQM74ROHtQneeAS4BvEbxjos7Mpg0xb+SvaftxywY6e9O6tFVExoVCD6LfAJwJtLn7e4HTgN0Hn6UgXwTeY2bPAO8BNgHpQmc2s+vMrMXMWtrb24sQztDSGWf56lbOaprKybMmDVtfRGS0KzRBdLt7N4CZVbr7b4Hjh5lnEzAnZ3x2WHaAu29290vc/TSC8xzZ5zwNO29Y9yZ3b3b35oaGhgI/yuH59W+3s2FXF0t1aauIjBOFJoiN4X0QPwUeNLOfAW3DzPMUcKyZzTOzCuAKYGVuBTOrN7NsDF8Bvh8OPwC8P7zfYgrw/rAsMrc8to6Zk6p4//zpUYYhIjJiCr2T+uPh4F+Z2cPAJOD+YeZJmdlnCBr2OPB9d3/JzG4EWtx9JXAu8DUzc+C/CR/f4e67zOxvCJIMwI3uvuvQPlrxvLp1H6t+t5MvffAEEnFd2ioi48MhP5HV3R89hLr3AfcNKvvLnOG7gLuGmPf79O9RRGrZqnVUJWNcedac4SuLiIwR+jk8jDc7evnJM5v4+GmzmFxTEXU4IiIjRgliGHc8tYHuvgxLF8+LOhQRkRGlBHEQqXSGH6xuZfHR0zj+bXott4iML0oQB/HLl7exeU83156jvQcRGX+UIA5i2WOtzJlazXknjNsH14rIOKYEMYQXN+3hydZdLFnURDxmUYcjIjLilCCGsGxVKzUVcT7ZrEtbRWR8UoLIY8f+HlY+u5lPnDGbSdXJqMMREYmEEkQetz+xnt50hiV6aquIjGNKEIP0pjL84PE23nNcA0c3TIg6HBGRyChBDPKLF7ewfV+PntoqIuOeEsQgtzzWytvra3nPsaV9fLiISLlTgsjxzPo3eXbDbpYsbiKmS1tFZJxTgsixbFUrdZUJLj1jdtShiIhETgkitG1vN/c+v4VPNs9hQuUhPwVdRGTMUYII3fZ4G2l3lixujDoUEZGyoAQB9KTS3PbEes4/4Sgap9VGHY6ISFlQggD+67kt7Ozo1VNbRURyjPsE4e7c8tg6jps+gcVHT4s6HBGRsjHuE0Tbzk7WbtvP0sXzMNOlrSIiWeP+cp2m+lpWfeU8XbkkIjKIWkWgfkJl1CGIiJSdcX+ISURE8lOCEBGRvMzdo46hKMysHWiLOo4jVA/siDqIMqLtMZC2Rz9ti4GOZHs0unvep5OOmQQxFphZi7s3Rx1HudD2GEjbo5+2xUCl2h46xCQiInkpQYiISF5KEOXlpqgDKDPaHgNpe/TTthioJNtD5yBERCQv7UGIiEheShBlwMzmmNnDZvaymb1kZjdEHVPUzCxuZs+Y2c+jjiVqZjbZzO4ys9+a2StmtijqmKJkZn8c/p+8aGa3m1lV1DGNJDP7vpltN7MXc8qmmtmDZrY27E8pxrqUIMpDCviCu88HFgLXm9n8iGOK2g3AK1EHUSa+Bdzv7icApzKOt4uZzQI+BzS7+8lAHLgi2qhG3DLgg4PKvgw85O7HAg+F40dMCaIMuPsWd386HN5H0ADMijaq6JjZbODDwM1RxxI1M5sEvBv4HoC797r77mijilwCqDazBFADbI44nhHl7v8N7BpUfDGwPBxeDnysGOtSgigzZtYEnAY8EW0kkfoX4E+BTNSBlIF5QDtwS3jI7WYzG7evPXT3TcA3gPXAFmCPu/8y2qjKwnR33xIObwWmF2OhShBlxMwmAHcDn3f3vVHHEwUz+wiw3d3XRB1LmUgApwPfdffTgA6KdPhgNAqPrV9MkDhnArVmdlW0UZUXDy5NLcrlqUoQZcLMkgTJ4TZ3vyfqeCJ0DnCRmbUCdwDnmdmt0YYUqY3ARnfP7lHeRZAwxqsLgHXu3u7ufcA9wOKIYyoH28xsBkDY316MhSpBlAELXmX3PeAVd/9m1PFEyd2/4u6z3b2J4OTjr9193P5CdPetwAYzOz4sOh94OcKQorYeWGhmNeH/zfmM45P2OVYCS8LhJcDPirFQJYjycA5wNcGv5WfD7kNRByVl47PAbWb2PLAA+LuI44lMuCd1F/A08AJBGzau7qo2s9uB1cDxZrbRzH4f+DrwPjNbS7CX9fWirEt3UouISD7agxARkbyUIEREJC8lCBERyUsJQkRE8lKCEBGRvJQgRMqAmZ2rJ9dKuVGCEBGRvJQgRA6BmV1lZk+GNzP+R/jeiv1m9s/hOwoeMrOGsO4CM3vczJ43s59kn9FvZseY2a/M7Dkze9rMjg4XPyHnvQ+3hXcKi0RGCUKkQGZ2InA5cI67LwDSwKeBWqDF3U8CHgW+Gs6yAviSu59CcNdvtvw24DvufirBc4SyT+E8Dfg8MB94O8Ed9iKRSUQdgMgocj5wBvBU+OO+muChaBngzrDOrcA94XscJrv7o2H5cuDHZlYHzHL3nwC4ezdAuLwn3X1jOP4s0AT8pvQfSyQ/JQiRwhmw3N2/MqDQ7C8G1Tvc59f05Ayn0f+nREyHmEQK9xDwCTM7Cg68B7iR4P/oE2GdTwG/cfc9wJtm9q6w/Grg0fCNgRvN7GPhMirNrGZEP4VIgfQLRaRA7v6ymf058EsziwF9wPUEL/E5K5y2neA8BQSPXf73MAG8AVwbll8N/IeZ3Rgu45Mj+DFECqanuYocITPb7+4Too5DpNh0iElERPLSHoSIiOSlPQgREclLCUJERPJSghARkbyUIEREJC8lCBERyUsJQkRE8vr/Bma/3Ibl3OwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hVwhL433vCVt",
        "outputId": "8c7bd0cc-c11f-48bf-ea01-cce1576159be"
      },
      "source": [
        "predictions, labels = predict(test_dl, clf)\n",
        "\n",
        "accuracy = accuracy_score(predictions, labels)\n",
        "ACCURACY_STATS.append((5, \"SMOTE\", accuracy))\n",
        "\n",
        "print(\"accuracy: {:.3f}\".format(accuracy))\n",
        "print(\"confusion matrix\")\n",
        "print(confusion_matrix(predictions, labels))"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy: 0.978\n",
            "confusion matrix\n",
            "[[481   5]\n",
            " [  7  64]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fwT1UzBFvN-C"
      },
      "source": [
        "## Text Attack Augmentation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A81i28NbvGB9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "54265347-8115-47b1-a2a6-66c934a0a7c9"
      },
      "source": [
        "spam_train_df = pd.read_csv(SPAM_TA_TRAIN_FILE, sep='\\t')\n",
        "spam_train_df.head()"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "      <th>embedding</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Ugh I don't wanna get out of bed. It's so warm.</td>\n",
              "      <td>ham</td>\n",
              "      <td>5.3949911e-02,5.8220369e-03,-3.1541418e-02,3.3...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Dear i have reache room</td>\n",
              "      <td>ham</td>\n",
              "      <td>4.7420025e-02,2.3147971e-03,1.6905151e-02,-1.1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I always chat with you. In fact i need money c...</td>\n",
              "      <td>ham</td>\n",
              "      <td>3.6402624e-02,-1.0346117e-01,5.0689392e-02,2.1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Yes just finished watching days of our lives. ...</td>\n",
              "      <td>ham</td>\n",
              "      <td>1.6084902e-02,-1.7980557e-03,-4.4794872e-02,3....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Nice.nice.how is it working?</td>\n",
              "      <td>ham</td>\n",
              "      <td>-4.0968325e-02,-2.6070992e-02,-1.6136026e-02,-...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            sentence  ...                                          embedding\n",
              "0    Ugh I don't wanna get out of bed. It's so warm.  ...  5.3949911e-02,5.8220369e-03,-3.1541418e-02,3.3...\n",
              "1                            Dear i have reache room  ...  4.7420025e-02,2.3147971e-03,1.6905151e-02,-1.1...\n",
              "2  I always chat with you. In fact i need money c...  ...  3.6402624e-02,-1.0346117e-01,5.0689392e-02,2.1...\n",
              "3  Yes just finished watching days of our lives. ...  ...  1.6084902e-02,-1.7980557e-03,-4.4794872e-02,3....\n",
              "4                       Nice.nice.how is it working?  ...  -4.0968325e-02,-2.6070992e-02,-1.6136026e-02,-...\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zkhh7a0o4qyD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5d884594-7632-49a2-a4d6-31fc2f390450"
      },
      "source": [
        "spam_train_df[\"label\"].value_counts()"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ham     3355\n",
              "spam    2712\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zVC_xZK14ur4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        },
        "outputId": "cb23240a-efe3-490e-99df-2d87afd89343"
      },
      "source": [
        "spam_train_df[\"label\"].value_counts().plot(kind=\"bar\")"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f685c464d30>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEMCAYAAAAoB2Y1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASTUlEQVR4nO3db5Bd9X3f8ffHEthOnQRRNlSV5Igk6mSw48h0I+gkD1w8BgEPRKatCw9ijcOM0g50ktbTVs4kJbbDjD1t7Bl3CFM5qBauY8ok9qDYcohK3CSeBqOVTQCBGbb8GUkVsI4wNvVEjei3D+5vk1uxq11Ju/cq+3u/Zu7cc77nd879nuHyuUfnnns2VYUkqQ9vGHcDkqTRMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjqyYOgneVOSh5P8WZJDST7U6p9O8mySR9pjc6snySeTTCd5NMkVQ9vanuTp9ti+fLslSZrL6kWMOQFcXVWvJrkA+GqSL7dl/7qqfueU8dcBm9rjSuAu4MokFwO3A5NAAQeT7K2ql5diRyRJC1sw9Gvw661X2+wF7XG6X3RtA+5p6z2U5KIka4F3Afur6jhAkv3AVuBz823okksuqY0bNy5iNyRJsw4ePPitqpqYa9lijvRJsgo4CPwYcGdVfS3JPwfuSPLvgAeBnVV1AlgHHB5a/UirzVef18aNG5mamlpMi5KkJsnz8y1b1Be5VfVaVW0G1gNbkrwd+CDw48BPARcD/3YJeiXJjiRTSaZmZmaWYpOSpOaMrt6pqm8DXwG2VtWxGjgB/GdgSxt2FNgwtNr6Vpuvfupr7KqqyaqanJiY818nkqSztJirdyaSXNSm3wy8B/hmO09PkgA3Ao+3VfYC72tX8VwFvFJVx4AHgGuSrEmyBrim1SRJI7KYc/prgT3tvP4bgPuq6otJ/jDJBBDgEeCftfH7gOuBaeB7wPsBqup4ko8AB9q4D89+qStJGo2cz7dWnpycLL/IlaQzk+RgVU3Otcxf5EpSRwx9SeqIoS9JHVnUj7N0eht3fmncLawoz330hnG3IK1YHulLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SerIgqGf5E1JHk7yZ0kOJflQq1+W5GtJppP81yQXtvob2/x0W75xaFsfbPWnkly7XDslSZrbYo70TwBXV9VPApuBrUmuAj4GfKKqfgx4Gbiljb8FeLnVP9HGkeRy4CbgbcBW4DeTrFrKnZEknd6CoV8Dr7bZC9qjgKuB32n1PcCNbXpbm6ctf3eStPq9VXWiqp4FpoEtS7IXkqRFWdQ5/SSrkjwCvATsB/4n8O2qOtmGHAHWtel1wGGAtvwV4G8P1+dYZ/i1diSZSjI1MzNz5nskSZrXokK/ql6rqs3AegZH5z++XA1V1a6qmqyqyYmJieV6GUnq0hldvVNV3wa+AvwD4KIkq9ui9cDRNn0U2ADQlv8g8OfD9TnWkSSNwGKu3plIclGbfjPwHuBJBuH/j9uw7cD9bXpvm6ct/8Oqqla/qV3dcxmwCXh4qXZEkrSw1QsPYS2wp11p8wbgvqr6YpIngHuT/DrwDeDuNv5u4DNJpoHjDK7YoaoOJbkPeAI4CdxaVa8t7e5Ikk5nwdCvqkeBd85Rf4Y5rr6pqr8A/sk827oDuOPM25QkLQV/kStJHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0JekjizmLpuS/gbbuPNL425hxXjuozeMu4Vz5pG+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1ZMHQT7IhyVeSPJHkUJJfbPVfS3I0ySPtcf3QOh9MMp3kqSTXDtW3ttp0kp3Ls0uSpPks5jYMJ4EPVNXXk3w/cDDJ/rbsE1X1H4YHJ7kcuAl4G/B3gf+W5O+1xXcC7wGOAAeS7K2qJ5ZiRyRJC1sw9KvqGHCsTX83yZPAutOssg24t6pOAM8mmQa2tGXTVfUMQJJ721hDX5JG5IzO6SfZCLwT+For3Zbk0SS7k6xptXXA4aHVjrTafPVTX2NHkqkkUzMzM2fSniRpAYsO/SRvAX4X+KWq+g5wF/CjwGYG/xL4jaVoqKp2VdVkVU1OTEwsxSYlSc2ibq2c5AIGgf/Zqvo8QFW9OLT8U8AX2+xRYMPQ6utbjdPUJUkjsJirdwLcDTxZVR8fqq8dGvazwONtei9wU5I3JrkM2AQ8DBwANiW5LMmFDL7s3bs0uyFJWozFHOn/NPBzwGNJHmm1XwZuTrIZKOA54BcAqupQkvsYfEF7Eri1ql4DSHIb8ACwCthdVYeWcF8kSQtYzNU7XwUyx6J9p1nnDuCOOer7TreeJGl5+YtcSeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUkQVDP8mGJF9J8kSSQ0l+sdUvTrI/ydPteU2rJ8knk0wneTTJFUPb2t7GP51k+/LtliRpLos50j8JfKCqLgeuAm5NcjmwE3iwqjYBD7Z5gOuATe2xA7gLBh8SwO3AlcAW4PbZDwpJ0mgsGPpVdayqvt6mvws8CawDtgF72rA9wI1tehtwTw08BFyUZC1wLbC/qo5X1cvAfmDrku6NJOm0zuicfpKNwDuBrwGXVtWxtugF4NI2vQ44PLTakVabr37qa+xIMpVkamZm5kzakyQtYNGhn+QtwO8Cv1RV3xleVlUF1FI0VFW7qmqyqiYnJiaWYpOSpGZRoZ/kAgaB/9mq+nwrv9hO29CeX2r1o8CGodXXt9p8dUnSiCzm6p0AdwNPVtXHhxbtBWavwNkO3D9Uf1+7iucq4JV2GugB4Joka9oXuNe0miRpRFYvYsxPAz8HPJbkkVb7ZeCjwH1JbgGeB97blu0Drgemge8B7weoquNJPgIcaOM+XFXHl2QvJEmLsmDoV9VXgcyz+N1zjC/g1nm2tRvYfSYNSpKWjr/IlaSOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHVkw9JPsTvJSkseHar+W5GiSR9rj+qFlH0wyneSpJNcO1be22nSSnUu/K5KkhSzmSP/TwNY56p+oqs3tsQ8gyeXATcDb2jq/mWRVklXAncB1wOXAzW2sJGmEVi80oKr+OMnGRW5vG3BvVZ0Ank0yDWxpy6ar6hmAJPe2sU+ccceSpLN2Luf0b0vyaDv9s6bV1gGHh8YcabX56q+TZEeSqSRTMzMz59CeJOlUZxv6dwE/CmwGjgG/sVQNVdWuqpqsqsmJiYml2qwkiUWc3plLVb04O53kU8AX2+xRYMPQ0PWtxmnqkqQROasj/SRrh2Z/Fpi9smcvcFOSNya5DNgEPAwcADYluSzJhQy+7N179m1Lks7Ggkf6ST4HvAu4JMkR4HbgXUk2AwU8B/wCQFUdSnIfgy9oTwK3VtVrbTu3AQ8Aq4DdVXVoyfdGknRai7l65+Y5ynefZvwdwB1z1PcB+86oO0nSkvIXuZLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSMLhn6S3UleSvL4UO3iJPuTPN2e17R6knwyyXSSR5NcMbTO9jb+6STbl2d3JEmns5gj/U8DW0+p7QQerKpNwINtHuA6YFN77ADugsGHBHA7cCWwBbh99oNCkjQ6C4Z+Vf0xcPyU8jZgT5veA9w4VL+nBh4CLkqyFrgW2F9Vx6vqZWA/r/8gkSQts7M9p39pVR1r0y8Al7bpdcDhoXFHWm2++usk2ZFkKsnUzMzMWbYnSZrLOX+RW1UF1BL0Mru9XVU1WVWTExMTS7VZSRJnH/ovttM2tOeXWv0osGFo3PpWm68uSRqhsw39vcDsFTjbgfuH6u9rV/FcBbzSTgM9AFyTZE37AveaVpMkjdDqhQYk+RzwLuCSJEcYXIXzUeC+JLcAzwPvbcP3AdcD08D3gPcDVNXxJB8BDrRxH66qU78cliQtswVDv6punmfRu+cYW8Ct82xnN7D7jLqTJC0pf5ErSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1JFzCv0kzyV5LMkjSaZa7eIk+5M83Z7XtHqSfDLJdJJHk1yxFDsgSVq8pTjS/4dVtbmqJtv8TuDBqtoEPNjmAa4DNrXHDuCuJXhtSdIZWI7TO9uAPW16D3DjUP2eGngIuCjJ2mV4fUnSPM419Av4gyQHk+xotUur6libfgG4tE2vAw4PrXuk1SRJI7L6HNf/mao6muSHgP1Jvjm8sKoqSZ3JBtuHxw6At771refYniRp2Dkd6VfV0fb8EvAFYAvw4uxpm/b8Uht+FNgwtPr6Vjt1m7uqarKqJicmJs6lPUnSKc469JP8rSTfPzsNXAM8DuwFtrdh24H72/Re4H3tKp6rgFeGTgNJkkbgXE7vXAp8Icnsdn67qn4/yQHgviS3AM8D723j9wHXA9PA94D3n8NrS5LOwlmHflU9A/zkHPU/B949R72AW8/29SRJ585f5EpSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI6MPPSTbE3yVJLpJDtH/fqS1LORhn6SVcCdwHXA5cDNSS4fZQ+S1LNRH+lvAaar6pmq+j/AvcC2EfcgSd0adeivAw4PzR9pNUnSCKwedwOnSrID2NFmX03y1Dj7WWEuAb417iYWko+NuwONyXn//vwb9N784fkWjDr0jwIbhubXt9pfqapdwK5RNtWLJFNVNTnuPqS5+P4cjVGf3jkAbEpyWZILgZuAvSPuQZK6NdIj/ao6meQ24AFgFbC7qg6NsgdJ6tnIz+lX1T5g36hfV4CnzXR+8/05AqmqcfcgSRoRb8MgSR0x9CWpI4a+JHXkvPtxlpZekncAGxn6711Vnx9bQxJ/dS+uG3j9e/Pj4+qpB4b+CpdkN/AO4BDwf1u5AENf4/Z7wF8Aj/HX700tM0N/5buqqryTqc5H66vqHeNuojee01/5/tTbV+s89eUk14y7id54pL/y3cMg+F8ATgAByiMsnQceAr6Q5A3AX/LX780fGG9bK5s/zlrhkkwD/4pTzptW1fNja0oCkjzL4O9pPFYG0ch4pL/yzVSVN7XT+egw8LiBP1qG/sr3jSS/zeBKiROzRS/Z1HngGeC/J/ky//9700s2l5Ghv/K9mcH/UMNfmHnJps4Hz7bHhe2hEfCcviR1xCP9FS7Jm4BbgLcBb5qtV9XPj60pCUgyAfwbXv/evHpsTXXA6/RXvs8Afwe4FvgjBn+i8rtj7Uga+CzwTeAy4EPAcwz+up6Wkad3Vrgk36iqdyZ5tKrekeQC4E+q6qpx96a+JTlYVX9/9r3Zageq6qfG3dtK5umdle8v2/O3k7wdeAH4oTH2I82afW8eS3ID8L+Ai8fYTxcM/ZVvV5I1wK8w+CP0bwF+dbwtSQD8epIfBD4A/EfgB4B/Od6WVj5P76xwSd4I/CMGt6+9oJWrqj48tqYkjY1f5K589zP4qftJ4NX2+N9j7UgCkvxIkt9L8q0kLyW5P8mPjLuvlc4j/RUuyeNV9fZx9yGdKslDwJ3A51rpJuBfVNWV4+tq5fNIf+X7H0l+YtxNSHP4vqr6TFWdbI//wtD1+loeHumvUEkeY3C7hdXAJgb3OfHWyjpvJPkY8DJwL4P36j8F1gD/HqCqjo+vu5XL0F+hkvzw6ZZ7a2WNW7u18qzZIMrsfFV5fn8ZGPqSxiLJe4Hfr6rvJPlV4ArgI1X19TG3tqJ5Tl/SuPxKC/yfAa4Gfgu4a8w9rXiGvqRxea093wB8qqq+hLdYXnaGvqRxOZrkPzH4Andf+yGhmbTMPKcvaSySfB+wlcHfyH06yVrgJ6rqD8bc2opm6EtSR/ynlCR1xNCXpI4Y+pLUEUNfkjpi6EtSR/4fbxoAyGghWLAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KDqS2TkC4zT3"
      },
      "source": [
        "train_ds = create_dataset(spam_train_df)\n",
        "\n",
        "train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\n",
        "val_dl = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False)\n",
        "test_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False)"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pF8XDxhG4_IV"
      },
      "source": [
        "clf = SpamClassifier(CLF_LAYER_DIMS)\n",
        "clf = clf.to(dev)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.AdamW(clf.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hI72tUFE5EFs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "62832436-fe6b-47c1-b1b7-3fc4fc80a310"
      },
      "source": [
        "train_losses, train_accs, val_losses, val_accs = [], [], [], []\n",
        "for epoch in range(1, NUM_EPOCHS + 1):\n",
        "  train_loss, train_acc = train(train_dl, clf, loss_fn, optimizer)\n",
        "  val_loss, val_acc = validate(val_dl, clf, loss_fn) \n",
        "  print(\"EPOCH {:2d} | train_loss: {:.3f}, acc: {:.3f} | val_loss: {:.3f}, acc: {:.3f}\"\n",
        "    .format(epoch, train_loss, train_acc, val_loss, val_acc))\n",
        "  train_losses.append(train_loss)\n",
        "  train_accs.append(train_acc)\n",
        "  val_losses.append(val_loss)\n",
        "  val_accs.append(val_acc)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH  1 | train_loss: 0.579, acc: 0.842 | val_loss: 0.402, acc: 0.972\n",
            "EPOCH  2 | train_loss: 0.358, acc: 0.970 | val_loss: 0.341, acc: 0.975\n",
            "EPOCH  3 | train_loss: 0.341, acc: 0.976 | val_loss: 0.337, acc: 0.981\n",
            "EPOCH  4 | train_loss: 0.336, acc: 0.980 | val_loss: 0.335, acc: 0.981\n",
            "EPOCH  5 | train_loss: 0.334, acc: 0.982 | val_loss: 0.335, acc: 0.980\n",
            "EPOCH  6 | train_loss: 0.332, acc: 0.983 | val_loss: 0.336, acc: 0.978\n",
            "EPOCH  7 | train_loss: 0.330, acc: 0.986 | val_loss: 0.339, acc: 0.972\n",
            "EPOCH  8 | train_loss: 0.329, acc: 0.986 | val_loss: 0.338, acc: 0.974\n",
            "EPOCH  9 | train_loss: 0.328, acc: 0.987 | val_loss: 0.335, acc: 0.980\n",
            "EPOCH 10 | train_loss: 0.327, acc: 0.987 | val_loss: 0.335, acc: 0.978\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LC41hTXO5IKG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        },
        "outputId": "6d074a7d-22e4-4d1b-aa5c-3792bf21c4b0"
      },
      "source": [
        "training_lossplots(\"Text Attack\", train_losses, train_accs, val_losses, val_accs)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEjCAYAAAAomJYLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwldZ3v/9fnbFm6k17opnc6iAi0yGbYGhUERNxAdFRUsHG5jjOozFwdxXHmjj/mOoNexxnvYxgdRKVZLoiAihuKiMw43WCnWRyhQRpM0+mdXtLpTic559Tn90dVkpPkpHO6c07qnOT9fDzqcaq+9a06n1SS76eWb1WZuyMiIjJcIu4ARESkOilBiIhIUUoQIiJSlBKEiIgUpQQhIiJFKUGIiEhRShAiNcrMWszMzSwVdywyOSlByKRhZvsKhsDMDhRMv/8w1neemXWUWPcLUWN95rDym83sfw8razezCw81HpGJpgQhk4a7T+8fgBeBtxWU3V6p7zUzAz4A7Io+RSYFJQiZ9MwsYWbXmtnzZrbTzO4ys9nRvK+b2T0Fdb9kZg+a2TTgZ8DCgqOQhaN8xWuBBcAngcvNLBOt66PA+4HPRMv/yMxuBY4CfhSVfSaq+z0z22pmnWb2H2b2yoKYGszsn8xsQzT/N2bWUOTnfGd0dHJiObabiBKETAWfAN4OnAssBHYDN0TzPgW8ysyuMrPXAh8GVrj7fuBNwOaCo5DNo6x/BfAj4K5o+m0A7n4jcDvw5Wj5t7n7lQw9uvlytMzPgGOBI4HHouX6fQV4NbAcmA18BggKAzCzDwJfAi50998f2uYRKU4Xt2Qq+BjwcXfvgPB6AfCimV3p7t1mdiVhA90FfKK/XinMrBF4F/ABd8+a2d2Ep5nuOfiSQ7n7twvW+QVgt5nNiGL6EHCWu2+KqqyK6vUv8hdRnfMOJXaRsegIQqaCpcD3zWyPme0B1gF5YB6Auz8KvAAYg0cBpboMyAE/jaZvB95kZnNLXYGZJc3s+ugU2F6gPZo1JxrqgecPsoq/Am5QcpByU4KQqWAj8CZ3n1kw1PfvkZvZ1UAdsJnw9E2/Uh51vAKYTnhEshX4HpAG3neQdQwvex9wKXAhMANoicoNeAnoAY45SAwXAX9jZu8sIV6RkilByFTwDeCLZrYUwMzmmtml0fgrgP8NXAFcSXhB+ZRouW3AEdGpnhHMbBFwAfBW4JRoOJnwWsAHCtbxsmGLDi9rAnqBnUAj8A/9M9w9AL4NfNXMFkZHG2ebWV3B8k8BFwM3mNklpW0SkbEpQchU8DXgPuAXZtYFPAKcGd1gdhvwJXd/0t2fA/4auNXM6tz9GeAO4IXo9NTwXkxXAk+4+y/cfWv/APxf4KSoN9G3gGXR8j+IlvtHwj3+PWb2aeAWYAOwCXg6iq/Qp4H/BtYQdqX9EsP+d939ScJE9U0ze9O4tpZIxPTCIBERKUZHECIiUpQShIiIFKUEISIiRSlBiIhIUUoQIiJSlBKEiIgUpQQhIiJFKUGIiEhRShAiIlKUEoSIiBSlBCEiIkUpQYiISFFKECIiUpQShIiIFKUEISIiRSlBiIhIUUoQIiJSVCruAMplzpw53tLSEncYIiI1Ze3atS+5+9xi8yZNgmhpaaGtrS3uMEREaoqZbRhtnk4xiYhIUUoQQC4fkA887jBERKrKlE8QL+7s5nVffoj7f7817lBERKrKlE8Qi2Y1kEomuHnVH+MORUSkqkz5BJFMGB84eylr2nfz+02dcYcjIlI1pnyCAHhX6xIa0klWrmqPOxQRkaqhBAHMaEjzjtMW8cMnN7Nrf1/c4YiIVAUliMhVy1voywXc8dsX4w5FRKQqKEFEjp3XxGtePofbHtlALh/EHY6ISOyUIAqsWN7Cls4efvH0trhDERGJnRJEgfOPP5Ilsxu4+b/a4w5FRCR2ShAFkgnjA2e18Nv2XTy1WV1eRWRqU4IY5t3q8ioiAihBjDCjMezy+oMn1OVVRKY2JYgiVkRdXu9coy6vIjJ1KUEU8Yp5TZzz8iO4bbW6vIrI1KUEMYoVZ7ewubOHB9TlVUSmKCWIUVxwwjwWz2rgO7pYLSJTlBLEKPqf8vrbP+7i6c174w5HRGTCKUEcxHtaj1KXVxGZspQgDmJGY5rLTlvED57YxG51eRWRKUYJYgwrzm6hNxdw55qNcYciIjKhlCDGcNz8JpYfcwS3rm5Xl1cRmVKUIEqwYnnY5fWX69TlVUSmDiWIElx4wjwWzWzgO3rKq4hMIUoQJUgmjBXLl/LoH3exbou6vIrI1KAEUaJ3ty6hPp1Ql1cRmTKUIEo0szHDZacu5vuPq8uriEwNsSQIM7vYzJ41s/Vmdm2R+VeZ2Q4zeyIaPhJHnMOtWL6U3lzAd9vU5VVEJr8JTxBmlgRuAN4ELAPea2bLilT9rrufEg03TWiQozh+fjNnv+wIbtVTXkVkCojjCOIMYL27v+DufcCdwKUxxHFYrjqnhU17DvDLddvjDkVEpKLiSBCLgMJzNB1R2XDvNLPfmdndZrak2IrM7KNm1mZmbTt27KhErCP0d3m9edUfJ+T7RETiUq0XqX8EtLj7ScADwMpildz9RndvdffWuXPnTkhg/U95feQFdXkVkcktjgSxCSg8IlgclQ1w953u3htN3gS8eoJiK8l7Tg+7vN6yuj3uUEREKiaOBLEGONbMjjazDHA5cF9hBTNbUDB5CbBuAuMbU9jldRHff3wTe7rV5VVEJqcJTxDungM+DvycsOG/y92fMrPrzOySqNonzewpM3sS+CRw1UTHOZYVy1voyQZ8V095FZFJytw97hjKorW11dva2ib0Oy+/cTUbdx3g4b86j1SyWi/niIiMzszWuntrsXlq1cbhquVHq8uriExaShDjcOEJR7JoZoOezyQik5ISxDikkgmuPHspq1/YyTNb1eVVRCYXJYhxek/rEupSCVau2hB3KCIiZTXuBGFm15hZs4W+ZWaPmdlF5QiuFsya1t/ltUNdXkVkUinHEcSH3H0vcBEwC7gSuL4M660Z6vIqIpNRORKERZ9vBm5196cKyqaEExY0c+bRs7ll9QbyweToNiwiUo4EsdbMfkGYIH5uZk3AlHsW9gcHnvK6Le5QRETKohwJ4sPAtcDp7t4NpIEPlmG9NeXCE+axcEa9uryKyKRRjgRxNvCsu+8xsyuAvwE6y7DemhJ2eW1h1fM7eXZrV9zhiIiMWzkSxNeBbjM7GfgU8DxwSxnWW3MuPz3q8rq6Pe5QRETGrRwJIufhA50uBf7V3W8Amsqw3poza1qGt5+yiHsf66CzOxt3OCIi41KOBNFlZp8j7N76EzNLEF6HmJIGury2vRh3KCIi41KOBPEeoJfwfoithC8A+j9lWG9NWrawmTPU5VVEJoFxJ4goKdwOzDCztwI97j4lr0H0++DyFjp2H+BBdXkVkRpWjkdtvBv4LfAu4N3Ao2b2J+Ndby17w7Koy+vq9rhDERE5bOU4xfR5wnsgVrj7B4AzgL8tw3prViqZ4Iqzl/Jf63fyh23q8ioitakcCSLh7oVvzNlZpvXWtMtPP4q6VIKbdeOciNSocjTk95vZz83sKjO7CvgJ8NMyrLemzZ6W4dJTFvL9xzapy6uI1KRyXKT+K+BG4KRouNHdPzve9U4GK5a3cCCb5642PeVVRGpPqhwrcfd7gHvKsa7J5JULZ3BGy2xueaSdD73maJKJKfWQWxGpcYd9BGFmXWa2t8jQZWZ6/2bkqnNa2LjrAL96ZvvYlUVEqshhJwh3b3L35iJDk7s3lzPIWnbRsnks0FNeRaQGTfneRpWWSia44qyl/Gb9SzynLq8iUkOUICbAe884ioy6vIpIjVGCmACzp2W49OSF3PvYJjoPqMuriNSGWBKEmV1sZs+a2Xozu/Yg9d5pZm5mrRMZXyX0d3n9nrq8ikiNmPAEYWZJ4AbgTcAy4L1mtqxIvSbgGuDRiY2wMk5cFHV51VNeRaRGxHEEcQaw3t1fcPc+4E7Clw0N9/fAl4CeikaTz8E9H4GNv63o10B4FPHirm4eUpdXEakBcSSIRUDheZaOqGyAmZ0GLHH3n1Q8ms4X4cVH4FsXwU8+DT2Vu4XjoleGXV51sVpEakHVXaSO3kj3VcL3W49V96Nm1mZmbTt27Di8L5z9MvjzR+DMj8Gam+CGM+GZyjxKKq0uryJSQ+JIEJuAJQXTi6Oyfk3AicCvzawdOAu4r9iFane/0d1b3b117ty5hx9R3XR40/XwkV9Cwyy4871w1wroKv8Lfy4/fQmZVELvihCRqhdHglgDHGtmR5tZBrgcuK9/prt3uvscd29x9xbgEeASd2+reGSLW+FPH4bz/xae/RnccDqsXQlevovKR0yvU5dXEakJE54g3D0HfBz4ObAOuMvdnzKz68zskomOZ4RkGl73afizVTDvVfCjT8LNb4WX1pftK1Ysb6G7T11eRaS6mZdx7zhOra2t3tZW5oOMIIDHb4UH/hayPXDuZ+Cca8IkMk7v+sYqtu3t5aFPn6envIpIbMxsrbsXvdes6i5SV5VEAl69Aq5eA8e9CX719/Dv50LH+BNRf5fXXz+rLq8iUp2UIErRNA/evRIuvwMO7IabLoSfXQu9+w57lW985XzmN6vLq4hULyWIQ3H8m+HqR+H0j8Cj34B/Owv+8IvDWlU6meDKs5fyn8+9xPrt6vIqItVHCeJQ1TfDW74CH/o5ZKbB/3sX3P0h2Hfo92EMdHldtaECgYqIjI8SxOE66kz40/+A8/4a1v0I/rUVHr/tkLrEHjG9jktOXsg9j3Wwt0ddXkWkuihBjEeqDs77LHzsN3DkCfDDq+GWS2HXCyWv4qqBLq8dFQxUROTQKUGUw9zj4Kqfwlu+Cpsfh387G37zz5Af+6jgxEUzaF06i1tWtxPoKa8iUkWUIMolkYDTPxxexH75hfDLL8A3Xx8mjDGsWN7Chp3d/PoP6vIqItVDCaLcmhfC5bfDu28NL1x/83z4+eehb/+oi1x84nzmNdfxnf9qn7g4RUTGoARRKcsuCY8mTlsBq/817BK7/pdFq6aTCa48K+zyesVNj/L3P36a77Vt5PebOunJ5ic4cBGRkB61MRHa/wt+dA3sfA5Oeg+88R9g2pwhVfb15vjy/c/wxMY9PLu1i95cAEDC4Og50zh+QTMnzG/i+PnNHDe/icWzGjDTIzpEZHwO9qgNJYiJku2B//yn8OJ1XRNc/I9hsijSyOcDp33nfp7d2sUzW/aybmsXz2zdy8ZdBwbqNNWlOG5+E8cvCJPG8fObOG5+E031439OlIhMHUoQ1WTb0+ETYjvWwDHnw1v/GWa1lLTovt5cmDS27uWZLV08u7WLdVv30tWTG6izeFbDQMLoTx4tRzSSSupsooiMpARRbYI8rPkWPPj/heOv/2s4688hmTrkVbk7mzt7eGbLXp7Z2hUOW/bywkv7yUfdZutSCY6dN30gcZywIDxNNWd6Xbl/MhGpMUoQ1aqzA37yKfjD/bDgZHjb12DBKUVPOx2qnmye53fs45kt0RFHlDx2dPUO1JkzvY4TFjRFp6fC5PHyI6dTn06O+/tFpDYoQVQzd3jq+/Czz8L+7YBBujF8zlNmGmSmR5+NQ6fTjQXz+ssPUidVB2a8tK83PDW1ZW90uqqLP2wbvCieTBiLZzXQVJ+iMZ2isS7JtEyKxkySaXUpGjJJpmWSNGZSTKsb/GxID51ujJZJ69RW7XKHIAe5Hsj1hkM+2sGwJFhilMEgcbD5ibLsBEl5HCxBHPo5DSkvMzjxHXDM6+HJO8PHiffth7590Nc9ON6zF/Zugez+qGx/+I9b8vckITOdOZlG5mSmcU5/Ipk9jWBeI/uCOnZm02zvSbGjN0FPPkF3t3GgK0F3zjiQT7A/l2BPDnqDJDmSZEmSJUXOU2QJy3L9ZdF8S6RJZzJkMnVkMhkydfVk0hka6zMjksmQpJNJUp9JUpdKREM4nikYr0snyCQTk+P6ijt4EDbIQS489dj/6dF4rhfyfQUNdg/khk3newvmjVZn+DoOUseDCv3AdvAEkjhYckmECShVPzik6yHVMOyzHtIN4c7RiHkFn6m6qF790M9kZsonMiWIatEwC876s0NbJp+LEkZBIunbD9nuwfFiyaa/PNsN+3eQ6NtPc99+mvv2cXTf/rAxGs3h/MU40BsN0ZPNAyxKKCmyXphskgPJJk8Sx/Co/gGM/RhgOEYQfToWdvk1wyyBRY2IWVSeSJCIyi0R1UkkSJiFn4lwXiKRIJGwsG4iSTJhYZlBwgOS5Eh4QII8Cc+TII95QMLzWDQkPGzQLchjQW6wcQ/yBY1+riAZRGVeoftdknVRI1pXMNQPfmamQ+OcgvLM0PmpuqHrSGai32kwyuDhz3Kw+cHB5o8xBMHgtutPbtmecAcqtx2yB6Kygk8O9yyJjZ1gkpmC2Ib/XF4wXuxn9lGWK1x2tG01bDsuOg0+dH+5/moGKEHUsmQKkjOgfkZ515vPhkOQDZNQMGw63zf6vIHpXEH58OkciSBLJh8OBFk8nyWf6yOX7SOfDceDfJYgcAIP8CAgCALcPfoMCALHo3nu4TyPGpAg+scKy3zIP5b3//MRjufdSRDggBOQJ0xBecLyKCWQI0lAghwJ8iQHy3xoWT46gnISBJYksCQeDSSi8UQKTyQgmcITSUikotMySSyRCqeTSczCz0QiRZDM4Mm6aIjGoz1oT2Ww/oY8XY+l6rFUhlQyQTJhpKPPVMKiz2g6GZYVThfWK7acWXgqMmFGwqju+3HcB4+Gsj2QOzDKZ8/IxDLw2Tu0Xv+87l3h3/OQo5vkyCMdSxeMFznVVmy5Qzpdl4AZiyuy+ZQgZKRkuizv3T4URvjHGMcfpLvTlw/ozQX0ZoNwPJvnQC4sy+bDIZf3aNzJBeF0X1SeCwL6cgG5wMlFdbL5cPrgyw7WyeYLls0G5HqHLpsLnHwQLp8PnGy+f884AHqioXPCt58ZJMxImg2OJ2xEIgk/o/HEyPHhy/cnnyHjZiQSg+vqX38yQfSdYZ2xy+tIJOpJ2Kxonf3zC+JJGYmMkez/OaI6if76I+IIY+z/3oFxK5w3GE+iYPsU/bmtYF5i6HqGfl/4NIb6CvxulSBkyjOz6LpGkor8l1VQEDjZKGHkAief9yGJJDdseqBe4AOJrdhyg/OdfDCYnLJ5J4iOwvIBg+PuBB5Oh0d94Q2fHpXno3pBQFTXw537IBwPonkD49G6wnUMjucDpy/vA+vO98cRrScffX++/7sK1t+/fH9s+YGfIyyrZacsmckPrj6n7OtVghCpYYmEUZdQt+TxGkhkwxJKf0I7WLkXJscRiS6aFwyr44NJbXhiHbJskaQ5tF64nrlNlbmnSQlCRKa88JROeMpHBk2C/oEiIlIJShAiIlLUpLmT2sx2ABvijmOc5gAvxR1EFdH2GErbY5C2xVDj2R5L3X1usRmTJkFMBmbWNtot71ORtsdQ2h6DtC2GqtT20CkmEREpSglCRESKUoKoLjfGHUCV0fYYSttjkLbFUBXZHroGISIiRekIQkREilKCEBGRopQgqoCZLTGzh8zsaTN7ysyuiTumuJlZ0sweN7Mfxx1L3MxsppndbWbPmNk6Mzs77pjiZGZ/Gf2f/N7M7jCzGnvE4viY2bfNbLuZ/b6gbLaZPWBmz0Wfs8rxXUoQ1SEHfMrdlwFnAVeb2bKYY4rbNcC6uIOoEl8D7nf344GTmcLbxcwWAZ8EWt39RCAJXB5vVBPuZuDiYWXXAg+6+7HAg9H0uClBVAF33+Luj0XjXYQNwKJ4o4qPmS0G3gLcFHcscTOzGcDrgG8BuHufu++JN6rYpYAGM0sBjcDmmOOZUO7+H8CuYcWXAiuj8ZXA28vxXUoQVcbMWoBTgUfjjSRW/wJ8hvBNOFPd0cAO4DvRKbebzGxa3EHFxd03AV8BXgS2AJ3u/ot4o6oK89x9SzS+FZhXjpUqQVQRM5sO3AP8hbvvjTueOJjZW4Ht7r427liqRAo4Dfi6u58K7KdMpw9qUXRu/VLCxLkQmGZmV8QbVXXx8N6Fsty/oARRJcwsTZgcbnf3e+OOJ0bnAJeYWTtwJ3C+md0Wb0ix6gA63L3/iPJuwoQxVV0I/NHdd7h7FrgXWB5zTNVgm5ktAIg+t5djpUoQVcDCt75/C1jn7l+NO544ufvn3H2xu7cQXnz8lbtP2T1Ed98KbDSz46KiC4CnYwwpbi8CZ5lZY/R/cwFT+KJ9gfuAFdH4CuCH5VipEkR1OAe4knBv+YloeHPcQUnV+ARwu5n9DjgF+IeY44lNdCR1N/AY8N+EbdiUeuyGmd0BrAaOM7MOM/swcD3wBjN7jvAo6/qyfJcetSEiIsXoCEJERIpSghARkaKUIEREpKhU3AGUy5w5c7ylpSXuMEREasratWtfGu2d1JMmQbS0tNDW1hZ3GCIiNcXMNow2T6eYRESkqElzBCEiUkn5wMnmA/KBk8s72SAgCJzAwYk+3XEHdwjccaLP/nkF00FB3cLlAwcGpsdYnrC8uT5Na8vssv/MShAiQi4f0J3Nc6AvHLr78hzI5jjQF9Ddl+NA/7xsPmrAwoZpuP4iL3gUUGE1L1rmI8qGrnPkfI9izgY+pOHO5p1c/3jg5INgoCwXNey5oHC8YF4QDC3rH4/qV/MtY6csmckPrj6n7OtVghCpcu5hI1jYSHf3N+ID07kh84aO56IGf7Dx7xlYR46ebEBfvjYfnJtOGsmEkU4kSCWNZCJBOmmkkkYqkSCViOYnE1FZWF6fjsaTiYHPdFR3sCxcLlx/uO6wbPB7EtY/gBmYGQYkzDAb/BxZDtC/3BjLE5Xb6MtPyyQrsn2VIESAIHB6cwE92Ty9uYDeXJ6ebPjZlwsb0GzeyeYCsvlwui8XleUHy7K5YdMF9cL5QTTfB+ZnC5brzRWUFdQ51L3XhEFjJkVDJklDOkljJjkwPrMxTUMmRUM6MWqdcDw1orwhnSQRtk5A2KgBWMF3W1Q4tKxgPJpTWHaweqN9TzJhA98llaEEMZXlc7BvK3Rugr2bYO/mcOh+afTj/QngQDYfkHPIWYacZchamixpspamj3Do9TS9pOj1ND2eosdTdAcpeoIU3UGS7iBFdy7FviDJ/lyKffkE+3JJunIJenNhQuiNEkI596DDPdZw77MulSCd7B/CskxqcLqpPk0mKu8fMqnh0+HebUMmSX3UYI/WgPePZ5IJNZ4ybkoQk1WuD7q2RI3+poLPgkSwbxv4sIYxPQ2mzQErTwc3B/LuAxfz8oETFE5H44V1+i/CJQnIWJYMOZrIkiFLysrTkOcsTc4y5NNp8nUZ8ok6PJEhSGbwZB2eykCyDlJ1WKpuyLil67B0PclUHYl0PYlMPcl0OJ7KNJBIhXUZWC4DqXpIZqLygvFkpviutFSX3n3F/5d6Ogt+l4fwex9RJxofvo5EvB1NlSBqUa53sJHfuxn2dhSMR3/A+7Yz4p0hmSaYsQiaF8KRJ0DzooJhYTjUzyjaYLk7+/vy7N7fx57uLLu7+9jdPTg+WJZlT3cfu6J6+3pzo/4YmVSC2Y0ZZk5PM6sxw6xpaWY2ZpjVGE4316epSyeoSyWpSyeoTyWpSwY0WJ76RJY6y1FP+JkhSyrow/J9kO8Nt1GuF/J9kOsZNt5HKt9LaqBOsfp9kO+G3G7oGaXO8OR6uMZsWKLGpXHO4O+peVH0u1wEDbOUZMajZ2+RHaiCHanOTdDbOXK5aXOhfmb49zDk76YXgtH/7g9JIj24I5GqL55EUnXh//Mbv1ie7yygBFFtsgeGNfabhk53bgpPAQ1XN2Ow8Z//qmENf/RZ3zxkEXen80CWLZ09bN3Tw9YNe9nauZ3tXb3s3j8yARzsNExTfSps5KPG/Zi505kZjc9q7G/4M2HZtLCsIZ2s7dMg+dzQxJHriRqKYomnN2w8htQpVr+/zrD6PXth29PhUaHnh8aRqi/4PS8amkSaF8KMxdB4xNRLIu7hHv5ojX9n9NnXNXLZ6fPCbTf7ZdDympHbtmkBpOtH/+4gP8bOR++w8bF2Vkar3wc9e6B7Z0U2oRJEOQVB+MfWsxd6u6B3bzS+t2C8q8h4ZzjeswcO7B653vqZ4T9580JYeOqwhn8RNC+AuqYhi+QDZ0dXL1v39rB1+wG2PreTLXs3sa2zhy2dPWzbG3725oY2+mZwxLRM1LBnWHpEI6csmcnMaUMb+9nTBsdnNqRJJafgPZfJVDhkJvAV0UE+PDocrcHbsAq6No/cg01mhiaNYslk2tzYT2mUzD38Xym2I9VZcESd3T9sQYOm+eHPPPcVcMz5IxNq04JwT308EknINAKN41tPzCbN+yBaW1t9XI/ayPVGjXRn1KB3FTTu/eOdQ8uHN/jF9kSGswTUNYdDfXPYsA+MN4eN/ZBTPwtGNEA92fxAA9//ubV/2Bt+7tjXSz4Y+rtNJ415zfUsmFE/7LOB+TPqmD+jgSOb6khPxcZ+MgkC2L/jIKdMOsIjkXzf0OUS6YK/v/5Gc/HQxnP6kYAN7sWWupdb5DTf2EdYBzkKy3aHn4UsETbuI2IvOCU3fR4k0xP2q6gFZrbW3VuLzdMRRNc2+JdXhX98Y0k1hA16f2Ne1wRN88LTO8PLB8abh5Znpo16qO/u7O3JDTb+f+xhS+fmqNE/wNa9vWztPMDu7uyIZafXpZg/o575zfW85tg5zG+uH5iePyMcZjdmhnRRlEkqkQj/LpvmwaJRXl8dBOFpidGSyKbHYN2Pi/xfGCOubR12nKnRr7n0n1uvawqvvQyvk26ApmGN/7QjwyM6KRttzfpmOOtjxRvzwka+rmn8h50H8YdtXbz/pkfZ0TUyUc2ZnmFecz2LZtZz2lEzR+z5z2uup6lee0VyCBIJmD43HBaeUryOO3TvKkgcHeEOVSJZvDEfrWfOaI1/ojI3d0n5KEGkG+AN18UdBd94+Hm6e3P89ZuPZ/6MBhZEe/9HNtdRl9I/ksTADKYdEQ4LToo7GomBEkQV2Lmvlx8/uYXLz1jCR193TNzhiIgAetx3VbhzzUb68gEfOHtp3KGIiAxQgohZLh9w2yMbeBs1MFYAABEgSURBVM3L5/DyI5vGXkBEZIIoQcTsF09vY0tnDyuWt8QdiojIEEoQMbt5VTuLZzVw/vFHxh2KiMgQShAxWrdlL7/94y4+cPZSkro/QUSqjBJEjFauaqc+neDdrUviDkVEZAQliJjs6e7jB09s4rJTFzGzsXI34ImIHK6KJggzu9jMnjWz9WZ2bZH5S83sQTP7nZn92swWF8zLm9kT0XBfJeOMw3fXbKQnG+jitIhUrYrdKGdmSeAG4A1AB7DGzO5z96cLqn0FuMXdV5rZ+cA/AldG8w64+yjPAKht+cC59ZENnHn0bI6f3zz2AiIiMajkEcQZwHp3f8Hd+4A7gUuH1VkG/Coaf6jI/EnpV89sp2P3AR09iEhVq2SCWARsLJjuiMoKPQm8Ixq/DGgysyOi6XozazOzR8zs7cW+wMw+GtVp27FjRzljr6iVq9pZMKOei5bNizsUEZFRxX2R+tPAuWb2OHAusAnof13W0ugZ5e8D/sXMRjykyN1vdPdWd2+dO3fuhAU9Huu3d/Gb9S9xxVlLp+ZLdkSkZlTyYX2bgML+m4ujsgHuvpnoCMLMpgPvdPc90bxN0ecLZvZr4FTg+QrGOyFWrtpAJpXg8tPVtVVEqltJu7Bmdq+ZvcXMDmWXdw1wrJkdbWYZ4HJgSG8kM5tTsM7PAd+OymeZWV1/HeAcoPDidk3a25Plnsc6eNtJCzliel3c4YiIHFSpDf6/EZ7qec7Mrjez48ZawN1zwMeBnwPrgLvc/Skzu87MLomqnQc8a2Z/AOYBX4zKTwDazOxJwovX1w/r/VST7m7roLsvz1W6OC0iNeCQ3kltZjOA9wKfJ7wA/U3gNncf+Q7MCTbud1JXWBA45//Tr5k9LcO9f35O3OGIiAAHfyd1yaeMot5FVwEfAR4HvgacBjxQhhgnvYef20H7zm51bRWRmlHSRWoz+z5wHHAr8DZ33xLN+q6ZVe9uexVZuaqduU11vOnEBXGHIiJSklJ7Mf1fd3+o2IzRDk1k0B9f2s+vn93BNRccSyalrq0iUhtKba2WmdnM/omol9GfVyimSeeW1e2kEsb7zzwq7lBEREpWaoL4H/33JwC4+27gf1QmpMllf2+Ou9s6ePOrFnBkc33c4YiIlKzUBJE0s4E32kQP4tMzqktw7+Ob6OrN6eK0iNScUq9B3E94Qfrfo+k/jcrkINydW1a186pFMzjtqJljLyAiUkVKTRCfJUwKfxZNPwDcVJGIJpFVz+/kue37+Mq7TqbgAExEpCaUlCDcPQC+Hg1SoptXtTN7Woa3nqSurSJSe0q9D+JYwpf5LAMGrrS6+8sqFFfN27irmwfXbePPzjuG+nQy7nBERA5ZqRepv0N49JADXg/cAtxWqaAmg9se2YCZ8f4zl8YdiojIYSk1QTS4+4OEz27a4O5fAN5SubBq24G+PHeu2chFy+axcGZD3OGIiByWUi9S90aP5X7OzD5O+F6H6ZULq7b98IlNdB7IqmuriNS0Uo8grgEagU8CrwauAFZUKqha5u7cvKqd4+c3cebRs+MOR0TksI2ZIKKb4t7j7vvcvcPdP+ju73T3RyYgvpqzpn03z2ztYsXyFnVtFZGaNmaCcPc88JoJiGVSWLmqnRkNad5+yqK4QxERGZdSr0E8bmb3Ad8D9vcXuvu9FYmqRm3pPMD9T23lw685moaMuraKSG0rNUHUAzuB8wvKHFCCKHD7Iy8SuHPlWeraKiK1r9Q7qT9Y6UBqXU82zx2/fZELjp/HktmNcYcjIjJupd5J/R3CI4Yh3P1DZY+oRv3kd1vYub+PFct19CAik0Opp5h+XDBeD1wGbC5/OLXJ3Vm5up1j5k7jNS+fE3c4IiJlUeoppnsKp83sDuA3FYmoBj2+cQ+/6+jkuktfqa6tIjJpHO4Lko8FjhyrkpldbGbPmtl6M7u2yPylZvagmf3OzH5tZosL5q0ws+eioapvylu5qp3pdSnecdrisSuLiNSIUq9BdDH0GsRWwndEHGyZJHAD8AagA1hjZve5+9MF1b4C3OLuK83sfMInxl5pZrOBvwNao+9dGy27u8Sfa8Js7+rhp/+9hfefuZTpdaWesRMRqX6lnmJqOox1nwGsd/cXAMzsTuBSoDBBLAP+ZzT+EPCDaPyNwAPuvita9gHgYuCOw4ijou54dCPZvPOBs3VxWkQml5JOMZnZZWY2o2B6ppm9fYzFFgEbC6Y7orJCTwLviMYvA5rM7IgSl41dXy7g9kc3cO4r5vKyuXp2oYhMLqVeg/g7d+/sn3D3PYSngMbr08C5ZvY4cC7hU2LzpS5sZh81szYza9uxY0cZwjk09z+1le1dvVylp7aKyCRUaoIoVm+s01ObgCUF04ujsgHuvtnd3+HupwKfj8r2lLJsVPdGd29199a5c+eO/VOU2cpV7Sw9opFzXzHx3y0iUmmlJog2M/uqmR0TDV8F1o6xzBrgWDM72swywOXAfYUVzGxO9J4JgM8B347Gfw5cZGazzGwWcFFUVjV+v6mTtRt2c+VZS0kk1LVVRCafUhPEJ4A+4LvAnUAPcPXBFnD3HPBxwoZ9HXCXuz9lZteZ2SVRtfOAZ83sD8A84IvRsruAvydMMmuA6/ovWFeLm1e105BO8q7WJWNXFhGpQeY+4gkaNam1tdXb2tom5Lt27uvl7Ot/xbtevZgvXvaqCflOEZFKMLO17t5abF6pvZgeMLOZBdOzzKyqTvlMpDvXbKQvF+iVoiIyqZV6imlOdPEYgOiGtTHvpJ6McvmA2x/ZwPJjjuAV8w7n9hARkdpQaoIIzOyo/gkza6HI012nggee3sbmzh4dPYjIpFfqsyE+D/zGzB4GDHgt8NGKRVXFVq5uZ9HMBi48YV7coYiIVFRJRxDufj/hc5GeJXzcxaeAAxWMqyo9s3Uvj7ywiyvPXkpSXVtFZJIr9WF9HwGuIbxh7QngLGA1Q19BOumtXLWBulSC96hrq4hMAaVeg7gGOB3Y4O6vB04F9hx8kcmlszvLDx7fxKWnLGTWtEzc4YiIVFypCaLH3XsAzKzO3Z8BjqtcWNXnrraNHMjmdXFaRKaMUi9Sd0T3QfwAeMDMdgMbKhdWdckHzi2PtHN6yyxeuXDG2AuIiEwCpb4P4rJo9Atm9hAwA7i/YlFVmYee2c7GXQf47MXHxx2KiMiEOeRXoLn7w5UIpJqtXN3O/OZ63vjK+XGHIiIyYQ73ndRTxvrt+/jP517i/WceRTqpzSUiU4davDHcsrqdTDLBe888asy6IiKTiRLEQXT1ZLlnbQdvPWkBc6bXxR2OiMiEUoI4iHvWdrC/T11bRWRqUoIYRRA4t6zewMlLZnLykpljLyAiMskoQYziP9e/xAsv7eeq5UvjDkVEJBZKEKNYuaqdOdMzvPlVC+IORUQkFkoQRWzYuZ+Hnt3O+844irpUMu5wRERioQRRxC2rN5A04/1n6fSSiExdShDD7O/NcVfbRi4+cT7zmuvjDkdEJDZKEMN8//FNdPXkuEpdW0VkiqtogjCzi83sWTNbb2bXFpl/lJk9ZGaPm9nvzOzNUXmLmR0wsyei4RuVjLOfu3PL6nZeubCZVy+dNRFfKSJStQ75YX2lMrMkcAPwBqADWGNm97n70wXV/ga4y92/bmbLgJ8CLdG85939lErFV8zq53fyh237+PKfnISZXikqIlNbJY8gzgDWu/sL7t4H3AlcOqyOA83R+AxgcwXjGdPK1e3MakxzyckL4wxDRKQqVDJBLAI2Fkx3RGWFvgBcYWYdhEcPnyiYd3R06ulhM3ttBeMMg9vdzQNPb+M9px9FfVpdW0VE4r5I/V7gZndfDLwZuNXMEsAW4Ch3PxX4n8D/M7Pm4Qub2UfNrM3M2nbs2DGuQG575EUArjhLT20VEYHKJohNwJKC6cVRWaEPA3cBuPtqoB6Y4+697r4zKl8LPA+8YvgXuPuN7t7q7q1z58497EB7snnuXPMib1g2j8WzGg97PSIik0klE8Qa4FgzO9rMMsDlwH3D6rwIXABgZicQJogdZjY3usiNmb0MOBZ4oVKB3vfEZvZ0Z/XUVhGRAhXrxeTuOTP7OPBzIAl8292fMrPrgDZ3vw/4FPBNM/tLwgvWV7m7m9nrgOvMLAsEwMfcfVeF4uTmVe0cN6+Js192RCW+QkSkJlUsQQC4+08JLz4Xlv2vgvGngXOKLHcPcE8lY+u3YWc367fv4+8uWaaurSIiBSqaIGpBy5xprP7c+TRmpvymEBEZQq0icIReJyoiMkLc3VxFRKRKKUGIiEhR5u5xx1AWZrYD2BB3HOM0B3gp7iCqiLbHUNoeg7QthhrP9ljq7kVvJJs0CWIyMLM2d2+NO45qoe0xlLbHIG2LoSq1PXSKSUREilKCEBGRopQgqsuNcQdQZbQ9htL2GKRtMVRFtoeuQYiISFE6ghARkaKUIKqAmS2J3s39tJk9ZWbXxB1T3MwsGb0w6sdxxxI3M5tpZneb2TNmts7Mzo47pjiZ2V9G/ye/N7M7zKw+7pgmkpl928y2m9nvC8pmm9kDZvZc9DmrHN+lBFEdcsCn3H0ZcBZwdfSO7qnsGmBd3EFUia8B97v78cDJTOHtYmaLgE8Cre5+IuGToi+PN6oJdzNw8bCya4EH3f1Y4MFoetyUIKqAu29x98ei8S7CBmD461mnDDNbDLwFuCnuWOJmZjOA1wHfAnD3PnffE29UsUsBDWaWAhqJ+V32E83d/wMY/vqDS4GV0fhK4O3l+C4liCpjZi3AqcCj8UYSq38BPkP4LpCp7mhgB/Cd6JTbTWY2Le6g4uLum4CvEL5sbAvQ6e6/iDeqqjDP3bdE41uBeeVYqRJEFTGz6YTvwfgLd98bdzxxMLO3AtujV81KuLd8GvD16B3t+ynT6YNaFJ1bv5QwcS4EppnZFfFGVV087Jpalu6pShBVwszShMnhdne/N+54YnQOcImZtQN3Aueb2W3xhhSrDqDD3fuPKO8mTBhT1YXAH919h7tngXuB5THHVA22mdkCgOhzezlWqgRRBSx8ld23gHXu/tW444mTu3/O3Re7ewvhxcdfufuU3UN0963ARjM7Liq6AHg6xpDi9iJwlpk1Rv83FzCFL9oXuA9YEY2vAH5YjpUqQVSHc4ArCfeWn4iGN8cdlFSNTwC3m9nvgFOAf4g5nthER1J3A48B/03Yhk2pu6rN7A5gNXCcmXWY2YeB64E3mNlzhEdZ15flu3QntYiIFKMjCBERKUoJQkREilKCEBGRopQgRESkKCUIEREpSglCpAqY2Xl6cq1UGyUIEREpSglC5BCY2RVm9tvoZsZ/j95bsc/M/jl6R8GDZjY3qnuKmT1iZr8zs+/3P6PfzF5uZr80syfN7DEzOyZa/fSC9z7cHt0pLBIbJQiREpnZCcB7gHPc/RQgD7wfmAa0ufsrgYeBv4sWuQX4rLufRHjXb3/57cAN7n4y4XOE+p/CeSrwF8Ay4GWEd9iLxCYVdwAiNeQC4NXAmmjnvoHwoWgB8N2ozm3AvdF7HGa6+8NR+Urge2bWBCxy9+8DuHsPQLS+37p7RzT9BNAC/KbyP5ZIcUoQIqUzYKW7f25IodnfDqt3uM+v6S0Yz6P/T4mZTjGJlO5B4E/M7EgYeA/wUsL/oz+J6rwP+I27dwK7zey1UfmVwMPRGwM7zOzt0TrqzKxxQn8KkRJpD0WkRO7+tJn9DfALM0sAWeBqwpf4nBHN2054nQLCxy5/I0oALwAfjMqvBP7dzK6L1vGuCfwxREqmp7mKjJOZ7XP36XHHIVJuOsUkIiJF6QhCRESK0hGEiIgUpQQhIiJFKUGIiEhRShAiIlKUEoSIiBSlBCEiIkX9//3rV4BzN1PYAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DCJr9gs85N_k",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "524f05a4-8f7b-4871-cde7-d79850f24400"
      },
      "source": [
        "predictions, labels = predict(test_dl, clf)\n",
        "\n",
        "accuracy = accuracy_score(predictions, labels)\n",
        "ACCURACY_STATS.append((6, \"text_attack\", accuracy))\n",
        "\n",
        "print(\"accuracy: {:.3f}\".format(accuracy))\n",
        "print(\"confusion matrix\")\n",
        "print(confusion_matrix(predictions, labels))"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy: 0.975\n",
            "confusion matrix\n",
            "[[478   4]\n",
            " [ 10  65]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "otEPj_QO5bTP"
      },
      "source": [
        "### Results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 339
        },
        "id": "Idiihxkg5aAp",
        "outputId": "d1a0ba06-c960-4af3-9c72-3413dfc91a39"
      },
      "source": [
        "accuracy_stats = sorted(ACCURACY_STATS, key=operator.itemgetter(0))\n",
        "xs = [x[1] for x in accuracy_stats]\n",
        "ys = [x[2] for x in accuracy_stats]\n",
        "\n",
        "plt.bar(np.arange(len(xs)), ys)\n",
        "plt.axhline(y=ys[0], color=\"r\", linestyle=\"--\", linewidth=1.5)\n",
        "plt.ylabel(\"accuracy\")\n",
        "plt.xticks(np.arange(len(xs)), xs, rotation=\"90\")\n",
        "plt.ylim([0.95, 1.0])\n",
        "\n",
        "_ = plt.show()"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAFCCAYAAADrMEr0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5wcVZ3+8c9DEgiXIAGyLksQAREYNQYY7igB1OUiYAC5KCzgb4mCKF6igi7CRm4iqMAiS1BYkFVQRIyIGyAQQC4uQxKCgMFsAElADZdwvyV+f3+cauiZqZnpTHdPdXWe9+vVr3TV6Z58K5mZp+tUnXMUEZiZmfW0UtEFmJlZa3JAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWa6mBYSkSyT9TdIf+miXpPMkzZc0V9JWVW1HSPpT9jiiWTWamVnfmnkG8V/AHv207wlsmj0mARcCSFobOBnYDtgWOFnS6CbWaWZmOZoWEBFxG/BMPy/ZD7g8kruBtSStB/wzcGNEPBMRzwI30n/QmJlZEwwv8O9eH3i8anthtq+v/b1ImkQ6+2D11VffevPNN29OpWZmberee+99KiLG5LUVGRB1i4ipwFSAzs7O6OrqKrgiM7NykfRYX21F3sW0CNigantstq+v/WZmNoSKDIhpwL9kdzNtDzwXEU8C04GPSBqdXZz+SLbPzMyGUNO6mCT9FJgArCtpIenOpBEAEfGfwPXAXsB84GXgqKztGUnfAu7JvtSUiOjvYreZmTVB0wIiIg4doD2Az/bRdglwSTPqMjOz2ngktZmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVqakBI2kPSPEnzJZ2Q076hpBmS5kqaKWlsVdu3Jf0hexzczDrNzKy3pgWEpGHABcCeQAdwqKSOHi87G7g8IsYBU4AzsvfuDWwFjAe2AyZLWrNZtZqZWW/NPIPYFpgfEQsi4nXgSmC/Hq/pAG7Ont9S1d4B3BYRSyPiJWAusEcTazUzsx6aGRDrA49XbS/M9lW7D9g/ez4RGCVpnWz/HpJWk7QusCuwQc+/QNIkSV2SuhYvXtzwAzAzW5EVfZF6MrCLpNnALsAiYFlE3ABcD9wJ/BS4C1jW880RMTUiOiOic8yYMUNYtplZ+2tmQCyi+6f+sdm+N0XEExGxf0RsCXwj27ck+/O0iBgfER8GBDzcxFrNzKyHZgbEPcCmkjaStDJwCDCt+gWS1pVUqeFE4JJs/7CsqwlJ44BxwA1NrNXMzHoY3qwvHBFLJR0HTAeGAZdExAOSpgBdETENmACcISmA24DPZm8fAdwuCeB54LCIWNqsWs3MrDdFRNE1NERnZ2d0dXUVXYaZWalIujciOvPair5IbWZmLcoBYWZmuRwQZmaWywFhZma5HBBmZpbLAWFmZrkcEGZmlssBYWZmuRwQZmaWywFhZma5HBBmZpbLAWFmZrkcEGZmlssBYWZmuRwQZmaWywFhZma5HBBmZpbLAWFmZrkcEGZmlssBYWZmuRwQZmaWywFhZma5HBBmZpbLAWFmZrkcEGZmlssBYWZmuRwQZmaWywFhZma5HBBmZpbLAWFmZrkcEGZmlssBYWZmuRwQZmaWywFhZma5mhoQkvaQNE/SfEkn5LRvKGmGpLmSZkoaW9V2lqQHJD0k6TxJamatZmbWXdMCQtIw4AJgT6ADOFRSR4+XnQ1cHhHjgCnAGdl7dwR2AsYB7wW2AXZpVq1mZtZbM88gtgXmR8SCiHgduBLYr8drOoCbs+e3VLUHMBJYGVgFGAH8tYm1mplZD80MiPWBx6u2F2b7qt0H7J89nwiMkrRORNxFCowns8f0iHio518gaZKkLkldixcvbvgBmJmtyIq+SD0Z2EXSbFIX0iJgmaR3AVsAY0mhspukD/R8c0RMjYjOiOgcM2bMUNZtZtb2agoISddI2lvS8gTKImCDqu2x2b43RcQTEbF/RGwJfCPbt4R0NnF3RLwYES8CvwV2WI6/28zM6lTrL/wfAJ8A/iTpTEmb1fCee4BNJW0kaWXgEGBa9QskrVsVOicCl2TP/0w6sxguaQTp7KJXF5OZmTVPTQERETdFxCeBrYBHgZsk3SnpqOwXeN57lgLHAdNJv9x/FhEPSJoiad/sZROAeZIeBt4OnJbtvxr4P+B+0nWK+yLi14M5QDMzGxxFRG0vlNYBDgMOB54A/hvYGXhfRExoVoG16uzsjK6urqLLMDMrFUn3RkRnXtvwGr/AL4HNgB8D+0TEk1nTVZL8W9nMrA3VFBDAeRFxS15DX8ljZv175wm/KbqEmjx65t41v7Ydj2lFVutF6g5Ja1U2JI2WdGyTajIzsxZQa0Acnd1+CkBEPAsc3ZySzMysFdQaEMOqJ8vL5llauTklmZlZK6j1GsT/kC5IX5RtfzrbZ2ZmbarWgPgaKRSOybZvBH7YlIrMzKwl1BQQEfF34MLsYWZmK4Bax0FsSlqroYM0DTcAEbFxk+oyM7OC1drFdClwMvA9YFfgKIqfCdb64fvRzaxetf6SXzUiZpCm5ngsIk4B/JNtZtbGaj2DeC2bdfVPko4jTdu9RvPKMjOzotV6BnE8sBrweWBr0qR9RzSrKDMzK96AZxDZoLiDI2Iy8CLp+oOZmbW5Ac8gImIZaVpvMzNbgdR6DWK2pGnAz4GXKjsj4pqmVGVmZoWrNSBGAk8Du1XtC6B1AmLePJgwofu+gw6CY4+Fl1+Gvfbq/Z4jj0yPp56CAw/s3X7MMXDwwfD443D44b3bv/xl2Gef9Hd/+tO92//t3+BDH4I5c+ALX+jdfvrpsOOOcOed8PWv927//vdh/Hi46SY49dTe7RddBJttBr/+NZxzTremKxc8zRc/+mWeXHMMH33oNg6bfX3vw/vYiTy72ts48P6bOPD+m3q1H/nxU3h1xEgOm/UbPvrH23u1H/KJMwE4+vfXsPv//W+3tleHr8KRB/07AJ+746fs9Nh93dqfXXVNjpmYHfOJJ8Jdd3X/4mPHwhVXpOdf+EL6N6z27nfD1Knp+aRJ8PDD3dvHj0//fgCHHQYLF3Zv32EHOOOM9PyAA+Dpp7u37747nHRSer7nnvDKK93bP/pRmDw5Pe/5fQe1fe8xhtEvP8eF157Rq/mKLffiui0+yHrPL+Z7153Tq/3ibScy413bsfHTCzl9+n/0aj9/x0O4453j6fjrAr45Y2qv9rM+eASzxm7BVgsf4qu3Xdarfcruk3jw7Ruz06NzYMJ3etffx/felQvSv2Orf+9Rub16Rf3e6+/3XpVaR1K3/XWHuxc83WvfFT+ZxXWz10g/pDntF192DzPuWCn9kOa0n//D33PHTa+lH9Kc9rN+cCezpj2bfkhz2qecezsPvn0ROz06h89l7dtvvM5gDs/MBuGS3z1Cx9+6/2wuWPJnvp6NMzr9f//Mxs90b3/wxUeYkrV/b/Yi1nuhe/us1+ZzVtZ+4R/+wuhXnu/WfseyeZz/Umr/r3mLGbn0tW7tM37zEBc/ldqvbPLvhZqWHJV0KemMoZuI+FQzihqMepccbbeBZe12PO2oHf+P2u2Y2u148tS95ChwXdXzkcBE0rrUZmbWpmrtYvpF9baknwK/a0pFZmbWEgY7n9KmwD80shAzM2sttc7m+gLdr0H8hbRGhJmZtalau5hGNbsQMzNrLTV1MUmaKOltVdtrSfpY88oyM7Oi1XoN4uSIeK6yERFLSOtDmJlZm6o1IPJeV+stsmZmVkK1BkSXpO9K2iR7fBe4t5mFmZlZsWoNiM8BrwNXAVcCrwKfbVZRZmZWvFrvYnoJOKHJtZiZWQup9S6mGyWtVbU9WtL05pVlZmZFq7WLad3sziUAIuJZPJLazKyt1RoQf5f0jsqGpHeSM7urmZm1j1pvVf0G8DtJtwICPgBMalpVZmZWuFovUv+PpE5SKMwGrgVe6f9dZmZWZrVepP5XYAbwZWAy8GPglBret4ekeZLmS+p1F5SkDSXNkDRX0kxJY7P9u0qaU/V41VN7mJkNrVqvQRwPbAM8FhG7AlsCS/p7g6RhwAXAnkAHcKikjh4vOxu4PCLGAVOAMwAi4paIGB8R40nrYL8M3FBjrWZm1gC1BsSrEfEqgKRVIuKPwGYDvGdbYH5ELIiI10kD7Pbr8ZoO4Obs+S057QAHAr+NiJdrrNXMzBqg1oBYmI2DuBa4UdKvgMcGeM/6wOPVXyPbV+0+YP/s+URglKSeq28fAvw07y+QNElSl6SuxYsX13AYZmZWq5oCIiImRsSSiDgFOAn4EdCIawKTgV0kzQZ2ARYByyqNktYD3gfkDsqLiKkR0RkRnWPGjGlAOWZmVrHcM7JGxK01vnQRsEHV9thsX/XXeoLsDELSGsAB1QPygIOAX0bEG8tbp5mZ1Wewa1LX4h5gU0kbSVqZ1FU0rfoFktaVVKnhROCSHl/jUProXjIzs+ZqWkBExFLgOFL30EPAzyLiAUlTJO2bvWwCME/Sw8DbgdMq789Ga28A1HrGYmZmDdTURX8i4nrg+h77vln1/Grg6j7e+yi9L2qbmdkQaWYXk5mZlZgDwszMcjkgzMwslwPCzMxyNfUitVkjvfOE3xRdQk0ePXPvokswawifQZiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVkuB4SZmeVyQJiZWS4HhJmZ5XJAmJlZLgeEmZnlckCYmVmupgaEpD0kzZM0X9IJOe0bSpohaa6kmZLGVrW9Q9INkh6S9KCkdzazVjMz665pASFpGHABsCfQARwqqaPHy84GLo+IccAU4IyqtsuB70TEFsC2wN+aVauZmfXWzDOIbYH5EbEgIl4HrgT26/GaDuDm7PktlfYsSIZHxI0AEfFiRLzcxFrNzKyHZgbE+sDjVdsLs33V7gP2z55PBEZJWgd4N7BE0jWSZkv6TnZG0o2kSZK6JHUtXry4CYdgZrbiKvoi9WRgF0mzgV2ARcAyYDjwgax9G2Bj4Mieb46IqRHRGRGdY8aMGbKizcxWBM0MiEXABlXbY7N9b4qIJyJi/4jYEvhGtm8J6WxjTtY9tRS4FtiqibWamVkPzQyIe4BNJW0kaWXgEGBa9QskrSupUsOJwCVV711LUuW0YDfgwSbWamZmPTQtILJP/scB04GHgJ9FxAOSpkjaN3vZBGCepIeBtwOnZe9dRupemiHpfkDAxc2q1czMehvezC8eEdcD1/fY982q51cDV/fx3huBcc2sz8zM+lb0RWozM2tRDggzM8vlgDAzs1wOCDMzy+WAMDOzXA4IMzPL5YAwM7NcDggzM8vlgDAzs1wOCDMzy+WAMDOzXA4IMzPL5YAwM7NcDggzM8vlgDAzs1wOCDMzy+WAMDOzXA4IMzPL5YAwM7NcDggzM8vlgDAzs1wOCDMzy+WAMDOzXA4IMzPL5YAwM7NcDggzM8vlgDAzs1wOCDMzy+WAMDOzXA4IMzPL5YAwM7NcDggzM8vlgDAzs1wOCDMzy+WAMDOzXE0NCEl7SJonab6kE3LaN5Q0Q9JcSTMlja1qWyZpTvaY1sw6zcyst+HN+sKShgEXAB8GFgL3SJoWEQ9Wvexs4PKIuEzSbsAZwOFZ2ysRMb5Z9ZmZWf+aeQaxLTA/IhZExOvAlcB+PV7TAdycPb8lp93MzAqiiGjOF5YOBPaIiH/Ntg8HtouI46pe8xPg9xFxrqT9gV8A60bE05KWAnOApcCZEXFtzt8xCZiUbW4GzGvKwQzeusBTRRfRQO12PNB+x9RuxwPtd0ytdjwbRsSYvIamdTHVaDLwH5KOBG4DFgHLsrYNI2KRpI2BmyXdHxH/V/3miJgKTB3KgpeHpK6I6Cy6jkZpt+OB9jumdjseaL9jKtPxNDMgFgEbVG2Pzfa9KSKeAPYHkLQGcEBELMnaFmV/LpA0E9gS6BYQZmbWPM28BnEPsKmkjSStDBwCdLsbSdK6kio1nAhcku0fLWmVymuAnYDqi9tmZtZkTQuIiFgKHAdMBx4CfhYRD0iaImnf7GUTgHmSHgbeDpyW7d8C6JJ0H+ni9Zk97n4qi5bt/hqkdjseaL9jarfjgfY7ptIcT9MuUpuZWbl5JLWZmeVyQJiZWS4HhJmZ5XJAmJlZLgdEE0jaWdJR2fMxkjYquqbBkrRJ1S3HEyR9XtJaRddVD0lb5Tw2kVT0wNEVmqTNq56v0qNt+6GvqHEk7Zmz7zNF1LI8HBANJulk4GukcR0AI4Ariquobr8Alkl6F+n2vA2AnxRbUt1+ANxNOp6LgbuAn5Nuuf5IkYUNhqQXJD3f4/G4pF9mMxGURfX31V092n4wlIU0wUnZhKQASPoqJZh7zgHReBOBfYGX4M3R4qMKrag+f8/GtEwEzo+IrwDrFVxTvZ4AtoyIzojYmjRKfwFp5uGzCq1scL4PfAVYnzRjwWTSL9sryQafloT6eJ63XTb7AqdL+oCk04DtcECskF6PNLgkACStXnA99XpD0qHAEcB12b4RBdbTCO+OiAcqG9kgzM0jYkGBNdVj34i4KCJeiIjnsznK/jkirgJGF13ccog+nudtl0pEPEUKiQuAfwIOzGa5bmnuc228n0m6CFhL0tHAp0jdGGV1FPAZ4LSIeCS7nvLjgmuq1wOSLiR9wgY4GHgw6/d+o7iyBu1lSQcBV2fbBwKvZs/L9It1rKTzSGcLledk2+sXV9bgSXqB9H+g7M+VgY2BAyVFRKxZZH0D8UjqJpD0YeAjpG+K6RFxY8ElDZqk4yPi3IH2lYmkVYFjgZ2zXXeQ+rhfBVaLiBeLqm0wsusM5wI7kH4J3Q18kTQ55tYR8bsCy6uZpCP6a4+Iy4aqFkscENYvSbMiYqse+2ZHxJZF1WTtSdLpEfH1outoBkkTgZsj4rlsey1gQt46N63EAdFg2cJH3wb+gXQGIaDlTyV7yq47fIL0Kfv2qqZRpAvXuxdSWANI2gk4BdiQqm7WiCjTHT9vkjQGOBp4J92P51NF1TQYeR9G2oWkOT2XUC7DBy1fg2i8s4B9IuKhogup053Ak6TVr86p2v8CMLeQihrnR6QumHt5a4GqMvsVKcRvotzHM0zSaPq4Yykinhniehop74aglv/96zOIBpN0R0TsVHQd1jdJv4+I7Yquo1HyPp2WkaTXSNdN8gIiynqGByDpEmAJ6S4mgM8Ca0fEkYUVVQMHRINJOhf4R+Ba4LXK/oi4prCi6tAuXWbVJJ0JDAOuofv/0azCiqqDpFOBOyPi+qJrqUcZulwGK7vd/STgQ9muG4FTI+Kl4qoamAOiwSRdmrM7ytYfXCFpPu3RZfYmSbfk7I6I2C1nf8vLbqVcnRR2b1DSEG/ngCgrB4T1y11mNlQkHRkR/5U9XwOgbLcc9yW7keCrwHuAkZX9rf6hxAHRIJK+GhFnSTqfnMFJEfH5AsoatKxrCWAX2qTLTNJhEXGFpC/ltUfEd4e6pnpI2jwi/igp986fMnaZSToWOIF0RiTSTRHfjohSz8Uk6QbgKtI0KJ8hzUywOCK+VmhhA2j5q+glUumC6Sq0isbZp+r5y6SBfxVB6r8vm8q0J2WeG6val0m3t56T0xZAS3867UnSN4CdSOMDFmT7NgbOlbR2RJxaaIH1WScifpQNMr0VuFXSPUUXNRCfQZhZS5A0D3h/RLzaY/+qwH0R8e5iKqufpLsjYntJ04HzSBNGXh0RmxRcWr98BtEgkn5NP/PeRMS+Q1hOw1TNh1PtOaArIn411PXUo49jeVOJuwFzlbAbMHqGQ7bzFUl/L6KgBjpV0ttIZ33nA2sCXyi2pIE5IBrn7KILaJKRwOak9RIADgAeAd4vadeIaPlv8ir3Fl1Ag+3TT1sZuwEXSdo9ImZU78zWUXiyoJoa5dlsmo3ngF3hzRH9Lc1dTE2QnRK/IyLmFV1LvSTdDewUEcuy7eGkUbs7A/dHREeR9dVD0pqkT60vFF2LgaT3kEaF/463wryTdF1iv+op2sumjznNWn5qEZ9BNJikfUhnEysDG0kaD0wpaxcTaT2BNUiffCBd6F07IpZlI19LR1IncCnpYrUkLQE+FRGlPMOQtA5wMim0g/QLdkpEPF1oYcspIh6Q9F7SHGDvyXbfBnw6r+upDCTtAOwIjOlx99yapMGaLc0B0XinANsCMwEiYo5KvCY1aW6pOZJmkm47/CBpZazVSXP/lNElwLERcTukNcRJgTGu0KoG70rSL9IDsu1Pkm6p/FCf72hRWRCUaRW8gaxM+oA1nO53zz1PWrejpbmLqcGq7lZ4c1SopLkRUdZfPkhajxR6APdky6iWVt6I3TKc7vdF0h8i4r099t0fEe8rqqbBqFpcp1cTJRwZXq0yTqrHvo9HxM/7ek8r8JKjjfeApE+QZqbcNBs4d2fRRS0vSZtnf25FWoP68ezxj30NzCqRWyVdJGmCpF0k/QCYKWmrkh7bDZIOkbRS9jgImF50UYMwA3gQOBV4X0SsmT1GlTkcMofk7DtxyKtYTj6DaDBJqwHfoGpFOeBbZetDlTQ1Iia127xF0OdcTBWlO7aquZgqt4KuBFQmgSvVJ+/sVtD9Sb9QR5K6yq4s61TfkvYE9gIOIh1LxZpAR0Rsm/vGFuGAaCJJw4DVI+L5omsxKxNJK5FC4jzg9LJNg1Ih6f3AeGAK8M2qpheAWyLi2UIKq5EDosEk/YQ018oy4B7SJ4VzI+I7hRY2SNkZ0ZdIt+1OkrQpsFlEXFdwaYOWLff4L/Rega1UA+WqSRpH7+Mp2zgIJO0IHAp8gHQ31lWVmwnKTNKIiHij6DqWlwOiwSqLt0j6JLAVaeKxe8t6kVrSVaR70v8lIt6bBcadZV6gRtKdwN3A/bzVLUNEXFZYUXXIFqMZBzzAW8dTuinmJT0GPEu6K+tmYGl1exknH6zIPlidAXTQfTbXll4Eybe5Nt4ISSOAjwH/ERFvSCpzCm8SEQcrrVFNRLwsKXdJyBIZGRG5M7qW1PZlHrBY5RHSXUz/nD16/tyU6tpQD5eSxqp8jzSS+ihKcJNQyxdYQhcBj5IuGt4maUPSPc9l9Xo2MjwAJG1C1bTfJfVjSUdLWk/S2pVH0UXV4S5J7RAQXwU+ERG7RsSuwGXAi8AfKMGYgQGsmk0hooh4LCJOAfYuuKYBuYtpCEgaHhFLB35l65H0EdJdWR3ADaRpD46MiJlF1lUPSZ8FTiOtEVz5AYhWP93vi6RdgGnAX0jhXRk3UKpuTUmzgA9FxDOSPkjqavoc6SLvFhFR2pDIujV3Bq4mdZ8tAs6MiM0KLWwADogmkLQ3vVeOmlJcRfXJpnLYnvSL5+6IeKrgkuoiaQGwbdmPo0JpWdgv0fuaymOFFTUIku6LiPdnzy8gLahzSrY9p+TXvbYhrRmzFvAt0s0r34mIuwstbAC+BtFgkv4TWI3Uz/hD0qnx/xZaVB0kXQHcCtweEX8sup4GmU9aBKldLI6IaUUX0QDDqs62dwcmVbWV+ndVRFQWB3qRdP2hG0nnR8TnhraqgZX6H71F7RgR47LpNf5d0jnAb4suqg4/It1yeH52/WE2cFtEnFtsWXV5iTS/1C10X0a1rLe5zs5ur/415V4W9qekUe5PAa+QZg1G0rt4a7LIdtWSU387IBrvlezPlyX9E/A0aaqKUoqIWyTdBmxDOiv6DKn7rMwBcW32aBerkoKh1MvCRsRpkmaQfl5uiLf6v1ciXYuwIeaAaLzrsoFYZ/HWnPY/LLCeumQ/sKsDd5E+0W0TEX8rtqr6lHW8Q18ioleXRVnl9clHxMNF1GIOiGY4GziG1C1T+aV6YaEV1WcusDXwXtJp/hJJd0XEK/2/rXWVddBSXySNBP4fvW+MKNVAuRVcS44t8jiIxruM9IN6Hmnt2Q7g8kIrqkNEfDEiPkiaQO1p0oCfJcVWVbdLSaG9lNRtdjlwRaEV1efHwD+SBpfdCowlzfVjLULSxwfY15Jdtr7NtcEkPdhzVGvevrKQdBzpbGhr0gDA20l3NN1cZF31kHRvRGxdvWZCZV/RtQ1GZX2Lyroj2Uj+2yNi+6JrsyRvvZEyrEHiLqbGmyVp+0pfqqTtgK6Ca6rHSOC7pPmkeg32kzS61WekzPFaNlvon7IAXERa9ausKpPALcmW7PwL8A8F1mOZqum+15d0XlXTmvSYa6oVOSAaRNL9pDtHRgB3Svpztr0hUNrxAxFx9gAvmUGalLBMjieNVfk8adDSbsARhVZUn6mSRgP/RhpRvQbdp5a24jxB+oC4L2/dtAKpC/CLhVS0HNzF1CDZnEt9Ktuo1lrlLd9ZJtmZxBpes8OaSdLYiFjYY99mETGvqJpq4YvUDZJNwNXno+j6mqh0nzAk/UTSmpJWJ00E96CkrxRd12BJOj47Hkn6oaRZ2Rxa1jpmZEvBAiDpy8AvC6ynJg4IWxF1ZGcMHyONct8IOLzYkuryqex4PgKsQzqWM4styXqYABwu6efZwNN3Ay293Cg4IKx+LXn/9gCq1+yYlq30VbozoSqV/4O9gMsj4gHK+f/StiLiSeB/gB1IK/9dFhEvFlpUDRwQ1i9Jm0haJXs+QdLns5HiFbsXVFo92m3Njnsl3UAKiOmSRlE1q6sVT9JNwHakAad7A9+XNNANIIXzRWrrl6Q5QCfpU8/1wK+A90TEXkXW1UjZCnnDKrfxSjqiTNNxZBfaxwMLImJJNj37+hExN2t/T3ZWYQWR9LGIuLZqexjw9Yj4VoFlDcgBYf2qDObJLuK+GhHnl/3OpYGUYQDT8mi34ykrSTsDm0bEpZLWBdwTwMAAAAS2SURBVEZFxCNF19UfdzHZQN7I1qM+Argu2zeiwHqGQrv137fb8ZSOpJOBrwEnZrtWpgTTuzggbCBHkS6snRYRj0jaiDT3Tztrt9PqdjueMppIGiz3EkBEPAGMKrSiGngktfUrIh4kjTgmG607KiK+XWxVTedP3NZor0dESAqAbAxOy/MZhPVL0sxsENbawCzgYknfLbquwZK0UvWApT7cMSTFNEA2OG6DAV72+pAUY/35maSLgLUkHQ3cBFxccE0DckDYQN6WDcLan3SP/XbAhwquadAi4u/AVwd4zXFDVE7dslXXrh/gNZ7VtXhjgKuBXwCbkebKGltoRTVwQNhAhktaDziIty5Sl91NkiZL2kDS2pVH0UXVYZakbYouwvr14Yi4MSK+EhGTI+JGYM+iixqIr0HYQKYA04HfRcQ9kjYG/lRwTfU6OPvzs1X7AijlinKkAViflPQY6SKoSCcX44otyyQdAxwLbCxpblXTKErQlelxEGYl19dMwm0+SWQpSHobMJq0xO0JVU0vRMQzxVRVOweE9asd1zuWtBrwJeAdETEpW6N6s4gobRdaj0FYY0hTmLf0ICxrfb4GYQNpx/WOLyXd2bNjtr0IOLW4cuqTMwhrBCUYhGWtzwFhA3lXRJwEvJTNT7Q3qc+7zDaJiLPIluqMiJcp99iHUg7CstbngLCB9Fzv+G2Uf73j1yWtSjbCWNImwGvFllSX17PbXUs1CMtan+9isoFU1js+ifZZ7/hk0tz8G0j6b2An4MhCK6pPz0FYn6IEg7Cs9fkita2Qsimxtyd1Ld0dEU8VXFJdJH2YtKKcgOnZffZmdXFAWC5JX+qvPSJKN92GpH6nvI6IWUNVSyNl/1dXRcSiomux9uIuJutL5SJn0PsCblk/VZyT/TmStAjSfaRjGwd0kWatLaNRwA2SngGuAn4eEX8tuCZrAz6DsH5Jugw4PiKWZNujgXNKPg7iGuDkiLg/234vcEpEHFhsZfWRNI40SvwAYGFElHbOLGsNvovJBjKuEg4AEfEsUPbV5DarhANARPwB2KLAehrlb8BfgKcp/51m1gIcEDaQlbKzBgCySe3K3jU5V9IPJU3IHhcDcwd8V4uSdKykmcAMYB3gaM/DZI1Q9h90a75zgLsk/Tzb/jhwWoH1NMJRwDHA8dn2bcCFxZVTtw1Ix/JB0vWhdl8S1oaIr0HYgCR1ALtlmzdnq8xZi5B0PPCvwDWki+4TgakRcX6hhVnpOSBshSNpJ+AUYEOqzqIjopTTfWfTSO8QES9l26sDd7mbyerlLiZbEf0I+CJwL7Cs4FoaQXQ/jmWUe24paxEOCFsRPRcRvy26iAa6FPi9pF9m2x8jhaBZXdzFZCscSWcCw0h99m9O0lfWkdTw5ijxnbPN2yNidpH1WHtwQNgKR9It2dPKN39lic7d+niL2QrJXUy2IpqZs8+flMx6cEDYiujFqucjgY8CDxVUi1nLcheTrfAkrUKaIntC0bWYtRJPtWEGq5HW2jazKu5ishWOpPt565rDMGAMMKW4isxak7uYbIUjacOqzaXAXyNiaVH1mLUqB4SZmeXyNQgzM8vlgDAzs1wOCDMzy+WAMDOzXP8fcp8Rdy8/rXcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "-QQpYAT75f2y",
        "outputId": "90ce6e13-ad49-4773-993a-fa64708568a5"
      },
      "source": [
        "result_df = pd.DataFrame(ACCURACY_STATS, columns=[\"expt_num\", \"expt_name\", \"accuracy\"])\n",
        "result_df.head(len(result_df))"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>expt_num</th>\n",
              "      <th>expt_name</th>\n",
              "      <th>accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>baseline</td>\n",
              "      <td>0.974865</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>class_weights</td>\n",
              "      <td>0.976661</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>under_sampling</td>\n",
              "      <td>0.967684</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>over_sampling</td>\n",
              "      <td>0.978456</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>SMOTE</td>\n",
              "      <td>0.978456</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>text_attack</td>\n",
              "      <td>0.974865</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   expt_num       expt_name  accuracy\n",
              "0         1        baseline  0.974865\n",
              "1         2   class_weights  0.976661\n",
              "2         3  under_sampling  0.967684\n",
              "3         4   over_sampling  0.978456\n",
              "4         5           SMOTE  0.978456\n",
              "5         6     text_attack  0.974865"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bXaw2dGSPBVT"
      },
      "source": [
        ""
      ],
      "execution_count": 54,
      "outputs": []
    }
  ]
}