financial-sentiment-classifier.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dfavenfre/financial-sentiment-classifier/blob/main/financial-sentiment-classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pHH4AzcPQ2yn"
      },
      "source": [
        "# ⚛ Packages and Important Libraries ⚛\n",
        "The imported libraries contain various modules that are helpful for model building in tensorflow and for data, financial_phrasebank dataset from Huggingface is downloaded. Important resources can be found down below. \n",
        "\n",
        "*   https://huggingface.co/datasets/financial_phrasebank (Financial PhraseBank Dataset)\n",
        "*   https://medium.com/prosus-ai-tech-blog/finbert-financial-sentiment-analysis-with-bert-b277a3607101 (FinBERT Article)\n",
        "*   https://medium.com/mlearning-ai/optimizing-deep-learning-models-with-pruning-a-practical-guide-163e990c02af (Model Pruning in TF)\n",
        "\n",
        "Complete list of libraries and modules are; \n",
        "*  tensorflow.keras.layers,  tensorflow.keras.models, tensorflow.keras.utils, sklearn.metrics, sklearn.model_selection\n",
        "*  nltk, string, re, random, io, pandas, numpy\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Y0VDqKYH5qC"
      },
      "source": [
        "paketleri yeniden düzenle, hatalar var"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8CCYxagt_ACp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a2d1ffd1-8b5c-4511-978b-053bcea4be0a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.12.0-py3-none-any.whl (474 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m474.6/474.6 kB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.22.4)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
            "Collecting dill<0.3.7,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.65.0)\n",
            "Collecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.4.0)\n",
            "Collecting aiohttp (from datasets)\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m26.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub<1.0.0,>=0.11.0 (from datasets)\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.1)\n",
            "Collecting responses<0.19 (from datasets)\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets)\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->datasets)\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting yarl<2.0,>=1.0 (from aiohttp->datasets)\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m35.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp->datasets)\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp->datasets)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.12.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: xxhash, multidict, frozenlist, dill, async-timeout, yarl, responses, multiprocess, huggingface-hub, aiosignal, aiohttp, datasets\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.12.0 dill-0.3.6 frozenlist-1.3.3 huggingface-hub-0.14.1 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 xxhash-3.2.0 yarl-1.9.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: imbalanced-learn in /usr/local/lib/python3.10/dist-packages (0.10.1)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.10/dist-packages (from imbalanced-learn) (1.22.4)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from imbalanced-learn) (1.10.1)\n",
            "Requirement already satisfied: scikit-learn>=1.0.2 in /usr/local/lib/python3.10/dist-packages (from imbalanced-learn) (1.2.2)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from imbalanced-learn) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from imbalanced-learn) (3.1.0)\n",
            "--2023-05-27 15:59:16--  https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.111.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10246 (10K) [text/plain]\n",
            "Saving to: ‘helper_functions.py’\n",
            "\n",
            "helper_functions.py 100%[===================>]  10.01K  --.-KB/s    in 0s      \n",
            "\n",
            "2023-05-27 15:59:17 (73.0 MB/s) - ‘helper_functions.py’ saved [10246/10246]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# model packages\n",
        "!pip install datasets\n",
        "!pip install imbalanced-learn\n",
        "!wget https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py\n",
        "from helper_functions import unzip_data, create_tensorboard_callback, plot_loss_curves, compare_historys\n",
        "from imblearn.over_sampling import SMOTE\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers \n",
        "from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n",
        "from tensorflow.python.keras.models import Sequential\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.layers import Embedding, Attention\n",
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
        "from keras.utils import pad_sequences\n",
        "from helper_functions import create_tensorboard_callback\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras import layers, models, utils\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "# data preprocessing packages\n",
        "import re\n",
        "import string\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "# uploading data\n",
        "import random\n",
        "import re\n",
        "import io\n",
        "import pandas as pd\n",
        "from google.colab import files\n",
        "from datasets import load_dataset\n",
        "\n",
        "# visualization\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eZLPon7DZ4rk"
      },
      "outputs": [],
      "source": [
        "# create a directory to save TensorBoard Logs\n",
        "SAVE_DIR = \"model_logs\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pA9eQ_wCSxVC"
      },
      "source": [
        "# Useful Functions ⛽\n",
        "\n",
        "\n",
        "\n",
        "*   **text_cleaning(text)** : Clean and remove unnecessary string elements from each sentence, and return it back to the original sentence data\n",
        "*   **calculate_results(y_true, y_pred)** :  Calculate various classification scores, including Accuracy, F1_score, Recall, and Precision. \n",
        "\n",
        "An addition to weight parameter for \"precision_recall_fscore_support(y_true, y_pred, average=\"weighted\")\", since the target variables (labels) might be imbalanced, setting average parameter to \"weighted\" solves the imbalance issue for us.   \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eVT-apGEz8mr"
      },
      "outputs": [],
      "source": [
        "def text_cleaning(text):\n",
        "\n",
        "  \"\"\" Clean and remove unnecessary string elements from each sentence, and return it back to the original sentence data.\"\"\"\n",
        "\n",
        "  text = re.sub(r'@[A-Za-z0-9]+', '', text)     # removing @mentions\n",
        "  text = re.sub(r'@[A-Za-zA-Z0-9]+', '', text)  # removing @mentions \n",
        "  text = re.sub(r'@[A-Za-z]+', '', text)        # removing @mentions\n",
        "  text = re.sub(r'@[-)]+', '', text)            # removing @mentions\n",
        "  text = re.sub(r'#', '', text )                # removing '#' sign\n",
        "  text = re.sub(r'RT[\\s]+', '', text)           # removing RT\n",
        "  text = re.sub(r'https?\\/\\/\\S+', '', text)     # removing the hyper link\n",
        "  text = re.sub(r'&[a-z;]+', '', text)          # removing '&gt;'\n",
        "\n",
        "  return text\n",
        "\n",
        "def calculate_results(y_true, y_pred):\n",
        "\n",
        "  \"\"\" Calculate various classification scores, including Accuracy, F1_score, Recall, and Precision.\"\"\"\n",
        "\n",
        "  model_accuracy = accuracy_score(y_true, y_pred) * 100\n",
        "  model_precision, model_recall, model_f1, _ = precision_recall_fscore_support(y_true, y_pred, average=\"weighted\")\n",
        "  model_results = {\"accuracy\": round(model_accuracy,3),\n",
        "                    \"precision\": round(model_precision,3),\n",
        "                    \"recall\": round(model_recall,3),\n",
        "                    \"f1\": round(model_f1,3)}\n",
        "  return model_results\n",
        "\n",
        "def pre_process(data):\n",
        "\n",
        "  \"\"\" Alternative to text_cleaning(), does the same job\"\"\"\n",
        "\n",
        "  # Convert all text to lowercase\n",
        "  data = data.str.lower()\n",
        "\n",
        "  # Remove numbers\n",
        "  data = data.apply(lambda x: re.sub(r'\\d+', '', x))\n",
        "\n",
        "  # Remove punctuation\n",
        "  punctuation = string.punctuation.replace(\"'\", \"\")\n",
        "  data = data.apply(lambda x: x.translate(str.maketrans('', '', punctuation)))\n",
        "\n",
        "  # Remove $ \n",
        "  punctuation = string.punctuation.replace(\"$\", \"\")\n",
        "  data = data.apply(lambda x: x.translate(str.maketrans('', '', punctuation)))\n",
        "\n",
        "  # Remove stop words\n",
        "  nltk.download('stopwords')\n",
        "  stop_words = set(stopwords.words('english'))\n",
        "  data = data.apply(lambda x: \" \".join(word for word in x.split() if word not in stop_words))\n",
        "\n",
        "  return data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q9HQhAVPUNt9"
      },
      "source": [
        "# Upload Financial_Phrasebank Data 🤗"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200,
          "referenced_widgets": [
            "d94e74a622b24c3ab39e9e7de437efc6",
            "8f7117cb09d6466fb43ee109cf513a3e",
            "9c7124b4dd0c408ea141b60b9285104a",
            "8eec3b46272e43bc8c59fe6fbf047a74",
            "a90e561f53a940a983fcad4a5f6838c6",
            "13768ae9f7eb45d78e14e8ba93cd22a2",
            "dfe77781db9849a9ae420d2fc97e4197",
            "8ca9c06729694ac08a48c751c4e0bc3c",
            "3bf54c950d134625bd4e59abc85cffc5",
            "c75971b274d84dd58cb31cbf88fcd3a3",
            "15f42b0e46a84cb49dfe00a6c666dac5",
            "047df9c754b74e8787ba2470612792c7",
            "69ae125c54ff43078af4961b650db3d1",
            "9b6e4d2ac08d4c7ba4f1cc4cc83d3891",
            "6ec91147ea0d48459cd558e8d8853407",
            "29d2c7efe50d4d1bbe902e66b1c73ebd",
            "8d22dad0578841a48d7a5175fbca555d",
            "a4b64694027645269be80205a594756b",
            "5e4a3851461b41899b1e4ac1fe428d62",
            "f46e655f67e1426b8c3815e4d390e115",
            "254167bf2e844d9ebcaa127d21067fde",
            "2374b23c7d6d43149ff52a569f2be993",
            "ed590ea5e5e34adf8b54ec9defaf57f1",
            "fe4565c8c72f416a80226ed34a3b084b",
            "bcb4e9547ff34a2e84988678780dc775",
            "682b839aed6641c389928fa5f5e07f2f",
            "beed5fc6ccf347b092700812bf136d02",
            "e6fda1df66864682af2f576dc9b8f907",
            "283e8da912754582bd498f47cc577ee3",
            "9ae25b72186540dab2fe22df932f365a",
            "c87e231443a64e32a40f0a52423a0786",
            "161fc3e1ece54727a996d64ac8053bba",
            "c41a9401f2e04378af2769e33feceeb1",
            "73fc020a8a1b415ba17d84f511eb1954",
            "e9164fdf4f924dcabf85bf90316b08eb",
            "e59fec5296804305998ff8d0eb4f4e86",
            "470f01239f21461d9980ed9c7381fbb6",
            "11ca43c181ae4e5c8f9401dc607d535c",
            "8b2cea7261c2457abb5b3388725ae970",
            "067ea20d276d4d74ac19bcd33a46f610",
            "5a743aa078ad4ca7bb33bf55500fbcd4",
            "4bfc97ca641043afa89b644d1470922c",
            "7ef53a3b847d4718adc0a2e1015be78a",
            "e115d386213040858ba56b464975211f",
            "23ca79e7f1ed495c8ae26d3f135a61fd",
            "1397b38634f44898b7c49592b8ace2d6",
            "f5c7ba46df244e81a325bceee46900c2",
            "76d87ae78a35450795cd853da7dd2830",
            "889e595b441a4b0d91aee3c4f77db2ab",
            "d29c114b2503409296e358a86cce7581",
            "86625afd990e4c7687bdcdc4046b9b1b",
            "d20b71c019e04dd488834853d94f4f3b",
            "c3e6d002173348dc9aa1343029b9c709",
            "e63b0c5e65f34e35902998e2547753ed",
            "80682a1c214b46389098c5d1338e64b2"
          ]
        },
        "id": "XYcyY31IQU5S",
        "outputId": "c0c000ef-dfc9-4f2e-bd9d-f08ce56c4068"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/6.04k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d94e74a622b24c3ab39e9e7de437efc6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/13.7k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "047df9c754b74e8787ba2470612792c7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/8.86k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ed590ea5e5e34adf8b54ec9defaf57f1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset financial_phrasebank/sentences_allagree to /root/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/682k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "73fc020a8a1b415ba17d84f511eb1954"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/2264 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "23ca79e7f1ed495c8ae26d3f135a61fd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset financial_phrasebank downloaded and prepared to /root/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141. Subsequent calls will reuse this data.\n",
            "Total Number of Sentences is: 2264\n"
          ]
        }
      ],
      "source": [
        "# load the data using load_dataset\n",
        "dataset = load_dataset(\"financial_phrasebank\",\"sentences_allagree\",split=\"train\")\n",
        "# Convert the DatasetDict to a Pandas dataframe (named it after HuggingFace DataFrame)\n",
        "text_df = pd.DataFrame(dataset)\n",
        "print(\"Total Number of Sentences is: {}\".format(len(text_df[\"sentence\"])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dxpg3g-KaAjY",
        "outputId": "1936a8b6-39a9-44fc-9745-01a5ed32655b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    0.614399\n",
              "2    0.251767\n",
              "0    0.133834\n",
              "Name: label, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "# Display the dataframe\n",
        "text_df[\"label\"].value_counts(normalize=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Caqjc8B-ZF4y",
        "outputId": "c99f7e4c-a00d-4cf2-b860-67d92b8b3dbc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        }
      ],
      "source": [
        "# clean and split huggingface dataset into training and validation\n",
        "sentences = pre_process(text_df[\"sentence\"])\n",
        "X_train_pre, X_test_pre, y_train_pre, y_test_pre = train_test_split(sentences,\n",
        "                                                                    text_df[\"label\"],\n",
        "                                                                    test_size=0.1,\n",
        "                                                                    random_state=42)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YEVrNaJrZoVe"
      },
      "source": [
        "# Upload Economic Sentiment text from Kaggle ⚓"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ST2wQFf2P8zg"
      },
      "outputs": [],
      "source": [
        "uploaded = files.upload()\n",
        "text_df = pd.read_csv(io.BytesIO(uploaded['economic_sentiment_text.csv']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZGWsS15iUeIL"
      },
      "outputs": [],
      "source": [
        "# clean and split economic sentiment dataset into training and validation\n",
        "text_df[\"Sentiment\"] = text_df[\"Sentiment\"].replace({\"neutral\":0, \"positive\":1, \"negative\":2})\n",
        "text_df[\"Sentiment\"].value_counts(normalize=True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0uq5lpfw_PYX"
      },
      "outputs": [],
      "source": [
        "sentences = pre_process(text_df[\"Sentence\"])\n",
        "X_train_pre, X_test_pre, y_train_pre, y_test_pre = train_test_split(sentences,text_df[\"Sentiment\"],test_size=0.1,random_state=42)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XoSvQi5zbHnN"
      },
      "source": [
        "# Data Pre-processing with Tensorflow Tokenizer 📑"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1f03aZbzS9Is",
        "outputId": "49b38c25-5c0c-40c2-d394-6572a9a7c217"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The max length is:  36\n"
          ]
        }
      ],
      "source": [
        "# Initialize the tokenizer \n",
        "tokenizer = Tokenizer()\n",
        "\n",
        "# Fit the tokenizer on the training data\n",
        "tokenizer.fit_on_texts(X_train_pre)\n",
        "\n",
        "# Convert the text data to sequences\n",
        "X_train_seq = tokenizer.texts_to_sequences(X_train_pre)\n",
        "X_test_seq = tokenizer.texts_to_sequences(X_test_pre)\n",
        "\n",
        "# Find max length\n",
        "max_length = max(len(seq) for seq in X_train_seq)\n",
        "print(\"The max length is: \", max_length)\n",
        "\n",
        "# Pad the sequences to have the same length\n",
        "X_train = pad_sequences(X_train_seq, max_length)\n",
        "X_test = pad_sequences(X_test_seq, max_length)\n",
        "\n",
        "# converting target variables into categorical values\n",
        "y_train = to_categorical(y_train_pre, num_classes=3)\n",
        "y_test = to_categorical(y_test_pre, num_classes=3)\n",
        "\n",
        "# Balance the labels\n",
        "smote = SMOTE(random_state=42)\n",
        "X_resampled, y_resampled = smote.fit_resample(X_train, y_train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oaorurNwg8Oj"
      },
      "source": [
        "# Baseline Model (Dense) ⚫"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6eyBG0Na121T",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9dbefbf0-c1c1-4db0-9e37-2bee27d6abdc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 36)]              0         \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 36, 256)           2560000   \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 36, 256)           0         \n",
            "                                                                 \n",
            " global_average_pooling1d (G  (None, 256)              0         \n",
            " lobalAveragePooling1D)                                          \n",
            "                                                                 \n",
            " dropout_1 (Dropout)         (None, 256)               0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 256)               65792     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 64)                16448     \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 3)                 195       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 2,642,435\n",
            "Trainable params: 2,642,435\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# construct the model architecture\n",
        "inputs = layers.Input(shape=(max_length,))\n",
        "x = layers.Embedding(input_dim=10000, output_dim=256, input_length=max_length)(inputs)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "x = layers.GlobalAveragePooling1D()(x)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "x = layers.Dense(256, activation=\"relu\")(x)\n",
        "x = layers.Dense(64, activation=\"relu\")(x)\n",
        "outputs = layers.Dense(3, activation=\"softmax\")(x)\n",
        "model = models.Model(inputs, outputs)\n",
        "\n",
        "# compile the model\n",
        "model.compile(loss=\"categorical_crossentropy\", \n",
        "              optimizer=tf.keras.optimizers.Adam(),\n",
        "              metrics=[\"accuracy\"])\n",
        "\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XFTbvlv_aE9F",
        "outputId": "93390450-a5a5-406d-ba13-1f735c439dbf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_dense/20230527-155945\n",
            "Epoch 1/4\n",
            "3738/3738 [==============================] - 46s 10ms/step - loss: 0.9047 - accuracy: 0.5372 - val_loss: 0.9953 - val_accuracy: 0.6079\n",
            "Epoch 2/4\n",
            "3738/3738 [==============================] - 17s 4ms/step - loss: 0.7673 - accuracy: 0.6089 - val_loss: 0.6293 - val_accuracy: 0.7401\n",
            "Epoch 3/4\n",
            "3738/3738 [==============================] - 18s 5ms/step - loss: 0.6692 - accuracy: 0.6873 - val_loss: 0.6014 - val_accuracy: 0.7269\n",
            "Epoch 4/4\n",
            "3738/3738 [==============================] - 17s 5ms/step - loss: 0.5967 - accuracy: 0.7207 - val_loss: 0.6596 - val_accuracy: 0.7489\n"
          ]
        }
      ],
      "source": [
        "# train the model\n",
        "model_history_hf = model.fit(X_resampled, y_resampled, \n",
        "          validation_data=(X_test, y_test),\n",
        "          epochs=4,\n",
        "          batch_size=1,\n",
        "          callbacks=[create_tensorboard_callback(SAVE_DIR,\"model_dense\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "Ez5-mI-0KTVg",
        "outputId": "9cf848d4-6b29-42e9-8072-d9e70be83b3f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       loss  accuracy  val_loss  val_accuracy\n",
              "1  0.767324  0.608882  0.629258      0.740088\n",
              "2  0.669200  0.687266  0.601377      0.726872"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-da6561d0-f3f7-4297-ac9f-46397d029366\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>val_loss</th>\n",
              "      <th>val_accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.767324</td>\n",
              "      <td>0.608882</td>\n",
              "      <td>0.629258</td>\n",
              "      <td>0.740088</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.669200</td>\n",
              "      <td>0.687266</td>\n",
              "      <td>0.601377</td>\n",
              "      <td>0.726872</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-da6561d0-f3f7-4297-ac9f-46397d029366')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-da6561d0-f3f7-4297-ac9f-46397d029366 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-da6561d0-f3f7-4297-ac9f-46397d029366');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "optimized = pd.DataFrame(model_history_hf.history) \n",
        "best_parameters = optimized.loc[optimized[\"loss\"] > optimized[\"val_loss\"]]\n",
        "best_parameters.sort_values(by=\"val_accuracy\", ascending=False).head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DtK7BVJAX1_A",
        "outputId": "4a9d23f0-88a6-4a1d-8c5b-1f2d94b1ad03"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8/8 [==============================] - 0s 2ms/step\n"
          ]
        }
      ],
      "source": [
        "# predict \n",
        "model_1_preds_probs = model.predict(X_test)\n",
        "# convert model prediction probabilities to label format\n",
        "model_1_preds = tf.squeeze(tf.round(model_1_preds_probs))\n",
        "# calculate model_1 results\n",
        "model_1 = calculate_results(y_test, model_1_preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gTUnvXRtMNYS"
      },
      "source": [
        "# Model 2 (LSTM) ⬛"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jq3YzuPrtsMt",
        "outputId": "4b0b617b-35a5-4d5c-db57-6e2663b41c55"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_4 (InputLayer)        [(None, 36)]              0         \n",
            "                                                                 \n",
            " embedding_3 (Embedding)     (None, 36, 256)           2560000   \n",
            "                                                                 \n",
            " dropout_8 (Dropout)         (None, 36, 256)           0         \n",
            "                                                                 \n",
            " lstm_1 (LSTM)               (None, 36, 256)           525312    \n",
            "                                                                 \n",
            " dropout_9 (Dropout)         (None, 36, 256)           0         \n",
            "                                                                 \n",
            " global_average_pooling1d_3   (None, 256)              0         \n",
            " (GlobalAveragePooling1D)                                        \n",
            "                                                                 \n",
            " dense_9 (Dense)             (None, 3)                 771       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 3,086,083\n",
            "Trainable params: 3,086,083\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# construct the model architecture\n",
        "inputs = layers.Input(shape=(max_length,))\n",
        "x = layers.Embedding(input_dim=10000, output_dim=256, input_length=max_length)(inputs)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "x = layers.LSTM(256, return_sequences=True)(x)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "x = layers.GlobalAveragePooling1D()(x)\n",
        "outputs = layers.Dense(3, activation=\"softmax\")(x)\n",
        "\n",
        "model_lstm = models.Model(inputs, outputs)\n",
        "\n",
        "# compile the model\n",
        "model_lstm.compile(loss=\"categorical_crossentropy\", \n",
        "              optimizer=tf.keras.optimizers.Adam(),\n",
        "              metrics=[\"accuracy\"])\n",
        "\n",
        "model_lstm.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o3hU3ADUMMi2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "656db6a9-c453-436a-8d64-18a6ed20f4b6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_lstm/20230527-162643\n",
            "Epoch 1/5\n",
            "117/117 [==============================] - 15s 109ms/step - loss: 0.9769 - accuracy: 0.4789 - val_loss: 0.7596 - val_accuracy: 0.7093\n",
            "Epoch 2/5\n",
            "117/117 [==============================] - 6s 50ms/step - loss: 0.7622 - accuracy: 0.6097 - val_loss: 0.6662 - val_accuracy: 0.6740\n",
            "Epoch 3/5\n",
            "117/117 [==============================] - 3s 24ms/step - loss: 0.6691 - accuracy: 0.6458 - val_loss: 0.6509 - val_accuracy: 0.7048\n",
            "Epoch 4/5\n",
            "117/117 [==============================] - 2s 14ms/step - loss: 0.6056 - accuracy: 0.6969 - val_loss: 0.6270 - val_accuracy: 0.7709\n",
            "Epoch 5/5\n",
            "117/117 [==============================] - 1s 12ms/step - loss: 0.5234 - accuracy: 0.7643 - val_loss: 0.5921 - val_accuracy: 0.7665\n"
          ]
        }
      ],
      "source": [
        "# train the model\n",
        "model_history_lstm = model_lstm.fit(X_resampled, y_resampled, \n",
        "          validation_data=(X_test, y_test),\n",
        "          epochs=5,\n",
        "          callbacks=[create_tensorboard_callback(SAVE_DIR,\"model_lstm\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bnAzxOF4469E",
        "outputId": "373f3a3b-eb16-4cba-e356-14fb37e30c90"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8/8 [==============================] - 0s 3ms/step\n"
          ]
        }
      ],
      "source": [
        "# predict \n",
        "model_2_preds_probs = model_lstm.predict(X_test)\n",
        "# convert model prediction probabilities to label format\n",
        "model_2_preds = tf.squeeze(tf.round(model_2_preds_probs))\n",
        "# calculate model_1 results\n",
        "model_2 = calculate_results(y_test, model_2_preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iLA6IjPCl16d"
      },
      "source": [
        "# Model 3 (GRU) ⬛"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3EqaQFrwlzWO",
        "outputId": "dcb5a0d3-418d-4d1f-b91e-3220b60c4fc0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 36)]              0         \n",
            "                                                                 \n",
            " embedding_1 (Embedding)     (None, 36, 128)           1280000   \n",
            "                                                                 \n",
            " dropout_4 (Dropout)         (None, 36, 128)           0         \n",
            "                                                                 \n",
            " gru (GRU)                   (None, 36, 128)           99072     \n",
            "                                                                 \n",
            " dropout_5 (Dropout)         (None, 36, 128)           0         \n",
            "                                                                 \n",
            " global_average_pooling1d_1   (None, 128)              0         \n",
            " (GlobalAveragePooling1D)                                        \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 3)                 387       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,379,459\n",
            "Trainable params: 1,379,459\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# construct the model architecture\n",
        "inputs = layers.Input(shape=(max_length, ))\n",
        "x = layers.Embedding(input_dim=10000, output_dim=128, input_length=max_length)(inputs)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "x = layers.GRU(128,return_sequences=True)(x)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "x = layers.GlobalAveragePooling1D()(x)\n",
        "outputs = layers.Dense(3, activation=\"softmax\")(x)\n",
        "model_gru = models.Model(inputs, outputs)\n",
        "\n",
        "# compile the model\n",
        "model_gru.compile(loss=\"categorical_crossentropy\", \n",
        "              optimizer=tf.keras.optimizers.Adam(),\n",
        "              metrics=[\"accuracy\"])\n",
        "model_gru.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "udTPCtTcl9Sj",
        "outputId": "8b286fdc-8d81-4099-9a5f-20707e449c89"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_gru/20230527-161144\n",
            "Epoch 1/5\n",
            "117/117 [==============================] - 19s 133ms/step - loss: 0.9955 - accuracy: 0.4666 - val_loss: 0.7336 - val_accuracy: 0.6608\n",
            "Epoch 2/5\n",
            "117/117 [==============================] - 8s 67ms/step - loss: 0.7796 - accuracy: 0.5835 - val_loss: 0.6529 - val_accuracy: 0.6652\n",
            "Epoch 3/5\n",
            "117/117 [==============================] - 4s 35ms/step - loss: 0.6864 - accuracy: 0.6327 - val_loss: 0.6489 - val_accuracy: 0.7048\n",
            "Epoch 4/5\n",
            "117/117 [==============================] - 1s 13ms/step - loss: 0.6220 - accuracy: 0.6739 - val_loss: 0.6596 - val_accuracy: 0.7137\n",
            "Epoch 5/5\n",
            "117/117 [==============================] - 2s 20ms/step - loss: 0.5617 - accuracy: 0.7218 - val_loss: 0.6308 - val_accuracy: 0.7621\n"
          ]
        }
      ],
      "source": [
        "# train the model\n",
        "model_history_gru = model_gru.fit(X_resampled, y_resampled, \n",
        "          validation_data=(X_test, y_test),\n",
        "          epochs=5,\n",
        "          callbacks=[create_tensorboard_callback(SAVE_DIR,\"model_gru\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b53FrpsTl9M7",
        "outputId": "96d02512-d0c2-4c20-aa80-ad204443a355"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8/8 [==============================] - 0s 3ms/step\n"
          ]
        }
      ],
      "source": [
        "# predict \n",
        "model_3_preds_probs = model_gru.predict(X_test)\n",
        "# convert model prediction probabilities to label format\n",
        "model_3_preds = tf.squeeze(tf.round(model_3_preds_probs))\n",
        "# calculate model_1 results\n",
        "model_3 = calculate_results(y_test, model_3_preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "veLOyc4hm5Yw"
      },
      "source": [
        "# Model 4 (Bidirectional LSTM) ⬛"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nLMnQjC1nM6_",
        "outputId": "1ca202f1-94d3-4005-9735-ebcd62f2f384"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_3 (InputLayer)        [(None, 36)]              0         \n",
            "                                                                 \n",
            " embedding_2 (Embedding)     (None, 36, 256)           2560000   \n",
            "                                                                 \n",
            " dropout_7 (Dropout)         (None, 36, 256)           0         \n",
            "                                                                 \n",
            " bidirectional (Bidirectiona  (None, 36, 256)          394240    \n",
            " l)                                                              \n",
            "                                                                 \n",
            " global_average_pooling1d_2   (None, 256)              0         \n",
            " (GlobalAveragePooling1D)                                        \n",
            "                                                                 \n",
            " dense_7 (Dense)             (None, 64)                16448     \n",
            "                                                                 \n",
            " dense_8 (Dense)             (None, 3)                 195       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 2,970,883\n",
            "Trainable params: 2,970,883\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# construct the model architecture\n",
        "inputs = layers.Input(shape=(max_length,))\n",
        "x = layers.Dropout(0.5)(x)\n",
        "x = layers.Embedding(input_dim=10000, output_dim=256, input_length=max_length)(inputs)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "x = layers.Bidirectional(layers.LSTM(128,return_sequences=True))(x)\n",
        "x = layers.GlobalAveragePooling1D()(x)\n",
        "x = layers.Dense(64, \"relu\")(x)\n",
        "outputs = layers.Dense(3, activation=\"softmax\")(x)\n",
        "model_bd = models.Model(inputs, outputs)\n",
        "\n",
        "# compile the model\n",
        "model_bd.compile(loss=\"categorical_crossentropy\", \n",
        "              optimizer=tf.keras.optimizers.Adam(),\n",
        "              metrics=[\"accuracy\"])\n",
        "\n",
        "model_bd.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5wCWzqJio82-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4e89a170-2ee3-445e-f08e-5b15710368f4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_bd/20230527-161220\n",
            "Epoch 1/8\n",
            "117/117 [==============================] - 21s 142ms/step - loss: 0.9316 - accuracy: 0.4960 - val_loss: 0.6228 - val_accuracy: 0.7225\n",
            "Epoch 2/8\n",
            "117/117 [==============================] - 8s 66ms/step - loss: 0.7240 - accuracy: 0.6049 - val_loss: 0.8006 - val_accuracy: 0.6344\n",
            "Epoch 3/8\n",
            "117/117 [==============================] - 3s 24ms/step - loss: 0.6299 - accuracy: 0.6651 - val_loss: 0.7028 - val_accuracy: 0.7533\n",
            "Epoch 4/8\n",
            "117/117 [==============================] - 4s 31ms/step - loss: 0.5566 - accuracy: 0.7199 - val_loss: 0.6088 - val_accuracy: 0.7885\n",
            "Epoch 5/8\n",
            "117/117 [==============================] - 3s 25ms/step - loss: 0.4803 - accuracy: 0.7809 - val_loss: 0.7387 - val_accuracy: 0.7313\n",
            "Epoch 6/8\n",
            "117/117 [==============================] - 3s 27ms/step - loss: 0.4138 - accuracy: 0.8277 - val_loss: 0.7261 - val_accuracy: 0.7533\n",
            "Epoch 7/8\n",
            "117/117 [==============================] - 2s 13ms/step - loss: 0.3418 - accuracy: 0.8571 - val_loss: 0.7303 - val_accuracy: 0.7665\n",
            "Epoch 8/8\n",
            "117/117 [==============================] - 1s 11ms/step - loss: 0.3114 - accuracy: 0.8721 - val_loss: 0.7777 - val_accuracy: 0.7357\n"
          ]
        }
      ],
      "source": [
        "# train the model\n",
        "model_history_bd = model_bd.fit(X_resampled, y_resampled, \n",
        "          validation_data=(X_test, y_test),\n",
        "          epochs=8,\n",
        "          callbacks=[create_tensorboard_callback(SAVE_DIR,\"model_bd\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WStdmnW6o-s-",
        "outputId": "81b20279-7845-4225-9158-b1362ecf889e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8/8 [==============================] - 1s 4ms/step\n"
          ]
        }
      ],
      "source": [
        "# predict \n",
        "model_4_preds_probs = model_bd.predict(X_test)\n",
        "# convert model prediction probabilities to label format\n",
        "model_4_preds = tf.squeeze(tf.round(model_4_preds_probs))\n",
        "# calculate model_1 results\n",
        "model_4 = calculate_results(y_test, model_4_preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J4Za9bC2YKle"
      },
      "source": [
        "# Feature Extraction #1 ⬛\n",
        "\n",
        "**A pre-trained sentence encoder (embedding) -> Universal Sentence Encoder (USE)**\n",
        "\n",
        "**Resources**\n",
        "* If you'd want to deploy the model for mobile, a lite-version of any pre-trained model would do the work. If otherwise, stick with the large ones. \n",
        "* https://tfhub.dev/google/universal-sentence-encoder/4 \n",
        "\n",
        "**Import Notes:** \n",
        "* Do not import tensorflow_hub outside the scope of hub.load(\"\"). If run outside, for some unknown reason the packages throws an error.    \n",
        "* leave the input_shape as a blank list, because it has a pre-defined input_shape that we have no idea of (probably it has an input shape of 512)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NB1A7nk6YKKd"
      },
      "outputs": [],
      "source": [
        "import tensorflow_hub as hub\n",
        "embedding_fn = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z4DsjIs-iy_Z"
      },
      "outputs": [],
      "source": [
        "# Create a Keras Layer using the USE pre-trained layer from tensorflow-hub \n",
        "sentence_encoder_layer = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n",
        "                                        input_shape=[], # check the important notes\n",
        "                                        dtype=tf.string,\n",
        "                                        trainable=False,\n",
        "                                        name=\"USE_encoder\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HS3Ev7lNoLpq"
      },
      "outputs": [],
      "source": [
        "# Run this only once, otherwise the shape of the label data would look like (0.9*len(y_train),3,3).\n",
        "y_train_pre = to_categorical(y_train_pre,num_classes=3)\n",
        "y_test_pre = to_categorical(y_test_pre,num_classes=3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SKYV_iCiq3WX",
        "outputId": "416faa01-07a8-48ca-b078-d2b753f420f9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(2037, 3) (227, 3)\n",
            "(2037,) (227,)\n"
          ]
        }
      ],
      "source": [
        "print(y_train_pre.shape, y_test_pre.shape)\n",
        "print(X_train_pre.shape, X_test_pre.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uipSeBlglTJ9",
        "outputId": "e234849c-9f12-4636-b39c-35aab6a73da0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE_encoder (KerasLayer)    (None, 512)               256797824 \n",
            "                                                                 \n",
            " dropout_2 (Dropout)         (None, 512)               0         \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 512)               262656    \n",
            "                                                                 \n",
            " dropout_3 (Dropout)         (None, 512)               0         \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 512)               262656    \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 3)                 1539      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 257,324,675\n",
            "Trainable params: 526,851\n",
            "Non-trainable params: 256,797,824\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# create model using the Sequential API\n",
        "model_use = tf.keras.Sequential([\n",
        "    sentence_encoder_layer,\n",
        "    layers.Dropout(0.5),\n",
        "    layers.Dense(512, activation=\"relu\"),\n",
        "    layers.Dropout(0.25),\n",
        "    layers.Dense(512, activation=\"relu\"),\n",
        "    layers.Dense(3, activation=\"softmax\")\n",
        "])\n",
        "\n",
        "# Compile\n",
        "model_use.compile(loss= \"categorical_crossentropy\",\n",
        "                  optimizer=tf.keras.optimizers.Adam(0.0068),\n",
        "                  metrics=[\"accuracy\"])\n",
        "model_use.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "24kATD0DoGWc",
        "outputId": "bae51377-5481-48d9-9677-776e53ccd17e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_use/20230527-161112\n",
            "Epoch 1/12\n",
            "64/64 [==============================] - 7s 25ms/step - loss: 0.6360 - accuracy: 0.7364 - val_loss: 0.4810 - val_accuracy: 0.8194\n",
            "Epoch 2/12\n",
            "64/64 [==============================] - 1s 15ms/step - loss: 0.4816 - accuracy: 0.7997 - val_loss: 0.4950 - val_accuracy: 0.8326\n",
            "Epoch 3/12\n",
            "64/64 [==============================] - 1s 15ms/step - loss: 0.4417 - accuracy: 0.8223 - val_loss: 0.4265 - val_accuracy: 0.8546\n",
            "Epoch 4/12\n",
            "64/64 [==============================] - 1s 15ms/step - loss: 0.3986 - accuracy: 0.8326 - val_loss: 0.4413 - val_accuracy: 0.8370\n",
            "Epoch 5/12\n",
            "64/64 [==============================] - 1s 15ms/step - loss: 0.3462 - accuracy: 0.8596 - val_loss: 0.3896 - val_accuracy: 0.8634\n",
            "Epoch 6/12\n",
            "64/64 [==============================] - 1s 16ms/step - loss: 0.3195 - accuracy: 0.8684 - val_loss: 0.4531 - val_accuracy: 0.8282\n",
            "Epoch 7/12\n",
            "64/64 [==============================] - 1s 16ms/step - loss: 0.3156 - accuracy: 0.8679 - val_loss: 0.4681 - val_accuracy: 0.8194\n",
            "Epoch 8/12\n",
            "64/64 [==============================] - 1s 14ms/step - loss: 0.2582 - accuracy: 0.8945 - val_loss: 0.4736 - val_accuracy: 0.8414\n",
            "Epoch 9/12\n",
            "64/64 [==============================] - 1s 16ms/step - loss: 0.2616 - accuracy: 0.8959 - val_loss: 0.4141 - val_accuracy: 0.8326\n",
            "Epoch 10/12\n",
            "64/64 [==============================] - 1s 14ms/step - loss: 0.2576 - accuracy: 0.8999 - val_loss: 0.4672 - val_accuracy: 0.8414\n",
            "Epoch 11/12\n",
            "64/64 [==============================] - 1s 21ms/step - loss: 0.2152 - accuracy: 0.9121 - val_loss: 0.4438 - val_accuracy: 0.8458\n",
            "Epoch 12/12\n",
            "64/64 [==============================] - 2s 25ms/step - loss: 0.2400 - accuracy: 0.8974 - val_loss: 0.4124 - val_accuracy: 0.8546\n"
          ]
        }
      ],
      "source": [
        "# train the model with USE embedding\n",
        "model_history_use = model_use.fit(X_train_pre, y_train_pre,\n",
        "                                  validation_data=(X_test_pre, y_test_pre),\n",
        "                                  epochs=12,\n",
        "                                  callbacks=[create_tensorboard_callback(SAVE_DIR,\"model_use\")])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cXNJGcS7pOjD",
        "outputId": "87ad1b62-4a21-4168-872a-b8c416b4549f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8/8 [==============================] - 0s 10ms/step\n"
          ]
        }
      ],
      "source": [
        "# predict \n",
        "model_5_preds_probs = model_use.predict(X_test_pre)\n",
        "# convert model prediction probabilities to label format\n",
        "model_5_preds = tf.squeeze(tf.round(model_5_preds_probs))\n",
        "# calculate model_1 results\n",
        "model_5 = calculate_results(y_test_pre, model_5_preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "86FiB7EWzt-O"
      },
      "source": [
        "# Model Performance Comparison ☑"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "AlExIAQdxpUK",
        "outputId": "645d227e-73e8-4734-d85d-5db604e6cfd9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                    accuracy  precision  recall     f1\n",
              "Pre-trained            0.815      0.862   0.815  0.837\n",
              "LSTM                   0.744      0.803   0.744  0.764\n",
              "GRU                    0.722      0.813   0.722  0.762\n",
              "Dense                  0.718      0.813   0.718  0.760\n",
              "Bidirectional LSTM     0.705      0.803   0.705  0.747"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bc79ddd6-a50c-4338-a462-809fc2b48be5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Pre-trained</th>\n",
              "      <td>0.815</td>\n",
              "      <td>0.862</td>\n",
              "      <td>0.815</td>\n",
              "      <td>0.837</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LSTM</th>\n",
              "      <td>0.744</td>\n",
              "      <td>0.803</td>\n",
              "      <td>0.744</td>\n",
              "      <td>0.764</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>GRU</th>\n",
              "      <td>0.722</td>\n",
              "      <td>0.813</td>\n",
              "      <td>0.722</td>\n",
              "      <td>0.762</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Dense</th>\n",
              "      <td>0.718</td>\n",
              "      <td>0.813</td>\n",
              "      <td>0.718</td>\n",
              "      <td>0.760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Bidirectional LSTM</th>\n",
              "      <td>0.705</td>\n",
              "      <td>0.803</td>\n",
              "      <td>0.705</td>\n",
              "      <td>0.747</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bc79ddd6-a50c-4338-a462-809fc2b48be5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-bc79ddd6-a50c-4338-a462-809fc2b48be5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-bc79ddd6-a50c-4338-a462-809fc2b48be5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ],
      "source": [
        "all_model_results = pd.DataFrame({\"Dense\":model_1,\n",
        "                                  \"LSTM\":model_2,\n",
        "                                  \"GRU\":model_3,\n",
        "                                  \"Bidirectional LSTM\":model_4,\n",
        "                                  \"Pre-trained\":model_5}).T\n",
        "all_model_results[\"accuracy\"] = round(all_model_results[\"accuracy\"] / 100,3)\n",
        "all_model_results.sort_values(by=\"accuracy\",ascending=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6-1L_L1yYIrE"
      },
      "source": [
        "# Model Testing 🧪"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vXKkB1l4E3HO",
        "outputId": "a76a7607-432c-4c13-d3d5-b9e573256030"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Enter the text: \"However, the Euro 7 proposal is simply not the right way to do this, as it would have an extremely low environmental impact at an extremely high cost.\"\n",
            "1/1 [==============================] - 0s 80ms/step\n",
            "Sentiment Prediction: [0.32501534 0.06486119 0.61012346]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "# Define the pre-processing function\n",
        "def pre_process2(text):\n",
        "    # Convert text to lowercase\n",
        "    text = text.lower()\n",
        "\n",
        "    # Remove numbers\n",
        "    text = re.sub(r'\\d+', '', text)\n",
        "\n",
        "    # Remove punctuation\n",
        "    punctuation = string.punctuation.replace(\"'\", \"\")\n",
        "    text = text.translate(str.maketrans('', '', punctuation))\n",
        "\n",
        "    # Remove $\n",
        "    punctuation = string.punctuation.replace(\"$\", \"\")\n",
        "    text = text.translate(str.maketrans('', '', punctuation))\n",
        "\n",
        "    # Remove stop words\n",
        "    nltk.download('stopwords')\n",
        "    stop_words = set(stopwords.words('english'))\n",
        "    text = \" \".join(word for word in text.split() if word not in stop_words)\n",
        "\n",
        "    return text\n",
        "\n",
        "# Function to make prediction on new text\n",
        "def predict_sentiment(text):\n",
        "    # Preprocess the text\n",
        "    processed_text = pre_process2(text)\n",
        "\n",
        "    # Tokenize the preprocessed text\n",
        "    text_seq = tokenizer.texts_to_sequences([processed_text])\n",
        "\n",
        "    # Pad the sequence\n",
        "    text_seq_padded = pad_sequences(text_seq, maxlen=max_length)\n",
        "\n",
        "    # Make prediction\n",
        "    prediction = model.predict(text_seq_padded)[0]\n",
        "\n",
        "    return prediction\n",
        "\n",
        "# Get user input\n",
        "text_input = input(\"Enter the text: \")\n",
        "\n",
        "# Make prediction\n",
        "prediction = predict_sentiment(text_input)\n",
        "print(\"Sentiment Prediction:\", prediction)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OJxj0hkVHrOh"
      },
      "outputs": [],
      "source": [
        "# Function to make prediction on new text\n",
        "def predict_sentiment2(text):\n",
        "    \n",
        "    # Make prediction\n",
        "    prediction = tf.squeeze(model_use.predict([text]))\n",
        "    \n",
        "    return prediction\n",
        "\n",
        "# Get user input\n",
        "text_input = input(\"Enter the text: \")\n",
        "\n",
        "# Make prediction\n",
        "prediction = predict_sentiment2(text_input)\n",
        "print(\"Sentiment Prediction:\", prediction)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save Model"
      ],
      "metadata": {
        "id": "2q_maHOjN8tH"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gN1iWeiKiR2l"
      },
      "outputs": [],
      "source": [
        "# Save Model & Pipeline\n",
        "import joblib\n",
        "model_file = open(\"sentiment_classifier_model.pkl\",\"wb\")\n",
        "joblib.dump(model, model_file)\n",
        "model_file.close()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "\n",
        "# Define the custom layer\n",
        "class USEEncoderLayer(tf.keras.layers.Layer):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(USEEncoderLayer, self).__init__(**kwargs)\n",
        "        self.use_layer = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n",
        "                                        input_shape=[],  # check the important notes\n",
        "                                        dtype=tf.string,\n",
        "                                        trainable=False,\n",
        "                                        name=\"USE_encoder\")\n",
        "\n",
        "    def call(self, inputs, **kwargs):\n",
        "        return self.use_layer(inputs)\n",
        "\n",
        "# Register the custom layer\n",
        "custom_objects = {\"USEEncoderLayer\": USEEncoderLayer, \"KerasLayer\": hub.KerasLayer}\n",
        "\n",
        "# Load the model with custom layer\n",
        "with tf.keras.utils.custom_object_scope(custom_objects):\n",
        "    model = joblib.load(\"/content/model_use2.pkl\")"
      ],
      "metadata": {
        "id": "HiAs1aExwNIi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to make prediction on new text\n",
        "def predict_sentiment2(text):\n",
        "    \n",
        "    # Make prediction\n",
        "    prediction = tf.squeeze(model.predict([text]))\n",
        "    \n",
        "    return prediction\n",
        "\n",
        "# Get user input\n",
        "text_input = input(\"Enter the text: \")\n",
        "\n",
        "# Make prediction\n",
        "prediction = predict_sentiment2(text_input)\n",
        "print(\"Sentiment Prediction:\", prediction)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QDft-wr5xI-a",
        "outputId": "3e4417fe-2912-4ec8-beeb-c6550ac2664b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Enter the text: \"However, the Euro 7 proposal is simply not the right way to do this, as it would have an extremely low environmental impact at an extremely high cost.\"\n",
            "1/1 [==============================] - 1s 536ms/step\n",
            "Sentiment Prediction: tf.Tensor([9.9942374e-01 1.5279527e-04 4.2358576e-04], shape=(3,), dtype=float32)\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "YEVrNaJrZoVe"
      ],
      "gpuType": "T4",
      "toc_visible": true,
      "authorship_tag": "ABX9TyOZN4cis4uWoZ+Ofxy/QPHw",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d94e74a622b24c3ab39e9e7de437efc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8f7117cb09d6466fb43ee109cf513a3e",
              "IPY_MODEL_9c7124b4dd0c408ea141b60b9285104a",
              "IPY_MODEL_8eec3b46272e43bc8c59fe6fbf047a74"
            ],
            "layout": "IPY_MODEL_a90e561f53a940a983fcad4a5f6838c6"
          }
        },
        "8f7117cb09d6466fb43ee109cf513a3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_13768ae9f7eb45d78e14e8ba93cd22a2",
            "placeholder": "​",
            "style": "IPY_MODEL_dfe77781db9849a9ae420d2fc97e4197",
            "value": "Downloading builder script: 100%"
          }
        },
        "9c7124b4dd0c408ea141b60b9285104a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8ca9c06729694ac08a48c751c4e0bc3c",
            "max": 6036,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3bf54c950d134625bd4e59abc85cffc5",
            "value": 6036
          }
        },
        "8eec3b46272e43bc8c59fe6fbf047a74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c75971b274d84dd58cb31cbf88fcd3a3",
            "placeholder": "​",
            "style": "IPY_MODEL_15f42b0e46a84cb49dfe00a6c666dac5",
            "value": " 6.04k/6.04k [00:00&lt;00:00, 189kB/s]"
          }
        },
        "a90e561f53a940a983fcad4a5f6838c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "13768ae9f7eb45d78e14e8ba93cd22a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dfe77781db9849a9ae420d2fc97e4197": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8ca9c06729694ac08a48c751c4e0bc3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bf54c950d134625bd4e59abc85cffc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c75971b274d84dd58cb31cbf88fcd3a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "15f42b0e46a84cb49dfe00a6c666dac5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "047df9c754b74e8787ba2470612792c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_69ae125c54ff43078af4961b650db3d1",
              "IPY_MODEL_9b6e4d2ac08d4c7ba4f1cc4cc83d3891",
              "IPY_MODEL_6ec91147ea0d48459cd558e8d8853407"
            ],
            "layout": "IPY_MODEL_29d2c7efe50d4d1bbe902e66b1c73ebd"
          }
        },
        "69ae125c54ff43078af4961b650db3d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8d22dad0578841a48d7a5175fbca555d",
            "placeholder": "​",
            "style": "IPY_MODEL_a4b64694027645269be80205a594756b",
            "value": "Downloading metadata: 100%"
          }
        },
        "9b6e4d2ac08d4c7ba4f1cc4cc83d3891": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5e4a3851461b41899b1e4ac1fe428d62",
            "max": 13677,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f46e655f67e1426b8c3815e4d390e115",
            "value": 13677
          }
        },
        "6ec91147ea0d48459cd558e8d8853407": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_254167bf2e844d9ebcaa127d21067fde",
            "placeholder": "​",
            "style": "IPY_MODEL_2374b23c7d6d43149ff52a569f2be993",
            "value": " 13.7k/13.7k [00:00&lt;00:00, 182kB/s]"
          }
        },
        "29d2c7efe50d4d1bbe902e66b1c73ebd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d22dad0578841a48d7a5175fbca555d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a4b64694027645269be80205a594756b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5e4a3851461b41899b1e4ac1fe428d62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f46e655f67e1426b8c3815e4d390e115": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "254167bf2e844d9ebcaa127d21067fde": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2374b23c7d6d43149ff52a569f2be993": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ed590ea5e5e34adf8b54ec9defaf57f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fe4565c8c72f416a80226ed34a3b084b",
              "IPY_MODEL_bcb4e9547ff34a2e84988678780dc775",
              "IPY_MODEL_682b839aed6641c389928fa5f5e07f2f"
            ],
            "layout": "IPY_MODEL_beed5fc6ccf347b092700812bf136d02"
          }
        },
        "fe4565c8c72f416a80226ed34a3b084b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e6fda1df66864682af2f576dc9b8f907",
            "placeholder": "​",
            "style": "IPY_MODEL_283e8da912754582bd498f47cc577ee3",
            "value": "Downloading readme: 100%"
          }
        },
        "bcb4e9547ff34a2e84988678780dc775": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ae25b72186540dab2fe22df932f365a",
            "max": 8862,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c87e231443a64e32a40f0a52423a0786",
            "value": 8862
          }
        },
        "682b839aed6641c389928fa5f5e07f2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_161fc3e1ece54727a996d64ac8053bba",
            "placeholder": "​",
            "style": "IPY_MODEL_c41a9401f2e04378af2769e33feceeb1",
            "value": " 8.86k/8.86k [00:00&lt;00:00, 114kB/s]"
          }
        },
        "beed5fc6ccf347b092700812bf136d02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e6fda1df66864682af2f576dc9b8f907": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "283e8da912754582bd498f47cc577ee3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ae25b72186540dab2fe22df932f365a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c87e231443a64e32a40f0a52423a0786": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "161fc3e1ece54727a996d64ac8053bba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c41a9401f2e04378af2769e33feceeb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "73fc020a8a1b415ba17d84f511eb1954": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e9164fdf4f924dcabf85bf90316b08eb",
              "IPY_MODEL_e59fec5296804305998ff8d0eb4f4e86",
              "IPY_MODEL_470f01239f21461d9980ed9c7381fbb6"
            ],
            "layout": "IPY_MODEL_11ca43c181ae4e5c8f9401dc607d535c"
          }
        },
        "e9164fdf4f924dcabf85bf90316b08eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8b2cea7261c2457abb5b3388725ae970",
            "placeholder": "​",
            "style": "IPY_MODEL_067ea20d276d4d74ac19bcd33a46f610",
            "value": "Downloading data: 100%"
          }
        },
        "e59fec5296804305998ff8d0eb4f4e86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5a743aa078ad4ca7bb33bf55500fbcd4",
            "max": 681890,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4bfc97ca641043afa89b644d1470922c",
            "value": 681890
          }
        },
        "470f01239f21461d9980ed9c7381fbb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ef53a3b847d4718adc0a2e1015be78a",
            "placeholder": "​",
            "style": "IPY_MODEL_e115d386213040858ba56b464975211f",
            "value": " 682k/682k [00:00&lt;00:00, 4.42MB/s]"
          }
        },
        "11ca43c181ae4e5c8f9401dc607d535c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8b2cea7261c2457abb5b3388725ae970": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "067ea20d276d4d74ac19bcd33a46f610": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5a743aa078ad4ca7bb33bf55500fbcd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4bfc97ca641043afa89b644d1470922c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7ef53a3b847d4718adc0a2e1015be78a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e115d386213040858ba56b464975211f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "23ca79e7f1ed495c8ae26d3f135a61fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1397b38634f44898b7c49592b8ace2d6",
              "IPY_MODEL_f5c7ba46df244e81a325bceee46900c2",
              "IPY_MODEL_76d87ae78a35450795cd853da7dd2830"
            ],
            "layout": "IPY_MODEL_889e595b441a4b0d91aee3c4f77db2ab"
          }
        },
        "1397b38634f44898b7c49592b8ace2d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d29c114b2503409296e358a86cce7581",
            "placeholder": "​",
            "style": "IPY_MODEL_86625afd990e4c7687bdcdc4046b9b1b",
            "value": "Generating train split:  81%"
          }
        },
        "f5c7ba46df244e81a325bceee46900c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d20b71c019e04dd488834853d94f4f3b",
            "max": 2264,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c3e6d002173348dc9aa1343029b9c709",
            "value": 2264
          }
        },
        "76d87ae78a35450795cd853da7dd2830": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e63b0c5e65f34e35902998e2547753ed",
            "placeholder": "​",
            "style": "IPY_MODEL_80682a1c214b46389098c5d1338e64b2",
            "value": " 1827/2264 [00:00&lt;00:00, 7863.59 examples/s]"
          }
        },
        "889e595b441a4b0d91aee3c4f77db2ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "d29c114b2503409296e358a86cce7581": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86625afd990e4c7687bdcdc4046b9b1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d20b71c019e04dd488834853d94f4f3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3e6d002173348dc9aa1343029b9c709": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e63b0c5e65f34e35902998e2547753ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80682a1c214b46389098c5d1338e64b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}