Multiple Choice/Strategy QA/Strategy QA - Multiple Choice Using BERT.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Strategy QA: Multiple Choice\n",
    "\n",
    "Dataset Source: https://huggingface.co/datasets/tasksource/bigbench/viewer/strategyqa/train"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "import os, sys, random, ast\n",
    "os.environ['TOKENIZERS_PARALLELISM']='false'\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import datasets\n",
    "from datasets import Dataset, DatasetDict, load_dataset\n",
    "\n",
    "import torch\n",
    "\n",
    "from dataclasses import dataclass\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModelForMultipleChoice\n",
    "from transformers import Trainer, TrainingArguments, set_seed\n",
    "from transformers.tokenization_utils_base import PreTrainedTokenizerBase, PaddingStrategy\n",
    "\n",
    "from typing import Optional, Union\n",
    "\n",
    "!git lfs install"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display Library Versions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Python: 3.9.12\n",
      "         NumPy: 1.25.1\n",
      "        Pandas: 2.0.3\n",
      "         Torch: 2.0.1\n",
      "      Datasets: 2.13.1\n",
      "  Transformers: 4.26.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Python:\".rjust(15), sys.version[0:6])\n",
    "print(\"NumPy:\".rjust(15), np.__version__)\n",
    "print(\"Pandas:\".rjust(15), pd.__version__)\n",
    "print(\"Torch:\".rjust(15), torch.__version__)\n",
    "print(\"Datasets:\".rjust(15), datasets.__version__)\n",
    "print(\"Transformers:\".rjust(15), transformers.__version__)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Dataset & Convert To Pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset bigbench (/Users/briandunn/.cache/huggingface/datasets/tasksource___bigbench/strategyqa/1.0.0/c5da5ac497141c7435da10444495b8577405d4ed01e524265b144a7063718c0c)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "913b27c14e7e4ed09a547b1da161d63d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>labels</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q: Was the Louisiana Purchase made with bitcoi...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q: Is someone more likely to survive having br...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q: Are you likely to hear Rammstein playing in...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q: Could the moon fit inside the Black Sea?\\nA:</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q: Can Arnold Schwarzenegger deadlift an adult...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1827</th>\n",
       "      <td>Q: Were there greater landslides than 1980 Uni...</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1828</th>\n",
       "      <td>Q: Would Eye surgery on a fly be in vain?\\nA:</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1829</th>\n",
       "      <td>Q: Was John Lennon known to be a good friend t...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1830</th>\n",
       "      <td>Q: Did Martin Luther believe in Satan?\\nA:</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1831</th>\n",
       "      <td>Q: Will Elijah Cummings cast a vote in the 202...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1832 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                context  labels    0   1\n",
       "0     Q: Was the Louisiana Purchase made with bitcoi...       1  Yes  No\n",
       "1     Q: Is someone more likely to survive having br...       1  Yes  No\n",
       "2     Q: Are you likely to hear Rammstein playing in...       1  Yes  No\n",
       "3       Q: Could the moon fit inside the Black Sea?\\nA:       1  Yes  No\n",
       "4     Q: Can Arnold Schwarzenegger deadlift an adult...       1  Yes  No\n",
       "...                                                 ...     ...  ...  ..\n",
       "1827  Q: Were there greater landslides than 1980 Uni...       0  Yes  No\n",
       "1828      Q: Would Eye surgery on a fly be in vain?\\nA:       0  Yes  No\n",
       "1829  Q: Was John Lennon known to be a good friend t...       1  Yes  No\n",
       "1830         Q: Did Martin Luther believe in Satan?\\nA:       0  Yes  No\n",
       "1831  Q: Will Elijah Cummings cast a vote in the 202...       1  Yes  No\n",
       "\n",
       "[1832 rows x 4 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = load_dataset(\"tasksource/bigbench\", \"strategyqa\")\n",
    "\n",
    "data = data.rename_column(\"inputs\", \"context\")\n",
    "\n",
    "data.set_format(type=\"pandas\")\n",
    "\n",
    "df = data['train'][:]\n",
    "\n",
    "df = df.dropna(how='all')\n",
    "\n",
    "df = df.drop_duplicates(subset='context')\n",
    "\n",
    "df['labels'] = df['targets'].apply(lambda x: x[0])\n",
    "\n",
    "mc_target_values = ['Yes', 'No']\n",
    "\n",
    "df['0'] = df['multiple_choice_targets'].apply(lambda x: mc_target_values[0])\n",
    "df['1'] = df['multiple_choice_targets'].apply(lambda x: mc_target_values[1])\n",
    "\n",
    "\n",
    "df['labels'] = df['multiple_choice_scores'].apply(lambda x: 1 if x[0] == 0 else 0)\n",
    "\n",
    "df = df.drop(columns=[\"multiple_choice_targets\", \"idx\", \"multiple_choice_scores\", \"targets\"])\n",
    "\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visual: Word Length of Training Dataset Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABb4AAAK9CAYAAADrKHpYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABSNUlEQVR4nO39ebhU9Z3vb7838wwCMkUExAERFQ8qciQISgTHoHiMxglDxxMDGkFx6CQymHaKRmLikFGiUaPpo5jgiCg4z0GjcSTaOAFGBQQVUOr5Iz/rcQsqWzYWLO/7uurqXWutWvWp2lTvvl6u/lZVqVQqBQAAAAAACqJOpQcAAAAAAIDaJHwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAsNa6du2aESNGVHoM1gMTJkxIVVVV/vWvf63z57rlllvSu3fvNGrUKFVVVVm4cOE6f04AADYMwjcAANVMmTIlVVVVeeSRR1a7f+DAgenVq9daP89NN92UCRMmrPV5iuill15KVVVVzjvvvEqP8qnOPPPMTJ06tWLP/+abb+bggw9O48aNc9FFF+WKK65I06ZNV3vs5/2b/rK9++67mTBhQmbOnFnpUQAACqtepQcAAGDD9+yzz6ZOnZpdU3HTTTfloosuEr83UGeeeWYOOuigDBs2rCLP//DDD+edd97JGWeckcGDB1dkhi/q3XffzcSJE5P8+z8kAQBQ+1zxDQDAWmvYsGHq169f6TFqZOnSpZUegbWwYMGCJEmrVq0qOwgAAOsl4RsAgLX2yTW+V6xYkYkTJ2aLLbZIo0aN0qZNm/Tv3z/Tp09PkowYMSIXXXRRkqSqqqp8+8jSpUtz4oknpnPnzmnYsGG22mqrnHfeeSmVStWe97333svxxx+ftm3bpnnz5tl///3z6quvpqqqqtqV5B+tO/2Pf/wj3/72t7PRRhulf//+SZInnngiI0aMyGabbZZGjRqlQ4cO+c53vpM333yz2nN9dI7nnnsuhx9+eFq2bJmNN944P/7xj1MqlfLyyy/nm9/8Zlq0aJEOHTrk/PPPX+V9mjt3bp555pm1eq8/btmyZRk/fnw233zzNGzYMJ07d87JJ5+cZcuWVTuuqqoqo0ePztSpU9OrV680bNgw22yzTW655ZZVzjlz5szsuOOOadSoUbp3755f/epX5df+8fMtXbo0f/jDH8q/u0+u8b5w4cKMGDEirVq1SsuWLXP00Ufn3XffXaPX9ec//zl9+vRJ48aN07Zt2xx++OF59dVXy/sHDhyYo446Kkmy0047rfb5P8+IESPSrFmzvPrqqxk2bFiaNWuWjTfeOCeddFI+/PDD8nEfX3bmggsuSJcuXdK4cePstttuefLJJ6udc+DAgau9gnvEiBHp2rVr+Xwbb7xxkmTixInl9++jf6/z5s3L0UcfnU022SQNGzZMx44d881vfjMvvfRSjV4fAMBXnaVOAABYrUWLFq32CwpXrFjxuY+dMGFCzjrrrPzHf/xHdt555yxevDiPPPJIHnvssXzjG9/I//2//zevvfZapk+fniuuuKLaY0ulUvbff//ceeedGTlyZHr37p1bb70148aNy6uvvpoLLrigfOyIESNy7bXX5ogjjsguu+ySWbNmZZ999vnUuf7P//k/2WKLLXLmmWeWI/r06dPzz3/+M0cffXQ6dOiQp556Kr/+9a/z1FNP5YEHHqgWfJPkW9/6VrbeeuucffbZufHGG/OTn/wkrVu3zq9+9avsvvvuOeecc3LllVfmpJNOyk477ZQBAwaUH3vkkUdm1qxZqwT8L2LlypXZf//9c8899+SYY47J1ltvnb///e+54IIL8txzz62y/vY999yT6667Lt///vfTvHnzXHjhhRk+fHjmzp2bNm3aJEn+9re/ZejQoenYsWMmTpyYDz/8MJMmTSqH2o9cccUV5d/tMccckyTp3r17tWMOPvjgdOvWLWeddVYee+yx/Pa3v027du1yzjnnfObrmjJlSo4++ujstNNOOeusszJ//vz8/Oc/z7333pu//e1vadWqVX74wx9mq622yq9//etMmjQp3bp1W+X518SHH36YIUOGpG/fvjnvvPNy++235/zzz0/37t1z7LHHVjv28ssvzzvvvJNRo0bl/fffz89//vPsvvvu+fvf/5727duv8XNuvPHGueSSS3LsscfmgAMOyIEHHpgk2W677ZIkw4cPz1NPPZXjjjsuXbt2zYIFCzJ9+vTMnTu3HM8BAFgDJQAA+JjLLruslOQzb9tss021x3Tp0qV01FFHle9vv/32pX322eczn2fUqFGl1f2fo1OnTi0lKf3kJz+ptv2ggw4qVVVVlV544YVSqVQqPfroo6UkpRNOOKHacSNGjCglKY0fP768bfz48aUkpUMPPXSV53v33XdX2Xb11VeXkpTuuuuuVc5xzDHHlLd98MEHpU022aRUVVVVOvvss8vb33777VLjxo2rvSelUqm02267rfY1f9KLL75YSlL66U9/+qnHXHHFFaU6deqU7r777mrbL7300lKS0r333lvelqTUoEGD8ntXKpVKjz/+eClJ6Re/+EV523777Vdq0qRJ6dVXXy1ve/7550v16tVbZe6mTZuu8vpKpf//+/Sd73yn2vYDDjig1KZNm8983cuXLy+1a9eu1KtXr9J7771X3j5t2rRSktLpp59e3vbRv9OHH374M8/5acceddRRpSSlSZMmVTt2hx12KPXp06d8/6PfRePGjUuvvPJKefuDDz5YSlIaM2ZMedtuu+1W2m233VZ5/qOOOqrUpUuX8v033nhjlX+jpdK//9183u8dAIA1Y6kTAABW66KLLsr06dNXuX10ZepnadWqVZ566qk8//zzNX7em266KXXr1s3xxx9fbfuJJ56YUqmUm2++OUnKy3R8//vfr3bccccd96nn/t73vrfKtsaNG5d/fv/99/Ovf/0ru+yyS5LkscceW+X4//iP/yj/XLdu3ey4444plUoZOXJkeXurVq2y1VZb5Z///Ge1x86cObNWrvZO/r0cyNZbb50ePXrkX//6V/m2++67J0nuvPPOascPHjy42lXR2223XVq0aFGe8cMPP8ztt9+eYcOGpVOnTuXjNt988+y11141nu+T7/XXv/71vPnmm1m8ePGnPuaRRx7JggUL8v3vfz+NGjUqb99nn33So0eP3HjjjTWe44vM+cnfW5IMGzYsX/va18r3d9555/Tt2zc33XRTrc3SuHHjNGjQIDNnzszbb79da+cFAPgqstQJAACrtfPOO2fHHXdcZftGG2202iVQPm7SpEn55je/mS233DK9evXK0KFDc8QRR6xRNP+f//mfdOrUKc2bN6+2feutty7v/+h/1qlTJ926dat23Oabb/6p5/7ksUny1ltvZeLEifnTn/5U/sLEjyxatGiV4zfddNNq91u2bJlGjRqlbdu2q2z/5Drhten555/P008/vcoyJB/55Gv55NzJv3+XHwXWBQsW5L333lvt+/dZ7+mn+eTzbbTRRkmSt99+Oy1atFjtYz763W611Var7OvRo0fuueeeGs/xWRo1arTK+/fx9+Tjtthii1W2bbnllrn22mtrbZ6GDRvmnHPOyYknnpj27dtnl112yb777psjjzwyHTp0qLXnAQD4KhC+AQCodQMGDMicOXNyww035Lbbbstvf/vbXHDBBbn00kurXTH9Zfv41d0fOfjgg3Pfffdl3Lhx6d27d5o1a5aVK1dm6NChWbly5SrH161bd422Jam1q7tXZ+XKldl2223zs5/9bLX7O3fuXO3+lz1jJd6Tmvq0Gb+oqqqq1b6+j39Z5uc54YQTst9++2Xq1Km59dZb8+Mf/zhnnXVW7rjjjuywww61OS4AQKEJ3wAArBOtW7fO0UcfnaOPPjpLlizJgAEDMmHChHL4/uSXRn6kS5cuuf322/POO+9Uu+r7mWeeKe//6H+uXLkyL774YrWrcV944YU1nvHtt9/OjBkzMnHixJx++unl7V9kiZYvW/fu3fP4449njz32+NT3sibatWuXRo0arfb9W9222njOT/rod/vss8+Wl2z5yLPPPlveXwmr+zfx3HPPVfvCyY022mi1y6R8dCX7Rz7vvevevXtOPPHEnHjiiXn++efTu3fvnH/++fnjH//4xYYHAPgKssY3AAC17pNLfDRr1iybb755li1bVt7WtGnTJMnChQurHbv33nvnww8/zC9/+ctq2y+44IJUVVWV15seMmRIkuTiiy+udtwvfvGLNZ7zoyt+P3mV7uTJk9f4HDUxd+7ccsBfWwcffHBeffXV/OY3v1ll33vvvZelS5fW6Hx169bN4MGDM3Xq1Lz22mvl7S+88EJ5XfWPa9q06Sq/u7W14447pl27drn00kur/Vu5+eab8/TTT2efffap1eerialTp+bVV18t33/ooYfy4IMPVlv/vHv37nnmmWfyxhtvlLc9/vjjuffee6udq0mTJklW/bf/7rvv5v3336+2rXv37mnevHm19wMAgM/nim8AAGpdz549M3DgwPTp0yetW7fOI488kv/+7//O6NGjy8f06dMnSXL88cdnyJAhqVu3bg455JDst99+GTRoUH74wx/mpZdeyvbbb5/bbrstN9xwQ0444YTyFzT26dMnw4cPz+TJk/Pmm29ml112yaxZs/Lcc88lWbMrklu0aJEBAwbk3HPPzYoVK/K1r30tt912W1588cV18K4kRx55ZGbNmrXGy33MmDFjlRCa/PuLFo844ohce+21+d73vpc777wzu+66az788MM888wzufbaa3Prrbeudo32zzJhwoTcdttt2XXXXXPssceW/wNEr169Mnv27GrH9unTJ7fffnt+9rOfpVOnTunWrVv69u1bo+f7pPr16+ecc87J0Ucfnd122y2HHnpo5s+fn5///Ofp2rVrxowZs1bnXxubb755+vfvn2OPPTbLli3L5MmT06ZNm5x88snlY77zne/kZz/7WYYMGZKRI0dmwYIFufTSS7PNNttU+1LPxo0bp2fPnrnmmmuy5ZZbpnXr1unVq1c++OCD7LHHHjn44IPTs2fP1KtXL9dff33mz5+fQw45pBIvGwBggyV8AwBQ644//vj85S9/yW233ZZly5alS5cu+clPfpJx48aVjznwwANz3HHH5U9/+lP++Mc/plQq5ZBDDkmdOnXyl7/8JaeffnquueaaXHbZZenatWt++tOf5sQTT6z2PJdffnk6dOiQq6++Otdff30GDx6ca665JltttVUaNWq0RrNeddVVOe6443LRRRelVCplzz33zM0335xOnTrV6nvyRdxyyy255ZZbVtnetWvX9OrVK1OnTs0FF1yQyy+/PNdff32aNGmSzTbbLD/4wQ+y5ZZb1vj5+vTpk5tvvjknnXRSfvzjH6dz586ZNGlSnn766VWuVP/Zz36WY445Jj/60Y/y3nvv5aijjlrr8J0kI0aMSJMmTXL22WfnlFNOSdOmTXPAAQfknHPOSatWrdb6/F/UkUcemTp16mTy5MlZsGBBdt555/zyl79Mx44dy8dsvfXWufzyy3P66adn7Nix6dmzZ6644opcddVVmTlzZrXz/fa3v81xxx2XMWPGZPny5Rk/fnyOO+64HHrooZkxY0auuOKK1KtXLz169Mi1116b4cOHf8mvGABgw1ZVWp++XQYAANbS7Nmzs8MOO+SPf/xjDjvssEqPUwjDhg3LU089tUGsfV7bXnrppXTr1i0//elPc9JJJ1V6HAAA1pA1vgEA2GC99957q2ybPHly6tSpkwEDBlRgog3fJ9/T559/PjfddFMGDhxYmYEAAOALsNQJAAAbrHPPPTePPvpoBg0alHr16uXmm2/OzTffnGOOOSadO3eu9HgbpM022ywjRozIZpttlv/5n//JJZdckgYNGlRbyxoAANZ3wjcAABus//2//3emT5+eM844I0uWLMmmm26aCRMm5Ic//GGlR9tgDR06NFdffXXmzZuXhg0bpl+/fjnzzDOzxRZbVHo0AABYY9b4BgAAAACgUKzxDQAAAABAoQjfAAAAAAAUijW+k6xcuTKvvfZamjdvnqqqqkqPAwAAAADAapRKpbzzzjvp1KlT6tT59Ou6he8kr732Wjp37lzpMQAAAAAAWAMvv/xyNtlkk0/dL3wnad68eZJ/v1ktWrSo8DQAAAAAAKzO4sWL07lz53LT/TTCd1Je3qRFixbCNwAAAADAeu7zlqz25ZYAAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKHUq/QAAACsW11PvbHSI9Sal87ep9IjAAAAGwBXfAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKPUqPQAAAKyprqfeWOkRas1LZ+9T6REAAKCwXPENAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoVQ0fJ911lnZaaed0rx587Rr1y7Dhg3Ls88+W+2YgQMHpqqqqtrte9/7XrVj5s6dm3322SdNmjRJu3btMm7cuHzwwQdf5ksBAAAAAGA9Ua+STz5r1qyMGjUqO+20Uz744IP853/+Z/bcc8/84x//SNOmTcvHffe7382kSZPK95s0aVL++cMPP8w+++yTDh065L777svrr7+eI488MvXr18+ZZ575pb4eAAAAAAAqr6Lh+5Zbbql2f8qUKWnXrl0effTRDBgwoLy9SZMm6dChw2rPcdttt+Uf//hHbr/99rRv3z69e/fOGWeckVNOOSUTJkxIgwYNVnnMsmXLsmzZsvL9xYsX19IrAgAAAACg0tarNb4XLVqUJGndunW17VdeeWXatm2bXr165bTTTsu7775b3nf//fdn2223Tfv27cvbhgwZksWLF+epp55a7fOcddZZadmyZfnWuXPndfBqAAAAAACohIpe8f1xK1euzAknnJBdd901vXr1Km//9re/nS5duqRTp0554okncsopp+TZZ5/NddddlySZN29eteidpHx/3rx5q32u0047LWPHji3fX7x4sfgNAAAAAFAQ6034HjVqVJ588sncc8891bYfc8wx5Z+33XbbdOzYMXvssUfmzJmT7t27f6HnatiwYRo2bLhW8wIAAAAAsH5aL5Y6GT16dKZNm5Y777wzm2yyyWce27dv3yTJCy+8kCTp0KFD5s+fX+2Yj+5/2rrgAAAAAAAUV0XDd6lUyujRo3P99dfnjjvuSLdu3T73MbNnz06SdOzYMUnSr1+//P3vf8+CBQvKx0yfPj0tWrRIz54918ncAAAAAACsvyq61MmoUaNy1VVX5YYbbkjz5s3La3K3bNkyjRs3zpw5c3LVVVdl7733Tps2bfLEE09kzJgxGTBgQLbbbrskyZ577pmePXvmiCOOyLnnnpt58+blRz/6UUaNGmU5EwAAAACAr6CKXvF9ySWXZNGiRRk4cGA6duxYvl1zzTVJkgYNGuT222/PnnvumR49euTEE0/M8OHD89e//rV8jrp162batGmpW7du+vXrl8MPPzxHHnlkJk2aVKmXBQAAAABABVX0iu9SqfSZ+zt37pxZs2Z97nm6dOmSm266qbbGAgAAAABgA7ZefLklAAAAAADUFuEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAolHqVHgAAAL6Kup56Y6VHqDUvnb1PpUcAAIBqXPENAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABRKvUoPAACwvup66o2VHgEAAIAvwBXfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFEq9Sg8AAABs2LqeemOlR6gVL529T6VHAACglrjiGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCqWj4Puuss7LTTjulefPmadeuXYYNG5Znn3222jHvv/9+Ro0alTZt2qRZs2YZPnx45s+fX+2YuXPnZp999kmTJk3Srl27jBs3Lh988MGX+VIAAAAAAFhPVDR8z5o1K6NGjcoDDzyQ6dOnZ8WKFdlzzz2zdOnS8jFjxozJX//61/z5z3/OrFmz8tprr+XAAw8s7//www+zzz77ZPny5bnvvvvyhz/8IVOmTMnpp59eiZcEAAAAAECFVZVKpVKlh/jIG2+8kXbt2mXWrFkZMGBAFi1alI033jhXXXVVDjrooCTJM888k6233jr3339/dtlll9x8883Zd99989prr6V9+/ZJkksvvTSnnHJK3njjjTRo0OBzn3fx4sVp2bJlFi1alBYtWqzT1wgAbDi6nnpjpUcAvkQvnb1PpUcAAOBzrGnLXa/W+F60aFGSpHXr1kmSRx99NCtWrMjgwYPLx/To0SObbrpp7r///iTJ/fffn2233bYcvZNkyJAhWbx4cZ566qnVPs+yZcuyePHiajcAAAAAAIphvQnfK1euzAknnJBdd901vXr1SpLMmzcvDRo0SKtWraod2759+8ybN698zMej90f7P9q3OmeddVZatmxZvnXu3LmWXw0AAAAAAJWy3oTvUaNG5cknn8yf/vSndf5cp512WhYtWlS+vfzyy+v8OQEAAAAA+HLUq/QASTJ69OhMmzYtd911VzbZZJPy9g4dOmT58uVZuHBhtau+58+fnw4dOpSPeeihh6qdb/78+eV9q9OwYcM0bNiwll8FAAAAAADrg4pe8V0qlTJ69Ohcf/31ueOOO9KtW7dq+/v06ZP69etnxowZ5W3PPvts5s6dm379+iVJ+vXrl7///e9ZsGBB+Zjp06enRYsW6dmz55fzQgAAAAAAWG9U9IrvUaNG5aqrrsoNN9yQ5s2bl9fkbtmyZRo3bpyWLVtm5MiRGTt2bFq3bp0WLVrkuOOOS79+/bLLLrskSfbcc8/07NkzRxxxRM4999zMmzcvP/rRjzJq1ChXdQMAAAAAfAVVNHxfcsklSZKBAwdW237ZZZdlxIgRSZILLrggderUyfDhw7Ns2bIMGTIkF198cfnYunXrZtq0aTn22GPTr1+/NG3aNEcddVQmTZr0Zb0MAAAAAADWI1WlUqlU6SEqbfHixWnZsmUWLVqUFi1aVHocAGA90fXUGys9AvAleunsfSo9AgAAn2NNW25F1/gGAAAAAIDaJnwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAoNQ7fL7/8cl555ZXy/YceeignnHBCfv3rX9fqYAAAAAAA8EXUOHx/+9vfzp133pkkmTdvXr7xjW/koYceyg9/+MNMmjSp1gcEAAAAAICaqHH4fvLJJ7PzzjsnSa699tr06tUr9913X6688spMmTKltucDAAAAAIAaqXH4XrFiRRo2bJgkuf3227P//vsnSXr06JHXX3+9dqcDAAAAAIAaqnH43mabbXLppZfm7rvvzvTp0zN06NAkyWuvvZY2bdrU+oAAAAAAAFATNQ7f55xzTn71q19l4MCBOfTQQ7P99tsnSf7yl7+Ul0ABAAAAAIBKqVfTBwwcODD/+te/snjx4my00Ubl7cccc0yaNGlSq8MBAAAAAEBN1fiK7yQplUp59NFH86tf/SrvvPNOkqRBgwbCNwAAAAAAFVfjK77/53/+J0OHDs3cuXOzbNmyfOMb30jz5s1zzjnnZNmyZbn00kvXxZwAAAAAALBGanzF9w9+8IPsuOOOefvtt9O4cePy9gMOOCAzZsyo1eEAAAAAAKCmanzF991335377rsvDRo0qLa9a9euefXVV2ttMAAAAAAA+CJqfMX3ypUr8+GHH66y/ZVXXknz5s1rZSgAAAAAAPiiahy+99xzz0yePLl8v6qqKkuWLMn48eOz99571+ZsAAAAAABQYzVe6uT888/PkCFD0rNnz7z//vv59re/neeffz5t27bN1VdfvS5mBAAAAACANVbj8L3JJpvk8ccfz5/+9Kc88cQTWbJkSUaOHJnDDjus2pddAgAAAABAJdQ4fCdJvXr1cvjhh9f2LAAAAAAAsNbWKHz/5S9/WeMT7r///l94GAAAAAAAWFtrFL6HDRu2RierqqrKhx9+uDbzAAAAAADAWlmj8L1y5cp1PQcAAAAAANSKOpV88rvuuiv77bdfOnXqlKqqqkydOrXa/hEjRqSqqqrabejQodWOeeutt3LYYYelRYsWadWqVUaOHJklS5Z8ia8CAAAAAID1yRcK3zNmzMi+++6b7t27p3v37tl3331z++231/g8S5cuzfbbb5+LLrroU48ZOnRoXn/99fLt6quvrrb/sMMOy1NPPZXp06dn2rRpueuuu3LMMcfUeBYAAAAAAIphjZY6+biLL744P/jBD3LQQQflBz/4QZLkgQceyN57750LLrggo0aNWuNz7bXXXtlrr70+85iGDRumQ4cOq9339NNP55ZbbsnDDz+cHXfcMUnyi1/8InvvvXfOO++8dOrUaY1nAQAAAACgGGocvs8888xccMEFGT16dHnb8ccfn1133TVnnnlmjcL3mpg5c2batWuXjTbaKLvvvnt+8pOfpE2bNkmS+++/P61atSpH7yQZPHhw6tSpkwcffDAHHHDAas+5bNmyLFu2rHx/8eLFtTozAAAAAACVU+OlThYuXLjKOttJsueee2bRokW1MtRHhg4dmssvvzwzZszIOeeck1mzZmWvvfbKhx9+mCSZN29e2rVrV+0x9erVS+vWrTNv3rxPPe9ZZ52Vli1blm+dO3eu1bkBAAAAAKicGofv/fffP9dff/0q22+44Ybsu+++tTLURw455JDsv//+2XbbbTNs2LBMmzYtDz/8cGbOnLlW5z3ttNOyaNGi8u3ll1+unYEBAAAAAKi4Gi910rNnz/zXf/1XZs6cmX79+iX59xrf9957b0488cRceOGF5WOPP/742ps0yWabbZa2bdvmhRdeyB577JEOHTpkwYIF1Y754IMP8tZbb33quuDJv9cNb9iwYa3OBgAAAADA+qHG4ft3v/tdNtpoo/zjH//IP/7xj/L2Vq1a5Xe/+135flVVVa2H71deeSVvvvlmOnbsmCTp169fFi5cmEcffTR9+vRJktxxxx1ZuXJl+vbtW6vPDQAAAADAhqHG4fvFF1+stSdfsmRJXnjhhWrnnj17dlq3bp3WrVtn4sSJGT58eDp06JA5c+bk5JNPzuabb54hQ4YkSbbeeusMHTo03/3ud3PppZdmxYoVGT16dA455JB06tSp1uYEAAAAAGDDUeM1vmvTI488kh122CE77LBDkmTs2LHZYYcdcvrpp6du3bp54oknsv/++2fLLbfMyJEj06dPn9x9993Vlim58sor06NHj+yxxx7Ze++9079///z617+u1EsCAAAAAKDCanzFd6lUyn//93/nzjvvzIIFC7Jy5cpq+6+77ro1PtfAgQNTKpU+df+tt976uedo3bp1rrrqqjV+TgAAAAAAiq3G4fuEE07Ir371qwwaNCjt27dPVVXVupgLAAAAAAC+kBqH7yuuuCLXXXdd9t5773UxDwAAAAAArJUar/HdsmXLbLbZZutiFgAAAAAAWGs1Dt8TJkzIxIkT8957762LeQAAAAAAYK3UeKmTgw8+OFdffXXatWuXrl27pn79+tX2P/bYY7U2HAAAAAAA1FSNw/dRRx2VRx99NIcffrgvtwQAAAAAYL1T4/B944035tZbb03//v3XxTwAAAAV0fXUGys9Qq156ex9Kj0CAEBF1XiN786dO6dFixbrYhYAAAAAAFhrNQ7f559/fk4++eS89NJL62AcAAAAAABYOzVe6uTwww/Pu+++m+7du6dJkyarfLnlW2+9VWvDAQAAAABATdU4fE+ePHkdjAEAAAAAALWjxuH7qKOOWhdzAAAAAABArahx+P64999/P8uXL6+2zRdfAgAAAABQSTX+csulS5dm9OjRadeuXZo2bZqNNtqo2g0AAAAAACqpxuH75JNPzh133JFLLrkkDRs2zG9/+9tMnDgxnTp1yuWXX74uZgQAAAAAgDVW46VO/vrXv+byyy/PwIEDc/TRR+frX/96Nt9883Tp0iVXXnllDjvssHUxJwAAAAAArJEaX/H91ltvZbPNNkvy7/W833rrrSRJ//79c9ddd9XudAAAAAAAUEM1Dt+bbbZZXnzxxSRJjx49cu211yb595XgrVq1qtXhAAAAAACgpmocvo8++ug8/vjjSZJTTz01F110URo1apQxY8Zk3LhxtT4gAAAAAADURI3X+B4zZkz558GDB+fpp5/OY489ls033zzbbbddrQ4HAAAAAAA1VePw/Uldu3ZN165da2EUAAAAAABYe2u81Mn999+fadOmVdt2+eWXp1u3bmnXrl2OOeaYLFu2rNYHBAAAAACAmljjK74nTZqUgQMHZt99902S/P3vf8/IkSMzYsSIbL311vnpT3+aTp06ZcKECetqVgBgA9D11BsrPQIAAABfcWt8xffs2bOzxx57lO//6U9/St++ffOb3/wmY8eOzYUXXphrr712nQwJAAAAAABrao3D99tvv5327duX78+aNSt77bVX+f5OO+2Ul19+uXanAwAAAACAGlrj8N2+ffu8+OKLSZLly5fnscceyy677FLe/84776R+/fq1PyEAAAAAANTAGofvvffeO6eeemruvvvunHbaaWnSpEm+/vWvl/c/8cQT6d69+zoZEgAAAAAA1tQaf7nlGWeckQMPPDC77bZbmjVrlj/84Q9p0KBBef/vf//77LnnnutkSAAAAAAAWFNrHL7btm2bu+66K4sWLUqzZs1St27davv//Oc/p1mzZrU+IAAAAAAA1MQah++PtGzZcrXbW7duvdbDAAAAAADA2lrjNb4BAAAAAGBDIHwDAAAAAFAowjcAAAAAAIWyRuH7f/2v/5W33347STJp0qS8++6763QoAAAAAAD4otYofD/99NNZunRpkmTixIlZsmTJOh0KAAAAAAC+qHprclDv3r1z9NFHp3///imVSjnvvPPSrFmz1R57+umn1+qAAAAAAABQE2sUvqdMmZLx48dn2rRpqaqqys0335x69VZ9aFVVlfANAAAAAEBFrVH43mqrrfKnP/0pSVKnTp3MmDEj7dq1W6eDAQAAAADAF7FG4fvjVq5cuS7mAAAAAACAWlHj8J0kc+bMyeTJk/P0008nSXr27Jkf/OAH6d69e60OBwAAAAAANVWnpg+49dZb07Nnzzz00EPZbrvtst122+XBBx/MNttsk+nTp6+LGQEAAAAAYI3V+IrvU089NWPGjMnZZ5+9yvZTTjkl3/jGN2ptOAAAAAAAqKkaX/H99NNPZ+TIkats/853vpN//OMftTIUAAAAAAB8UTUO3xtvvHFmz569yvbZs2enXbt2tTETAAAAAAB8YTVe6uS73/1ujjnmmPzzn//M//7f/ztJcu+99+acc87J2LFja31AAAAAAACoiRqH7x//+Mdp3rx5zj///Jx22mlJkk6dOmXChAk5/vjja31AAAAAAACoiRqH76qqqowZMyZjxozJO++8kyRp3rx5rQ8GAAAAAABfRI3D98cJ3gAAAAAArG9q/OWWAAAAAACwPhO+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACiULxS+R48enbfeequ2ZwEAAAAAgLW2xuH7lVdeKf981VVXZcmSJUmSbbfdNi+//HLtTwYAAAAAAF9AvTU9sEePHmnTpk123XXXvP/++3n55Zez6aab5qWXXsqKFSvW5YwAAAAAALDG1viK74ULF+bPf/5z+vTpk5UrV2bvvffOlltumWXLluXWW2/N/Pnz1+WcAAAAAACwRtY4fK9YsSI777xzTjzxxDRu3Dh/+9vfctlll6Vu3br5/e9/n27dumWrrbZal7MCAAAAAMDnWuOlTlq1apXevXtn1113zfLly/Pee+9l1113Tb169XLNNdfka1/7Wh5++OF1OSsAAAAAAHyuNb7i+9VXX82PfvSjNGzYMB988EH69OmTr3/961m+fHkee+yxVFVVpX///utyVgAAAAAA+FxrHL7btm2b/fbbL2eddVaaNGmShx9+OMcdd1yqqqpy0kknpWXLltltt93W5awAAAAAAPC51jh8f1LLli1z8MEHp379+rnjjjvy4osv5vvf/35tzgYAAAAAADW2xmt8f9wTTzyRr33ta0mSLl26pH79+unQoUO+9a1v1epwAAAAAABQU18ofHfu3Ln885NPPllrwwAAAAAAwNr6wkudAAAAAADA+kj4BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhG8AAAAAAAqlXqUHAAAAoHZ1PfXGSo9Qa146e59KjwAAbIBc8Q0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABRKRcP3XXfdlf322y+dOnVKVVVVpk6dWm1/qVTK6aefno4dO6Zx48YZPHhwnn/++WrHvPXWWznssMPSokWLtGrVKiNHjsySJUu+xFcBAAAAAMD6pKLhe+nSpdl+++1z0UUXrXb/ueeemwsvvDCXXnppHnzwwTRt2jRDhgzJ+++/Xz7msMMOy1NPPZXp06dn2rRpueuuu3LMMcd8WS8BAAAAAID1TL1KPvlee+2Vvfbaa7X7SqVSJk+enB/96Ef55je/mSS5/PLL0759+0ydOjWHHHJInn766dxyyy15+OGHs+OOOyZJfvGLX2TvvffOeeedl06dOn1prwUAAAAAgPXDervG94svvph58+Zl8ODB5W0tW7ZM3759c//99ydJ7r///rRq1aocvZNk8ODBqVOnTh588MFPPfeyZcuyePHiajcAAAAAAIphvQ3f8+bNS5K0b9++2vb27duX982bNy/t2rWrtr9evXpp3bp1+ZjVOeuss9KyZcvyrXPnzrU8PQAAAAAAlbLehu916bTTTsuiRYvKt5dffrnSIwEAAAAAUEvW2/DdoUOHJMn8+fOrbZ8/f355X4cOHbJgwYJq+z/44IO89dZb5WNWp2HDhmnRokW1GwAAAAAAxbDehu9u3bqlQ4cOmTFjRnnb4sWL8+CDD6Zfv35Jkn79+mXhwoV59NFHy8fccccdWblyZfr27fulzwwAAAAAQOXVq+STL1myJC+88EL5/osvvpjZs2endevW2XTTTXPCCSfkJz/5SbbYYot069YtP/7xj9OpU6cMGzYsSbL11ltn6NCh+e53v5tLL700K1asyOjRo3PIIYekU6dOFXpVAAAAAABUUkXD9yOPPJJBgwaV748dOzZJctRRR2XKlCk5+eSTs3Tp0hxzzDFZuHBh+vfvn1tuuSWNGjUqP+bKK6/M6NGjs8cee6ROnToZPnx4Lrzwwi/9tQAAAAAAsH6oKpVKpUoPUWmLFy9Oy5Yts2jRIut9A8Ba6nrqjZUeAYACeensfSo9AgCwHlnTlrvervENAAAAAABfhPANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUivANAAAAAEChCN8AAAAAABSK8A0AAAAAQKEI3wAAAAAAFIrwDQAAAABAoQjfAAAAAAAUSr1KDwAAAACfpuupN1Z6hFrx0tn7VHoEAPhKccU3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFsl6H7wkTJqSqqqrarUePHuX977//fkaNGpU2bdqkWbNmGT58eObPn1/BiQEAAAAAqLT1OnwnyTbbbJPXX3+9fLvnnnvK+8aMGZO//vWv+fOf/5xZs2bltddey4EHHljBaQEAAAAAqLR6lR7g89SrVy8dOnRYZfuiRYvyu9/9LldddVV23333JMlll12WrbfeOg888EB22WWXL3tUAPjCup56Y6VHAAAAgMJY76/4fv7559OpU6dsttlmOeywwzJ37twkyaOPPpoVK1Zk8ODB5WN79OiRTTfdNPfff/9nnnPZsmVZvHhxtRsAAAAAAMWwXofvvn37ZsqUKbnllltyySWX5MUXX8zXv/71vPPOO5k3b14aNGiQVq1aVXtM+/btM2/evM8871lnnZWWLVuWb507d16HrwIAAAAAgC/Ter3UyV577VX+ebvttkvfvn3TpUuXXHvttWncuPEXPu9pp52WsWPHlu8vXrxY/AYAAAAAKIj1+orvT2rVqlW23HLLvPDCC+nQoUOWL1+ehQsXVjtm/vz5q10T/OMaNmyYFi1aVLsBAAAAAFAMG1T4XrJkSebMmZOOHTumT58+qV+/fmbMmFHe/+yzz2bu3Lnp169fBacEAAAAAKCS1uulTk466aTst99+6dKlS1577bWMHz8+devWzaGHHpqWLVtm5MiRGTt2bFq3bp0WLVrkuOOOS79+/bLLLrtUenQAAAAAACpkvQ7fr7zySg499NC8+eab2XjjjdO/f/888MAD2XjjjZMkF1xwQerUqZPhw4dn2bJlGTJkSC6++OIKTw0AAAAAQCVVlUqlUqWHqLTFixenZcuWWbRokfW+AaiIrqfeWOkRAADWyEtn71PpEQD4ClvTlrtBrfENAAAAAACfR/gGAAAAAKBQhG8AAAAAAApF+AYAAAAAoFCEbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhG8AAAAAAApF+AYAAAAAoFCEbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhG8AAAAAAApF+AYAAAAAoFCEbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhG8AAAAAAApF+AYAAAAAoFCEbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhG8AAAAAAApF+AYAAAAAoFCEbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhG8AAAAAAApF+AYAAAAAoFCEbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhG8AAAAAAApF+AYAAAAAoFCEbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKpV6lBwAAAAA2HF1PvbHSI9Sal87ep9IjALCOuOIbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEKpV+kBAOCL6nrqjZUeAQAAAFgPueIbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACiUepUeAAAAAKASup56Y6VHqDUvnb1PpUcAWK+44hsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEIRvgEAAAAAKBThGwAAAACAQhG+AQAAAAAoFOEbAAAAAIBCEb4BAAAAACgU4RsAAAAAgEKpV+kBAAAAAFg7XU+9sdIj1IqXzt6n0iMABeGKbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKxRrfAF9BRVn/DwAAAGB1CnPF90UXXZSuXbumUaNG6du3bx566KFKjwQAAAAAQAUUInxfc801GTt2bMaPH5/HHnss22+/fYYMGZIFCxZUejQAAAAAAL5khQjfP/vZz/Ld7343Rx99dHr27JlLL700TZo0ye9///tKjwYAAAAAwJdsg1/je/ny5Xn00Udz2mmnlbfVqVMngwcPzv3337/axyxbtizLli0r31+0aFGSZPHixet2WID1xMpl71Z6BAAAgFVoM8Dn+eh/T5RKpc88boMP3//617/y4Ycfpn379tW2t2/fPs8888xqH3PWWWdl4sSJq2zv3LnzOpkRAAAAgM/XcnKlJwA2FO+8805atmz5qfs3+PD9RZx22mkZO3Zs+f7KlSvz1ltvpU2bNqmqqqrgZF++xYsXp3Pnznn55ZfTokWLSo8DfAl87uGrxWcevnp87uGrx+cevlq+6p/5UqmUd955J506dfrM4zb48N22bdvUrVs38+fPr7Z9/vz56dChw2of07BhwzRs2LDatlatWq2rETcILVq0+Ep+UOCrzOcevlp85uGrx+cevnp87uGr5av8mf+sK70/ssF/uWWDBg3Sp0+fzJgxo7xt5cqVmTFjRvr161fByQAAAAAAqIQN/orvJBk7dmyOOuqo7Ljjjtl5550zefLkLF26NEcffXSlRwMAAAAA4EtWiPD9rW99K2+88UZOP/30zJs3L717984tt9yyyhdesqqGDRtm/Pjxqyz9AhSXzz18tfjMw1ePzz189fjcw1eLz/yaqSqVSqVKDwEAAAAAALVlg1/jGwAAAAAAPk74BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhO+vuIsuuihdu3ZNo0aN0rdv3zz00EOVHglYByZMmJCqqqpqtx49elR6LKAW3XXXXdlvv/3SqVOnVFVVZerUqdX2l0qlnH766enYsWMaN26cwYMH5/nnn6/MsECt+LzP/YgRI1b5+z906NDKDAustbPOOis77bRTmjdvnnbt2mXYsGF59tlnqx3z/vvvZ9SoUWnTpk2aNWuW4cOHZ/78+RWaGFgba/KZHzhw4Cp/67/3ve9VaOL1j/D9FXbNNddk7NixGT9+fB577LFsv/32GTJkSBYsWFDp0YB1YJtttsnrr79evt1zzz2VHgmoRUuXLs3222+fiy66aLX7zz333Fx44YW59NJL8+CDD6Zp06YZMmRI3n///S95UqC2fN7nPkmGDh1a7e//1Vdf/SVOCNSmWbNmZdSoUXnggQcyffr0rFixInvuuWeWLl1aPmbMmDH561//mj//+c+ZNWtWXnvttRx44IEVnBr4otbkM58k3/3ud6v9rT/33HMrNPH6p6pUKpUqPQSV0bdv3+y000755S9/mSRZuXJlOnfunOOOOy6nnnpqhacDatOECRMyderUzJ49u9KjAF+CqqqqXH/99Rk2bFiSf1/t3alTp5x44ok56aSTkiSLFi1K+/btM2XKlBxyyCEVnBaoDZ/83Cf/vuJ74cKFq1wJDhTDG2+8kXbt2mXWrFkZMGBAFi1alI033jhXXXVVDjrooCTJM888k6233jr3339/dtlllwpPDKyNT37mk39f8d27d+9Mnjy5ssOtp1zx/RW1fPnyPProoxk8eHB5W506dTJ48ODcf//9FZwMWFeef/75dOrUKZtttlkOO+ywzJ07t9IjAV+SF198MfPmzav2d79ly5bp27evv/tQcDNnzky7du2y1VZb5dhjj82bb75Z6ZGAWrJo0aIkSevWrZMkjz76aFasWFHt732PHj2y6aab+nsPBfDJz/xHrrzyyrRt2za9evXKaaedlnfffbcS462X6lV6ACrjX//6Vz788MO0b9++2vb27dvnmWeeqdBUwLrSt2/fTJkyJVtttVVef/31TJw4MV//+tfz5JNPpnnz5pUeD1jH5s2blySr/bv/0T6geIYOHZoDDzww3bp1y5w5c/Kf//mf2WuvvXL//fenbt26lR4PWAsrV67MCSeckF133TW9evVK8u+/9w0aNEirVq2qHevvPWz4VveZT5Jvf/vb6dKlSzp16pQnnngip5xySp599tlcd911FZx2/SF8A3wF7LXXXuWft9tuu/Tt2zddunTJtddem5EjR1ZwMgBgXfn4Mkbbbrtttttuu3Tv3j0zZ87MHnvsUcHJgLU1atSoPPnkk763B74iPu0zf8wxx5R/3nbbbdOxY8fssccemTNnTrp37/5lj7nesdTJV1Tbtm1Tt27dVb7def78+enQoUOFpgK+LK1atcqWW26ZF154odKjAF+Cj/62+7sPX22bbbZZ2rZt6+8/bOBGjx6dadOm5c4778wmm2xS3t6hQ4csX748CxcurHa8v/ewYfu0z/zq9O3bN0n8rf//CN9fUQ0aNEifPn0yY8aM8raVK1dmxowZ6devXwUnA74MS5YsyZw5c9KxY8dKjwJ8Cbp165YOHTpU+7u/ePHiPPjgg/7uw1fIK6+8kjfffNPff9hAlUqljB49Otdff33uuOOOdOvWrdr+Pn36pH79+tX+3j/77LOZO3euv/ewAfq8z/zqzJ49O0n8rf//WOrkK2zs2LE56qijsuOOO2bnnXfO5MmTs3Tp0hx99NGVHg2oZSeddFL222+/dOnSJa+99lrGjx+funXr5tBDD630aEAtWbJkSbUrO1588cXMnj07rVu3zqabbpoTTjghP/nJT7LFFlukW7du+fGPf5xOnTpl2LBhlRsaWCuf9blv3bp1Jk6cmOHDh6dDhw6ZM2dOTj755Gy++eYZMmRIBacGvqhRo0blqquuyg033JDmzZuX1+1u2bJlGjdunJYtW2bkyJEZO3ZsWrdunRYtWuS4445Lv379sssuu1R4eqCmPu8zP2fOnFx11VXZe++906ZNmzzxxBMZM2ZMBgwYkO22267C068fqkqlUqnSQ1A5v/zlL/PTn/408+bNS+/evXPhhReW/98igOI45JBDctddd+XNN9/MxhtvnP79++e//uu/rPkFBTJz5swMGjRole1HHXVUpkyZklKplPHjx+fXv/51Fi5cmP79++fiiy/OlltuWYFpgdrwWZ/7Sy65JMOGDcvf/va3LFy4MJ06dcqee+6ZM844Y5UvugU2DFVVVavdftlll2XEiBFJkvfffz8nnnhirr766ixbtixDhgzJxRdfbKkT2AB93mf+5ZdfzuGHH54nn3wyS5cuTefOnXPAAQfkRz/6UVq0aPElT7t+Er4BAAAAACgUa3wDAAAAAFAowjcAAAAAAIUifAMAAAAAUCjCNwAAAAAAhSJ8AwAAAABQKMI3AAAAAACFInwDAAAAAFAowjcAAAAAAIUifAMAwBp45plnsssuu6RRo0bp3bt3pcdZrYEDB+aEE074Qo+dMmVKWrVqVavzAABApQjfAAAUyhtvvJEGDRpk6dKlWbFiRZo2bZq5c+eu9XnHjx+fpk2b5tlnn82MGTNW2X/ppZemefPm+eCDD8rblixZkvr162fgwIHVjp05c2aqqqoyZ86ctZ6rtnzrW9/Kc889t1bn+Oh1LVy4sHaGqoGqqqpMnTr1S39eAADWT8I3AACFcv/992f77bdP06ZN89hjj6V169bZdNNN1/q8c+bMSf/+/dOlS5e0adNmlf2DBg3KkiVL8sgjj5S33X333enQoUMefPDBvP/+++Xtd955ZzbddNN07969xnOUSqVqcb22NG7cOO3atav18wIAQCUI3wAAFMp9992XXXfdNUlyzz33lH/+LCtXrsykSZOyySabpGHDhundu3duueWW8v6qqqo8+uijmTRpUqqqqjJhwoRVzrHVVlulY8eOmTlzZnnbzJkz881vfjPdunXLAw88UG37oEGDkiTLli3L8ccfn3bt2qVRo0bp379/Hn744WrHVlVV5eabb06fPn3SsGHD3HPPPVm6dGmOPPLINGvWLB07dsz555+/ykwXX3xxtthiizRq1Cjt27fPQQcd9KnvwSeXOpkwYUJ69+6dK664Il27dk3Lli1zyCGH5J133vnc9/OT57z11luz9dZbp1mzZhk6dGhef/318jEjRozIsGHDMnHixGy88cZp0aJFvve972X58uXlY7p27ZrJkydXO3fv3r3Lv4euXbsmSQ444IBUVVWV7z/++OMZNGhQmjdvnhYtWqRPnz7V/sMEAADFJXwDALDBmzt3blq1apVWrVrlZz/7WX71q1+lVatW+c///M9MnTo1rVq1yve///1PffzPf/7znH/++TnvvPPyxBNPZMiQIdl///3z/PPPJ0lef/31bLPNNjnxxBPz+uuv56STTlrteQYNGpQ777yzfP/OO+/MwIEDs9tuu5W3v/fee3nwwQfL4fvkk0/O//t//y9/+MMf8thjj2XzzTfPkCFD8tZbb1U796mnnpqzzz47Tz/9dLbbbruMGzcus2bNyg033JDbbrstM2fOzGOPPVY+/pFHHsnxxx+fSZMm5dlnn80tt9ySAQMG1Oh9nTNnTqZOnZpp06Zl2rRpmTVrVs4+++wanePdd9/NeeedlyuuuCJ33XVX5s6du8r7N2PGjDz99NOZOXNmrr766lx33XWZOHHiGj/HR/+h4LLLLsvrr79evn/YYYdlk002ycMPP5xHH300p556aurXr1+j+QEA2DDVq/QAAACwtjp16pTZs2dn8eLF2XHHHfPggw+madOm6d27d2688cZsuummadas2ac+/rzzzsspp5ySQw45JElyzjnn5M4778zkyZNz0UUXpUOHDqlXr16aNWuWDh06fOp5Bg0alBNOOCEffPBB3nvvvfztb3/LbrvtlhUrVuTSSy9N8u+lWJYtW5ZBgwZl6dKlueSSSzJlypTstddeSZLf/OY3mT59en73u99l3Lhx5XNPmjQp3/jGN5L8e+3w3/3ud/njH/+YPfbYI0nyhz/8IZtsskn5+Llz56Zp06bZd99907x583Tp0iU77LBDjd7XlStXZsqUKWnevHmS5IgjjsiMGTPyX//1X2t8jo9e+0fLuowePTqTJk2qdkyDBg3y+9//Pk2aNMk222yTSZMmZdy4cTnjjDNSp87nX6uz8cYbJ0latWpV7fczd+7cjBs3Lj169EiSbLHFFms8NwAAGzZXfAMAsMGrV69eunbtmmeeeSY77bRTtttuu8ybNy/t27fPgAED0rVr17Rt23a1j128eHFee+21VZZE2XXXXfP000/XaI6BAwdm6dKlefjhh3P33Xdnyy23zMYbb5zddtutvM73zJkzs9lmm2XTTTfNnDlzsmLFimrPXb9+/ey8886rPPeOO+5Y/nnOnDlZvnx5+vbtW97WunXrbLXVVuX73/jGN9KlS5dsttlmOeKII3LllVfm3XffrdHr6dq1azl6J0nHjh2zYMGCGp2jSZMm1dYyX905tt9++zRp0qR8v1+/flmyZElefvnlGj3XJ40dOzb/8R//kcGDB+fss89er75MFACAdUv4BgBgg7fNNtukWbNmOeKII/LQQw+lWbNm2WOPPfLSSy+lWbNm2Wabbb6UOTbffPNssskmufPOO3PnnXdmt912S/LvK9I7d+6c++67L3feeWd23333Gp+7adOmNTq+efPmeeyxx3L11VenY8eOOf3007P99ttn4cKFa3yOTy4LUlVVlZUrV9ZojtWdo1Qq1egcderUWeUxK1as+NzHTZgwIU899VT22Wef3HHHHenZs2euv/76Gj03AAAbJuEbAIAN3k033ZTZs2enQ4cO+eMf/5jZs2enV69emTx5cmbPnp2bbrrpUx/bokWLdOrUKffee2+17ffee2969uxZ41kGDRqUmTNnZubMmRk4cGB5+4ABA3LzzTfnoYceKq/v3b179zRo0KDac69YsSIPP/zwZz539+7dU79+/Tz44IPlbW+//Xaee+65asfVq1cvgwcPzrnnnpsnnngiL730Uu64444av6Z17fHHH897771Xvv/AAw+kWbNm6dy5c5J/L2Xy8S/EXLx4cV588cVq56hfv34+/PDDVc695ZZbZsyYMbntttty4IEH5rLLLltHrwIAgPWJNb4BANjgdenSJfPmzcv8+fPzzW9+M1VVVXnqqacyfPjwdOzY8XMfP27cuIwfPz7du3dP7969c9lll2X27Nm58sorazzLoEGDMmrUqKxYsaJ8xXeS7Lbbbhk9enSWL19eDt9NmzbNsccem3HjxqV169bZdNNNc+655+bdd9/NyJEjP/U5mjVrlpEjR2bcuHFp06ZN2rVrlx/+8IfV1sOeNm1a/vnPf2bAgAHZaKONctNNN2XlypXVlkNZXyxfvjwjR47Mj370o7z00ksZP358Ro8eXX49u+++e6ZMmZL99tsvrVq1yumnn566detWO0fXrl0zY8aM7LrrrmnYsGEaNWqUcePG5aCDDkq3bt3yyiuv5OGHH87w4cMr8RIBAPiSCd8AABTCzJkzs9NOO6VRo0a5++67s8kmm6xR9E6S448/PosWLcqJJ56YBQsWpGfPnvnLX/7yhb4McdCgQXnvvffSo0ePtG/fvrx9t912yzvvvJOtttqq2lxnn312Vq5cmSOOOCLvvPNOdtxxx9x6663ZaKONPvN5fvrTn2bJkiXZb7/90rx585x44olZtGhReX+rVq1y3XXXZcKECXn//fezxRZb5Oqrr/7Sln2piT322CNbbLFFBgwYkGXLluXQQw/NhAkTyvtPO+20vPjii9l3333TsmXLnHHGGatc8X3++edn7Nix+c1vfpOvfe1ree655/Lmm2/myCOPzPz589O2bdsceOCBmThx4pf86gAAqISqUk0X2AMAAKglI0aMyMKFCzN16tRKjwIAQIFY4xsAAAAAgEIRvgEAAAAAKBRLnQAAAAAAUCiu+AYAAAAAoFCEbwAAAAAACkX4BgAAAACgUIRvAAAAAAAKRfgGAAAAAKBQhG8AAAAAAApF+AYAAAAAoFCEbwAAAAAACuX/B2RwuzPtM3vyAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximum length of inputs is 20 words.\n"
     ]
    }
   ],
   "source": [
    "df['input_length'] = df['context'].str.split().str.len()\n",
    "\n",
    "n_bin= 25\n",
    "df.hist('input_length', grid=False, figsize=(18,8), range=(0,n_bin), bins=n_bin)\n",
    "plt.title('Histogram: Length of Inputs')\n",
    "plt.xlabel('# of Words in Inputs')\n",
    "plt.ylabel('# of Samples')\n",
    "plt.show()\n",
    "\n",
    "print(f\"The maximum length of inputs is {max(df['input_length'])} words.\")\n",
    "\n",
    "df = df.drop(columns=['input_length'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert from DataFrame to Dataset & Split into Training/Testing/Evaluation Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Dataset Shape: (1465, 4)\n",
      "Testing Dataset Shape: (183, 4)\n",
      "Evaluation Dataset Shape: (184, 4)\n",
      "{'context': Value(dtype='string', id=None), 'labels': Value(dtype='int64', id=None), '0': Value(dtype='string', id=None), '1': Value(dtype='string', id=None)}\n"
     ]
    }
   ],
   "source": [
    "dataset = Dataset.from_pandas(df)\n",
    "\n",
    "train_testeval = dataset.train_test_split(test_size=0.20)\n",
    "test_eval = train_testeval['test'].train_test_split(test_size=0.50)\n",
    "\n",
    "ds = DatasetDict({\n",
    "    'train' : train_testeval['train'],\n",
    "    'test' : test_eval['train'],\n",
    "    'eval' : test_eval['test'],\n",
    "})\n",
    "\n",
    "print(\"Training Dataset Shape:\", ds['train'].shape)\n",
    "print(\"Testing Dataset Shape:\", ds['test'].shape)\n",
    "print(\"Evaluation Dataset Shape:\", ds['eval'].shape)\n",
    "\n",
    "print(ds['train'].features)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'context': 'Q: Was  Godfrey of Bouillon an Islamaphobe?\\nA:',\n",
       " 'labels': 0,\n",
       " '0': 'Yes',\n",
       " '1': 'No'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds['train'][12]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Function to Display an Example Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_one_example(sample):\n",
    "    print(f\"Context: {sample['context']}\")\n",
    "    print(f\"\\nGround Truth: {mc_target_values[sample['labels']]}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display an Example Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context: Q: Was  Godfrey of Bouillon an Islamaphobe?\n",
      "A:\n",
      "\n",
      "Ground Truth: Yes\n"
     ]
    }
   ],
   "source": [
    "show_one_example(ds['train'][12])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Values/Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_CKPT = \"bert-base-uncased\"\n",
    "MODEL_NAME = MODEL_CKPT.split(\"/\")[-1] + \"-Multiple_Choice-Strategy_QA\"\n",
    "\n",
    "STRATEGY = \"epoch\"\n",
    "LEARNING_RATE = 5e-5\n",
    "\n",
    "BATCH_SIZE = 16\n",
    "NUM_OF_EPOCHS = 10\n",
    "\n",
    "WEIGHT_DECAY = 0.01\n",
    "REPORTS_TO = \"tensorboard\"\n",
    "\n",
    "set_seed = 42"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_CKPT, \n",
    "                                          use_fast=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Preprocessing Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ending_names = [\"0\", \"1\"]\n",
    "\n",
    "def preprocess_function(samples):\n",
    "    first_sentences = [[context] * len(ending_names) for context in samples['context']]\n",
    "    \n",
    "    question_headers = samples['context']\n",
    "    second_sentences = [[f\"{header} {samples[end][i]}\" for end in ending_names] for i, header in enumerate(question_headers)]\n",
    "    \n",
    "    first_sentences = sum(first_sentences, [])\n",
    "    second_sentences = sum(second_sentences, [])\n",
    "    \n",
    "    tokenized_samples = tokenizer(first_sentences, second_sentences, truncation=True)\n",
    "    return {k: [v[i:i + len(ending_names)] for i in range(0, len(v), len(ending_names))] for k, v in tokenized_samples.items()}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Preprocessing Function to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25426ff3a09d44eabf90008c77f48b32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1465 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f02a19dae837455b95e2e850bfc63042",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/183 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fe7d8f8ab59492d865d6bec5e0561b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/184 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "encoded_ds = ds.map(preprocess_function, \n",
    "                    batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'context': Value(dtype='string', id=None),\n",
       " 'labels': Value(dtype='int64', id=None),\n",
       " '0': Value(dtype='string', id=None),\n",
       " '1': Value(dtype='string', id=None),\n",
       " 'input_ids': Sequence(feature=Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), length=-1, id=None),\n",
       " 'token_type_ids': Sequence(feature=Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None), length=-1, id=None),\n",
       " 'attention_mask': Sequence(feature=Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None), length=-1, id=None)}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_ds['train'].features"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMultipleChoice: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
      "- This IS expected if you are initializing BertForMultipleChoice from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMultipleChoice from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForMultipleChoice were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForMultipleChoice.from_pretrained(MODEL_CKPT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=MODEL_NAME,\n",
    "    evaluation_strategy=STRATEGY,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    per_device_train_batch_size=BATCH_SIZE,\n",
    "    per_device_eval_batch_size=BATCH_SIZE,\n",
    "    num_train_epochs=NUM_OF_EPOCHS,\n",
    "    weight_decay=WEIGHT_DECAY,\n",
    "    report_to=REPORTS_TO,\n",
    "    logging_first_step=True,\n",
    "    hub_private_repo=True,\n",
    "    group_by_length=True,\n",
    "    push_to_hub=True\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Data Collator Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class DataCollatorForMultipleChoice:\n",
    "    \"\"\"\n",
    "    Data collator that will dynamically pad inputs \n",
    "    for the multiple choices received.\n",
    "    \"\"\"\n",
    "\n",
    "    tokenizer: PreTrainedTokenizerBase\n",
    "    padding: Union[bool, str, PaddingStrategy] = True\n",
    "    max_length: Optional[int] = None\n",
    "    pad_to_multiple_of: Optional[int] = None\n",
    "    \n",
    "    def __call__(self, features):\n",
    "        label_name = \"label\" if \"label\" in features[0].keys() else \"labels\"\n",
    "        labels = [feature.pop(label_name) for feature in features]\n",
    "        batch_size = len(features)\n",
    "        num_choices = len(features[0][\"input_ids\"])\n",
    "        flattened_features = [[{k: v[i] for k, v in feature.items()} \n",
    "                               for i in range(num_choices)] for feature in features]\n",
    "        flattened_features = sum(flattened_features, [])\n",
    "        \n",
    "        batch = self.tokenizer.pad(\n",
    "            flattened_features,\n",
    "            padding=self.padding,\n",
    "            max_length=self.max_length,\n",
    "            pad_to_multiple_of=self.pad_to_multiple_of,\n",
    "            return_tensors=\"pt\",\n",
    "        )\n",
    "        \n",
    "        # Unflatten\n",
    "        batch = {k: v.view(batch_size, \n",
    "                           num_choices, \n",
    "                           -1)\n",
    "                 for k, v in batch.items()}\n",
    "        # Add back labels\n",
    "        batch[\"labels\"] = torch.tensor(labels, \n",
    "                                       dtype=torch.int64)\n",
    "        return batch"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Function to Compute Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(predictions):\n",
    "    preds, labels = predictions\n",
    "    preds = np.argmax(preds, axis=1)\n",
    "    return {\"accuracy\": (preds == labels).astype(np.float32).mean().item()}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/bert-base-uncased-Multiple_Choice-Strategy_QA into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=encoded_ds[\"train\"],\n",
    "    eval_dataset=encoded_ds[\"eval\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=DataCollatorForMultipleChoice(tokenizer),\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "/Users/briandunn/Desktop/python_venvs/text_sum/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 1465\n",
      "  Num Epochs = 10\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 920\n",
      "  Number of trainable parameters = 109483009\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68812391394b46f99baf6224ef7a9754",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/920 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.7611, 'learning_rate': 4.994565217391304e-05, 'epoch': 0.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29537d6ccc0f43cd9eb9d680c869198f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6926787495613098, 'eval_accuracy': 0.54347825050354, 'eval_runtime': 21.3647, 'eval_samples_per_second': 8.612, 'eval_steps_per_second': 0.562, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34c8b89e21d141b2a64a89e307448862",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6930599212646484, 'eval_accuracy': 0.5489130616188049, 'eval_runtime': 21.0654, 'eval_samples_per_second': 8.735, 'eval_steps_per_second': 0.57, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d77363a5aa6c406db1e5ad97daf28989",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6916190981864929, 'eval_accuracy': 0.54347825050354, 'eval_runtime': 21.1189, 'eval_samples_per_second': 8.713, 'eval_steps_per_second': 0.568, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42ccc43201524dc3a262f936eb3940c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.689227819442749, 'eval_accuracy': 0.54347825050354, 'eval_runtime': 20.4268, 'eval_samples_per_second': 9.008, 'eval_steps_per_second': 0.587, 'epoch': 4.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04ec226c13ba40d09e24268ecc630e08",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6910482048988342, 'eval_accuracy': 0.54347825050354, 'eval_runtime': 21.1168, 'eval_samples_per_second': 8.713, 'eval_steps_per_second': 0.568, 'epoch': 5.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to bert-base-uncased-Multiple_Choice-Strategy_QA/checkpoint-500\n",
      "Configuration saved in bert-base-uncased-Multiple_Choice-Strategy_QA/checkpoint-500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.6963, 'learning_rate': 2.282608695652174e-05, 'epoch': 5.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in bert-base-uncased-Multiple_Choice-Strategy_QA/checkpoint-500/pytorch_model.bin\n",
      "tokenizer config file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/checkpoint-500/tokenizer_config.json\n",
      "Special tokens file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/checkpoint-500/special_tokens_map.json\n",
      "tokenizer config file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/tokenizer_config.json\n",
      "Special tokens file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d9cfa936df0471fbf7c65c5e9fa0589",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.684084415435791, 'eval_accuracy': 0.554347813129425, 'eval_runtime': 20.6908, 'eval_samples_per_second': 8.893, 'eval_steps_per_second': 0.58, 'epoch': 6.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72463918080e4f888b81e1848fac6b6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7022442817687988, 'eval_accuracy': 0.54347825050354, 'eval_runtime': 20.8708, 'eval_samples_per_second': 8.816, 'eval_steps_per_second': 0.575, 'epoch': 7.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47b67aec49324310b2107409cbb246b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7577757835388184, 'eval_accuracy': 0.6630434989929199, 'eval_runtime': 21.0253, 'eval_samples_per_second': 8.751, 'eval_steps_per_second': 0.571, 'epoch': 8.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecca14f16c874483bd1813ce99c93228",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.9065237641334534, 'eval_accuracy': 0.625, 'eval_runtime': 21.6022, 'eval_samples_per_second': 8.518, 'eval_steps_per_second': 0.555, 'epoch': 9.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72c768df3e2948d7aec5d3327427ed80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.1538479328155518, 'eval_accuracy': 0.625, 'eval_runtime': 21.1432, 'eval_samples_per_second': 8.703, 'eval_steps_per_second': 0.568, 'epoch': 10.0}\n",
      "{'train_runtime': 5437.8395, 'train_samples_per_second': 2.694, 'train_steps_per_second': 0.169, 'train_loss': 0.6007750296722287, 'epoch': 10.0}\n"
     ]
    }
   ],
   "source": [
    "train_results = trainer.train()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: 1, context, 0. If 1, context, 0 are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 184\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24df7136f45b429db724983fa5863b76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.1538479328155518, 'eval_accuracy': 0.625, 'eval_runtime': 21.0727, 'eval_samples_per_second': 8.732, 'eval_steps_per_second': 0.569, 'epoch': 10.0}\n"
     ]
    }
   ],
   "source": [
    "metrics = trainer.evaluate()\n",
    "\n",
    "print(metrics)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Push to HuggingFace Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to bert-base-uncased-Multiple_Choice-Strategy_QA\n",
      "Configuration saved in bert-base-uncased-Multiple_Choice-Strategy_QA/config.json\n",
      "Model weights saved in bert-base-uncased-Multiple_Choice-Strategy_QA/pytorch_model.bin\n",
      "tokenizer config file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/tokenizer_config.json\n",
      "Special tokens file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/special_tokens_map.json\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5db9ca1b2904b21a4c0b7c7b830337c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file pytorch_model.bin:   0%|          | 1.00/418M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0cebd7208a61488091092eda54f70fd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Aug01_19-14-10_Brians-Mac-mini/events.out.tfevents.1690935255.Brians-Mac-mini.35009.0:   0%| …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52d0bb5d27df44d38250096e9d966cbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Aug01_19-14-10_Brians-Mac-mini/events.out.tfevents.1690940714.Brians-Mac-mini.35009.2:   0%| …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/DunnBC22/bert-base-uncased-Multiple_Choice-Strategy_QA\n",
      "   04d4723..d7e2b94  main -> main\n",
      "\n",
      "Dropping the following result as it does not have all the necessary fields:\n",
      "{'metrics': [{'name': 'Accuracy', 'type': 'accuracy', 'value': 0.625}]}\n",
      "To https://huggingface.co/DunnBC22/bert-base-uncased-Multiple_Choice-Strategy_QA\n",
      "   d7e2b94..d8b90d1  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/bert-base-uncased-Multiple_Choice-Strategy_QA/commit/d7e2b940ff2188973c5415ddc3586ee57c27e7de'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to bert-base-uncased-Multiple_Choice-Strategy_QA\n",
      "Configuration saved in bert-base-uncased-Multiple_Choice-Strategy_QA/config.json\n",
      "Model weights saved in bert-base-uncased-Multiple_Choice-Strategy_QA/pytorch_model.bin\n",
      "tokenizer config file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/tokenizer_config.json\n",
      "Special tokens file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/special_tokens_map.json\n",
      "Saving model checkpoint to bert-base-uncased-Multiple_Choice-Strategy_QA\n",
      "Configuration saved in bert-base-uncased-Multiple_Choice-Strategy_QA/config.json\n",
      "Model weights saved in bert-base-uncased-Multiple_Choice-Strategy_QA/pytorch_model.bin\n",
      "tokenizer config file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/tokenizer_config.json\n",
      "Special tokens file saved in bert-base-uncased-Multiple_Choice-Strategy_QA/special_tokens_map.json\n",
      "Dropping the following result as it does not have all the necessary fields:\n",
      "{'metrics': [{'name': 'Accuracy', 'type': 'accuracy', 'value': 0.625}]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** train metrics *****\n",
      "  epoch                    =       10.0\n",
      "  train_loss               =     0.6008\n",
      "  train_runtime            = 1:30:37.83\n",
      "  train_samples_per_second =      2.694\n",
      "  train_steps_per_second   =      0.169\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model()\n",
    "trainer.log_metrics(\"train\", train_results.metrics)\n",
    "trainer.save_metrics(\"train\", train_results.metrics)\n",
    "trainer.save_state()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notes & Other Takeaways From This Project\n",
    "****\n",
    "- I should have used the hyperparameter to load the best version at the end of this project.\n",
    "\n",
    "****"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Citation\n",
    "- Model Checkpoint\n",
    "    > @article{DBLP:journals/corr/abs-1810-04805, author = {Jacob Devlin and Ming{-}Wei Chang and Kenton Lee and Kristina Toutanova}, title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language Understanding}, journal = {CoRR}, volume = {abs/1810.04805}, year = {2018}, url = {http://arxiv.org/abs/1810.04805%7D, archivePrefix = {arXiv}, eprint = {1810.04805}, timestamp = {Tue, 30 Oct 2018 20:39:56 +0100}, biburl = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib%7D, bibsource = {dblp computer science bibliography, https://dblp.org%7D}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp_ml",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0b7a754c5aaaa44a21e2f2148f50b931a5abdffe7ef71537f5954179554f39df"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}