Causal Language Modeling/US Economic News Articles/US Economic News Articles - CLM.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## US Economical News Articles: Causal Language Modeling\n",
    "\n",
    "Dataset Source: https://www.kaggle.com/datasets/heeraldedhia/us-economic-news-articles"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Necessary Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "NOTE: Redirects are currently not supported in Windows or MacOs.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "import os, sys, random, math\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import datasets\n",
    "from datasets import ClassLabel, Dataset, DatasetDict\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForLanguageModeling\n",
    "from transformers import TrainingArguments, Trainer, set_seed\n",
    "\n",
    "!git lfs install"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display Library Versions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Python: 3.9.12\n",
      "        Pandas: 1.5.0\n",
      "      Datasets: 2.9.0\n",
      "  Transformers: 4.26.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Python:\".rjust(15), sys.version[0:6])\n",
    "print(\"Pandas:\".rjust(15), pd.__version__)\n",
    "print(\"Datasets:\".rjust(15), datasets.__version__)\n",
    "print(\"Transformers:\".rjust(15), transformers.__version__)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NEW YORK -- Yields on most certificates of dep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Wall Street Journal Online&lt;/br&gt;&lt;/br&gt;The Mo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WASHINGTON -- In an effort to achieve banking ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The statistics on the enormous costs of employ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NEW YORK -- Indecision marked the dollar's ton...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7995</th>\n",
       "      <td>Secretary of Commerce Charles W. Sawyer said y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7996</th>\n",
       "      <td>U.S. stocks inched up last week, overcoming co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7997</th>\n",
       "      <td>Ben S. Bernanke cleared a key hurdle Thursday ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7998</th>\n",
       "      <td>The White House's push to contract out many fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7999</th>\n",
       "      <td>NEW YORK. April 17-Automobile stocks put on th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     NEW YORK -- Yields on most certificates of dep...\n",
       "1     The Wall Street Journal Online</br></br>The Mo...\n",
       "2     WASHINGTON -- In an effort to achieve banking ...\n",
       "3     The statistics on the enormous costs of employ...\n",
       "4     NEW YORK -- Indecision marked the dollar's ton...\n",
       "...                                                 ...\n",
       "7995  Secretary of Commerce Charles W. Sawyer said y...\n",
       "7996  U.S. stocks inched up last week, overcoming co...\n",
       "7997  Ben S. Bernanke cleared a key hurdle Thursday ...\n",
       "7998  The White House's push to contract out many fe...\n",
       "7999  NEW YORK. April 17-Automobile stocks put on th...\n",
       "\n",
       "[8000 rows x 1 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parent_dir = \"/Users/briandunn/Desktop/nlp_ds/Causal Language Modeling/US Economic News Articles\"\n",
    "\n",
    "data_file = os.path.join(parent_dir, \"US-Economic-News.csv\")\n",
    "\n",
    "data = pd.read_csv(data_file, encoding_errors=\"ignore\", usecols=[\"text\"])\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visual: Histogram of Input Lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximum length of this dataset is approximately 863 words.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABaIAAAK9CAYAAADWuhs+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSLUlEQVR4nO3de5zVdZ0/8NdwR2BAUGdkFQeVxLuFF0jzFkksma5Ua5nrrWwTL4hpw5oXTAV1S9dCTTO0zUvZpmWTlotKFwEV0zV10UyCVcHNBBTjopzfH/vg/Biuc2C+zAw8n4/HeXjO5/v5fj7v7/ec+c7h5efxnapSqVQKAAAAAAAUpF1LFwAAAAAAwOZNEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQCwhaurq8vJJ5/c0mVs9q655prsvPPOad++ffbbb7+WLmeLU1VVlTPPPLOlywAA2GIJogEANiO33XZbqqqq8uSTT65x++GHH5699tpro+f5xS9+kUsvvXSjx9lS/OpXv8oFF1yQgw8+OJMmTcqVV165Wp9HH300VVVVTXo0l8ceeyyXXnpp5s+f36T+J598crp3795s8ze3So8HAIBNp0NLFwAAQMuaOXNm2rWrbH3CL37xi0ycOFEY3UQPP/xw2rVrl1tvvTWdOnVaY5/dd989//7v/96obezYsenevXsuvPDCQup67LHHMm7cuJx88snp1atXIXNsSpvb8QAAbE4E0QAAW7jOnTu3dAkVW7RoUbp169bSZTTZG2+8ka5du641hE6SmpqafP7zn2/UNmHChGyzzTartQMAQFvj1hwAAFu4Ve8RvWzZsowbNy4DBgxIly5d0qdPnxxyyCF56KGHkvzf7RkmTpyYJGu8XcSiRYty3nnnZccdd0znzp2z22675V//9V9TKpUazfu3v/0tZ599drbZZpv06NEjn/zkJ/Pqq6+mqqqq0UrrSy+9NFVVVXn++efzuc99LltvvXUOOeSQJMl//dd/5eSTT87OO++cLl26pLa2NqeeemrefPPNRnOtGOPFF1/M5z//+fTs2TPbbrttLrroopRKpcyZMyfHHHNMqqurU1tbm2984xtNOnfvvfdevv71r2eXXXZJ586dU1dXl3/5l3/JkiVLyn2qqqoyadKkLFq0qHyubrvttiaNvybz58/P6NGjy+d31113zVVXXZXly5cnSUqlUo444ohsu+22eeONN8r7LV26NHvvvXd22WWXLFq0KJdeemnOP//8JEn//v3Ltc2aNWuDa1th+vTp+fjHP56ePXtmq622ymGHHZbf/e53jfqseE/++Mc/llcw9+zZM6ecckrefffdRn2b8llp6vHcd9992WuvvdK5c+fsueeeefDBBzf6eAEAWD8rogEANkMLFizIX/7yl9Xaly1btt59L7300owfPz5f+MIXcuCBB2bhwoV58skn89RTT+VjH/tYvvSlL+W1117LQw89tNqtJEqlUj75yU/mkUceyWmnnZb99tsvv/zlL3P++efn1VdfzbXXXlvue/LJJ+dHP/pRTjzxxAwePDhTpkzJiBEj1lrXpz/96QwYMCBXXnllOdR+6KGH8qc//SmnnHJKamtr89xzz+Xmm2/Oc889l2nTpq12P+V//Md/zO67754JEyakoaEhl19+eXr37p3vfOc7OfLII3PVVVfljjvuyFe+8pUccMABOfTQQ9d5rr7whS/k9ttvz6c+9amcd955mT59esaPH58XXngh9957b5Lk3//933PzzTfn8ccfz3e/+90kyYc//OH1vg9r8u677+awww7Lq6++mi996Uvp169fHnvssYwdOzavv/56rrvuulRVVeV73/te9tlnn/zzP/9zfvKTnyRJLrnkkjz33HN59NFH061btxx33HF58cUXc9ddd+Xaa6/NNttskyTZdtttN6i2FR5++OEMHz48gwYNyiWXXJJ27dpl0qRJOfLII/Ob3/wmBx54YKP+n/nMZ9K/f/+MHz8+Tz31VL773e9mu+22y1VXXVXu05TPSlOO57e//W1+8pOf5IwzzkiPHj1y/fXXZ+TIkZk9e3b69OmzUccNAMB6lAAA2GxMmjSplGSdjz333LPRPjvttFPppJNOKr/ed999SyNGjFjnPKNGjSqt6avkfffdV0pSuvzyyxu1f+pTnypVVVWV/vjHP5ZKpVJpxowZpSSl0aNHN+p38sknl5KULrnkknLbJZdcUkpS+uxnP7vafO++++5qbXfddVcpSenXv/71amOcfvrp5bb33nuvtMMOO5SqqqpKEyZMKLe/9dZbpa5duzY6J2vy9NNPl5KUvvCFLzRq/8pXvlJKUnr44YfLbSeddFKpW7du6xxvTfbcc8/SYYcdVn799a9/vdStW7fSiy++2KhffX19qX379qXZs2eX277zne+UkpR+8IMflKZNm1Zq3779auf7mmuuKSUpvfLKK02qZ33HsXz58tKAAQNKw4YNKy1fvrzc/u6775b69+9f+tjHPlZuW/GenHrqqY3G+Id/+IdSnz59yq8r+ays63iSlDp16lT+DJZKpdIzzzxTSlL61re+td5jBwBg47g1BwDAZmjixIl56KGHVnvss88+6923V69eee655/LSSy9VPO8vfvGLtG/fPmeffXaj9vPOOy+lUikPPPBAkpRvh3DGGWc06nfWWWetdex//ud/Xq2ta9eu5eeLFy/OX/7ylwwePDhJ8tRTT63W/wtf+EL5efv27bP//vunVCrltNNOK7f36tUru+22W/70pz+ttZbk/441ScaMGdOo/bzzzkuSNDQ0rHP/DXHPPffkIx/5SLbeeuv85S9/KT+GDh2a999/P7/+9a/LfU8//fQMGzYsZ511Vk488cTssssuufLKK5u9ppU9/fTTeemll/K5z30ub775Zrm+RYsW5aMf/Wh+/etfl28hssKq7+tHPvKRvPnmm1m4cGGSDfusrM3QoUOzyy67lF/vs88+qa6uXu97DQDAxnNrDgCAzdCBBx6Y/ffff7X2FQHmulx22WU55phj8oEPfCB77bVXPv7xj+fEE09sUoj95z//OX379k2PHj0ate++++7l7Sv+265du/Tv379Rv1133XWtY6/aN0n++te/Zty4cbn77rsb3Q85+b/bk6yqX79+jV737NkzXbp0Kd/GYeX2Ve8zvaoVx7BqzbW1tenVq1f5WJvTSy+9lP/6r/9a6+0zVj0Ht956a3bZZZe89NJLeeyxxxoF90VY8T8vTjrppLX2WbBgQbbeeuvy61XfkxXb3nrrrVRXV2/QZ2VtVp1rxXxvvfVWxWMBAFAZQTQAAI0ceuihefnll/PTn/40v/rVr/Ld73431157bW666aZGK4o3tTWFqJ/5zGfy2GOP5fzzz89+++2X7t27Z/ny5fn4xz++2srb5P9WQTelLclqf1xxbVa9D3WRli9fno997GO54IIL1rj9Ax/4QKPXjz76aPkPJz777LMZMmRI4fUlyTXXXJP99ttvjX26d+/e6PXGnv9KbMq5AABoTBANAMBqevfunVNOOSWnnHJK3nnnnRx66KG59NJLy0H02sLXnXbaKf/5n/+Zt99+u9Gq6P/+7/8ub1/x3+XLl+eVV17JgAEDyv3++Mc/NrnGt956K5MnT864ceNy8cUXl9s35JYiG2LFMbz00kvlFd9JMm/evMyfP798rM1pl112yTvvvJOhQ4eut+/rr7+es846K0cddVQ6deqUr3zlKxk2bFijupo7RF9x24vq6uom1dgUlXxWNuX/FAAAoDLuEQ0AQCOr3pKie/fu2XXXXcsra5OkW7duSZL58+c36vv3f//3ef/99/Ptb3+7Ufu1116bqqqqDB8+PEkybNiwJMkNN9zQqN+3vvWtJte5YnXrqqtZr7vuuiaPsTH+/u//fo3zffOb30ySjBgxotnn/MxnPpOpU6fml7/85Wrb5s+fn/fee6/8+otf/GKWL1+eW2+9NTfffHM6dOiQ0047rdH5Wtv7uKEGDRqUXXbZJf/6r/+ad955Z7Xt//u//1vxmJV8Vpr7eAAAaD5WRAMA0Mgee+yRww8/PIMGDUrv3r3z5JNP5sc//nHOPPPMcp9BgwYlSc4+++wMGzYs7du3z/HHH5+jjz46RxxxRC688MLMmjUr++67b371q1/lpz/9aUaPHl1eMTto0KCMHDky1113Xd58880MHjw4U6ZMyYsvvpikaStbq6urc+ihh+bqq6/OsmXL8nd/93f51a9+lVdeeaWAs7K6fffdNyeddFJuvvnmzJ8/P4cddlgef/zx3H777Tn22GNzxBFHNPuc559/fn72s5/lE5/4RE4++eQMGjQoixYtyrPPPpsf//jHmTVrVrbZZptMmjQpDQ0Nue2227LDDjsk+b/g9vOf/3xuvPHG8h/+W/E+XnjhhTn++OPTsWPHHH300eVAd02WLVuWyy+/fLX23r1754wzzsh3v/vdDB8+PHvuuWdOOeWU/N3f/V1effXVPPLII6murs79999f0TFX8lnZkOMBAGDTEEQDANDI2WefnZ/97Gf51a9+lSVLlmSnnXbK5ZdfnvPPP7/c57jjjstZZ52Vu+++Oz/4wQ9SKpVy/PHHp127dvnZz36Wiy++OD/84Q8zadKk1NXV5Zprrsl5553XaJ7vf//7qa2tzV133ZV77703Q4cOzQ9/+MPstttu6dKlS5NqvfPOO3PWWWdl4sSJKZVKOeqoo/LAAw+kb9++zXpO1ua73/1udt5559x222259957U1tbm7Fjx+aSSy4pZL6tttoqU6ZMyZVXXpl77rkn3//+91NdXZ0PfOADGTduXHr27Jn/+Z//ybnnnpujjz660R8NPOGEE/If//EfueCCCzJ8+PD0798/BxxwQL7+9a/npptuyoMPPli+Bca6gtulS5fmoosuWq19l112yRlnnJHDDz88U6dOzde//vV8+9vfzjvvvJPa2tocdNBB+dKXvrRBx93Uz8qGHA8AAJtGVclf5gAAoJV4+umn88EPfjA/+MEPcsIJJ7R0ObRiPisAAG2Le0QDANAi/va3v63Wdt1116Vdu3Y59NBDW6AiWiufFQCAts+tOQAAaBFXX311ZsyYkSOOOCIdOnTIAw88kAceeCCnn356dtxxx5Yuj1bEZwUAoO1zaw4AAFrEQw89lHHjxuX555/PO++8k379+uXEE0/MhRdemA4drJfg//NZAQBo+wTRAAAAAAAUyj2iAQAAAAAolCAaAAAAAIBCtbobqi1fvjyvvfZaevTokaqqqpYuBwAAAACANSiVSnn77bfTt2/ftGu37jXPrS6Ifu211/zlawAAAACANmLOnDnZYYcd1tmn1QXRPXr0SPJ/xVdXV7dwNQAAAAAArMnChQuz4447ljPddakoiH7//fdz6aWX5gc/+EHmzp2bvn375uSTT87Xvva18m00SqVSLrnkktxyyy2ZP39+Dj744Nx4440ZMGBAk+ZYMU51dbUgGgAAAACglWvKLZYr+mOFV111VW688cZ8+9vfzgsvvJCrrroqV199db71rW+V+1x99dW5/vrrc9NNN2X69Onp1q1bhg0blsWLF1d+BAAAAAAAtHlVpVKp1NTOn/jEJ1JTU5Nbb7213DZy5Mh07do1P/jBD1IqldK3b9+cd955+cpXvpIkWbBgQWpqanLbbbfl+OOPX+8cCxcuTM+ePbNgwQIrogEAAAAAWqlKstyKVkR/+MMfzuTJk/Piiy8mSZ555pn89re/zfDhw5Mkr7zySubOnZuhQ4eW9+nZs2cOOuigTJ06dY1jLlmyJAsXLmz0AAAAAABg81HRPaLr6+uzcOHCDBw4MO3bt8/777+fK664IieccEKSZO7cuUmSmpqaRvvV1NSUt61q/PjxGTdu3IbUDgAAAABAG1DRiugf/ehHueOOO3LnnXfmqaeeyu23355//dd/ze23377BBYwdOzYLFiwoP+bMmbPBYwEAAAAA0PpUtCL6/PPPT319fflez3vvvXf+/Oc/Z/z48TnppJNSW1ubJJk3b16233778n7z5s3Lfvvtt8YxO3funM6dO29g+QAAAAAAtHYVrYh+9913065d413at2+f5cuXJ0n69++f2traTJ48ubx94cKFmT59eoYMGdIM5QIAAAAA0NZUtCL66KOPzhVXXJF+/fplzz33zO9///t885vfzKmnnpokqaqqyujRo3P55ZdnwIAB6d+/fy666KL07ds3xx57bBH1AwAAAADQylUURH/rW9/KRRddlDPOOCNvvPFG+vbtmy996Uu5+OKLy30uuOCCLFq0KKeffnrmz5+fQw45JA8++GC6dOnS7MUDAAAAAND6VZVKpVJLF7GyhQsXpmfPnlmwYEGqq6tbuhwAAAAAANagkiy3ontEAwAAAABApQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA2wmaqrb2jpEgAAAACSCKIBAAAAACiYIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBqAwtXVN7R0CQAAAEALEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTTAFq6uvmGNzwEAAACaiyAaAAAAAIBCCaIBAAAAAChURUF0XV1dqqqqVnuMGjUqSbJ48eKMGjUqffr0Sffu3TNy5MjMmzevkMIBAAAAAGgbKgqin3jiibz++uvlx0MPPZQk+fSnP50kOffcc3P//ffnnnvuyZQpU/Laa6/luOOOa/6qAQAAAABoMzpU0nnbbbdt9HrChAnZZZddcthhh2XBggW59dZbc+edd+bII49MkkyaNCm77757pk2blsGDBzdf1QAAAAAAtBkbfI/opUuX5gc/+EFOPfXUVFVVZcaMGVm2bFmGDh1a7jNw4MD069cvU6dOXes4S5YsycKFCxs9AAAAAADYfGxwEH3fffdl/vz5Ofnkk5Mkc+fOTadOndKrV69G/WpqajJ37ty1jjN+/Pj07Nmz/Nhxxx03tCQAAAAAAFqhDQ6ib7311gwfPjx9+/bdqALGjh2bBQsWlB9z5szZqPEAAAAAAGhdKrpH9Ap//vOf85//+Z/5yU9+Um6rra3N0qVLM3/+/EaroufNm5fa2tq1jtW5c+d07tx5Q8oAAAAAAKAN2KAV0ZMmTcp2222XESNGlNsGDRqUjh07ZvLkyeW2mTNnZvbs2RkyZMjGVwoAAAAAQJtU8Yro5cuXZ9KkSTnppJPSocP/371nz5457bTTMmbMmPTu3TvV1dU566yzMmTIkAwePLhZiwYAAAAAoO2oOIj+z//8z8yePTunnnrqatuuvfbatGvXLiNHjsySJUsybNiw3HDDDc1SKAAAAAAAbVPFQfRRRx2VUqm0xm1dunTJxIkTM3HixI0uDAAAAACAzcMG3SMaAAAAAACaShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANwCZRV9+QuvqGli4DAAAAaAGCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBmhD6uobUlff0NJlAAAAAFREEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QDNrK6+oaVLWKu6+oZNWt+mng8AAABonQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAM2grr6hpUsAAAAAaLUE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFKpDSxcAQMuoq29o6RIAAACALYQV0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUquIg+tVXX83nP//59OnTJ127ds3ee++dJ598sry9VCrl4osvzvbbb5+uXbtm6NCheemll5q1aAAAAAAA2o6Kgui33norBx98cDp27JgHHnggzz//fL7xjW9k6623Lve5+uqrc/311+emm27K9OnT061btwwbNiyLFy9u9uIBAAAAAGj9OlTS+aqrrsqOO+6YSZMmldv69+9ffl4qlXLdddfla1/7Wo455pgkyfe///3U1NTkvvvuy/HHH99MZQMAAAAA0FZUtCL6Zz/7Wfbff/98+tOfznbbbZcPfvCDueWWW8rbX3nllcydOzdDhw4tt/Xs2TMHHXRQpk6dusYxlyxZkoULFzZ6AAAAAACw+agoiP7Tn/6UG2+8MQMGDMgvf/nLfPnLX87ZZ5+d22+/PUkyd+7cJElNTU2j/WpqasrbVjV+/Pj07Nmz/Nhxxx035DgANit19Q2pq2/Y5PsCAAAAFKGiIHr58uX50Ic+lCuvvDIf/OAHc/rpp+eLX/xibrrppg0uYOzYsVmwYEH5MWfOnA0eCwAAAACA1qeiIHr77bfPHnvs0aht9913z+zZs5MktbW1SZJ58+Y16jNv3rzytlV17tw51dXVjR4AAAAAAGw+KgqiDz744MycObNR24svvpiddtopyf/94cLa2tpMnjy5vH3hwoWZPn16hgwZ0gzlAgAAAADQ1nSopPO5556bD3/4w7nyyivzmc98Jo8//nhuvvnm3HzzzUmSqqqqjB49OpdffnkGDBiQ/v3756KLLkrfvn1z7LHHFlE/AAAAAACtXEVB9AEHHJB77703Y8eOzWWXXZb+/fvnuuuuywknnFDuc8EFF2TRokU5/fTTM3/+/BxyyCF58MEH06VLl2YvHgAAAACA1q+iIDpJPvGJT+QTn/jEWrdXVVXlsssuy2WXXbZRhQEAAAAAsHmoOIgGYM3q6htafO5ZE0a0WA0AAAAAa1PRHysEAAAAAIBKCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBClZX35C6+oaWLgMAAACgxQiiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiATYjdfUNG7zfhuy76j4bOv/G2NDaAQAAgE1HEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAFuguvqGdW5b33YAAACASgiiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAdqguvqGivpW0r/ocTZkXgAAAKBtE0QDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTRAK1BX37De563VxtTYFo4PAAAA2HiCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBmij6uobUlffsMnmAgAAANhQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQHVq6AIDNWV19QyF9NycrjnvWhBEtXAkAAABQFCuiAQAAAAAolCAaAAAAAIBCCaIBAAAAAChURUH0pZdemqqqqkaPgQMHlrcvXrw4o0aNSp8+fdK9e/eMHDky8+bNa/aiAQAAAABoOypeEb3nnnvm9ddfLz9++9vflrede+65uf/++3PPPfdkypQpee2113Lcccc1a8EAAAAAALQtHSreoUOH1NbWrta+YMGC3Hrrrbnzzjtz5JFHJkkmTZqU3XffPdOmTcvgwYM3vloAAAAAANqcildEv/TSS+nbt2923nnnnHDCCZk9e3aSZMaMGVm2bFmGDh1a7jtw4MD069cvU6dOXet4S5YsycKFCxs9AAAAAADYfFS0Ivqggw7Kbbfdlt122y2vv/56xo0bl4985CP5wx/+kLlz56ZTp07p1atXo31qamoyd+7ctY45fvz4jBs3boOKB2it6uobWrqENs85BAAAgM1HRUH08OHDy8/32WefHHTQQdlpp53yox/9KF27dt2gAsaOHZsxY8aUXy9cuDA77rjjBo0FAAAAAEDrU/GtOVbWq1evfOADH8gf//jH1NbWZunSpZk/f36jPvPmzVvjPaVX6Ny5c6qrqxs9AAAAAADYfGxUEP3OO+/k5Zdfzvbbb59BgwalY8eOmTx5cnn7zJkzM3v27AwZMmSjCwUAAAAAoG2q6NYcX/nKV3L00Udnp512ymuvvZZLLrkk7du3z2c/+9n07Nkzp512WsaMGZPevXunuro6Z511VoYMGZLBgwcXVT8AAAAAAK1cRUH0//zP/+Szn/1s3nzzzWy77bY55JBDMm3atGy77bZJkmuvvTbt2rXLyJEjs2TJkgwbNiw33HBDIYUDAAAAANA2VBRE33333evc3qVLl0ycODETJ07cqKIAAAAAANh8bNQ9ogGgCHX1DZt0rk05HwAAAGyJBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA1sEerqG1JX39DSZRRiXce1uR4zAAAA0LYIogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGqBCdfUNqatv2KD9Kmnf0jT1PGzouXeeAQAAoOUIogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEJ1aOkCAFi3uvqGFhmnrr4hsyaMaNJYK79e8Xxt+wIAAABbHiuiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaYBOqq29IXX1DS5exSTTnca5trC3lXAIAAEBbJ4gGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAK1aGlCwBoq+rqG1q6hMIVdYxrG7ep820J5x4AAAA2J1ZEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0sMWqq29ocntdfcNa+9M6rPz+VPLermu8IvsDAADAlkQQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoTq0dAEAW6K6+oaWLmGL4VwDAABAy7MiGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKFSHli4AoGh19Q0tXQKbyMa+1yv2nzVhRHOU0+rmAwAAgJZiRTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhdqoIHrChAmpqqrK6NGjy22LFy/OqFGj0qdPn3Tv3j0jR47MvHnzNrZOAAAAAADaqA0Oop944ol85zvfyT777NOo/dxzz83999+fe+65J1OmTMlrr72W4447bqMLBQAAAACgbdqgIPqdd97JCSeckFtuuSVbb711uX3BggW59dZb881vfjNHHnlkBg0alEmTJuWxxx7LtGnTmq1oAAAAAADajg0KokeNGpURI0Zk6NChjdpnzJiRZcuWNWofOHBg+vXrl6lTp65xrCVLlmThwoWNHgAAAAAAbD4qDqLvvvvuPPXUUxk/fvxq2+bOnZtOnTqlV69ejdpramoyd+7cNY43fvz49OzZs/zYcccdKy0JYKPV1Te0dAkAAAAAm62Kgug5c+bknHPOyR133JEuXbo0SwFjx47NggULyo85c+Y0y7gAAAAAALQOFQXRM2bMyBtvvJEPfehD6dChQzp06JApU6bk+uuvT4cOHVJTU5OlS5dm/vz5jfabN29eamtr1zhm586dU11d3egBAAAAAMDmo0MlnT/60Y/m2WefbdR2yimnZODAgfnqV7+aHXfcMR07dszkyZMzcuTIJMnMmTMze/bsDBkypPmqBgAAAACgzagoiO7Ro0f22muvRm3dunVLnz59yu2nnXZaxowZk969e6e6ujpnnXVWhgwZksGDBzdf1QAAAAAAtBkVBdFNce2116Zdu3YZOXJklixZkmHDhuWGG25o7mkAAAAAAGgjNjqIfvTRRxu97tKlSyZOnJiJEydu7NAAAAAAAGwGKvpjhQAAAAAAUClBNLBZqqtvaJVjAQAAAGyJBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAFairb2jpEmgm3ksAAADYdATRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUqkNLFwCwqdXVNzS5z6wJI4ouZ4vTlPMPAAAAbF6siAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAZgs1RX35C6+oZmGaeSdgAAAGB1gmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAbarLr6hpYugc2IzxMAAAAURxANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNHAFqWuvqGlS6AV8XkAAACATUMQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDbRpdfUNLV0Cm0hdfUOLv99rm7/SdgAAANjSCKIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBAAAAAChUh5YuAKC1qKtvaFIbbdumeE9XnqOuviGzJoxYbdvKbQAAALC5syIaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCdWjpAgCaU119Q0uXAAAAAMAqrIgGAAAAAKBQgmgAAAAAAApVURB94403Zp999kl1dXWqq6szZMiQPPDAA+XtixcvzqhRo9KnT5907949I0eOzLx585q9aAAAAAAA2o6KgugddtghEyZMyIwZM/Lkk0/myCOPzDHHHJPnnnsuSXLuuefm/vvvzz333JMpU6bktddey3HHHVdI4QAAAAAAtA0V/bHCo48+utHrK664IjfeeGOmTZuWHXbYIbfeemvuvPPOHHnkkUmSSZMmZffdd8+0adMyePDg5qsaAAAAAIA2Y4PvEf3+++/n7rvvzqJFizJkyJDMmDEjy5Yty9ChQ8t9Bg4cmH79+mXq1KlrHWfJkiVZuHBhowcAAAAAAJuPilZEJ8mzzz6bIUOGZPHixenevXvuvffe7LHHHnn66afTqVOn9OrVq1H/mpqazJ07d63jjR8/PuPGjau4cADYVOrqG9b4vIjxZ00YsdY+K7at/BwAAADagopXRO+22255+umnM3369Hz5y1/OSSedlOeff36DCxg7dmwWLFhQfsyZM2eDxwIAAAAAoPWpeEV0p06dsuuuuyZJBg0alCeeeCL/9m//ln/8x3/M0qVLM3/+/EaroufNm5fa2tq1jte5c+d07ty58soBAAAAAGgTNvge0SssX748S5YsyaBBg9KxY8dMnjy5vG3mzJmZPXt2hgwZsrHTAAAAAADQRlW0Inrs2LEZPnx4+vXrl7fffjt33nlnHn300fzyl79Mz549c9ppp2XMmDHp3bt3qqurc9ZZZ2XIkCEZPHhwUfUDAAAAANDKVRREv/HGG/mnf/qnvP766+nZs2f22Wef/PKXv8zHPvaxJMm1116bdu3aZeTIkVmyZEmGDRuWG264oZDCAQAAAABoGyoKom+99dZ1bu/SpUsmTpyYiRMnblRRAAAAAABsPjb6HtEAAAAAALAugmigxdXVN6SuvqHJ7RvaDwAAAICWIYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGNrm6+oZNsm9dfcN6+zelD2xKG/t53BSfZz8zAAAAVEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoTq0dAEAK6urb0iSzJowYo3t0JY19XO8tp8DAAAAaKusiAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFAdWroAYMtWV9/QLH2gNWuuz7CfBQAAANoqK6IBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaKBwtTVN6SuvmGTzgctrSU+h02Z088HAAAALUkQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRQIuoq29IXX3DJt8XWouN/Qw318/BymP4uQIAAKAogmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQHVq6AABoberqG5q136a2cl2zJoxotjGbaywAAAC2PFZEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0ALBGdfUNqatvaOkyAAAA2AwIogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGtggdfUNqatvaOkyYLNW5M+Yn2EAAAA2JUE0AAAAAACFEkQDAAAAAFCoioLo8ePH54ADDkiPHj2y3Xbb5dhjj83MmTMb9Vm8eHFGjRqVPn36pHv37hk5cmTmzZvXrEUDAAAAANB2VBRET5kyJaNGjcq0adPy0EMPZdmyZTnqqKOyaNGicp9zzz03999/f+65555MmTIlr732Wo477rhmLxwAAAAAgLahQyWdH3zwwUavb7vttmy33XaZMWNGDj300CxYsCC33npr7rzzzhx55JFJkkmTJmX33XfPtGnTMnjw4OarHAAAAACANmGj7hG9YMGCJEnv3r2TJDNmzMiyZcsydOjQcp+BAwemX79+mTp16hrHWLJkSRYuXNjoAQAAAADA5mODg+jly5dn9OjROfjgg7PXXnslSebOnZtOnTqlV69ejfrW1NRk7ty5axxn/Pjx6dmzZ/mx4447bmhJQCtQV9+QuvqGli4Dthgb8vPmZxQAAIBNbYOD6FGjRuUPf/hD7r777o0qYOzYsVmwYEH5MWfOnI0aDwAAAACA1qWie0SvcOaZZ+bnP/95fv3rX2eHHXYot9fW1mbp0qWZP39+o1XR8+bNS21t7RrH6ty5czp37rwhZQAAAAAA0AZUtCK6VCrlzDPPzL333puHH344/fv3b7R90KBB6dixYyZPnlxumzlzZmbPnp0hQ4Y0T8UAAAAAALQpFa2IHjVqVO6888789Kc/TY8ePcr3fe7Zs2e6du2anj175rTTTsuYMWPSu3fvVFdX56yzzsqQIUMyePDgQg4AAAAAAIDWraIg+sYbb0ySHH744Y3aJ02alJNPPjlJcu2116Zdu3YZOXJklixZkmHDhuWGG25olmIBAAAAAGh7KgqiS6XSevt06dIlEydOzMSJEze4KAAAAAAANh8V3SMaAAAAAAAqVdGKaIBNpa6+oaVLgDaluX5mVowza8KIZhkPAAAAEiuiAQAAAAAomCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaKFxdfUNLlwBtVl19Q5v5Gaq0zlX7t5XjBAAAoHKCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaGCTqKtvaJY+QMupq2+o6Od05f6V7tsc+wMAANB6CKIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBrYKHX1DRW1A61PXX3DOn9m/TwDAACwsQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoTq0dAG0LnX1DZk1YURLl0ErUlffkCSZNWFEo+dA27Di53bV55ty3kq2bcx8rk0AAACtlxXRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAKymrr6hWfttzPxFzQEAAMCmI4gGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAoM2oq29o6RIAAADYAIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJooNnU1Te0dAkArkUAAACtkCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCdWjpAoC2p66+YZ2vm7ofsHmo5Ge7Oa8DGzJWXX1DZk0Y0Ww1bMj8SZpUw4bWWskc/H/OGwAAFMuKaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKVXEQ/etf/zpHH310+vbtm6qqqtx3332NtpdKpVx88cXZfvvt07Vr1wwdOjQvvfRSc9ULAAAAAEAbU3EQvWjRouy7776ZOHHiGrdfffXVuf7663PTTTdl+vTp6datW4YNG5bFixdvdLEAAAAAALQ9HSrdYfjw4Rk+fPgat5VKpVx33XX52te+lmOOOSZJ8v3vfz81NTW57777cvzxx29ctQAAAAAAtDnNeo/oV155JXPnzs3QoUPLbT179sxBBx2UqVOnrnGfJUuWZOHChY0eAAAAAABsPipeEb0uc+fOTZLU1NQ0aq+pqSlvW9X48eMzbty45iwDaIK6+oYkyawJI1q4EmBzsuLaUvQ+lYy9ruucayEAAMCm0awrojfE2LFjs2DBgvJjzpw5LV0SAAAAAADNqFmD6Nra2iTJvHnzGrXPmzevvG1VnTt3TnV1daMHAAAAAACbj2YNovv375/a2tpMnjy53LZw4cJMnz49Q4YMac6pAAAAAABoIyq+R/Q777yTP/7xj+XXr7zySp5++un07t07/fr1y+jRo3P55ZdnwIAB6d+/fy666KL07ds3xx57bHPWDQAAAABAG1FxEP3kk0/miCOOKL8eM2ZMkuSkk07KbbfdlgsuuCCLFi3K6aefnvnz5+eQQw7Jgw8+mC5dujRf1QAAAAAAtBkVB9GHH354SqXSWrdXVVXlsssuy2WXXbZRhQEAAAAAsHmoOIgGthx19Q1rfA7QGq8JzVnTirFmTRixUXU0Zf+6+oYNmqc12xyPCQAA2DjN+scKAQAAAABgVYJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUB1augBgw9XVNyRJZk0YUUh/gKKtuC4VvU9zWdd1dH11tWTdAAAALc2KaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFAdWroAoFh19Q1JklkTRlTUH6C5VHJdKeIatOqYa7ou1tU3lF+v3H/l9qLrKmL85qy90t8nAAAAK7MiGgAAAACAQgmiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQgmioRWoq29o1n5r6r++fevqGyoeH6AtW/WaV8l1cNXr68ZeP9c1xopt65qj0hpWHbMpz1fed2MUea6ae6wNmcfvUgAAWDNBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEwxaurr6hpUsAaNXq6hvWeK1cW/uqfZprvqaM25T2ptbUlH6rjru+c7XqtkrPz4bs3xK/55ryHrY1m+MxAQCwaQmiAQAAAAAolCAaAAAAAIBCCaIBAAAAACiUIBoAAAAAgEIJogEAAAAAKJQgGgAAAACAQnVo6QJo/erqG5IksyaMaOFK1qy117cxKjm2FX2b0n/lvgA0zYZeOzfmmrvqvkVcvzdkzI2tY33719U3FPK7rCnjNqdNPd+KOZN1fxdoiboAAMCKaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAAAAACtWhpQuA1q6uviGzJozY6DGSNBpn1XHX1KcpY676fGPqA2DTWNt1d0Oux+vapzl/V2zI/JXsP2vCiDWO1dTx1/Z7dUPHW9c8SdN/XzdlvErHaso+K/dZX/+Vz0lzHRcAAKzKimgAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQHVq6AFqfuvqGJMmsCSNadKwV+66wpjFW7bPyfOuae+X9Vt6+tn3W1H/Vvmsbc301N2X7+vbZ2P4Am5st4TpYxDE255iVfgdojrmb4ztMXX1Dk/Zv6u/9NY3b1DlWnmdjvltUMu6a2lcef9Xam3IslRzv+uZoyvONmXvl/VbY2O/ETTnXGzvfhu6/sedtQ8/vptQWagSALYUV0QAAAAAAFEoQDQAAAABAoQTRAAAAAAAUShANAAAAAEChBNEAAAAAABRKEA0AAAAAQKEE0QAAAAAAFKpDSxfAhqurb2j0etaEEattW7mtkrFWtK1t/5W3rWnf9Y21rvrWNt765ll5+6p917XvmmppSv919V3XuWtqXc25DwBsjLb+u6eS7xCrvt6Q7zlNmXttc67t+8jGvgeVfr+qZL6Va1/T81X7rGmO9fWv5Fw05dyua451fYer5Hvjqse0IdZ3btc3x6rneH19NrbGSsZc33le+X1pyr8Z1tVnTfVU+j43p6b+W2Ft/ZtyfjamthUq+ffMun62K92/OftvCs39HmzI/EnrOieb2sa8By39/m0I73nr4z1pOiuiAQAAAAAolCAaAAAAAIBCCaIBAAAAAChUYUH0xIkTU1dXly5duuSggw7K448/XtRUAAAAAAC0YoUE0T/84Q8zZsyYXHLJJXnqqaey7777ZtiwYXnjjTeKmA4AAAAAgFaskCD6m9/8Zr74xS/mlFNOyR577JGbbropW221Vb73ve8VMR0AAAAAAK1Yh+YecOnSpZkxY0bGjh1bbmvXrl2GDh2aqVOnrtZ/yZIlWbJkSfn1ggULkiQLFy5s7tI2O8uXvNvo9crnbMW2pp7HVcdaecw1jbV8ybvl12vbd01jNaW+po7XVCsfQ3P0q2TeFVY+3uY+PgBozTb0d19Tv2esqV9L/L5tSr1r67Ou71vrG2td/Sr9rtZUa5t71eNY07xN2XfV/k09p+s73lW/d656ztdW+/q+wzble/T65m/KfOt7vq791zT/umpfW/917d/Uz25Txl/1/WxK7c15fJX+W6pSazumpvZvyvnZmNpWqOQzUMnnbn37N2f/TaG534MNmT9pXedkU9uY96Cl378N4T1vfbb092TFcZdKpfX2rSo1pVcFXnvttfzd3/1dHnvssQwZMqTcfsEFF2TKlCmZPn16o/6XXnppxo0b15wlAAAAAACwicyZMyc77LDDOvs0+4roSo0dOzZjxowpv16+fHn++te/pk+fPqmqqmrBylrGwoULs+OOO2bOnDmprq5u6XIAXJeAVsd1CWhNXJOA1sZ1iU2pVCrl7bffTt++fdfbt9mD6G222Sbt27fPvHnzGrXPmzcvtbW1q/Xv3LlzOnfu3KitV69ezV1Wm1NdXe1iAbQqrktAa+O6BLQmrklAa+O6xKbSs2fPJvVr9j9W2KlTpwwaNCiTJ08uty1fvjyTJ09udKsOAAAAAAC2DIXcmmPMmDE56aSTsv/+++fAAw/Mddddl0WLFuWUU04pYjoAAAAAAFqxQoLof/zHf8z//u//5uKLL87cuXOz33775cEHH0xNTU0R021WOnfunEsuuWS125UAtBTXJaC1cV0CWhPXJKC1cV2itaoqlUqlli4CAAAAAIDNV7PfIxoAAAAAAFYmiAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUILoVmTixImpq6tLly5dctBBB+Xxxx9v6ZKAzdT48eNzwAEHpEePHtluu+1y7LHHZubMmY36LF68OKNGjUqfPn3SvXv3jBw5MvPmzWvUZ/bs2RkxYkS22mqrbLfddjn//PPz3nvvbcpDATZDEyZMSFVVVUaPHl1uc00CNrVXX301n//859OnT5907do1e++9d5588sny9lKplIsvvjjbb799unbtmqFDh+all15qNMZf//rXnHDCCamurk6vXr1y2mmn5Z133tnUhwJsBt5///1cdNFF6d+/f7p27ZpddtklX//611Mqlcp9XJdo7QTRrcQPf/jDjBkzJpdcckmeeuqp7Lvvvhk2bFjeeOONli4N2AxNmTIlo0aNyrRp0/LQQw9l2bJlOeqoo7Jo0aJyn3PPPTf3339/7rnnnkyZMiWvvfZajjvuuPL2999/PyNGjMjSpUvz2GOP5fbbb89tt92Wiy++uCUOCdhMPPHEE/nOd76TffbZp1G7axKwKb311ls5+OCD07FjxzzwwAN5/vnn841vfCNbb711uc/VV1+d66+/PjfddFOmT5+ebt26ZdiwYVm8eHG5zwknnJDnnnsuDz30UH7+85/n17/+dU4//fSWOCSgjbvqqqty44035tvf/nZeeOGFXHXVVbn66qvzrW99q9zHdYlWr0SrcOCBB5ZGjRpVfv3++++X+vbtWxo/fnwLVgVsKd54441SktKUKVNKpVKpNH/+/FLHjh1L99xzT7nPCy+8UEpSmjp1aqlUKpV+8YtflNq1a1eaO3duuc+NN95Yqq6uLi1ZsmTTHgCwWXj77bdLAwYMKD300EOlww47rHTOOeeUSiXXJGDT++pXv1o65JBD1rp9+fLlpdra2tI111xTbps/f36pc+fOpbvuuqtUKpVKzz//fClJ6Yknnij3eeCBB0pVVVWlV199tbjigc3SiBEjSqeeemqjtuOOO650wgknlEol1yXaBiuiW4GlS5dmxowZGTp0aLmtXbt2GTp0aKZOndqClQFbigULFiRJevfunSSZMWNGli1b1ui6NHDgwPTr1698XZo6dWr23nvv1NTUlPsMGzYsCxcuzHPPPbcJqwc2F6NGjcqIESMaXXsS1yRg0/vZz36W/fffP5/+9Kez3Xbb5YMf/GBuueWW8vZXXnklc+fObXRd6tmzZw466KBG16VevXpl//33L/cZOnRo2rVrl+nTp2+6gwE2Cx/+8IczefLkvPjii0mSZ555Jr/97W8zfPjwJK5LtA0dWroAkr/85S95//33G/3DKUlqamry3//93y1UFbClWL58eUaPHp2DDz44e+21V5Jk7ty56dSpU3r16tWob01NTebOnVvus6br1optAJW4++6789RTT+WJJ55YbZtrErCp/elPf8qNN96YMWPG5F/+5V/yxBNP5Oyzz06nTp1y0kknla8ra7rurHxd2m677Rpt79ChQ3r37u26BFSsvr4+CxcuzMCBA9O+ffu8//77ueKKK3LCCSckiesSbYIgGmALN2rUqPzhD3/Ib3/725YuBdhCzZkzJ+ecc04eeuihdOnSpaXLAcjy5cuz//7758orr0ySfPCDH8wf/vCH3HTTTTnppJNauDpgS/SjH/0od9xxR+68887sueeeefrppzN69Oj07dvXdYk2w605WoFtttkm7du3X+0vv8+bNy+1tbUtVBWwJTjzzDPz85//PI888kh22GGHcnttbW2WLl2a+fPnN+q/8nWptrZ2jdetFdsAmmrGjBl544038qEPfSgdOnRIhw4dMmXKlFx//fXp0KFDampqXJOATWr77bfPHnvs0aht9913z+zZs5P8/+vKuv4NV1tbu9ofn3/vvffy17/+1XUJqNj555+f+vr6HH/88dl7771z4okn5txzz8348eOTuC7RNgiiW4FOnTpl0KBBmTx5crlt+fLlmTx5coYMGdKClQGbq1KplDPPPDP33ntvHn744fTv37/R9kGDBqVjx46NrkszZ87M7Nmzy9elIUOG5Nlnn230Reahhx5KdXX1av9wA1iXj370o3n22Wfz9NNPlx/7779/TjjhhPJz1yRgUzr44IMzc+bMRm0vvvhidtpppyRJ//79U1tb2+i6tHDhwkyfPr3RdWn+/PmZMWNGuc/DDz+c5cuX56CDDtoERwFsTt599920a9c4xmvfvn2WL1+exHWJtsGtOVqJMWPG5KSTTsr++++fAw88MNddd10WLVqUU045paVLAzZDo0aNyp133pmf/vSn6dGjR/l+YD179kzXrl3Ts2fPnHbaaRkzZkx69+6d6urqnHXWWRkyZEgGDx6cJDnqqKOyxx575MQTT8zVV1+duXPn5mtf+1pGjRqVzp07t+ThAW1Mjx49yveoX6Fbt27p06dPud01CdiUzj333Hz4wx/OlVdemc985jN5/PHHc/PNN+fmm29OklRVVWX06NG5/PLLM2DAgPTv3z8XXXRR+vbtm2OPPTbJ/62g/vjHP54vfvGLuemmm7Js2bKceeaZOf7449O3b98WPDqgLTr66KNzxRVXpF+/ftlzzz3z+9//Pt/85jdz6qmnJnFdoo0o0Wp861vfKvXr16/UqVOn0oEHHliaNm1aS5cEbKaSrPExadKkcp+//e1vpTPOOKO09dZbl7baaqvSP/zDP5Ref/31RuPMmjWrNHz48FLXrl1L22yzTem8884rLVu2bBMfDbA5Ouyww0rnnHNO+bVrErCp3X///aW99tqr1Llz59LAgQNLN998c6Pty5cvL1100UWlmpqaUufOnUsf/ehHSzNnzmzU58033yx99rOfLXXv3r1UXV1dOuWUU0pvv/32pjwMYDOxcOHC0jnnnFPq169fqUuXLqWdd965dOGFF5aWLFlS7uO6RGtXVSqVSi0ZhAMAAAAAsHlzj2gAAAAAAAoliAYAAAAAoFCCaAAAAAAACiWIBgAAAACgUIJoAAAAAAAKJYgGAAAAAKBQgmgAAAAAAAoliAYAAAAAoFCCaAAAtiiHH354Ro8e3dJltHmzZs1KVVVVnn766ZYuBQCANkAQDQDAJnPTTTelR48eee+998pt77zzTjp27JjDDz+8Ud9HH300VVVVefnllzdxla3HK6+8ks997nPp27dvunTpkh122CHHHHNM/vu//7ulSwMAgIoIogEA2GSOOOKIvPPOO3nyySfLbb/5zW9SW1ub6dOnZ/HixeX2Rx55JP369csuu+xS8TylUqlR2N3aLVu2bI1tH/vYx7JgwYL85Cc/ycyZM/PDH/4we++9d+bPn7/piwQAgI0giAYAYJPZbbfdsv322+fRRx8ttz366KM55phj0r9//0ybNq1R+xFHHJEkWbJkSc4+++xst9126dKlSw455JA88cQTjfpWVVXlgQceyKBBg9K5c+f89re/zaJFi/JP//RP6d69e7bffvt84xvfWK2mG264IQMGDEiXLl1SU1OTT33qU2ut/7bbbkuvXr1y3333lfcZNmxY5syZ06jfT3/603zoQx9Kly5dsvPOO2fcuHGNgvGqqqrceOON+eQnP5lu3brliiuuWG2u5557Li+//HJuuOGGDB48ODvttFMOPvjgXH755Rk8eHC531e/+tV84AMfyFZbbZWdd945F110UaNg+9JLL81+++2X733ve+nXr1+6d++eM844I++//36uvvrq1NbWZrvttluthhU1Dh8+PF27ds3OO++cH//4x2s9N0nyhz/8IcOHD0/37t1TU1OTE088MX/5y1/WuQ8AAFsGQTQAAJvUEUcckUceeaT8+pFHHsnhhx+eww47rNz+t7/9LdOnTy8H0RdccEH+4z/+I7fffnueeuqp7Lrrrhk2bFj++te/Nhq7vr4+EyZMyAsvvJB99tkn559/fqZMmZKf/vSn+dWvfpVHH300Tz31VLn/k08+mbPPPjuXXXZZZs6cmQcffDCHHnroOut/9913c8UVV+T73/9+fve732X+/Pk5/vjjy9t/85vf5J/+6Z9yzjnn5Pnnn893vvOd3HbbbasFvZdeemn+4R/+Ic8++2xOPfXU1ebZdttt065du/z4xz/O+++/v9Z6evTokdtuuy3PP/98/u3f/i233HJLrr322kZ9Xn755TzwwAN58MEHc9ddd+XWW2/NiBEj8j//8z+ZMmVKrrrqqnzta1/L9OnTG+130UUXZeTIkXnmmWdywgkn5Pjjj88LL7ywxjrmz5+fI488Mh/84Afz5JNP5sEHH8y8efPymc98Zp3nEwCALUQJAAA2oVtuuaXUrVu30rJly0oLFy4sdejQofTGG2+U7rzzztKhhx5aKpVKpcmTJ5eSlP785z+X3nnnnVLHjh1Ld9xxR3mMpUuXlvr27Vu6+uqrS6VSqfTII4+UkpTuu+++cp+333671KlTp9KPfvSjctubb75Z6tq1a+mcc84plUql0n/8x3+UqqurSwsXLmxS7ZMmTSolKU2bNq3c9sILL5SSlKZPn14qlUqlj370o6Urr7yy0X7//u//Xtp+++3Lr5OURo8evd75vv3tb5e22mqrUo8ePUpHHHFE6bLLLiu9/PLL69znmmuuKQ0aNKj8+pJLLilttdVWjY5x2LBhpbq6utL7779fbtttt91K48ePb1TjP//zPzca+6CDDip9+ctfLpVKpdIrr7xSSlL6/e9/XyqVSqWvf/3rpaOOOqpR/zlz5pSSlGbOnLneYwUAYPNmRTQAAJvU4YcfnkWLFuWJJ57Ib37zm3zgAx/Itttum8MOO6x8n+hHH300O++8c/r165eXX345y5Yty8EHH1weo2PHjjnwwANXW527//77l5+//PLLWbp0aQ466KByW+/evbPbbruVX3/sYx/LTjvtlJ133jknnnhi7rjjjrz77rvrrL9Dhw454IADyq8HDhyYXr16lWt55plnctlll6V79+7lxxe/+MW8/vrrjcZeuda1GTVqVObOnZs77rgjQ4YMyT333JM999wzDz30ULnPD3/4wxx88MGpra1N9+7d87WvfS2zZ89uNE5dXV169OhRfl1TU5M99tgj7dq1a9T2xhtvNNpvyJAhq71e24roZ555Jo888kij4x44cGCSbNF/cBIAgP8jiAYAYJPadddds8MOO+SRRx7JI488ksMOOyxJ0rdv3+y444557LHH8sgjj+TII4+seOxu3bpV1L9Hjx556qmnctddd2X77bfPxRdfnH333Xej/hjgO++8k3HjxuXpp58uP5599tm89NJL6dKlS8W19ujRI0cffXSuuOKKPPPMM/nIRz6Syy+/PEkyderUnHDCCfn7v//7/PznP8/vf//7XHjhhVm6dGmjMTp27NjodVVV1Rrbli9fviGHnOT/jvvoo49udNxPP/10XnrppfXe7gQAgM2fIBoAgE3uiCOOyKOPPppHH300hx9+eLn90EMPzQMPPJDHH3+8fH/oXXbZJZ06dcrvfve7cr9ly5bliSeeyB577LHWOXbZZZd07Nix0X2P33rrrbz44ouN+nXo0CFDhw7N1Vdfnf/6r//KrFmz8vDDD6913Pfeey9PPvlk+fXMmTMzf/787L777kmSD33oQ5k5c2Z23XXX1R4rr0DeEFVVVRk4cGAWLVqUJHnsscey00475cILL8z++++fAQMG5M9//vNGzbGylf945IrXK45zVR/60Ify3HPPpa6ubrXjrvR/EAAAsPnp0NIFAACw5TniiCMyatSoLFu2rLwiOkkOO+ywnHnmmVm6dGk5iO7WrVu+/OUv5/zzz0/v3r3Tr1+/XH311Xn33Xdz2mmnrXWO7t2757TTTsv555+fPn36ZLvttsuFF17YKAz++c9/nj/96U859NBDs/XWW+cXv/hFli9f3uj2Havq2LFjzjrrrFx//fXp0KFDzjzzzAwePDgHHnhgkuTiiy/OJz7xifTr1y+f+tSn0q5duzzzzDP5wx/+UF7J3BRPP/10Lrnkkpx44onZY4890qlTp0yZMiXf+9738tWvfjVJMmDAgMyePTt33313DjjggDQ0NOTee+9t8hzrc88992T//ffPIYcckjvuuCOPP/54br311jX2HTVqVG655ZZ89rOfzQUXXJDevXvnj3/8Y+6+++5897vfTfv27ZutLgAA2h5BNAAAm9wRRxyRv/3tbxk4cGBqamrK7Ycddljefvvt7Lbbbtl+++3L7RMmTMjy5ctz4okn5u23387++++fX/7yl9l6663XOc8111xTvmVEjx49ct5552XBggXl7b169cpPfvKTXHrppVm8eHEGDBiQu+66K3vuuedax9xqq63y1a9+NZ/73Ofy6quv5iMf+UijcHbYsGH5+c9/nssuuyxXXXVVOnbsmIEDB+YLX/hCRedohx12SF1dXcaNG5dZs2alqqqq/Prcc89Nknzyk5/MueeemzPPPDNLlizJiBEjctFFF+XSSy+taK61GTduXO6+++6cccYZ2X777XPXXXetdRV6375987vf/S5f/epXc9RRR2XJkiXZaaed8vGPf3yjV4IDAND2VZVKpVJLFwEAAG3BbbfdltGjR2/UPaTbiqqqqtx777059thjW7oUAAA2A5YmAAAAAABQKEE0AAAAAACFcmsOAAAAAAAKZUU0AAAAAACFEkQDAAAAAFAoQTQAAAAAAIUSRAMAAAAAUChBNAAAAAAAhRJEAwAAAABQKEE0AAAAAACFEkQDAAAAAFCo/weqR+VAc9pIYQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['input_len'] = data['text'].str.split().str.len()\n",
    "\n",
    "max_len = max(data['input_len'])\n",
    "\n",
    "print(f\"The maximum length of this dataset is approximately {max_len} words.\")\n",
    "\n",
    "n = max_len\n",
    "data.hist(\"input_len\", grid=False, figsize=(18,8), range=(0, n), bins=n)\n",
    "plt.suptitle('')\n",
    "plt.title('Histogram of Text Length')\n",
    "plt.ylabel('')\n",
    "plt.xlabel('Words per Sample')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trim Overly Lengthy Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NEW YORK -- Yields on most certificates of dep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Wall Street Journal Online&lt;/br&gt;&lt;/br&gt;The Mo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WASHINGTON -- In an effort to achieve banking ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The statistics on the enormous costs of employ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NEW YORK -- Indecision marked the dollar's ton...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7995</th>\n",
       "      <td>Secretary of Commerce Charles W. Sawyer said y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7996</th>\n",
       "      <td>U.S. stocks inched up last week, overcoming co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7997</th>\n",
       "      <td>Ben S. Bernanke cleared a key hurdle Thursday ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7998</th>\n",
       "      <td>The White House's push to contract out many fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7999</th>\n",
       "      <td>NEW YORK. April 17-Automobile stocks put on th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     NEW YORK -- Yields on most certificates of dep...\n",
       "1     The Wall Street Journal Online</br></br>The Mo...\n",
       "2     WASHINGTON -- In an effort to achieve banking ...\n",
       "3     The statistics on the enormous costs of employ...\n",
       "4     NEW YORK -- Indecision marked the dollar's ton...\n",
       "...                                                 ...\n",
       "7995  Secretary of Commerce Charles W. Sawyer said y...\n",
       "7996  U.S. stocks inched up last week, overcoming co...\n",
       "7997  Ben S. Bernanke cleared a key hurdle Thursday ...\n",
       "7998  The White House's push to contract out many fe...\n",
       "7999  NEW YORK. April 17-Automobile stocks put on th...\n",
       "\n",
       "[8000 rows x 1 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.drop(columns=[\"input_len\"])\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert DataFrame to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text'],\n",
       "    num_rows: 8000\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = Dataset.from_pandas(data)\n",
    "\n",
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Split Dataset into Training & Evaluation Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data shape: (6400, 1)\n",
      "Evaluation data shape: (1600, 1)\n",
      "{'text': Value(dtype='string', id=None)}\n"
     ]
    }
   ],
   "source": [
    "train_testvalid = dataset.train_test_split(test_size=0.20)\n",
    "\n",
    "ds = DatasetDict({\n",
    "    'train' : train_testvalid['train'],\n",
    "    'eval' : train_testvalid['test']\n",
    "})\n",
    "\n",
    "print('Training data shape:', ds['train'].shape)\n",
    "print('Evaluation data shape:', ds['eval'].shape)\n",
    "print(ds['train'].features)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Values/Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "BLOCK_SIZE = 128\n",
    "MODEL_CKPT = \"distilgpt2\"\n",
    "\n",
    "MODEL_NAME = MODEL_CKPT.split(\"/\")[-1] + \"-CLM_US_Economic_News_Articles\"\n",
    "set_seed = 42\n",
    "\n",
    "BATCH_SIZE = 1000 \n",
    "NUM_OF_EPOCHS = 3\n",
    "\n",
    "WEIGHT_DECAY = 0.01\n",
    "STRATEGY = \"epoch\"\n",
    "\n",
    "REPORTS_TO = \"tensorboard\"\n",
    "LEARNING_RATE = 2e-5"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenize Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_CKPT, use_fast=True)\n",
    "\n",
    "def tokenizer_function(samples):\n",
    "    return tokenizer(samples[\"text\"])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Tokenizer Function to Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7a6b2b234b24c1fa08eaf402f9b92c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1371 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "318d9d415b9541a49bb486fa914f6bec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask'],\n",
       "    num_rows: 6400\n",
       "})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_ds = ds.map(tokenizer_function, \n",
    "                      batched=True,\n",
    "                      remove_columns=ds[\"train\"].column_names,)\n",
    "\n",
    "tokenized_ds[\"train\"]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess Tokenized Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_texts(samples):\n",
    "    concatenated_examples = {k: sum(samples[k], []) for k in samples.keys()}\n",
    "    total_length = len(concatenated_examples[list(samples.keys())[0]])\n",
    "    total_length = (total_length // BLOCK_SIZE) * BLOCK_SIZE\n",
    "    result = {\n",
    "        k: [t[i : i + BLOCK_SIZE] for i in range(0, total_length, BLOCK_SIZE)]\n",
    "        for k, t in concatenated_examples.items()\n",
    "    }\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "    return result"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Preprocessing Function to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49710b9faf8248df8571218ff33687aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f3a3676f1df4c9ca0077ad214fbaaba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff8e5f6fd98b44d2b5a258099a60096c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68ad238bb81b46c391d88c16d4ee28a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee1b40c741a9478aa4f947699f61b3b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a2b12d6d31e4b109871ead02682e2dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "789c84ccb3d046c099ff9e97be1417b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47b09bafda29432baf897876c51dfe40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clm_ds = tokenized_ds.map(\n",
    "    group_texts,\n",
    "    batched=True,\n",
    "    num_proc=4,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(MODEL_CKPT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=MODEL_NAME,\n",
    "    evaluation_strategy=STRATEGY,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    weight_decay=WEIGHT_DECAY,\n",
    "    num_train_epochs=NUM_OF_EPOCHS,\n",
    "    report_to=REPORTS_TO,\n",
    "    logging_first_step=True,\n",
    "    hub_private_repo=True,\n",
    "    push_to_hub=True\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Data Collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/distilgpt2-CLM_US_Economic_News_Articles into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=clm_ds[\"train\"],\n",
    "    eval_dataset=clm_ds[\"eval\"],\n",
    "    data_collator=data_collator\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 14950\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 5607\n",
      "  Number of trainable parameters = 81912576\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f38ba08db7fd4ccfbe3dcdf9bf80cf29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 4.3704, 'learning_rate': 1.9996433030140896e-05, 'epoch': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-500\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-500/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-500/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.7389, 'learning_rate': 1.8216515070447654e-05, 'epoch': 0.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-500/pytorch_model.bin\n",
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-1000\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-1000/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-1000/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.6512, 'learning_rate': 1.643303014089531e-05, 'epoch': 0.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-1000/pytorch_model.bin\n",
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-1500\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-1500/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-1500/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.6225, 'learning_rate': 1.4649545211342966e-05, 'epoch': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-1500/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 3787\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c419c41ed8194171980ccdf277e59f8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.485297203063965, 'eval_runtime': 2160.3583, 'eval_samples_per_second': 1.753, 'eval_steps_per_second': 0.219, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-2000\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-2000/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-2000/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.5652, 'learning_rate': 1.286606028179062e-05, 'epoch': 1.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-2000/pytorch_model.bin\n",
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-2500\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-2500/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-2500/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.5178, 'learning_rate': 1.1082575352238275e-05, 'epoch': 1.34}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-2500/pytorch_model.bin\n",
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-3000\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-3000/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-3000/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.505, 'learning_rate': 9.299090422685929e-06, 'epoch': 1.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-3000/pytorch_model.bin\n",
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-3500\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-3500/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-3500/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.5092, 'learning_rate': 7.515605493133583e-06, 'epoch': 1.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-3500/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 3787\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7ed4bf4bc07434386a44c5bbc015319",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.455538749694824, 'eval_runtime': 2171.8891, 'eval_samples_per_second': 1.744, 'eval_steps_per_second': 0.218, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-4000\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-4000/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-4000/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.4834, 'learning_rate': 5.732120563581238e-06, 'epoch': 2.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-4000/pytorch_model.bin\n",
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-4500\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-4500/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-4500/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.4671, 'learning_rate': 3.948635634028893e-06, 'epoch': 2.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-4500/pytorch_model.bin\n",
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-5000\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-5000/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-5000/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.4611, 'learning_rate': 2.1651507044765475e-06, 'epoch': 2.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-5000/pytorch_model.bin\n",
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles/checkpoint-5500\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-5500/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-5500/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.4514, 'learning_rate': 3.816657749242019e-07, 'epoch': 2.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/checkpoint-5500/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 3787\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cd32c5b3ca04e339930d8abaade6d0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.44724440574646, 'eval_runtime': 2180.7631, 'eval_samples_per_second': 1.737, 'eval_steps_per_second': 0.217, 'epoch': 3.0}\n",
      "{'train_runtime': 85410.3703, 'train_samples_per_second': 0.525, 'train_steps_per_second': 0.066, 'train_loss': 3.541630748675982, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "train_results = trainer.train()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 3787\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "008c10fe872b4d82947c04146f69cc40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity: 31.41\n"
     ]
    }
   ],
   "source": [
    "evaluation_results = trainer.evaluate()\n",
    "print(f\"Perplexity: {math.exp(evaluation_results['eval_loss']):.2f}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Push Model to HuggingFace Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to distilgpt2-CLM_US_Economic_News_Articles\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/config.json\n",
      "Configuration saved in distilgpt2-CLM_US_Economic_News_Articles/generation_config.json\n",
      "Model weights saved in distilgpt2-CLM_US_Economic_News_Articles/pytorch_model.bin\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ff13fd4dbdf4eb2866377b13d2c7748",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4470c2aeb68b44799199a91003cd009d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75e3c326e3cf44e0976a25a58ee5e980",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "remote: Scanning LFS files of refs/heads/main for validity...        \n",
      "remote: LFS file scan complete.        \n",
      "To https://huggingface.co/DunnBC22/distilgpt2-CLM_US_Economic_News_Articles\n",
      "   9df1997..4099801  main -> main\n",
      "\n",
      "Dropping the following result as it does not have all the necessary fields:\n",
      "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n",
      "To https://huggingface.co/DunnBC22/distilgpt2-CLM_US_Economic_News_Articles\n",
      "   4099801..cd0ba72  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/distilgpt2-CLM_US_Economic_News_Articles/commit/4099801e21b12b35eaccc11adc98855c0e276eb9'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notes & Other Takeaways From This Project\n",
    "****\n",
    "- Not what I was looking for as far as results. I should try this project using GPT2 instead of DistilGPT2.\n",
    "****"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp_ml",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0b7a754c5aaaa44a21e2f2148f50b931a5abdffe7ef71537f5954179554f39df"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}