Text Summarization/Text-Summarized Data - Comparison/Flan-T5 - Text Summarization - 1 Epoch.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Summarization Using Flan-T5 Transformer Model\n",
    "\n",
    "Dataset Source: https://www.kaggle.com/datasets/cuitengfeui/textsummarization-data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Install Missing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sentencepiece in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (0.1.97)\n",
      "Requirement already satisfied: rouge-score in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (0.1.2)\n",
      "Requirement already satisfied: nltk in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (3.7)\n",
      "Requirement already satisfied: tensorboard in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (2.11.0)\n",
      "Requirement already satisfied: six>=1.14.0 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from rouge-score) (1.16.0)\n",
      "Requirement already satisfied: numpy in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from rouge-score) (1.24.2)\n",
      "Requirement already satisfied: absl-py in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from rouge-score) (1.2.0)\n",
      "Requirement already satisfied: joblib in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from nltk) (1.2.0)\n",
      "Requirement already satisfied: click in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from nltk) (8.0.4)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from nltk) (2022.9.13)\n",
      "Requirement already satisfied: tqdm in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from nltk) (4.64.1)\n",
      "Requirement already satisfied: protobuf<4,>=3.9.2 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (3.20.1)\n",
      "Requirement already satisfied: grpcio>=1.24.3 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (1.51.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (58.1.0)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (2.2.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (3.4.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (0.6.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (0.4.6)\n",
      "Requirement already satisfied: wheel>=0.26 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (0.38.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (2.28.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (1.8.1)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from tensorboard) (2.15.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard) (5.2.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from markdown>=2.6.8->tensorboard) (5.0.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard) (2.10)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard) (2.1.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard) (1.26.12)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from werkzeug>=1.0.1->tensorboard) (2.1.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard) (3.8.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard) (3.2.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install sentencepiece rouge-score nltk tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Importing Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "import os, sys\n",
    "os.environ['TOKENIZERS_PARALLELISM']='false'\n",
    "\n",
    "import torch\n",
    "import torch.nn\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "import nltk\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, set_seed, AutoModelForSeq2SeqLM\n",
    "from transformers import DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "\n",
    "from datasets import Dataset, DatasetDict, load_dataset\n",
    "\n",
    "import evaluate\n",
    "\n",
    "!git lfs install"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Versions of Some Necessary Libraries Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             Python: 3.9.12\n",
      "              NumPy: 1.24.2\n",
      "             Pandas: 1.5.0\n",
      "       Transformers: 4.26.1\n",
      "              Torch: 1.12.1\n"
     ]
    }
   ],
   "source": [
    "library_Spacing = 20\n",
    "print(\"Python:\".rjust(library_Spacing), sys.version[0:6])\n",
    "print(\"NumPy:\".rjust(library_Spacing), np.__version__)\n",
    "print(\"Pandas:\".rjust(library_Spacing), pd.__version__)\n",
    "print(\"Transformers:\".rjust(library_Spacing), transformers.__version__)\n",
    "print(\"Torch:\".rjust(library_Spacing), torch.__version__)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest & Start Preprocessing Training Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jason blake of the islanders will miss the res...</td>\n",
       "      <td>blake missing rest of season</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>the u.s. military on wednesday captured a wife...</td>\n",
       "      <td>u.s. arrests wife and daughter of saddam deput...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>craig bellamy 's future at west ham appeared i...</td>\n",
       "      <td>west ham drops bellamy amid transfer turmoil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cambridge - when barack obama sought advice be...</td>\n",
       "      <td>in search for expertise harvard looms large</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wall street held on to steep gains on monday ,...</td>\n",
       "      <td>wall street ends a three-day losing streak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>a rapidly growing world population and threat ...</td>\n",
       "      <td>u.n. food agency must help more people with le...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>ahead of nuclear talks between senior iranian ...</td>\n",
       "      <td>us urges iran to change course on its nuclear ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>airbus ceo thomas enders dismissed a french in...</td>\n",
       "      <td>airbus ceo thomas enders calls eads insider tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>albania 's central bank said friday it has iss...</td>\n",
       "      <td>central bank licenses a local bank</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>south africa 's deputy president phumzile mlam...</td>\n",
       "      <td>s. african deputy president burgled</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19966 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 article  \\\n",
       "0      jason blake of the islanders will miss the res...   \n",
       "1      the u.s. military on wednesday captured a wife...   \n",
       "2      craig bellamy 's future at west ham appeared i...   \n",
       "3      cambridge - when barack obama sought advice be...   \n",
       "4      wall street held on to steep gains on monday ,...   \n",
       "...                                                  ...   \n",
       "19995  a rapidly growing world population and threat ...   \n",
       "19996  ahead of nuclear talks between senior iranian ...   \n",
       "19997  airbus ceo thomas enders dismissed a french in...   \n",
       "19998  albania 's central bank said friday it has iss...   \n",
       "19999  south africa 's deputy president phumzile mlam...   \n",
       "\n",
       "                                                 summary  \n",
       "0                           blake missing rest of season  \n",
       "1      u.s. arrests wife and daughter of saddam deput...  \n",
       "2           west ham drops bellamy amid transfer turmoil  \n",
       "3            in search for expertise harvard looms large  \n",
       "4             wall street ends a three-day losing streak  \n",
       "...                                                  ...  \n",
       "19995  u.n. food agency must help more people with le...  \n",
       "19996  us urges iran to change course on its nuclear ...  \n",
       "19997  airbus ceo thomas enders calls eads insider tr...  \n",
       "19998                 central bank licenses a local bank  \n",
       "19999                s. african deputy president burgled  \n",
       "\n",
       "[19966 rows x 2 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parent_dir = \"/Users/briandunn/Desktop/nlp_ds/Text Summarization/text-summarized data\"\n",
    "\n",
    "train_data_file = os.path.join(parent_dir, \"train.csv\")\n",
    "\n",
    "train_df = pd.read_csv(train_data_file)\n",
    "\n",
    "train_df.drop_duplicates(inplace=True)\n",
    "\n",
    "train_df.dropna(inplace=True)\n",
    "\n",
    "train_df.rename(columns={\"document\": \"article\", \"title\": \"summary\"}, inplace=True)\n",
    "\n",
    "train_df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest & Start Preprocessing Evaluation Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mr. emmons also was part of the design team fo...</td>\n",
       "      <td>donn emmons architect of northern california l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>the secret 's out : in `` in &amp;amp; out , '' to...</td>\n",
       "      <td>in &amp;amp; out brings gay sensibilities to mains...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>indonesia 's top security official said thursd...</td>\n",
       "      <td>top security minister suspects foreign terrori...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;unk&gt; on the tongue , pungent to the nose , gi...</td>\n",
       "      <td>warm up to ginger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>thirty years after they were written off as de...</td>\n",
       "      <td>vietnamese commandos seek payment from us</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>`` who wants to die with money in the stock ma...</td>\n",
       "      <td>records set for ## contemporary artists</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>the iraqi government 's failure to grasp the s...</td>\n",
       "      <td>iraq far behind in mine clearance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>the provisional results of the european parlia...</td>\n",
       "      <td>&lt;unk&gt; to remain biggest party group in europea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>british prime minister tony blair saturday wel...</td>\n",
       "      <td>gore blair agree to boost cyber commerce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>a small earthquake rocked russia 's buryatia r...</td>\n",
       "      <td>small earthquake reported in siberia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               article  \\\n",
       "0    mr. emmons also was part of the design team fo...   \n",
       "1    the secret 's out : in `` in &amp; out , '' to...   \n",
       "2    indonesia 's top security official said thursd...   \n",
       "3    <unk> on the tongue , pungent to the nose , gi...   \n",
       "4    thirty years after they were written off as de...   \n",
       "..                                                 ...   \n",
       "995  `` who wants to die with money in the stock ma...   \n",
       "996  the iraqi government 's failure to grasp the s...   \n",
       "997  the provisional results of the european parlia...   \n",
       "998  british prime minister tony blair saturday wel...   \n",
       "999  a small earthquake rocked russia 's buryatia r...   \n",
       "\n",
       "                                               summary  \n",
       "0    donn emmons architect of northern california l...  \n",
       "1    in &amp; out brings gay sensibilities to mains...  \n",
       "2    top security minister suspects foreign terrori...  \n",
       "3                                    warm up to ginger  \n",
       "4            vietnamese commandos seek payment from us  \n",
       "..                                                 ...  \n",
       "995            records set for ## contemporary artists  \n",
       "996                  iraq far behind in mine clearance  \n",
       "997  <unk> to remain biggest party group in europea...  \n",
       "998           gore blair agree to boost cyber commerce  \n",
       "999               small earthquake reported in siberia  \n",
       "\n",
       "[1000 rows x 2 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_data_file = os.path.join(parent_dir, \"val.csv\")\n",
    "\n",
    "eval_df = pd.read_csv(eval_data_file)\n",
    "\n",
    "eval_df.drop_duplicates(inplace=True)\n",
    "\n",
    "eval_df.dropna(inplace=True)\n",
    "\n",
    "eval_df.rename(columns={\"document\": \"article\", \"title\": \"summary\"}, inplace=True)\n",
    "\n",
    "eval_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove Unnecessary Whitespace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df[\"article\"] = train_df[\"article\"].str.strip()\n",
    "train_df[\"summary\"] = train_df[\"summary\"].str.strip()\n",
    "eval_df[\"article\"] = eval_df[\"article\"].str.strip()\n",
    "eval_df[\"summary\"] = eval_df[\"summary\"].str.strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize Length of Each Article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The longest string (in words) in this dataset is 74 words.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/YAAAHWCAYAAADdKxJLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVCElEQVR4nO3df3zN9f//8fvZ7Cf2C9ushqX8jIgs+V0zJBG9/Whv7yXRW1OxUrzffk0iQt5+RCLyfk+qdyV5R5bftGZoCPkRRdhWrZn5sc12vn/0cb5Oszlnztk5x27Xy2WXnOfreV6vx+t4XKb7ef0yGI1GowAAAAAAgEtyc3QBAAAAAACg7Aj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAA3OKefPJJ1alTx+r31alTR08++aTN63E1P/74owwGg2bMmOHoUgAAuC6CPQAATuKtt96SwWBQZGSk1e89c+aMJk6cqLS0NNsXZiMdO3bU3Xff7egySvTFF19o4sSJji4DAACrEewBAHASiYmJqlOnjnbu3Kljx45Z9d4zZ84oISHhusH+nXfe0eHDh21U5a3riy++UEJCgqPLAADAagR7AACcwIkTJ/T1119r1qxZqlGjhhITEy1635UrV5Sfn1/qHA8PD3l5edmiTAAA4IQI9gAAOIHExEQFBgaqe/fuevzxx68b7K+91nv27NmqW7euvLy89NZbb+m+++6TJA0aNEgGg0EGg0HLli2TdP1r7IuKivSvf/1LTZo0kbe3t2rUqKGuXbtq165dpdaZnZ2tESNGKDw8XF5eXrrzzjs1bdo0FRUV2eRzkKS1a9eqXbt2qly5sqpWraru3bvrwIEDZnOefPJJValSRadPn1avXr1UpUoV1ahRQy+99JIKCwvN5v72228aOHCg/Pz8FBAQoNjYWO3du7fYZzR//nxJMn1+BoOhWG2LFi0yfe733XefUlNTbbbfAACUVSVHFwAAAP4I9r1795anp6cGDBigBQsWKDU11RTYr7V06VJdvnxZQ4cOlZeXlx577DGdP39e48eP19ChQ9WuXTtJ0gMPPFDi9gYPHqxly5apW7duevrpp3XlyhVt27ZN33zzjVq2bHnd91y8eFEdOnTQ6dOn9cwzz6hWrVr6+uuvNWbMGJ09e1azZ8++6c/h3//+t2JjY9WlSxdNmzZNFy9e1IIFC9S2bVt9++23Zl9QFBYWqkuXLoqMjNSMGTP01VdfaebMmapbt66GDRsm6Y8vMHr06KGdO3dq2LBhatCggT777DPFxsaabfeZZ57RmTNnlJSUpH//+9/XrW3FihU6f/68nnnmGRkMBk2fPl29e/fW8ePH5eHhcdP7DgBAmRkBAIBD7dq1yyjJmJSUZDQajcaioiLj7bffbnzhhRfM5p04ccIoyejn52fMzMw0W5aammqUZFy6dGmx9cfGxhpr165ter1x40ajJOPzzz9fbG5RUZHpz7Vr1zbGxsaaXr/66qvGypUrG48cOWL2ntGjRxvd3d2NJ0+eLHU/O3ToYGzcuHGJy8+fP28MCAgwDhkyxGw8PT3d6O/vbzYeGxtrlGScNGmS2dzmzZsbW7RoYXr98ccfGyUZZ8+ebRorLCw0Pvjgg8U+r7i4OOP1/tfo6uderVo1Y1ZWlmn8s88+M0oyfv7556XuNwAA9sap+AAAOFhiYqJCQkLUqVMnSX+cCt6vXz+tXLmy2GnlktSnTx/VqFGjzNv7+OOPZTAYNGHChGLLrnf6+VUfffSR2rVrp8DAQP3666+mn6ioKBUWFmrr1q1lrkmSkpKSlJ2drQEDBpit393dXZGRkdq0aVOx9/z97383e92uXTsdP37c9HrdunXy8PDQkCFDTGNubm6Ki4uzur5+/fopMDDQbFuSzLYHAIAjcCo+AAAOVFhYqJUrV6pTp046ceKEaTwyMlIzZ87Uhg0bFB0dbfaeiIiIm9rmDz/8oLCwMAUFBVn1vqNHj2rfvn0lfqmQmZl5U3UdPXpUkvTggw9ed7mfn5/Z66v3BrhWYGCgfv/9d9Prn376STVr1pSvr6/ZvDvvvNPq+mrVqlVsW5LMtgcAgCMQ7AEAcKCNGzfq7NmzWrlypVauXFlseWJiYrFg7+PjU17lmSkqKlLnzp318ssvX3d5vXr1bnr90h/X2YeGhhZbXqmS+f+2uLu739T2rFXS9oxGY7nWAQDAnxHsAQBwoMTERAUHB5vuyH6tTz75RJ9++qkWLlx4wzBf2in0f1a3bl19+eWXysrKsuqofd26dZWbm6uoqCiL32ONunXrSpKCg4Ntto3atWtr06ZNunjxotlR+2PHjhWba81nCACAM+EaewAAHOTSpUv65JNP9Mgjj+jxxx8v9jN8+HCdP39eq1evvuG6KleuLOmPx9HdSJ8+fWQ0GpWQkFBsWWlHn/v27avk5GR9+eWXxZZlZ2frypUrN9x2abp06SI/Pz9NmTJFBQUFxZb/8ssvZVpnQUGB3nnnHdNYUVHRdb9IseYzBADAmXDEHgAAB1m9erXOnz+vRx999LrL77//ftWoUUOJiYnq169fqeuqW7euAgICtHDhQlWtWlWVK1dWZGTkda/H79SpkwYOHKg5c+bo6NGj6tq1q4qKirRt2zZ16tRJw4cPv+42Ro0apdWrV+uRRx7Rk08+qRYtWujChQvav3+//vvf/+rHH39U9erVS63zl19+0eTJk4uNR0REKCYmRgsWLNDAgQN17733qn///qpRo4ZOnjyp//3vf2rTpo3mzZtX6vr/rFevXmrVqpVefPFFHTt2TA0aNNDq1auVlZUlyfwofYsWLSRJzz//vLp06SJ3d3f179/fqu0BAOAIBHsAABwkMTFR3t7e6ty583WXu7m5qXv37kpMTNRvv/1W6ro8PDz03nvvacyYMfr73/+uK1euaOnSpSXeaG/p0qVq2rSplixZolGjRsnf318tW7bUAw88UOI2fH19tWXLFk2ZMkUfffSRli9fLj8/P9WrV08JCQny9/e/4T5nZmZq3LhxxcYfeughxcTE6IknnlBYWJhef/11vfHGG8rLy9Ntt92mdu3aadCgQTdc/5+5u7vrf//7n1544QW99957cnNz02OPPaYJEyaoTZs28vb2Ns3t3bu3nnvuOa1cuVL/+c9/ZDQaCfYAAJdgMHLHFwAAUMGsWrVKjz32mLZv3642bdo4uhwAAG4KwR4AANzSLl26ZHbzwcLCQkVHR2vXrl1KT0932FMGAACwFU7FBwAAt7TnnntOly5dUuvWrZWXl6dPPvlEX3/9taZMmUKoBwDcEjhiDwAAbmkrVqzQzJkzdezYMV2+fFl33nmnhg0bVuJNAgEAcDUEewAAAAAAXBjPsQcAAAAAwIUR7AEAAAAAcGHcPM8CRUVFOnPmjKpWrSqDweDocgAAAAAAtzij0ajz588rLCxMbm6lH5Mn2FvgzJkzCg8Pd3QZAAAAAIAK5tSpU7r99ttLnUOwt0DVqlUl/fGB+vn5Obia0hUUFGj9+vWKjo6Wh4eHo8sBiqFH4ezoUTg7ehTOjh6Fs3OVHs3JyVF4eLgpj5aGYG+Bq6ff+/n5uUSw9/X1lZ+fn1M3KSouehTOjh6Fs6NH4ezoUTg7V+tRSy4Hd+jN87Zu3aoePXooLCxMBoNBq1atMi0rKCjQK6+8oiZNmqhy5coKCwvT3/72N505c8ZsHVlZWYqJiZGfn58CAgI0ePBg5ebmms3Zt2+f2rVrJ29vb4WHh2v69OnlsXsAAAAAANidQ4P9hQsXdM8992j+/PnFll28eFF79uzRuHHjtGfPHn3yySc6fPiwHn30UbN5MTExOnDggJKSkrRmzRpt3bpVQ4cONS3PyclRdHS0ateurd27d+uNN97QxIkTtWjRIrvvHwAAAAAA9ubQU/G7deumbt26XXeZv7+/kpKSzMbmzZunVq1a6eTJk6pVq5YOHTqkdevWKTU1VS1btpQkzZ07Vw8//LBmzJihsLAwJSYmKj8/X++++648PT3VuHFjpaWladasWWZfAAAAAAAA4Ipc6hr7c+fOyWAwKCAgQJKUnJysgIAAU6iXpKioKLm5uSklJUWPPfaYkpOT1b59e3l6eprmdOnSRdOmTdPvv/+uwMDAYtvJy8tTXl6e6XVOTo6kPy4PKCgosNPe2cbV+py9TlRc9CicHT0KZ0ePwtnRo3B2rtKj1tTnMsH+8uXLeuWVVzRgwADTDezS09MVHBxsNq9SpUoKCgpSenq6aU5ERITZnJCQENOy6wX7qVOnKiEhodj4+vXr5evra5P9sbc/n+0AOBt6FM6OHoWzo0fh7OhRODtn79GLFy9aPNclgn1BQYH69u0ro9GoBQsW2H17Y8aMUXx8vOn11ccMREdHu8Rd8ZOSktS5c2eXuMMjKh56FM6OHoWzo0fh7OhRODtX6dGrZ45bwumD/dVQ/9NPP2njxo1mwTo0NFSZmZlm869cuaKsrCyFhoaa5mRkZJjNufr66pw/8/LykpeXV7FxDw8Pp/6Lv5Yr1YqKiR6Fs6NH4ezoUTg7ehTOztl71JraHHpX/Bu5GuqPHj2qr776StWqVTNb3rp1a2VnZ2v37t2msY0bN6qoqEiRkZGmOVu3bjW7PiEpKUn169e/7mn4AAAAAAC4EocG+9zcXKWlpSktLU2SdOLECaWlpenkyZMqKCjQ448/rl27dikxMVGFhYVKT09Xenq68vPzJUkNGzZU165dNWTIEO3cuVM7duzQ8OHD1b9/f4WFhUmSnnjiCXl6emrw4ME6cOCAPvjgA/3rX/8yO9UeAAAAAABX5dBT8Xft2qVOnTqZXl8N27GxsZo4caJWr14tSWrWrJnZ+zZt2qSOHTtKkhITEzV8+HA99NBDcnNzU58+fTRnzhzTXH9/f61fv15xcXFq0aKFqlevrvHjx/OoOwAAAADALcGhwb5jx44yGo0lLi9t2VVBQUFasWJFqXOaNm2qbdu2WV0fAAAAAADOzqmvsQcAAAAAAKUj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALsyhN88DADgHg+HGcyy4nykAAAAcgCP2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCuHkeAFQAltwcDwAAAK6JI/YAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACAC+M59gDg4nhGPQAAQMVGsAcAWORGXyAYjeVTBwAAAMxxKj4AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALq+ToAgAApTMYHF0BAAAAnBlH7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYQR7AAAAAABcGMEeAAAAAAAXRrAHAAAAAMCFEewBAAAAAHBhBHsAAAAAAFyYQ4P91q1b1aNHD4WFhclgMGjVqlVmy41Go8aPH6+aNWvKx8dHUVFROnr0qNmcrKwsxcTEyM/PTwEBARo8eLByc3PN5uzbt0/t2rWTt7e3wsPDNX36dHvvGgAAAAAA5cKhwf7ChQu65557NH/+/Osunz59uubMmaOFCxcqJSVFlStXVpcuXXT58mXTnJiYGB04cEBJSUlas2aNtm7dqqFDh5qW5+TkKDo6WrVr19bu3bv1xhtvaOLEiVq0aJHd9w8AAAAAAHur5MiNd+vWTd26dbvuMqPRqNmzZ2vs2LHq2bOnJGn58uUKCQnRqlWr1L9/fx06dEjr1q1TamqqWrZsKUmaO3euHn74Yc2YMUNhYWFKTExUfn6+3n33XXl6eqpx48ZKS0vTrFmzzL4AAAAAAADAFTk02JfmxIkTSk9PV1RUlGnM399fkZGRSk5OVv/+/ZWcnKyAgABTqJekqKgoubm5KSUlRY899piSk5PVvn17eXp6muZ06dJF06ZN0++//67AwMBi287Ly1NeXp7pdU5OjiSpoKBABQUF9thdm7lan7PXiYqLHrWej4+jK7DMrfJXSo/C2dGjcHb0KJydq/SoNfU5bbBPT0+XJIWEhJiNh4SEmJalp6crODjYbHmlSpUUFBRkNiciIqLYOq4uu16wnzp1qhISEoqNr1+/Xr6+vmXco/KVlJTk6BKAUtGjlnv/fUdXYJkvvnB0BbZFj8LZ0aNwdvQonJ2z9+jFixctnuu0wd6RxowZo/j4eNPrnJwchYeHKzo6Wn5+fg6s7MYKCgqUlJSkzp07y8PDw9HlAMXQo9bz93d0BZY5d87RFdgGPQpnR4/C2dGjcHau0qNXzxy3hNMG+9DQUElSRkaGatasaRrPyMhQs2bNTHMyMzPN3nflyhVlZWWZ3h8aGqqMjAyzOVdfX53zZ15eXvLy8io27uHh4dR/8ddypVpRMdGjlrt0ydEVWOZW++ukR+Hs6FE4O3oUzs7Ze9Sa2pz2OfYREREKDQ3Vhg0bTGM5OTlKSUlR69atJUmtW7dWdna2du/ebZqzceNGFRUVKTIy0jRn69atZtcnJCUlqX79+tc9DR8AAAAAAFfi0GCfm5urtLQ0paWlSfrjhnlpaWk6efKkDAaDRowYocmTJ2v16tXav3+//va3vyksLEy9evWSJDVs2FBdu3bVkCFDtHPnTu3YsUPDhw9X//79FRYWJkl64okn5OnpqcGDB+vAgQP64IMP9K9//cvsVHsAAAAAAFyVQ0/F37Vrlzp16mR6fTVsx8bGatmyZXr55Zd14cIFDR06VNnZ2Wrbtq3WrVsnb29v03sSExM1fPhwPfTQQ3Jzc1OfPn00Z84c03J/f3+tX79ecXFxatGihapXr67x48fzqDsAAAAAwC3BocG+Y8eOMhqNJS43GAyaNGmSJk2aVOKcoKAgrVixotTtNG3aVNu2bStznQAAAAAAOCunvcYeAAAAAADcmNPeFR8AKgKDwdEVAAAAwNVxxB4AAAAAABdGsAcAAAAAwIUR7AEAAAAAcGFcYw8AsAlL7hdQyoNQAAAAUEYcsQcAAAAAwIVxxB4AUG5udFSfI/oAAADW44g9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCuMYeAOzIkjvFAwAAADeDI/YAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MKuDfUZGhgYOHKiwsDBVqlRJ7u7uZj8AAAAAAKD8VLL2DU8++aROnjypcePGqWbNmjIYDPaoCwAAAAAAWMDqYL99+3Zt27ZNzZo1s0M5AAAAAADAGlafih8eHi6j0WiPWgAAAAAAgJWsDvazZ8/W6NGj9eOPP9qhHAAAAAAAYA2LTsUPDAw0u5b+woULqlu3rnx9feXh4WE2Nysry7YVAgAAAACAElkU7GfPnm3nMgAAAAAAQFlYFOxjY2PtXQcAAAAAACgDq6+xd3d3V2ZmZrHx3377jefYAwAAAABQzqwO9iXdET8vL0+enp43XRAAAAAAALCcxc+xnzNnjiTJYDBo8eLFqlKlimlZYWGhtm7dqgYNGti+QgAAAAAAUCKLg/2bb74p6Y8j9gsXLjQ77d7T01N16tTRwoULbV8hAAAAAAAokcXB/sSJE5KkTp066ZNPPlFgYKDdigIAAAAAAJaxONhftWnTJnvUAQAAAAAAysDqYB8fH3/dcYPBIG9vb915553q2bOngoKCbro4AAAAAABQOquD/bfffqs9e/aosLBQ9evXlyQdOXJE7u7uatCggd566y29+OKL2r59uxo1amTzggEAAAAAwP9n9ePuevbsqaioKJ05c0a7d+/W7t279fPPP6tz584aMGCATp8+rfbt22vkyJH2qBcAAAAAAFzD6mD/xhtv6NVXX5Wfn59pzN/fXxMnTtT06dPl6+ur8ePHa/fu3TYtFAAAAAAAFGd1sD937pwyMzOLjf/yyy/KycmRJAUEBCg/P//mqwMAAAAAAKUq06n4Tz31lD799FP9/PPP+vnnn/Xpp59q8ODB6tWrlyRp586dqlevnq1rBQCnYjDc+AcAAACwN6tvnvf2229r5MiR6t+/v65cufLHSipVUmxsrN58801JUoMGDbR48WLbVgoAuOXd6MsQo7F86gAAAHAlVgf7KlWq6J133tGbb76p48ePS5LuuOMOValSxTSnWbNmNisQAAAAAACUzOpgf1WVKlXUtGlTW9YCAAAAAACsZHWwv3Dhgl5//XVt2LBBmZmZKioqMlt+9Sg+AAAAAACwP6uD/dNPP60tW7Zo4MCBqlmzpgzcHQoAAAAAAIexOtivXbtW//vf/9SmTRt71AMAAAAAAKxg9ePuAgMDFRQUZI9aAAAAAACAlawO9q+++qrGjx+vixcv2qMeAAAAAABgBatPxZ85c6Z++OEHhYSEqE6dOvLw8DBbvmfPHpsVBwAAAAAASmd1sO/Vq5cdygAAAAAAAGVhdbCfMGGCPeoAAAAAAABlYPU19pKUnZ2txYsXa8yYMcrKypL0xyn4p0+ftmlxhYWFGjdunCIiIuTj46O6devq1VdfldFoNM0xGo0aP368atasKR8fH0VFReno0aNm68nKylJMTIz8/PwUEBCgwYMHKzc316a1AgAAAADgCFYH+3379qlevXqaNm2aZsyYoezsbEnSJ598ojFjxti0uGnTpmnBggWaN2+eDh06pGnTpmn69OmaO3euac706dM1Z84cLVy4UCkpKapcubK6dOmiy5cvm+bExMTowIEDSkpK0po1a7R161YNHTrUprUCAAAAAOAIVgf7+Ph4Pfnkkzp69Ki8vb1N4w8//LC2bt1q0+K+/vpr9ezZU927d1edOnX0+OOPKzo6Wjt37pT0x9H62bNna+zYserZs6eaNm2q5cuX68yZM1q1apUk6dChQ1q3bp0WL16syMhItW3bVnPnztXKlSt15swZm9YLAAAAAEB5s/oa+9TUVL399tvFxm+77Talp6fbpKirHnjgAS1atEhHjhxRvXr1tHfvXm3fvl2zZs2SJJ04cULp6emKiooyvcff31+RkZFKTk5W//79lZycrICAALVs2dI0JyoqSm5ubkpJSdFjjz1WbLt5eXnKy8szvc7JyZEkFRQUqKCgwKb7aGtX63P2OlFx3Uo96uPj6AoqnvJom1upR3Frokfh7OhRODtX6VFr6rM62Ht5eZmC7rWOHDmiGjVqWLu6Uo0ePVo5OTlq0KCB3N3dVVhYqNdee00xMTGSZPoiISQkxOx9ISEhpmXp6ekKDg42W16pUiUFBQWV+EXE1KlTlZCQUGx8/fr18vX1ven9Kg9JSUmOLgEo1a3Qo++/7+gKKp4vvii/bd0KPYpbGz0KZ0ePwtk5e49evHjR4rlWB/tHH31UkyZN0ocffihJMhgMOnnypF555RX16dPH2tWV6sMPP1RiYqJWrFihxo0bKy0tTSNGjFBYWJhiY2Ntuq1rjRkzRvHx8abXOTk5Cg8PV3R0tPz8/Oy2XVsoKChQUlKSOnfuLA8PD0eXAxRzK/Wov7+jK6h4zp2z/zZupR7FrYkehbOjR+HsXKVHr3dAvSRWB/uZM2fq8ccfV3BwsC5duqQOHTooPT1d999/v1577TVrV1eqUaNGafTo0erfv78kqUmTJvrpp580depUxcbGKjQ0VJKUkZGhmjVrmt6XkZGhZs2aSZJCQ0OVmZlptt4rV64oKyvL9P4/8/LykpeXV7FxDw8Pp/6Lv5Yr1YqK6Vbo0UuXHF1BxVOeLXMr9ChubfQonB09Cmfn7D1qTW1WB3t/f38lJSVpx44d2rt3r3Jzc3XvvfeaXeduKxcvXpSbm/n9/dzd3VVUVCRJioiIUGhoqDZs2GAK8jk5OUpJSdGwYcMkSa1bt1Z2drZ2796tFi1aSJI2btyooqIiRUZG2rxmAAAAAADKk9XB/qo2bdqoTZs2ptfff/+9Hn30UR05csQmhUlSjx499Nprr6lWrVpq3Lixvv32W82aNUtPPfWUpD8uAxgxYoQmT56su+66SxERERo3bpzCwsLUq1cvSVLDhg3VtWtXDRkyRAsXLlRBQYGGDx+u/v37KywszGa1Arj1GAyOrgAAAAC4sTIH+z/Ly8vTDz/8YKvVSZLmzp2rcePG6dlnn1VmZqbCwsL0zDPPaPz48aY5L7/8si5cuKChQ4cqOztbbdu21bp168wexZeYmKjhw4froYcekpubm/r06aM5c+bYtFYAAAAAABzBZsHeHqpWrarZs2dr9uzZJc4xGAyaNGmSJk2aVOKcoKAgrVixwg4VAgAAAADgWG43ngIAAAAAAJwVwR4AAAAAABdm8an4gYGBMpRyJ6krV67YpCAAAAAAAGA5i4N9ade5AwAAAAAAx7A42MfGxtqzDgAAbsiSRxAajfavAwAAwJlwjT0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCrg/2kSZN08eLFYuOXLl3SpEmTbFIUAAAAAACwjNXBPiEhQbm5ucXGL168qISEBJsUBQAAAAAALGN1sDcajTJc53lDe/fuVVBQkE2KAgAAAAAAlrH4OfaBgYEyGAwyGAyqV6+eWbgvLCxUbm6u/v73v9ulSAAAAAAAcH0WB/vZs2fLaDTqqaeeUkJCgvz9/U3LPD09VadOHbVu3douRQIAAAAAgOuzONjHxsZKkiIiIvTAAw/Iw8PDbkUBAAAAAADLWBzsr+rQoYPpz5cvX1Z+fr7Zcj8/v5uvCgAAAAAAWMTqm+ddvHhRw4cPV3BwsCpXrqzAwECzHwAAAAAAUH6sDvajRo3Sxo0btWDBAnl5eWnx4sVKSEhQWFiYli9fbo8aAQAAAABACaw+Ff/zzz/X8uXL1bFjRw0aNEjt2rXTnXfeqdq1aysxMVExMTH2qBMAAAAAAFyH1Ufss7KydMcdd0j643r6rKwsSVLbtm21detW21YHAAAAAABKZXWwv+OOO3TixAlJUoMGDfThhx9K+uNIfkBAgE2LAwAAAAAApbM62A8aNEh79+6VJI0ePVrz58+Xt7e3Ro4cqVGjRtm8QAAAAAAAUDKrr7EfOXKk6c9RUVH6/vvvtXv3bt15551q2rSpTYsDAAAAAAClszrY/1nt2rVVu3ZtW9QCAAAAAACsVKZgv2HDBm3YsEGZmZkqKioyW/buu+/apDAAAAAAAHBjVgf7hIQETZo0SS1btlTNmjVlMBjsURcAAAAAALCA1cF+4cKFWrZsmQYOHGiPegCg3PC9JAAAAG4FVt8VPz8/Xw888IA9agEAAAAAAFayOtg//fTTWrFihT1qAQAAAAAAVrLoVPz4+HjTn4uKirRo0SJ99dVXatq0qTw8PMzmzpo1y7YVAgAAAACAElkU7L/99luz182aNZMkfffdd2bj3EgPAAAAAIDyZVGw37Rpk73rAAAAAAAAZWDxNfbu7u7KzMy0Zy0AAAAAAMBKFgd7o9FozzoAAAAAAEAZWH1XfAAAAAAA4Dwsusb+qsWLF6tKlSqlznn++edvqiAAAAAAAGA5q4L9woUL5e7uXuJyg8FAsAcAAAAAoBxZFex37dql4OBge9UCAAAAAACsZPE19jyjHgAAAAAA58Nd8QEAAAAAcGEWB/sJEybc8MZ5AAAAAACgfFl8jf2ECRPsWQcAAAAAACgDnmMPAAAAAIALI9gDAAAAAODCLA72Fy9etGcdAAAAAACgDCwO9tWrV9cjjzyiRYsWKT093Z41AQAAAAAAC1kc7L///nt16dJFH374oerUqaPIyEi99tpr2r9/vz3rAwAAAAAApbA42NeqVUvPPfecvvrqK2VkZGjEiBHav3+/2rVrpzvuuEMjRozQxo0bVVhYaM96AQAAAADANcp08zx/f38NGDBAK1eu1C+//KK3335bhYWFGjRokGrUqKHExERb1wkAAAAAAK7D4ufYl8TDw0OdO3dW586dNXfuXH377be6cuWKLWoDgDIzGBxdAQAAAFA+rA727du3V8eOHdWhQwe1adNG3t7eZsubN29us+IAAAAAAEDprD4VPzo6Wt9884169uypgIAAtW3bVmPHjlVSUhKPxAMAOJzBUPoPAADArcbqYD927FitX79e2dnZ2rRpkx555BHt2rVL3bt3V1BQkM0LPH36tP7617+qWrVq8vHxUZMmTbRr1y7TcqPRqPHjx6tmzZry8fFRVFSUjh49araOrKwsxcTEyM/PTwEBARo8eLByc3NtXisAAAAAAOWtTDfPk6Tjx49r//792rt3r/bt26eqVauqW7dutqxNv//+u9q0aSMPDw+tXbtWBw8e1MyZMxUYGGiaM336dM2ZM0cLFy5USkqKKleurC5duujy5cumOTExMTpw4ICSkpK0Zs0abd26VUOHDrVprQAAAAAAOILV19g/8cQT2rJli/Ly8tS+fXt16NBBo0ePVtOmTWWw8TmO06ZNU3h4uJYuXWoai4iIMP3ZaDRq9uzZGjt2rHr27ClJWr58uUJCQrRq1Sr1799fhw4d0rp165SamqqWLVtKkubOnauHH35YM2bMUFhYmE1rBgAAAACgPFkd7FeuXKnq1avr6aef1oMPPqi2bdvK19fXHrVp9erV6tKli/7yl79oy5Ytuu222/Tss89qyJAhkqQTJ04oPT1dUVFRpvf4+/srMjJSycnJ6t+/v5KTkxUQEGAK9ZIUFRUlNzc3paSk6LHHHiu23by8POXl5Zle5+TkSJIKCgpUUFBgl321lav1OXudqLjKq0d9fOy6eriwG7Uev0fh7OhRODt6FM7OVXrUmvqsDva//fabtm3bps2bN2vMmDE6dOiQmjVrpo4dO6pjx46Kjo62dpUlOn78uBYsWKD4+Hj94x//UGpqqp5//nl5enoqNjZW6enpkqSQkBCz94WEhJiWpaenKzg42Gx5pUqVFBQUZJrzZ1OnTlVCQkKx8fXr19vtSwxbS0pKcnQJQKns3aPvv2/X1cOFffGFZfP4PQpnR4/C2dGjcHbO3qPW3Jze6mAfGBioRx99VI8++qgk6dixY5o8ebLeeOMNTZs2TYWFhdauskRFRUVq2bKlpkyZIumPR+l99913WrhwoWJjY222nT8bM2aM4uPjTa9zcnIUHh6u6Oho+fn52W27tlBQUKCkpCR17txZHh4eji4HKKa8etTf326rhos7d6705fwehbOjR+Hs6FE4O1fp0atnjluiTEfst2zZos2bN2vz5s06ePCgAgIC1KNHD3Xo0MHa1ZWqZs2aatSokdlYw4YN9fHHH0uSQkNDJUkZGRmqWbOmaU5GRoaaNWtmmpOZmWm2jitXrigrK8v0/j/z8vKSl5dXsXEPDw+n/ou/livViorJ3j166ZLdVg0XZ2nb8XsUzo4ehbOjR+HsnL1HranN6mAfHBys6tWrq127dhoyZIg6duyoJk2aWLsai7Rp00aHDx82Gzty5Ihq164t6Y8b6YWGhmrDhg2mIJ+Tk6OUlBQNGzZMktS6dWtlZ2dr9+7datGihSRp48aNKioqUmRkpF3qBgAAAACgvFgd7Pft26fGjRvbo5ZiRo4cqQceeEBTpkxR3759tXPnTi1atEiLFi2SJBkMBo0YMUKTJ0/WXXfdpYiICI0bN05hYWHq1auXpD+O8Hft2lVDhgzRwoULVVBQoOHDh6t///7cER8AAAAA4PKsDvblFeol6b777tOnn36qMWPGaNKkSYqIiNDs2bMVExNjmvPyyy/rwoULGjp0qLKzs9W2bVutW7dO3t7epjmJiYkaPny4HnroIbm5ualPnz6aM2dOue0HAAAAAAD2YnWwL2+PPPKIHnnkkRKXGwwGTZo0SZMmTSpxTlBQkFasWGGP8gAAAAAAcCg3RxcAAAAAAADKjmAPAAAAAIALs0mwz87OtsVqAAAAAACAlawO9tOmTdMHH3xget23b19Vq1ZNt912m/bu3WvT4gAAAAAAQOmsDvYLFy5UeHi4JCkpKUlJSUlau3atunXrplGjRtm8QAAAAAAAUDKr74qfnp5uCvZr1qxR3759FR0drTp16igyMtLmBQIAAAAAgJJZfcQ+MDBQp06dkiStW7dOUVFRkiSj0ajCwkLbVgcAgI0ZDKX/+Ps7ukIAAADrWH3Evnfv3nriiSd011136bffflO3bt0kSd9++63uvPNOmxcIAAAAAABKZnWwf/PNN1WnTh2dOnVK06dPV5UqVSRJZ8+e1bPPPmvzAgEAAAAAQMmsDvYeHh566aWXio2PHDnSJgUBAAAAAADLlek59v/+97/Vtm1bhYWF6aeffpIkzZ49W5999plNiwMAAAAAAKWzOtgvWLBA8fHx6tatm7Kzs003zAsICNDs2bNtXR8AAAAAACiF1cF+7ty5euedd/TPf/5T7u7upvGWLVtq//79Ni0OAEpyozubAwAAABWF1cH+xIkTat68ebFxLy8vXbhwwSZFAQAAAAAAy1gd7CMiIpSWllZsfN26dWrYsKEtagIAAAAAABay+q748fHxiouL0+XLl2U0GrVz5069//77mjp1qhYvXmyPGgEAAAAAQAmsDvZPP/20fHx8NHbsWF28eFFPPPGEwsLC9K9//Uv9+/e3R40AAAAAAKAEVgd7SYqJiVFMTIwuXryo3NxcBQcH27ouAAAAAABggTIF+6t8fX3l6+trq1oAAAAAAICVLAr2zZs3l8HC50ft2bPnpgoCAAAAAACWsyjY9+rVy85lAAAAAACAsrAo2E+YMMHedQAAAAAAgDKw+jn2qampSklJKTaekpKiXbt22aQoAAAAAABgGauDfVxcnE6dOlVs/PTp04qLi7NJUQAAAAAAwDJWB/uDBw/q3nvvLTbevHlzHTx40CZFAQAAAAAAy1gd7L28vJSRkVFs/OzZs6pU6aaengcAAAAAAKxkdbCPjo7WmDFjdO7cOdNYdna2/vGPf6hz5842LQ4AAEfx95cMhuv/AAAAOBOrD7HPmDFD7du3V+3atdW8eXNJUlpamkJCQvTvf//b5gUCAAAAAICSWR3sb7vtNu3bt0+JiYnau3evfHx8NGjQIA0YMEAeHh72qBEAAAAAAJSgTBfFV65cWUOHDrV1LQAAAAAAwEoWBfvVq1erW7du8vDw0OrVq0ud++ijj9qkMAAAAAAAcGMWBftevXopPT1dwcHB6tWrV4nzDAaDCgsLbVUbgAqKm5MBAAAAlrMo2BcVFV33zwAAAAAAwLGsftzd8uXLlZeXV2w8Pz9fy5cvt0lRAAAAAADAMlYH+0GDBpk9w/6q8+fPa9CgQTYpCgAAAAAAWMbqYG80GmW4zgWwP//8s/z9/W1SFAAAAAAAsIzFj7tr3ry5DAaDDAaDHnroIVWq9P/fWlhYqBMnTqhr1652KRIAAAAAAFyfxcH+6t3w09LS1KVLF1WpUsW0zNPTU3Xq1FGfPn1sXiAAAAAAACiZxcF+woQJKiwsVJ06dRQdHa2aNWvasy4AAAAAAGABq66xd3d31zPPPKPLly/bqx4AAAAAAGAFq2+ed/fdd+v48eP2qAUAAAAAAFjJ6mA/efJkvfTSS1qzZo3Onj2rnJwcsx8AAAAAAFB+LL7G/qqHH35YkvToo4+aPfbu6mPwCgsLbVcdAAAAAAAoldXBftOmTfaoAwAAAAAAlIHVwb5Dhw4lLvvuu+9uqhgAAAAAAGAdq6+x/7Pz589r0aJFatWqle655x5b1AQAAAAAACxU5mC/detWxcbGqmbNmpoxY4YefPBBffPNN7asDQAAAAAA3IBVp+Knp6dr2bJlWrJkiXJyctS3b1/l5eVp1apVatSokb1qBAAAAAAAJbD4iH2PHj1Uv3597du3T7Nnz9aZM2c0d+5ce9YGAAAAAABuwOIj9mvXrtXzzz+vYcOG6a677rJnTQAAOLVrnvZaIqPR/nUAAABIVhyx3759u86fP68WLVooMjJS8+bN06+//mrP2gAAAAAAwA1YHOzvv/9+vfPOOzp79qyeeeYZrVy5UmFhYSoqKlJSUpLOnz9vzzoBAAAAAMB1WH1X/MqVK+upp57S9u3btX//fr344ot6/fXXFRwcrEcffdQeNZq8/vrrMhgMGjFihGns8uXLiouLU7Vq1VSlShX16dNHGRkZZu87efKkunfvLl9fXwUHB2vUqFG6cuWKXWsFAAAAAKA83NRz7OvXr6/p06fr559/1vvvv2+rmq4rNTVVb7/9tpo2bWo2PnLkSH3++ef66KOPtGXLFp05c0a9e/c2LS8sLFT37t2Vn5+vr7/+Wu+9956WLVum8ePH27VeAAAAAADKw00F+6vc3d3Vq1cvrV692harKyY3N1cxMTF65513FBgYaBo/d+6clixZolmzZunBBx9UixYttHTpUn399df65ptvJEnr16/XwYMH9Z///EfNmjVTt27d9Oqrr2r+/PnKz8+3S70AAAAAAJQXq55j7yhxcXHq3r27oqKiNHnyZNP47t27VVBQoKioKNNYgwYNVKtWLSUnJ+v+++9XcnKymjRpopCQENOcLl26aNiwYTpw4ICaN29ebHt5eXnKy8szvc7JyZEkFRQUqKCgwB67aDNX63P2OlFxWdKjPj7lVQ1QnI9Pgdl/y4pfw7AX/q2Hs6NH4excpUetqc/pg/3KlSu1Z88epaamFluWnp4uT09PBQQEmI2HhIQoPT3dNOfaUH91+dVl1zN16lQlJCQUG1+/fr18fX3LshvlLikpydElAKUqrUftfGUPYJF3372536NffGGjQoAS8G89nB09Cmfn7D168eJFi+c6dbA/deqUXnjhBSUlJcnb27vctjtmzBjFx8ebXufk5Cg8PFzR0dHy8/MrtzrKoqCgQElJSercubM8PDwcXQ5QjCU96u9fzkUB1/DxKdC77ybpqac669Klsv8ePXfOhkUB1+Dfejg7ehTOzlV69OqZ45awKNjfe++92rBhgwIDAzVp0iS99NJL5XLkevfu3crMzNS9995rGissLNTWrVs1b948ffnll8rPz1d2drbZUfuMjAyFhoZKkkJDQ7Vz506z9V69a/7VOX/m5eUlLy+vYuMeHh5O/Rd/LVeqFRVTaT166VI5FwNcx6VLHjcV7PkVDHvj33o4O3oUzs7Ze9Sa2iy6ed6hQ4d04cIFSVJCQoJyc3PLVpmVHnroIe3fv19paWmmn5YtWyomJsb0Zw8PD23YsMH0nsOHD+vkyZNq3bq1JKl169bav3+/MjMzTXOSkpLk5+enRo0alct+AAAAAABgLxYdsW/WrJkGDRqktm3bymg0asaMGapSpcp159ryMXJVq1bV3XffbTZWuXJlVatWzTQ+ePBgxcfHKygoSH5+fnruuefUunVr3X///ZKk6OhoNWrUSAMHDtT06dOVnp6usWPHKi4u7rpH5QEAAAAAcCUWBftly5ZpwoQJWrNmjQwGg9auXatKlYq/1WAwlPvz4d988025ubmpT58+ysvLU5cuXfTWW2+Zlru7u2vNmjUaNmyYWrdurcqVKys2NlaTJk0q1zoBAAAAALAHi4J9/fr1tXLlSkmSm5ubNmzYoODgYLsWVpLNmzebvfb29tb8+fM1f/78Et9Tu3ZtfcHtiQEAAAAAtyCr74pfVFRkjzoAAAAAAEAZlOlxdz/88INmz56tQ4cOSZIaNWqkF154QXXr1rVpcQAAAAAAoHQW3RX/Wl9++aUaNWqknTt3qmnTpmratKlSUlLUuHFjJSUl2aNGAAAAAABQAquP2I8ePVojR47U66+/Xmz8lVdeUefOnW1WHAAAAAAAKJ3VR+wPHTqkwYMHFxt/6qmndPDgQZsUBQAAAAAALGN1sK9Ro4bS0tKKjaelpTnsTvkAADgbg6H0HwAAAFux+lT8IUOGaOjQoTp+/LgeeOABSdKOHTs0bdo0xcfH27xAAAAAAABQMquD/bhx41S1alXNnDlTY8aMkSSFhYVp4sSJev75521eIAAAAAAAKJnVwd5gMGjkyJEaOXKkzp8/L0mqWrWqzQsDAAAAAAA3Vqbn2F9FoAcAAAAAwLFuKtgDAICyseQGekaj/esAAACuz+q74gMAAAAAAOdBsAcAAAAAwIUR7AE4hL8/z/cGAAAAbKFMwX748OHKysqydS0AAAAAAMBKFgf7n3/+2fTnFStWKDc3V5LUpEkTnTp1yvaVAQAAAACAG7L4rvgNGjRQtWrV1KZNG12+fFmnTp1SrVq19OOPP6qgoMCeNQIAAAAAgBJYfMQ+OztbH330kVq0aKGioiI9/PDDqlevnvLy8vTll18qIyPDnnUCAAAAAIDrsDjYFxQUqFWrVnrxxRfl4+Ojb7/9VkuXLpW7u7veffddRUREqH79+vasFQAAAAAA/InFp+IHBASoWbNmatOmjfLz83Xp0iW1adNGlSpV0gcffKDbbrtNqamp9qwVAAAAAAD8icVH7E+fPq2xY8fKy8tLV65cUYsWLdSuXTvl5+drz549MhgMatu2rT1rBQAAAAAAf2JxsK9evbp69OihqVOnytfXV6mpqXruuedkMBj00ksvyd/fXx06dLBnrQAAAAAA4E/K9Bx7SfL391ffvn3l4eGhjRs36sSJE3r22WdtWRsAAAAAALiBMgX7ffv26fbbb5ck1a5dWx4eHgoNDVW/fv1sWhwA12MwlP7j7+/oCgEAAIBbi8U3z7tWeHi46c/fffedzYoBAAAAAADWKfOp+AAAAAAAwPEI9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACAC6vk6AIAuBaDwdEVAAAAALgWwR4AACd1oy/SjMbyqQMAADg3TsUHAAAAAMCFEewBAAAAAHBhBHsAAAAAAFwYwR4AAAAAABdGsAcAAAAAwIUR7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhTl1sJ86daruu+8+Va1aVcHBwerVq5cOHz5sNufy5cuKi4tTtWrVVKVKFfXp00cZGRlmc06ePKnu3bvL19dXwcHBGjVqlK5cuVKeuwIAAAAAgF04dbDfsmWL4uLi9M033ygpKUkFBQWKjo7WhQsXTHNGjhypzz//XB999JG2bNmiM2fOqHfv3qblhYWF6t69u/Lz8/X111/rvffe07JlyzR+/HhH7BIAAAAAADZVydEFlGbdunVmr5ctW6bg4GDt3r1b7du317lz57RkyRKtWLFCDz74oCRp6dKlatiwob755hvdf//9Wr9+vQ4ePKivvvpKISEhatasmV599VW98sormjhxojw9PR2xawAAAAAA2IRTB/s/O3funCQpKChIkrR7924VFBQoKirKNKdBgwaqVauWkpOTdf/99ys5OVlNmjRRSEiIaU6XLl00bNgwHThwQM2bNy+2nby8POXl5Zle5+TkSJIKCgpUUFBgl32zlav1OXudcF0+Pjf7/gKz/wLOxpV6lF/1FRP/1sPZ0aNwdq7So9bU5zLBvqioSCNGjFCbNm109913S5LS09Pl6empgIAAs7khISFKT083zbk21F9dfnXZ9UydOlUJCQnFxtevXy9fX9+b3ZVykZSU5OgScIt6/33brOfdd+lRODdX6NEvvnB0BXAk/q2Hs6NH4eycvUcvXrxo8VyXCfZxcXH67rvvtH37drtva8yYMYqPjze9zsnJUXh4uKKjo+Xn52f37d+MgoICJSUlqXPnzvLw8HB0ObgF+fvf3Pt9fAr07rtJeuqpzrp0iR6F83GlHv2/E9lQwfBvPZwdPQpn5yo9evXMcUu4RLAfPny41qxZo61bt+r22283jYeGhio/P1/Z2dlmR+0zMjIUGhpqmrNz506z9V29a/7VOX/m5eUlLy+vYuMeHh5O/Rd/LVeqFa7l0iVbrcfD6UMTKjZX6FFLbhNjNNq/DjgG/9bD2dGjcHbO3qPW1ObUd8U3Go0aPny4Pv30U23cuFERERFmy1u0aCEPDw9t2LDBNHb48GGdPHlSrVu3liS1bt1a+/fvV2ZmpmlOUlKS/Pz81KhRo/LZEQAAAAAA7MSpj9jHxcVpxYoV+uyzz1S1alXTNfH+/v7y8fGRv7+/Bg8erPj4eAUFBcnPz0/PPfecWrdurfvvv1+SFB0drUaNGmngwIGaPn260tPTNXbsWMXFxV33qDwAAAAAAK7EqYP9ggULJEkdO3Y0G1+6dKmefPJJSdKbb74pNzc39enTR3l5eerSpYveeust01x3d3etWbNGw4YNU+vWrVW5cmXFxsZq0qRJ5bUbAAA4jMFQ+nJO1QcAwPU5dbA3WvB/G97e3po/f77mz59f4pzatWvrC24dDAAAAAC4BTn1NfYAAAAAAKB0Tn3EHkD5utEpuwAAAACcD0fsAQAAAABwYRyxByoQjsgDAAAAtx6O2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwrgrPgAAFdiNnpZhNJZPHQAAoOw4Yg8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwnjcHXCLuNEjqwAAAADcmjhiDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MK6xB1wE19ADAAAAuB6CPQAAKJElXyoajfavAwAAlIxT8QEAAAAAcGEEewAAAAAAXBin4gNOgmvoAbiqG/3+4lR9AADsiyP2AAAAAAC4MI7YAwAAu+IGfAAA2BdH7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYdwVHwAAONyN7pzPXfMBACgZwR4oB5Y86gkAAAAAyoJT8QEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYdwVH7AB7noPAPZlye9ZHokHAKioOGIPAAAAAIAL44g9AACoEDjqDwC4VXHEHgAAAAAAF0awBwAAAADAhXEqPgAAuCVwI1MAQEXFEXsAAAAAAFwYR+wBAAD+z42O+nNzPQCAM+KIPQAAAAAALoxgDwAAAACAC+NUfAAAAAtZcoM+TtcHAJQ3jtgDAADYkMFQ+s/Nvt/f3/77AABwLRyxBwAAKEc8lg8AYGsV6oj9/PnzVadOHXl7eysyMlI7d+50dEkAAAAAANyUChPsP/jgA8XHx2vChAnas2eP7rnnHnXp0kWZmZmOLg0AAMCmbnQ6P2cNAMCtpcIE+1mzZmnIkCEaNGiQGjVqpIULF8rX11fvvvuuo0sDAACwmr//zYV2WwR/S75AKI8vGPgSA0BFVyGusc/Pz9fu3bs1ZswY05ibm5uioqKUnJxcbH5eXp7y8vJMr8+dOydJysrKUkFBgf0LvgkFBQW6ePGifvvtN3l4eDi6nArD29vRFbgOb+8/etTb+zcZjfQonA89CmdXXj3q42NJLeWznZutwxbbuJGff77xnNtvL5/tOBr/Pwpn5yo9ev78eUmS0YLHrVSIYP/rr7+qsLBQISEhZuMhISH6/vvvi82fOnWqEhISio1HRETYrUagorh8WXriCUdXAZSMHoWzo0edU/Xqt9Z2ADiP8+fPy/8Gj0SpEMHeWmPGjFF8fLzpdVFRkbKyslStWjUZnPx8rpycHIWHh+vUqVPy8/NzdDlAMfQonB09CmdHj8LZ0aNwdq7So0ajUefPn1dYWNgN51aIYF+9enW5u7srIyPDbDwjI0OhoaHF5nt5ecnLy8tsLCAgwJ4l2pyfn59TNylAj8LZ0aNwdvQonB09CmfnCj16oyP1V1WIm+d5enqqRYsW2rBhg2msqKhIGzZsUOvWrR1YGQAAAAAAN6dCHLGXpPj4eMXGxqply5Zq1aqVZs+erQsXLmjQoEGOLg0AAAAAgDKrMMG+X79++uWXXzR+/Hilp6erWbNmWrduXbEb6rk6Ly8vTZgwodilBICzoEfh7OhRODt6FM6OHoWzuxV71GC05N75AAAAAADAKVWIa+wBAAAAALhVEewBAAAAAHBhBHsAAAAAAFwYwR4AAAAAABdGsL+FzJ8/X3Xq1JG3t7ciIyO1c+dOR5eECmrq1Km67777VLVqVQUHB6tXr146fPiw2ZzLly8rLi5O1apVU5UqVdSnTx9lZGQ4qGJUdK+//roMBoNGjBhhGqNH4WinT5/WX//6V1WrVk0+Pj5q0qSJdu3aZVpuNBo1fvx41axZUz4+PoqKitLRo0cdWDEqksLCQo0bN04RERHy8fFR3bp19eqrr+ra+3LToyhvW7duVY8ePRQWFiaDwaBVq1aZLbekJ7OyshQTEyM/Pz8FBARo8ODBys3NLce9KBuC/S3igw8+UHx8vCZMmKA9e/bonnvuUZcuXZSZmeno0lABbdmyRXFxcfrmm2+UlJSkgoICRUdH68KFC6Y5I0eO1Oeff66PPvpIW7Zs0ZkzZ9S7d28HVo2KKjU1VW+//baaNm1qNk6PwpF+//13tWnTRh4eHlq7dq0OHjyomTNnKjAw0DRn+vTpmjNnjhYuXKiUlBRVrlxZXbp00eXLlx1YOSqKadOmacGCBZo3b54OHTqkadOmafr06Zo7d65pDj2K8nbhwgXdc889mj9//nWXW9KTMTExOnDggJKSkrRmzRpt3bpVQ4cOLa9dKDsjbgmtWrUyxsXFmV4XFhYaw8LCjFOnTnVgVcAfMjMzjZKMW7ZsMRqNRmN2drbRw8PD+NFHH5nmHDp0yCjJmJyc7KgyUQGdP3/eeNdddxmTkpKMHTp0ML7wwgtGo5EeheO98sorxrZt25a4vKioyBgaGmp84403TGPZ2dlGLy8v4/vvv18eJaKC6969u/Gpp54yG+vdu7cxJibGaDTSo3A8ScZPP/3U9NqSnjx48KBRkjE1NdU0Z+3atUaDwWA8ffp0udVeFhyxvwXk5+dr9+7dioqKMo25ubkpKipKycnJDqwM+MO5c+ckSUFBQZKk3bt3q6CgwKxnGzRooFq1atGzKFdxcXHq3r27WS9K9Cgcb/Xq1WrZsqX+8pe/KDg4WM2bN9c777xjWn7ixAmlp6eb9ai/v78iIyPpUZSLBx54QBs2bNCRI0ckSXv37tX27dvVrVs3SfQonI8lPZmcnKyAgAC1bNnSNCcqKkpubm5KSUkp95qtUcnRBeDm/frrryosLFRISIjZeEhIiL7//nsHVQX8oaioSCNGjFCbNm109913S5LS09Pl6empgIAAs7khISFKT093QJWoiFauXKk9e/YoNTW12DJ6FI52/PhxLViwQPHx8frHP/6h1NRUPf/88/L09FRsbKypD6/3bz89ivIwevRo5eTkqEGDBnJ3d1dhYaFee+01xcTESBI9CqdjSU+mp6crODjYbHmlSpUUFBTk9H1LsAdgV3Fxcfruu++0fft2R5cCmJw6dUovvPCCkpKS5O3t7ehygGKKiorUsmVLTZkyRZLUvHlzfffdd1q4cKFiY2MdXB0gffjhh0pMTNSKFSvUuHFjpaWlacSIEQoLC6NHAQfgVPxbQPXq1eXu7l7sbs0ZGRkKDQ11UFWANHz4cK1Zs0abNm3S7bffbhoPDQ1Vfn6+srOzzebTsygvu3fvVmZmpu69915VqlRJlSpV0pYtWzRnzhxVqlRJISEh9CgcqmbNmmrUqJHZWMOGDXXy5ElJMvUh//bDUUaNGqXRo0erf//+atKkiQYOHKiRI0dq6tSpkuhROB9LejI0NLTYzcevXLmirKwsp+9bgv0twNPTUy1atNCGDRtMY0VFRdqwYYNat27twMpQURmNRg0fPlyffvqpNm7cqIiICLPlLVq0kIeHh1nPHj58WCdPnqRnUS4eeugh7d+/X2lpaaafli1bKiYmxvRnehSO1KZNm2KPCT1y5Ihq164tSYqIiFBoaKhZj+bk5CglJYUeRbm4ePGi3NzMo4S7u7uKiook0aNwPpb0ZOvWrZWdna3du3eb5mzcuFFFRUWKjIws95qtwan4t4j4+HjFxsaqZcuWatWqlWbPnq0LFy5o0KBBji4NFVBcXJxWrFihzz77TFWrVjVdk+Tv7y8fHx/5+/tr8ODBio+PV1BQkPz8/PTcc8+pdevWuv/++x1cPSqCqlWrmu75cFXlypVVrVo10zg9CkcaOXKkHnjgAU2ZMkV9+/bVzp07tWjRIi1atEiSZDAYNGLECE2ePFl33XWXIiIiNG7cOIWFhalXr16OLR4VQo8ePfTaa6+pVq1aaty4sb799lvNmjVLTz31lCR6FI6Rm5urY8eOmV6fOHFCaWlpCgoKUq1atW7Ykw0bNlTXrl01ZMgQLVy4UAUFBRo+fLj69++vsLAwB+2VhRx9W37Yzty5c421atUyenp6Glu1amX85ptvHF0SKihJ1/1ZunSpac6lS5eMzz77rDEwMNDo6+trfOyxx4xnz551XNGo8K593J3RSI/C8T7//HPj3XffbfTy8jI2aNDAuGjRIrPlRUVFxnHjxhlDQkKMXl5exoceesh4+PBhB1WLiiYnJ8f4wgsvGGvVqmX09vY23nHHHcZ//vOfxry8PNMcehTlbdOmTdf9f9DY2Fij0WhZT/7222/GAQMGGKtUqWL08/MzDho0yHj+/HkH7I11DEaj0eig7xQAAAAAAMBN4hp7AAAAAABcGMEeAAAAAAAXRrAHAAAAAMCFEewBAAAAAHBhBHsAAAAAAFwYwR4AAAAAABdGsAcAAAAAwIUR7AEAAAAAcGEEewAA7MxgMGjVqlUWzZ04caKaNWtm13oc6cknn1SvXr3K9N727dtrxYoVptfWfK7OytJ9yM/PV506dbRr1y77FwUAcDkEewAAriM5OVnu7u7q3r27xe8pKZSfPXtW3bp1s2F1pbuZ8GwrP/74owwGg9LS0myyvtWrVysjI0P9+/c3jd3s59q/f3917drVbGzdunUyGAyaOHGi2fjEiRNVq1atMm/rZnl6euqll17SK6+84rAaAADOi2APAMB1LFmyRM8995y2bt2qM2fOlDrXaDTqypUrJS4PDQ2Vl5eXrUusUObMmaNBgwbJze3//6/LzX6unTp10o4dO8z+7jZt2qTw8HBt3rzZbO6mTZvUqVOnMm0nPz+/zDVeKyYmRtu3b9eBAwdssj4AwK2DYA8AwJ/k5ubqgw8+0LBhw9S9e3ctW7bMbPnmzZtlMBi0du1atWjRQl5eXvrPf/6jhIQE7d27VwaDQQaDwfS+P59u/fPPP2vAgAEKCgpS5cqV1bJlS6WkpJRYz+LFi9WwYUN5e3urQYMGeuutt25q/7777jt169ZNVapUUUhIiAYOHKhff/3VtLxjx456/vnn9fLLLysoKEihoaHFjmB///33atu2rby9vdWoUSN99dVXZvsZEREhSWrevLkMBoM6duxo9v4ZM2aoZs2aqlatmuLi4lRQUFBivb/88os2btyoHj16mI1fu72rZwh88skn6tSpk3x9fXXPPfcoOTm5xPV26tRJubm5Zqe3b968WaNHj1ZKSoouX74sSbp8+bJSUlJMwf7kyZPq2bOnqlSpIj8/P/Xt21cZGRmmdVw9c2Px4sWKiIiQt7e3JOno0aNq37696TNLSkoyqyc/P1/Dhw9XzZo15e3trdq1a2vq1Kmm5YGBgWrTpo1WrlxZ4j4BAComgj0AAH/y4YcfqkGDBqpfv77++te/6t1335XRaCw2b/To0Xr99dd16NAhde7cWS+++KIaN26ss2fP6uzZs+rXr1+x9+Tm5qpDhw46ffq0Vq9erb179+rll19WUVHRdWtJTEzU+PHj9dprr+nQoUOaMmWKxo0bp/fee69M+5adna0HH3xQzZs3165du7Ru3TplZGSob9++ZvPee+89Va5cWSkpKZo+fbomTZpkCqKFhYXq1auXfH19lZKSokWLFumf//yn2ft37twpSfrqq6909uxZffLJJ6ZlmzZt0g8//KBNmzbpvffe07Jly4p9eXKt7du3y9fXVw0bNrzh/v3zn//USy+9pLS0NNWrV08DBgwo8WyKevXqKSwsTJs2bZIknT9/Xnv27NFf/vIX1alTx/SlwNdff628vDx16tRJRUVF6tmzp7KysrRlyxYlJSXp+PHjxf6ujx07po8//liffPKJ0tLSVFRUpN69e8vT01MpKSlauHBhsdPq58yZo9WrV+vDDz/U4cOHlZiYqDp16pjNadWqlbZt23bDzwEAULFUcnQBAAA4myVLluivf/2rJKlr1646d+6ctmzZUuyo86RJk9S5c2fT6ypVqqhSpUoKDQ0tcd0rVqzQL7/8otTUVAUFBUmS7rzzzhLnT5gwQTNnzlTv3r0l/XEk/ODBg3r77bcVGxtr9b7NmzdPzZs315QpU0xj7777rsLDw3XkyBHVq1dPktS0aVNNmDBBknTXXXdp3rx52rBhgzp37qykpCT98MMP2rx5s2lfX3vtNbPPokaNGpKkatWqFfs8AgMDNW/ePLm7u6tBgwbq3r27NmzYoCFDhly35p9++kkhISFmp+GX5KWXXjLdFyEhIUGNGzfWsWPH1KBBg+vO79SpkzZv3qwxY8Zo27ZtqlevnmrUqKH27dtr8+bNpuURERGqXbu2kpKStH//fp04cULh4eGSpOXLl6tx48ZKTU3VfffdJ+mPo+/Lly83fQ7r16/X999/ry+//FJhYWGSpClTppjdI+DkyZO666671LZtWxkMBtWuXbtYvWFhYfrpp59u+DkAACoWjtgDAHCNw4cPa+fOnRowYIAkqVKlSurXr5+WLFlSbG7Lli2tXn9aWpqaN29uCvWluXDhgn744QcNHjxYVapUMf1MnjxZP/zwg9XblqS9e/dq06ZNZuu7GnqvXWfTpk3N3lezZk1lZmZK+uMzCg8PNwvsrVq1sriGxo0by93d/brrvp5Lly6ZTme/kWvrrlmzpiSVuu6OHTtqx44dKigo0ObNm01f3nTo0MF0nf3VgC9Jhw4dUnh4uCnUS1KjRo0UEBCgQ4cOmcZq165tCvXXvu9qqJek1q1bm9Xy5JNPKi0tTfXr19fzzz+v9evXF6vXx8dHFy9evNHHAACoYDhiDwDANZYsWaIrV66YBTCj0SgvLy/NmzdP/v7+pvHKlStbvX4fHx+L5+bm5kqS3nnnHUVGRpotuzYYWyM3N1c9evTQtGnTii27GoQlycPDw2yZwWAo8XIBa1m77urVq+v333+3et0Gg0GSSl13p06ddOHCBaWmpmrTpk0aNWqUpD+C/VNPPaWsrCylpKTomWeesWj7V5WlN+69916dOHFCa9eu1VdffaW+ffsqKipK//3vf01zsrKyzL4wAABAItgDAGBy5coVLV++XDNnzlR0dLTZsl69eun999/X3//+9xLf7+npqcLCwlK30bRpUy1evFhZWVk3PGofEhKisLAwHT9+XDExMZbvSCnuvfdeffzxx6pTp44qVSrb/wbUr19fp06dUkZGhkJCQiRJqampZnM8PT0l6YafhyWaN2+u9PR0/f777woMDLzp9V2rbt26Cg8P1+rVq5WWlqYOHTpIkm677TbddtttmjlzpvLz801H7Bs2bKhTp07p1KlTpqP2Bw8eVHZ2tho1alTidq6+7+zZs6YvUL755pti8/z8/NSvXz/169dPjz/+uLp27WrWK999952aN29u088AAOD6OBUfAID/s2bNGv3+++8aPHiw7r77brOfPn36XPd0/GvVqVNHJ06cUFpamn799Vfl5eUVmzNgwACFhoaqV69e2rFjh44fP66PP/64xLu3JyQkaOrUqZozZ46OHDmi/fv3a+nSpZo1a1aptZw7d05paWlmP6dOnVJcXJyysrI0YMAApaam6ocfftCXX36pQYMGWRzCO3furLp16yo2Nlb79u3Tjh07NHbsWEn//yh5cHCwfHx8TDfnO3funEXrvp7mzZurevXq2rFjR5nXUZpOnTrprbfe0p133mn6okL646j93LlzTTfZk6SoqCg1adJEMTEx2rNnj3bu3Km//e1v6tChQ6mXZkRFRalevXqKjY3V3r17tW3btmI3HJw1a5bef/99ff/99zpy5Ig++ugjhYaGKiAgwDRn27Ztxb50AgCAYA8AwP9ZsmSJoqKizE63v6pPnz7atWuX9u3bV+L7+/Tpo65du6pTp06qUaOG3n///WJzPD09tX79egUHB+vhhx9WkyZN9Prrr5d4av3TTz+txYsXa+nSpWrSpIk6dOigZcuWmR4nV5LNmzerefPmZj8JCQkKCwvTjh07VFhYqOjoaDVp0kQjRoxQQECARTenk/64DGDVqlXKzc3Vfffdp6efftoUUq9eC1+pUiXNmTNHb7/9tsLCwtSzZ0+L1l3S9gYNGqTExMQyr6M0nTp10vnz54vdHLFDhw46f/682fPrDQaDPvvsMwUGBqp9+/aKiorSHXfcoQ8++KDUbbi5uenTTz/VpUuX1KpVKz399NN67bXXzOZUrVpV06dPV8uWLXXffffpxx9/1BdffGH6e0lOTta5c+f0+OOP22bHAQC3DIPxes/vAQAAsMKOHTvUtm1bHTt2THXr1rX5+tPT09W4cWPt2bPnuneLrwj69eune+65R//4xz8cXQoAwMlwjT0AALDap59+qipVquiuu+7SsWPH9MILL6hNmzZ2CfWSFBoaqiVLlujkyZMVMtjn5+erSZMmGjlypKNLAQA4IY7YAwAAqy1fvlyTJ0/WyZMnVb16dUVFRWnmzJmqVq2ao0sDAKDCIdgDAAAAAODCuHkeAAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuLD/BxGVERS+Jy3vAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1200x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df['article_lengths'] = train_df['article'].str.split().apply(len)\n",
    "eval_df['article_lengths'] = eval_df['article'].str.split().apply(len)\n",
    "\n",
    "article_lengths = pd.concat([pd.Series(train_df['article_lengths']), pd.Series(eval_df['article_lengths'])])\n",
    "\n",
    "print(f\"The longest string (in words) in this dataset is {max(article_lengths)} words.\")\n",
    "\n",
    "n = 100\n",
    "article_lengths.hist(grid=True, color=\"Blue\", figsize=(12,5), bins=n, range=(0, n))\n",
    "plt.suptitle('')\n",
    "plt.xlabel('Article Length (in Words)')\n",
    "plt.ylabel('# of Articles w/ That Length')\n",
    "plt.title('Article Length')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize Length of Each Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The longest string (in words) in this dataset is 26 words.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/YAAAHWCAYAAADdKxJLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABdU0lEQVR4nO3dfXzO9f////thdmq2OdtZZhY1JxmiZghhW/JxUt7vnH2Y00qjWCK95awTUsK7RCUn9TaibzpBMTKSiWQRUpbe9Gb0JoadmO31+6Pfjo+jGceLYzt2rNv1ctmlHc/X83i9Hq/tcTnW3evMYhiGIQAAAAAA4JIqObsAAAAAAABw4wj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAKXgl19+kcVi0SuvvOLsUgAAFRzBHgAAB9q3b5/+9re/KTw8XF5eXrrlllsUGxur1157zdmluawOHTrojjvucHYZJVq3bp2mTJni7DIAAH9hBHsAABxk+/btatmypb777jsNHz5cr7/+uoYNG6ZKlSpp7ty5zi4PpWTdunWaOnWqs8sAAPyFVXZ2AQAAVBQvvPCC/P39tWvXLgUEBNgsO3XqlHOKciLDMJSbmytvb29nlwIAQIXGEXsAABwkIyNDjRs3LhbqJSkwMND6fdG110uWLCk2z2Kx2JzWPWXKFFksFv3444/63//9X/n7+6tWrVp69tlnZRiGjh07ph49esjPz0/BwcGaNWuWzfpSU1NlsVi0cuVKTZ06VbfccouqVq2qv/3tbzp37pzy8vI0evRoBQYGytfXV4MHD1ZeXp7NOhYvXqyOHTsqMDBQnp6eatSokebPn1+s9rp16+p//ud/tH79erVs2VLe3t5688031b59ezVt2vSqP7PIyEjFx8df46dqv88++0z33HOPqlSpoqpVq6pr167av3+/zZxBgwbJ19dX//nPf9SzZ0/5+vqqVq1aGjt2rAoKCmzmnj59WgMGDJCfn58CAgKUkJCg7777zuZ3N2jQIM2bN0/SH7+7oq8/e+utt1SvXj15enrqrrvu0q5duxyyzwAASByxBwDAYcLDw5WWlqbvv//e4deE9+7dWw0bNtSMGTO0du1aPf/886pevbrefPNNdezYUS+99JKWLVumsWPH6q677lK7du1s3j99+nR5e3vr6aef1uHDh/Xaa6/J3d1dlSpV0u+//64pU6Zox44dWrJkiSIiIjRp0iTre+fPn6/GjRure/fuqly5sj799FM99thjKiwsVGJios12Dh06pL59++qRRx7R8OHDFRkZKV9fXw0fPrzYz2XXrl368ccfNXHixJv++bz33ntKSEhQfHy8XnrpJWVnZ2v+/Plq27at9uzZo7p161rnFhQUKD4+XtHR0XrllVe0ceNGzZo1S/Xq1dOIESMkSYWFherWrZt27typESNGqEGDBvr444+VkJBgs91HHnlEx48fV0pKit57772r1pacnKzz58/rkUcekcVi0cyZM/Xggw/q559/lru7+03vOwAAMgAAgENs2LDBcHNzM9zc3IyYmBhj3Lhxxvr1641Lly7ZzDty5IghyVi8eHGxdUgyJk+ebH09efJkQ5Lx8MMPW8cuX75s1K5d27BYLMaMGTOs47///rvh7e1tJCQkWMc2b95sSDLuuOMOmzr69u1rWCwWo0uXLjbbj4mJMcLDw23GsrOzi9UZHx9v3HrrrTZj4eHhhiTj888/txk/e/as4eXlZYwfP95m/PHHHzeqVKliXLhwodj6r9S+fXujcePGJS4/f/68ERAQYAwfPtxmPDMz0/D397cZT0hIMCQZ06ZNs5nbvHlzo0WLFtbX/+///T9DkjFnzhzrWEFBgdGxY8div7vExETjav9LVfR7rlGjhnHmzBnr+Mcff2xIMj799NNr7jcAAPbiVHwAABwkNjZWaWlp6t69u7777jvNnDlT8fHxuuWWW/TJJ5/c1LqHDRtm/d7NzU0tW7aUYRgaOnSodTwgIECRkZH6+eefi71/4MCBNkeHo6OjZRiGhgwZYjMvOjpax44d0+XLl61jV14jf+7cOf33v/9V+/bt9fPPP+vcuXM274+IiCh2ar2/v7969Oih5cuXyzAMSX8cNX///ffVs2dPValSxcyPopiUlBSdPXtWffv21X//+1/rl5ubm6Kjo7V58+Zi73n00UdtXt9zzz02P7fPP/9c7u7uGj58uHWsUqVKxc5QsEfv3r1VrVo1m21JuurvCQCAG0GwBwDAge666y59+OGH+v3337Vz505NmDBB58+f19/+9jcdOHDghtdbp04dm9f+/v7y8vJSzZo1i43//vvvdr1fksLCwoqNFxYW2gT2r776Sp07d1aVKlUUEBCgWrVq6ZlnnpGkqwb7qxk4cKCOHj2qL7/8UpK0ceNGnTx5UgMGDChxn+31008/SZI6duyoWrVq2Xxt2LCh2I0Lvby8VKtWLZuxatWq2fzc/v3vfyskJEQ+Pj428+rXr2+6vj//7ItC/tV+TwAA3AiusQcAoBR4eHjorrvu0l133aXbb79dgwcP1qpVqzR58uSr3lxNUrGbt13Jzc3NrjFJ1qPi9sy93joyMjLUqVMnNWjQQK+++qrCwsLk4eGhdevWafbs2SosLLR5X0l3wI+Pj1dQUJD+9a9/qV27dvrXv/6l4OBgde7c+arzzSiq4b333lNwcHCx5ZUr2/7vTkn7XFrM/J4AALgRBHsAAEpZy5YtJUknTpyQ9H9HbM+ePWsz79///neZ1mWPTz/9VHl5efrkk09sjjxf7fT2a3Fzc1O/fv20ZMkSvfTSS/roo480fPhwh4TsevXqSfrjyQOO+IcC6Y8bIW7evFnZ2dk2R+0PHz5cbG5J/1ADAEBZ4VR8AAAcZPPmzVc9Crtu3TpJfzzaTZL8/PxUs2ZNbd261WbeG2+8UfpFmlQUvK/cr3Pnzmnx4sWm1zVgwAD9/vvveuSRR3ThwgX97//+r0NqjI+Pl5+fn1588UXl5+cXW/7bb7/d0Drz8/P19ttvW8cKCwutj7a7UtE9Av78DzUAAJQVjtgDAOAgo0aNUnZ2th544AE1aNBAly5d0vbt2/X++++rbt26Gjx4sHXusGHDNGPGDA0bNkwtW7bU1q1b9eOPPzqx+quLi4uTh4eHunXrZg3kb7/9tgIDA61nINirefPmuuOOO7Rq1So1bNhQd955p93v/e233/T8888XG4+IiFD//v01f/58DRgwQHfeeaf69OmjWrVq6ejRo1q7dq3atGmj119/3VStPXv21N13360nn3xShw8fVoMGDfTJJ5/ozJkzkmyP0rdo0UKS9Pjjjys+Pl5ubm7q06ePqe0BAHAzCPYAADjIK6+8olWrVmndunV66623dOnSJdWpU0ePPfaYJk6cqICAAOvcSZMm6bffftMHH3yglStXqkuXLvrss88UGBjovB24isjISH3wwQeaOHGixo4dq+DgYI0YMUK1atUqdkd9ewwcOFDjxo0zfdO8U6dO6dlnny023qlTJ/Xv31/9+vVTaGioZsyYoZdffll5eXm65ZZbdM8999j8g4q93NzctHbtWj3xxBNaunSpKlWqpAceeECTJ09WmzZt5OXlZZ374IMPatSoUVqxYoX+9a9/yTAMgj0AoExZDO7cAgAAysjcuXM1ZswY/fLLL8XuFu8KPvroIz3wwAPatm2b2rRp4+xyAACQRLAHAABlxDAMNW3aVDVq1DB98z1nyMnJsbnLf0FBgeLi4vTNN98oMzOzxCcAAABQ1jgVHwAAlKqLFy/qk08+0ebNm7Vv3z59/PHHzi7JLqNGjVJOTo5iYmKUl5enDz/8UNu3b9eLL75IqAcAlCscsQcAAKXql19+UUREhAICAvTYY4/phRdecHZJdklOTtasWbN0+PBh5ebmqn79+hoxYoRGjhzp7NIAALBBsAcAAAAAwIXxHHsAAAAAAFwYwR4AAAAAABfGzfPsUFhYqOPHj6tq1aqyWCzOLgcAAAAAUMEZhqHz588rNDRUlSpd+5g8wd4Ox48fV1hYmLPLAAAAAAD8xRw7dky1a9e+5hyCvR2qVq0q6Y8fqJ+fn5Orubb8/Hxt2LBBcXFxcnd3d3Y5cAH0DMygX2AWPQOz6BmYRc/ALFfpmaysLIWFhVnz6LUQ7O1QdPq9n5+fSwR7Hx8f+fn5lesmRflBz8AM+gVm0TMwi56BWfQMzHK1nrHncnBungcAAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuLDKzi4AACoai6Vst2cYZbs9AAAAlC8csQcAAAAAwIUR7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYQR7AAAAAABcGMEeAAAAAAAXRrAHAAAAAMCFEewBAAAAAHBhBHsAAAAAAFwYwR4AAAAAABdGsAcAAAAAwIUR7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYU4N9vPnz1dUVJT8/Pzk5+enmJgYffbZZ9blHTp0kMVisfl69NFHbdZx9OhRde3aVT4+PgoMDNRTTz2ly5cv28xJTU3VnXfeKU9PT9WvX19Lliwpi90DAAAAAKDUVXbmxmvXrq0ZM2botttuk2EYWrp0qXr06KE9e/aocePGkqThw4dr2rRp1vf4+PhYvy8oKFDXrl0VHBys7du368SJExo4cKDc3d314osvSpKOHDmirl276tFHH9WyZcu0adMmDRs2TCEhIYqPjy/bHQYAAAAAwMGcGuy7detm8/qFF17Q/PnztWPHDmuw9/HxUXBw8FXfv2HDBh04cEAbN25UUFCQmjVrpueee07jx4/XlClT5OHhoQULFigiIkKzZs2SJDVs2FDbtm3T7NmzCfYAAAAAAJfn1GB/pYKCAq1atUoXL15UTEyMdXzZsmX617/+peDgYHXr1k3PPvus9ah9WlqamjRpoqCgIOv8+Ph4jRgxQvv371fz5s2Vlpamzp0722wrPj5eo0ePLrGWvLw85eXlWV9nZWVJkvLz85Wfn++I3S01RfWV9zpRftAzjuftXbbbK8tfHf0Cs+gZmEXPwCx6Bma5Ss+Yqc/pwX7fvn2KiYlRbm6ufH19tXr1ajVq1EiS1K9fP4WHhys0NFR79+7V+PHjdejQIX344YeSpMzMTJtQL8n6OjMz85pzsrKylJOTI++r/B/49OnTNXXq1GLjGzZssLkUoDxLSUlxdglwMfSM4yxfXrbbW7eubLcn0S8wj56BWfQMzKJnYFZ575ns7Gy75zo92EdGRio9PV3nzp3TBx98oISEBG3ZskWNGjXSww8/bJ3XpEkThYSEqFOnTsrIyFC9evVKraYJEyYoKSnJ+jorK0thYWGKi4uTn59fqW3XEfLz85WSkqLY2Fi5u7s7uxy4gL9Cz/j7O7uC0nXuXNlt66/QL3AsegZm0TMwi56BWa7SM0VnjtvD6cHew8ND9evXlyS1aNFCu3bt0ty5c/Xmm28WmxsdHS1JOnz4sOrVq6fg4GDt3LnTZs7JkyclyXpdfnBwsHXsyjl+fn5XPVovSZ6envL09Cw27u7uXq5/8VdypVpRPlTknsnJcXYFpcsZv7aK3C8oHfQMzKJnYBY9A7PKe8+Yqa3cPce+sLDQ5vr2K6Wnp0uSQkJCJEkxMTHat2+fTp06ZZ2TkpIiPz8/6+n8MTEx2rRpk816UlJSbK7jBwAAAADAVTn1iP2ECRPUpUsX1alTR+fPn1dycrJSU1O1fv16ZWRkKDk5Wffff79q1KihvXv3asyYMWrXrp2ioqIkSXFxcWrUqJEGDBigmTNnKjMzUxMnTlRiYqL1iPujjz6q119/XePGjdOQIUP0xRdfaOXKlVq7dq0zdx0AAAAAAIdwarA/deqUBg4cqBMnTsjf319RUVFav369YmNjdezYMW3cuFFz5szRxYsXFRYWpl69emnixInW97u5uWnNmjUaMWKEYmJiVKVKFSUkJNg89z4iIkJr167VmDFjNHfuXNWuXVsLFy7kUXcAAAAAgArBqcH+nXfeKXFZWFiYtmzZct11hIeHa911bgndoUMH7dmzx3R9AAAAAACUd+XuGnsAAAAAAGA/gj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYA4OIslrL78vd39t4CAADgzwj2AAAAAAC4MKcG+/nz5ysqKkp+fn7y8/NTTEyMPvvsM+vy3NxcJSYmqkaNGvL19VWvXr108uRJm3UcPXpUXbt2lY+PjwIDA/XUU0/p8uXLNnNSU1N15513ytPTU/Xr19eSJUvKYvcAAAAAACh1Tg32tWvX1owZM7R7925988036tixo3r06KH9+/dLksaMGaNPP/1Uq1at0pYtW3T8+HE9+OCD1vcXFBSoa9euunTpkrZv366lS5dqyZIlmjRpknXOkSNH1LVrV917771KT0/X6NGjNWzYMK1fv77M9xcAAAAAAEer7MyNd+vWzeb1Cy+8oPnz52vHjh2qXbu23nnnHSUnJ6tjx46SpMWLF6thw4basWOHWrVqpQ0bNujAgQPauHGjgoKC1KxZMz333HMaP368pkyZIg8PDy1YsEARERGaNWuWJKlhw4batm2bZs+erfj4+DLfZwAAAAAAHMmpwf5KBQUFWrVqlS5evKiYmBjt3r1b+fn56ty5s3VOgwYNVKdOHaWlpalVq1ZKS0tTkyZNFBQUZJ0THx+vESNGaP/+/WrevLnS0tJs1lE0Z/To0SXWkpeXp7y8POvrrKwsSVJ+fr7y8/MdtMelo6i+8l4nyo+/Qs94ezu7gorD27vi9wsc66/wGQPHomdgFj0Ds1ylZ8zU5/Rgv2/fPsXExCg3N1e+vr5avXq1GjVqpPT0dHl4eCggIMBmflBQkDIzMyVJmZmZNqG+aHnRsmvNycrKUk5Ojryv8n/806dP19SpU4uNb9iwQT4+Pje8r2UpJSXF2SXAxVTknlm+3NkVVDwVuV9QOugZmEXPwCx6BmaV957Jzs62e67Tg31kZKTS09N17tw5ffDBB0pISNCWLVucWtOECROUlJRkfZ2VlaWwsDDFxcXJz8/PiZVdX35+vlJSUhQbGyt3d3dnlwMX8FfoGR7R5jje3vlatKhi9wsc66/wGQPHomdgFj0Ds1ylZ4rOHLeH04O9h4eH6tevL0lq0aKFdu3apblz56p37966dOmSzp49a3PU/uTJkwoODpYkBQcHa+fOnTbrK7pr/pVz/nwn/ZMnT8rPz++qR+slydPTU56ensXG3d3dy/Uv/kquVCvKh4rcMzk5zq6g4qnI/YLSQc/ALHoGZtEzMKu894yZ2srdc+wLCwuVl5enFi1ayN3dXZs2bbIuO3TokI4ePaqYmBhJUkxMjPbt26dTp05Z56SkpMjPz0+NGjWyzrlyHUVzitYBAAAAAIArc+oR+wkTJqhLly6qU6eOzp8/r+TkZKWmpmr9+vXy9/fX0KFDlZSUpOrVq8vPz0+jRo1STEyMWrVqJUmKi4tTo0aNNGDAAM2cOVOZmZmaOHGiEhMTrUfcH330Ub3++usaN26chgwZoi+++EIrV67U2rVrnbnrAAAAAAA4hFOD/alTpzRw4ECdOHFC/v7+ioqK0vr16xUbGytJmj17tipVqqRevXopLy9P8fHxeuONN6zvd3Nz05o1azRixAjFxMSoSpUqSkhI0LRp06xzIiIitHbtWo0ZM0Zz585V7dq1tXDhQh51BwAAAACoEJwa7N95551rLvfy8tK8efM0b968EueEh4dr3bp111xPhw4dtGfPnhuqEQAAAACA8qzcXWMPAAAAAADsZzrYnzx5UgMGDFBoaKgqV64sNzc3my8AAAAAAFB2TJ+KP2jQIB09elTPPvusQkJCZLFYSqMuAAAAAABgB9PBftu2bfryyy/VrFmzUigHAAAAAACYYfpU/LCwMBmGURq1AAAAAAAAk0wH+zlz5ujpp5/WL7/8UgrlAAAAAAAAM+w6Fb9atWo219JfvHhR9erVk4+Pj9zd3W3mnjlzxrEVAgAAAACAEtkV7OfMmVPKZQAAAAAAgBthV7BPSEgo7ToAAAAAAMANMH2NvZubm06dOlVs/PTp0zzHHgAAAACAMmY62Jd0R/y8vDx5eHjcdEEAAAAAAMB+dj/H/p///KckyWKxaOHChfL19bUuKygo0NatW9WgQQPHVwgAAAAAAEpkd7CfPXu2pD+O2C9YsMDmtHsPDw/VrVtXCxYscHyFAAAAAACgRHYH+yNHjkiS7r33Xn344YeqVq1aqRUFAAAAAADsY3ewL7J58+bSqAMAAAAAANwA08E+KSnpquMWi0VeXl6qX7++evTooerVq990cQAAAAAA4NpMB/s9e/bo22+/VUFBgSIjIyVJP/74o9zc3NSgQQO98cYbevLJJ7Vt2zY1atTI4QUDAAAAAID/Y/pxdz169FDnzp11/Phx7d69W7t379avv/6q2NhY9e3bV//5z3/Url07jRkzpjTqBQAAAAAAVzAd7F9++WU999xz8vPzs475+/trypQpmjlzpnx8fDRp0iTt3r3boYUCAAAAAIDiTAf7c+fO6dSpU8XGf/vtN2VlZUmSAgICdOnSpZuvDkCFZLGU7RcAAABQkd3QqfhDhgzR6tWr9euvv+rXX3/V6tWrNXToUPXs2VOStHPnTt1+++2OrhUAAAAAAPyJ6ZvnvfnmmxozZoz69Omjy5cv/7GSypWVkJCg2bNnS5IaNGighQsXOrZSAAAAAABQjOlg7+vrq7fffluzZ8/Wzz//LEm69dZb5evra53TrFkzhxUIAAAAAABKZjrYF/H19VVUVJQjawEAAAAAACaZDvYXL17UjBkztGnTJp06dUqFhYU2y4uO4gMAAAAAgNJnOtgPGzZMW7Zs0YABAxQSEiILt5wGAAAAAMBpTAf7zz77TGvXrlWbNm1Kox4AAAAAAGCC6cfdVatWTdWrVy+NWgAAAAAAgEmmg/1zzz2nSZMmKTs7uzTqAQAAAAAAJpg+FX/WrFnKyMhQUFCQ6tatK3d3d5vl3377rcOKAwAAAAAA12Y62Pfs2bMUygAAAAAAADfCdLCfPHlyadQBAAAAAABugOlr7CXp7NmzWrhwoSZMmKAzZ85I+uMU/P/85z8OLQ4AAAAAAFyb6SP2e/fuVefOneXv769ffvlFw4cPV/Xq1fXhhx/q6NGjevfdd0ujTgAAAAAAcBWmj9gnJSVp0KBB+umnn+Tl5WUdv//++7V161aHFgcAAAAAAK7NdLDftWuXHnnkkWLjt9xyizIzMx1SFAAAAAAAsI/pYO/p6amsrKxi4z/++KNq1arlkKIAAAAAAIB9TAf77t27a9q0acrPz5ckWSwWHT16VOPHj1evXr0cXiAAAAAAACiZ6WA/a9YsXbhwQYGBgcrJyVH79u1Vv359+fr66oUXXjC1runTp+uuu+5S1apVFRgYqJ49e+rQoUM2czp06CCLxWLz9eijj9rMOXr0qLp27SofHx8FBgbqqaee0uXLl23mpKam6s4775Snp6fq16+vJUuWmN11AAAAAADKHdN3xff391dKSoq++uorfffdd7pw4YLuvPNOde7c2fTGt2zZosTERN111126fPmynnnmGcXFxenAgQOqUqWKdd7w4cM1bdo062sfHx/r9wUFBeratauCg4O1fft2nThxQgMHDpS7u7tefPFFSdKRI0fUtWtXPfroo1q2bJk2bdqkYcOGKSQkRPHx8abrBgAAAACgvDAd7Iu0adNGbdq0sb7+4Ycf1L17d/344492r+Pzzz+3eb1kyRIFBgZq9+7dateunXXcx8dHwcHBV13Hhg0bdODAAW3cuFFBQUFq1qyZnnvuOY0fP15TpkyRh4eHFixYoIiICM2aNUuS1LBhQ23btk2zZ88m2AMAAAAAXNoNB/s/y8vLU0ZGxk2t49y5c5Kk6tWr24wvW7ZM//rXvxQcHKxu3brp2WeftR61T0tLU5MmTRQUFGSdHx8frxEjRmj//v1q3ry50tLSip1REB8fr9GjR5e4L3l5edbXRTcLzM/Pt95boLwqqq+814nywxk94+1dZpuCg3l78xkDc/i7BLPoGZhFz8AsV+kZM/U5LNjfrMLCQo0ePVpt2rTRHXfcYR3v16+fwsPDFRoaqr1792r8+PE6dOiQPvzwQ0lSZmamTaiXZH1d9Pi9kuZkZWUpJydH3n9KGdOnT9fUqVOL1bhhwwabywDKs5SUFGeXABdTlj2zfHmZbQqlhM8YmEXPwCx6BmbRMzCrvPdMdna23XPLTbBPTEzU999/r23bttmMP/zww9bvmzRpopCQEHXq1EkZGRmqV69eqdQyYcIEJSUlWV9nZWUpLCxMcXFx8vPzK5VtOkp+fr5SUlIUGxsrd3d3Z5cDF+CMnvH3L5PNoBR4e+dr0SI+Y2A//i7BLHoGZtEzMMtVeuZqj5kvSbkI9iNHjtSaNWu0detW1a5d+5pzo6OjJUmHDx9WvXr1FBwcrJ07d9rMOXnypCRZr8sPDg62jl05x8/Pr9jRekny9PSUp6dnsXF3d/dy/Yu/kivVivKhLHsmJ6dMNoNSxGcMzKJnYBY9A7PoGZhV3nvGTG12B/tq1arJYrGUuPzPj5ezh2EYGjVqlFavXq3U1FRFRERc9z3p6emSpJCQEElSTEyMXnjhBZ06dUqBgYGS/jilws/PT40aNbLOWbdunc16UlJSFBMTY7pmAAAAAADKE7uD/Zw5cxy+8cTERCUnJ+vjjz9W1apVrdfE+/v7y9vbWxkZGUpOTtb999+vGjVqaO/evRozZozatWunqKgoSVJcXJwaNWqkAQMGaObMmcrMzNTEiROVmJhoPer+6KOP6vXXX9e4ceM0ZMgQffHFF1q5cqXWrl3r8H0CAAAAAKAs2R3sExISHL7x+fPnS5I6dOhgM7548WINGjRIHh4e2rhxo+bMmaOLFy8qLCxMvXr10sSJE61z3dzctGbNGo0YMUIxMTGqUqWKEhISbJ57HxERobVr12rMmDGaO3euateurYULF/KoOwAAAACAy3PqNfaGYVxzeVhYmLZs2XLd9YSHhxc71f7POnTooD179piqDwAAAACA8q6SswsAAAAAAAA3jmAPAAAAAIALI9gDAAAAAODCTAf7adOmKTs7u9h4Tk6OzQ3rAAAAAABA6TMd7KdOnaoLFy4UG8/OztbUqVMdUhQAAAAAALCP6WBvGIYsFkux8e+++07Vq1d3SFEAAAAAAMA+dj/urlq1arJYLLJYLLr99tttwn1BQYEuXLigRx99tFSKBAAAAAAAV2d3sJ8zZ44Mw9CQIUM0depU+fv7W5d5eHiobt26iomJKZUiAQAAAADA1dkd7BMSEiRJERERat26tdzd3UutKAAAAAAAYB+7g32R9u3bW7/Pzc3VpUuXbJb7+fndfFUAAAAAAMAupm+el52drZEjRyowMFBVqlRRtWrVbL4AAAAAAEDZMR3sn3rqKX3xxReaP3++PD09tXDhQk2dOlWhoaF69913S6NGAAAAAABQAtOn4n/66ad699131aFDBw0ePFj33HOP6tevr/DwcC1btkz9+/cvjToBAAAAAMBVmD5if+bMGd16662S/rie/syZM5Kktm3bauvWrY6tDgAAAAAAXJPpYH/rrbfqyJEjkqQGDRpo5cqVkv44kh8QEODQ4gAAAAAAwLWZDvaDBw/Wd999J0l6+umnNW/ePHl5eWnMmDF66qmnHF4gAKD88feXLJay+wIAAEDJTF9jP2bMGOv3nTt31g8//KDdu3erfv36ioqKcmhxAAAAAADg2kwH+z8LDw9XeHi4I2oBAAAAAAAm3VCw37RpkzZt2qRTp06psLDQZtmiRYscUhgAAAAAALg+08F+6tSpmjZtmlq2bKmQkBBZuPgRAAAAAACnMR3sFyxYoCVLlmjAgAGlUQ8AAAAAADDB9F3xL126pNatW5dGLQAAAAAAwCTTwX7YsGFKTk4ujVoAAAAAAIBJdp2Kn5SUZP2+sLBQb731ljZu3KioqCi5u7vbzH311VcdWyEAAAAAACiRXcF+z549Nq+bNWsmSfr+++9txrmRHgAAAAAAZcuuYL958+bSrgMAAAAAANwAu6+xd3Nz06lTp0qzFgAAAAAAYJLdwd4wjNKsAwAAAAAA3ADTd8UHAAAAAADlh13X2BdZuHChfH19rznn8ccfv6mCAAAAAACA/UwF+wULFsjNza3E5RaLhWAPAAAAAEAZMhXsv/nmGwUGBpZWLQAAAAAAwCS7r7HnGfUAAAAAAJQ/3BUfAAAAAAAXZnewnzx58nVvnAcAAAAAAMqW3dfYT548uTTrAAAAAAAAN4Dn2AMAAAAA4MII9gAAAAAAuDC7g312drbDNz59+nTdddddqlq1qgIDA9WzZ08dOnTIZk5ubq4SExNVo0YN+fr6qlevXjp58qTNnKNHj6pr167y8fFRYGCgnnrqKV2+fNlmTmpqqu688055enqqfv36WrJkicP3BwAAAACAsmZ3sK9Zs6b+53/+R2+99ZYyMzMdsvEtW7YoMTFRO3bsUEpKivLz8xUXF6eLFy9a54wZM0affvqpVq1apS1btuj48eN68MEHrcsLCgrUtWtXXbp0Sdu3b9fSpUu1ZMkSTZo0yTrnyJEj6tq1q+69916lp6dr9OjRGjZsmNavX++Q/QAAAAAAwFnsvnneDz/8oI8//lgrV67U448/rqZNm6p79+7q3r27mjRpckMb//zzz21eL1myRIGBgdq9e7fatWunc+fO6Z133lFycrI6duwoSVq8eLEaNmyoHTt2qFWrVtqwYYMOHDigjRs3KigoSM2aNdNzzz2n8ePHa8qUKfLw8NCCBQsUERGhWbNmSZIaNmyobdu2afbs2YqPj7+h2gEAAAAAKA/sDvZ16tTRqFGjNGrUKJ07d07r1q3Txx9/rJdfflnVq1e3hvz27dvLzc3thoo5d+6cJKl69eqSpN27dys/P1+dO3e2zmnQoIHq1KmjtLQ0tWrVSmlpaWrSpImCgoKsc+Lj4zVixAjt379fzZs3V1pams06iuaMHj36qnXk5eUpLy/P+jorK0uSlJ+fr/z8/Bvat7JSVF95rxPlhzN6xtu7zDYFB/P2zrf5b1nhI8118XcJZtEzMIuegVmu0jNm6rM72F/J399fffv2Vd++fZWfn6/U1FR98sknGjx4sM6fP6/XXntN/fv3N7XOwsJCjR49Wm3atNEdd9whScrMzJSHh4cCAgJs5gYFBVkvB8jMzLQJ9UXLi5Zda05WVpZycnLk/aeUMX36dE2dOrVYjRs2bJCPj4+p/XKWlJQUZ5cAF1OWPbN8eZltCqVk0aKy/YxZt65MN4dSwN8lmEXPwCx6BmaV954xc5+7Gwr2V3J3d1dsbKxiY2P12muvac+ePcVuXGePxMREff/999q2bdvNlnTTJkyYoKSkJOvrrKwshYWFKS4uTn5+fk6s7Pry8/OVkpKi2NhYubu7O7scuABn9Iy/f5lsBqXA2ztfixalaMiQWOXklN1nzP9/QhdcEH+XYBY9A7PoGZjlKj1TdOa4PUwH+3bt2qlDhw5q37692rRpIy8vL5vlzZs3N7tKjRw5UmvWrNHWrVtVu3Zt63hwcLAuXbqks2fP2hy1P3nypIKDg61zdu7cabO+orvmXznnz3fSP3nypPz8/IodrZckT09PeXp6Fht3d3cv17/4K7lSrSgfyrJncnLKZDMoRTk57mUa7Pk4c338XYJZ9AzMomdgVnnvGTO1mX6OfVxcnHbs2KEePXooICBAbdu21cSJE5WSkmL6kXiGYWjkyJFavXq1vvjiC0VERNgsb9Gihdzd3bVp0ybr2KFDh3T06FHFxMRIkmJiYrRv3z6dOnXKOiclJUV+fn5q1KiRdc6V6yiaU7QOAAAAAABclekj9hMnTpQkXb58Wbt27dKWLVuUmpqqmTNnqlKlSsrNzbV7XYmJiUpOTtbHH3+sqlWrWq+J9/f3l7e3t/z9/TV06FAlJSWpevXq8vPz06hRoxQTE6NWrVpJ+uMfGho1aqQBAwZo5syZyszM1MSJE5WYmGg96v7oo4/q9ddf17hx4zRkyBB98cUXWrlypdauXWt29wEAAAAAKFdu+Br7n3/+Wfv27dN3332nvXv3qmrVqmrXrp2pdcyfP1+S1KFDB5vxxYsXa9CgQZKk2bNnq1KlSurVq5fy8vIUHx+vN954wzrXzc1Na9as0YgRIxQTE6MqVaooISFB06ZNs86JiIjQ2rVrNWbMGM2dO1e1a9fWwoULedQdAAAAAMDlmQ72/fr105YtW5SXl6d27dqpffv2evrppxUVFSWLxWJqXYZhXHeOl5eX5s2bp3nz5pU4Jzw8XOuuc8vkDh06aM+ePabqAwAAAACgvDMd7FesWKGaNWtq2LBh6tixo9q2besyj4ADAAAAAKCiMX3zvNOnT2vhwoW6dOmSJkyYoJo1a6p169Z65plntGHDhtKoEQAAAAAAlMB0sK9WrZq6d++uV199Vbt379bevXt1++236+WXX1aXLl1Ko0YAAAAAAFAC06finz592non/NTUVB04cEABAQHq1q2b2rdvXxo1AgAAAACAEpgO9oGBgapZs6buueceDR8+XB06dFCTJk1KozYAAAAAAHAdpoP93r171bhx49KoBQAAAAAAmGT6GntCPQAAAAAA5YfpYA8AAAAAAMoPgj0AAAAAAC6MYA8AAAAAgAtzSLA/e/asI1YDAAAAAABMMh3sX3rpJb3//vvW1w899JBq1KihW265Rd99951DiwMAAAAAANdmOtgvWLBAYWFhkqSUlBSlpKTos88+U5cuXfTUU085vEAAAAAAAFAy08+xz8zMtAb7NWvW6KGHHlJcXJzq1q2r6OhohxcIAAAAAABKZvqIfbVq1XTs2DFJ0ueff67OnTtLkgzDUEFBgWOrAwAAAAAA12T6iP2DDz6ofv366bbbbtPp06fVpUsXSdKePXtUv359hxcIAAAAAABKZjrYz549W3Xr1tWxY8c0c+ZM+fr6SpJOnDihxx57zOEFAgAAAACAkpkO9u7u7ho7dmyx8TFjxjikIAAAAAAAYL8beo79e++9p7Zt2yo0NFT//ve/JUlz5szRxx9/7NDiAAAAAADAtZkO9vPnz1dSUpK6dOmis2fPWm+YFxAQoDlz5ji6PgAAAAAAcA2mg/1rr72mt99+W//4xz/k5uZmHW/ZsqX27dvn0OIAAAAAAMC1mQ72R44cUfPmzYuNe3p66uLFiw4pCgAAAAAA2Md0sI+IiFB6enqx8c8//1wNGzZ0RE0AAAAAAMBOpu+Kn5SUpMTEROXm5sowDO3cuVPLly/X9OnTtXDhwtKoEQAAAAAAlMB0sB82bJi8vb01ceJEZWdnq1+/fgoNDdXcuXPVp0+f0qgRAAAAAACUwHSwl6T+/furf//+ys7O1oULFxQYGOjougAAAAAAgB1uKNgX8fHxkY+Pj6NqAQAAAAAAJtkV7Js3by6LxWLXCr/99tubKggAAAAAANjPrmDfs2fPUi4DAAAAAADcCLuC/eTJk0u7DgAAAAAAcANMP8d+165d+vrrr4uNf/311/rmm28cUhQAAAAAALCP6WCfmJioY8eOFRv/z3/+o8TERIcUBQAAAAAA7GM62B84cEB33nlnsfHmzZvrwIEDDikKAAAAAADYx3Sw9/T01MmTJ4uNnzhxQpUr39TT8wAAAAAAgEmmg31cXJwmTJigc+fOWcfOnj2rZ555RrGxsQ4tDgAAAAAAXJvpQ+yvvPKK2rVrp/DwcDVv3lySlJ6erqCgIL333nsOLxAAAAAAAJTMdLC/5ZZbtHfvXi1btkzfffedvL29NXjwYPXt21fu7u6lUSMAAAAAACjBDV0UX6VKFT388MOOrgWAE/n7Szk5zq4CAAAAgFl2XWP/ySefKD8/3/r9tb7M2Lp1q7p166bQ0FBZLBZ99NFHNssHDRoki8Vi83XffffZzDlz5oz69+8vPz8/BQQEaOjQobpw4YLNnL179+qee+6Rl5eXwsLCNHPmTFN1AgAAAABQXtl1xL5nz57KzMxUYGCgevbsWeI8i8WigoICuzd+8eJFNW3aVEOGDNGDDz541Tn33XefFi9ebH3t6elps7x///46ceKEUlJSlJ+fr8GDB+vhhx9WcnKyJCkrK0txcXHq3LmzFixYoH379mnIkCEKCAjgrAMAAAAAgMuzK9gXFhZe9fub1aVLF3Xp0uWaczw9PRUcHHzVZQcPHtTnn3+uXbt2qWXLlpKk1157Tffff79eeeUVhYaGatmyZbp06ZIWLVokDw8PNW7cWOnp6Xr11VcJ9gAAAAAAl2f6Gvt3331XvXv3Lnbk/NKlS1qxYoUGDhzosOIkKTU1VYGBgapWrZo6duyo559/XjVq1JAkpaWlKSAgwBrqJalz586qVKmSvv76az3wwANKS0tTu3bt5OHhYZ0THx+vl156Sb///ruqVatWbJt5eXnKy8uzvs7KypIk5efnWy9JKK+K6ivvdaL8KOoVb296BtdX1Cdl3S98pLku/i7BLHoGZtEzMMtVesZMfRbDMAwzK3dzc9OJEycUGBhoM3769GkFBgaaOhXfphCLRatXr7Y51X/FihXy8fFRRESEMjIy9Mwzz8jX11dpaWlyc3PTiy++qKVLl+rQoUM26woMDNTUqVM1YsQIxcXFKSIiQm+++aZ1+YEDB9S4cWMdOHBADRs2LFbLlClTNHXq1GLjycnJ8vHxuaH9AwAAAADAXtnZ2erXr5/OnTsnPz+/a841fcTeMAxZLJZi47/++qv8/f3Nru6a+vTpY/2+SZMmioqKUr169ZSamqpOnTo5dFtXmjBhgpKSkqyvs7KyFBYWpri4uOv+QJ0tPz9fKSkpio2N5fGDsEtRzwwZEqucHHoG1+btna9Fi8q+X86dK7NNwcH4uwSz6BmYRc/ALFfpmaIzx+1hd7Bv3ry59c70nTp1UuXK//fWgoICHTlypNgd6x3t1ltvVc2aNXX48GF16tRJwcHBOnXqlM2cy5cv68yZM9br8oODg3Xy5EmbOUWvS7p239PTs9ilBpLk7u5ern/xV3KlWlE+5OS4E+xht7LuFz7OXB9/l2AWPQOz6BmYVd57xkxtdgf7olPk09PTFR8fL19fX+syDw8P1a1bV7169bK/yhvw66+/6vTp0woJCZEkxcTE6OzZs9q9e7datGghSfriiy9UWFio6Oho65x//OMfys/Pt/5gUlJSFBkZedXr6wEAAAAAcCV2B/vJkyeroKBAdevWVVxcnDVc34wLFy7o8OHD1tdHjhxRenq6qlevrurVq2vq1Knq1auXgoODlZGRoXHjxql+/fqKj4+XJDVs2FD33Xefhg8frgULFig/P18jR45Unz59FBoaKknq16+fpk6dqqFDh2r8+PH6/vvvNXfuXM2ePfum6wcAAAAAwNkqmZns5uamRx55RLm5uQ7Z+DfffKPmzZurefPmkqSkpCQ1b95ckyZNkpubm/bu3avu3bvr9ttv19ChQ9WiRQt9+eWXNqfJL1u2TA0aNFCnTp10//33q23btnrrrbesy/39/bVhwwYdOXJELVq00JNPPqlJkybxqDsAAAAAQIVg+uZ5d9xxh37++WdFRETc9MY7dOiga92Uf/369dddR/Xq1ZWcnHzNOVFRUfryyy9N1wcAAAAAQHln6oi9JD3//PMaO3as1qxZoxMnTigrK8vmCwAAAAAAlB3TR+zvv/9+SVL37t1tHntX9Bi8G32OPQAAAAAAMM90sN+8eXNp1AEAQImu+HfkMnGNq8QAAADKHdPBvn379iUu+/7772+qGAAAAAAAYI7pa+z/7Pz583rrrbd09913q2nTpo6oCQAAAAAA2OmGg/3WrVuVkJCgkJAQvfLKK+rYsaN27NjhyNoAAAAAAMB1mDoVPzMzU0uWLNE777yjrKwsPfTQQ8rLy9NHH32kRo0alVaNAAAAAACgBHYfse/WrZsiIyO1d+9ezZkzR8ePH9drr71WmrUBAAAAAIDrsPuI/WeffabHH39cI0aM0G233VaaNQEAAAAAADvZfcR+27ZtOn/+vFq0aKHo6Gi9/vrr+u9//1uatQEAAAAAgOuwO9i3atVKb7/9tk6cOKFHHnlEK1asUGhoqAoLC5WSkqLz58+XZp0AAAAAAOAqTN8Vv0qVKhoyZIi2bdumffv26cknn9SMGTMUGBio7t27l0aNAAAAAACgBDf1HPvIyEjNnDlTv/76q5YvX+6omgAAAAAAgJ1uKtgXcXNzU8+ePfXJJ584YnUAAAAAAMBODgn2AAAAAADAOQj2AAAAAAC4MII9AAAAAAAuzK5gf+edd+r333+XJE2bNk3Z2dmlWhQAAAAAALCPXcH+4MGDunjxoiRp6tSpunDhQqkWBQAAAAAA7FPZnknNmjXT4MGD1bZtWxmGoVdeeUW+vr5XnTtp0iSHFggAAAAAAEpmV7BfsmSJJk+erDVr1shiseizzz5T5crF32qxWAj2AAAAAACUIbuCfWRkpFasWCFJqlSpkjZt2qTAwMBSLQwAAAAAAFyfXcH+SoWFhaVRBwAAAAAAuAGmg70kZWRkaM6cOTp48KAkqVGjRnriiSdUr149hxYHAAAAAACuzfRz7NevX69GjRpp586dioqKUlRUlL7++ms1btxYKSkppVEjAAAAAAAogekj9k8//bTGjBmjGTNmFBsfP368YmNjHVYcAAAAAAC4NtNH7A8ePKihQ4cWGx8yZIgOHDjgkKIAAAAAAIB9TAf7WrVqKT09vdh4eno6d8oHAAAAAKCMmT4Vf/jw4Xr44Yf1888/q3Xr1pKkr776Si+99JKSkpIcXiAAAAAAACiZ6WD/7LPPqmrVqpo1a5YmTJggSQoNDdWUKVP0+OOPO7xAAAAAAABQMtPB3mKxaMyYMRozZozOnz8vSapatarDCwMAAAAAANd3Q8+xL0KgBwAAAADAuUzfPA8AAAAAAJQfBHsAAAAAAFwYwR4AAAAAABdGsAcAAAAAwIXdULAfOXKkzpw54+haAAAAAACASXYH+19//dX6fXJysi5cuCBJatKkiY4dO+b4ygAAAAAAwHXZ/bi7Bg0aqEaNGmrTpo1yc3N17Ngx1alTR7/88ovy8/NLs0YAAAAAAFACu4/Ynz17VqtWrVKLFi1UWFio+++/X7fffrvy8vK0fv16nTx50vTGt27dqm7duik0NFQWi0UfffSRzXLDMDRp0iSFhITI29tbnTt31k8//WQz58yZM+rfv7/8/PwUEBCgoUOHWs8mKLJ3717dc8898vLyUlhYmGbOnGm6VgAAAAAAyiO7g31+fr7uvvtuPfnkk/L29taePXu0ePFiubm5adGiRYqIiFBkZKSpjV+8eFFNmzbVvHnzrrp85syZ+uc//6kFCxbo66+/VpUqVRQfH6/c3FzrnP79+2v//v1KSUnRmjVrtHXrVj388MPW5VlZWYqLi1N4eLh2796tl19+WVOmTNFbb71lqlYAAAAAAMoju0/FDwgIULNmzdSmTRtdunRJOTk5atOmjSpXrqz3339ft9xyi3bt2mVq4126dFGXLl2uuswwDM2ZM0cTJ05Ujx49JEnvvvuugoKC9NFHH6lPnz46ePCgPv/8c+3atUstW7aUJL322mu6//779corryg0NFTLli3TpUuXtGjRInl4eKhx48ZKT0/Xq6++avMPAFfKy8tTXl6e9XVWVpakP/5xo7xfdlBUX3mvE+VHUa94e9MzuL6iPqno/cJHqOPwdwlm0TMwi56BWa7SM2bqszvY/+c//1FaWpq2b9+uy5cvq0WLFrrrrrt06dIlffvtt6pdu7batm17QwVfzZEjR5SZmanOnTtbx/z9/RUdHa20tDT16dNHaWlpCggIsIZ6SercubMqVaqkr7/+Wg888IDS0tLUrl07eXh4WOfEx8frpZde0u+//65q1aoV2/b06dM1derUYuMbNmyQj4+Pw/axNKWkpDi7BLiYRYvoGdivovfLunXOrqDi4e8SzKJnYBY9A7PKe89kZ2fbPdfuYF+zZk1169ZN3bp104IFC7R161YdPHhQAwcO1NixYzVgwADdfffd2rJlyw0V/WeZmZmSpKCgIJvxoKAg67LMzEwFBgbaLK9cubKqV69uMyciIqLYOoqWXS3YT5gwQUlJSdbXWVlZCgsLU1xcnPz8/G5yz0pXfn6+UlJSFBsbK3d3d2eXAxdQ1DNDhsQqJ4eewbV5e+dr0aKK3y/nzjm7goqDv0swi56BWfQMzHKVnik6c9wedgf7P/P399dDDz2koUOH6osvvpCPj4/DQr2zeXp6ytPTs9i4u7t7uf7FX8mVakX5kJPjXqGDGhyrovcLH5+Ox98lmEXPwCx6BmaV954xU5vdN8+70t69e1W7dm1JUnh4uNzd3RUcHKzevXvfyOquKjg4WJKK3W3/5MmT1mXBwcE6deqUzfLLly/rzJkzNnOuto4rtwEAAAAAgKu6oWAfFhamSpX+eOv333+vsLAwhxYlSREREQoODtamTZusY1lZWfr6668VExMjSYqJidHZs2e1e/du65wvvvhChYWFio6Ots7ZunWrzY0HUlJSFBkZedXT8AEAAAAAcCU3FOwd5cKFC0pPT1d6erqkP26Yl56erqNHj8pisWj06NF6/vnn9cknn2jfvn0aOHCgQkND1bNnT0lSw4YNdd9992n48OHauXOnvvrqK40cOVJ9+vRRaGioJKlfv37y8PDQ0KFDtX//fr3//vuaO3euzTX0AAAAAAC4qhu+xt4RvvnmG917773W10VhOyEhQUuWLNG4ceN08eJFPfzwwzp79qzatm2rzz//XF5eXtb3LFu2TCNHjlSnTp1UqVIl9erVS//85z+ty/39/bVhwwYlJiaqRYsWqlmzpiZNmlTio+4AAAAAAHAlTg32HTp0kGEYJS63WCyaNm2apk2bVuKc6tWrKzk5+ZrbiYqK0pdffnnDdQIAAAAAUF459VR8AAAAAABwcwj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC7Mqc+xBwCgPLJYynZ7hlG22wMAABULR+wBAAAAAHBhBHsAAAAAAFwYwR4AAAAAABdGsAcAAAAAwIUR7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYQR7AAAAAABcGMEeAAAAAAAXRrAHAAAAAMCFEewBAAAAAHBhBHsAAAAAAFwYwR4oZyyWsv3y93f2HgMAAAC4GQR7AAAAAABcGMEeAAAAAAAXRrAHAAAAAMCFEewBAAAAAHBhBHsAAAAAAFwYwR4AAAAAABdGsAcAAAAAwIUR7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYQR7AAAAAABcGMEeAAAAAAAXRrAHAAAAAMCFletgP2XKFFksFpuvBg0aWJfn5uYqMTFRNWrUkK+vr3r16qWTJ0/arOPo0aPq2rWrfHx8FBgYqKeeekqXL18u610BAAAAAKBUVHZ2AdfTuHFjbdy40fq6cuX/K3nMmDFau3atVq1aJX9/f40cOVIPPvigvvrqK0lSQUGBunbtquDgYG3fvl0nTpzQwIED5e7urhdffLHM9wUAAAAAAEcr98G+cuXKCg4OLjZ+7tw5vfPOO0pOTlbHjh0lSYsXL1bDhg21Y8cOtWrVShs2bNCBAwe0ceNGBQUFqVmzZnruuec0fvx4TZkyRR4eHmW9OwAAAAAAOFS5D/Y//fSTQkND5eXlpZiYGE2fPl116tTR7t27lZ+fr86dO1vnNmjQQHXq1FFaWppatWqltLQ0NWnSREFBQdY58fHxGjFihPbv36/mzZtfdZt5eXnKy8uzvs7KypIk5efnKz8/v5T21DGK6ivvdaJk3t5lvb18m/8C10K/lI6K/JHN3yWYRc/ALHoGZrlKz5ipr1wH++joaC1ZskSRkZE6ceKEpk6dqnvuuUfff/+9MjMz5eHhoYCAAJv3BAUFKTMzU5KUmZlpE+qLlhctK8n06dM1derUYuMbNmyQj4/PTe5V2UhJSXF2CbhBy5c7Z7uLFtEzsB/94ljr1jm7gtLH3yWYRc/ALHoGZpX3nsnOzrZ7brkO9l26dLF+HxUVpejoaIWHh2vlypXyLsXDmhMmTFBSUpL1dVZWlsLCwhQXFyc/P79S264j5OfnKyUlRbGxsXJ3d3d2ObgB/v5luz1v73wtWpSiIUNilZNDz+Da6JeK4dy5stsWf5dgFj0Ds+gZmOUqPVN05rg9ynWw/7OAgADdfvvtOnz4sGJjY3Xp0iWdPXvW5qj9yZMnrdfkBwcHa+fOnTbrKLpr/tWu2y/i6ekpT0/PYuPu7u7l+hd/JVeqFbZycpy1XXeCGuxGv7g2Z/x54O8SzKJnYBY9A7PKe8+Yqa1cP+7uzy5cuKCMjAyFhISoRYsWcnd316ZNm6zLDx06pKNHjyomJkaSFBMTo3379unUqVPWOSkpKfLz81OjRo3KvH4AAAAAABytXB+xHzt2rLp166bw8HAdP35ckydPlpubm/r27St/f38NHTpUSUlJql69uvz8/DRq1CjFxMSoVatWkqS4uDg1atRIAwYM0MyZM5WZmamJEycqMTHxqkfkAQAAAABwNeU62P/666/q27evTp8+rVq1aqlt27basWOHatWqJUmaPXu2KlWqpF69eikvL0/x8fF64403rO93c3PTmjVrNGLECMXExKhKlSpKSEjQtGnTnLVLAAAAAAA4VLkO9itWrLjmci8vL82bN0/z5s0rcU54eLjW/RVuNwwAAAAA+EtyqWvsAQAAAACALYI9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAMBfjMVSdl/+/s7eWwAAKj6CPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAAAAALgwgj0AAAAAAC6ssrMLAAAAFZ+/v5STU3bbM4yy2xYAAM7GEXsAAAAAAFwYwR4AAAAAABdGsAcAAAAAwIUR7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYZWdXQAAAICjWSxluz3DKNvtAQBwJY7YAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGvsAQAAbhLX9AMAnOkvdcR+3rx5qlu3rry8vBQdHa2dO3c6uyQAAAAAAG7KXybYv//++0pKStLkyZP17bffqmnTpoqPj9epU6ecXRoAAAAAADfsLxPsX331VQ0fPlyDBw9Wo0aNtGDBAvn4+GjRokXOLg3lnMVStl8AAFwPf5sAAFf6S1xjf+nSJe3evVsTJkywjlWqVEmdO3dWWlpasfl5eXnKy8uzvj537pwk6cyZM8rPzy/9gm9Cfn6+srOzdfr0abm7uzu7nArBy8vZFZQuL68/esbL67QMg57BtdEvMIueqRi8vctyW/maN69s/1+mdu0y2YyNX38t+21WVPz/L8xylZ45f/68JMmw48Yqf4lg/9///lcFBQUKCgqyGQ8KCtIPP/xQbP706dM1derUYuMRERGlViPgLLm5Ur9+zq4CroJ+gVn0DMz6q/RMzZrOrgCAqzh//rz8/f2vOecvEezNmjBhgpKSkqyvCwsLdebMGdWoUUOWcn4+WlZWlsLCwnTs2DH5+fk5uxy4AHoGZtAvMIuegVn0DMyiZ2CWq/SMYRg6f/68QkNDrzv3LxHsa9asKTc3N508edJm/OTJkwoODi4239PTU56enjZjAQEBpVmiw/n5+ZXrJkX5Q8/ADPoFZtEzMIuegVn0DMxyhZ653pH6In+Jm+d5eHioRYsW2rRpk3WssLBQmzZtUkxMjBMrAwAAAADg5vwljthLUlJSkhISEtSyZUvdfffdmjNnji5evKjBgwc7uzQAAAAAAG7YXybY9+7dW7/99psmTZqkzMxMNWvWTJ9//nmxG+q5Ok9PT02ePLnYpQRASegZmEG/wCx6BmbRMzCLnoFZFbFnLIY9984HAAAAAADl0l/iGnsAAAAAACoqgj0AAAAAAC6MYA8AAAAAgAsj2AMAAAAA4MII9hXIvHnzVLduXXl5eSk6Olo7d+50dkkop6ZMmSKLxWLz1aBBA2eXhXJk69at6tatm0JDQ2WxWPTRRx/ZLDcMQ5MmTVJISIi8vb3VuXNn/fTTT84pFuXC9Xpm0KBBxT537rvvPucUi3Jh+vTpuuuuu1S1alUFBgaqZ8+eOnTokM2c3NxcJSYmqkaNGvL19VWvXr108uRJJ1UMZ7KnXzp06FDsc+bRRx91UsVwtvnz5ysqKkp+fn7y8/NTTEyMPvvsM+vyivb5QrCvIN5//30lJSVp8uTJ+vbbb9W0aVPFx8fr1KlTzi4N5VTjxo114sQJ69e2bducXRLKkYsXL6pp06aaN2/eVZfPnDlT//znP7VgwQJ9/fXXqlKliuLj45Wbm1vGlaK8uF7PSNJ9991n87mzfPnyMqwQ5c2WLVuUmJioHTt2KCUlRfn5+YqLi9PFixetc8aMGaNPP/1Uq1at0pYtW3T8+HE9+OCDTqwazmJPv0jS8OHDbT5nZs6c6aSK4Wy1a9fWjBkztHv3bn3zzTfq2LGjevToof3790uqgJ8vBiqEu+++20hMTLS+LigoMEJDQ43p06c7sSqUV5MnTzaaNm3q7DLgIiQZq1evtr4uLCw0goODjZdfftk6dvbsWcPT09NYvny5EypEefPnnjEMw0hISDB69OjhlHrgGk6dOmVIMrZs2WIYxh+fK+7u7saqVauscw4ePGhIMtLS0pxVJsqJP/eLYRhG+/btjSeeeMJ5RaHcq1atmrFw4cIK+fnCEfsK4NKlS9q9e7c6d+5sHatUqZI6d+6stLQ0J1aG8uynn35SaGiobr31VvXv319Hjx51dklwEUeOHFFmZqbNZ46/v7+io6P5zME1paamKjAwUJGRkRoxYoROnz7t7JJQjpw7d06SVL16dUnS7t27lZ+fb/NZ06BBA9WpU4fPGhTrlyLLli1TzZo1dccdd2jChAnKzs52RnkoZwoKCrRixQpdvHhRMTExFfLzpbKzC8DN++9//6uCggIFBQXZjAcFBemHH35wUlUoz6Kjo7VkyRJFRkbqxIkTmjp1qu655x59//33qlq1qrPLQzmXmZkpSVf9zClaBvzZfffdpwcffFARERHKyMjQM888oy5duigtLU1ubm7OLg9OVlhYqNGjR6tNmza64447JP3xWePh4aGAgACbuXzW4Gr9Ikn9+vVTeHi4QkNDtXfvXo0fP16HDh3Shx9+6MRq4Uz79u1TTEyMcnNz5evrq9WrV6tRo0ZKT0+vcJ8vBHvgL6hLly7W76OiohQdHa3w8HCtXLlSQ4cOdWJlACqqPn36WL9v0qSJoqKiVK9ePaWmpqpTp05OrAzlQWJior7//nvu9wK7lNQvDz/8sPX7Jk2aKCQkRJ06dVJGRobq1atX1mWiHIiMjFR6errOnTunDz74QAkJCdqyZYuzyyoVnIpfAdSsWVNubm7F7uJ48uRJBQcHO6kquJKAgADdfvvtOnz4sLNLgQso+lzhMwc349Zbb1XNmjX53IFGjhypNWvWaPPmzapdu7Z1PDg4WJcuXdLZs2dt5vNZ89dWUr9cTXR0tCTxOfMX5uHhofr166tFixaaPn26mjZtqrlz51bIzxeCfQXg4eGhFi1aaNOmTdaxwsJCbdq0STExMU6sDK7iwoULysjIUEhIiLNLgQuIiIhQcHCwzWdOVlaWvv76az5zYLdff/1Vp0+f5nPnL8wwDI0cOVKrV6/WF198oYiICJvlLVq0kLu7u81nzaFDh3T06FE+a/6CrtcvV5Oeni5JfM7AqrCwUHl5eRXy84VT8SuIpKQkJSQkqGXLlrr77rs1Z84cXbx4UYMHD3Z2aSiHxo4dq27duik8PFzHjx/X5MmT5ebmpr59+zq7NJQTFy5csDnCceTIEaWnp6t69eqqU6eORo8ereeff1633XabIiIi9Oyzzyo0NFQ9e/Z0XtFwqmv1TPXq1TV16lT16tVLwcHBysjI0Lhx41S/fn3Fx8c7sWo4U2JiopKTk/Xxxx+ratWq1uta/f395e3tLX9/fw0dOlRJSUmqXr26/Pz8NGrUKMXExKhVq1ZOrh5l7Xr9kpGRoeTkZN1///2qUaOG9u7dqzFjxqhdu3aKiopycvVwhgkTJqhLly6qU6eOzp8/r+TkZKWmpmr9+vUV8/PF2bflh+O89tprRp06dQwPDw/j7rvvNnbs2OHsklBO9e7d2wgJCTE8PDyMW265xejdu7dx+PBhZ5eFcmTz5s2GpGJfCQkJhmH88ci7Z5991ggKCjI8PT2NTp06GYcOHXJu0XCqa/VMdna2ERcXZ9SqVctwd3c3wsPDjeHDhxuZmZnOLhtOdLV+kWQsXrzYOicnJ8d47LHHjGrVqhk+Pj7GAw88YJw4ccJ5RcNprtcvR48eNdq1a2dUr17d8PT0NOrXr2889dRTxrlz55xbOJxmyJAhRnh4uOHh4WHUqlXL6NSpk7Fhwwbr8or2+WIxDMMoy39IAAAAAAAAjsM19gAAAAAAuDCCPQAAAAAALoxgDwAAAACACyPYAwAAAADgwgj2AAAAAAC4MII9AAAAAAAujGAPAAAAAIALI9gDAAAAAODCCPYAAKBc+uWXX2SxWJSenm76vZs2bVLDhg1VUFAgSZoyZYqaNWvm2ALLmJl9WLBggbp161a6BQEAyg2CPQCgwvvtt980YsQI1alTR56engoODlZ8fLy++uorZ5fmdDcTnh1p0KBB6tmzp8PWN27cOE2cOFFubm6SpLFjx2rTpk03vL4ffvhBFotFO3bssBlv1aqVvLy8lJubax3Lzc2Vl5eX3nnnnRve3s0aMmSIvv32W3355ZdOqwEAUHYI9gCACq9Xr17as2ePli5dqh9//FGffPKJOnTooNOnTzu7NIe5dOmSs0soN7Zt26aMjAz16tXLOubr66saNWrc8DobNGig4OBgpaamWsfOnz+vb7/9VrVq1bIJ/GlpacrLy1PHjh1vaFv5+fk3XGcRDw8P9evXT//85z9vel0AgPKPYA8AqNDOnj2rL7/8Ui+99JLuvfdehYeH6+6779aECRPUvXt3SVc/an327FlZLBZrkEtNTZXFYtH69evVvHlzeXt7q2PHjjp16pQ+++wzNWzYUH5+furXr5+ys7Ot6+nQoYNGjRql0aNHq1q1agoKCtLbb7+tixcvavDgwapatarq16+vzz77zPqegoICDR06VBEREfL29lZkZKTmzp1rs19FR7hfeOEFhYaGKjIyUtOmTdMdd9xR7GfQrFkzPfvsszf08yssLNT06dOttTRt2lQffPCBdXnRz2XTpk1q2bKlfHx81Lp1ax06dMhmPc8//7wCAwNVtWpVDRs2TE8//bT1tPIpU6Zo6dKl+vjjj2WxWGx+7pL0888/695775WPj4+aNm2qtLS0a9a8YsUKxcbGysvLyzr259PYi35+r7zyikJCQlSjRg0lJiZeM1Tfe++9NnVt27ZNt99+u7p162YznpqaqvDwcEVEREiS5s+fr3r16snDw0ORkZF67733bNZrsVg0f/58de/eXVWqVNELL7wgSZoxY4aCgoJUtWpVDR061OasgKLt3H333apSpYoCAgLUpk0b/fvf/7Yu79atmz755BPl5ORc8+cFAHB9BHsAQIXm6+srX19fffTRR8rLy7vp9U2ZMkWvv/66tm/frmPHjumhhx7SnDlzlJycrLVr12rDhg167bXXbN6zdOlS1axZUzt37tSoUaM0YsQI/f3vf1fr1q317bffKi4uTgMGDLD+g0BhYaFq166tVatW6cCBA5o0aZKeeeYZrVy50ma9mzZt0qFDh5SSkqI1a9ZoyJAhOnjwoHbt2mWds2fPHu3du1eDBw++of2dPn263n33XS1YsED79+/XmDFj9L//+7/asmWLzbx//OMfmjVrlr755htVrlxZQ4YMsS5btmyZXnjhBb300kvavXu36tSpo/nz51uXjx07Vg899JDuu+8+nThxQidOnFDr1q1t1j127Filp6fr9ttvV9++fXX58uUSa/7yyy/VsmXL6+7b5s2blZGRoc2bN2vp0qVasmSJlixZUuL8e++9V9u2bbNue/PmzerQoYPat2+vzZs326z33nvvlSStXr1aTzzxhJ588kl9//33euSRRzR48GCb+dIfffXAAw9o3759GjJkiFauXKkpU6boxRdf1DfffKOQkBC98cYb1vmXL19Wz5491b59e+3du1dpaWl6+OGHZbFYrHNatmypy5cv6+uvv77uzwIA4OIMAAAquA8++MCoVq2a4eXlZbRu3dqYMGGC8d1331mXHzlyxJBk7Nmzxzr2+++/G5KMzZs3G4ZhGJs3bzYkGRs3brTOmT59uiHJyMjIsI498sgjRnx8vPV1+/btjbZt21pfX7582ahSpYoxYMAA69iJEycMSUZaWlqJ+5CYmGj06tXL+johIcEICgoy8vLybOZ16dLFGDFihPX1qFGjjA4dOpS43qvte5Hc3FzDx8fH2L59u8340KFDjb59+xqGcfWfy9q1aw1JRk5OjmEYhhEdHW0kJibarKNNmzZG06ZNbfanR48eV61t4cKF1rH9+/cbkoyDBw+WuE/+/v7Gu+++azM2efLkYtsLDw83Ll++bB37+9//bvTu3bvE9f7000+GJOvP46677jJWrlxpHD9+3PD09DRycnKM7Oxsw9PT01i6dKlhGIbRunVrY/jw4Tbr+fvf/27cf//91teSjNGjR9vMiYmJMR577DGbsejoaOs+nD592pBkpKamllivYRhGtWrVjCVLllxzDgDA9XHEHgBQ4fXq1UvHjx/XJ598ovvuu0+pqam68847r3l0tiRRUVHW74OCguTj46Nbb73VZuzUqVMlvsfNzU01atRQkyZNbN4jyeZ98+bNU4sWLVSrVi35+vrqrbfe0tGjR23W26RJE3l4eNiMDR8+XMuXL1dubq4uXbqk5ORkm6PnZhw+fFjZ2dmKjY21nvng6+urd999VxkZGSXuY0hIiM3+HDp0SHfffbfN/D+/vpZrrftqcnJybE7DL0njxo2tN9crWve11lu/fn3Vrl1bqampysrK0p49e9S+fXuFhISoTp06SktLs15fX3TE/uDBg2rTpo3Netq0aaODBw/ajP35DIODBw8qOjraZiwmJsb6ffXq1TVo0CDFx8erW7dumjt3rk6cOFGsZm9vb5tLQwAAFVNlZxcAAEBZ8PLyUmxsrGJjY/Xss89q2LBhmjx5sgYNGqRKlf74d27DMKzzS7rW2t3d3fq9xWKxeV00VlhYWOJ7rva+otOni963YsUKjR07VrNmzVJMTIyqVq2ql19+udgp1VWqVClWX7du3eTp6anVq1fLw8ND+fn5+tvf/nb1H8p1XLhwQZK0du1a3XLLLTbLPD09S9zHP+/PzTK77po1a+r33383td6idV+v5g4dOmjz5s2KiorSbbfdpsDAQEmyno5vGIbq16+vsLCw627/Slf7XV7P4sWL9fjjj+vzzz/X+++/r4kTJyolJUWtWrWyzjlz5oxq1aplet0AANfCEXsAwF9So0aNdPHiRUmyBp8rj3g68/FvX331lVq3bq3HHntMzZs3V/369YsdIS9J5cqVlZCQoMWLF2vx4sXq06ePvL29b6iORo0aydPTU0ePHlX9+vVtvswE18jISJvr/iUVe+3h4WF95vzNat68uQ4cOOCQdf3Zvffeq+3btyslJUUdOnSwjrdr106pqalKTU21Hq2XpIYNGxZ7rOJXX32lRo0aXXM7DRs2LPYPOX9+1J70x75OmDBB27dv1x133KHk5GTrsoyMDOXm5qp58+ZmdhEA4II4Yg8AqNBOnz6tv//97xoyZIiioqJUtWpVffPNN5o5c6Z69Ogh6Y/TlVu1aqUZM2YoIiJCp06d0sSJE51W82233aZ3331X69evV0REhN577z3t2rXLepf16xk2bJgaNmwoScVCZUn+fBd76Y9T1ceOHasxY8aosLBQbdu21blz5/TVV1/Jz89PCQkJdq171KhRGj58uFq2bKnWrVvr/fff1969e20uYahbt67Wr1+vQ4cOqUaNGvL397dr3VcTHx+vpUuX3vD7r+Xee+/VxYsXtWjRIr399tvW8fbt22vYsGGSpMcee8w6/tRTT+mhhx5S8+bN1blzZ3366af68MMPtXHjxmtu54knntCgQYPUsmVLtWnTRsuWLdP+/futP7MjR47orbfeUvfu3RUaGqpDhw7pp59+0sCBA63r+PLLL3XrrbeqXr16jvwRAADKIYI9AKBC8/X1VXR0tGbPnq2MjAzl5+crLCxMw4cP1zPPPGOdt2jRIg0dOlQtWrRQZGSkZs6cqbi4OKfU/Mgjj2jPnj3q3bu3LBaL+vbtq8cee8zmkXjXctttt6l169Y6c+ZMseu0S9KnT59iY8eOHdNzzz2nWrVqafr06fr5558VEBCgO++80+Zndz39+/fXzz//rLFjxyo3N1cPPfSQBg0apJ07d1rnDB8+XKmpqWrZsqUuXLigzZs3q27dunZv48/bGzdunA4dOqTIyMgbWkdJIiIiFB4ern//+99q3769dbxOnToKDQ3VL7/8YnMkv2fPnpo7d65eeeUVPfHEE4qIiNDixYtt5lxN7969lZGRoXHjxik3N1e9evXSiBEjtH79ekmSj4+PfvjhBy1dulSnT59WSEiIEhMT9cgjj1jXsXz5cg0fPtyh+w8AKJ8sxpUXFAIAAJdnGIZuu+02PfbYY0pKSnJ2OVcVGxur4ODgYs90d5SnnnpKWVlZevPNN0tl/eXd/v371bFjR/344483dfYDAMA1cMQeAIAK5LffftOKFSuUmZl5w8+ud7Ts7GwtWLBA8fHxcnNz0/Lly7Vx40alpKSU2jb/8Y9/6I033lBhYaH15oh/JSdOnNC7775LqAeAvwiO2AMAUIFYLBbVrFlTc+fOVb9+/ZxdjqQ/Hj/XrVs37dmzR7m5uYqMjNTEiRP14IMPOrs0AAAqBII9AAAAAAAu7K93bhoAAAAAABUIwR4AAAAAABdGsAcAAAAAwIUR7AEAAAAAcGEEewAAAAAAXBjBHgAAAAAAF0awBwAAAADAhRHsAQAAAABwYf8fQ0iYaIBSTNwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1200x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df['summary_lengths'] = train_df['summary'].str.split().apply(len)\n",
    "eval_df['summary_lengths'] = eval_df['summary'].str.split().apply(len)\n",
    "\n",
    "summary_lengths = pd.concat([pd.Series(train_df['summary_lengths']), pd.Series(eval_df['summary_lengths'])])\n",
    "\n",
    "print(f\"The longest string (in words) in this dataset is {max(summary_lengths)} words.\")\n",
    "\n",
    "n = 30\n",
    "summary_lengths.hist(grid=True, color=\"Blue\", figsize=(12,5), bins=n, range=(0, n))\n",
    "plt.suptitle('')\n",
    "plt.xlabel('Summary Length (in Words)')\n",
    "plt.ylabel('# of Articles w/ That Length')\n",
    "plt.title('Summary Length')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trimmed Dataset Based on Article &/or Summary Length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The longest string (in words) in this dataset is 60 words.\n",
      "The longest string (in words) in this dataset is 18 words.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jason blake of the islanders will miss the res...</td>\n",
       "      <td>blake missing rest of season</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>the u.s. military on wednesday captured a wife...</td>\n",
       "      <td>u.s. arrests wife and daughter of saddam deput...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>craig bellamy 's future at west ham appeared i...</td>\n",
       "      <td>west ham drops bellamy amid transfer turmoil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cambridge - when barack obama sought advice be...</td>\n",
       "      <td>in search for expertise harvard looms large</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wall street held on to steep gains on monday ,...</td>\n",
       "      <td>wall street ends a three-day losing streak</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             article  \\\n",
       "0  jason blake of the islanders will miss the res...   \n",
       "1  the u.s. military on wednesday captured a wife...   \n",
       "2  craig bellamy 's future at west ham appeared i...   \n",
       "3  cambridge - when barack obama sought advice be...   \n",
       "4  wall street held on to steep gains on monday ,...   \n",
       "\n",
       "                                             summary  \n",
       "0                       blake missing rest of season  \n",
       "1  u.s. arrests wife and daughter of saddam deput...  \n",
       "2       west ham drops bellamy amid transfer turmoil  \n",
       "3        in search for expertise harvard looms large  \n",
       "4         wall street ends a three-day losing streak  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# trim training & evaluation article lengths to have no more than 60 words\n",
    "train_df = train_df[train_df['article_lengths'] < 61]\n",
    "train_df = train_df[train_df['article_lengths'] > 17]\n",
    "\n",
    "eval_df = eval_df[eval_df['article_lengths'] < 61]\n",
    "eval_df = eval_df[eval_df['article_lengths'] > 17]\n",
    "\n",
    "# trim training & evaluation article lengths to have no less than 18 words\n",
    "train_df = train_df[train_df['summary_lengths'] < 19]\n",
    "eval_df = eval_df[eval_df['summary_lengths'] < 19]\n",
    "\n",
    "train_df.reset_index(drop=True, inplace=True)\n",
    "eval_df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "max_article_length = max(max(train_df['article_lengths']), max(eval_df['article_lengths']))\n",
    "max_summary_length = max(max(train_df['summary_lengths']), max(eval_df['summary_lengths']))\n",
    "\n",
    "print(f\"The longest string (in words) in this dataset is {max_article_length} words.\")\n",
    "print(f\"The longest string (in words) in this dataset is {max_summary_length} words.\")\n",
    "\n",
    "# drop unnecessary features\n",
    "train_df = train_df.drop(columns=['article_lengths', 'summary_lengths'])\n",
    "eval_df = eval_df.drop(columns=['article_lengths', 'summary_lengths'])\n",
    "\n",
    "train_df.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Determine Longest Token Lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The longest article length is 153 tokens.\n",
      "The longest summary length is 50 tokens.\n"
     ]
    }
   ],
   "source": [
    "MODEL_CKPT = \"google/flan-t5-base\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_CKPT, \n",
    "                                             truncation=True, \n",
    "                                             padding=True,\n",
    "                                             batched=True, \n",
    "                                             batch_size=None)\n",
    "\n",
    "train_a_len = [len(tokenizer.encode(a)) for a in train_df[\"article\"]]\n",
    "train_s_len = [len(tokenizer.encode(a)) for a in train_df[\"summary\"]]\n",
    "\n",
    "eval_a_len = [len(tokenizer.encode(a)) for a in eval_df[\"article\"]]\n",
    "eval_s_len = [len(tokenizer.encode(a)) for a in eval_df[\"summary\"]]\n",
    "\n",
    "max_art_len = max(max(train_a_len), max(eval_a_len))\n",
    "max_sum_len = max(max(train_s_len), max(eval_s_len))\n",
    "\n",
    "print(f\"The longest article length is {max_art_len} tokens.\")\n",
    "print(f\"The longest summary length is {max_sum_len} tokens.\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert DataFrame to Dataset & Split into Train & Eval Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Shape: (19150, 2)\n",
      "Validation Data Shape: (945, 2)\n"
     ]
    }
   ],
   "source": [
    "# Convert pandas dataframe to a dataset\n",
    "train_ds = Dataset.from_pandas(train_df)\n",
    "eval_ds = Dataset.from_pandas(eval_df)\n",
    "\n",
    "# Combine the train/test/valid into one datasetdict\n",
    "dataset = DatasetDict({\n",
    "    'train' : train_ds,\n",
    "    'eval' : eval_ds\n",
    "})\n",
    "\n",
    "print('Training Data Shape:', dataset['train'].shape)\n",
    "print('Validation Data Shape:', dataset['eval'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Basic Constants/Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_CKPT = \"google/flan-t5-base\"\n",
    "MODEL_NAME = MODEL_CKPT.split(\"/\")[-1] + \"-text_summarization_data\"\n",
    "\n",
    "DEVICE = torch.device(\"cpu\")\n",
    "set_seed(42)\n",
    "\n",
    "BATCH_SIZE = 16\n",
    "NUM_OF_EPOCHS = 1\n",
    "\n",
    "LEARNING_RATE = 2e-5\n",
    "STRATEGY = \"epoch\"\n",
    "\n",
    "MAX_INPUT_LENGTH = max_art_len + 3\n",
    "MAX_TARGET_LENGTH = max_sum_len + 3\n",
    "\n",
    "WEIGHT_DECAY = 0.01\n",
    "REPORTS_TO = \"tensorboard\"\n",
    "\n",
    "PREFIX = \"summarize: \""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate both Tokenizer and Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_CKPT, \n",
    "                                             truncation=True, \n",
    "                                             padding=True, \n",
    "                                             batched=True, \n",
    "                                             batch_size=None)\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_CKPT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Function to Tokenize & Return Necessary Inputs for Batches of Samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_examples_to_features(ex_batch, padding=\"max_length\"):\n",
    "    inputs = [PREFIX + doc for doc in ex_batch[\"article\"]]\n",
    "    \n",
    "    input_encodings = tokenizer(inputs, \n",
    "                                max_length=MAX_INPUT_LENGTH, \n",
    "                                padding=\"max_length\",\n",
    "                                truncation=True)\n",
    "    \n",
    "    with tokenizer.as_target_tokenizer():\n",
    "        target_encodings = tokenizer(ex_batch['summary'], \n",
    "                                     max_length=MAX_TARGET_LENGTH, \n",
    "                                     truncation=True,\n",
    "                                     padding=\"max_length\")\n",
    "    \n",
    "    if padding == \"max_length\":\n",
    "        target_encodings[\"input_ids\"] = [\n",
    "            [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in target_encodings[\"input_ids\"]\n",
    "        ]\n",
    "    \n",
    "    return {\"input_ids\" : input_encodings[\"input_ids\"],\n",
    "            \"attention_mask\" : input_encodings[\"attention_mask\"],\n",
    "            \"labels\" : target_encodings[\"input_ids\"],}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Above Function to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ba5feb3f3c44a3883bb780eb72d7d9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages/transformers/tokenization_utils_base.py:3581: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18c3eb46ba894d43ac265aafaa6e3dd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "encoded_ds = dataset.map(convert_examples_to_features, batched=True)\n",
    "encoded_ds.set_format(type=\"torch\", columns=['input_ids', 'labels', 'attention_mask'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=MODEL_NAME,\n",
    "    num_train_epochs=NUM_OF_EPOCHS,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    per_device_train_batch_size=BATCH_SIZE,\n",
    "    per_device_eval_batch_size=BATCH_SIZE,\n",
    "    weight_decay=WEIGHT_DECAY,\n",
    "    logging_strategy=STRATEGY,\n",
    "    evaluation_strategy=STRATEGY,\n",
    "    logging_first_step=True,\n",
    "    disable_tqdm=False,\n",
    "    report_to=REPORTS_TO,\n",
    "    predict_with_generate=True,\n",
    "    push_to_hub=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Function to Computes Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rouge_metric = evaluate.load(\"rouge\")\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
    "    # Replace -100 in the labels as we cannot decode them.\n",
    "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "    \n",
    "    # Rouge expects a newline after each sentence\n",
    "    decoded_preds = [\"\\n\".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]\n",
    "    decoded_labels = [\"\\n\".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]\n",
    "    \n",
    "    result = rouge_metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)\n",
    "    # Extract a few results\n",
    "    result = {key: value * 100 for key, value in result.items()}\n",
    "    \n",
    "    # Add mean generated length\n",
    "    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]\n",
    "    result[\"gen_len\"] = np.mean(prediction_lens)\n",
    "    \n",
    "    return {k: round(v, 4) for k, v in result.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Data Collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/flan-t5-base-text_summarization_data into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = Seq2SeqTrainer(model=model, \n",
    "                  args=training_args,\n",
    "                  data_collator=seq2seq_data_collator,\n",
    "                  train_dataset=encoded_ds['train'], \n",
    "                  eval_dataset=encoded_ds['eval'], \n",
    "                  tokenizer=tokenizer,\n",
    "                  compute_metrics=compute_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: article, summary. If article, summary are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "/Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 19150\n",
      "  Num Epochs = 1\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 1197\n",
      "  Number of trainable parameters = 247577856\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ae1b68dfdf34806a7c355216ce1b611",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a T5TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 1.988, 'learning_rate': 1.998329156223893e-05, 'epoch': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to flan-t5-base-text_summarization_data/checkpoint-500\n",
      "Configuration saved in flan-t5-base-text_summarization_data/checkpoint-500/config.json\n",
      "Configuration saved in flan-t5-base-text_summarization_data/checkpoint-500/generation_config.json\n",
      "Model weights saved in flan-t5-base-text_summarization_data/checkpoint-500/pytorch_model.bin\n",
      "tokenizer config file saved in flan-t5-base-text_summarization_data/checkpoint-500/tokenizer_config.json\n",
      "Special tokens file saved in flan-t5-base-text_summarization_data/checkpoint-500/special_tokens_map.json\n",
      "Copy vocab file to flan-t5-base-text_summarization_data/checkpoint-500/spiece.model\n",
      "tokenizer config file saved in flan-t5-base-text_summarization_data/tokenizer_config.json\n",
      "Special tokens file saved in flan-t5-base-text_summarization_data/special_tokens_map.json\n",
      "Copy vocab file to flan-t5-base-text_summarization_data/spiece.model\n",
      "Saving model checkpoint to flan-t5-base-text_summarization_data/checkpoint-1000\n",
      "Configuration saved in flan-t5-base-text_summarization_data/checkpoint-1000/config.json\n",
      "Configuration saved in flan-t5-base-text_summarization_data/checkpoint-1000/generation_config.json\n",
      "Model weights saved in flan-t5-base-text_summarization_data/checkpoint-1000/pytorch_model.bin\n",
      "tokenizer config file saved in flan-t5-base-text_summarization_data/checkpoint-1000/tokenizer_config.json\n",
      "Special tokens file saved in flan-t5-base-text_summarization_data/checkpoint-1000/special_tokens_map.json\n",
      "Copy vocab file to flan-t5-base-text_summarization_data/checkpoint-1000/spiece.model\n",
      "tokenizer config file saved in flan-t5-base-text_summarization_data/tokenizer_config.json\n",
      "Special tokens file saved in flan-t5-base-text_summarization_data/special_tokens_map.json\n",
      "Copy vocab file to flan-t5-base-text_summarization_data/spiece.model\n",
      "The following columns in the evaluation set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: article, summary. If article, summary are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 945\n",
      "  Batch size = 16\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.0287, 'learning_rate': 0.0, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de8f5a8c6ba140f5ade7f2f57ab691c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.7385940551757812, 'eval_rouge1': 43.6615, 'eval_rouge2': 20.349, 'eval_rougeL': 40.1032, 'eval_rougeLsum': 40.1589, 'eval_gen_len': 14.6434, 'eval_runtime': 2080.5615, 'eval_samples_per_second': 0.454, 'eval_steps_per_second': 0.029, 'epoch': 1.0}\n",
      "{'train_runtime': 74941.3155, 'train_samples_per_second': 0.256, 'train_steps_per_second': 0.016, 'train_loss': 2.028625026283009, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1197, training_loss=2.028625026283009, metrics={'train_runtime': 74941.3155, 'train_samples_per_second': 0.256, 'train_steps_per_second': 0.016, 'train_loss': 2.028625026283009, 'epoch': 1.0})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compute & Print Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: article, summary. If article, summary are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 945\n",
      "  Batch size = 16\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "157535b375b1403d99bd7d1fb5f0c738",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Generate config GenerationConfig {\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "score = trainer.evaluate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print Out Full Rouge Metric Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eval_loss</th>\n",
       "      <th>eval_rouge1</th>\n",
       "      <th>eval_rouge2</th>\n",
       "      <th>eval_rougeL</th>\n",
       "      <th>eval_rougeLsum</th>\n",
       "      <th>eval_gen_len</th>\n",
       "      <th>eval_runtime</th>\n",
       "      <th>eval_samples_per_second</th>\n",
       "      <th>eval_steps_per_second</th>\n",
       "      <th>epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>FLAN-T5</th>\n",
       "      <td>1.738594</td>\n",
       "      <td>43.6615</td>\n",
       "      <td>20.349</td>\n",
       "      <td>40.1032</td>\n",
       "      <td>40.1589</td>\n",
       "      <td>14.6434</td>\n",
       "      <td>2104.9768</td>\n",
       "      <td>0.449</td>\n",
       "      <td>0.029</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         eval_loss  eval_rouge1  eval_rouge2  eval_rougeL  eval_rougeLsum  \\\n",
       "FLAN-T5   1.738594      43.6615       20.349      40.1032         40.1589   \n",
       "\n",
       "         eval_gen_len  eval_runtime  eval_samples_per_second  \\\n",
       "FLAN-T5       14.6434     2104.9768                    0.449   \n",
       "\n",
       "         eval_steps_per_second  epoch  \n",
       "FLAN-T5                  0.029    1.0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(score, index=[f\"FLAN-T5\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Push Fine-Tuned Model to HuggingFace Hub (My Profile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to flan-t5-base-text_summarization_data\n",
      "Configuration saved in flan-t5-base-text_summarization_data/config.json\n",
      "Configuration saved in flan-t5-base-text_summarization_data/generation_config.json\n",
      "Model weights saved in flan-t5-base-text_summarization_data/pytorch_model.bin\n",
      "tokenizer config file saved in flan-t5-base-text_summarization_data/tokenizer_config.json\n",
      "Special tokens file saved in flan-t5-base-text_summarization_data/special_tokens_map.json\n",
      "Copy vocab file to flan-t5-base-text_summarization_data/spiece.model\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67166f63ee6a4d21970c6666927c1e6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aecf9866dcda4439a886302d28213dc8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90778c80c7e7486ba697b718c6ec5d7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "remote: Scanning LFS files of refs/heads/main for validity...        \n",
      "remote: LFS file scan complete.        \n",
      "To https://huggingface.co/DunnBC22/flan-t5-base-text_summarization_data\n",
      "   efba34a..9cb4fde  main -> main\n",
      "\n",
      "WARNING:huggingface_hub.repository:remote: Scanning LFS files of refs/heads/main for validity...        \n",
      "remote: LFS file scan complete.        \n",
      "To https://huggingface.co/DunnBC22/flan-t5-base-text_summarization_data\n",
      "   efba34a..9cb4fde  main -> main\n",
      "\n",
      "Dropping the following result as it does not have all the necessary fields:\n",
      "{'task': {'name': 'Sequence-to-sequence Language Modeling', 'type': 'text2text-generation'}, 'metrics': [{'name': 'Rouge1', 'type': 'rouge', 'value': 43.6615}]}\n",
      "To https://huggingface.co/DunnBC22/flan-t5-base-text_summarization_data\n",
      "   9cb4fde..3684006  main -> main\n",
      "\n",
      "WARNING:huggingface_hub.repository:To https://huggingface.co/DunnBC22/flan-t5-base-text_summarization_data\n",
      "   9cb4fde..3684006  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/flan-t5-base-text_summarization_data/commit/9cb4fde1e0096a4b77726c9cadeaefb855a4a78f'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub(commit_message=\"All Dunn!\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notes & Other Takeaways From This Project\n",
    "****\n",
    "- Because of how well the model worked with tests on small sample sizes, I thought I could get away with only training for a single epoch. While there are no results for this project using multiple epochs, I am confident that the results will improve.\n",
    "****"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Citation\n",
    "- Model Checkpoint\n",
    "    - @misc{https://doi.org/10.48550/arxiv.2210.11416,\n",
    "  doi = {10.48550/ARXIV.2210.11416},\n",
    "  url = {https://arxiv.org/abs/2210.11416},\n",
    "  author = {Chung, Hyung Won and Hou, Le and Longpre, Shayne and Zoph, Barret and Tay, Yi and Fedus, William and Li, Eric and Wang, Xuezhi and Dehghani, Mostafa and Brahma, Siddhartha and Webson, Albert and Gu, Shixiang Shane and Dai, Zhuyun and Suzgun, Mirac and Chen, Xinyun and Chowdhery, Aakanksha and Narang, Sharan and Mishra, Gaurav and Yu, Adams and Zhao, Vincent and Huang, Yanping and Dai, Andrew and Yu, Hongkun and Petrov, Slav and Chi, Ed H. and Dean, Jeff and Devlin, Jacob and Roberts, Adam and Zhou, Denny and Le, Quoc V. and Wei, Jason},\n",
    "  keywords = {Machine Learning (cs.LG), Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences},\n",
    "  title = {Scaling Instruction-Finetuned Language Models},\n",
    "  publisher = {arXiv},\n",
    "  year = {2022},\n",
    "  copyright = {Creative Commons Attribution 4.0 International}\n",
    "}\n",
    "****\n",
    "- Rouge Metric Citation\n",
    "    - @inproceedings{lin-2004-rouge,\n",
    "            title = \"{ROUGE}: A Package for Automatic Evaluation of Summaries\",\n",
    "            author = \"Lin, Chin-Yew\",\n",
    "            booktitle = \"Text Summarization Branches Out\",\n",
    "            month = jul,\n",
    "            year = \"2004\",\n",
    "            address = \"Barcelona, Spain\",\n",
    "            publisher = \"Association for Computational Linguistics\",\n",
    "            url = \"https://www.aclweb.org/anthology/W04-1013\",\n",
    "            pages = \"74--81\",\n",
    "        }"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpcv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0f2f65fb703e64d564cd9efcad5c1927cf411d56f2690ec4a46f5d87d6a6451f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}