Causal LM for Chatbot/Company FAQs/v1/FAQs - CLM for DialoGPT Chatbot - Medium.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FAQs: Causal Language Modeling\n",
    "\n",
    "Dataset Source: FAQs From Large Transportation Company"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Necessary Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "import os, sys, random, math\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import datasets\n",
    "from datasets import Dataset\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForLanguageModeling\n",
    "from transformers import TrainingArguments, Trainer, set_seed\n",
    "\n",
    "!git lfs install"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display Library Versions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Python: 3.9.12\n",
      "        Pandas: 1.5.0\n",
      "      Datasets: 2.9.0\n",
      "  Transformers: 4.26.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Python:\".rjust(15), sys.version[0:6])\n",
    "print(\"Pandas:\".rjust(15), pd.__version__)\n",
    "print(\"Datasets:\".rjust(15), datasets.__version__)\n",
    "print(\"Transformers:\".rjust(15), transformers.__version__)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 45 entries, 0 to 44\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   Question   45 non-null     object\n",
      " 1   Answer     45 non-null     object\n",
      " 2   input      45 non-null     object\n",
      " 3   input_len  45 non-null     int64 \n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 1.5+ KB\n"
     ]
    }
   ],
   "source": [
    "data_file = \"Company FAQs.csv\"\n",
    "\n",
    "data = pd.read_csv(data_file, usecols=[0,1])\n",
    "\n",
    "data = data.rename(columns={\"question\": \"Question\"})\n",
    "\n",
    "data['input'] = \"Question: \" + data['Question'] + \" Answer: \" + data['Answer']\n",
    "\n",
    "data['input_len'] = data['input'].str.len()\n",
    "\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visual: Histogram of Input Lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximum length of this dataset is approximately 5323 characters.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABZkAAAK9CAYAAABGqWLxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/nklEQVR4nO3deZxVdf348fewDeuwKDAg+yIkBiYo8lVZFEUis+2bGhGSZiYupKHyMwXUBLXSFkJNkxYT00JLBSVlKREUEEU0RMIkN3KBYdEBmfP7owf36xVQ5iMDM/h8Ph73Iefcz73nc+7M54G+PI9zC7IsywIAAAAAABJU29sTAAAAAACg6hKZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAPZx7dq1i9NOO21vT2Ofd91110WHDh2ievXqccghh+zt6XziFBQUxDnnnLO3pwEA8IkkMgMAVCFTpkyJgoKCWLhw4Q6f79+/fxx88MEf+zgPPPBAjBs37mO/zyfFQw89FBdddFEceeSRcdttt8XVV1+93ZjZs2dHQUHBLj12l3nz5sW4ceNi7dq1uzT+tNNOi/r16++24+9u5T0fAAD2jBp7ewIAAFSs5cuXR7Vq5bu24IEHHohJkyYJzbvokUceiWrVqsWtt94atWrV2uGYT33qU/Hb3/42b9+YMWOifv36cemll1bIvObNmxfjx4+P0047LRo1alQhx9iT9rXzAQDYV4jMAAD7uMLCwr09hXLbuHFj1KtXb29PY5etWbMm6tSps9PAHBHRvHnz+PrXv563b+LEibH//vtvtx8AAKoSt8sAANjHffCezFu2bInx48dH586do3bt2rHffvvFUUcdFTNnzoyI/94yYdKkSRERO7yFw8aNG+PCCy+M1q1bR2FhYXTp0iV++MMfRpZlecd955134rzzzov9998/GjRoEJ///Ofj5ZdfjoKCgrwrpMeNGxcFBQXx7LPPxte+9rVo3LhxHHXUURER8fTTT8dpp50WHTp0iNq1a0dxcXF885vfjDfffDPvWNve4/nnn4+vf/3r0bBhw2jatGlcdtllkWVZrF69Ok466aQoKiqK4uLi+NGPfrRLn917770XV155ZXTs2DEKCwujXbt28f/+3/+L0tLS3JiCgoK47bbbYuPGjbnPasqUKbv0/juydu3aGDVqVO7z7dSpU1xzzTVRVlYWERFZlsWAAQOiadOmsWbNmtzrNm/eHJ/+9KejY8eOsXHjxhg3blyMHj06IiLat2+fm9uLL76YPLdtFixYECeccEI0bNgw6tatG/369YtHH300b8y2n8kLL7yQu/K4YcOGMWLEiNi0aVPe2F35XdnV87nnnnvi4IMPjsLCwujWrVvMmDHjY58vAAAfzpXMAABV0Lp16+KNN97Ybv+WLVs+8rXjxo2LCRMmxBlnnBGHH354lJSUxMKFC2Px4sVx3HHHxbe//e145ZVXYubMmdvd3iHLsvj85z8fs2bNitNPPz0OOeSQePDBB2P06NHx8ssvx/XXX58be9ppp8Uf/vCHGDZsWBxxxBExZ86cGDJkyE7n9b//+7/RuXPnuPrqq3PBeubMmfHPf/4zRowYEcXFxbFs2bK4+eabY9myZTF//vzt7l988sknx6c+9amYOHFi3H///XHVVVdFkyZN4qabbopjjjkmrrnmmrj99tvje9/7Xhx22GHRt2/fD/2szjjjjPj1r38dX/nKV+LCCy+MBQsWxIQJE+K5556LadOmRUTEb3/727j55pvj8ccfj1tuuSUiIv7nf/7nI38OO7Jp06bo169fvPzyy/Htb3872rRpE/PmzYsxY8bEq6++GjfccEMUFBTEr371q+jevXucddZZ8ac//SkiIsaOHRvLli2L2bNnR7169eJLX/pSPP/883HHHXfE9ddfH/vvv39ERDRt2jRpbts88sgjMXjw4OjZs2eMHTs2qlWrFrfddlscc8wx8be//S0OP/zwvPFf/epXo3379jFhwoRYvHhx3HLLLdGsWbO45pprcmN25XdlV87n73//e/zpT3+Ks88+Oxo0aBA//elP48tf/nK89NJLsd9++32s8wYA4ENkAABUGbfddlsWER/66NatW95r2rZtmw0fPjy33aNHj2zIkCEfepyRI0dmO/pXxXvuuSeLiOyqq67K2/+Vr3wlKygoyF544YUsy7Js0aJFWURko0aNyht32mmnZRGRjR07Nrdv7NixWURkp5566nbH27Rp03b77rjjjiwisrlz5273HmeeeWZu33vvvZe1atUqKygoyCZOnJjb//bbb2d16tTJ+0x2ZMmSJVlEZGeccUbe/u9973tZRGSPPPJIbt/w4cOzevXqfej77Ui3bt2yfv365bavvPLKrF69etnzzz+fN+6SSy7Jqlevnr300ku5fTfddFMWEdnvfve7bP78+Vn16tW3+7yvu+66LCKyVatW7dJ8Puo8ysrKss6dO2eDBg3KysrKcvs3bdqUtW/fPjvuuONy+7b9TL75zW/mvccXv/jFbL/99sttl+d35cPOJyKyWrVq5X4HsyzLnnrqqSwisp/97Gcfee4AAKRzuwwAgCpo0qRJMXPmzO0e3bt3/8jXNmrUKJYtWxYrVqwo93EfeOCBqF69epx33nl5+y+88MLIsiymT58eEZG7RcHZZ5+dN+7cc8/d6XufddZZ2+2rU6dO7s/vvvtuvPHGG3HEEUdERMTixYu3G3/GGWfk/ly9evXo1atXZFkWp59+em5/o0aNokuXLvHPf/5zp3OJ+O+5RkRccMEFefsvvPDCiIi4//77P/T1Ke666644+uijo3HjxvHGG2/kHgMHDoytW7fG3Llzc2PPPPPMGDRoUJx77rkxbNiw6NixY1x99dW7fU7vt2TJklixYkV87WtfizfffDM3v40bN8axxx4bc+fOzd3WY5sP/lyPPvroePPNN6OkpCQi0n5XdmbgwIHRsWPH3Hb37t2jqKjoI3/WAAB8PG6XAQBQBR1++OHRq1ev7fZvi5Mf5oorroiTTjopDjzwwDj44IPjhBNOiGHDhu1SoP7Xv/4VLVu2jAYNGuTt/9SnPpV7fts/q1WrFu3bt88b16lTp52+9wfHRkS89dZbMX78+Jg6dWre/Ycj/nvLkA9q06ZN3nbDhg2jdu3auVsrvH//B+/r/EHbzuGDcy4uLo5GjRrlznV3WrFiRTz99NM7vaXFBz+DW2+9NTp27BgrVqyIefPm5UX5irDtf0wMHz58p2PWrVsXjRs3zm1/8Gey7bm33347ioqKkn5XduaDx9p2vLfffrvc7wUAwK4TmQEAPmH69u0bK1eujHvvvTceeuihuOWWW+L666+PG2+8Me9K4D1tR4H0q1/9asybNy9Gjx4dhxxySNSvXz/KysrihBNO2O6K2Yj/Xr28K/siYrsvKtyZD973uSKVlZXFcccdFxdddNEOnz/wwAPztmfPnp37EsKlS5dGnz59Knx+ERHXXXddHHLIITscU79+/bztj/v5l8eePBYAAP9HZAYA+ARq0qRJjBgxIkaMGBEbNmyIvn37xrhx43KReWdhtW3btvHXv/411q9fn3c18z/+8Y/c89v+WVZWFqtWrYrOnTvnxr3wwgu7PMe33347Hn744Rg/fnxcfvnluf0pt/lIse0cVqxYkbtSOyLi9ddfj7Vr1+bOdXfq2LFjbNiwIQYOHPiRY1999dU499xz4/jjj49atWrF9773vRg0aFDevHZ3IN92K4qioqJdmuOuKM/vyp4M/gAA7Dr3ZAYA+IT54G0i6tevH506dcpdERsRUa9evYiIWLt2bd7Yz372s7F169b4+c9/nrf/+uuvj4KCghg8eHBERAwaNCgiIn7xi1/kjfvZz362y/PcdlXqB69CveGGG3b5PT6Oz372szs83o9//OOIiBgyZMhuP+ZXv/rVeOyxx+LBBx/c7rm1a9fGe++9l9v+1re+FWVlZXHrrbfGzTffHDVq1IjTTz897/Pa2c8xVc+ePaNjx47xwx/+MDZs2LDd8//5z3/K/Z7l+V3Z3ecDAMDu4UpmAIBPmIMOOij69+8fPXv2jCZNmsTChQvj7rvvjnPOOSc3pmfPnhERcd5558WgQYOievXqccopp8SJJ54YAwYMiEsvvTRefPHF6NGjRzz00ENx7733xqhRo3JXuvbs2TO+/OUvxw033BBvvvlmHHHEETFnzpx4/vnnI2LXrkgtKiqKvn37xrXXXhtbtmyJAw44IB566KFYtWpVBXwq2+vRo0cMHz48br755li7dm3069cvHn/88fj1r38dX/jCF2LAgAG7/ZijR4+OP//5z/G5z30uTjvttOjZs2ds3Lgxli5dGnfffXe8+OKLsf/++8dtt90W999/f0yZMiVatWoVEf+Nsl//+tdj8uTJuS/R2/ZzvPTSS+OUU06JmjVrxoknnpiLtTuyZcuWuOqqq7bb36RJkzj77LPjlltuicGDB0e3bt1ixIgRccABB8TLL78cs2bNiqKiovjLX/5SrnMuz+9KyvkAAFDxRGYAgE+Y8847L/785z/HQw89FKWlpdG2bdu46qqrYvTo0bkxX/rSl+Lcc8+NqVOnxu9+97vIsixOOeWUqFatWvz5z3+Oyy+/PO6888647bbbol27dnHdddfFhRdemHec3/zmN1FcXBx33HFHTJs2LQYOHBh33nlndOnSJWrXrr1Lc/39738f5557bkyaNCmyLIvjjz8+pk+fHi1bttytn8nO3HLLLdGhQ4eYMmVKTJs2LYqLi2PMmDExduzYCjle3bp1Y86cOXH11VfHXXfdFb/5zW+iqKgoDjzwwBg/fnw0bNgw/v3vf8d3v/vdOPHEE/O+gG/o0KHxxz/+MS666KIYPHhwtG/fPg477LC48sor48Ybb4wZM2bkbkvxYVF28+bNcdlll223v2PHjnH22WdH//7947HHHosrr7wyfv7zn8eGDRuiuLg4evfuHd/+9reTzntXf1dSzgcAgIpXkPkWDAAA9pAlS5bEZz7zmfjd734XQ4cO3dvToRLzuwIAUHW4JzMAABXinXfe2W7fDTfcENWqVYu+ffvuhRlRWfldAQCo2twuAwCACnHttdfGokWLYsCAAVGjRo2YPn16TJ8+Pc4888xo3br13p4elYjfFQCAqs3tMgAAqBAzZ86M8ePHx7PPPhsbNmyINm3axLBhw+LSSy+NGjVc68D/8bsCAFC1icwAAAAAACRzT2YAAAAAAJKJzAAAAAAAJNvjNzgrKyuLV155JRo0aBAFBQV7+vAAAAAAAOyCLMti/fr10bJly6hWbefXK+/xyPzKK6/4hmgAAAAAgCpi9erV0apVq50+v8cjc4MGDSLivxMrKira04cHAAAAAGAXlJSUROvWrXNNd2f2eGTedouMoqIikRkAAAAAoJL7qNse++I/AAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAECyckXmcePGRUFBQd6ja9euFTU3AAAAAAAquRrlfUG3bt3ir3/96/+9QY1yvwUAAAAAAPuIchfiGjVqRHFxcUXMBQAAAACAKqbc92ResWJFtGzZMjp06BBDhw6Nl1566UPHl5aWRklJSd4DAAAAAIB9Q7kic+/evWPKlCkxY8aMmDx5cqxatSqOPvroWL9+/U5fM2HChGjYsGHu0bp16489aQAAAAAAKoeCLMuy1BevXbs22rZtGz/+8Y/j9NNP3+GY0tLSKC0tzW2XlJRE69atY926dVFUVJR6aAAAAAAAKlBJSUk0bNjwI1vux/rWvkaNGsWBBx4YL7zwwk7HFBYWRmFh4cc5DAAAAAAAlVS578n8fhs2bIiVK1dGixYtdtd8AAAAAACoQsoVmb/3ve/FnDlz4sUXX4x58+bFF7/4xahevXqceuqpFTU/AAAAAAAqsXLdLuPf//53nHrqqfHmm29G06ZN46ijjor58+dH06ZNK2p+AAAAAABUYuWKzFOnTq2oeQAAAAAAUAV9rHsyAwAAAADwySYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJDsY0XmiRMnRkFBQYwaNWo3TQcAAAAAgKokOTI/8cQTcdNNN0X37t1353wAAAAAAKhCkiLzhg0bYujQofHLX/4yGjduvLvnBAAAAABAFZEUmUeOHBlDhgyJgQMHfuTY0tLSKCkpyXsAAAAAALBvKHdknjp1aixevDgmTJiwS+MnTJgQDRs2zD1at25d7knC7tDukvv39hQAAAAAYJ9Trsi8evXqOP/88+P222+P2rVr79JrxowZE+vWrcs9Vq9enTRRAAAAAAAqnxrlGbxo0aJYs2ZNHHroobl9W7dujblz58bPf/7zKC0tjerVq+e9prCwMAoLC3fPbAEAAAAAqFTKFZmPPfbYWLp0ad6+ESNGRNeuXePiiy/eLjADAAAAALBvK1dkbtCgQRx88MF5++rVqxf77bffdvsBAAAAANj3lfuL/wAAAAAAYJtyXcm8I7Nnz94N0wAAAAAAoCpyJTMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQrV2SePHlydO/ePYqKiqKoqCj69OkT06dPr6i5AQAAAABQyZUrMrdq1SomTpwYixYtioULF8YxxxwTJ510Uixbtqyi5gcAAAAAQCVWozyDTzzxxLztH/zgBzF58uSYP39+dOvWbbdODAAAAACAyq9ckfn9tm7dGnfddVds3Lgx+vTps9NxpaWlUVpamtsuKSlJPSQAAAAAAJVMub/4b+nSpVG/fv0oLCyMs846K6ZNmxYHHXTQTsdPmDAhGjZsmHu0bt36Y00YAAAAAIDKo9yRuUuXLrFkyZJYsGBBfOc734nhw4fHs88+u9PxY8aMiXXr1uUeq1ev/lgTBgAAAACg8ij37TJq1aoVnTp1ioiInj17xhNPPBE/+clP4qabbtrh+MLCwigsLPx4swQAAAAAoFIq95XMH1RWVpZ3z2UAAAAAAD45ynUl85gxY2Lw4MHRpk2bWL9+ffz+97+P2bNnx4MPPlhR8wMAAAAAoBIrV2Res2ZNfOMb34hXX301GjZsGN27d48HH3wwjjvuuIqaHwAAAAAAlVi5IvOtt95aUfMAAAAAAKAK+tj3ZAYAAAAA4JNLZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgWbki84QJE+Kwww6LBg0aRLNmzeILX/hCLF++vKLmBgAAAABAJVeuyDxnzpwYOXJkzJ8/P2bOnBlbtmyJ448/PjZu3FhR8wMAAAAAoBKrUZ7BM2bMyNueMmVKNGvWLBYtWhR9+/bdrRMDAAAAAKDyK1dk/qB169ZFRESTJk12Oqa0tDRKS0tz2yUlJR/nkAAAAAAAVCLJX/xXVlYWo0aNiiOPPDIOPvjgnY6bMGFCNGzYMPdo3bp16iEBAAAAAKhkkiPzyJEj45lnnompU6d+6LgxY8bEunXrco/Vq1enHhIAAAAAgEom6XYZ55xzTtx3330xd+7caNWq1YeOLSwsjMLCwqTJAQAAAABQuZUrMmdZFueee25MmzYtZs+eHe3bt6+oeQEAAAAAUAWUKzKPHDkyfv/738e9994bDRo0iNdeey0iIho2bBh16tSpkAkCAAAAAFB5leuezJMnT45169ZF//79o0WLFrnHnXfeWVHzAwAAAACgEiv37TIAAAAAAGCbcl3JDAAAAAAA7ycyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJCs3JF57ty5ceKJJ0bLli2joKAg7rnnngqYFgAAAAAAVUG5I/PGjRujR48eMWnSpIqYDwAAAAAAVUiN8r5g8ODBMXjw4IqYCwAAAAAAVUy5I3N5lZaWRmlpaW67pKSkog8JAAAAAMAeUuFf/DdhwoRo2LBh7tG6deuKPiSVVLtL7q/0xyrv6/bkOe1te+ozperwswUAAGBf5799d02FR+YxY8bEunXrco/Vq1dX9CEBAAAAANhDKvx2GYWFhVFYWFjRhwEAAAAAYC+o8CuZAQAAAADYd5X7SuYNGzbECy+8kNtetWpVLFmyJJo0aRJt2rTZrZMDAAAAAKByK3dkXrhwYQwYMCC3fcEFF0RExPDhw2PKlCm7bWIAAAAAAFR+5Y7M/fv3jyzLKmIuAAAAAABUMe7JDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyZIi86RJk6Jdu3ZRu3bt6N27dzz++OO7e14AAAAAAFQB5Y7Md955Z1xwwQUxduzYWLx4cfTo0SMGDRoUa9asqYj5AQAAAABQiZU7Mv/4xz+Ob33rWzFixIg46KCD4sYbb4y6devGr371q4qYHwAAAAAAlViN8gzevHlzLFq0KMaMGZPbV61atRg4cGA89thjO3xNaWlplJaW5rbXrVsXERElJSUp86UKKyvdtMd+7js61q4cv7xz3JPntLelnusn6TP6pPGzBQAAYF/3Sf9v323nnmXZh44ryD5qxPu88sorccABB8S8efOiT58+uf0XXXRRzJkzJxYsWLDda8aNGxfjx4/f1UMAAAAAAFCJrF69Olq1arXT58t1JXOKMWPGxAUXXJDbLisri7feeiv222+/KCgoqOjDVyolJSXRunXrWL16dRQVFe3t6QC7wLqFqse6harJ2oWqx7qFqse6pbyyLIv169dHy5YtP3RcuSLz/vvvH9WrV4/XX389b//rr78excXFO3xNYWFhFBYW5u1r1KhReQ67zykqKrKQoYqxbqHqsW6harJ2oeqxbqHqsW4pj4YNG37kmHJ98V+tWrWiZ8+e8fDDD+f2lZWVxcMPP5x3+wwAAAAAAD4Zyn27jAsuuCCGDx8evXr1isMPPzxuuOGG2LhxY4wYMaIi5gcAAAAAQCVW7sh88sknx3/+85+4/PLL47XXXotDDjkkZsyYEc2bN6+I+e1TCgsLY+zYsdvdPgSovKxbqHqsW6iarF2oeqxbqHqsWypKQZZl2d6eBAAAAAAAVVO57skMAAAAAADvJzIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpF5D5o0aVK0a9cuateuHb17947HH398b08JPhHmzp0bJ554YrRs2TIKCgrinnvuyXs+y7K4/PLLo0WLFlGnTp0YOHBgrFixIm/MW2+9FUOHDo2ioqJo1KhRnH766bFhw4a8MU8//XQcffTRUbt27WjdunVce+21FX1qsM+aMGFCHHbYYdGgQYNo1qxZfOELX4jly5fnjXn33Xdj5MiRsd9++0X9+vXjy1/+crz++ut5Y1566aUYMmRI1K1bN5o1axajR4+O9957L2/M7Nmz49BDD43CwsLo1KlTTJkypaJPD/ZJkydPju7du0dRUVEUFRVFnz59Yvr06bnnrVmo/CZOnBgFBQUxatSo3D5rFyqfcePGRUFBQd6ja9euueetW/YGkXkPufPOO+OCCy6IsWPHxuLFi6NHjx4xaNCgWLNmzd6eGuzzNm7cGD169IhJkybt8Plrr702fvrTn8aNN94YCxYsiHr16sWgQYPi3XffzY0ZOnRoLFu2LGbOnBn33XdfzJ07N84888zc8yUlJXH88cdH27ZtY9GiRXHdddfFuHHj4uabb67w84N90Zw5c2LkyJExf/78mDlzZmzZsiWOP/742LhxY27Md7/73fjLX/4Sd911V8yZMydeeeWV+NKXvpR7fuvWrTFkyJDYvHlzzJs3L37961/HlClT4vLLL8+NWbVqVQwZMiQGDBgQS5YsiVGjRsUZZ5wRDz744B49X9gXtGrVKiZOnBiLFi2KhQsXxjHHHBMnnXRSLFu2LCKsWajsnnjiibjpppuie/fuefutXaicunXrFq+++mru8fe//z33nHXLXpGxRxx++OHZyJEjc9tbt27NWrZsmU2YMGEvzgo+eSIimzZtWm67rKwsKy4uzq677rrcvrVr12aFhYXZHXfckWVZlj377LNZRGRPPPFEbsz06dOzgoKC7OWXX86yLMt+8YtfZI0bN85KS0tzYy6++OKsS5cuFXxG8MmwZs2aLCKyOXPmZFn233Vas2bN7K677sqNee6557KIyB577LEsy7LsgQceyKpVq5a99tpruTGTJ0/OioqKcmv1oosuyrp165Z3rJNPPjkbNGhQRZ8SfCI0btw4u+WWW6xZqOTWr1+fde7cOZs5c2bWr1+/7Pzzz8+yzN+3UFmNHTs269Gjxw6fs27ZW1zJvAds3rw5Fi1aFAMHDsztq1atWgwcODAee+yxvTgzYNWqVfHaa6/lrc+GDRtG7969c+vzsccei0aNGkWvXr1yYwYOHBjVqlWLBQsW5Mb07ds3atWqlRszaNCgWL58ebz99tt76Gxg37Vu3bqIiGjSpElERCxatCi2bNmSt3a7du0abdq0yVu7n/70p6N58+a5MYMGDYqSkpLclZWPPfZY3ntsG+PvZ/h4tm7dGlOnTo2NGzdGnz59rFmo5EaOHBlDhgzZbn1Zu1B5rVixIlq2bBkdOnSIoUOHxksvvRQR1i17j8i8B7zxxhuxdevWvMUbEdG8efN47bXX9tKsgIjIrcEPW5+vvfZaNGvWLO/5GjVqRJMmTfLG7Og93n8MIE1ZWVmMGjUqjjzyyDj44IMj4r/rqlatWtGoUaO8sR9cux+1Lnc2pqSkJN55552KOB3Ypy1dujTq168fhYWFcdZZZ8W0adPioIMOsmahEps6dWosXrw4JkyYsN1z1i5UTr17944pU6bEjBkzYvLkybFq1ao4+uijY/369dYte02NvT0BAIAPM3LkyHjmmWfy7jMHVE5dunSJJUuWxLp16+Luu++O4cOHx5w5c/b2tICdWL16dZx//vkxc+bMqF279t6eDrCLBg8enPtz9+7do3fv3tG2bdv4wx/+EHXq1NmLM+OTzJXMe8D+++8f1atX3+6bPF9//fUoLi7eS7MCIiK3Bj9sfRYXF2/3JZ3vvfdevPXWW3ljdvQe7z8GUH7nnHNO3HfffTFr1qxo1apVbn9xcXFs3rw51q5dmzf+g2v3o9blzsYUFRX5F3RIUKtWrejUqVP07NkzJkyYED169Iif/OQn1ixUUosWLYo1a9bEoYceGjVq1IgaNWrEnDlz4qc//WnUqFEjmjdvbu1CFdCoUaM48MAD44UXXvB3LnuNyLwH1KpVK3r27BkPP/xwbl9ZWVk8/PDD0adPn704M6B9+/ZRXFyctz5LSkpiwYIFufXZp0+fWLt2bSxatCg35pFHHomysrLo3bt3bszcuXNjy5YtuTEzZ86MLl26ROPGjffQ2cC+I8uyOOecc2LatGnxyCOPRPv27fOe79mzZ9SsWTNv7S5fvjxeeumlvLW7dOnSvP9JNHPmzCgqKoqDDjooN+b977FtjL+fYfcoKyuL0tJSaxYqqWOPPTaWLl0aS5YsyT169eoVQ4cOzf3Z2oXKb8OGDbFy5cpo0aKFv3PZe/b2Nw9+UkydOjUrLCzMpkyZkj377LPZmWeemTVq1CjvmzyBirF+/frsySefzJ588sksIrIf//jH2ZNPPpn961//yrIsyyZOnJg1atQou/fee7Onn346O+mkk7L27dtn77zzTu49TjjhhOwzn/lMtmDBguzvf/971rlz5+zUU0/NPb927dqsefPm2bBhw7Jnnnkmmzp1ala3bt3spptu2uPnC/uC73znO1nDhg2z2bNnZ6+++mrusWnTptyYs846K2vTpk32yCOPZAsXLsz69OmT9enTJ/f8e++9lx188MHZ8ccfny1ZsiSbMWNG1rRp02zMmDG5Mf/85z+zunXrZqNHj86ee+65bNKkSVn16tWzGTNm7NHzhX3BJZdcks2ZMydbtWpV9vTTT2eXXHJJVlBQkD300ENZllmzUFX069cvO//883Pb1i5UPhdeeGE2e/bsbNWqVdmjjz6aDRw4MNt///2zNWvWZFlm3bJ3iMx70M9+9rOsTZs2Wa1atbLDDz88mz9//t6eEnwizJo1K4uI7R7Dhw/PsizLysrKsssuuyxr3rx5VlhYmB177LHZ8uXL897jzTffzE499dSsfv36WVFRUTZixIhs/fr1eWOeeuqp7KijjsoKCwuzAw44IJs4ceKeOkXY5+xozUZEdtttt+XGvPPOO9nZZ5+dNW7cOKtbt272xS9+MXv11Vfz3ufFF1/MBg8enNWpUyfbf//9swsvvDDbsmVL3phZs2ZlhxxySFarVq2sQ4cOeccAdt03v/nNrG3btlmtWrWypk2bZscee2wuMGeZNQtVxQcjs7ULlc/JJ5+ctWjRIqtVq1Z2wAEHZCeffHL2wgsv5J63btkbCrIsy/bONdQAAAAAAFR17skMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAwD6lf//+MWrUqL09jSrvxRdfjIKCgliyZMnengoAAJWcyAwAwG5z4403RoMGDeK9997L7duwYUPUrFkz+vfvnzd29uzZUVBQECtXrtzDs6w8Vq1aFV/72teiZcuWUbt27WjVqlWcdNJJ8Y9//GNvTw0AAHaZyAwAwG4zYMCA2LBhQyxcuDC3729/+1sUFxfHggUL4t13383tnzVrVrRp0yY6duxY7uNkWZYXsiu7LVu27HDfcccdF+vWrYs//elPsXz58rjzzjvj05/+dKxdu3bPTxIAABKJzAAA7DZdunSJFi1axOzZs3P7Zs+eHSeddFK0b98+5s+fn7d/wIABERFRWloa5513XjRr1ixq164dRx11VDzxxBN5YwsKCmL69OnRs2fPKCwsjL///e+xcePG+MY3vhH169ePFi1axI9+9KPt5vSLX/wiOnfuHLVr147mzZvHV77ylZ3Of8qUKdGoUaO45557cq8ZNGhQrF69Om/cvffeG4ceemjUrl07OnToEOPHj8+L3gUFBTF58uT4/Oc/H/Xq1Ysf/OAH2x1r2bJlsXLlyvjFL34RRxxxRLRt2zaOPPLIuOqqq+KII47Ijbv44ovjwAMPjLp160aHDh3isssuy4vW48aNi0MOOSR+9atfRZs2baJ+/fpx9tlnx9atW+Paa6+N4uLiaNas2XZz2DbHwYMHR506daJDhw5x99137/SziYh45plnYvDgwVG/fv1o3rx5DBs2LN54440PfQ0AAPs+kRkAgN1qwIABMWvWrNz2rFmzon///tGvX7/c/nfeeScWLFiQi8wXXXRR/PGPf4xf//rXsXjx4ujUqVMMGjQo3nrrrbz3vuSSS2LixInx3HPPRffu3WP06NExZ86cuPfee+Ohhx6K2bNnx+LFi3PjFy5cGOedd15cccUVsXz58pgxY0b07dv3Q+e/adOm+MEPfhC/+c1v4tFHH421a9fGKaecknv+b3/7W3zjG9+I888/P5599tm46aabYsqUKdtF3HHjxsUXv/jFWLp0aXzzm9/c7jhNmzaNatWqxd133x1bt27d6XwaNGgQU6ZMiWeffTZ+8pOfxC9/+cu4/vrr88asXLkypk+fHjNmzIg77rgjbr311hgyZEj8+9//jjlz5sQ111wT3//+92PBggV5r7vsssviy1/+cjz11FMxdOjQOOWUU+K5557b4TzWrl0bxxxzTHzmM5+JhQsXxowZM+L111+Pr371qx/6eQIA8AmQAQDAbvTLX/4yq1evXrZly5aspKQkq1GjRrZmzZrs97//fda3b98sy7Ls4YcfziIi+9e//pVt2LAhq1mzZnb77bfn3mPz5s1Zy5Yts2uvvTbLsiybNWtWFhHZPffckxuzfv36rFatWtkf/vCH3L4333wzq1OnTnb++ednWZZlf/zjH7OioqKspKRkl+Z+2223ZRGRzZ8/P7fvueeeyyIiW7BgQZZlWXbsscdmV199dd7rfvvb32YtWrTIbUdENmrUqI883s9//vOsbt26WYMGDbIBAwZkV1xxRbZy5coPfc11112X9ezZM7c9duzYrG7dunnnOGjQoKxdu3bZ1q1bc/u6dOmSTZgwIW+OZ511Vt579+7dO/vOd76TZVmWrVq1KouI7Mknn8yyLMuuvPLK7Pjjj88bv3r16iwisuXLl3/kuQIAsO9yJTMAALtV//79Y+PGjfHEE0/E3/72tzjwwAOjadOm0a9fv9x9mWfPnh0dOnSINm3axMqVK2PLli1x5JFH5t6jZs2acfjhh293VW2vXr1yf165cmVs3rw5evfundvXpEmT6NKlS277uOOOi7Zt20aHDh1i2LBhcfvtt8emTZs+dP41atSIww47LLfdtWvXaNSoUW4uTz31VFxxxRVRv3793ONb3/pWvPrqq3nv/f657szIkSPjtddei9tvvz369OkTd911V3Tr1i1mzpyZG3PnnXfGkUceGcXFxVG/fv34/ve/Hy+99FLe+7Rr1y4aNGiQ227evHkcdNBBUa1atbx9a9asyXtdnz59ttve2ZXMTz31VMyaNSvvvLt27RoR8Yn+8kYAANwuAwCA3axTp07RqlWrmDVrVsyaNSv69esXEREtW7aM1q1bx7x582LWrFlxzDHHlPu969WrV67xDRo0iMWLF8cdd9wRLVq0iMsvvzx69Ojxsb5Yb8OGDTF+/PhYsmRJ7rF06dJYsWJF1K5du9xzbdCgQZx44onxgx/8IJ566qk4+uij46qrroqIiMceeyyGDh0an/3sZ+O+++6LJ598Mi699NLYvHlz3nvUrFkzb7ugoGCH+8rKylJOOSL+e94nnnhi3nkvWbIkVqxY8ZG3IAEAYN8mMgMAsNsNGDAgZs+eHbNnz47+/fvn9vft2zemT58ejz/+eO5+zB07doxatWrFo48+mhu3ZcuWeOKJJ+Kggw7a6TE6duwYNWvWzLvP8Ntvvx3PP/983rgaNWrEwIED49prr42nn346XnzxxXjkkUd2+r7vvfdeLFy4MLe9fPnyWLt2bXzqU5+KiIhDDz00li9fHp06ddru8f4rh1MUFBRE165dY+PGjRERMW/evGjbtm1ceuml0atXr+jcuXP861//+ljHeL/3fxHjtu1t5/lBhx56aCxbtizatWu33XmXN/4DALBvqbG3JwAAwL5nwIABMXLkyNiyZUvuSuaIiH79+sU555wTmzdvzkXmevXqxXe+850YPXp0NGnSJNq0aRPXXnttbNq0KU4//fSdHqN+/fpx+umnx+jRo2O//faLZs2axaWXXpoXeu+777745z//GX379o3GjRvHAw88EGVlZXm31PigmjVrxrnnnhs//elPo0aNGnHOOefEEUccEYcffnhERFx++eXxuc99Ltq0aRNf+cpXolq1avHUU0/FM888k7sCeVcsWbIkxo4dG8OGDYuDDjooatWqFXPmzIlf/epXcfHFF0dEROfOneOll16KqVOnxmGHHRb3339/TJs2bZeP8VHuuuuu6NWrVxx11FFx++23x+OPPx633nrrDseOHDkyfvnLX8app54aF110UTRp0iReeOGFmDp1atxyyy1RvXr13TYvAACqFpEZAIDdbsCAAfHOO+9E165do3nz5rn9/fr1i/Xr10eXLl2iRYsWuf0TJ06MsrKyGDZsWKxfvz569eoVDz74YDRu3PhDj3PdddflbuPQoEGDuPDCC2PdunW55xs1ahR/+tOfYty4cfHuu+9G586d44477ohu3brt9D3r1q0bF198cXzta1+Ll19+OY4++ui88Dpo0KC477774oorrohrrrkmatasGV27do0zzjijXJ9Rq1atol27djF+/Ph48cUXo6CgILf93e9+NyIiPv/5z8d3v/vdOOecc6K0tDSGDBkSl112WYwbN65cx9qZ8ePHx9SpU+Pss8+OFi1axB133LHTq8dbtmwZjz76aFx88cVx/PHHR2lpabRt2zZOOOGEj30FNwAAVVtBlmXZ3p4EAABUBlOmTIlRo0Z9rHs2VxUFBQUxbdq0+MIXvrC3pwIAQBXnkgMAAAAAAJKJzAAAAAAAJHO7DAAAAAAAkrmSGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAsv8PYeRfOK8a2BYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "max_len = max(data['input_len'])\n",
    "\n",
    "print(f\"The maximum length of this dataset is approximately {max_len} characters.\")\n",
    "\n",
    "n = max_len\n",
    "data.hist(\"input_len\", grid=False, figsize=(18,8), range=(0, n + 1), bins=n + 1)\n",
    "plt.suptitle('')\n",
    "plt.title('Histogram of Text Length')\n",
    "plt.ylabel('')\n",
    "plt.xlabel('Words per Sample')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trim Overly Lengthy Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Question: How can I talk to a real person? Ans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Question: How do I contact customer service? A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Question: How do I file a complaint? Answer: W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Question: How to file a complaint? Answer: You...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Question: What do I do if my package was damag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Question: How can I pay my bill? Answer: You h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Question: How can I get help with my bill? Ans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Question: What do I do if I’m locked out of my...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Question: How do I find out my account number?...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Question: How do I create a FedEx account? Ans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Question: I submitted a request to change my d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Question: How can I stop the delivery of my Fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Question: How can I request to have a FedEx Ex...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Question: Did You Know? Answer: You can use th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Question: How do I know what time the package ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Question: How can I change or correct the deli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Question: How can I report a missing FedEx Exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Question: When I track my FedEx Express® packa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Question: How do I know what time the package ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Question: How can I add delivery instructions ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Question: When and where can I pick up my pack...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Question: How can I manage my incoming deliver...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Question: How can I schedule a delivery time f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Question: How can I request to have my FedEx® ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Question: What does the “delivery exception” s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Question: What does the “delivery now pending”...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Question: If I missed a delivery, how do I kno...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Question: How can I report a missing FedEx Exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Question: What do I do if I missed my delivery...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Question: Did You Know? Answer: With FedEx Del...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Question: What do I do if I don’t have a track...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Question: Why isn’t my package moving when I c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Question: How can I track the exact location o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Question: How do I know what time my FedEx® pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Question: Did You Know? Answer: You use FedEx ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Question: When will my package arrive? Answer:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                input\n",
       "0   Question: How can I talk to a real person? Ans...\n",
       "1   Question: How do I contact customer service? A...\n",
       "2   Question: How do I file a complaint? Answer: W...\n",
       "3   Question: How to file a complaint? Answer: You...\n",
       "4   Question: What do I do if my package was damag...\n",
       "5   Question: How can I pay my bill? Answer: You h...\n",
       "6   Question: How can I get help with my bill? Ans...\n",
       "7   Question: What do I do if I’m locked out of my...\n",
       "8   Question: Did You Know? Answer: Did you know? ...\n",
       "9   Question: How do I find out my account number?...\n",
       "10  Question: Did You Know? Answer: Did you know? ...\n",
       "11  Question: How do I create a FedEx account? Ans...\n",
       "12  Question: Did You Know? Answer: Did you know? ...\n",
       "13  Question: I submitted a request to change my d...\n",
       "14  Question: Did You Know? Answer: Did you know? ...\n",
       "15  Question: How can I stop the delivery of my Fe...\n",
       "16  Question: Did You Know? Answer: Did you know? ...\n",
       "17  Question: How can I request to have a FedEx Ex...\n",
       "18  Question: Did You Know? Answer: You can use th...\n",
       "19  Question: How do I know what time the package ...\n",
       "20  Question: Did You Know? Answer: Did you know? ...\n",
       "21  Question: How can I change or correct the deli...\n",
       "22  Question: Did You Know? Answer: Did you know? ...\n",
       "23  Question: How can I report a missing FedEx Exp...\n",
       "24  Question: When I track my FedEx Express® packa...\n",
       "25  Question: How do I know what time the package ...\n",
       "26  Question: Did You Know? Answer: Did you know? ...\n",
       "27  Question: How can I add delivery instructions ...\n",
       "28  Question: When and where can I pick up my pack...\n",
       "29  Question: How can I manage my incoming deliver...\n",
       "30  Question: How can I schedule a delivery time f...\n",
       "31  Question: How can I request to have my FedEx® ...\n",
       "32  Question: What does the “delivery exception” s...\n",
       "33  Question: What does the “delivery now pending”...\n",
       "34  Question: If I missed a delivery, how do I kno...\n",
       "35  Question: How can I report a missing FedEx Exp...\n",
       "36  Question: What do I do if I missed my delivery...\n",
       "37  Question: Did You Know? Answer: With FedEx Del...\n",
       "38  Question: What do I do if I don’t have a track...\n",
       "39  Question: Why isn’t my package moving when I c...\n",
       "40  Question: How can I track the exact location o...\n",
       "41  Question: How do I know what time my FedEx® pa...\n",
       "42  Question: Did You Know? Answer: You use FedEx ...\n",
       "43  Question: When will my package arrive? Answer:...\n",
       "44  Question: Did You Know? Answer: Did you know? ..."
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.drop(columns=[\"input_len\", \"Question\", \"Answer\"])\n",
    "\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert DataFrame to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input'],\n",
       "    num_rows: 45\n",
       "})"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = Dataset.from_pandas(data)\n",
    "\n",
    "ds"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Values/Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DialoGPT-medium-FAQs-Tuned\n"
     ]
    }
   ],
   "source": [
    "BLOCK_SIZE = 128\n",
    "MODEL_CKPT = \"microsoft/DialoGPT-medium\"\n",
    "\n",
    "MODEL_NAME = MODEL_CKPT.split(\"/\")[-1] + \"-FAQs-Tuned\"\n",
    "set_seed = 42\n",
    "\n",
    "BATCH_SIZE = 500 \n",
    "NUM_OF_EPOCHS = 12\n",
    "\n",
    "WEIGHT_DECAY = 0.01\n",
    "STRATEGY = \"epoch\"\n",
    "\n",
    "REPORTS_TO = \"tensorboard\"\n",
    "LEARNING_RATE = 2e-5\n",
    "\n",
    "print(MODEL_NAME)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenize Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5150b6cd37b142baa7525b31a4b07504",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "530d5b9937e5490cb89150e3cb02c5b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file config.json from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/config.json\n",
      "Model config GPT2Config {\n",
      "  \"_name_or_path\": \"microsoft/DialoGPT-medium\",\n",
      "  \"activation_function\": \"gelu_new\",\n",
      "  \"architectures\": [\n",
      "    \"GPT2LMHeadModel\"\n",
      "  ],\n",
      "  \"attn_pdrop\": 0.1,\n",
      "  \"bos_token_id\": 50256,\n",
      "  \"embd_pdrop\": 0.1,\n",
      "  \"eos_token_id\": 50256,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_epsilon\": 1e-05,\n",
      "  \"model_type\": \"gpt2\",\n",
      "  \"n_ctx\": 1024,\n",
      "  \"n_embd\": 1024,\n",
      "  \"n_head\": 16,\n",
      "  \"n_inner\": null,\n",
      "  \"n_layer\": 24,\n",
      "  \"n_positions\": 1024,\n",
      "  \"reorder_and_upcast_attn\": false,\n",
      "  \"resid_pdrop\": 0.1,\n",
      "  \"scale_attn_by_inverse_layer_idx\": false,\n",
      "  \"scale_attn_weights\": true,\n",
      "  \"summary_activation\": null,\n",
      "  \"summary_first_dropout\": 0.1,\n",
      "  \"summary_proj_to_labels\": true,\n",
      "  \"summary_type\": \"cls_index\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"task_specific_params\": {\n",
      "    \"conversational\": {\n",
      "      \"max_length\": 1000\n",
      "    }\n",
      "  },\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 50257\n",
      "}\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd8a8e06e639413599def28013395ca5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "708c92d113ba4b46a8a03fd330d26316",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading file vocab.json from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/vocab.json\n",
      "loading file merges.txt from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/merges.txt\n",
      "loading file tokenizer.json from cache at None\n",
      "loading file added_tokens.json from cache at None\n",
      "loading file special_tokens_map.json from cache at None\n",
      "loading file tokenizer_config.json from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/tokenizer_config.json\n",
      "loading configuration file config.json from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/config.json\n",
      "Model config GPT2Config {\n",
      "  \"_name_or_path\": \"microsoft/DialoGPT-medium\",\n",
      "  \"activation_function\": \"gelu_new\",\n",
      "  \"architectures\": [\n",
      "    \"GPT2LMHeadModel\"\n",
      "  ],\n",
      "  \"attn_pdrop\": 0.1,\n",
      "  \"bos_token_id\": 50256,\n",
      "  \"embd_pdrop\": 0.1,\n",
      "  \"eos_token_id\": 50256,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_epsilon\": 1e-05,\n",
      "  \"model_type\": \"gpt2\",\n",
      "  \"n_ctx\": 1024,\n",
      "  \"n_embd\": 1024,\n",
      "  \"n_head\": 16,\n",
      "  \"n_inner\": null,\n",
      "  \"n_layer\": 24,\n",
      "  \"n_positions\": 1024,\n",
      "  \"reorder_and_upcast_attn\": false,\n",
      "  \"resid_pdrop\": 0.1,\n",
      "  \"scale_attn_by_inverse_layer_idx\": false,\n",
      "  \"scale_attn_weights\": true,\n",
      "  \"summary_activation\": null,\n",
      "  \"summary_first_dropout\": 0.1,\n",
      "  \"summary_proj_to_labels\": true,\n",
      "  \"summary_type\": \"cls_index\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"task_specific_params\": {\n",
      "    \"conversational\": {\n",
      "      \"max_length\": 1000\n",
      "    }\n",
      "  },\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 50257\n",
      "}\n",
      "\n",
      "loading configuration file config.json from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/config.json\n",
      "Model config GPT2Config {\n",
      "  \"_name_or_path\": \"microsoft/DialoGPT-medium\",\n",
      "  \"activation_function\": \"gelu_new\",\n",
      "  \"architectures\": [\n",
      "    \"GPT2LMHeadModel\"\n",
      "  ],\n",
      "  \"attn_pdrop\": 0.1,\n",
      "  \"bos_token_id\": 50256,\n",
      "  \"embd_pdrop\": 0.1,\n",
      "  \"eos_token_id\": 50256,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_epsilon\": 1e-05,\n",
      "  \"model_type\": \"gpt2\",\n",
      "  \"n_ctx\": 1024,\n",
      "  \"n_embd\": 1024,\n",
      "  \"n_head\": 16,\n",
      "  \"n_inner\": null,\n",
      "  \"n_layer\": 24,\n",
      "  \"n_positions\": 1024,\n",
      "  \"reorder_and_upcast_attn\": false,\n",
      "  \"resid_pdrop\": 0.1,\n",
      "  \"scale_attn_by_inverse_layer_idx\": false,\n",
      "  \"scale_attn_weights\": true,\n",
      "  \"summary_activation\": null,\n",
      "  \"summary_first_dropout\": 0.1,\n",
      "  \"summary_proj_to_labels\": true,\n",
      "  \"summary_type\": \"cls_index\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"task_specific_params\": {\n",
      "    \"conversational\": {\n",
      "      \"max_length\": 1000\n",
      "    }\n",
      "  },\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 50257\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_CKPT, use_fast=True)\n",
    "\n",
    "def tokenizer_function(samples):\n",
    "    return tokenizer(samples[\"input\"])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Tokenizer Function to Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fd9592f2cb242d6b25bf009a9a05ebb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1839 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask'],\n",
       "    num_rows: 45\n",
       "})"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_ds = ds.map(tokenizer_function, \n",
    "                      batched=True,\n",
    "                      remove_columns=ds.column_names,)\n",
    "\n",
    "tokenized_ds"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess Tokenized Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_texts(samples):\n",
    "    # Concatenate all text\n",
    "    concatenated_examples = {k: sum(samples[k], []) for k in samples.keys()}\n",
    "    \n",
    "    # Determine length of all text's concatenated\n",
    "    total_length = len(concatenated_examples[list(samples.keys())[0]])\n",
    "    \n",
    "    # Determine number of blocks\n",
    "    total_length = (total_length // BLOCK_SIZE) * BLOCK_SIZE\n",
    "    \n",
    "    # Split text into chunks of block_size\n",
    "    result = {\n",
    "        k: [t[i : i + BLOCK_SIZE] for i in range(0, total_length, BLOCK_SIZE)]\n",
    "        for k, t in concatenated_examples.items()\n",
    "    }\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "    return result"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Preprocessing Function to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a380ee49e4f4935a307558c6de5541e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "032f6b0faf994219a98040e8d4a0db76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7406012bd86463e9b3d92528a2946bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d7d6363c4e34192acfcf3f4ccf29c3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clm_ds = tokenized_ds.map(\n",
    "    group_texts,\n",
    "    batched=True,\n",
    "    num_proc=4,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file config.json from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/config.json\n",
      "Model config GPT2Config {\n",
      "  \"_name_or_path\": \"microsoft/DialoGPT-medium\",\n",
      "  \"activation_function\": \"gelu_new\",\n",
      "  \"architectures\": [\n",
      "    \"GPT2LMHeadModel\"\n",
      "  ],\n",
      "  \"attn_pdrop\": 0.1,\n",
      "  \"bos_token_id\": 50256,\n",
      "  \"embd_pdrop\": 0.1,\n",
      "  \"eos_token_id\": 50256,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_epsilon\": 1e-05,\n",
      "  \"model_type\": \"gpt2\",\n",
      "  \"n_ctx\": 1024,\n",
      "  \"n_embd\": 1024,\n",
      "  \"n_head\": 16,\n",
      "  \"n_inner\": null,\n",
      "  \"n_layer\": 24,\n",
      "  \"n_positions\": 1024,\n",
      "  \"reorder_and_upcast_attn\": false,\n",
      "  \"resid_pdrop\": 0.1,\n",
      "  \"scale_attn_by_inverse_layer_idx\": false,\n",
      "  \"scale_attn_weights\": true,\n",
      "  \"summary_activation\": null,\n",
      "  \"summary_first_dropout\": 0.1,\n",
      "  \"summary_proj_to_labels\": true,\n",
      "  \"summary_type\": \"cls_index\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"task_specific_params\": {\n",
      "    \"conversational\": {\n",
      "      \"max_length\": 1000\n",
      "    }\n",
      "  },\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 50257\n",
      "}\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56a0f2129aa04e50a7c64ca1665700d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading weights file pytorch_model.bin from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/pytorch_model.bin\n",
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 50256,\n",
      "  \"eos_token_id\": 50256,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
      "\n",
      "All the weights of GPT2LMHeadModel were initialized from the model checkpoint at microsoft/DialoGPT-medium.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da15b51d17e14f689cb0b950c43184e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file generation_config.json from cache at /Users/briandunn/.cache/huggingface/hub/models--microsoft--DialoGPT-medium/snapshots/9d5c5fadcc072b693fb5a5e29416bbf3f503c26c/generation_config.json\n",
      "Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"bos_token_id\": 50256,\n",
      "  \"eos_token_id\": 50256,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(MODEL_CKPT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n"
     ]
    }
   ],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=MODEL_NAME,\n",
    "    evaluation_strategy=STRATEGY,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    weight_decay=WEIGHT_DECAY,\n",
    "    num_train_epochs=NUM_OF_EPOCHS,\n",
    "    report_to=REPORTS_TO,\n",
    "    logging_first_step=True,\n",
    "    logging_strategy=\"steps\",\n",
    "    logging_steps=11,\n",
    "    save_strategy=STRATEGY,\n",
    "    hub_private_repo=True,\n",
    "    push_to_hub=True\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Data Collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/DialoGPT-medium-FAQs-Tuned into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=clm_ds,\n",
    "    eval_dataset=clm_ds,\n",
    "    data_collator=data_collator\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/briandunn/Documents/nlpnn/nlp_ml/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 85\n",
      "  Num Epochs = 12\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 132\n",
      "  Number of trainable parameters = 354823168\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59861a3d80774c46a2344edd80069042",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.1467, 'learning_rate': 1.984848484848485e-05, 'epoch': 0.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 6.2671, 'learning_rate': 1.8333333333333333e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73eec6dceb014b80aa8d3ffaa3bea642",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-11\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-11/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-11/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 4.803851127624512, 'eval_runtime': 201.1259, 'eval_samples_per_second': 0.423, 'eval_steps_per_second': 0.055, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-11/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 4.8143, 'learning_rate': 1.6666666666666667e-05, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce492cb1d6464e44b7b4c2c3404b24cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-22\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-22/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-22/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 4.026591777801514, 'eval_runtime': 210.6257, 'eval_samples_per_second': 0.404, 'eval_steps_per_second': 0.052, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-22/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 4.072, 'learning_rate': 1.5000000000000002e-05, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd852c180ab344b5b759dcb3c986b791",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-33\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-33/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-33/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.4453070163726807, 'eval_runtime': 209.7368, 'eval_samples_per_second': 0.405, 'eval_steps_per_second': 0.052, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-33/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.5482, 'learning_rate': 1.3333333333333333e-05, 'epoch': 4.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97f34b15f1a84aab95f92065bd9caf3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-44\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-44/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-44/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.0559096336364746, 'eval_runtime': 203.0742, 'eval_samples_per_second': 0.419, 'eval_steps_per_second': 0.054, 'epoch': 4.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-44/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.2654, 'learning_rate': 1.1666666666666668e-05, 'epoch': 5.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10e54bccf9cd4ca1bad273bd6aaf1b01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-55\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-55/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-55/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.815948486328125, 'eval_runtime': 196.9651, 'eval_samples_per_second': 0.432, 'eval_steps_per_second': 0.056, 'epoch': 5.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-55/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.9732, 'learning_rate': 1e-05, 'epoch': 6.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e004794ffa9742d5a8230866a4ec923d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-66\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-66/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-66/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.597230911254883, 'eval_runtime': 197.0769, 'eval_samples_per_second': 0.431, 'eval_steps_per_second': 0.056, 'epoch': 6.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-66/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.8302, 'learning_rate': 8.333333333333334e-06, 'epoch': 7.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a344f80856c47959d4694fe27a41e4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-77\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-77/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-77/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.4588348865509033, 'eval_runtime': 196.3558, 'eval_samples_per_second': 0.433, 'eval_steps_per_second': 0.056, 'epoch': 7.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-77/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.6841, 'learning_rate': 6.666666666666667e-06, 'epoch': 8.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5fac560832c45c284e3d21039ae0a69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-88\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-88/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-88/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.337815284729004, 'eval_runtime': 196.5914, 'eval_samples_per_second': 0.432, 'eval_steps_per_second': 0.056, 'epoch': 8.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-88/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.5855, 'learning_rate': 5e-06, 'epoch': 9.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75d874ae3c2c4d7baad51e940a35cb38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-99\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-99/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-99/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.2652175426483154, 'eval_runtime': 198.2924, 'eval_samples_per_second': 0.429, 'eval_steps_per_second': 0.055, 'epoch': 9.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-99/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.5088, 'learning_rate': 3.3333333333333333e-06, 'epoch': 10.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4adeb574e9f14bd3b9840cd7601c9c4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-110\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-110/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-110/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.2036728858947754, 'eval_runtime': 197.3854, 'eval_samples_per_second': 0.431, 'eval_steps_per_second': 0.056, 'epoch': 10.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-110/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.4637, 'learning_rate': 1.6666666666666667e-06, 'epoch': 11.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ffe1902dd5141408f191ee6b61262ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-121\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-121/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-121/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.170647382736206, 'eval_runtime': 203.0223, 'eval_samples_per_second': 0.419, 'eval_steps_per_second': 0.054, 'epoch': 11.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-121/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.4402, 'learning_rate': 0.0, 'epoch': 12.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a24f0e1450d84dafa6e2a913ccc5f714",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned/checkpoint-132\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-132/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/checkpoint-132/generation_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.160059928894043, 'eval_runtime': 199.1187, 'eval_samples_per_second': 0.427, 'eval_steps_per_second': 0.055, 'epoch': 12.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/checkpoint-132/pytorch_model.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_runtime': 10171.3234, 'train_samples_per_second': 0.1, 'train_steps_per_second': 0.013, 'train_loss': 3.385300874710083, 'epoch': 12.0}\n"
     ]
    }
   ],
   "source": [
    "train_results = trainer.train()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 85\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19b4b672118741d79c81f9457d5014c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity: 8.67\n"
     ]
    }
   ],
   "source": [
    "evaluation_results = trainer.evaluate()\n",
    "print(f\"Perplexity: {math.exp(evaluation_results['eval_loss']):.2f}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Push Model to HuggingFace Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to DialoGPT-medium-FAQs-Tuned\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/config.json\n",
      "Configuration saved in DialoGPT-medium-FAQs-Tuned/generation_config.json\n",
      "Model weights saved in DialoGPT-medium-FAQs-Tuned/pytorch_model.bin\n",
      "Several commits (2) will be pushed upstream.\n",
      "The progress bars may be unreliable.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cebcc0e6830e49fcbe2172a7063aa847",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30a9e4732dcc4dda9ca797cc3ecf857c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c562bf80510643d89451c7a6b969d220",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/DunnBC22/DialoGPT-medium-FAQs-Tuned\n",
      "   9ac1e66..44c500f  main -> main\n",
      "\n",
      "Dropping the following result as it does not have all the necessary fields:\n",
      "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n",
      "Adding files tracked by Git LFS: ['.DS_Store']. This may take a bit of time if the files are large.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8e7add0bbee4dab82037c3e09030af6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1eb11c96e2bf4bdab3ff8848cedfb732",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/DunnBC22/DialoGPT-medium-FAQs-Tuned\n",
      "   44c500f..a3a521a  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/DialoGPT-medium-FAQs-Tuned/commit/44c500f72c2501f604f8a5d5542296bbdf9ee5f5'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notes & Other Key Takeaways\n",
    "****\n",
    "- Training this model on the Frequently Asked Questions for a different company or department would create a fantastic way to automate some mundane and easily answered questions.\n",
    "- With a perplexity value of 8.67, this model is well suited already, but I want to try training it with the larger version of this transformer to see if there is substanital improvement.\n",
    "****"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp_ml",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0b7a754c5aaaa44a21e2f2148f50b931a5abdffe7ef71537f5954179554f39df"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}