Causal LM for Chatbot/Company FAQs/v2/FAQs - CLM for DialoGPT Chatbot-DialoGPT-Large_v2.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FAQs: Causal Language Modeling\n",
    "\n",
    "Dataset Source: FAQ Page From a Large Company"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Necessary Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "import os, sys, math\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import datasets\n",
    "from datasets import Dataset\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForLanguageModeling\n",
    "from transformers import TrainingArguments, Trainer, set_seed\n",
    "\n",
    "!git lfs install"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display Library Versions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Python: 3.9.12\n",
      "        Pandas: 2.0.0\n",
      "      Datasets: 2.11.0\n",
      "  Transformers: 4.27.4\n"
     ]
    }
   ],
   "source": [
    "print(\"Python:\".rjust(15), sys.version[0:6])\n",
    "print(\"Pandas:\".rjust(15), pd.__version__)\n",
    "print(\"Datasets:\".rjust(15), datasets.__version__)\n",
    "print(\"Transformers:\".rjust(15), transformers.__version__)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 45 entries, 0 to 44\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   Question   45 non-null     object\n",
      " 1   Answer     45 non-null     object\n",
      " 2   input      45 non-null     object\n",
      " 3   input_len  45 non-null     int64 \n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 1.5+ KB\n"
     ]
    }
   ],
   "source": [
    "data_file = \"Company FAQs.csv\"\n",
    "\n",
    "data = pd.read_csv(data_file, usecols=[0,1])\n",
    "\n",
    "data = data.rename(columns={\"question\": \"Question\"})\n",
    "\n",
    "data['input'] = \"Question: \" + data['Question'] + \" Answer: \" + data['Answer']\n",
    "\n",
    "data['input_len'] = data['input'].str.len()\n",
    "\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visual: Histogram of Input Lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximum length of this dataset is approximately 5323 characters.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABZkAAAK9CAYAAABGqWLxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/nklEQVR4nO3deZxVdf348fewDeuwKDAg+yIkBiYo8lVZFEUis+2bGhGSZiYupKHyMwXUBLXSFkJNkxYT00JLBSVlKREUEEU0RMIkN3KBYdEBmfP7owf36xVQ5iMDM/h8Ph73Iefcz73nc+7M54G+PI9zC7IsywIAAAAAABJU29sTAAAAAACg6hKZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAPZx7dq1i9NOO21vT2Ofd91110WHDh2ievXqccghh+zt6XziFBQUxDnnnLO3pwEA8IkkMgMAVCFTpkyJgoKCWLhw4Q6f79+/fxx88MEf+zgPPPBAjBs37mO/zyfFQw89FBdddFEceeSRcdttt8XVV1+93ZjZs2dHQUHBLj12l3nz5sW4ceNi7dq1uzT+tNNOi/r16++24+9u5T0fAAD2jBp7ewIAAFSs5cuXR7Vq5bu24IEHHohJkyYJzbvokUceiWrVqsWtt94atWrV2uGYT33qU/Hb3/42b9+YMWOifv36cemll1bIvObNmxfjx4+P0047LRo1alQhx9iT9rXzAQDYV4jMAAD7uMLCwr09hXLbuHFj1KtXb29PY5etWbMm6tSps9PAHBHRvHnz+PrXv563b+LEibH//vtvtx8AAKoSt8sAANjHffCezFu2bInx48dH586do3bt2rHffvvFUUcdFTNnzoyI/94yYdKkSRERO7yFw8aNG+PCCy+M1q1bR2FhYXTp0iV++MMfRpZlecd955134rzzzov9998/GjRoEJ///Ofj5ZdfjoKCgrwrpMeNGxcFBQXx7LPPxte+9rVo3LhxHHXUURER8fTTT8dpp50WHTp0iNq1a0dxcXF885vfjDfffDPvWNve4/nnn4+vf/3r0bBhw2jatGlcdtllkWVZrF69Ok466aQoKiqK4uLi+NGPfrRLn917770XV155ZXTs2DEKCwujXbt28f/+3/+L0tLS3JiCgoK47bbbYuPGjbnPasqUKbv0/juydu3aGDVqVO7z7dSpU1xzzTVRVlYWERFZlsWAAQOiadOmsWbNmtzrNm/eHJ/+9KejY8eOsXHjxhg3blyMHj06IiLat2+fm9uLL76YPLdtFixYECeccEI0bNgw6tatG/369YtHH300b8y2n8kLL7yQu/K4YcOGMWLEiNi0aVPe2F35XdnV87nnnnvi4IMPjsLCwujWrVvMmDHjY58vAAAfzpXMAABV0Lp16+KNN97Ybv+WLVs+8rXjxo2LCRMmxBlnnBGHH354lJSUxMKFC2Px4sVx3HHHxbe//e145ZVXYubMmdvd3iHLsvj85z8fs2bNitNPPz0OOeSQePDBB2P06NHx8ssvx/XXX58be9ppp8Uf/vCHGDZsWBxxxBExZ86cGDJkyE7n9b//+7/RuXPnuPrqq3PBeubMmfHPf/4zRowYEcXFxbFs2bK4+eabY9myZTF//vzt7l988sknx6c+9amYOHFi3H///XHVVVdFkyZN4qabbopjjjkmrrnmmrj99tvje9/7Xhx22GHRt2/fD/2szjjjjPj1r38dX/nKV+LCCy+MBQsWxIQJE+K5556LadOmRUTEb3/727j55pvj8ccfj1tuuSUiIv7nf/7nI38OO7Jp06bo169fvPzyy/Htb3872rRpE/PmzYsxY8bEq6++GjfccEMUFBTEr371q+jevXucddZZ8ac//SkiIsaOHRvLli2L2bNnR7169eJLX/pSPP/883HHHXfE9ddfH/vvv39ERDRt2jRpbts88sgjMXjw4OjZs2eMHTs2qlWrFrfddlscc8wx8be//S0OP/zwvPFf/epXo3379jFhwoRYvHhx3HLLLdGsWbO45pprcmN25XdlV87n73//e/zpT3+Ks88+Oxo0aBA//elP48tf/nK89NJLsd9++32s8wYA4ENkAABUGbfddlsWER/66NatW95r2rZtmw0fPjy33aNHj2zIkCEfepyRI0dmO/pXxXvuuSeLiOyqq67K2/+Vr3wlKygoyF544YUsy7Js0aJFWURko0aNyht32mmnZRGRjR07Nrdv7NixWURkp5566nbH27Rp03b77rjjjiwisrlz5273HmeeeWZu33vvvZe1atUqKygoyCZOnJjb//bbb2d16tTJ+0x2ZMmSJVlEZGeccUbe/u9973tZRGSPPPJIbt/w4cOzevXqfej77Ui3bt2yfv365bavvPLKrF69etnzzz+fN+6SSy7Jqlevnr300ku5fTfddFMWEdnvfve7bP78+Vn16tW3+7yvu+66LCKyVatW7dJ8Puo8ysrKss6dO2eDBg3KysrKcvs3bdqUtW/fPjvuuONy+7b9TL75zW/mvccXv/jFbL/99sttl+d35cPOJyKyWrVq5X4HsyzLnnrqqSwisp/97Gcfee4AAKRzuwwAgCpo0qRJMXPmzO0e3bt3/8jXNmrUKJYtWxYrVqwo93EfeOCBqF69epx33nl5+y+88MLIsiymT58eEZG7RcHZZ5+dN+7cc8/d6XufddZZ2+2rU6dO7s/vvvtuvPHGG3HEEUdERMTixYu3G3/GGWfk/ly9evXo1atXZFkWp59+em5/o0aNokuXLvHPf/5zp3OJ+O+5RkRccMEFefsvvPDCiIi4//77P/T1Ke666644+uijo3HjxvHGG2/kHgMHDoytW7fG3Llzc2PPPPPMGDRoUJx77rkxbNiw6NixY1x99dW7fU7vt2TJklixYkV87WtfizfffDM3v40bN8axxx4bc+fOzd3WY5sP/lyPPvroePPNN6OkpCQi0n5XdmbgwIHRsWPH3Hb37t2jqKjoI3/WAAB8PG6XAQBQBR1++OHRq1ev7fZvi5Mf5oorroiTTjopDjzwwDj44IPjhBNOiGHDhu1SoP7Xv/4VLVu2jAYNGuTt/9SnPpV7fts/q1WrFu3bt88b16lTp52+9wfHRkS89dZbMX78+Jg6dWre/Ycj/nvLkA9q06ZN3nbDhg2jdu3auVsrvH//B+/r/EHbzuGDcy4uLo5GjRrlznV3WrFiRTz99NM7vaXFBz+DW2+9NTp27BgrVqyIefPm5UX5irDtf0wMHz58p2PWrVsXjRs3zm1/8Gey7bm33347ioqKkn5XduaDx9p2vLfffrvc7wUAwK4TmQEAPmH69u0bK1eujHvvvTceeuihuOWWW+L666+PG2+8Me9K4D1tR4H0q1/9asybNy9Gjx4dhxxySNSvXz/KysrihBNO2O6K2Yj/Xr28K/siYrsvKtyZD973uSKVlZXFcccdFxdddNEOnz/wwAPztmfPnp37EsKlS5dGnz59Knx+ERHXXXddHHLIITscU79+/bztj/v5l8eePBYAAP9HZAYA+ARq0qRJjBgxIkaMGBEbNmyIvn37xrhx43KReWdhtW3btvHXv/411q9fn3c18z/+8Y/c89v+WVZWFqtWrYrOnTvnxr3wwgu7PMe33347Hn744Rg/fnxcfvnluf0pt/lIse0cVqxYkbtSOyLi9ddfj7Vr1+bOdXfq2LFjbNiwIQYOHPiRY1999dU499xz4/jjj49atWrF9773vRg0aFDevHZ3IN92K4qioqJdmuOuKM/vyp4M/gAA7Dr3ZAYA+IT54G0i6tevH506dcpdERsRUa9evYiIWLt2bd7Yz372s7F169b4+c9/nrf/+uuvj4KCghg8eHBERAwaNCgiIn7xi1/kjfvZz362y/PcdlXqB69CveGGG3b5PT6Oz372szs83o9//OOIiBgyZMhuP+ZXv/rVeOyxx+LBBx/c7rm1a9fGe++9l9v+1re+FWVlZXHrrbfGzTffHDVq1IjTTz897/Pa2c8xVc+ePaNjx47xwx/+MDZs2LDd8//5z3/K/Z7l+V3Z3ecDAMDu4UpmAIBPmIMOOij69+8fPXv2jCZNmsTChQvj7rvvjnPOOSc3pmfPnhERcd5558WgQYOievXqccopp8SJJ54YAwYMiEsvvTRefPHF6NGjRzz00ENx7733xqhRo3JXuvbs2TO+/OUvxw033BBvvvlmHHHEETFnzpx4/vnnI2LXrkgtKiqKvn37xrXXXhtbtmyJAw44IB566KFYtWpVBXwq2+vRo0cMHz48br755li7dm3069cvHn/88fj1r38dX/jCF2LAgAG7/ZijR4+OP//5z/G5z30uTjvttOjZs2ds3Lgxli5dGnfffXe8+OKLsf/++8dtt90W999/f0yZMiVatWoVEf+Nsl//+tdj8uTJuS/R2/ZzvPTSS+OUU06JmjVrxoknnpiLtTuyZcuWuOqqq7bb36RJkzj77LPjlltuicGDB0e3bt1ixIgRccABB8TLL78cs2bNiqKiovjLX/5SrnMuz+9KyvkAAFDxRGYAgE+Y8847L/785z/HQw89FKWlpdG2bdu46qqrYvTo0bkxX/rSl+Lcc8+NqVOnxu9+97vIsixOOeWUqFatWvz5z3+Oyy+/PO6888647bbbol27dnHdddfFhRdemHec3/zmN1FcXBx33HFHTJs2LQYOHBh33nlndOnSJWrXrr1Lc/39738f5557bkyaNCmyLIvjjz8+pk+fHi1bttytn8nO3HLLLdGhQ4eYMmVKTJs2LYqLi2PMmDExduzYCjle3bp1Y86cOXH11VfHXXfdFb/5zW+iqKgoDjzwwBg/fnw0bNgw/v3vf8d3v/vdOPHEE/O+gG/o0KHxxz/+MS666KIYPHhwtG/fPg477LC48sor48Ybb4wZM2bkbkvxYVF28+bNcdlll223v2PHjnH22WdH//7947HHHosrr7wyfv7zn8eGDRuiuLg4evfuHd/+9reTzntXf1dSzgcAgIpXkPkWDAAA9pAlS5bEZz7zmfjd734XQ4cO3dvToRLzuwIAUHW4JzMAABXinXfe2W7fDTfcENWqVYu+ffvuhRlRWfldAQCo2twuAwCACnHttdfGokWLYsCAAVGjRo2YPn16TJ8+Pc4888xo3br13p4elYjfFQCAqs3tMgAAqBAzZ86M8ePHx7PPPhsbNmyINm3axLBhw+LSSy+NGjVc68D/8bsCAFC1icwAAAAAACRzT2YAAAAAAJKJzAAAAAAAJNvjNzgrKyuLV155JRo0aBAFBQV7+vAAAAAAAOyCLMti/fr10bJly6hWbefXK+/xyPzKK6/4hmgAAAAAgCpi9erV0apVq50+v8cjc4MGDSLivxMrKira04cHAAAAAGAXlJSUROvWrXNNd2f2eGTedouMoqIikRkAAAAAoJL7qNse++I/AAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAECyckXmcePGRUFBQd6ja9euFTU3AAAAAAAquRrlfUG3bt3ir3/96/+9QY1yvwUAAAAAAPuIchfiGjVqRHFxcUXMBQAAAACAKqbc92ResWJFtGzZMjp06BBDhw6Nl1566UPHl5aWRklJSd4DAAAAAIB9Q7kic+/evWPKlCkxY8aMmDx5cqxatSqOPvroWL9+/U5fM2HChGjYsGHu0bp16489aQAAAAAAKoeCLMuy1BevXbs22rZtGz/+8Y/j9NNP3+GY0tLSKC0tzW2XlJRE69atY926dVFUVJR6aAAAAAAAKlBJSUk0bNjwI1vux/rWvkaNGsWBBx4YL7zwwk7HFBYWRmFh4cc5DAAAAAAAlVS578n8fhs2bIiVK1dGixYtdtd8AAAAAACoQsoVmb/3ve/FnDlz4sUXX4x58+bFF7/4xahevXqceuqpFTU/AAAAAAAqsXLdLuPf//53nHrqqfHmm29G06ZN46ijjor58+dH06ZNK2p+AAAAAABUYuWKzFOnTq2oeQAAAAAAUAV9rHsyAwAAAADwySYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJDsY0XmiRMnRkFBQYwaNWo3TQcAAAAAgKokOTI/8cQTcdNNN0X37t1353wAAAAAAKhCkiLzhg0bYujQofHLX/4yGjduvLvnBAAAAABAFZEUmUeOHBlDhgyJgQMHfuTY0tLSKCkpyXsAAAAAALBvKHdknjp1aixevDgmTJiwS+MnTJgQDRs2zD1at25d7knC7tDukvv39hQAAAAAYJ9Trsi8evXqOP/88+P222+P2rVr79JrxowZE+vWrcs9Vq9enTRRAAAAAAAqnxrlGbxo0aJYs2ZNHHroobl9W7dujblz58bPf/7zKC0tjerVq+e9prCwMAoLC3fPbAEAAAAAqFTKFZmPPfbYWLp0ad6+ESNGRNeuXePiiy/eLjADAAAAALBvK1dkbtCgQRx88MF5++rVqxf77bffdvsBAAAAANj3lfuL/wAAAAAAYJtyXcm8I7Nnz94N0wAAAAAAoCpyJTMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQrV2SePHlydO/ePYqKiqKoqCj69OkT06dPr6i5AQAAAABQyZUrMrdq1SomTpwYixYtioULF8YxxxwTJ510Uixbtqyi5gcAAAAAQCVWozyDTzzxxLztH/zgBzF58uSYP39+dOvWbbdODAAAAACAyq9ckfn9tm7dGnfddVds3Lgx+vTps9NxpaWlUVpamtsuKSlJPSQAAAAAAJVMub/4b+nSpVG/fv0oLCyMs846K6ZNmxYHHXTQTsdPmDAhGjZsmHu0bt36Y00YAAAAAIDKo9yRuUuXLrFkyZJYsGBBfOc734nhw4fHs88+u9PxY8aMiXXr1uUeq1ev/lgTBgAAAACg8ij37TJq1aoVnTp1ioiInj17xhNPPBE/+clP4qabbtrh+MLCwigsLPx4swQAAAAAoFIq95XMH1RWVpZ3z2UAAAAAAD45ynUl85gxY2Lw4MHRpk2bWL9+ffz+97+P2bNnx4MPPlhR8wMAAAAAoBIrV2Res2ZNfOMb34hXX301GjZsGN27d48HH3wwjjvuuIqaHwAAAAAAlVi5IvOtt95aUfMAAAAAAKAK+tj3ZAYAAAAA4JNLZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgWbki84QJE+Kwww6LBg0aRLNmzeILX/hCLF++vKLmBgAAAABAJVeuyDxnzpwYOXJkzJ8/P2bOnBlbtmyJ448/PjZu3FhR8wMAAAAAoBKrUZ7BM2bMyNueMmVKNGvWLBYtWhR9+/bdrRMDAAAAAKDyK1dk/qB169ZFRESTJk12Oqa0tDRKS0tz2yUlJR/nkAAAAAAAVCLJX/xXVlYWo0aNiiOPPDIOPvjgnY6bMGFCNGzYMPdo3bp16iEBAAAAAKhkkiPzyJEj45lnnompU6d+6LgxY8bEunXrco/Vq1enHhIAAAAAgEom6XYZ55xzTtx3330xd+7caNWq1YeOLSwsjMLCwqTJAQAAAABQuZUrMmdZFueee25MmzYtZs+eHe3bt6+oeQEAAAAAUAWUKzKPHDkyfv/738e9994bDRo0iNdeey0iIho2bBh16tSpkAkCAAAAAFB5leuezJMnT45169ZF//79o0WLFrnHnXfeWVHzAwAAAACgEiv37TIAAAAAAGCbcl3JDAAAAAAA7ycyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJCs3JF57ty5ceKJJ0bLli2joKAg7rnnngqYFgAAAAAAVUG5I/PGjRujR48eMWnSpIqYDwAAAAAAVUiN8r5g8ODBMXjw4IqYCwAAAAAAVUy5I3N5lZaWRmlpaW67pKSkog8JAAAAAMAeUuFf/DdhwoRo2LBh7tG6deuKPiSVVLtL7q/0xyrv6/bkOe1te+ozperwswUAAGBf5799d02FR+YxY8bEunXrco/Vq1dX9CEBAAAAANhDKvx2GYWFhVFYWFjRhwEAAAAAYC+o8CuZAQAAAADYd5X7SuYNGzbECy+8kNtetWpVLFmyJJo0aRJt2rTZrZMDAAAAAKByK3dkXrhwYQwYMCC3fcEFF0RExPDhw2PKlCm7bWIAAAAAAFR+5Y7M/fv3jyzLKmIuAAAAAABUMe7JDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAAAAAQDKRGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpEZAAAAAIBkIjMAAAAAAMlEZgAAAAAAkonMAAAAAAAkE5kBAAAAAEgmMgMAAAAAkExkBgAAAAAgmcgMAAAAAEAykRkAAAAAgGQiMwAAAAAAyZIi86RJk6Jdu3ZRu3bt6N27dzz++OO7e14AAAAAAFQB5Y7Md955Z1xwwQUxduzYWLx4cfTo0SMGDRoUa9asqYj5AQAAAABQiZU7Mv/4xz+Ob33rWzFixIg46KCD4sYbb4y6devGr371q4qYHwAAAAAAlViN8gzevHlzLFq0KMaMGZPbV61atRg4cGA89thjO3xNaWlplJaW5rbXrVsXERElJSUp86UKKyvdtMd+7js61q4cv7xz3JPntLelnusn6TP6pPGzBQAAYF/3Sf9v323nnmXZh44ryD5qxPu88sorccABB8S8efOiT58+uf0XXXRRzJkzJxYsWLDda8aNGxfjx4/f1UMAAAAAAFCJrF69Olq1arXT58t1JXOKMWPGxAUXXJDbLisri7feeiv222+/KCgoqOjDVyolJSXRunXrWL16dRQVFe3t6QC7wLqFqse6harJ2oWqx7qFqse6pbyyLIv169dHy5YtP3RcuSLz/vvvH9WrV4/XX389b//rr78excXFO3xNYWFhFBYW5u1r1KhReQ67zykqKrKQoYqxbqHqsW6harJ2oeqxbqHqsW4pj4YNG37kmHJ98V+tWrWiZ8+e8fDDD+f2lZWVxcMPP5x3+wwAAAAAAD4Zyn27jAsuuCCGDx8evXr1isMPPzxuuOGG2LhxY4wYMaIi5gcAAAAAQCVW7sh88sknx3/+85+4/PLL47XXXotDDjkkZsyYEc2bN6+I+e1TCgsLY+zYsdvdPgSovKxbqHqsW6iarF2oeqxbqHqsWypKQZZl2d6eBAAAAAAAVVO57skMAAAAAADvJzIDAAAAAJBMZAYAAAAAIJnIDAAAAABAMpF5D5o0aVK0a9cuateuHb17947HH398b08JPhHmzp0bJ554YrRs2TIKCgrinnvuyXs+y7K4/PLLo0WLFlGnTp0YOHBgrFixIm/MW2+9FUOHDo2ioqJo1KhRnH766bFhw4a8MU8//XQcffTRUbt27WjdunVce+21FX1qsM+aMGFCHHbYYdGgQYNo1qxZfOELX4jly5fnjXn33Xdj5MiRsd9++0X9+vXjy1/+crz++ut5Y1566aUYMmRI1K1bN5o1axajR4+O9957L2/M7Nmz49BDD43CwsLo1KlTTJkypaJPD/ZJkydPju7du0dRUVEUFRVFnz59Yvr06bnnrVmo/CZOnBgFBQUxatSo3D5rFyqfcePGRUFBQd6ja9euueetW/YGkXkPufPOO+OCCy6IsWPHxuLFi6NHjx4xaNCgWLNmzd6eGuzzNm7cGD169IhJkybt8Plrr702fvrTn8aNN94YCxYsiHr16sWgQYPi3XffzY0ZOnRoLFu2LGbOnBn33XdfzJ07N84888zc8yUlJXH88cdH27ZtY9GiRXHdddfFuHHj4uabb67w84N90Zw5c2LkyJExf/78mDlzZmzZsiWOP/742LhxY27Md7/73fjLX/4Sd911V8yZMydeeeWV+NKXvpR7fuvWrTFkyJDYvHlzzJs3L37961/HlClT4vLLL8+NWbVqVQwZMiQGDBgQS5YsiVGjRsUZZ5wRDz744B49X9gXtGrVKiZOnBiLFi2KhQsXxjHHHBMnnXRSLFu2LCKsWajsnnjiibjpppuie/fuefutXaicunXrFq+++mru8fe//z33nHXLXpGxRxx++OHZyJEjc9tbt27NWrZsmU2YMGEvzgo+eSIimzZtWm67rKwsKy4uzq677rrcvrVr12aFhYXZHXfckWVZlj377LNZRGRPPPFEbsz06dOzgoKC7OWXX86yLMt+8YtfZI0bN85KS0tzYy6++OKsS5cuFXxG8MmwZs2aLCKyOXPmZFn233Vas2bN7K677sqNee6557KIyB577LEsy7LsgQceyKpVq5a99tpruTGTJ0/OioqKcmv1oosuyrp165Z3rJNPPjkbNGhQRZ8SfCI0btw4u+WWW6xZqOTWr1+fde7cOZs5c2bWr1+/7Pzzz8+yzN+3UFmNHTs269Gjxw6fs27ZW1zJvAds3rw5Fi1aFAMHDsztq1atWgwcODAee+yxvTgzYNWqVfHaa6/lrc+GDRtG7969c+vzsccei0aNGkWvXr1yYwYOHBjVqlWLBQsW5Mb07ds3atWqlRszaNCgWL58ebz99tt76Gxg37Vu3bqIiGjSpElERCxatCi2bNmSt3a7du0abdq0yVu7n/70p6N58+a5MYMGDYqSkpLclZWPPfZY3ntsG+PvZ/h4tm7dGlOnTo2NGzdGnz59rFmo5EaOHBlDhgzZbn1Zu1B5rVixIlq2bBkdOnSIoUOHxksvvRQR1i17j8i8B7zxxhuxdevWvMUbEdG8efN47bXX9tKsgIjIrcEPW5+vvfZaNGvWLO/5GjVqRJMmTfLG7Og93n8MIE1ZWVmMGjUqjjzyyDj44IMj4r/rqlatWtGoUaO8sR9cux+1Lnc2pqSkJN55552KOB3Ypy1dujTq168fhYWFcdZZZ8W0adPioIMOsmahEps6dWosXrw4JkyYsN1z1i5UTr17944pU6bEjBkzYvLkybFq1ao4+uijY/369dYte02NvT0BAIAPM3LkyHjmmWfy7jMHVE5dunSJJUuWxLp16+Luu++O4cOHx5w5c/b2tICdWL16dZx//vkxc+bMqF279t6eDrCLBg8enPtz9+7do3fv3tG2bdv4wx/+EHXq1NmLM+OTzJXMe8D+++8f1atX3+6bPF9//fUoLi7eS7MCIiK3Bj9sfRYXF2/3JZ3vvfdevPXWW3ljdvQe7z8GUH7nnHNO3HfffTFr1qxo1apVbn9xcXFs3rw51q5dmzf+g2v3o9blzsYUFRX5F3RIUKtWrejUqVP07NkzJkyYED169Iif/OQn1ixUUosWLYo1a9bEoYceGjVq1IgaNWrEnDlz4qc//WnUqFEjmjdvbu1CFdCoUaM48MAD44UXXvB3LnuNyLwH1KpVK3r27BkPP/xwbl9ZWVk8/PDD0adPn704M6B9+/ZRXFyctz5LSkpiwYIFufXZp0+fWLt2bSxatCg35pFHHomysrLo3bt3bszcuXNjy5YtuTEzZ86MLl26ROPGjffQ2cC+I8uyOOecc2LatGnxyCOPRPv27fOe79mzZ9SsWTNv7S5fvjxeeumlvLW7dOnSvP9JNHPmzCgqKoqDDjooN+b977FtjL+fYfcoKyuL0tJSaxYqqWOPPTaWLl0aS5YsyT169eoVQ4cOzf3Z2oXKb8OGDbFy5cpo0aKFv3PZe/b2Nw9+UkydOjUrLCzMpkyZkj377LPZmWeemTVq1CjvmzyBirF+/frsySefzJ588sksIrIf//jH2ZNPPpn961//yrIsyyZOnJg1atQou/fee7Onn346O+mkk7L27dtn77zzTu49TjjhhOwzn/lMtmDBguzvf/971rlz5+zUU0/NPb927dqsefPm2bBhw7Jnnnkmmzp1ala3bt3spptu2uPnC/uC73znO1nDhg2z2bNnZ6+++mrusWnTptyYs846K2vTpk32yCOPZAsXLsz69OmT9enTJ/f8e++9lx188MHZ8ccfny1ZsiSbMWNG1rRp02zMmDG5Mf/85z+zunXrZqNHj86ee+65bNKkSVn16tWzGTNm7NHzhX3BJZdcks2ZMydbtWpV9vTTT2eXXHJJVlBQkD300ENZllmzUFX069cvO//883Pb1i5UPhdeeGE2e/bsbNWqVdmjjz6aDRw4MNt///2zNWvWZFlm3bJ3iMx70M9+9rOsTZs2Wa1atbLDDz88mz9//t6eEnwizJo1K4uI7R7Dhw/PsizLysrKsssuuyxr3rx5VlhYmB177LHZ8uXL897jzTffzE499dSsfv36WVFRUTZixIhs/fr1eWOeeuqp7KijjsoKCwuzAw44IJs4ceKeOkXY5+xozUZEdtttt+XGvPPOO9nZZ5+dNW7cOKtbt272xS9+MXv11Vfz3ufFF1/MBg8enNWpUyfbf//9swsvvDDbsmVL3phZs2ZlhxxySFarVq2sQ4cOeccAdt03v/nNrG3btlmtWrWypk2bZscee2wuMGeZNQtVxQcjs7ULlc/JJ5+ctWjRIqtVq1Z2wAEHZCeffHL2wgsv5J63btkbCrIsy/bONdQAAAAAAFR17skMAAAAAEAykRkAAAAAgGQiMwAAAAAAyURmAAAAAACSicwAAAAAACQTmQEAAAAASCYyAwAAAACQTGQGAAAAACCZyAwAwD6lf//+MWrUqL09jSrvxRdfjIKCgliyZMnengoAAJWcyAwAwG5z4403RoMGDeK9997L7duwYUPUrFkz+vfvnzd29uzZUVBQECtXrtzDs6w8Vq1aFV/72teiZcuWUbt27WjVqlWcdNJJ8Y9//GNvTw0AAHaZyAwAwG4zYMCA2LBhQyxcuDC3729/+1sUFxfHggUL4t13383tnzVrVrRp0yY6duxY7uNkWZYXsiu7LVu27HDfcccdF+vWrYs//elPsXz58rjzzjvj05/+dKxdu3bPTxIAABKJzAAA7DZdunSJFi1axOzZs3P7Zs+eHSeddFK0b98+5s+fn7d/wIABERFRWloa5513XjRr1ixq164dRx11VDzxxBN5YwsKCmL69OnRs2fPKCwsjL///e+xcePG+MY3vhH169ePFi1axI9+9KPt5vSLX/wiOnfuHLVr147mzZvHV77ylZ3Of8qUKdGoUaO45557cq8ZNGhQrF69Om/cvffeG4ceemjUrl07OnToEOPHj8+L3gUFBTF58uT4/Oc/H/Xq1Ysf/OAH2x1r2bJlsXLlyvjFL34RRxxxRLRt2zaOPPLIuOqqq+KII47Ijbv44ovjwAMPjLp160aHDh3isssuy4vW48aNi0MOOSR+9atfRZs2baJ+/fpx9tlnx9atW+Paa6+N4uLiaNas2XZz2DbHwYMHR506daJDhw5x99137/SziYh45plnYvDgwVG/fv1o3rx5DBs2LN54440PfQ0AAPs+kRkAgN1qwIABMWvWrNz2rFmzon///tGvX7/c/nfeeScWLFiQi8wXXXRR/PGPf4xf//rXsXjx4ujUqVMMGjQo3nrrrbz3vuSSS2LixInx3HPPRffu3WP06NExZ86cuPfee+Ohhx6K2bNnx+LFi3PjFy5cGOedd15cccUVsXz58pgxY0b07dv3Q+e/adOm+MEPfhC/+c1v4tFHH421a9fGKaecknv+b3/7W3zjG9+I888/P5599tm46aabYsqUKdtF3HHjxsUXv/jFWLp0aXzzm9/c7jhNmzaNatWqxd133x1bt27d6XwaNGgQU6ZMiWeffTZ+8pOfxC9/+cu4/vrr88asXLkypk+fHjNmzIg77rgjbr311hgyZEj8+9//jjlz5sQ111wT3//+92PBggV5r7vsssviy1/+cjz11FMxdOjQOOWUU+K5557b4TzWrl0bxxxzTHzmM5+JhQsXxowZM+L111+Pr371qx/6eQIA8AmQAQDAbvTLX/4yq1evXrZly5aspKQkq1GjRrZmzZrs97//fda3b98sy7Ls4YcfziIi+9e//pVt2LAhq1mzZnb77bfn3mPz5s1Zy5Yts2uvvTbLsiybNWtWFhHZPffckxuzfv36rFatWtkf/vCH3L4333wzq1OnTnb++ednWZZlf/zjH7OioqKspKRkl+Z+2223ZRGRzZ8/P7fvueeeyyIiW7BgQZZlWXbsscdmV199dd7rfvvb32YtWrTIbUdENmrUqI883s9//vOsbt26WYMGDbIBAwZkV1xxRbZy5coPfc11112X9ezZM7c9duzYrG7dunnnOGjQoKxdu3bZ1q1bc/u6dOmSTZgwIW+OZ511Vt579+7dO/vOd76TZVmWrVq1KouI7Mknn8yyLMuuvPLK7Pjjj88bv3r16iwisuXLl3/kuQIAsO9yJTMAALtV//79Y+PGjfHEE0/E3/72tzjwwAOjadOm0a9fv9x9mWfPnh0dOnSINm3axMqVK2PLli1x5JFH5t6jZs2acfjhh293VW2vXr1yf165cmVs3rw5evfundvXpEmT6NKlS277uOOOi7Zt20aHDh1i2LBhcfvtt8emTZs+dP41atSIww47LLfdtWvXaNSoUW4uTz31VFxxxRVRv3793ONb3/pWvPrqq3nv/f657szIkSPjtddei9tvvz369OkTd911V3Tr1i1mzpyZG3PnnXfGkUceGcXFxVG/fv34/ve/Hy+99FLe+7Rr1y4aNGiQ227evHkcdNBBUa1atbx9a9asyXtdnz59ttve2ZXMTz31VMyaNSvvvLt27RoR8Yn+8kYAANwuAwCA3axTp07RqlWrmDVrVsyaNSv69esXEREtW7aM1q1bx7x582LWrFlxzDHHlPu969WrV67xDRo0iMWLF8cdd9wRLVq0iMsvvzx69Ojxsb5Yb8OGDTF+/PhYsmRJ7rF06dJYsWJF1K5du9xzbdCgQZx44onxgx/8IJ566qk4+uij46qrroqIiMceeyyGDh0an/3sZ+O+++6LJ598Mi699NLYvHlz3nvUrFkzb7ugoGCH+8rKylJOOSL+e94nnnhi3nkvWbIkVqxY8ZG3IAEAYN8mMgMAsNsNGDAgZs+eHbNnz47+/fvn9vft2zemT58ejz/+eO5+zB07doxatWrFo48+mhu3ZcuWeOKJJ+Kggw7a6TE6duwYNWvWzLvP8Ntvvx3PP/983rgaNWrEwIED49prr42nn346XnzxxXjkkUd2+r7vvfdeLFy4MLe9fPnyWLt2bXzqU5+KiIhDDz00li9fHp06ddru8f4rh1MUFBRE165dY+PGjRERMW/evGjbtm1ceuml0atXr+jcuXP861//+ljHeL/3fxHjtu1t5/lBhx56aCxbtizatWu33XmXN/4DALBvqbG3JwAAwL5nwIABMXLkyNiyZUvuSuaIiH79+sU555wTmzdvzkXmevXqxXe+850YPXp0NGnSJNq0aRPXXnttbNq0KU4//fSdHqN+/fpx+umnx+jRo2O//faLZs2axaWXXpoXeu+777745z//GX379o3GjRvHAw88EGVlZXm31PigmjVrxrnnnhs//elPo0aNGnHOOefEEUccEYcffnhERFx++eXxuc99Ltq0aRNf+cpXolq1avHUU0/FM888k7sCeVcsWbIkxo4dG8OGDYuDDjooatWqFXPmzIlf/epXcfHFF0dEROfOneOll16KqVOnxmGHHRb3339/TJs2bZeP8VHuuuuu6NWrVxx11FFx++23x+OPPx633nrrDseOHDkyfvnLX8app54aF110UTRp0iReeOGFmDp1atxyyy1RvXr13TYvAACqFpEZAIDdbsCAAfHOO+9E165do3nz5rn9/fr1i/Xr10eXLl2iRYsWuf0TJ06MsrKyGDZsWKxfvz569eoVDz74YDRu3PhDj3PdddflbuPQoEGDuPDCC2PdunW55xs1ahR/+tOfYty4cfHuu+9G586d44477ohu3brt9D3r1q0bF198cXzta1+Ll19+OY4++ui88Dpo0KC477774oorrohrrrkmatasGV27do0zzjijXJ9Rq1atol27djF+/Ph48cUXo6CgILf93e9+NyIiPv/5z8d3v/vdOOecc6K0tDSGDBkSl112WYwbN65cx9qZ8ePHx9SpU+Pss8+OFi1axB133LHTq8dbtmwZjz76aFx88cVx/PHHR2lpabRt2zZOOOGEj30FNwAAVVtBlmXZ3p4EAABUBlOmTIlRo0Z9rHs2VxUFBQUxbdq0+MIXvrC3pwIAQBXnkgMAAAAAAJKJzAAAAAAAJHO7DAAAAAAAkrmSGQAAAACAZCIzAAAAAADJRGYAAAAAAJKJzAAAAAAAJBOZAQAAAABIJjIDAAAAAJBMZAYAAAAAIJnIDAAAAABAsv8PYeRfOK8a2BYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "max_len = max(data['input_len'])\n",
    "\n",
    "print(f\"The maximum length of this dataset is approximately {max_len} characters.\")\n",
    "\n",
    "n = max_len\n",
    "data.hist(\"input_len\", grid=False, figsize=(18,8), range=(0, n + 1), bins=n + 1)\n",
    "plt.suptitle('')\n",
    "plt.title('Histogram of Text Length')\n",
    "plt.ylabel('')\n",
    "plt.xlabel('Words per Sample')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Drop Unnecessary Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Question: How can I talk to a real person? Ans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Question: How do I contact customer service? A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Question: How do I file a complaint? Answer: W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Question: How to file a complaint? Answer: You...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Question: What do I do if my package was damag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Question: How can I pay my bill? Answer: You h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Question: How can I get help with my bill? Ans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Question: What do I do if I’m locked out of my...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Question: How do I find out my account number?...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Question: How do I create a FedEx account? Ans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Question: I submitted a request to change my d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Question: How can I stop the delivery of my Fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Question: How can I request to have a FedEx Ex...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Question: Did You Know? Answer: You can use th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Question: How do I know what time the package ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Question: How can I change or correct the deli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Question: How can I report a missing FedEx Exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Question: When I track my FedEx Express® packa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Question: How do I know what time the package ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Question: How can I add delivery instructions ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Question: When and where can I pick up my pack...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Question: How can I manage my incoming deliver...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Question: How can I schedule a delivery time f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Question: How can I request to have my FedEx® ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Question: What does the “delivery exception” s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Question: What does the “delivery now pending”...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Question: If I missed a delivery, how do I kno...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Question: How can I report a missing FedEx Exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Question: What do I do if I missed my delivery...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Question: Did You Know? Answer: With FedEx Del...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Question: What do I do if I don’t have a track...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Question: Why isn’t my package moving when I c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Question: How can I track the exact location o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Question: How do I know what time my FedEx® pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Question: Did You Know? Answer: You use FedEx ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Question: When will my package arrive? Answer:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Question: Did You Know? Answer: Did you know? ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                input\n",
       "0   Question: How can I talk to a real person? Ans...\n",
       "1   Question: How do I contact customer service? A...\n",
       "2   Question: How do I file a complaint? Answer: W...\n",
       "3   Question: How to file a complaint? Answer: You...\n",
       "4   Question: What do I do if my package was damag...\n",
       "5   Question: How can I pay my bill? Answer: You h...\n",
       "6   Question: How can I get help with my bill? Ans...\n",
       "7   Question: What do I do if I’m locked out of my...\n",
       "8   Question: Did You Know? Answer: Did you know? ...\n",
       "9   Question: How do I find out my account number?...\n",
       "10  Question: Did You Know? Answer: Did you know? ...\n",
       "11  Question: How do I create a FedEx account? Ans...\n",
       "12  Question: Did You Know? Answer: Did you know? ...\n",
       "13  Question: I submitted a request to change my d...\n",
       "14  Question: Did You Know? Answer: Did you know? ...\n",
       "15  Question: How can I stop the delivery of my Fe...\n",
       "16  Question: Did You Know? Answer: Did you know? ...\n",
       "17  Question: How can I request to have a FedEx Ex...\n",
       "18  Question: Did You Know? Answer: You can use th...\n",
       "19  Question: How do I know what time the package ...\n",
       "20  Question: Did You Know? Answer: Did you know? ...\n",
       "21  Question: How can I change or correct the deli...\n",
       "22  Question: Did You Know? Answer: Did you know? ...\n",
       "23  Question: How can I report a missing FedEx Exp...\n",
       "24  Question: When I track my FedEx Express® packa...\n",
       "25  Question: How do I know what time the package ...\n",
       "26  Question: Did You Know? Answer: Did you know? ...\n",
       "27  Question: How can I add delivery instructions ...\n",
       "28  Question: When and where can I pick up my pack...\n",
       "29  Question: How can I manage my incoming deliver...\n",
       "30  Question: How can I schedule a delivery time f...\n",
       "31  Question: How can I request to have my FedEx® ...\n",
       "32  Question: What does the “delivery exception” s...\n",
       "33  Question: What does the “delivery now pending”...\n",
       "34  Question: If I missed a delivery, how do I kno...\n",
       "35  Question: How can I report a missing FedEx Exp...\n",
       "36  Question: What do I do if I missed my delivery...\n",
       "37  Question: Did You Know? Answer: With FedEx Del...\n",
       "38  Question: What do I do if I don’t have a track...\n",
       "39  Question: Why isn’t my package moving when I c...\n",
       "40  Question: How can I track the exact location o...\n",
       "41  Question: How do I know what time my FedEx® pa...\n",
       "42  Question: Did You Know? Answer: You use FedEx ...\n",
       "43  Question: When will my package arrive? Answer:...\n",
       "44  Question: Did You Know? Answer: Did you know? ..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.drop(columns=[\"input_len\", \"Question\", \"Answer\"])\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert DataFrame to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input'],\n",
       "    num_rows: 45\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = Dataset.from_pandas(data)\n",
    "\n",
    "ds"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Values/Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DialoGPT-large-FAQs-Tuned_v2\n"
     ]
    }
   ],
   "source": [
    "BLOCK_SIZE = 128\n",
    "MODEL_CKPT = \"microsoft/DialoGPT-large\"\n",
    "\n",
    "MODEL_NAME = MODEL_CKPT.split(\"/\")[-1] + \"-FAQs-Tuned_v2\"\n",
    "set_seed = 42\n",
    "\n",
    "BATCH_SIZE = 500 \n",
    "NUM_OF_EPOCHS = 20\n",
    "\n",
    "WEIGHT_DECAY = 0.01\n",
    "STRATEGY = \"epoch\"\n",
    "\n",
    "REPORTS_TO = \"tensorboard\"\n",
    "LEARNING_RATE = 2e-5\n",
    "\n",
    "print(MODEL_NAME)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenize Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_CKPT, padding_side='left')\n",
    "\n",
    "def tokenizer_function(samples):\n",
    "    return tokenizer(samples[\"input\"])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Tokenizer Function to Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "762da19b0c184e7a9621b67bb70e5dd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/45 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1839 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask'],\n",
       "    num_rows: 45\n",
       "})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_ds = ds.map(tokenizer_function, \n",
    "                      batched=False,\n",
    "                      remove_columns=ds.column_names,)\n",
    "\n",
    "tokenized_ds"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess Tokenized Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_texts(samples):\n",
    "    # Concatenate all text\n",
    "    concatenated_examples = {k: sum(samples[k], []) for k in samples.keys()}\n",
    "    \n",
    "    # Determine length of all text's concatenated\n",
    "    total_length = len(concatenated_examples[list(samples.keys())[0]])\n",
    "    \n",
    "    # Determine number of blocks\n",
    "    total_length = (total_length // BLOCK_SIZE) * BLOCK_SIZE\n",
    "    \n",
    "    # Split text into chunks of block_size\n",
    "    result = {\n",
    "        k: [t[i : i + BLOCK_SIZE] for i in range(0, total_length, BLOCK_SIZE)]\n",
    "        for k, t in concatenated_examples.items()\n",
    "    }\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "    return result"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Preprocessing Function to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b291bcae218f470781bc45b5742fe747",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=4):   0%|          | 0/45 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clm_ds = tokenized_ds.map(\n",
    "    group_texts,\n",
    "    batched=True,\n",
    "    num_proc=4,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(MODEL_CKPT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=MODEL_NAME,\n",
    "    evaluation_strategy=STRATEGY,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    weight_decay=WEIGHT_DECAY,\n",
    "    num_train_epochs=NUM_OF_EPOCHS,\n",
    "    report_to=REPORTS_TO,\n",
    "    logging_first_step=True,\n",
    "    logging_strategy=STRATEGY,\n",
    "    save_strategy=STRATEGY,\n",
    "    hub_private_repo=True,\n",
    "    push_to_hub=True\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save Tokenizer & Push it to Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/DunnBC22/DialoGPT-large-FAQs-Tuned_v2/commit/2f16f0d9b4fb72b91ea5c1d8dc31d147ac855736', commit_message='Upload tokenizer', commit_description='', oid='2f16f0d9b4fb72b91ea5c1d8dc31d147ac855736', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(\"tokenizer\")\n",
    "tokenizer.push_to_hub(MODEL_NAME)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Data Collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/DialoGPT-large-FAQs-Tuned_v2 into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=clm_ds,\n",
    "    eval_dataset=clm_ds,\n",
    "    data_collator=data_collator\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/briandunn/Documents/deep_learning/dl/lib/python3.9/site-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16cca3bed17c49c7ab7311ee6278b6c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/220 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.2139, 'learning_rate': 1.9909090909090913e-05, 'epoch': 0.09}\n",
      "{'loss': 7.5377, 'learning_rate': 1.9e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ddd0a38c74a4db7b58b2db284d2e66e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 5.818904876708984, 'eval_runtime': 108.0189, 'eval_samples_per_second': 0.787, 'eval_steps_per_second': 0.102, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Adding files tracked by Git LFS: ['.DS_Store']. This may take a bit of time if the files are large.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 5.4966, 'learning_rate': 1.8e-05, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "376a0451579649bbb5e86e68b5eb0748",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 4.189695835113525, 'eval_runtime': 107.9932, 'eval_samples_per_second': 0.787, 'eval_steps_per_second': 0.102, 'epoch': 2.0}\n",
      "{'loss': 4.2105, 'learning_rate': 1.7e-05, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fdbfceb41ed45ac9ef107d5af49887f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.307358741760254, 'eval_runtime': 107.4434, 'eval_samples_per_second': 0.791, 'eval_steps_per_second': 0.102, 'epoch': 3.0}\n",
      "{'loss': 3.4495, 'learning_rate': 1.6000000000000003e-05, 'epoch': 4.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a60b19a1f9f84cf3a53b4f9c5f6a4996",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.732468843460083, 'eval_runtime': 108.0362, 'eval_samples_per_second': 0.787, 'eval_steps_per_second': 0.102, 'epoch': 4.0}\n",
      "{'loss': 2.9218, 'learning_rate': 1.5000000000000002e-05, 'epoch': 5.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8162c54a4fa04d26911f7c602e059aec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.2970614433288574, 'eval_runtime': 108.6261, 'eval_samples_per_second': 0.783, 'eval_steps_per_second': 0.101, 'epoch': 5.0}\n",
      "{'loss': 2.4316, 'learning_rate': 1.4e-05, 'epoch': 6.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de61dc2db20943f79e27543e78c8df96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.9358689785003662, 'eval_runtime': 107.2349, 'eval_samples_per_second': 0.793, 'eval_steps_per_second': 0.103, 'epoch': 6.0}\n",
      "{'loss': 2.1323, 'learning_rate': 1.3000000000000001e-05, 'epoch': 7.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e666139c90964beba55b710a03d41bbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.6539617776870728, 'eval_runtime': 107.9108, 'eval_samples_per_second': 0.788, 'eval_steps_per_second': 0.102, 'epoch': 7.0}\n",
      "{'loss': 1.8443, 'learning_rate': 1.2e-05, 'epoch': 8.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21323e1d48464dd383afac3a9dd4d8b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.4227427244186401, 'eval_runtime': 107.6831, 'eval_samples_per_second': 0.789, 'eval_steps_per_second': 0.102, 'epoch': 8.0}\n",
      "{'loss': 1.6379, 'learning_rate': 1.1000000000000001e-05, 'epoch': 9.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb4fc1323fd246bc826fc2897f72fdfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.2491848468780518, 'eval_runtime': 106.9677, 'eval_samples_per_second': 0.795, 'eval_steps_per_second': 0.103, 'epoch': 9.0}\n",
      "{'loss': 1.496, 'learning_rate': 1e-05, 'epoch': 10.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16d66c9dbbd146df95f24194fa1220ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.0815372467041016, 'eval_runtime': 107.1591, 'eval_samples_per_second': 0.793, 'eval_steps_per_second': 0.103, 'epoch': 10.0}\n",
      "{'loss': 1.3339, 'learning_rate': 9e-06, 'epoch': 11.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4004dbd59b514c818fb3494f9eb7c09f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.968041181564331, 'eval_runtime': 107.9521, 'eval_samples_per_second': 0.787, 'eval_steps_per_second': 0.102, 'epoch': 11.0}\n",
      "{'loss': 1.2226, 'learning_rate': 8.000000000000001e-06, 'epoch': 12.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b90abc2754849c19878c55e972c268e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.86135333776474, 'eval_runtime': 107.4336, 'eval_samples_per_second': 0.791, 'eval_steps_per_second': 0.102, 'epoch': 12.0}\n",
      "{'loss': 1.1263, 'learning_rate': 7e-06, 'epoch': 13.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eac7adede13b4d6fbea21ce75d0c6135",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7849896550178528, 'eval_runtime': 107.4846, 'eval_samples_per_second': 0.791, 'eval_steps_per_second': 0.102, 'epoch': 13.0}\n",
      "{'loss': 1.0404, 'learning_rate': 6e-06, 'epoch': 14.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a996a83d4e6c4b8c9def80ffc9da6994",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7081905007362366, 'eval_runtime': 107.0061, 'eval_samples_per_second': 0.794, 'eval_steps_per_second': 0.103, 'epoch': 14.0}\n",
      "{'loss': 0.9794, 'learning_rate': 5e-06, 'epoch': 15.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35199c4cc8fa48ba80275bb7f4dc3408",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6539128422737122, 'eval_runtime': 109.0894, 'eval_samples_per_second': 0.779, 'eval_steps_per_second': 0.101, 'epoch': 15.0}\n",
      "{'loss': 0.9351, 'learning_rate': 4.000000000000001e-06, 'epoch': 16.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5648bac2c6754dbfb6c5d261930b785d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6111845374107361, 'eval_runtime': 106.7368, 'eval_samples_per_second': 0.796, 'eval_steps_per_second': 0.103, 'epoch': 16.0}\n",
      "{'loss': 0.8775, 'learning_rate': 3e-06, 'epoch': 17.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c220131a53a94ed7b0b0810cedb9bb90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5726211667060852, 'eval_runtime': 107.0777, 'eval_samples_per_second': 0.794, 'eval_steps_per_second': 0.103, 'epoch': 17.0}\n",
      "{'loss': 0.855, 'learning_rate': 2.0000000000000003e-06, 'epoch': 18.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a883aaa22bd44c1987773791f0c5b947",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5511290431022644, 'eval_runtime': 107.4179, 'eval_samples_per_second': 0.791, 'eval_steps_per_second': 0.102, 'epoch': 18.0}\n",
      "{'loss': 0.8269, 'learning_rate': 1.0000000000000002e-06, 'epoch': 19.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3720d5877e4b43cc826d9a9df55b4f22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5350754857063293, 'eval_runtime': 107.1959, 'eval_samples_per_second': 0.793, 'eval_steps_per_second': 0.103, 'epoch': 19.0}\n",
      "{'loss': 0.8218, 'learning_rate': 0.0, 'epoch': 20.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1428ab9733944a0dbcfb23d84311af37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5300764441490173, 'eval_runtime': 106.7251, 'eval_samples_per_second': 0.796, 'eval_steps_per_second': 0.103, 'epoch': 20.0}\n",
      "{'train_runtime': 13665.8658, 'train_samples_per_second': 0.124, 'train_steps_per_second': 0.016, 'train_loss': 2.1710222547704525, 'epoch': 20.0}\n"
     ]
    }
   ],
   "source": [
    "train_results = trainer.train()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d7fdd9fde304ed0889829746bb60b2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity: 1.70\n",
      "***** eval metrics *****\n",
      "  epoch                   =       20.0\n",
      "  eval_loss               =     0.5301\n",
      "  eval_runtime            = 0:01:46.93\n",
      "  eval_samples_per_second =      0.795\n",
      "  eval_steps_per_second   =      0.103\n"
     ]
    }
   ],
   "source": [
    "metrics = trainer.evaluate()\n",
    "print(f\"Perplexity: {math.exp(metrics['eval_loss']):.2f}\")\n",
    "\n",
    "trainer.log_metrics(\"eval\", metrics)\n",
    "trainer.save_metrics(\"eval\", metrics)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save Model to Repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Several commits (2) will be pushed upstream.\n",
      "The progress bars may be unreliable.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae9213790b594292919d8f849e02b015",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file pytorch_model.bin:   0%|          | 1.00/2.92G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f41c4e8fb83a47478c66feab6cdde7a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/.DS_Store:   0%|          | 1.00/6.00k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6aea731844c4ecd90e2ef6976199a77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Apr19_22-42-22_Brians-Mac-mini.local/events.out.tfevents.1681962152.Brians-Mac-mini.local.120…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "164b8162199840e0821a892c877be38b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Apr19_22-42-22_Brians-Mac-mini.local/events.out.tfevents.1681975925.Brians-Mac-mini.local.120…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9769fbc85a024c4baa5752f066d220ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file .DS_Store:   0%|          | 1.00/12.0k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/DunnBC22/DialoGPT-large-FAQs-Tuned_v2\n",
      "   fb071f8..cf97b7b  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55fd68ff3f6a46f1b12565641cf08cee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file .DS_Store: 100%|##########| 12.0k/12.0k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/DunnBC22/DialoGPT-large-FAQs-Tuned_v2\n",
      "   cf97b7b..506667a  main -> main\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** train metrics *****\n",
      "  epoch                    =       20.0\n",
      "  train_loss               =      2.171\n",
      "  train_runtime            = 3:47:45.86\n",
      "  train_samples_per_second =      0.124\n",
      "  train_steps_per_second   =      0.016\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model()\n",
    "trainer.log_metrics(\"train\", train_results.metrics)\n",
    "trainer.save_metrics(\"train\", train_results.metrics)\n",
    "trainer.save_state()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Push Model to HuggingFace Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c0616afc6714daa9a0fce9525692211",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file .DS_Store: 100%|##########| 12.0k/12.0k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/DunnBC22/DialoGPT-large-FAQs-Tuned_v2\n",
      "   506667a..c568c58  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c27f96367c0c482292effd6351b28215",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file .DS_Store: 100%|##########| 12.0k/12.0k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/DunnBC22/DialoGPT-large-FAQs-Tuned_v2\n",
      "   c568c58..89e451f  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/DialoGPT-large-FAQs-Tuned_v2/commit/c568c5847b15a70b79cc3173cf9730ce3889dc4c'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notes & Other Takeaways From This Project\n",
    "****\n",
    "- This is definitely the model to use for the FAQ chatbot!\n",
    "****"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp_ml",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0b7a754c5aaaa44a21e2f2148f50b931a5abdffe7ef71537f5954179554f39df"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}