Optical Character Recognition (OCR)/Captcha/OCR_captcha.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OCR of Captcha Images\n",
    "\n",
    "Dataset Source: https://www.kaggle.com/datasets/alizahidraja/captcha-data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Install Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: torch in ./nlp_ch4/lib/python3.9/site-packages (1.12.1)\n",
      "Requirement already satisfied: torchvision in ./nlp_ch4/lib/python3.9/site-packages (0.13.1)\n",
      "Requirement already satisfied: torchaudio in ./nlp_ch4/lib/python3.9/site-packages (0.12.1)\n",
      "Requirement already satisfied: typing-extensions in ./nlp_ch4/lib/python3.9/site-packages (from torch) (4.3.0)\n",
      "Requirement already satisfied: requests in ./nlp_ch4/lib/python3.9/site-packages (from torchvision) (2.28.1)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in ./nlp_ch4/lib/python3.9/site-packages (from torchvision) (9.2.0)\n",
      "Requirement already satisfied: numpy in ./nlp_ch4/lib/python3.9/site-packages (from torchvision) (1.22.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./nlp_ch4/lib/python3.9/site-packages (from requests->torchvision) (1.26.11)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./nlp_ch4/lib/python3.9/site-packages (from requests->torchvision) (2022.6.15)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in ./nlp_ch4/lib/python3.9/site-packages (from requests->torchvision) (2.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in ./nlp_ch4/lib/python3.9/site-packages (from requests->torchvision) (3.3)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.2.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m22.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.2.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m22.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install torch torchvision torchaudio\n",
    "%pip install -q datasets jiwer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Import Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "NOTE: Redirects are currently not supported in Windows or MacOs.\n"
     ]
    }
   ],
   "source": [
    "import os, sys, itertools\n",
    "os.environ['TOKENIZERS_PARALLELISM']='false'\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "import datasets\n",
    "from datasets import load_dataset\n",
    "\n",
    "import transformers\n",
    "from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "from transformers import VisionEncoderDecoderModel, TrOCRProcessor, default_data_collator\n",
    "\n",
    "import evaluate"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Display Versions of Relevant Software & Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Python: 3.9.12\n",
      "        Pandas: 1.4.3\n",
      "      Datasets: 2.4.0\n",
      "  Transformers: 4.22.1\n",
      "         Torch: 1.12.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Python:\".rjust(15), sys.version[0:6])\n",
    "print(\"Pandas:\".rjust(15), pd.__version__)\n",
    "print(\"Datasets:\".rjust(15), datasets.__version__)\n",
    "print(\"Transformers:\".rjust(15), transformers.__version__)\n",
    "print(\"Torch:\".rjust(15), torch.__version__)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Ingest & Preprocess Training DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>p5g5m.png</td>\n",
       "      <td>p5g5m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e72cd.png</td>\n",
       "      <td>e72cd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pgmn2.png</td>\n",
       "      <td>pgmn2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6cm6m.png</td>\n",
       "      <td>6cm6m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>68wfd.png</td>\n",
       "      <td>68wfd</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_name   text\n",
       "0  p5g5m.png  p5g5m\n",
       "1  e72cd.png  e72cd\n",
       "2  pgmn2.png  pgmn2\n",
       "3  6cm6m.png  6cm6m\n",
       "4  68wfd.png  68wfd"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_path = \"/Users/briandunn/Desktop/Vit Image Datasets/OCR/captcha data/train\"\n",
    "train_dir_list = os.listdir(train_path)\n",
    "\n",
    "train_dataset = pd.DataFrame(train_dir_list, columns=['file_name'])\n",
    "\n",
    "train_dataset['text'] = train_dataset['file_name'].map(lambda x: x.split(\".\")[0])\n",
    "\n",
    "train_dataset.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Ingest & Preprocess Testing DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c4527.png</td>\n",
       "      <td>c4527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>n8ydd.png</td>\n",
       "      <td>n8ydd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c6we6.png</td>\n",
       "      <td>c6we6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mmg2m.png</td>\n",
       "      <td>mmg2m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5nm6d.png</td>\n",
       "      <td>5nm6d</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_name   text\n",
       "0  c4527.png  c4527\n",
       "1  n8ydd.png  n8ydd\n",
       "2  c6we6.png  c6we6\n",
       "3  mmg2m.png  mmg2m\n",
       "4  5nm6d.png  5nm6d"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_path = \"/Users/briandunn/Desktop/Vit Image Datasets/OCR/captcha data/test\"\n",
    "test_dir_list = os.listdir(test_path)\n",
    "\n",
    "test_dataset = pd.DataFrame(test_dir_list, columns=['file_name'])\n",
    "\n",
    "test_dataset['text'] = test_dataset['file_name'].map(lambda x: x.split(\".\")[0])\n",
    "\n",
    "test_dataset.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create Dataset Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Captcha_Dataset(Dataset):\n",
    "    \n",
    "    def __init__(self, root_dir, df, processor, max_target_length=128):\n",
    "        self.root_dir = root_dir\n",
    "        self.df = df\n",
    "        self.processor = processor\n",
    "        self.max_target_length = max_target_length\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        # get file name + text\n",
    "        file_name = self.df['file_name'][idx]\n",
    "        text = self.df['text'][idx]\n",
    "        # prepare image (i.e. resize + normalize)\n",
    "        image = Image.open(self.root_dir + file_name).convert(\"RGB\")\n",
    "        pixel_values = self.processor(image, return_tensors=\"pt\").pixel_values\n",
    "        # add labels (input_ids) by encoding the text\n",
    "        labels = self.processor.tokenizer(text, padding=\"max_length\", max_length=self.max_target_length).input_ids\n",
    "        # important: make sure that PAD tokens are ignored by the loss function\n",
    "        labels = [label if label != self.processor.tokenizer.pad_token_id \n",
    "                  else -100 for label in labels]\n",
    "        \n",
    "        encoding = {\"pixel_values\" : pixel_values.squeeze(), \"labels\" : torch.tensor(labels)}\n",
    "        return encoding"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Basic Values/Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_CKPT = \"microsoft/trocr-base-printed\"\n",
    "MODEL_NAME =  MODEL_CKPT.split(\"/\")[-1] + \"_captcha_ocr\"\n",
    "NUM_OF_EPOCHS = 3"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Instantiate Processor, Create Training, & Testing Dataset Instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "processor = TrOCRProcessor.from_pretrained(MODEL_CKPT)\n",
    "\n",
    "train_ds = Captcha_Dataset(root_dir=\"/Users/briandunn/Desktop/Vit Image Datasets/OCR/captcha data/train/\",\n",
    "                             df=train_dataset,\n",
    "                             processor=processor)\n",
    "\n",
    "test_ds = Captcha_Dataset(root_dir=\"/Users/briandunn/Desktop/Vit Image Datasets/OCR/captcha data/test/\",\n",
    "                             df=test_dataset,\n",
    "                             processor=processor)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Print Length of Training & Testing Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The training dataset has 856 samples in it.\n",
      "The testing dataset has 214 samples in it.\n"
     ]
    }
   ],
   "source": [
    "print(f\"The training dataset has {len(train_ds)} samples in it.\")\n",
    "print(f\"The testing dataset has {len(test_ds)} samples in it.\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Example of Input Data Shapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pixel_values  :  torch.Size([3, 384, 384])\n",
      "labels  :  torch.Size([128])\n"
     ]
    }
   ],
   "source": [
    "encoding = train_ds[10]\n",
    "\n",
    "for k,v in encoding.items():\n",
    "    print(k, \" : \", v.shape)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Show Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMgAAAAyCAIAAACWMwO2AAApoklEQVR4nO18eXBc1ZX+fUvv/XpRL5JamyVrsyUk5AVb3ndhbIyDkxBDCJgwBAjlYTIzgcykJuUxVFE1GEgCTBmmjImxIcaAscHIYC1eZGzhVbKs1VpaW6v37b3ufv26+/fHN+5yZX6piYKcOFO+f6ik1uu33HfuOd/5zncu1dzcTAghhFAUlf554/gTP/+DX6b8+P+r5/m/ev80uT1uj5swbhvW7XFTxm3Duj1uyrhtWLfHTRm3Dev2uCnjtmHdHjdl3Das2+OmjD9qWNQkx2TP88eOl8lkcrmcpmlRFCVJwsGSJLEsy7JsJBKRyWSSJCmVykQiQQiJxWIMw0iSlP56+ltTcv8ymYyiqPRPhmFEUaRpOplMKpXKZDJJ07QkSTRNJxIJ3Hw0Gk0mkyzLyuXyVColiqJCoZiqeZuq8zAMgydSq9UKhSKRSCQSCeb6UCqV0WgUPxOJRDKZnOx1bzmPFQgEIpEIIUSlUqnVaqVSqVAo8LZYltXr9QqFQhTFUCgkSZJCodDr9VqtNh6PY0YEQZDL5YFAYKruh6KoWCwWj8dpmvb7/Q6Hw2Aw0DStUCjC4bDBYIhGowMDAwqFghASj8fD4bDZbDYajeFw2O/3K5VKjuPwRLfUoCiKpuloNOrxeMLhsFwuVygUeMxQKOT3+1mWxZ9yuVyj0Uz2/OzNuOlvM7RaLfwB1gpN0xRFpVIplUolSZIoioQQGJwoivF43OfzyeVyjuNkMlk4HE6lUhRFmUymtA/7lsPn85lMJoVCEQgEKIqCZb/33nstLS2hUGjt2rWlpaWRSAQuKhgMajQat9udSqU4jpPL5cFgMJVKGY3GW822otGoTCZTqVQqlSqVShFCkskkwzCEEK1Wi/jgdrtVKlUikfB4PJO1LeqPlXSov16JgKZpPGcqlaJpGgEokUiIohiNRhmGkcvlKpUqFotFo1GbzRYMBmOxmEwmQ6CUyWTxeFwul0/J/ev1+k8//TQ/P7+goMDj8Xz44YeCIJSWlubl5TkcjvHxcZVKVVtbu2fPng0bNuh0OoVCUVJSEo/HBUHgOE6tVkuS5PP50i/mJs3bZM+DpZJIJLRaLYJAMpmUyWR+v99iscRisd7e3vPnz9fW1paXl8fj8WQyOanr3nIeSxRFhmEAVlKplCRJkUgklUrBmHQ6XSQSSSQS0WhULpfL5fKxsTG1Wh2LxSKRSEdHh0wms1gshYWFU3U/+/fvP3PmTHZ29ltvvRUMBq1Wq8PhWLZsGRCJ1+uVJOnEiRPPP//8zp07lyxZUl9fv3Xr1pqaGpi+x+OBV5iq+5mqEYvFNBoNy7KEEEmS4LQSiYQkSfv27du5c2dLSwvDMDt27MjOzv4zQuEt57EQ4wCHEQQJIcDyLMsqlUpBENRqNaJkOBxOJBIZGRnbt2/neb6pqclsNv/4xz+uq6tLv8tvef/hcLizs/ORRx5JJpPFxcU9PT0ajSYUCuXm5jocjqqqKp7nXS6XUqmcP39+U1PTvn37qqqqfD4fwzAmkwmYj6IopBo3b94mex54IFEUE4mESqUKBoPNzc1ffvnlBx98ABQxa9asurq6H/7whyUlJTzPK5XKyV33VjMslUoFnySXy2UyWTKZxHpSqVRwZsFgUKfTBQIBr9fb399/7NixQ4cO5efn2+12QsjBgwe3bdu2a9cuoOlvf//PP//88ePHEYjD4bBCoYjFYk8++eTKlSu/973vEUK0Wm1OTk53d/fixYt1Op3D4Xj77bdnzpzpcrmi0SghJCMjIxQKwTfcvHmb7HkoilKr1YFAoKmpae/evV999VUoFCKEVFVVLVu2bPPmzVVVVTA45E94lj/9urdcKIRVATyGw+GBgYHu7m6Hw5GXl4cEvr+/PxAIXL16dWxsbGRkRK/XE0LsdrtarSaEjI+Pm83mNMD69mPr1q0+n6+hoYEQotFo5syZ09bW1tLSYrfbn3vuOZqmd+zYUVtbe+3atY6Ojh07dgwMDJhMJpfLJUlSRkYGz/OSJOGJpuqWpmQcPnz44MGDR48eDQQCDMPk5uY++uij3/ve98rLy7VarUqlcrlccrmcZVmKooLB4GSnlDp+/Ph///btVkA0GjUajYQQQRAIIcjsKIriOM7tdisUCrA+hJB4PE4IkSRJpVKBQQHzRAjBV4C+CSEsy168ePHZZ5/1+/1gE+RyOY7HFwVBwMEajYbneZZlly9f/g//8A9ZWVlarTYQCGRmZgaDQZZlY7GYUqlkWRY5Nj7R6/XA+7FYDDmRUqlMpVKxWIxl2WQyGQgEnnzySbvd7vF4EolEWVnZypUrs7KyrFar0+k8evRoRUWFy+Wy2+0vvvjis88+a7PZdu7cqVKpGIaJx+NIPiKRiEajEUXRYDBMTEwgr/R6vXq9Ph6PI1kRRRE4khCSSqVwG4lEAkggHo8jHUEqkEqlZDIZ5hBxNhwO0zStVqt5nieEGAwGQRB4nrdarXgdeCM8z1+6dOmDDz747LPPvF4vwzAFBQWLFy++//77ly5dijgw2aTnj9nDlC0jZPswizSqQEIEjiQSicRiMa1WC9fCMIzf749EIohZ6WQQU4k/k8lkKBQCeGdZNhqNxmIxXA4sJcMwKpUqHo8DfmVkZFgsFhBghBC5XB6LxRBANRqNIAhYeRqNBsbE8zxFUXhtyKth0BRFMQyD1GHbtm1vvPFGQ0NDIpHo7u5+4oknZs2a9f777z/++ONZWVnhcHjFihUtLS1dXV3PPPOMw+EYGxszGAxqtRpPjScCahFFkeM4QghyW1EUlUplKBRSKBS4epraFQQBv+NfaXuiaRosMdiNRCIRiUTwCFhCer2epmm32y2XyzMzM8PhMKb6ypUru3fv/uijj7xer0Kh4Dju6aefXrBgweLFi61WazwelySJ53lcaErsYcoMy2Qy+f1+4G7wk0qlkqbpjIwMpLIajUatVouiyPN8IpFQq9UMw2i1WoZhfD4fTdM6nU6SpFgshpUKLhvnJITggWmaBqEFH6PVaoPBYE5OzqpVqyorK4uLi7Ozs7Ozs6PRKHgaWHMikbBarbgE+HGsdZgmUm6GYWKxGGgwmHg0GvX7/b/4xS/Gx8eVSqUkSStWrNi0adPVq1c3bNhgtVrvueceEDw1NTU8z5tMJlDVYIaQouNCsVhMrVaDb4ShYyoikYhSqYQ3ksvloiiCW+J5XqfT4QwgwNL0t0qlYllWoVDQNI0DtFot7InneZ7nwXGoVCqfz9fW1rZ///7Tp093dHQQQmw223e/+93169cvWbLEaDTCs0qShFAAVw0q69uPKTMsh8NB07TBYJDJZHDIcrlckqTh4WGTyQTwC1uhKEoUxffee290dFQURbPZXFZWtnDhQo7jnE5nRkYGljjDMJgsvGl8kkwmeZ6naZrjuNzc3IKCgkceecRoNBoMBgQyQkgwGCSEpFKpSCTidDr9fj9iIkVRZWVlhJBoNIo0B0hcEASDwYD8CKkoIQSMhiAIu3bt2r179+uvv04IKSwsDIfDxcXFhJBwOJyTkwPyMC8vz2KxpJ9UEAQwQ3j3KpUK9oSTw9OkUimwJFqttq+vz+v1ZmRkoHJF03QaSOAr+BbAQLqowDAMLIwQ4na7g8FgQUEBKNn+/v6PPvro97///cjICJz98uXLH3300ZUrV3IcR9M0FmQ8Ho/FYkjAYaksyyJifPsxZYalVqtpmo7FYqDdgKgEQSgoKBAE4eLFizRNV1dXK5XKs2fPHjhwYGho6Nq1a5j30tJSv9+/Zs2aoqIiuA2sRbj9dK4+c+bM8vJyjuNycnJKSkqKiopsNtvIyIjZbMb7oChKkiRk+DRNd3Z2XrhwARgIi3ju3LllZWXFxcUGgwGcKiHkxvwRFTRcDuv4wIEDH374oVqtFgQhlUp1dnYGg8G6ujq5XB4KhYLBYFZWlsvlMhqNDMMMDg5i2ZSXl8MJCYKAZ4nH48gzWJYNhULRaFShUASDwaGhoZMnTyaTydmzZ2dnZwOToagAspcQAkNEmEZpjxACWhi5cyqVKi8vv3Llyp49ew4dOtTf3w92Zt68eY899tgdd9yB+8HcJhIJt9sN53ejc00z0lNiD1NmWDRNp1KpUChEUZTFYiGETExM+P3+nTt3nj9/vqen5/HHH7/jjjva2toOHDhw9OhRlmXTtdvLly9fvny5o6Pj6aefzs3NVSgUmNBUKsXzPNaQSqVau3btunXrVCqVUqmUy+U8zzscDrPZrFAokCrH43HEX5VKNTAwcOzYsa+//trn87lcLpqmlUpld3d3SUnJpk2bZs6cabfbbTabTCajadrj8Wi1Wixu2BaujmyuoqKisbFRLpdnZ2cPDQ0pFIrf/OY38+bNC4VCWVlZarUatbYDBw5Mnz49mUz29vamUqlUKpWZmYlsg+M4jUbjdDpFUYTBKRSKtra2pqam7u7ub775RqPRxGKxFStWaLVaQghMCiaOJADuihCSTCY9Hk8ymVSr1TabLRqNHjlyZP/+/VeuXEG8M5lMtbW169atu++++/Lz81FlYhgGSzRN7yFx8Xg88J2I9bDUKbGHKTOsaDQKFIVl+vHHH+/evdvhcAA6MAxTWFg4Ojr65ptvnj59Gg6AEIKFgj/PnDmj1WofffRRm82GMIqwgtcciUQATeCcZDIZx3EGg4HneZwETkitVodCIZ7nT5061dDQMDIygoVOCEkkEleuXGlrawsGg1VVVYODgwsXLkQC6Pf7jUajRqPJzc3Nzc2FaxEEwel0ulyujo4OrVYrSVJDQ8O9995bUFBw4cKFc+fOsSwrk8mys7MzMzMVCsXg4KBGo+ns7Dx06NDExEQqlaqoqMjPz5ckqbCwkKKopqYmq9UqSVJOTk5hYeHBgwc/+ugjl8sViURYli0tLa2oqMjKyiKEQLsBF4J4h2wjHA7r9XqWZYeHhxsbGw8ePNjW1gYMmkwm58yZ86Mf/Wjp0qUajcZgMHAc5/f79Xo9pggxxOv1EkJg5Uql0mAwYIWn3eEtB96BCfC7IAgDAwMOhyOd+1AUdfXqVZfL1dnZSa5TEhzHQaQAOOX3+48fP15XV2cwGKAzwXpNq2JkMhkoA2SLYBBUKlU0Gk3j3GQy6XQ6g8Hg+fPnIToA3CGEaLVaFO1bW1uHhoY8Ho/VakU45nl+YmIiJyentrZ28eLFZWVlCoVCEAS32/3WW2/V1dW1t7ePjo62t7d3dnZmZ2cnEgmFQpGdnX3p0qWtW7fi9W/evLmhoSGVSrnd7vHxceTIDoejr68vNzdXqVQ2Njbed999Ho/H4XAQQk6fPm232xF6lEplIBCA0ACRDrEMHjQej9vt9osXL3Z0dBw7dqynpwfpHhLhnJwco9G4ZcuWJUuWmEwmUJpIXDiOw59pbY/RaAT65DgO5TJguPRLvOmh8I/xE3jHCoWC5/m+vr62tra/+7u/Qxk8Go2mUimfz2c2m0VRBIdOrpeivF7vmTNnfD5fWVnZ/Pnzc3Nz5XJ5Q0PDpUuXQqEQvL3L5Tpw4MC//Mu/IJREIhHESkKIWq3G+VmW9Xg8yCghgsAUIy/z+Xz9/f0+n8/v96dvoKioKBgMbtq0KRAInDx50uVy9fX1aTSaS5cuTUxMAMgPDg4+++yzV65cOXPmjMViuXDhwty5c7u6urZt2/bFF18gISgqKkJFEmKYy5cvazSasrKyoqKiGTNmVFRUvPvuu/v27WNZ9vPPP6coqqioyGKxfPHFF01NTclkMhwOMwyj0+kef/zxPXv2XLp0iaIoOGzYUzAYDAQCFosFi2dkZOTcuXMnT55sbW212+2xWAxvfdq0aevXr6+tra2urs7Ly+M4Ls2+gtiDt4N1KhQKxHR8F54Jx6Qx5f983f/zvf+BYfzPz/9Uw/pjA0Hqo48+2rFjRyKR0Ol099xzT3Z2dloRRQiBNeClwo2Hw+GTJ08KglBWVvb9739/yZIlOTk50Wg0JyeHZdnTp0+nRTLHjx9fsWLFsmXLdDpdNBrlOA45vFqt7uvr+/zzz5H6oSwaDAYtFksqlSoqKoL5UhQ1NDTU3d0NkowQYjKZli1bxvN8fn7+ypUrv/Od71y7du3DDz+8dOnS119/fWOxorOzMxaLlZWVZWdn33vvvTzPL1++/MyZM1evXs3IyIjH4xMTEz6fD7lkmov6+OOPKYq69957N27cKIqiTqcLBoNqtVqr1fI8f+7cuUAg4Pf78abPnj27ZMmS5ubmY8eOAU2T6/4brGlLS8vnn3/e3d195swZQRDgmTQaTXV19dy5c0tLS1etWqVQKEAHpvnCNAN364xJG1YqlQJ3wnGcKIrBYPCrr776wQ9+AIzJsizoGSBuENz4YjgcLikpeeihh5YtW2a1Wt1uN8Mws2fP7unpaW9vR8Ki1WojkcjQ0BCmCcaBIo/b7T5y5AjqBCB74vF4IBCARmXx4sVqtbqmpmbWrFnJZHJoaAhgQqlUejwemqYXL14sSZLBYMjNzZ0+fXpvb28ikUBcBpOkVCr37t37ox/9aMGCBYFAQKfTEUKqq6sbGxuLiormzJnjcrlMJlN7e3s4HHa73U6nE7hnfHxcoVD4fL6urq7u7m6j0RgMBpHBtLa2Op1OSDHBQUiSNDo62tXVlUgk1q5dOzIycvHiRbBHZ8+e/fjjj5PJJA5TqVQVFRUrVqxAaIa6EFkbMGUkEklnNqBSp8wopmJM2rAYhjEajRUVFX6/nxBCUdTg4CCymLSAmGVZZB/4BNFTq9XOmzcvjQMg+zSbzfPmzWttbT179mwkEkHEaWtrW7JkSUZGBsMwHMdxHAdVDCwpjS7hxtvb2+PxuNvtRg1Hq9VGo1G73e5wOJD5E0JisdjcuXPj8ThESEqlcsWKFXfeeWd9ff3Ro0dRzInH44lEwmKxAHv5fL7x8XGLxaLX6wcGBubPn79ly5ZYLGY2m5VKpdPpPHjwIER8MP0jR44AzbS3t2dkZHi93uHhYUEQFi1aJElSZ2cnojZN09euXaNpuri4eGxs7NKlS3CryWRyeHjYaDQuWrRo6dKlNTU11dXVsCTIG3meh5oIc44KB6JbmnibGouYojFpw/L7/TzPZ2dnFxQUDA0NaTSa9vZ2ZPvJZFKlUlEUBf0kjsfyoml6xowZS5cuzczMhIe3Wq2hUGh8fNxkMuXk5BBCEGIYhrHb7aOjozU1NVqtFqVAYExEunQlEQgaJbbx8XHkjAhtqB5CUDpz5kytVutwOAoKCsCDE0Lmz58PiP3VV18lk8lEIlFQUDA+Pq7RaCRJcjqdeXl5+fn59fX1wWDwueeeY1m2ubk5KyururrabrezLLtx48bm5mb4PIqivF7vkSNH4J4Rv7BIzp49yzBMOBzGSoPGmhDS3d09NDSECgTKl3K5/KmnnnrsscfUajXHceCBURtA9iOTyViWFQQBn2NuUQjCeabEIKZqTNqwiouL+/v7KYqaM2eOw+EIh8NwD8XFxaA0w+FwLBaDZ4atEELUanVZWVllZSVwmF6vBzUAhgIpDFK8SCTi9/uHhoYEQcjJyVm5cqVGowkEAleuXOF5PhqNIpcJBoM8z6drGuDNe3t7eZ7PycmRyWQej4cQwrKszWZzOByHDx9evHhxIpFobW0VBMFkMo2Ojg4MDKDCQ9N0f38/y7Lbt2/X6XTgr3GGNDOCHPbGqaAoCoUXBGvwValUCmYUiUQKCws9Hk9+fj7DMJcvX+Y4Dt6lvLzcYDAsXLgwGo3u2LEDZ0OHiMViiUQigiAAXGNgccIiUcTEf0G7IIP+m8dYPT09JpOJZVmw5IQQURT/9V//9aWXXjKbzVarFcUQLCCU4YAGQqFQPB5HfwE4iEAgEI1Gh4eHh4eHCSFpEC1JUmNjYygU4jjO4/FAkYwcUJIkKGjTalp8BchDEIS+vj7EGng4SZJaW1vhPD799FOXy4XQnLYSZAbp6xJCeJ7XaDQKhWL27NkGgyEYDObn59M0rdFoUA1kWdZqtQ4NDYVCocHBwZaWFmjlAoEAfCQhJDMz0+VyLVq06LnnntNoNP/1X//V1dWFKyoUiqVLlz777LOCILzzzjurVq06ffq0XC73+/0DAwPDw8MWiwWcU3oQQhAH0eAAR4XZgPf6X3O0v/yYtGEhTfP5fPPmzbNYLC6XixDS19f34IMPokZbWFjIcRzAryRJiAuEEGTagiAMDQ2dP3++v7+/sbHR5/OhsJhmE5AcXLlypaurK01tpDMADKxUuVyu1+vD4XD6Eij0pgXvgC8Wi6W6uloUxaqqKpjFyMhIbm5uPB7v6+tTKpUNDQ2hUEij0dTV1c2ZM+ebb77Ztm0bOoIgbgmHw1qtFvmXKIomkwn8p8PhOHXqVEdHh8PhwOtPC1cKCgpWr17t8/kikci1a9ccDsfy5cvr6+tRjOvt7ZXJZBcuXBgfH1+zZk1jY2Naa2Q0GnU63ejoKNxbun6XSqUwP0CWN5YRCSHwan+uDdyUwf6vvMUfDJlMhtKYKIo/+clPxsbGrl27dvz4cVEUjx8//tVXX8FLpc8JDJRIJBoaGi5cuOD1eqFdge8hhABGQK2Ab0mSVFJSsm3btlgsptPp0D6lVCrNZnM6LqhUqmQyefXq1X379hkMhj179qRjVk5OTvpNIyEvLy8vLy+vq6tDMxMhxOVyDQ8Pt7e3r1+//ujRo2gN6uvrq6mpmTFjRl9f3+zZs2FVqHIwDON2u7OyshKJxOjoaFZWlsPh6O7uHhsbW716dW9v74kTJyiKAsROJBJOp3PRokUjIyOxWGzXrl3BYPCJJ55oaWmJRqMlJSUoCNrt9oGBgXvuuQfAi6Ko0dFRuE8sS3ID7QzOifz/CCdCCNDnpN7jlB//bfVYyKFSqVRGRkZdXZ1Wqx0dHZ0xY8a7775bVlZmMpkg4R0bGwP8NBgMACuBQCAtp0FoC4fD6H2gadrlcqVLvzRNm0ymadOmoQysVqtR3ECWB6tFOOA4ThCEFStWHDhwAKgWR2ZlZZnN5t7eXkEQBEE4ffr097///eHhYbRqaTQaq9XKMIzZbP7888+RunMc53K59u/ff+jQoc7OTrlcrtPpwuEwUrxEIlFSUnLt2rVkMllVVfXNN9/wPD9//vyJiYlr1661tbWhcgXjNplMnZ2dL774YnV1tSAIWq12aGjIaDSGQqHi4mKbzeZyuVKpFGAowBN0iz6fLzs7e2xsDNT5ZF/NLTUmzd8D4kBEADFaUVHRQw899Lvf/e6VV1555ZVXfvvb327durWyshKxA6wEJhHYxWAwmEymtAZNEASPx4NgBwlePB7X6XToySGEhMNhoGNBEBAOEokEKByz2azVaru6uvB1g8FQUlICaJ+Xl4cl7vV6QTr8+te/TteVZTJZZ2fn1q1bu7q6ysrKWJY1GAzr1q0LhUL/8R//ATLF7Xbn5uaiNMtx3MjIiMFgsNls586ds9lsOp1Oq9UODAwcPHgQojGZTAZhBRbSrFmzfvazn9XX12/btq28vLyvr08mk7nd7v7+/oGBAZVKdc8992zZsuXVV18lhOD+Z86cuXfvXo1G82d0xdxqgzpx4sR///YnNztAlokyXDQaReFFo9FwHNfT07Nv375Dhw4BaiAuwFZUKtXOnTtLSkoAFDiOYximt7f3k08+qa+vHx0dvfGimzZteu6554xGI2R6sFEYClTI0WgUyq09e/bs2LEjXe2577774Feqq6vfffddVH4CgcDTTz9911131dbWymQypAUvv/zy888/n5+fv3bt2i+//BIRfNWqVR0dHadPn/Z6vWaz2eFw6HQ6JJ5utxsgXaPR7N27t6mpSS6Xz5kz57XXXjMYDHa7XRAEPC/K5KIoTkxMZGZm7t27d9asWVVVVRzHgbZ9/PHHq6qqHnjggWQymZ2dnZub63Q6GYZ55513/vM///PAgQM3YqapKrl8y/Pc9FAIASuKmphEUAZut3v//v2ffPJJb28vIQQCUSRuoLYjkUhXVxchxGAwoHElmUw6HA6Hw4GsDRGWEJJKpfLz82OxGOh4wHmI0SRJQvEHsWNiYgJ2BtJVFEWj0bhp06a8vLyXX355+fLlTU1NyMV279798MMPv/322w8//LBer29sbHzkkUc2bNiQn59/5swZWG04HK6trV2wYAFEi7h5QRB6enqmT58OIHjo0KGampqSkhIIEE6cOJGZmdnR0YF2Z5/PRwiBlK+urm7jxo2/+tWvRkZGVqxYodFoioqK5HK52+3evHnz5s2bc3NzZ82alZOT43Q6kVS2t7ffcccdECv/rYfCSRsW4kJaMqtUKl0u18WLF1955ZW+vr40v5LW8BNC4CFCodDLL79sMBjmzJljMpmcTqfP53M6nV6vF+4NGAsavYKCght7KaFlw0Uht1IqlV6v9+LFi8eOHTObzV6vN5lMGo1Go9G4cOFCm82mUqnuuOOOvr6+4eFhg8Hg9/t/+tOfDg8Py2Qyo9Ho8XgGBwcHBwexPHDdRCLxb//2b6tXr66urp4zZ04gENBoNMPDw2fOnBkeHt69e7fP56uuro7FYlevXgVR3tHRAbUMz/MrV64Ej8/zvEKhOHr0qEaj+cd//MesrKxDhw6tXLlycHAQuoZLly4tXLgwPz9fLpdXVVU1Nzejf/+ll17avXs39oOYovf7VxuTNiwIxCRJQiaFWHb48GFBEJRKJZK7GTNmbNiwwePx7N69GygKdiaKotfrbW5uFgQBL4MQgkwt7f9EUaQoKiMjQ6PRQP1NUZTBYIA0AEsZ3YXd3d1nz549ceKEXC6HRmD9+vXV1dWo6vzyl79855137rrrLoPB0N7ertPpLl++bLVaX3vttWnTpkHPDupVFEXcTG5ubnFx8cyZM/1+f2tra3l5Ofqili9fvmvXrurq6nXr1rW1tTU0NBw/fnzRokXBYNDpdKKfc9u2bY2NjbNmzTp37hx1fV+ahx9+OBqNer3ew4cP19bWAkgpFIrf/va3Tz31VCwW27lz59y5cxmG+eyzz6xW68TEBJi2KX/Nf/kx6WdAYoymrg8//PCDDz4APIdG0Wq1/uAHP9i4cSPHce+//z6mMl2EKSoqQlIN1Q0SwHg8jgUKrUiaXJAkyeFwnD9/PhQK2Wy2wsJCnU4HsOVyuVD//+KLLxKJRCgUysjIKC8vnzdvXmVlJQpKlZWVc+bMQTmvs7MTAXR8fJwQMjY2hrQUghmwkTzPQ7mQm5u7bNmycDgsCAIKhb/61a9aWlosFsuuXbsIIei9zsrKqqurQ9kgOzv79ddfX7x48dDQEIR1SE3QYu9wOBYuXBgMBiORCErUg4ODHo+nrKzs1KlTWKWEkImJicLCQofDAWT2t+60/nce6w8+j8fjNpvt/ffff/PNN5E2p/979913P/PMM1lZWaCa0hsowKqwYca2bdtaW1vb2trsdjsUeUgYGYax2WxFRUU0TdfX1584caKmpoZhGI/Hc+DAAZVKBd+jVquDwWBXVxfOkEa4lZWVjzzySE1NDUQ1oVBIFMUNGzbU19fL5XLs3mG32xGmA4EAMo+JiYlVq1ZRFFVfX88wDM/zV69e1ev1Pp9v8+bNMplsaGjowIEDOp2utLR0aGjIZDLdfffdKpXqyJEjDQ0N8HmhUKi5ubm6unrt2rXz5s177733KisrT5w4IYriuXPnaJq2WCxr1qzp6+t77bXXYPQKhcJsNh8+fPjBBx88evRofX09RVF6vR55bkNDw+bNm282z/Rnn+dPPH7SHisrK6u3t7erqyvtcliWnTZt2r//+79zHJefn49eAGwtRAhBWRrcRCqVKisry8rKWrly5cTERDAYRIgEyjEajRCJNzc3NzY2PvHEE2gH2LZtG03TIOLTuhFETETGioqKNWvWzJgxQ6fTpZv4pk2b1t3dff/99587d27u3LlPPfXUQw89tH//fkC0vLw8m8324x//OBKJ9PX1gawihIC+ysvLM5vNoKB++tOfnj59+uTJkw888MCiRYu6urra29ufeeYZ0G9nz56dMWMGxFuhUKiwsPDLL78MBAIoHLW3t0+fPn379u1gGUpLSwOBwODgYHV1dWlp6dmzZ1evXm02m9G5ynFcXV3dggULwNHcakXlyY5JG9b4+HhOTs7GjRtbW1v7+/uzs7MfeOCBhx56SCaTiaKYVrQplUo0xaf3hRIEYWJiAjNotVrz8vKAq5LXd2dAtzQhZPr06QDFJpMJjQAqlQrBBaWxtFLZarWuWbOmtrb2zjvvBC8AglSpVA4MDBQVFQ0ODpaVlTEM09DQ8M///M9///d/39ra+t3vfveTTz7ZsmWLIAgLFy5cvnw5Lh2NRq9du0YI0ev1VVVViHe//OUvOzs7165dW1tb++mnnw4MDKxevbqioqKqqurLL79cu3btXXfdderUqd7e3rlz5wqC8NZbb/385z8fHh7u7+83mUz33nvvq6+++p3vfOf9999/5ZVX7r77bkLI5cuXt2zZcvz4cei0nnzyyVmzZkGMiqrXLbg7zWQHdfLkyf/+7U/mM1C63717tyAIP/zhDy0WC4COUqmEdgXNoqIorlu3zul0QuOB2tYnn3wC9SPqfdCagk1A/Ucmk73xxhtvvvnmCy+8sGHDBplMtnfv3paWlpGREb/fHwwG0QtlMBgqKysrKys3bNhQWFiInipAZmhjQB+Mjo5euXJFr9cvWbLEbrcbjca+vj6bzSYIQl5ens/n27lzp9VqffXVV+ELjUbjypUrH3vssdmzZ/v9frCjBQUFoVAIomFowsxmM1obLl++DC0N9jPKy8tDq/EvfvGLgYGBBx544PDhw0ajcfv27Wq1+rPPPtu+ffvo6Oi8efNcLtehQ4eQNFAUhS2poJUAir3ZPNNUnWfKQiHkSkVFRY8++ij2PXO73UjK0nvDgYJSq9XTp08PBoPwSXAzIyMj1dXVYFmp6330aU0z2kXuv/9+mUwG6lwmk23atGn69OmBQGBkZAR9WjnXh8ViwYafIOIVCgU6+CiKwoaRWq22p6fnm2++SSQS4XB4/fr1d911F1KKZDJZXFw8Y8aM8+fPT5s2bXh4mKIon883NjYWCARcLhca8ymKGhgYgN7Q5XJxHJeXl4e9JBQKxZIlS7B/BuDR+Pg4mslefPFFlUr1k5/8pKqqatWqVeBQ5s+fv2rVqtmzZzc1Nb3wwgsZGRlId9Rq9djYGIoZ2P8ieX1zvb/dMWnDslgsdrsdogZsAIdWTHAKEHKQ6/6prKzs66+/vvHrPT09s2fPvlFii+CFAJpMJmOx2LRp05588kmZTDY+Pq7X60VRXLVqlc/nA1+a3poB7fyoRsOSoD5Ato+ts9rb2wkhDofjZz/72T/90z+9/fbbDz74IApHhBCv11tXV9fa2jo2NoYbkMlkqDiBjyWEZGdnO51O7HcA5AT9D0SnINjQoY+nNhgMaBt0uVwPPvggy7Jz585FWPT7/ffdd9+0adPq6uogi9VqtRMTEwUFBXq9Hk3x5HrRbKo6kv9aY9KGNTo6qtPpgEhAi6vVavyE6AAK4Hg8LoritGnTyHUNMfxTX19fOlqlwTh1fYM17O2BeGcwGKCs53ne7/fj1aKF1e12A4Nj39t0Yx3q0GmtZlZW1oULF06dOgWN7xtvvFFXV4fds5qbm2UymU6n++yzz1wu142yJzQpoKIcj8cHBwf1ej1FUWBV4FPxyBqNBkAbUFKSJFSaOY7r7OycPn16aWkpOh8zMzMjkciCBQvARGAqkJ9ardZgMIjKGHN9J2O0xE3le/6Lj0kXodGFHAqFIC4ATsKmOeihwB7D8CJlZWXU9Z0XsFMAuoSxCwPiI3V9Ew5CCIRswGeQFzscDpvNFovF/H5/uhqtUqn0ej02iQiHw3AhEJZotVqYJuxszZo18+bNQ2em1+utr68XRRHJ/69//Wun03n+/PnLly9DRkFRlFqtRv07LbDJzMyEa4RHhAgMHgt8m1arTSQSGo0G259ge4U777wTJo5KJc48NDQ0MTEBt4cWyLQYJCMjA70CODi9fd7f7vhzwPukjl+8eDHqMDAvk8n0wQcf5Ofnp2s+N+m6aK7CvpIvvPBCTk7O66+/jg1n4/F4SUmJWq0eHx+HvaYzWbPZ/NJLL9XU1CBU3UiC/4WLuLfaeSZ7/E3f5x1oFIwXIQRu4C9AK8NpQQHx85//PJVK/eY3v1m/fn1paSlFUT09PZcuXUp3gcI/YXstgCRCCEVR/zeqK3+V8f8AMLv1p/o0Wz0AAAAASUVORK5CYII=",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=200x50>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image = Image.open(train_ds.root_dir + train_dataset['file_name'][0]).convert(\"RGB\")\n",
    "\n",
    "image"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Show Label for Above Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37ep6\n"
     ]
    }
   ],
   "source": [
    "labels = encoding['labels']\n",
    "labels[labels == -100] = processor.tokenizer.pad_token_id\n",
    "label_str = processor.decode(labels, skip_special_tokens=True)\n",
    "print(label_str)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of VisionEncoderDecoderModel were not initialized from the model checkpoint at microsoft/trocr-base-printed and are newly initialized: ['encoder.pooler.dense.bias', 'encoder.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = VisionEncoderDecoderModel.from_pretrained(MODEL_CKPT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Model Configuration Modifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.config.decoder_start_token_id = processor.tokenizer.cls_token_id\n",
    "model.config.pad_token_id = processor.tokenizer.pad_token_id\n",
    "\n",
    "model.config.vocab_size = model.config.decoder.vocab_size\n",
    "\n",
    "model.config.eos_token_id = processor.tokenizer.sep_token_id\n",
    "model.config.max_length = 64\n",
    "model.config.early_stopping = True\n",
    "model.config.no_repeat_ngram_size = 3\n",
    "model.config.length_penalty = 2.0\n",
    "model.config.num_beams = 4"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Define Metrics Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "cer_metric = evaluate.load(\"cer\")\n",
    "\n",
    "def compute_metrics(pred):\n",
    "    label_ids = pred.label_ids\n",
    "    pred_ids = pred.predictions\n",
    "    \n",
    "    pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)\n",
    "    label_ids[label_ids == -100] = processor.tokenizer.pad_token_id\n",
    "    label_str = processor.batch_decode(label_ids, skip_special_tokens=True)\n",
    "    \n",
    "    cer = cer_metric.compute(predictions=pred_str, references=label_str)\n",
    "    \n",
    "    return {\"cer\" : cer}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Define Training Arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = Seq2SeqTrainingArguments(\n",
    "    output_dir = MODEL_NAME,\n",
    "    num_train_epochs=NUM_OF_EPOCHS,\n",
    "    predict_with_generate=True,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    per_device_train_batch_size=8,\n",
    "    per_device_eval_batch_size=8,\n",
    "    logging_first_step=True,\n",
    "    hub_private_repo=True,\n",
    "    push_to_hub=True\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Define Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/trocr-base-printed_captcha_ocr into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = Seq2SeqTrainer(\n",
    "    model=model,\n",
    "    tokenizer=processor.feature_extractor,\n",
    "    args=args,\n",
    "    compute_metrics=compute_metrics,\n",
    "    train_dataset=train_ds,\n",
    "    eval_dataset=test_ds,\n",
    "    data_collator=default_data_collator\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Fit/Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/briandunn/Documents/nlpnn/nlp_ch4/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 856\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 321\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93b2700198ff46efba2e7005c419395e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/321 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.4464, 'learning_rate': 4.9844236760124614e-05, 'epoch': 0.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 214\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a984e7f93514a7d8c2b76aa6dce7864",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/27 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to trocr-base-printed_captcha_ocr/checkpoint-107\n",
      "Configuration saved in trocr-base-printed_captcha_ocr/checkpoint-107/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5615314245223999, 'eval_cer': 0.08785046728971962, 'eval_runtime': 5533.5267, 'eval_samples_per_second': 0.039, 'eval_steps_per_second': 0.005, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in trocr-base-printed_captcha_ocr/checkpoint-107/pytorch_model.bin\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/checkpoint-107/preprocessor_config.json\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/preprocessor_config.json\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 214\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba5a98d649df46d9a6570cf183fb2ee4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/27 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to trocr-base-printed_captcha_ocr/checkpoint-214\n",
      "Configuration saved in trocr-base-printed_captcha_ocr/checkpoint-214/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.24318334460258484, 'eval_cer': 0.026168224299065422, 'eval_runtime': 4970.2478, 'eval_samples_per_second': 0.043, 'eval_steps_per_second': 0.005, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in trocr-base-printed_captcha_ocr/checkpoint-214/pytorch_model.bin\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/checkpoint-214/preprocessor_config.json\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/preprocessor_config.json\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 214\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "649d8cfe1ec44ebd8713e9ee1ecc191a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/27 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to trocr-base-printed_captcha_ocr/checkpoint-321\n",
      "Configuration saved in trocr-base-printed_captcha_ocr/checkpoint-321/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.13803276419639587, 'eval_cer': 0.007476635514018692, 'eval_runtime': 5028.1446, 'eval_samples_per_second': 0.043, 'eval_steps_per_second': 0.005, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in trocr-base-printed_captcha_ocr/checkpoint-321/pytorch_model.bin\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/checkpoint-321/preprocessor_config.json\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/preprocessor_config.json\n",
      "Adding files tracked by Git LFS: ['.DS_Store']. This may take a bit of time if the files are large.\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_runtime': 54834.3401, 'train_samples_per_second': 0.047, 'train_steps_per_second': 0.006, 'train_loss': 0.40612717209575333, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=321, training_loss=0.40612717209575333, metrics={'train_runtime': 54834.3401, 'train_samples_per_second': 0.047, 'train_steps_per_second': 0.006, 'train_loss': 0.40612717209575333, 'epoch': 3.0})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save Model & Model State"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to trocr-base-printed_captcha_ocr\n",
      "Configuration saved in trocr-base-printed_captcha_ocr/config.json\n",
      "Model weights saved in trocr-base-printed_captcha_ocr/pytorch_model.bin\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/preprocessor_config.json\n",
      "Saving model checkpoint to trocr-base-printed_captcha_ocr\n",
      "Configuration saved in trocr-base-printed_captcha_ocr/config.json\n",
      "Model weights saved in trocr-base-printed_captcha_ocr/pytorch_model.bin\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/preprocessor_config.json\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model()\n",
    "trainer.save_state()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 214\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98d3df02dc2c4dd58fc42a9e39920bb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/27 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.13803276419639587,\n",
       " 'eval_cer': 0.007476635514018692,\n",
       " 'eval_runtime': 5008.3445,\n",
       " 'eval_samples_per_second': 0.043,\n",
       " 'eval_steps_per_second': 0.005,\n",
       " 'epoch': 3.0}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Push Model to Hub (My Profile!!!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to trocr-base-printed_captcha_ocr\n",
      "Configuration saved in trocr-base-printed_captcha_ocr/config.json\n",
      "Model weights saved in trocr-base-printed_captcha_ocr/pytorch_model.bin\n",
      "Feature extractor saved in trocr-base-printed_captcha_ocr/preprocessor_config.json\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02027faf846d4568b1c95984d6b84121",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Jan12_22-58-23_Brians-Mac-mini.local/events.out.tfevents.1673585911.Brians-Mac-mini.local.438…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "627d0c03e782431fbc6787e7b2af6f8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Jan12_22-58-23_Brians-Mac-mini.local/events.out.tfevents.1673645753.Brians-Mac-mini.43820.2: …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "remote: Scanning LFS files for validity, may be slow...        \n",
      "remote: LFS file scan complete.        \n",
      "To https://huggingface.co/DunnBC22/trocr-base-printed_captcha_ocr\n",
      "   c0145d1..a66a6da  main -> main\n",
      "\n",
      "To https://huggingface.co/DunnBC22/trocr-base-printed_captcha_ocr\n",
      "   a66a6da..7cc8af5  main -> main\n",
      "\n"
     ]
    }
   ],
   "source": [
    "kwargs = {\n",
    "    \"finetuned_from\" : model.config._name_or_path,\n",
    "    \"tasks\" : \"image-to-text\",\n",
    "    \"tags\" : [\"image-to-text\"],\n",
    "}\n",
    "\n",
    "if args.push_to_hub:\n",
    "    trainer.push_to_hub(\"All Dunn!!!\")\n",
    "else:\n",
    "    trainer.create_model_card(**kwargs)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notes & Other Takeaways From This Project\n",
    "****\n",
    "- The Character Error Rate (CER) was 0.0075. I am pleased with that result.\n",
    "- Context about metric: Zero (0) is perfection. One the worst score (unless there is an insertion error).\n",
    "****"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Citations\n",
    "\n",
    "##### For Transformer Checkpoint\n",
    "- @misc{li2021trocr,\n",
    "      title={TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models}, \n",
    "      author={Minghao Li and Tengchao Lv and Lei Cui and Yijuan Lu and Dinei Florencio and Cha Zhang and Zhoujun Li and Furu Wei},\n",
    "      year={2021},\n",
    "      eprint={2109.10282},\n",
    "      archivePrefix={arXiv},\n",
    "      primaryClass={cs.CL}\n",
    "}\n",
    "\n",
    "##### For CER Metric\n",
    "- @inproceedings{morris2004,\n",
    "author = {Morris, Andrew and Maier, Viktoria and Green, Phil},\n",
    "year = {2004},\n",
    "month = {01},\n",
    "pages = {},\n",
    "title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}\n",
    "}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp_ch4",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a52fe47989fdc78fafbb981021cec52a6b82df6453830b9ffbd04250493e6cab"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}