04. Pretrained Models/PretrainedModel.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PretrainedModel.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "957e095a73a04591bb5fab8696af061b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bc08769e79eb4c2890dbfef83d60a9b5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_97a100676e024babb68562fa29104237",
              "IPY_MODEL_727208403fe1401a9bbaa24c00874fc7",
              "IPY_MODEL_828124b3ba894e1d8d5dec057991e495"
            ]
          }
        },
        "bc08769e79eb4c2890dbfef83d60a9b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "97a100676e024babb68562fa29104237": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fa9ec3fc52bc4176bb2b0e247cb0d73b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_31ce7c77e7f14b139d2f06840d291b02"
          }
        },
        "727208403fe1401a9bbaa24c00874fc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_785e6e4979fa401883bed2871ee92efd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_113f430a406847d69ec9e635faa5bd35"
          }
        },
        "828124b3ba894e1d8d5dec057991e495": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_48888bda89b84a64ac37fe485f93f37c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [00:00&lt;00:00, 58.84it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0bee05318e8147f596e3046701ff24fc"
          }
        },
        "fa9ec3fc52bc4176bb2b0e247cb0d73b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "31ce7c77e7f14b139d2f06840d291b02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "785e6e4979fa401883bed2871ee92efd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "113f430a406847d69ec9e635faa5bd35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48888bda89b84a64ac37fe485f93f37c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0bee05318e8147f596e3046701ff24fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ba551b3375564a7b9ea59576cde50f50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_726ccfe3b3fb4c429f6745fe0c59062c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_555f9571507c43c997a5a434e71fec44",
              "IPY_MODEL_c24b3ef3c3c8453b95c9f92cac448ecc",
              "IPY_MODEL_f592f52d24af46ac9b9e21c7e801206f"
            ]
          }
        },
        "726ccfe3b3fb4c429f6745fe0c59062c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "555f9571507c43c997a5a434e71fec44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_87aba6a79c924802be023c6601f47090",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3b752510457846f99d590e042db7a8c6"
          }
        },
        "c24b3ef3c3c8453b95c9f92cac448ecc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dd0139ca19074aed86585e27f7c3c1c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c7469b58d5a64256b7a44a084e8b3ffa"
          }
        },
        "f592f52d24af46ac9b9e21c7e801206f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f3483fa8082b437cb9ee9ccd1bc25aba",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4/4 [00:00&lt;00:00,  6.91ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_89d97214d04c4d4188d7f12c72cd34d5"
          }
        },
        "87aba6a79c924802be023c6601f47090": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3b752510457846f99d590e042db7a8c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd0139ca19074aed86585e27f7c3c1c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c7469b58d5a64256b7a44a084e8b3ffa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3483fa8082b437cb9ee9ccd1bc25aba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "89d97214d04c4d4188d7f12c72cd34d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "**INITIALIZATION:**\n",
        "- I use these three lines of code on top of my each notebooks because it will help to prevent any problems while reloading the same project. And the third line of code helps to make visualization within the notebook."
      ],
      "metadata": {
        "id": "NTM4eJtYoUrW"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "vrzwsW9LnzLu"
      },
      "outputs": [],
      "source": [
        "#@ INITIALIZATION: \n",
        "%reload_ext autoreload\n",
        "%autoreload 2\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**LIBRARIES AND DEPENDENCIES:**\n",
        "- I have downloaded all the libraries and dependencies required for the project in one particular cell."
      ],
      "metadata": {
        "id": "TeFko4nqodm1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSTALLING DEPENDENCIES: UNCOMMENT BELOW: \n",
        "# !pip install datasets transformers[sentencepiece]"
      ],
      "metadata": {
        "id": "Alj-bR9rogkb"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ DOWNLOADING LIBRARIES AND DEPENDENCIES:\n",
        "import numpy as np\n",
        "import torch\n",
        "import transformers\n",
        "import datasets\n",
        "from datasets import load_dataset\n",
        "from datasets import load_metric\n",
        "from transformers import AdamW\n",
        "from transformers import AutoTokenizer\n",
        "from transformers import AutoModelForSequenceClassification\n",
        "from transformers import DataCollatorWithPadding\n",
        "from transformers import TrainingArguments\n",
        "from transformers import Trainer\n",
        "\n",
        "#@ IGNORING WARNINGS: \n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "metadata": {
        "id": "xvPCBSX5oluM"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**PROCESSING THE DATA:**"
      ],
      "metadata": {
        "id": "3MupS6cJpHLb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ PROCESSING THE DATA:\n",
        "checkpoint = \"bert-base-uncased\"                                        # Initialization. \n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)                   # Initializing Tokenizer. \n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)  # Initializing Sequence Model. \n",
        "sequences = [\n",
        "        \"I've been waiting for a HuggingFace course my whole life\",\n",
        "        \"This course is amazing!\"\n",
        "]                                                                       # Text Sequences. \n",
        "batch = tokenizer(sequences, padding=True, truncation=True, \n",
        "                  return_tensors=\"pt\")                                  # Getting Batch of Tensors. \n",
        "batch[\"labels\"] = torch.tensor([1, 1])                                  # Initializing Labels. \n",
        "\n",
        "#@ INITIALIZING MODEL TRAINING PARAMETERS:\n",
        "optimizer = AdamW(model.parameters())                                   # Initializing Optimizer. \n",
        "loss = model(**batch).loss                                              # Initializing Loss. \n",
        "loss.backward()                                                         # Initializing Back Propagation. \n",
        "optimizer.step()                                                        # Updating Parameters. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V45HKkHQpD61",
        "outputId": "19fc89cb-50df-4a6d-b6aa-c2d54f87e76e"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**GETTING THE DATASET:**\n",
        "- In this notebook, we will use MRPC (Microsoft Research Paraphrase Corpus) dataset introduced by William B. Dolan and Chris Brockett. The dataset consist of 5801 pairs of sentences, with a label indicating if they are paraphrases or not. It is one of the 10 datasets composing the GLUE benchmark, which is an academic benchmark that is used to measure the performance of ML models across 10 different text classification tasks. "
      ],
      "metadata": {
        "id": "h-BFfURztEfK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ GETTING THE DATASET:\n",
        "raw_datasets = load_dataset(\"glue\", \"mrpc\")             # Getting MRPC Dataset. \n",
        "raw_datasets                                            # Inspecting Dataset. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 309,
          "referenced_widgets": [
            "957e095a73a04591bb5fab8696af061b",
            "bc08769e79eb4c2890dbfef83d60a9b5",
            "97a100676e024babb68562fa29104237",
            "727208403fe1401a9bbaa24c00874fc7",
            "828124b3ba894e1d8d5dec057991e495",
            "fa9ec3fc52bc4176bb2b0e247cb0d73b",
            "31ce7c77e7f14b139d2f06840d291b02",
            "785e6e4979fa401883bed2871ee92efd",
            "113f430a406847d69ec9e635faa5bd35",
            "48888bda89b84a64ac37fe485f93f37c",
            "0bee05318e8147f596e3046701ff24fc"
          ]
        },
        "id": "TVvxjHqhsbcx",
        "outputId": "c9f700eb-66ba-4e7c-964e-30525a858706"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reusing dataset glue (/root/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "957e095a73a04591bb5fab8696af061b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
              "        num_rows: 3668\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
              "        num_rows: 408\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
              "        num_rows: 1725\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING TRAINING DATASET: \n",
        "raw_train_dataset = raw_datasets[\"train\"]               # Training Dataset. \n",
        "raw_train_dataset[15]                                    # Inspection. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YFr08GBQ0xtW",
        "outputId": "e67d005c-cba1-43d2-b921-51e269784f4c"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'idx': 16,\n",
              " 'label': 0,\n",
              " 'sentence1': 'Rudder was most recently senior vice president for the Developer & Platform Evangelism Business .',\n",
              " 'sentence2': 'Senior Vice President Eric Rudder , formerly head of the Developer and Platform Evangelism unit , will lead the new entity .'}"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING TYPE OF COLUMNS:\n",
        "raw_train_dataset.features"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AGVarErY3vIe",
        "outputId": "7952f95d-c049-4fbb-93d2-232438c24bde"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'idx': Value(dtype='int32', id=None),\n",
              " 'label': ClassLabel(num_classes=2, names=['not_equivalent', 'equivalent'], id=None),\n",
              " 'sentence1': Value(dtype='string', id=None),\n",
              " 'sentence2': Value(dtype='string', id=None)}"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**PREPROCESSING THE DATASET:**\n",
        "- To preprocess the dataset, we will convert the text to numbers the model can make sense of, with the help of tokenizer. "
      ],
      "metadata": {
        "id": "BiiE5nTe5nZo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING TOKENIZATION:\n",
        "tokenized_1 = tokenizer(raw_datasets[\"train\"][\"sentence1\"])                  # Tokenization. \n",
        "tokenized_2 = tokenizer(raw_datasets[\"train\"][\"sentence2\"])                  # Tokenization. "
      ],
      "metadata": {
        "id": "Fz6lI8xN4GQ9"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPLEMENTING TOKENIZER:\n",
        "inputs = tokenizer(\"This is a first sentence\", \n",
        "                   \"This is a second sentence\")                              # Tokenization. \n",
        "inputs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tnz8kEcyjPH8",
        "outputId": "a0315c47-0bdc-430a-84ab-057919e768d2"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [101, 2023, 2003, 1037, 2034, 6251, 102, 2023, 2003, 1037, 2117, 6251, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ DEFINING TOKENIZATION FUNCTION:\n",
        "def tokenize_function(example):                                               # Defining Function. \n",
        "    return tokenizer(example[\"sentence1\"], example[\"sentence2\"], \n",
        "                     truncation=True)                                         # Implementation of Tokenizer. \n",
        "\n",
        "#@ IMPLEMENTATION OF FUNCTION:\n",
        "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)        # Initializing Tokenization. \n",
        "tokenized_datasets                                                            # Inspection. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347,
          "referenced_widgets": [
            "ba551b3375564a7b9ea59576cde50f50",
            "726ccfe3b3fb4c429f6745fe0c59062c",
            "555f9571507c43c997a5a434e71fec44",
            "c24b3ef3c3c8453b95c9f92cac448ecc",
            "f592f52d24af46ac9b9e21c7e801206f",
            "87aba6a79c924802be023c6601f47090",
            "3b752510457846f99d590e042db7a8c6",
            "dd0139ca19074aed86585e27f7c3c1c9",
            "c7469b58d5a64256b7a44a084e8b3ffa",
            "f3483fa8082b437cb9ee9ccd1bc25aba",
            "89d97214d04c4d4188d7f12c72cd34d5"
          ]
        },
        "id": "YORtBNZOjgdR",
        "outputId": "8e35bc74-c445-4ab1-ed9f-34b341b03ddb"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ba551b3375564a7b9ea59576cde50f50",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/4 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-c44a7f7bf29f5b58.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-3e01f4e4c8ccd211.arrow\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
              "        num_rows: 3668\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
              "        num_rows: 408\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
              "        num_rows: 1725\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**DYNAMIC PADDING**\n",
        "- The function that is responsible for putting together samples inside a batch is called a `collate function`. **Dynamic Padding** means the samples in the batch should all be padded to the maximum length inside the batch. "
      ],
      "metadata": {
        "id": "W73wiTROuhkF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPLEMENTATION OF COLLATOR FUNCTION: \n",
        "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)    # Initialization. \n",
        "\n",
        "#@ IMPLEMENTATION OF COLLATOR FUNCTION: INITIALIZATION: \n",
        "samples = tokenized_datasets[\"train\"][:8]  \n",
        "samples = {k:v for k,v in samples.items() if \n",
        "           k not in [\"idx\", \"sentence1\", \"sentence2\"]}\n",
        "[len(x) for x in samples[\"input_ids\"]]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SZrfuz8DmD7l",
        "outputId": "a6353f97-dfb1-4bda-8610-ce5b38c678a0"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[50, 59, 47, 67, 59, 50, 62, 32]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPLEMENTATION OF COLLATOR FUNCTION: \n",
        "batch = data_collator(samples)                  # Implementation. \n",
        "{k:v.shape for k, v in batch.items()}           # Inspection. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ddpwArvGwCjW",
        "outputId": "7f75509f-be96-4c99-e881-e9c6aa531797"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'attention_mask': torch.Size([8, 67]),\n",
              " 'input_ids': torch.Size([8, 67]),\n",
              " 'labels': torch.Size([8]),\n",
              " 'token_type_ids': torch.Size([8, 67])}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**TRAINING:**"
      ],
      "metadata": {
        "id": "4W_RLfRwriWC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING TRAINING:\n",
        "training_args = TrainingArguments(checkpoint)                                           # Initializing Training Arguments. \n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)    # Initializing Classification Model. \n",
        "\n",
        "\n",
        "#@ INITIALIZING TRAINER:\n",
        "trainer = Trainer(model, training_args, train_dataset=tokenized_datasets[\"train\"],\n",
        "                  eval_dataset=tokenized_datasets[\"validation\"], \n",
        "                  data_collator=data_collator, tokenizer=tokenizer)                     # Initializing Trainer.\n",
        "trainer.train()                                                                         # Initializing Training.  "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 662
        },
        "id": "JIeHvtSMrR6R",
        "outputId": "ad36abcb-f740-4579-917e-0e511a06a135"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following columns in the training set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence2, idx, sentence1. If sentence2, idx, sentence1 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running training *****\n",
            "  Num examples = 3668\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1377\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1377' max='1377' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1377/1377 06:37, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.562600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.366000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to bert-base-uncased/checkpoint-500\n",
            "Configuration saved in bert-base-uncased/checkpoint-500/config.json\n",
            "Model weights saved in bert-base-uncased/checkpoint-500/pytorch_model.bin\n",
            "tokenizer config file saved in bert-base-uncased/checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in bert-base-uncased/checkpoint-500/special_tokens_map.json\n",
            "Saving model checkpoint to bert-base-uncased/checkpoint-1000\n",
            "Configuration saved in bert-base-uncased/checkpoint-1000/config.json\n",
            "Model weights saved in bert-base-uncased/checkpoint-1000/pytorch_model.bin\n",
            "tokenizer config file saved in bert-base-uncased/checkpoint-1000/tokenizer_config.json\n",
            "Special tokens file saved in bert-base-uncased/checkpoint-1000/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1377, training_loss=0.40232000856531125, metrics={'train_runtime': 397.7197, 'train_samples_per_second': 27.668, 'train_steps_per_second': 3.462, 'total_flos': 405470580750720.0, 'train_loss': 0.40232000856531125, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**EVALUATION:**"
      ],
      "metadata": {
        "id": "6Gg-pF6aDLA1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING MODEL EVALUATION:\n",
        "predictions = trainer.predict(tokenized_datasets[\"validation\"])         # Getting Predictions. \n",
        "print(predictions.predictions.shape, predictions.label_ids.shape)       # Inspecting Predictions. "
      ],
      "metadata": {
        "id": "8ZJnOI-xuEX1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 144
        },
        "outputId": "d8518103-a3ae-4813-c50e-8243f7cc6164"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the test set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence2, idx, sentence1. If sentence2, idx, sentence1 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Prediction *****\n",
            "  Num examples = 408\n",
            "  Batch size = 8\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='102' max='51' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [51/51 01:27]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(408, 2) (408,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING MODEL PREDICTION:\n",
        "preds = np.argmax(predictions.predictions, axis=-1)                     # Getting Maximum Index.\n",
        "metric = load_metric(\"glue\", \"mrpc\")                                    # Initializing Metrics. \n",
        "metric.compute(predictions=preds, references=predictions.label_ids)     # Computing Metrices. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h0evBLawFnVV",
        "outputId": "4a39c8cd-6035-492a-9b2f-9b5f641353d0"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 0.8676470588235294, 'f1': 0.9072164948453608}"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ DEFINING FUNCTION FOR COMPUTING METRICS:\n",
        "def compute_metrics(eval_preds):                                        # Defining Function. \n",
        "    metric = load_metric(\"glue\", \"mrpc\")                                # Initializing Metrics. \n",
        "    logits, labels = eval_preds\n",
        "    predictions = np.argmax(logits, axis=-1)                            # Getting Maximum Index. \n",
        "    return metric.compute(predictions=predictions, references=labels)    # Getting Metrices. "
      ],
      "metadata": {
        "id": "ZF4eRpM6HYwk"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ DEFINING NEW TRAINER: \n",
        "training_args = TrainingArguments(\"test-trainer\", \n",
        "                                  evaluation_strategy=\"epoch\")          # Initializing Training Arguments. \n",
        "trainer = Trainer(model, training_args, \n",
        "                  train_dataset=tokenized_datasets[\"train\"],            # Initializing Training Datasets. \n",
        "                  eval_dataset=tokenized_datasets[\"validation\"],        # Initializing Validation Datasets. \n",
        "                  data_collator=data_collator, tokenizer=tokenizer, \n",
        "                  compute_metrics=compute_metrics)                      # Initializing Trainer. \n",
        "trainer.train()                                                         # Training the Model. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 849
        },
        "id": "xeTSv_p7KJFX",
        "outputId": "1d44da42-9627-4acb-b812-0e1d43d3c73c"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "The following columns in the training set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence2, idx, sentence1. If sentence2, idx, sentence1 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running training *****\n",
            "  Num examples = 3668\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1377\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1377' max='1377' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1377/1377 06:38, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.838388</td>\n",
              "      <td>0.835784</td>\n",
              "      <td>0.879713</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.156100</td>\n",
              "      <td>0.802094</td>\n",
              "      <td>0.848039</td>\n",
              "      <td>0.892734</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.088100</td>\n",
              "      <td>0.924342</td>\n",
              "      <td>0.835784</td>\n",
              "      <td>0.886248</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence2, idx, sentence1. If sentence2, idx, sentence1 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 408\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-trainer/checkpoint-500\n",
            "Configuration saved in test-trainer/checkpoint-500/config.json\n",
            "Model weights saved in test-trainer/checkpoint-500/pytorch_model.bin\n",
            "tokenizer config file saved in test-trainer/checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in test-trainer/checkpoint-500/special_tokens_map.json\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence2, idx, sentence1. If sentence2, idx, sentence1 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 408\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-trainer/checkpoint-1000\n",
            "Configuration saved in test-trainer/checkpoint-1000/config.json\n",
            "Model weights saved in test-trainer/checkpoint-1000/pytorch_model.bin\n",
            "tokenizer config file saved in test-trainer/checkpoint-1000/tokenizer_config.json\n",
            "Special tokens file saved in test-trainer/checkpoint-1000/special_tokens_map.json\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence2, idx, sentence1. If sentence2, idx, sentence1 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 408\n",
            "  Batch size = 8\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1377, training_loss=0.10326220942478553, metrics={'train_runtime': 398.7433, 'train_samples_per_second': 27.597, 'train_steps_per_second': 3.453, 'total_flos': 405470580750720.0, 'train_loss': 0.10326220942478553, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    }
  ]
}