LightweightFineTuning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/eljandoubi/Copilot/blob/main/LightweightFineTuning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "bf3f9130",
      "metadata": {
        "id": "bf3f9130"
      },
      "source": [
        "# Lightweight Fine-Tuning Project"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "6e3f7aa6",
      "metadata": {
        "id": "6e3f7aa6"
      },
      "source": [
        "* PEFT technique: LoftQ initialization & QLoRA-style training\n",
        "* Model: facebook/opt-125m\n",
        "* Evaluation approach: Perplexity\n",
        "* Fine-tuning dataset: codeparrot/github-code"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "gQDzrQjHi3gN",
      "metadata": {
        "id": "gQDzrQjHi3gN"
      },
      "source": [
        "If you are running this in Colab, please restart the notebook after executing the next cell."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8P8iWpw_6YRb",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8P8iWpw_6YRb",
        "outputId": "1aaf1d6a-b00c-4325-e520-d0951c17bbe5",
        "tags": []
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: ipywidgets==8.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 1)) (8.1.1)\n",
            "Requirement already satisfied: notebook==7.0.7 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 2)) (7.0.7)\n",
            "Requirement already satisfied: datasets==2.16.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 3)) (2.16.1)\n",
            "Requirement already satisfied: transformers==4.37.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 4)) (4.37.2)\n",
            "Requirement already satisfied: accelerate==0.26.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 5)) (0.26.1)\n",
            "Requirement already satisfied: bitsandbytes==0.42.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 6)) (0.42.0)\n",
            "Requirement already satisfied: peft==0.8.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 7)) (0.8.1)\n",
            "Requirement already satisfied: gradio==4.16.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 8)) (4.16.0)\n",
            "Requirement already satisfied: torch==2.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from -r requirements.txt (line 9)) (2.2.0)\n",
            "Requirement already satisfied: comm>=0.1.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.1.4)\n",
            "Requirement already satisfied: ipython>=6.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipywidgets==8.1.1->-r requirements.txt (line 1)) (8.16.1)\n",
            "Requirement already satisfied: traitlets>=4.3.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipywidgets==8.1.1->-r requirements.txt (line 1)) (5.12.0)\n",
            "Requirement already satisfied: widgetsnbextension~=4.0.9 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipywidgets==8.1.1->-r requirements.txt (line 1)) (4.0.9)\n",
            "Requirement already satisfied: jupyterlab-widgets~=3.0.9 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipywidgets==8.1.1->-r requirements.txt (line 1)) (3.0.9)\n",
            "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from notebook==7.0.7->-r requirements.txt (line 2)) (2.9.1)\n",
            "Requirement already satisfied: jupyterlab-server<3,>=2.22.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from notebook==7.0.7->-r requirements.txt (line 2)) (2.25.0)\n",
            "Requirement already satisfied: jupyterlab<5,>=4.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from notebook==7.0.7->-r requirements.txt (line 2)) (4.0.7)\n",
            "Requirement already satisfied: notebook-shim<0.3,>=0.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from notebook==7.0.7->-r requirements.txt (line 2)) (0.2.3)\n",
            "Requirement already satisfied: tornado>=6.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from notebook==7.0.7->-r requirements.txt (line 2)) (6.3.3)\n",
            "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (3.12.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (1.26.1)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (13.0.0)\n",
            "Requirement already satisfied: pyarrow-hotfix in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (0.6)\n",
            "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (0.3.7)\n",
            "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (4.66.1)\n",
            "Requirement already satisfied: xxhash in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (0.70.15)\n",
            "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets==2.16.1->-r requirements.txt (line 3)) (2023.10.0)\n",
            "Requirement already satisfied: aiohttp in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (3.9.3)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (0.20.3)\n",
            "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (21.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from datasets==2.16.1->-r requirements.txt (line 3)) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers==4.37.2->-r requirements.txt (line 4)) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers==4.37.2->-r requirements.txt (line 4)) (0.15.1)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers==4.37.2->-r requirements.txt (line 4)) (0.4.2)\n",
            "Requirement already satisfied: psutil in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from accelerate==0.26.1->-r requirements.txt (line 5)) (5.9.5)\n",
            "Requirement already satisfied: scipy in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from bitsandbytes==0.42.0->-r requirements.txt (line 6)) (1.11.3)\n",
            "Requirement already satisfied: aiofiles<24.0,>=22.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (23.2.1)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (5.2.0)\n",
            "Requirement already satisfied: fastapi in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.109.1)\n",
            "Requirement already satisfied: ffmpy in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.3.1)\n",
            "Requirement already satisfied: gradio-client==0.8.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.8.1)\n",
            "Requirement already satisfied: httpx in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.26.0)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (6.1.0)\n",
            "Requirement already satisfied: jinja2<4.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (3.1.2)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (2.1.3)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (3.8.0)\n",
            "Requirement already satisfied: orjson~=3.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (3.9.13)\n",
            "Requirement already satisfied: pillow<11.0,>=8.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (10.1.0)\n",
            "Requirement already satisfied: pydantic>=2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (2.4.2)\n",
            "Requirement already satisfied: pydub in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.25.1)\n",
            "Requirement already satisfied: python-multipart in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.0.7)\n",
            "Requirement already satisfied: ruff>=0.1.7 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.2.0)\n",
            "Requirement already satisfied: semantic-version~=2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (2.10.0)\n",
            "Requirement already satisfied: tomlkit==0.12.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.12.0)\n",
            "Requirement already satisfied: typer<1.0,>=0.9 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from typer[all]<1.0,>=0.9->gradio==4.16.0->-r requirements.txt (line 8)) (0.9.0)\n",
            "Requirement already satisfied: typing-extensions~=4.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (4.8.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio==4.16.0->-r requirements.txt (line 8)) (0.27.0.post1)\n",
            "Requirement already satisfied: sympy in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (1.12)\n",
            "Requirement already satisfied: networkx in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (3.2)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch==2.2.0->-r requirements.txt (line 9)) (2.2.0)\n",
            "Requirement already satisfied: websockets<12.0,>=10.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from gradio-client==0.8.1->gradio==4.16.0->-r requirements.txt (line 8)) (11.0.3)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.2.0->-r requirements.txt (line 9)) (12.3.101)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from altair<6.0,>=4.2.0->gradio==4.16.0->-r requirements.txt (line 8)) (4.19.1)\n",
            "Requirement already satisfied: toolz in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from altair<6.0,>=4.2.0->gradio==4.16.0->-r requirements.txt (line 8)) (0.12.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from aiohttp->datasets==2.16.1->-r requirements.txt (line 3)) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from aiohttp->datasets==2.16.1->-r requirements.txt (line 3)) (23.1.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from aiohttp->datasets==2.16.1->-r requirements.txt (line 3)) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from aiohttp->datasets==2.16.1->-r requirements.txt (line 3)) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from aiohttp->datasets==2.16.1->-r requirements.txt (line 3)) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from aiohttp->datasets==2.16.1->-r requirements.txt (line 3)) (4.0.3)\n",
            "Requirement already satisfied: backcall in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.2.0)\n",
            "Requirement already satisfied: decorator in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (5.1.1)\n",
            "Requirement already satisfied: jedi>=0.16 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.19.1)\n",
            "Requirement already satisfied: matplotlib-inline in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.1.6)\n",
            "Requirement already satisfied: pickleshare in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.7.5)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (3.0.39)\n",
            "Requirement already satisfied: pygments>=2.4.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (2.16.1)\n",
            "Requirement already satisfied: stack-data in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.6.2)\n",
            "Requirement already satisfied: exceptiongroup in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (1.1.3)\n",
            "Requirement already satisfied: pexpect>4.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (4.8.0)\n",
            "Requirement already satisfied: anyio>=3.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (4.0.0)\n",
            "Requirement already satisfied: argon2-cffi in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (23.1.0)\n",
            "Requirement already satisfied: jupyter-client>=7.4.4 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (8.5.0)\n",
            "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (5.4.0)\n",
            "Requirement already satisfied: jupyter-events>=0.6.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.8.0)\n",
            "Requirement already satisfied: jupyter-server-terminals in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.4.4)\n",
            "Requirement already satisfied: nbconvert>=6.4.4 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (7.9.2)\n",
            "Requirement already satisfied: nbformat>=5.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (5.9.2)\n",
            "Requirement already satisfied: overrides in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (7.4.0)\n",
            "Requirement already satisfied: prometheus-client in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.17.1)\n",
            "Requirement already satisfied: pyzmq>=24 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (25.1.1)\n",
            "Requirement already satisfied: send2trash>=1.8.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.8.2)\n",
            "Requirement already satisfied: terminado>=0.8.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.17.1)\n",
            "Requirement already satisfied: websocket-client in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.6.4)\n",
            "Requirement already satisfied: async-lru>=1.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyterlab<5,>=4.0.2->notebook==7.0.7->-r requirements.txt (line 2)) (2.0.4)\n",
            "Requirement already satisfied: ipykernel in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyterlab<5,>=4.0.2->notebook==7.0.7->-r requirements.txt (line 2)) (6.26.0)\n",
            "Requirement already satisfied: jupyter-lsp>=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyterlab<5,>=4.0.2->notebook==7.0.7->-r requirements.txt (line 2)) (2.2.0)\n",
            "Requirement already satisfied: tomli in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyterlab<5,>=4.0.2->notebook==7.0.7->-r requirements.txt (line 2)) (2.0.1)\n",
            "Requirement already satisfied: babel>=2.10 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.22.1->notebook==7.0.7->-r requirements.txt (line 2)) (2.13.1)\n",
            "Requirement already satisfied: json5>=0.9.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.22.1->notebook==7.0.7->-r requirements.txt (line 2)) (0.9.14)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from matplotlib~=3.0->gradio==4.16.0->-r requirements.txt (line 8)) (1.1.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from matplotlib~=3.0->gradio==4.16.0->-r requirements.txt (line 8)) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from matplotlib~=3.0->gradio==4.16.0->-r requirements.txt (line 8)) (4.43.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from matplotlib~=3.0->gradio==4.16.0->-r requirements.txt (line 8)) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from matplotlib~=3.0->gradio==4.16.0->-r requirements.txt (line 8)) (3.1.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from matplotlib~=3.0->gradio==4.16.0->-r requirements.txt (line 8)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from pandas->datasets==2.16.1->-r requirements.txt (line 3)) (2023.3.post1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from pydantic>=2.0->gradio==4.16.0->-r requirements.txt (line 8)) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.10.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from pydantic>=2.0->gradio==4.16.0->-r requirements.txt (line 8)) (2.10.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests>=2.19.0->datasets==2.16.1->-r requirements.txt (line 3)) (3.3.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests>=2.19.0->datasets==2.16.1->-r requirements.txt (line 3)) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests>=2.19.0->datasets==2.16.1->-r requirements.txt (line 3)) (1.26.18)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests>=2.19.0->datasets==2.16.1->-r requirements.txt (line 3)) (2023.7.22)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from typer<1.0,>=0.9->typer[all]<1.0,>=0.9->gradio==4.16.0->-r requirements.txt (line 8)) (8.1.7)\n",
            "Requirement already satisfied: colorama<0.5.0,>=0.4.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from typer[all]<1.0,>=0.9->gradio==4.16.0->-r requirements.txt (line 8)) (0.4.4)\n",
            "Requirement already satisfied: shellingham<2.0.0,>=1.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from typer[all]<1.0,>=0.9->gradio==4.16.0->-r requirements.txt (line 8)) (1.5.4)\n",
            "Requirement already satisfied: rich<14.0.0,>=10.11.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from typer[all]<1.0,>=0.9->gradio==4.16.0->-r requirements.txt (line 8)) (13.7.0)\n",
            "Requirement already satisfied: h11>=0.8 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from uvicorn>=0.14.0->gradio==4.16.0->-r requirements.txt (line 8)) (0.14.0)\n",
            "Requirement already satisfied: starlette<0.36.0,>=0.35.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from fastapi->gradio==4.16.0->-r requirements.txt (line 8)) (0.35.1)\n",
            "Requirement already satisfied: httpcore==1.* in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from httpx->gradio==4.16.0->-r requirements.txt (line 8)) (1.0.2)\n",
            "Requirement already satisfied: sniffio in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from httpx->gradio==4.16.0->-r requirements.txt (line 8)) (1.3.0)\n",
            "Requirement already satisfied: mpmath>=0.19 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from sympy->torch==2.2.0->-r requirements.txt (line 9)) (1.3.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.8.3)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.16.0->-r requirements.txt (line 8)) (2023.7.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.16.0->-r requirements.txt (line 8)) (0.30.2)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.16.0->-r requirements.txt (line 8)) (0.10.6)\n",
            "Requirement already satisfied: platformdirs>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-core!=5.0.*,>=4.12->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (3.11.0)\n",
            "Requirement already satisfied: python-json-logger>=2.0.4 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (2.0.7)\n",
            "Requirement already satisfied: rfc3339-validator in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.1.4)\n",
            "Requirement already satisfied: rfc3986-validator>=0.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.1.1)\n",
            "Requirement already satisfied: beautifulsoup4 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (4.12.2)\n",
            "Requirement already satisfied: bleach!=5.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (6.1.0)\n",
            "Requirement already satisfied: defusedxml in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.7.1)\n",
            "Requirement already satisfied: jupyterlab-pygments in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.2.2)\n",
            "Requirement already satisfied: mistune<4,>=2.0.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (3.0.1)\n",
            "Requirement already satisfied: nbclient>=0.5.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.8.0)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.5.0)\n",
            "Requirement already satisfied: tinycss2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.2.1)\n",
            "Requirement already satisfied: fastjsonschema in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from nbformat>=5.3.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (2.18.1)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.2.8)\n",
            "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio==4.16.0->-r requirements.txt (line 8)) (1.16.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio==4.16.0->-r requirements.txt (line 8)) (3.0.0)\n",
            "Requirement already satisfied: argon2-cffi-bindings in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (21.2.0)\n",
            "Requirement already satisfied: debugpy>=1.6.5 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipykernel->jupyterlab<5,>=4.0.2->notebook==7.0.7->-r requirements.txt (line 2)) (1.8.0)\n",
            "Requirement already satisfied: nest-asyncio in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from ipykernel->jupyterlab<5,>=4.0.2->notebook==7.0.7->-r requirements.txt (line 2)) (1.5.8)\n",
            "Requirement already satisfied: executing>=1.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (1.2.0)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (2.4.1)\n",
            "Requirement already satisfied: pure-eval in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.1.1->-r requirements.txt (line 1)) (0.2.2)\n",
            "Requirement already satisfied: webencodings in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (0.5.1)\n",
            "Requirement already satisfied: fqdn in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.5.1)\n",
            "Requirement already satisfied: isoduration in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (20.11.0)\n",
            "Requirement already satisfied: jsonpointer>1.13 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (2.4)\n",
            "Requirement already satisfied: uri-template in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.3.0)\n",
            "Requirement already satisfied: webcolors>=1.11 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.13)\n",
            "Requirement already satisfied: mdurl~=0.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio==4.16.0->-r requirements.txt (line 8)) (0.1.2)\n",
            "Requirement already satisfied: cffi>=1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.16.0)\n",
            "Requirement already satisfied: soupsieve>1.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (2.5)\n",
            "Requirement already satisfied: pycparser in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (2.21)\n",
            "Requirement already satisfied: arrow>=0.15.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (1.3.0)\n",
            "Requirement already satisfied: types-python-dateutil>=2.8.10 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from arrow>=0.15.0->isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook==7.0.7->-r requirements.txt (line 2)) (2.8.19.14)\n"
          ]
        }
      ],
      "source": [
        "!pip install -r requirements.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f20bf789",
      "metadata": {
        "id": "f20bf789"
      },
      "source": [
        "## Loading and Evaluating a Foundation Model\n",
        "\n",
        "In the cells below, I will load the pre-trained Hugging Face model and evaluate its performance prior to fine-tuning. This step includes loading an appropriate tokenizer and dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4598eee2",
      "metadata": {
        "id": "4598eee2"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b789a0c2",
      "metadata": {
        "id": "b789a0c2"
      },
      "outputs": [],
      "source": [
        "train_size=1_000"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ebf52adb",
      "metadata": {
        "id": "ebf52adb"
      },
      "outputs": [],
      "source": [
        "val_size=train_size//10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b2c55ae4",
      "metadata": {
        "id": "b2c55ae4"
      },
      "outputs": [],
      "source": [
        "test_size=val_size"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7687ef06",
      "metadata": {
        "id": "7687ef06"
      },
      "outputs": [],
      "source": [
        "seed=42"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "poY-pIVXIGRD",
      "metadata": {
        "id": "poY-pIVXIGRD"
      },
      "source": [
        "I will load the dataset in streaming mode to avoid downloading the entire 1TB."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ced74482",
      "metadata": {
        "id": "ced74482"
      },
      "outputs": [],
      "source": [
        "iter_ds=load_dataset(\"codeparrot/github-code\", streaming=True, trust_remote_code=True,\n",
        "                split=\"train\").shuffle(seed=seed,\n",
        "                                       buffer_size=train_size+val_size+test_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "36a5d650",
      "metadata": {
        "id": "36a5d650"
      },
      "outputs": [],
      "source": [
        "iter_train_ds=iter_ds.take(train_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6eb60851",
      "metadata": {
        "id": "6eb60851"
      },
      "outputs": [],
      "source": [
        "iter_val_ds=iter_ds.skip(train_size).take(val_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dc5c566b",
      "metadata": {
        "id": "dc5c566b"
      },
      "outputs": [],
      "source": [
        "iter_test_ds=iter_ds.skip(train_size+val_size).take(test_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dac38f18",
      "metadata": {
        "id": "dac38f18"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6d14a2f4",
      "metadata": {
        "id": "6d14a2f4"
      },
      "outputs": [],
      "source": [
        "model_id = \"facebook/opt-125m\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5810fba9",
      "metadata": {
        "id": "5810fba9"
      },
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_id)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "eda3d146",
      "metadata": {
        "id": "eda3d146"
      },
      "outputs": [],
      "source": [
        "if tokenizer.pad_token is None:\n",
        "  print(\"It was None\")\n",
        "  tokenizer.pad_token = tokenizer.eos_token"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "91c72362",
      "metadata": {
        "id": "91c72362"
      },
      "outputs": [],
      "source": [
        "from transformers import PreTrainedTokenizer"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "YiYpL-Q1IJUt",
      "metadata": {
        "id": "YiYpL-Q1IJUt"
      },
      "source": [
        "I will segment the text so that it can be processed by the model within the context length."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3c8faa9a",
      "metadata": {
        "id": "3c8faa9a"
      },
      "outputs": [],
      "source": [
        "def chunk_and_encode(\n",
        "        samples: dict[str,  str],\n",
        "        tokenizer: PreTrainedTokenizer,\n",
        "        max_len: int,\n",
        "        stride: int,\n",
        "        col_name: str) -> dict[str, list[list[int]]]:\n",
        "    \"\"\"\n",
        "    Split test in chunks and encode them\n",
        "    Args:\n",
        "        samples (dict[str, str]):  batch of data raws from hugging face dataset\n",
        "        tokenizer (PreTrainedTokenizer): hugging face tokenizer\n",
        "        max_len (int): the length of chunk\n",
        "        stride (int): the number of overlapping tokens\n",
        "        col_name (str): the name of the text column\n",
        "    Return:\n",
        "        tokenized chunks (dict[str, list[list[int]]])\n",
        "    \"\"\"\n",
        "    chunks = tokenizer(\n",
        "        samples[col_name],\n",
        "        truncation=True,\n",
        "        padding=True,\n",
        "        max_length=max_len,\n",
        "        stride=stride,\n",
        "        return_overflowing_tokens=True,\n",
        "        )\n",
        "\n",
        "    return {\n",
        "        'input_ids': chunks['input_ids'],\n",
        "        'attention_mask': chunks['attention_mask']\n",
        "        }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8b5b4bcd",
      "metadata": {
        "id": "8b5b4bcd"
      },
      "outputs": [],
      "source": [
        "max_length=2**11"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "44ce57f5",
      "metadata": {
        "id": "44ce57f5"
      },
      "outputs": [],
      "source": [
        "stride=max_length//16"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d3804066",
      "metadata": {
        "id": "d3804066"
      },
      "outputs": [],
      "source": [
        "col_name=\"code\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6c2703ac",
      "metadata": {
        "id": "6c2703ac"
      },
      "outputs": [],
      "source": [
        "from functools import partial"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "83499b73",
      "metadata": {
        "id": "83499b73"
      },
      "outputs": [],
      "source": [
        "process_text = partial(chunk_and_encode,\n",
        "                tokenizer=tokenizer,\n",
        "                max_len=max_length,\n",
        "                stride=stride,\n",
        "                col_name=col_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "51b3474a",
      "metadata": {
        "id": "51b3474a"
      },
      "outputs": [],
      "source": [
        "from datasets import Dataset,IterableDataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bf634de7-84ac-4cfa-a8ad-5270cb96d820",
      "metadata": {
        "id": "bf634de7-84ac-4cfa-a8ad-5270cb96d820"
      },
      "outputs": [],
      "source": [
        "def gen_from_iterable_dataset(iterable_ds: IterableDataset)->dict:\n",
        "    \"\"\"Create a generator from an iterable dataset\"\"\"\n",
        "    yield from iterable_ds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ca5101c6-3da5-40b4-bc35-faa3255b26c4",
      "metadata": {
        "id": "ca5101c6-3da5-40b4-bc35-faa3255b26c4"
      },
      "outputs": [],
      "source": [
        "def create_dataset(iterable_ds: IterableDataset)->Dataset:\n",
        "    \"\"\"Create a dataset from an iterable dataset\"\"\"\n",
        "    iter_token=iterable_ds.map(process_text,\n",
        "                              remove_columns=iter_ds.column_names,\n",
        "                              batched=True)\n",
        "    return Dataset.from_generator(partial(gen_from_iterable_dataset, iter_token))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e23b182b",
      "metadata": {
        "id": "e23b182b",
        "outputId": "c941409d-92e8-4997-eb99-22321d6653c1",
        "colab": {
          "referenced_widgets": [
            "fc92ddeb4a5a4350be06b352fbfff756"
          ]
        }
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fc92ddeb4a5a4350be06b352fbfff756",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "train_ds=create_dataset(iter_train_ds).shuffle(seed=seed)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "54644c11-7de4-422c-93f8-9acb6118ec66",
      "metadata": {
        "id": "54644c11-7de4-422c-93f8-9acb6118ec66",
        "outputId": "12e7f3f9-4360-4e6e-f799-3ff574e007a9",
        "colab": {
          "referenced_widgets": [
            "f1f9a1c9fc174742b84b2799eeab99a6"
          ]
        }
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f1f9a1c9fc174742b84b2799eeab99a6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "val_ds=create_dataset(iter_val_ds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "41287f0c-2cdc-45d4-861c-143fe3bc0456",
      "metadata": {
        "id": "41287f0c-2cdc-45d4-861c-143fe3bc0456",
        "outputId": "898c769e-166e-4713-df50-d16e171865e0",
        "colab": {
          "referenced_widgets": [
            "3708c1d220f94b20a2e48685f660f86c"
          ]
        }
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3708c1d220f94b20a2e48685f660f86c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "test_ds=create_dataset(iter_test_ds)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "gPRV6SZ2KHYg",
      "metadata": {
        "id": "gPRV6SZ2KHYg"
      },
      "source": [
        "I will load the model in NF4 and use double quantization, as described in the QLoRA paper. The computation will be performed using Brain Float 16-bit precision."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cf4f842a",
      "metadata": {
        "id": "cf4f842a"
      },
      "outputs": [],
      "source": [
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "uQYoGKNXrezk",
      "metadata": {
        "id": "uQYoGKNXrezk"
      },
      "outputs": [],
      "source": [
        "from transformers import BitsAndBytesConfig\n",
        "\n",
        "config = BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_quant_type=\"nf4\",\n",
        "    bnb_4bit_use_double_quant=True,\n",
        "    bnb_4bit_compute_dtype=torch.bfloat16,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3YmlQ8OLrzdw",
      "metadata": {
        "id": "3YmlQ8OLrzdw"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModelForCausalLM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e61de881",
      "metadata": {
        "id": "e61de881"
      },
      "outputs": [],
      "source": [
        "model = AutoModelForCausalLM.from_pretrained(model_id, device_map=\"auto\", quantization_config=config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "da478f8f",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "da478f8f",
        "outputId": "b2de492f-07c3-4c0c-912f-5656bacef83d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "OPTForCausalLM(\n",
              "  (model): OPTModel(\n",
              "    (decoder): OPTDecoder(\n",
              "      (embed_tokens): Embedding(50272, 768, padding_idx=1)\n",
              "      (embed_positions): OPTLearnedPositionalEmbedding(2050, 768)\n",
              "      (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (layers): ModuleList(\n",
              "        (0-11): 12 x OPTDecoderLayer(\n",
              "          (self_attn): OPTAttention(\n",
              "            (k_proj): Linear4bit(in_features=768, out_features=768, bias=True)\n",
              "            (v_proj): Linear4bit(in_features=768, out_features=768, bias=True)\n",
              "            (q_proj): Linear4bit(in_features=768, out_features=768, bias=True)\n",
              "            (out_proj): Linear4bit(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (activation_fn): ReLU()\n",
              "          (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear4bit(in_features=768, out_features=3072, bias=True)\n",
              "          (fc2): Linear4bit(in_features=3072, out_features=768, bias=True)\n",
              "          (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=50272, bias=False)\n",
              ")"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "ee67a30f",
      "metadata": {
        "id": "ee67a30f"
      },
      "source": [
        "Perplexity (PPL) is one of the most common metrics for evaluating language models.\n",
        "\n",
        "It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a69ce1a7",
      "metadata": {
        "id": "a69ce1a7"
      },
      "outputs": [],
      "source": [
        "from transformers import PreTrainedModel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e2bb3b6e",
      "metadata": {
        "id": "e2bb3b6e"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e88061ae",
      "metadata": {
        "id": "e88061ae"
      },
      "outputs": [],
      "source": [
        "def evaluate(model: PreTrainedModel,\n",
        "             eval_ds: Dataset,\n",
        "             batch_size: int,\n",
        "            )->dict[str,float]:\n",
        "\n",
        "    \"\"\"\n",
        "    Compute the perplexity of a model over an evaluation dataset\n",
        "    \"\"\"\n",
        "    model.eval()\n",
        "    losses = []\n",
        "    device = model.device\n",
        "\n",
        "    for batch in tqdm(eval_ds.iter(batch_size)):\n",
        "        input_ids=torch.LongTensor(batch[\"input_ids\"]).to(device)\n",
        "        with torch.no_grad():\n",
        "            batch_loss = model(input_ids, labels=input_ids).loss.reshape(1,-1)\n",
        "\n",
        "        losses.append(batch_loss)\n",
        "    loss = torch.mean(torch.cat(losses))\n",
        "    try:\n",
        "        perplexity = torch.exp(loss).item()\n",
        "    except OverflowError:\n",
        "        perplexity = float(\"inf\")\n",
        "    return {\"perplexity\":perplexity}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ruDRSmN3Ix6f",
      "metadata": {
        "id": "ruDRSmN3Ix6f"
      },
      "outputs": [],
      "source": [
        "batch_size=16"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "89da212f",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "89da212f",
        "outputId": "988e46b3-4618-4798-9070-dfa15977271b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "164it [02:42,  1.01it/s]\n"
          ]
        }
      ],
      "source": [
        "base_score=evaluate(model,test_ds,batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b474fe72",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b474fe72",
        "outputId": "6b2acc79-fff7-41fa-937a-9940a2b94a59"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'perplexity': 24.5625}"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "base_score"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "74f8704b-8a3f-4492-8560-81273a02cd53",
      "metadata": {
        "id": "74f8704b-8a3f-4492-8560-81273a02cd53"
      },
      "source": [
        "Free GPU RAM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "rRMWDSaoVt1I",
      "metadata": {
        "id": "rRMWDSaoVt1I"
      },
      "outputs": [],
      "source": [
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f3db148d",
      "metadata": {
        "id": "f3db148d"
      },
      "source": [
        "## Performing Parameter-Efficient Fine-Tuning\n",
        "\n",
        "In the cells below, I will create a PEFT model from your loaded model, run a training loop, and save the PEFT model weights."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3e92f8af",
      "metadata": {
        "id": "3e92f8af"
      },
      "outputs": [],
      "source": [
        "model = AutoModelForCausalLM.from_pretrained(model_id, device_map=\"auto\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fb7f6534",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fb7f6534",
        "outputId": "95ff5a00-a099-47bb-a9fa-ba9fbcd87623"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "OPTForCausalLM(\n",
              "  (model): OPTModel(\n",
              "    (decoder): OPTDecoder(\n",
              "      (embed_tokens): Embedding(50272, 768, padding_idx=1)\n",
              "      (embed_positions): OPTLearnedPositionalEmbedding(2050, 768)\n",
              "      (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (layers): ModuleList(\n",
              "        (0-11): 12 x OPTDecoderLayer(\n",
              "          (self_attn): OPTAttention(\n",
              "            (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (activation_fn): ReLU()\n",
              "          (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=50272, bias=False)\n",
              ")"
            ]
          },
          "execution_count": 41,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "91a49281-f6f6-4192-af44-e5cc2e65fef0",
      "metadata": {
        "id": "91a49281-f6f6-4192-af44-e5cc2e65fef0"
      },
      "outputs": [],
      "source": [
        "from peft import LoftQConfig, LoraConfig, get_peft_model"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "5ecf7d23-73f5-49e4-bf32-384cb496dcaf",
      "metadata": {
        "id": "5ecf7d23-73f5-49e4-bf32-384cb496dcaf"
      },
      "source": [
        "Adjust the quantization bit to 4 and incorporate 10 iterations of LoftQ in the configuration."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "949db024",
      "metadata": {
        "id": "949db024"
      },
      "outputs": [],
      "source": [
        "loftq_config = LoftQConfig(loftq_bits=4,loftq_iter=10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ETJ2Qrjc0WeH",
      "metadata": {
        "id": "ETJ2Qrjc0WeH"
      },
      "outputs": [],
      "source": [
        "lora_config = LoraConfig(\n",
        "    init_lora_weights=\"loftq\",\n",
        "    loftq_config=loftq_config,\n",
        "    r=64,\n",
        "    lora_alpha=32,\n",
        "    target_modules=\"all-linear\",\n",
        "    lora_dropout=0.01,\n",
        "    bias=\"none\",\n",
        "    task_type=\"CAUSAL_LM\"\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "E-swK20o0ppj",
      "metadata": {
        "id": "E-swK20o0ppj"
      },
      "outputs": [],
      "source": [
        "model = get_peft_model(model, lora_config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "BUIRl7c_2BvX",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BUIRl7c_2BvX",
        "outputId": "e4f1399f-558c-4ba2-83de-eb802745d6c3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "trainable params: 10,616,832 || all params: 135,856,128 || trainable%: 7.814761215629522\n"
          ]
        }
      ],
      "source": [
        "model.print_trainable_parameters()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6nW2gu321IEB",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6nW2gu321IEB",
        "outputId": "7ee6ea0a-3289-403e-b885-b904c97019ba"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "PeftModelForCausalLM(\n",
              "  (base_model): LoraModel(\n",
              "    (model): OPTForCausalLM(\n",
              "      (model): OPTModel(\n",
              "        (decoder): OPTDecoder(\n",
              "          (embed_tokens): Embedding(50272, 768, padding_idx=1)\n",
              "          (embed_positions): OPTLearnedPositionalEmbedding(2050, 768)\n",
              "          (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "          (layers): ModuleList(\n",
              "            (0-11): 12 x OPTDecoderLayer(\n",
              "              (self_attn): OPTAttention(\n",
              "                (k_proj): lora.Linear(\n",
              "                  (base_layer): Linear(in_features=768, out_features=768, bias=True)\n",
              "                  (lora_dropout): ModuleDict(\n",
              "                    (default): Dropout(p=0.01, inplace=False)\n",
              "                  )\n",
              "                  (lora_A): ModuleDict(\n",
              "                    (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                  )\n",
              "                  (lora_B): ModuleDict(\n",
              "                    (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                  )\n",
              "                  (lora_embedding_A): ParameterDict()\n",
              "                  (lora_embedding_B): ParameterDict()\n",
              "                )\n",
              "                (v_proj): lora.Linear(\n",
              "                  (base_layer): Linear(in_features=768, out_features=768, bias=True)\n",
              "                  (lora_dropout): ModuleDict(\n",
              "                    (default): Dropout(p=0.01, inplace=False)\n",
              "                  )\n",
              "                  (lora_A): ModuleDict(\n",
              "                    (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                  )\n",
              "                  (lora_B): ModuleDict(\n",
              "                    (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                  )\n",
              "                  (lora_embedding_A): ParameterDict()\n",
              "                  (lora_embedding_B): ParameterDict()\n",
              "                )\n",
              "                (q_proj): lora.Linear(\n",
              "                  (base_layer): Linear(in_features=768, out_features=768, bias=True)\n",
              "                  (lora_dropout): ModuleDict(\n",
              "                    (default): Dropout(p=0.01, inplace=False)\n",
              "                  )\n",
              "                  (lora_A): ModuleDict(\n",
              "                    (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                  )\n",
              "                  (lora_B): ModuleDict(\n",
              "                    (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                  )\n",
              "                  (lora_embedding_A): ParameterDict()\n",
              "                  (lora_embedding_B): ParameterDict()\n",
              "                )\n",
              "                (out_proj): lora.Linear(\n",
              "                  (base_layer): Linear(in_features=768, out_features=768, bias=True)\n",
              "                  (lora_dropout): ModuleDict(\n",
              "                    (default): Dropout(p=0.01, inplace=False)\n",
              "                  )\n",
              "                  (lora_A): ModuleDict(\n",
              "                    (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                  )\n",
              "                  (lora_B): ModuleDict(\n",
              "                    (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                  )\n",
              "                  (lora_embedding_A): ParameterDict()\n",
              "                  (lora_embedding_B): ParameterDict()\n",
              "                )\n",
              "              )\n",
              "              (activation_fn): ReLU()\n",
              "              (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (fc1): lora.Linear(\n",
              "                (base_layer): Linear(in_features=768, out_features=3072, bias=True)\n",
              "                (lora_dropout): ModuleDict(\n",
              "                  (default): Dropout(p=0.01, inplace=False)\n",
              "                )\n",
              "                (lora_A): ModuleDict(\n",
              "                  (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                )\n",
              "                (lora_B): ModuleDict(\n",
              "                  (default): Linear(in_features=64, out_features=3072, bias=False)\n",
              "                )\n",
              "                (lora_embedding_A): ParameterDict()\n",
              "                (lora_embedding_B): ParameterDict()\n",
              "              )\n",
              "              (fc2): lora.Linear(\n",
              "                (base_layer): Linear(in_features=3072, out_features=768, bias=True)\n",
              "                (lora_dropout): ModuleDict(\n",
              "                  (default): Dropout(p=0.01, inplace=False)\n",
              "                )\n",
              "                (lora_A): ModuleDict(\n",
              "                  (default): Linear(in_features=3072, out_features=64, bias=False)\n",
              "                )\n",
              "                (lora_B): ModuleDict(\n",
              "                  (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                )\n",
              "                (lora_embedding_A): ParameterDict()\n",
              "                (lora_embedding_B): ParameterDict()\n",
              "              )\n",
              "              (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            )\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (lm_head): Linear(in_features=768, out_features=50272, bias=False)\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "XOa3NknE5Lld",
      "metadata": {
        "id": "XOa3NknE5Lld"
      },
      "outputs": [],
      "source": [
        "from transformers import DataCollatorForLanguageModeling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3yCVWGBa2EE-",
      "metadata": {
        "id": "3yCVWGBa2EE-"
      },
      "outputs": [],
      "source": [
        "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c-RBrxAG3GJ8",
      "metadata": {
        "id": "c-RBrxAG3GJ8"
      },
      "outputs": [],
      "source": [
        "from transformers import TrainingArguments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "JOc20SvhJJtt",
      "metadata": {
        "id": "JOc20SvhJJtt"
      },
      "outputs": [],
      "source": [
        "import torch.multiprocessing as mp"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "de45243c-5bf1-45f6-accb-82319023794c",
      "metadata": {
        "id": "de45243c-5bf1-45f6-accb-82319023794c"
      },
      "outputs": [],
      "source": [
        "model_name = model_id.split(\"/\")[1]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a6339037-2aa7-4817-b5b2-726a980f1e58",
      "metadata": {
        "id": "a6339037-2aa7-4817-b5b2-726a980f1e58"
      },
      "source": [
        "I set the optimizer to lion with have record for only one momentom and I will quantized to 8bit and move the paged memory to cpu."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fp0Z8HdR5PxE",
      "metadata": {
        "id": "fp0Z8HdR5PxE"
      },
      "outputs": [],
      "source": [
        "training_args = TrainingArguments(\n",
        "        f\"{model_name}-finetuned-lora\",\n",
        "        optim=\"paged_lion_8bit\",\n",
        "        learning_rate=5e-6,\n",
        "        weight_decay=0.01,\n",
        "        auto_find_batch_size=True,\n",
        "        gradient_accumulation_steps=4,\n",
        "        num_train_epochs=3,\n",
        "        evaluation_strategy=\"epoch\",\n",
        "        save_strategy=\"epoch\",\n",
        "        dataloader_num_workers=mp.cpu_count(),\n",
        "        fp16=True,\n",
        "        logging_steps=100,\n",
        "        load_best_model_at_end=True,\n",
        "        metric_for_best_model=\"eval_loss\",\n",
        "        push_to_hub=False,\n",
        "        greater_is_better=False,\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5rZ8eHhdA90N",
      "metadata": {
        "id": "5rZ8eHhdA90N"
      },
      "outputs": [],
      "source": [
        "from transformers import Trainer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "u53rFXrLA1KU",
      "metadata": {
        "id": "u53rFXrLA1KU"
      },
      "outputs": [],
      "source": [
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_ds,\n",
        "    eval_dataset= val_ds,\n",
        "    data_collator=data_collator,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8c315c9b-bee2-49b7-a4d3-c81abc4ff1cc",
      "metadata": {
        "id": "8c315c9b-bee2-49b7-a4d3-c81abc4ff1cc"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "nAeg-QF2Bgxf",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 159
        },
        "id": "nAeg-QF2Bgxf",
        "outputId": "bf5090b2-f498-4112-cbaa-9090deddefe7"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='25608' max='25608' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [25608/25608 5:10:23, Epoch 2/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.600400</td>\n",
              "      <td>1.527814</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.468100</td>\n",
              "      <td>1.490909</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.390700</td>\n",
              "      <td>1.482334</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "TrainOutput(global_step=25608, training_loss=1.5175122290765892, metrics={'train_runtime': 18624.7305, 'train_samples_per_second': 5.5, 'train_steps_per_second': 1.375, 'total_flos': 1.2042190267298611e+17, 'train_loss': 1.5175122290765892, 'epoch': 3.0})"
            ]
          },
          "execution_count": 57,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "trainer.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dfvltZPOdR2D",
      "metadata": {
        "id": "dfvltZPOdR2D"
      },
      "outputs": [],
      "source": [
        " model_saved = f\"best-{model_name}-finetuned-lora\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "vNeQjJiSc4UB",
      "metadata": {
        "id": "vNeQjJiSc4UB"
      },
      "outputs": [],
      "source": [
        "model.save_pretrained(model_saved)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fefc49a4-da32-4dae-bc91-0d209ac2015d",
      "metadata": {
        "id": "fefc49a4-da32-4dae-bc91-0d209ac2015d"
      },
      "outputs": [],
      "source": [
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "9e995479",
      "metadata": {
        "id": "9e995479"
      },
      "source": [
        "## Performing Inference with a PEFT Model\n",
        "\n",
        "In the cells below, I will load the saved PEFT model weights and evaluate the performance of the trained PEFT model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "16e7dbe9-cf68-4d6b-8c4b-ef4608652157",
      "metadata": {
        "id": "16e7dbe9-cf68-4d6b-8c4b-ef4608652157"
      },
      "outputs": [],
      "source": [
        "from peft import AutoPeftModelForCausalLM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "031c10e7",
      "metadata": {
        "id": "031c10e7"
      },
      "outputs": [],
      "source": [
        "model = AutoPeftModelForCausalLM.from_pretrained(model_saved, config=lora_config, device_map=\"auto\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3ebd553d",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ebd553d",
        "outputId": "f4f75282-94e0-4549-b30c-ca5003717fca"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "PeftModelForCausalLM(\n",
              "  (base_model): LoraModel(\n",
              "    (model): OPTForCausalLM(\n",
              "      (model): OPTModel(\n",
              "        (decoder): OPTDecoder(\n",
              "          (embed_tokens): Embedding(50272, 768, padding_idx=1)\n",
              "          (embed_positions): OPTLearnedPositionalEmbedding(2050, 768)\n",
              "          (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "          (layers): ModuleList(\n",
              "            (0-11): 12 x OPTDecoderLayer(\n",
              "              (self_attn): OPTAttention(\n",
              "                (k_proj): lora.Linear(\n",
              "                  (base_layer): Linear(in_features=768, out_features=768, bias=True)\n",
              "                  (lora_dropout): ModuleDict(\n",
              "                    (default): Dropout(p=0.01, inplace=False)\n",
              "                  )\n",
              "                  (lora_A): ModuleDict(\n",
              "                    (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                  )\n",
              "                  (lora_B): ModuleDict(\n",
              "                    (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                  )\n",
              "                  (lora_embedding_A): ParameterDict()\n",
              "                  (lora_embedding_B): ParameterDict()\n",
              "                )\n",
              "                (v_proj): lora.Linear(\n",
              "                  (base_layer): Linear(in_features=768, out_features=768, bias=True)\n",
              "                  (lora_dropout): ModuleDict(\n",
              "                    (default): Dropout(p=0.01, inplace=False)\n",
              "                  )\n",
              "                  (lora_A): ModuleDict(\n",
              "                    (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                  )\n",
              "                  (lora_B): ModuleDict(\n",
              "                    (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                  )\n",
              "                  (lora_embedding_A): ParameterDict()\n",
              "                  (lora_embedding_B): ParameterDict()\n",
              "                )\n",
              "                (q_proj): lora.Linear(\n",
              "                  (base_layer): Linear(in_features=768, out_features=768, bias=True)\n",
              "                  (lora_dropout): ModuleDict(\n",
              "                    (default): Dropout(p=0.01, inplace=False)\n",
              "                  )\n",
              "                  (lora_A): ModuleDict(\n",
              "                    (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                  )\n",
              "                  (lora_B): ModuleDict(\n",
              "                    (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                  )\n",
              "                  (lora_embedding_A): ParameterDict()\n",
              "                  (lora_embedding_B): ParameterDict()\n",
              "                )\n",
              "                (out_proj): lora.Linear(\n",
              "                  (base_layer): Linear(in_features=768, out_features=768, bias=True)\n",
              "                  (lora_dropout): ModuleDict(\n",
              "                    (default): Dropout(p=0.01, inplace=False)\n",
              "                  )\n",
              "                  (lora_A): ModuleDict(\n",
              "                    (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                  )\n",
              "                  (lora_B): ModuleDict(\n",
              "                    (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                  )\n",
              "                  (lora_embedding_A): ParameterDict()\n",
              "                  (lora_embedding_B): ParameterDict()\n",
              "                )\n",
              "              )\n",
              "              (activation_fn): ReLU()\n",
              "              (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (fc1): lora.Linear(\n",
              "                (base_layer): Linear(in_features=768, out_features=3072, bias=True)\n",
              "                (lora_dropout): ModuleDict(\n",
              "                  (default): Dropout(p=0.01, inplace=False)\n",
              "                )\n",
              "                (lora_A): ModuleDict(\n",
              "                  (default): Linear(in_features=768, out_features=64, bias=False)\n",
              "                )\n",
              "                (lora_B): ModuleDict(\n",
              "                  (default): Linear(in_features=64, out_features=3072, bias=False)\n",
              "                )\n",
              "                (lora_embedding_A): ParameterDict()\n",
              "                (lora_embedding_B): ParameterDict()\n",
              "              )\n",
              "              (fc2): lora.Linear(\n",
              "                (base_layer): Linear(in_features=3072, out_features=768, bias=True)\n",
              "                (lora_dropout): ModuleDict(\n",
              "                  (default): Dropout(p=0.01, inplace=False)\n",
              "                )\n",
              "                (lora_A): ModuleDict(\n",
              "                  (default): Linear(in_features=3072, out_features=64, bias=False)\n",
              "                )\n",
              "                (lora_B): ModuleDict(\n",
              "                  (default): Linear(in_features=64, out_features=768, bias=False)\n",
              "                )\n",
              "                (lora_embedding_A): ParameterDict()\n",
              "                (lora_embedding_B): ParameterDict()\n",
              "              )\n",
              "              (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            )\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (lm_head): Linear(in_features=768, out_features=50272, bias=False)\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "execution_count": 63,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e475c790-bfbd-4e52-97e7-ac9eb8121244",
      "metadata": {
        "id": "e475c790-bfbd-4e52-97e7-ac9eb8121244"
      },
      "source": [
        "I merged the QLoRA weights with the original ones to enable faster inference."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8b58ba20-c528-4a18-afa5-40b868e9df79",
      "metadata": {
        "id": "8b58ba20-c528-4a18-afa5-40b868e9df79"
      },
      "outputs": [],
      "source": [
        "model = model.merge_and_unload()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cfb5a17f-6db4-4ab2-ba8b-e1e9efc5ad69",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cfb5a17f-6db4-4ab2-ba8b-e1e9efc5ad69",
        "outputId": "ea293e93-5d3c-4b50-f3f4-f204d541aabb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "OPTForCausalLM(\n",
              "  (model): OPTModel(\n",
              "    (decoder): OPTDecoder(\n",
              "      (embed_tokens): Embedding(50272, 768, padding_idx=1)\n",
              "      (embed_positions): OPTLearnedPositionalEmbedding(2050, 768)\n",
              "      (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (layers): ModuleList(\n",
              "        (0-11): 12 x OPTDecoderLayer(\n",
              "          (self_attn): OPTAttention(\n",
              "            (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (activation_fn): ReLU()\n",
              "          (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=50272, bias=False)\n",
              ")"
            ]
          },
          "execution_count": 65,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "H0NKE9eQgIt1",
      "metadata": {
        "id": "H0NKE9eQgIt1"
      },
      "outputs": [],
      "source": [
        "batch_size=8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a0132683-8f71-44fb-840f-0e604354a33c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a0132683-8f71-44fb-840f-0e604354a33c",
        "outputId": "da5be471-26aa-463c-ba21-451155f806a6"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "327it [03:14,  1.68it/s]\n"
          ]
        }
      ],
      "source": [
        "base_score=evaluate(model,test_ds,batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5e3bd1ef-55df-4b9c-95d1-7374f23d40a3",
      "metadata": {
        "id": "5e3bd1ef-55df-4b9c-95d1-7374f23d40a3",
        "outputId": "73cbc49d-23a8-4198-d986-4738fc985510"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'perplexity': 9.781842231750488}"
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "base_score"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e6473cf4-e9a7-4fd2-8bc5-d955dd27733c",
      "metadata": {
        "id": "e6473cf4-e9a7-4fd2-8bc5-d955dd27733c"
      },
      "source": [
        "We can see that fine-tuning has improved perplexity."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "JPf7HLjLkPuT",
      "metadata": {
        "id": "JPf7HLjLkPuT"
      },
      "outputs": [],
      "source": [
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3MY8Vv_nQ110",
      "metadata": {
        "id": "3MY8Vv_nQ110"
      },
      "source": [
        "Try it yourself"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5b96642c-9695-4432-9eb5-c62e6546e95a",
      "metadata": {
        "id": "5b96642c-9695-4432-9eb5-c62e6546e95a",
        "tags": [],
        "outputId": "378adc37-f3f1-4610-a4a3-7fed24adffc2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Running on local URL:  http://127.0.0.1:7860\n",
            "Sagemaker notebooks may require sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
            "\n",
            "Running on public URL: https://503b9c0a530b2e72c1.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div><iframe src=\"https://503b9c0a530b2e72c1.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 70,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from transformers import pipeline\n",
        "import gradio as gr\n",
        "\n",
        "pipe = pipeline(\"text-generation\", model=model, tokenizer=tokenizer)\n",
        "\n",
        "demo = gr.Interface.from_pipeline(pipe)\n",
        "demo.launch()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "conda_pytorch_p310",
      "language": "python",
      "name": "conda_pytorch_p310"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.13"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}