FFT_PEFT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "475a2904-98e5-4cfd-bef3-3af9678c290d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset, load_metric\n",
    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, GenerationConfig, TrainingArguments, Trainer\n",
    "import torch\n",
    "import time\n",
    "import evaluate\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5a829a23-effb-45cd-8a06-8b9466433caf",
   "metadata": {},
   "outputs": [],
   "source": [
    "rouge = evaluate.load('rouge')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e6f76402-ba46-4a8c-aa2c-3116076b1b9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'dialogue', 'summary'],\n",
       "        num_rows: 14732\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'dialogue', 'summary'],\n",
       "        num_rows: 819\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'dialogue', 'summary'],\n",
       "        num_rows: 818\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "huggingface_dataset_name = \"samsum\"\n",
    "\n",
    "dataset = load_dataset(huggingface_dataset_name)\n",
    "\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3c2106a9-a8b2-41f8-a46d-a543bc9746c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name='google/flan-t5-base'\n",
    "\n",
    "original_model = AutoModelForSeq2SeqLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f22442ce-fb1f-461d-bbf9-ff0e66601784",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable model parameters: 247577856\n",
      "all model parameters: 247577856\n",
      "percentage of trainable model parameters: 100.00%\n"
     ]
    }
   ],
   "source": [
    "def print_number_of_trainable_model_parameters(model):\n",
    "    trainable_model_params = 0\n",
    "    all_model_params = 0\n",
    "    for _, param in model.named_parameters():\n",
    "        all_model_params += param.numel()\n",
    "        if param.requires_grad:\n",
    "            trainable_model_params += param.numel()\n",
    "    return f\"trainable model parameters: {trainable_model_params}\\nall model parameters: {all_model_params}\\npercentage of trainable model parameters: {100 * trainable_model_params / all_model_params:.2f}%\"\n",
    "\n",
    "print(print_number_of_trainable_model_parameters(original_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5772369f-201f-44e8-b7b2-5bca826b9d0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_function(example):\n",
    "    start_prompt = 'Summarize the following conversation.\\n\\n'\n",
    "    end_prompt = '\\n\\nSummary: '\n",
    "    prompt = [start_prompt + dialogue + end_prompt for dialogue in example[\"dialogue\"]]\n",
    "    example['input_ids'] = tokenizer(prompt, padding=\"max_length\", truncation=True, return_tensors=\"pt\").input_ids\n",
    "    example['labels'] = tokenizer(example[\"summary\"], padding=\"max_length\", truncation=True, return_tensors=\"pt\").input_ids\n",
    "    \n",
    "    return example\n",
    "\n",
    "# The dataset actually contains 3 diff splits: train, validation, test.\n",
    "# The tokenize_function code is handling all data across all splits in batches.\n",
    "tokenized_datasets = dataset.map(tokenize_function, batched=True)\n",
    "tokenized_datasets = tokenized_datasets.remove_columns(['id',  'dialogue', 'summary',])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "90cda14c-ba03-41e2-9a5c-be291901da9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shapes of the datasets:\n",
      "Training: (14732, 2)\n",
      "Validation: (818, 2)\n",
      "Test: (819, 2)\n",
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['input_ids', 'labels'],\n",
      "        num_rows: 14732\n",
      "    })\n",
      "    test: Dataset({\n",
      "        features: ['input_ids', 'labels'],\n",
      "        num_rows: 819\n",
      "    })\n",
      "    validation: Dataset({\n",
      "        features: ['input_ids', 'labels'],\n",
      "        num_rows: 818\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "print(f\"Shapes of the datasets:\")\n",
    "print(f\"Training: {tokenized_datasets['train'].shape}\")\n",
    "print(f\"Validation: {tokenized_datasets['validation'].shape}\")\n",
    "print(f\"Test: {tokenized_datasets['test'].shape}\")\n",
    "\n",
    "print(tokenized_datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "491df3e1-b23c-4d1a-8bd6-0cb9a5ce2bf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "instruct_model = AutoModelForSeq2SeqLM.from_pretrained(\"dialogue-summary-training-1692394116\", torch_dtype=torch.bfloat16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bf8d0311-2fbf-4655-be22-87294af798ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import PeftModel, PeftConfig\n",
    "\n",
    "peft_model_base = AutoModelForSeq2SeqLM.from_pretrained(\"google/flan-t5-base\", torch_dtype=torch.bfloat16)\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"google/flan-t5-base\")\n",
    "\n",
    "peft_model = PeftModel.from_pretrained(peft_model_base, \n",
    "                                       'peft-dialogue-summary-checkpoint-local/', \n",
    "                                       torch_dtype=torch.bfloat16,\n",
    "                                       is_trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7b65568d-eae5-48a6-bdac-92234af571d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dash_line = '-'.join('' for x in range(100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "25147a66-1d27-466b-868e-e44e965b3e9b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------------------------\n",
      "BASELINE HUMAN SUMMARY:\n",
      "Sam won't finish work till 5. Sam is bringing him over about 9 am. Sam will see Abdellilah in the morning. \n",
      "---------------------------------------------------------------------------------------------------\n",
      "ORIGINAL MODEL:\n",
      "Sam is at work. He finishes at 5 and is not bringing Abdellilah over tonight. Sam will bring Abdellilah to work at about 9.\n",
      "---------------------------------------------------------------------------------------------------\n",
      "INSTRUCT MODEL:\n",
      "Sam is working at 5 and will bring Abdellilah over at 9 tonight.\n",
      "---------------------------------------------------------------------------------------------------\n",
      "PEFT MODEL: Sam finishes work at 5 and is not bringing Abdellilah over tonight. Sam will bring Abdellilah to work at about 9.\n"
     ]
    }
   ],
   "source": [
    "index = 200\n",
    "dialogue = dataset['test'][index]['dialogue']\n",
    "human_baseline_summary = dataset['test'][index]['summary']\n",
    "\n",
    "prompt = f\"\"\"\n",
    "Summarize the following conversation.\n",
    "\n",
    "{dialogue}\n",
    "\n",
    "Summary: \"\"\"\n",
    "\n",
    "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
    "\n",
    "original_model_outputs = original_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\n",
    "original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "instruct_model_outputs = instruct_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\n",
    "instruct_model_text_output = tokenizer.decode(instruct_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "peft_model_outputs = peft_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\n",
    "peft_model_text_output = tokenizer.decode(peft_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "print(dash_line)\n",
    "print(f'BASELINE HUMAN SUMMARY:\\n{human_baseline_summary}')\n",
    "print(dash_line)\n",
    "print(f'ORIGINAL MODEL:\\n{original_model_text_output}')\n",
    "print(dash_line)\n",
    "print(f'INSTRUCT MODEL:\\n{instruct_model_text_output}')\n",
    "print(dash_line)\n",
    "print(f'PEFT MODEL: {peft_model_text_output}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dc67f1b0-68e1-4545-a777-ebddab6106c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>human_baseline_summaries</th>\n",
       "      <th>original_model_summaries</th>\n",
       "      <th>peft_model_summaries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hannah needs Betty's number but Amanda doesn't...</td>\n",
       "      <td>Amanda can't find Betty's number. Amanda will ...</td>\n",
       "      <td>Amanda can't find Betty's number. Amanda will ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Eric and Rob are going to watch a stand-up on ...</td>\n",
       "      <td>Eric and Rob are watching a stand-up. Eric and...</td>\n",
       "      <td>Eric and Rob are watching a stand-up. Eric and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lenny can't decide which trousers to buy. Bob ...</td>\n",
       "      <td>Lenny wants to buy two pairs of purple trouser...</td>\n",
       "      <td>Lenny wants to buy two pairs of purple trouser...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Emma will be home soon and she will let Will k...</td>\n",
       "      <td>Emma will be home soon. Will will pick her up.</td>\n",
       "      <td>Emma will be home soon. Will will pick her up.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jane is in Warsaw. Ollie and Jane has a party....</td>\n",
       "      <td>Jane lost her calendar. Ollie and Jane have lu...</td>\n",
       "      <td>Jane lost her calendar. Ollie and Jane have lu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Hilary has the keys to the apartment. Benjamin...</td>\n",
       "      <td>Hilary and Elliot are meeting at the conferenc...</td>\n",
       "      <td>Hilary and Elliot are meeting at the conferenc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Payton provides Max with websites selling clot...</td>\n",
       "      <td>Payton likes shopping but he doesn't always bu...</td>\n",
       "      <td>Payton likes shopping but he doesn't always bu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Rita and Tina are bored at work and have still...</td>\n",
       "      <td>Rita is tired and is not able to concentrate a...</td>\n",
       "      <td>Rita is tired and is not able to concentrate a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Beatrice wants to buy Leo a scarf, but he does...</td>\n",
       "      <td>Beatrice is in town, shopping. She has a scarf...</td>\n",
       "      <td>Beatrice is in town, shopping. She has a scarf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Eric doesn't know if his parents let him go to...</td>\n",
       "      <td>Eric is coming to Ivan's brother's wedding. Er...</td>\n",
       "      <td>Eric is coming to Ivan's brother's wedding. Er...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            human_baseline_summaries  \\\n",
       "0  Hannah needs Betty's number but Amanda doesn't...   \n",
       "1  Eric and Rob are going to watch a stand-up on ...   \n",
       "2  Lenny can't decide which trousers to buy. Bob ...   \n",
       "3  Emma will be home soon and she will let Will k...   \n",
       "4  Jane is in Warsaw. Ollie and Jane has a party....   \n",
       "5  Hilary has the keys to the apartment. Benjamin...   \n",
       "6  Payton provides Max with websites selling clot...   \n",
       "7  Rita and Tina are bored at work and have still...   \n",
       "8  Beatrice wants to buy Leo a scarf, but he does...   \n",
       "9  Eric doesn't know if his parents let him go to...   \n",
       "\n",
       "                            original_model_summaries  \\\n",
       "0  Amanda can't find Betty's number. Amanda will ...   \n",
       "1  Eric and Rob are watching a stand-up. Eric and...   \n",
       "2  Lenny wants to buy two pairs of purple trouser...   \n",
       "3     Emma will be home soon. Will will pick her up.   \n",
       "4  Jane lost her calendar. Ollie and Jane have lu...   \n",
       "5  Hilary and Elliot are meeting at the conferenc...   \n",
       "6  Payton likes shopping but he doesn't always bu...   \n",
       "7  Rita is tired and is not able to concentrate a...   \n",
       "8  Beatrice is in town, shopping. She has a scarf...   \n",
       "9  Eric is coming to Ivan's brother's wedding. Er...   \n",
       "\n",
       "                                peft_model_summaries  \n",
       "0  Amanda can't find Betty's number. Amanda will ...  \n",
       "1  Eric and Rob are watching a stand-up. Eric and...  \n",
       "2  Lenny wants to buy two pairs of purple trouser...  \n",
       "3     Emma will be home soon. Will will pick her up.  \n",
       "4  Jane lost her calendar. Ollie and Jane have lu...  \n",
       "5  Hilary and Elliot are meeting at the conferenc...  \n",
       "6  Payton likes shopping but he doesn't always bu...  \n",
       "7  Rita is tired and is not able to concentrate a...  \n",
       "8  Beatrice is in town, shopping. She has a scarf...  \n",
       "9  Eric is coming to Ivan's brother's wedding. Er...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dialogues = dataset['test'][0:10]['dialogue']\n",
    "human_baseline_summaries = dataset['test'][0:10]['summary']\n",
    "\n",
    "original_model_summaries = []\n",
    "instruct_model_summaries = []\n",
    "peft_model_summaries = []\n",
    "\n",
    "for idx, dialogue in enumerate(dialogues):\n",
    "    prompt = f\"\"\"\n",
    "Summarize the following conversation.\n",
    "\n",
    "{dialogue}\n",
    "\n",
    "Summary: \"\"\"\n",
    "    \n",
    "    input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
    "\n",
    "    human_baseline_text_output = human_baseline_summaries[idx]\n",
    "    \n",
    "    original_model_outputs = original_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    instruct_model_outputs = instruct_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    instruct_model_text_output = tokenizer.decode(instruct_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    peft_model_outputs = peft_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    peft_model_text_output = tokenizer.decode(peft_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    original_model_summaries.append(original_model_text_output)\n",
    "    instruct_model_summaries.append(instruct_model_text_output)\n",
    "    peft_model_summaries.append(peft_model_text_output)\n",
    "\n",
    "zipped_summaries = list(zip(human_baseline_summaries, original_model_summaries, peft_model_summaries))\n",
    " \n",
    "df = pd.DataFrame(zipped_summaries, columns = ['human_baseline_summaries', 'original_model_summaries', 'peft_model_summaries'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6fec3df3-be5b-4bcd-a9c8-b90ba44e071a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ORIGINAL MODEL:\n",
      "{'rouge1': 0.47194198278924726, 'rouge2': 0.23162475619947837, 'rougeL': 0.37803363632249615, 'rougeLsum': 0.3778719127883262}\n",
      "INSTRUCT MODEL:\n",
      "{'rouge1': 0.4846259662103536, 'rouge2': 0.2762756042079696, 'rougeL': 0.40429002404429004, 'rougeLsum': 0.4033549184740315}\n",
      "PEFT MODEL:\n",
      "{'rouge1': 0.4733787055598633, 'rouge2': 0.23056743645098504, 'rougeL': 0.3801720208397835, 'rougeLsum': 0.37960358600779665}\n"
     ]
    }
   ],
   "source": [
    "# rouge = evaluate.load('rouge')\n",
    "\n",
    "original_model_results = rouge.compute(\n",
    "    predictions=original_model_summaries,\n",
    "    references=human_baseline_summaries[0:len(original_model_summaries)],\n",
    "    use_aggregator=True,\n",
    "    use_stemmer=True,\n",
    ")\n",
    "\n",
    "\n",
    "instruct_model_results = rouge.compute(\n",
    "    predictions=instruct_model_summaries,\n",
    "    references=human_baseline_summaries[0:len(instruct_model_summaries)],\n",
    "    use_aggregator=True,\n",
    "    use_stemmer=True,\n",
    ")\n",
    "peft_model_results = rouge.compute(\n",
    "    predictions=peft_model_summaries,\n",
    "    references=human_baseline_summaries[0:len(peft_model_summaries)],\n",
    "    use_aggregator=True,\n",
    "    use_stemmer=True,\n",
    ")\n",
    "\n",
    "print('ORIGINAL MODEL:')\n",
    "print(original_model_results)\n",
    "dialogues = dataset['test'][0:10]['dialogue']\n",
    "human_baseline_summaries = dataset['test'][0:10]['summary']\n",
    "\n",
    "original_model_summaries = []\n",
    "instruct_model_summaries = []\n",
    "peft_model_summaries = []\n",
    "\n",
    "for idx, dialogue in enumerate(dialogues):\n",
    "    prompt = f\"\"\"\n",
    "Summarize the following conversation.\n",
    "\n",
    "\n",
    "{dialogue}\n",
    "\n",
    "Summary: \"\"\"\n",
    "    \n",
    "    input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
    "\n",
    "    human_baseline_text_output = human_baseline_summaries[idx]\n",
    "    \n",
    "    original_model_outputs = original_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    instruct_model_outputs = instruct_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    instruct_model_text_output = tokenizer.decode(instruct_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    peft_model_outputs = peft_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    peft_model_text_output = tokenizer.decode(peft_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    original_model_summaries.append(original_model_text_output)\n",
    "    instruct_model_summaries.append(instruct_model_text_output)\n",
    "    peft_model_summaries.append(peft_model_text_output)\n",
    "\n",
    "zipped_summaries = list(zip(human_baseline_summaries, original_model_summaries, peft_model_summaries))\n",
    " \n",
    "df = pd.DataFrame(zipped_summaries, columns = ['human_baseline_summaries', 'original_model_summaries', 'peft_model_summaries'])\n",
    "df\n",
    "print('INSTRUCT MODEL:')\n",
    "print(instruct_model_results)\n",
    "print('PEFT MODEL:')\n",
    "print(peft_model_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1674bc92-f3a2-4933-98c7-0cee9a660fa9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f8703a1e-419f-484f-8350-089b9cdbfed8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ORIGINAL MODEL:\n",
      "{'rouge1': 0.4856239903068369, 'rouge2': 0.23049530503225096, 'rougeL': 0.39967566075816996, 'rougeLsum': 0.3977338945201747}\n",
      "INSTRUCT MODEL:\n",
      "{'rouge1': 0.45671542330414133, 'rouge2': 0.22082225739346958, 'rougeL': 0.39229977493722634, 'rougeLsum': 0.38998498980949936}\n",
      "PEFT MODEL:\n",
      "{'rouge1': 0.49103582380731736, 'rouge2': 0.24042832542071296, 'rougeL': 0.4069284822836028, 'rougeLsum': 0.4052615183106413}\n"
     ]
    }
   ],
   "source": [
    "# rouge = evaluate.load('rouge')\n",
    "\n",
    "original_model_results = rouge.compute(\n",
    "    predictions=original_model_summaries,\n",
    "    references=human_baseline_summaries[0:len(original_model_summaries)],\n",
    "    use_aggregator=True,\n",
    "    use_stemmer=True,\n",
    ")\n",
    "\n",
    "\n",
    "instruct_model_results = rouge.compute(\n",
    "    predictions=instruct_model_summaries,\n",
    "    references=human_baseline_summaries[0:len(instruct_model_summaries)],\n",
    "    use_aggregator=True,\n",
    "    use_stemmer=True,\n",
    ")\n",
    "peft_model_results = rouge.compute(\n",
    "    predictions=peft_model_summaries,\n",
    "    references=human_baseline_summaries[0:len(peft_model_summaries)],\n",
    "    use_aggregator=True,\n",
    "    use_stemmer=True,\n",
    ")\n",
    "\n",
    "print('ORIGINAL MODEL:')\n",
    "print(original_model_results)\n",
    "dialogues = dataset['test'][0:50]['dialogue']\n",
    "human_baseline_summaries = dataset['test'][0:50]['summary']\n",
    "\n",
    "original_model_summaries = []\n",
    "instruct_model_summaries = []\n",
    "peft_model_summaries = []\n",
    "\n",
    "for idx, dialogue in enumerate(dialogues):\n",
    "    prompt = f\"\"\"\n",
    "Summarize the following conversation.\n",
    "\n",
    "\n",
    "{dialogue}\n",
    "\n",
    "Summary: \"\"\"\n",
    "    \n",
    "    input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
    "\n",
    "    human_baseline_text_output = human_baseline_summaries[idx]\n",
    "    \n",
    "    original_model_outputs = original_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    instruct_model_outputs = instruct_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    instruct_model_text_output = tokenizer.decode(instruct_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    peft_model_outputs = peft_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
    "    peft_model_text_output = tokenizer.decode(peft_model_outputs[0], skip_special_tokens=True)\n",
    "\n",
    "    original_model_summaries.append(original_model_text_output)\n",
    "    instruct_model_summaries.append(instruct_model_text_output)\n",
    "    peft_model_summaries.append(peft_model_text_output)\n",
    "\n",
    "zipped_summaries = list(zip(human_baseline_summaries, original_model_summaries, peft_model_summaries))\n",
    " \n",
    "df = pd.DataFrame(zipped_summaries, columns = ['human_baseline_summaries', 'original_model_summaries', 'peft_model_summaries'])\n",
    "df\n",
    "print('INSTRUCT MODEL:')\n",
    "print(instruct_model_results)\n",
    "print('PEFT MODEL:')\n",
    "print(peft_model_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8192e6c8-d234-4e1d-be6f-7ee94b436b1a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daf680fe-f25e-42cd-905d-2ccfb30b76bd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}