notebooks/t5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c32c924c",
   "metadata": {},
   "source": [
    "## Off the shelf results with T5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "928f6d17",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import T5ForConditionalGeneration, AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d883b7d3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "085ebfa59a144c2c9b9199fdde24fea9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/892M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ed09811cf3e4da1b5d2c112e27d34fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
      "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
      "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
      "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
      "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "base_model = T5ForConditionalGeneration.from_pretrained('t5-base')\n",
    "base_tokenizer = AutoTokenizer.from_pretrained('t5-base')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6c21f6b",
   "metadata": {},
   "source": [
    "## Abstractive Summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cd740019",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original text preprocessed:\n",
      " Sinan Ozdemir is a data scientist, startup founder, and educator living in the San Francisco Bay Area with his dog, Charlie; and cat, Euclid. He spent his academic career studying pure mathematics at Johns Hopkins University before transitioning to education. He spent several years conducting lectures on data science at Johns Hopkins University and at the General Assembly before founding his own startup, Kylie.ai, which uses artificial intelligence to build chatbots from historical transcripts. After completing a Fellowship at the Y Combinator accelerator, Sinan spent most of his time working on his fast-growing company, while creating educational material for data science.\n"
     ]
    }
   ],
   "source": [
    "text_to_summarize =\"\"\"Sinan Ozdemir is a data scientist, startup founder, and educator living in the San Francisco Bay Area with his dog, \n",
    "Charlie; and cat, Euclid. He spent his academic career studying pure mathematics \n",
    "at Johns Hopkins University before transitioning to education. He spent several years conducting lectures \n",
    "on data science at Johns Hopkins University and at the General Assembly before founding his own startup, \n",
    "Kylie.ai, which uses artificial intelligence to build chatbots from historical transcripts. \n",
    "After completing a Fellowship at the Y Combinator accelerator, Sinan spent most of his time working on \n",
    "his fast-growing company, while creating educational material for data science.\n",
    "\"\"\"\n",
    "\n",
    "preprocess_text = text_to_summarize.strip().replace(\"\\n\",\"\")\n",
    "\n",
    "print (\"original text preprocessed:\\n\", preprocess_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b987bdf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summarized text: \n",
      "Sinan Ozdemir is a data scientist, startup founder, and educator. he founded his own startup, Kylie.ai, which uses artificial intelligence to build chatbots.\n"
     ]
    }
   ],
   "source": [
    "# known prompt for summarization with T5\n",
    "t5_prepared_text = \"summarize: \" + preprocess_text\n",
    "\n",
    "input_ids = base_tokenizer.encode(t5_prepared_text, return_tensors=\"pt\")\n",
    "\n",
    "# summmarize \n",
    "summary_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    num_beams=4,\n",
    "    no_repeat_ngram_size=3,\n",
    "    min_length=30,\n",
    "    max_length=50,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(summary_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print (f\"Summarized text: \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a65fa05",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "cba2c628",
   "metadata": {},
   "source": [
    "## English -> German Translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "97f956b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Translated text:\n",
      "Wo ist die Schokolade?\n"
     ]
    }
   ],
   "source": [
    "input_ids = base_tokenizer.encode('translate English to German: Where is the chocolate?', return_tensors='pt')\n",
    "\n",
    "# translate \n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    num_beams=4,\n",
    "    no_repeat_ngram_size=3,\n",
    "    max_length=20,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print (f\"Translated text:\\n{output}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6bb64ef5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 3488,   229,    67, 31267,    58,     1]]),\n",
       " tensor(0.1136, grad_fn=<NllLossBackward0>))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pass labels in to calculate loss\n",
    "\n",
    "input_ids = base_tokenizer('translate English to German: Where is the chocolate?', return_tensors='pt').input_ids\n",
    "labels = base_tokenizer('Wo ist die Schokolade?', return_tensors='pt').input_ids\n",
    "\n",
    "loss = base_model(input_ids=input_ids, labels=labels).loss\n",
    "\n",
    "labels, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0397d662",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3012d5e3",
   "metadata": {},
   "source": [
    "## CoLA: The Corpus of Linguistic Acceptability\n",
    "checking for grammatical correctess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ada39336",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is grammatically correct?: \n",
      "acceptable\n"
     ]
    }
   ],
   "source": [
    "input_ids = base_tokenizer.encode('cola sentence: Where is the chocolate?', return_tensors='pt')\n",
    "\n",
    "# CoLA \n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    num_beams=4,\n",
    "    no_repeat_ngram_size=3,\n",
    "    max_length=20,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"is grammatically correct?: \\n{output}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2200dc26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is grammatically correct?: \n",
      "unacceptable\n"
     ]
    }
   ],
   "source": [
    "input_ids = base_tokenizer.encode('cola sentence: Where be a chocolates?', return_tensors='pt')\n",
    "\n",
    "# summmarize \n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    max_length=20,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"is grammatically correct?: \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb731953",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "509b5967",
   "metadata": {},
   "source": [
    "## STSB - Semantic Text Similarity Benchmark\n",
    "Are two sentences semantically similar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ff07d3b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "semantically similar? (0-5): \n",
      "3.2\n"
     ]
    }
   ],
   "source": [
    "sentence_one = 'How to fish'\n",
    "sentence_two = 'Fishing Manual for beginnners'\n",
    "\n",
    "\n",
    "input_ids = base_tokenizer.encode(f\"stsb sentence1: {sentence_one} sentence2: {sentence_two}\", return_tensors='pt')\n",
    "\n",
    "# calculate semantic similarity \n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    max_length=3,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"semantically similar? (0-5): \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "b361b234",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "semantically similar? (0-5): \n",
      "0.4\n"
     ]
    }
   ],
   "source": [
    "sentence_one = 'How to fish'\n",
    "sentence_two = 'Hiking Manual for beginnners'\n",
    "\n",
    "\n",
    "input_ids = base_tokenizer.encode(f\"stsb sentence1: {sentence_one} sentence2: {sentence_two}\", return_tensors='pt')\n",
    "\n",
    "# calculate semantic similarity\n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    max_length=3,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"semantically similar? (0-5): \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81ea1065",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a3081d51",
   "metadata": {},
   "source": [
    "## MNLI - Multi-Genre Natural Language Inference\n",
    "Whether a premise implies (“entailment”), contradicts (“contradiction”), or neither (“neutral”) a hypothesis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d3808dda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Response: \n",
      "entailment\n"
     ]
    }
   ],
   "source": [
    "input_ids = base_tokenizer.encode(\n",
    "    'mnli premise: I am active in politics. hypothesis: I am running for mayor', return_tensors='pt'\n",
    ")\n",
    "\n",
    "# mnli \n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"Response: \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4054466d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Response: \n",
      "contradiction\n"
     ]
    }
   ],
   "source": [
    "input_ids = base_tokenizer.encode(\n",
    "    'mnli premise: I am active in politics. hypothesis: I do not really vote', return_tensors='pt'\n",
    ")\n",
    "\n",
    "# mnli \n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"Response: \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ab54bbec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Response: \n",
      "neutral\n"
     ]
    }
   ],
   "source": [
    "input_ids = base_tokenizer.encode(\n",
    "    'mnli premise: I am active in politics. hypothesis: I code for a living', return_tensors='pt'\n",
    ")\n",
    "\n",
    "# mnli \n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"Response: \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49aecb68",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "ca923c09",
   "metadata": {},
   "source": [
    "## Q/A - Question/Answering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "31f2923f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Response: \n",
      "California\n"
     ]
    }
   ],
   "source": [
    "input_ids = base_tokenizer.encode(\n",
    "    'question: Where does Sinan live? context: Sinan lives in California but Matt lives in Boston.', return_tensors='pt'\n",
    ")\n",
    "\n",
    "# Q/A\n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"Response: \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4845657",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b0cd275e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Response: \n",
      "Boston\n"
     ]
    }
   ],
   "source": [
    "input_ids = base_tokenizer.encode(\n",
    "    'question: Where does Matt live? context: Sinan lives in California but Matt lives in Boston.', return_tensors='pt'\n",
    ")\n",
    "\n",
    "# Q/A\n",
    "translate_ids = base_model.generate(\n",
    "    input_ids,\n",
    "    early_stopping=True\n",
    ")\n",
    "\n",
    "output = base_tokenizer.decode(translate_ids[0], skip_special_tokens=True)\n",
    "\n",
    "print(f\"Response: \\n{output}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f2d4fee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "336c91b9",
   "metadata": {},
   "source": [
    "## Fine-tuning T5 for abstractive summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8f4fcdff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline, T5ForConditionalGeneration, TrainingArguments, Trainer, \\\n",
    "                         DataCollatorForSeq2Seq\n",
    "import pandas as pd\n",
    "from datasets import Dataset\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ceea57f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/transformers/models/t5/tokenization_t5.py:164: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
      "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
      "- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding.\n",
      "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
      "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "base_model = T5ForConditionalGeneration.from_pretrained('t5-small')\n",
    "base_tokenizer = T5Tokenizer.from_pretrained('t5-small')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "33f3a084",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(568411, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>B001E4KFG0</td>\n",
       "      <td>A3SGXH7AUHU8GW</td>\n",
       "      <td>delmartian</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1303862400</td>\n",
       "      <td>Good Quality Dog Food.</td>\n",
       "      <td>I have bought several of the Vitality canned d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>B00813GRG4</td>\n",
       "      <td>A1D87F6ZCVE5NK</td>\n",
       "      <td>dll pa</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1346976000</td>\n",
       "      <td>Not as Advertised.</td>\n",
       "      <td>Product arrived labeled as Jumbo Salted Peanut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>B000LQOCH0</td>\n",
       "      <td>ABXLMWJIXXAIN</td>\n",
       "      <td>Natalia Corres \"Natalia Corres\"</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1219017600</td>\n",
       "      <td>\"Delight\" says it all.</td>\n",
       "      <td>This is a confection that has been around a fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>B000UA0QIQ</td>\n",
       "      <td>A395BORC6FGVXV</td>\n",
       "      <td>Karl</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1307923200</td>\n",
       "      <td>Cough Medicine.</td>\n",
       "      <td>If you are looking for the secret ingredient i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>B006K2ZZ7K</td>\n",
       "      <td>A1UQRSCLF8GW1T</td>\n",
       "      <td>Michael D. Bigham \"M. Wassir\"</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1350777600</td>\n",
       "      <td>Great taffy.</td>\n",
       "      <td>Great taffy at a great price.  There was a wid...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id   ProductId          UserId                      ProfileName  \\\n",
       "0   1  B001E4KFG0  A3SGXH7AUHU8GW                       delmartian   \n",
       "1   2  B00813GRG4  A1D87F6ZCVE5NK                           dll pa   \n",
       "2   3  B000LQOCH0   ABXLMWJIXXAIN  Natalia Corres \"Natalia Corres\"   \n",
       "3   4  B000UA0QIQ  A395BORC6FGVXV                             Karl   \n",
       "4   5  B006K2ZZ7K  A1UQRSCLF8GW1T    Michael D. Bigham \"M. Wassir\"   \n",
       "\n",
       "   HelpfulnessNumerator  HelpfulnessDenominator  Score        Time  \\\n",
       "0                     1                       1      5  1303862400   \n",
       "1                     0                       0      1  1346976000   \n",
       "2                     1                       1      4  1219017600   \n",
       "3                     3                       3      2  1307923200   \n",
       "4                     0                       0      5  1350777600   \n",
       "\n",
       "                  Summary                                               Text  \n",
       "0  Good Quality Dog Food.  I have bought several of the Vitality canned d...  \n",
       "1      Not as Advertised.  Product arrived labeled as Jumbo Salted Peanut...  \n",
       "2  \"Delight\" says it all.  This is a confection that has been around a fe...  \n",
       "3         Cough Medicine.  If you are looking for the secret ingredient i...  \n",
       "4            Great taffy.  Great taffy at a great price.  There was a wid...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://www.kaggle.com/snap/amazon-fine-food-reviews?select=Reviews.csv\n",
    "\n",
    "reviews = pd.read_csv('../data/reviews.csv')\n",
    "\n",
    "# Pre-processing step\n",
    "# Punctuation is important in grammar and important for complex decoding architectures to know when to stop!\n",
    "def add_punc(s):\n",
    "    if s[-1] not in ('.', '!', '?'):\n",
    "        s = s + '.'\n",
    "    return s\n",
    "\n",
    "reviews.dropna(inplace=True)\n",
    "\n",
    "reviews['Summary'] = reviews['Summary'].map(add_punc)\n",
    "\n",
    "print(reviews.shape)\n",
    "\n",
    "reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "da4c1dcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(157786, 10)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews = reviews[(reviews['Summary'].str.len() < 100) & (reviews['Summary'].str.len() >= 30)]\n",
    "\n",
    "reviews.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "41ba36f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(0)\n",
    "\n",
    "reviews_dataset = Dataset.from_pandas(reviews.astype(str).sample(5000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6e4aff70",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We have a prompt but only as a prefix in the encoder\n",
    "prefix = \"summarize: \"\n",
    "\n",
    "# we will manually add our own labels because unlike GPT, we cannot assume the labels are based on the inputs\n",
    "def preprocess_function(examples):\n",
    "    inputs = [prefix + doc for doc in examples[\"Text\"]]\n",
    "    model_inputs = base_tokenizer(inputs, max_length=1024, truncation=True)\n",
    "\n",
    "    labels = base_tokenizer(examples[\"Summary\"], max_length=128, truncation=True)\n",
    "\n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f78ae0a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "822208b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45c1e698fef941acb13da1c4e4dc50aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_reviews_dataset = reviews_dataset.map(preprocess_function, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "28641597",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Id': '404224',\n",
       " 'ProductId': 'B003WH04CM',\n",
       " 'UserId': 'A1IW2NOBWMD985',\n",
       " 'ProfileName': 'Barbara \"ouma geek\"',\n",
       " 'HelpfulnessNumerator': '0',\n",
       " 'HelpfulnessDenominator': '0',\n",
       " 'Score': '5',\n",
       " 'Time': '1349827200',\n",
       " 'Summary': 'My Dog Gives These Five Woofs!',\n",
       " 'Text': 'Sarah, just loves these treats, they are her very favorite, and I love seeing her so happy.  She is a rescue, thrown from a passing car, and I feel nothing is too good for her!  I plan on never being without them and I plan on watching her enjoy these terrific Bone Treats until we are both too old to chew!',\n",
       " '__index_level_0__': 404223,\n",
       " 'input_ids': [21603,\n",
       "  10,\n",
       "  8077,\n",
       "  6,\n",
       "  131,\n",
       "  5682,\n",
       "  175,\n",
       "  11954,\n",
       "  6,\n",
       "  79,\n",
       "  33,\n",
       "  160,\n",
       "  182,\n",
       "  1305,\n",
       "  6,\n",
       "  11,\n",
       "  27,\n",
       "  333,\n",
       "  2492,\n",
       "  160,\n",
       "  78,\n",
       "  1095,\n",
       "  5,\n",
       "  451,\n",
       "  19,\n",
       "  3,\n",
       "  9,\n",
       "  9635,\n",
       "  6,\n",
       "  3,\n",
       "  12618,\n",
       "  45,\n",
       "  3,\n",
       "  9,\n",
       "  5792,\n",
       "  443,\n",
       "  6,\n",
       "  11,\n",
       "  27,\n",
       "  473,\n",
       "  1327,\n",
       "  19,\n",
       "  396,\n",
       "  207,\n",
       "  21,\n",
       "  160,\n",
       "  55,\n",
       "  27,\n",
       "  515,\n",
       "  30,\n",
       "  470,\n",
       "  271,\n",
       "  406,\n",
       "  135,\n",
       "  11,\n",
       "  27,\n",
       "  515,\n",
       "  30,\n",
       "  3355,\n",
       "  160,\n",
       "  777,\n",
       "  175,\n",
       "  16138,\n",
       "  4523,\n",
       "  15,\n",
       "  16494,\n",
       "  7,\n",
       "  552,\n",
       "  62,\n",
       "  33,\n",
       "  321,\n",
       "  396,\n",
       "  625,\n",
       "  12,\n",
       "  15196,\n",
       "  55,\n",
       "  1],\n",
       " 'attention_mask': [1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1],\n",
       " 'labels': [499, 6751, 6434, 7, 506, 9528, 3488, 858, 7, 55, 1]}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# inspect a single datapoint\n",
    "tokenized_reviews_dataset['train'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e71d418e",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_reviews_dataset = tokenized_reviews_dataset.train_test_split(test_size=.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ca5565f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data collator specifically for generic sequence to sequence tasks\n",
    "# Use when we are translating one sequence to another like translation, summarization, etc\n",
    "data_collator = DataCollatorForSeq2Seq(tokenizer=base_tokenizer, model=base_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "dd24d2d9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator. If Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 500\n",
      "  Batch size = 32\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='32' max='16' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [16/16 38:17]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 4.67218542098999,\n",
       " 'eval_runtime': 34.1269,\n",
       " 'eval_samples_per_second': 14.651,\n",
       " 'eval_steps_per_second': 0.469}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./t5_summary_results\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=32,\n",
    "    num_train_epochs=2,\n",
    "    load_best_model_at_end=True,\n",
    "    logging_steps=50,\n",
    "    save_strategy='epoch'\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=base_model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_reviews_dataset[\"train\"],\n",
    "    eval_dataset=tokenized_reviews_dataset[\"test\"],\n",
    "    data_collator=data_collator,\n",
    ")\n",
    "\n",
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2f1671d4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator. If Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "/opt/homebrew/lib/python3.10/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 4500\n",
      "  Num Epochs = 2\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 282\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='282' max='282' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [282/282 1:18:16, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.557000</td>\n",
       "      <td>3.387323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.468700</td>\n",
       "      <td>3.360325</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator. If Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 500\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./t5_summary_results/checkpoint-141\n",
      "Configuration saved in ./t5_summary_results/checkpoint-141/config.json\n",
      "Model weights saved in ./t5_summary_results/checkpoint-141/pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator. If Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 500\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./t5_summary_results/checkpoint-282\n",
      "Configuration saved in ./t5_summary_results/checkpoint-282/config.json\n",
      "Model weights saved in ./t5_summary_results/checkpoint-282/pytorch_model.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./t5_summary_results/checkpoint-282 (score: 3.3603246212005615).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=282, training_loss=3.5456864106739667, metrics={'train_runtime': 4704.1548, 'train_samples_per_second': 1.913, 'train_steps_per_second': 0.06, 'total_flos': 1522107824603136.0, 'train_loss': 3.5456864106739667, 'epoch': 2.0})"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "54510733",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator. If Id, ProfileName, Summary, Score, Time, __index_level_0__, UserId, ProductId, Text, HelpfulnessDenominator, HelpfulnessNumerator are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 500\n",
      "  Batch size = 32\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='16' max='16' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [16/16 00:31]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 3.3603246212005615,\n",
       " 'eval_runtime': 33.7016,\n",
       " 'eval_samples_per_second': 14.836,\n",
       " 'eval_steps_per_second': 0.475,\n",
       " 'epoch': 2.0}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "221a71f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./t5_summary_results\n",
      "Configuration saved in ./t5_summary_results/config.json\n",
      "Model weights saved in ./t5_summary_results/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6ab1d16d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file ./t5_summary_results/config.json\n",
      "Model config T5Config {\n",
      "  \"_name_or_path\": \"t5-small\",\n",
      "  \"architectures\": [\n",
      "    \"T5ForConditionalGeneration\"\n",
      "  ],\n",
      "  \"d_ff\": 2048,\n",
      "  \"d_kv\": 64,\n",
      "  \"d_model\": 512,\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"dense_act_fn\": \"relu\",\n",
      "  \"dropout_rate\": 0.1,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"feed_forward_proj\": \"relu\",\n",
      "  \"initializer_factor\": 1.0,\n",
      "  \"is_encoder_decoder\": true,\n",
      "  \"is_gated_act\": false,\n",
      "  \"layer_norm_epsilon\": 1e-06,\n",
      "  \"model_type\": \"t5\",\n",
      "  \"n_positions\": 512,\n",
      "  \"num_decoder_layers\": 6,\n",
      "  \"num_heads\": 8,\n",
      "  \"num_layers\": 6,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"relative_attention_max_distance\": 128,\n",
      "  \"relative_attention_num_buckets\": 32,\n",
      "  \"task_specific_params\": {\n",
      "    \"summarization\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"length_penalty\": 2.0,\n",
      "      \"max_length\": 200,\n",
      "      \"min_length\": 30,\n",
      "      \"no_repeat_ngram_size\": 3,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"summarize: \"\n",
      "    },\n",
      "    \"translation_en_to_de\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to German: \"\n",
      "    },\n",
      "    \"translation_en_to_fr\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to French: \"\n",
      "    },\n",
      "    \"translation_en_to_ro\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to Romanian: \"\n",
      "    }\n",
      "  },\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.23.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32128\n",
      "}\n",
      "\n",
      "loading weights file ./t5_summary_results/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing T5ForConditionalGeneration.\n",
      "\n",
      "All the weights of T5ForConditionalGeneration were initialized from the model checkpoint at ./t5_summary_results.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use T5ForConditionalGeneration for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "loaded_model = T5ForConditionalGeneration.from_pretrained('./t5_summary_results')\n",
    "\n",
    "# summarization pipeline prepends a default prefix of summarize: \n",
    "generator = pipeline(\n",
    "    'summarization', model=loaded_model, tokenizer=base_tokenizer\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "ad6b8780",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "418718    best beans for the buck you will find.\n",
      "Name: Summary, dtype: object\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Very happy with this coffee. mellow, and rich, organic and fair trade, what more can you ask for? Oh yeah, what a great deal! THanks'"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sam = reviews.sample(1)\n",
    "\n",
    "print(sam['Summary'])\n",
    "\n",
    "text = sam['Text'].tolist()[0]\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "c325f6c4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'summary_text': 'Great coffee.'}]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate a summary\n",
    "generator(text, min_length=3, max_length=15, early_stopping=True, num_beams=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "c7d83a20",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file config.json from cache at /Users/sinanozdemir/.cache/huggingface/hub/models--t5-small/snapshots/9507060efcd5189100109e25df8326eb07274a36/config.json\n",
      "Model config T5Config {\n",
      "  \"_name_or_path\": \"t5-small\",\n",
      "  \"architectures\": [\n",
      "    \"T5ForConditionalGeneration\"\n",
      "  ],\n",
      "  \"d_ff\": 2048,\n",
      "  \"d_kv\": 64,\n",
      "  \"d_model\": 512,\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"dense_act_fn\": \"relu\",\n",
      "  \"dropout_rate\": 0.1,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"feed_forward_proj\": \"relu\",\n",
      "  \"initializer_factor\": 1.0,\n",
      "  \"is_encoder_decoder\": true,\n",
      "  \"is_gated_act\": false,\n",
      "  \"layer_norm_epsilon\": 1e-06,\n",
      "  \"model_type\": \"t5\",\n",
      "  \"n_positions\": 512,\n",
      "  \"num_decoder_layers\": 6,\n",
      "  \"num_heads\": 8,\n",
      "  \"num_layers\": 6,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"relative_attention_max_distance\": 128,\n",
      "  \"relative_attention_num_buckets\": 32,\n",
      "  \"task_specific_params\": {\n",
      "    \"summarization\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"length_penalty\": 2.0,\n",
      "      \"max_length\": 200,\n",
      "      \"min_length\": 30,\n",
      "      \"no_repeat_ngram_size\": 3,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"summarize: \"\n",
      "    },\n",
      "    \"translation_en_to_de\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to German: \"\n",
      "    },\n",
      "    \"translation_en_to_fr\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to French: \"\n",
      "    },\n",
      "    \"translation_en_to_ro\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to Romanian: \"\n",
      "    }\n",
      "  },\n",
      "  \"transformers_version\": \"4.23.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32128\n",
      "}\n",
      "\n",
      "loading configuration file config.json from cache at /Users/sinanozdemir/.cache/huggingface/hub/models--t5-small/snapshots/9507060efcd5189100109e25df8326eb07274a36/config.json\n",
      "Model config T5Config {\n",
      "  \"_name_or_path\": \"t5-small\",\n",
      "  \"architectures\": [\n",
      "    \"T5ForConditionalGeneration\"\n",
      "  ],\n",
      "  \"d_ff\": 2048,\n",
      "  \"d_kv\": 64,\n",
      "  \"d_model\": 512,\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"dense_act_fn\": \"relu\",\n",
      "  \"dropout_rate\": 0.1,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"feed_forward_proj\": \"relu\",\n",
      "  \"initializer_factor\": 1.0,\n",
      "  \"is_encoder_decoder\": true,\n",
      "  \"is_gated_act\": false,\n",
      "  \"layer_norm_epsilon\": 1e-06,\n",
      "  \"model_type\": \"t5\",\n",
      "  \"n_positions\": 512,\n",
      "  \"num_decoder_layers\": 6,\n",
      "  \"num_heads\": 8,\n",
      "  \"num_layers\": 6,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"relative_attention_max_distance\": 128,\n",
      "  \"relative_attention_num_buckets\": 32,\n",
      "  \"task_specific_params\": {\n",
      "    \"summarization\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"length_penalty\": 2.0,\n",
      "      \"max_length\": 200,\n",
      "      \"min_length\": 30,\n",
      "      \"no_repeat_ngram_size\": 3,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"summarize: \"\n",
      "    },\n",
      "    \"translation_en_to_de\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to German: \"\n",
      "    },\n",
      "    \"translation_en_to_fr\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to French: \"\n",
      "    },\n",
      "    \"translation_en_to_ro\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to Romanian: \"\n",
      "    }\n",
      "  },\n",
      "  \"transformers_version\": \"4.23.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32128\n",
      "}\n",
      "\n",
      "loading weights file pytorch_model.bin from cache at /Users/sinanozdemir/.cache/huggingface/hub/models--t5-small/snapshots/9507060efcd5189100109e25df8326eb07274a36/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing T5ForConditionalGeneration.\n",
      "\n",
      "All the weights of T5ForConditionalGeneration were initialized from the model checkpoint at t5-small.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use T5ForConditionalGeneration for predictions without further training.\n",
      "Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
      "loading configuration file config.json from cache at /Users/sinanozdemir/.cache/huggingface/hub/models--t5-small/snapshots/9507060efcd5189100109e25df8326eb07274a36/config.json\n",
      "Model config T5Config {\n",
      "  \"_name_or_path\": \"t5-small\",\n",
      "  \"architectures\": [\n",
      "    \"T5ForConditionalGeneration\"\n",
      "  ],\n",
      "  \"d_ff\": 2048,\n",
      "  \"d_kv\": 64,\n",
      "  \"d_model\": 512,\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"dense_act_fn\": \"relu\",\n",
      "  \"dropout_rate\": 0.1,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"feed_forward_proj\": \"relu\",\n",
      "  \"initializer_factor\": 1.0,\n",
      "  \"is_encoder_decoder\": true,\n",
      "  \"is_gated_act\": false,\n",
      "  \"layer_norm_epsilon\": 1e-06,\n",
      "  \"model_type\": \"t5\",\n",
      "  \"n_positions\": 512,\n",
      "  \"num_decoder_layers\": 6,\n",
      "  \"num_heads\": 8,\n",
      "  \"num_layers\": 6,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"relative_attention_max_distance\": 128,\n",
      "  \"relative_attention_num_buckets\": 32,\n",
      "  \"task_specific_params\": {\n",
      "    \"summarization\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"length_penalty\": 2.0,\n",
      "      \"max_length\": 200,\n",
      "      \"min_length\": 30,\n",
      "      \"no_repeat_ngram_size\": 3,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"summarize: \"\n",
      "    },\n",
      "    \"translation_en_to_de\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to German: \"\n",
      "    },\n",
      "    \"translation_en_to_fr\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to French: \"\n",
      "    },\n",
      "    \"translation_en_to_ro\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to Romanian: \"\n",
      "    }\n",
      "  },\n",
      "  \"transformers_version\": \"4.23.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32128\n",
      "}\n",
      "\n",
      "loading file spiece.model from cache at /Users/sinanozdemir/.cache/huggingface/hub/models--t5-small/snapshots/9507060efcd5189100109e25df8326eb07274a36/spiece.model\n",
      "loading file tokenizer.json from cache at /Users/sinanozdemir/.cache/huggingface/hub/models--t5-small/snapshots/9507060efcd5189100109e25df8326eb07274a36/tokenizer.json\n",
      "loading file added_tokens.json from cache at None\n",
      "loading file special_tokens_map.json from cache at None\n",
      "loading file tokenizer_config.json from cache at None\n",
      "loading configuration file config.json from cache at /Users/sinanozdemir/.cache/huggingface/hub/models--t5-small/snapshots/9507060efcd5189100109e25df8326eb07274a36/config.json\n",
      "Model config T5Config {\n",
      "  \"_name_or_path\": \"t5-small\",\n",
      "  \"architectures\": [\n",
      "    \"T5ForConditionalGeneration\"\n",
      "  ],\n",
      "  \"d_ff\": 2048,\n",
      "  \"d_kv\": 64,\n",
      "  \"d_model\": 512,\n",
      "  \"decoder_start_token_id\": 0,\n",
      "  \"dense_act_fn\": \"relu\",\n",
      "  \"dropout_rate\": 0.1,\n",
      "  \"eos_token_id\": 1,\n",
      "  \"feed_forward_proj\": \"relu\",\n",
      "  \"initializer_factor\": 1.0,\n",
      "  \"is_encoder_decoder\": true,\n",
      "  \"is_gated_act\": false,\n",
      "  \"layer_norm_epsilon\": 1e-06,\n",
      "  \"model_type\": \"t5\",\n",
      "  \"n_positions\": 512,\n",
      "  \"num_decoder_layers\": 6,\n",
      "  \"num_heads\": 8,\n",
      "  \"num_layers\": 6,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"relative_attention_max_distance\": 128,\n",
      "  \"relative_attention_num_buckets\": 32,\n",
      "  \"task_specific_params\": {\n",
      "    \"summarization\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"length_penalty\": 2.0,\n",
      "      \"max_length\": 200,\n",
      "      \"min_length\": 30,\n",
      "      \"no_repeat_ngram_size\": 3,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"summarize: \"\n",
      "    },\n",
      "    \"translation_en_to_de\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to German: \"\n",
      "    },\n",
      "    \"translation_en_to_fr\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to French: \"\n",
      "    },\n",
      "    \"translation_en_to_ro\": {\n",
      "      \"early_stopping\": true,\n",
      "      \"max_length\": 300,\n",
      "      \"num_beams\": 4,\n",
      "      \"prefix\": \"translate English to Romanian: \"\n",
      "    }\n",
      "  },\n",
      "  \"transformers_version\": \"4.23.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32128\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Try the base t5 on the same text\n",
    "base_generator = pipeline(\n",
    "    'summarization', model='t5-small', tokenizer='t5-small'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "d02031ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'summary_text': 'THanks coffee is a great coffee . mel'}]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Summary is a bit more extractive than our fine-tuned version and style isn't quite the same as our dataset\n",
    "base_generator(text, min_length=3, max_length=15, early_stopping=True, num_beams=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f84c0a7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}