1_lora_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "64c5640b-4d9a-4cd5-90d6-82b8172713a2",
   "metadata": {},
   "source": [
    "# LoRA from Scratch\n",
    "\n",
    "## Environment Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5f594ecd-615a-47d2-9904-9d95e0cf2445",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f61b5f2-3f1b-46ab-b0c4-2bab4fee27c7",
   "metadata": {},
   "source": [
    "## Inspecting RoBERTa Base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0500cad5-fb26-4291-baa5-29b9397487f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:train:torch version: 2.0.1\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('src')\n",
    "from train import load_model_and_tokenizer_and_collator\n",
    "import util\n",
    "import logging\n",
    "util.logger.setLevel(logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ecd286f-d034-4964-99ed-ac9e9021b207",
   "metadata": {},
   "source": [
    "### RoBERTa Modules\n",
    "We start witht the RoBERTa Base model as provided through Hugging Face. Let's have a quick look at the modules."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8cff11b4-2c21-4a10-b4c3-6d7af2e84269",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "model = AutoModelForSequenceClassification.from_pretrained('roberta-base')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5fa3fa2f-67c4-4c1b-8185-a5d0e3489bcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RobertaForSequenceClassification(\n",
       "  (roberta): RobertaModel(\n",
       "    (embeddings): RobertaEmbeddings(\n",
       "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
       "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
       "      (token_type_embeddings): Embedding(1, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): RobertaEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): RobertaClassificationHead(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (out_proj): Linear(in_features=768, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65f2bd24-91c5-43fe-9408-8e5a1a03b6d8",
   "metadata": {},
   "source": [
    "^^^ Above we see the encoder layers with their attention and feed forward layers and the classifier at the top of the network (at the bottom of the printed summary).\n",
    "\n",
    "### RoBERTa Parameters\n",
    "Below we see all parameters. All of these would be trained during a full finetuning. In the below output the parameters that we train are marked with a `1` and then are followed by the number of parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "42b162ff-d276-4ea4-a70f-62f3fe22b154",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DEBUG:util:Parameters (name, tunable, count):\n",
      "DEBUG:util: roberta.embeddings.word_embeddings.weight                      1    38603520\n",
      "DEBUG:util: roberta.embeddings.position_embeddings.weight                  1      394752\n",
      "DEBUG:util: roberta.embeddings.token_type_embeddings.weight                1         768\n",
      "DEBUG:util: roberta.embeddings.LayerNorm.weight                            1         768\n",
      "DEBUG:util: roberta.embeddings.LayerNorm.bias                              1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.0.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.0.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.0.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.0.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.1.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.1.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.1.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.1.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.2.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.2.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.2.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.2.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.3.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.3.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.3.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.3.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.4.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.4.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.4.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.4.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.5.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.5.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.5.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.5.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.6.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.6.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.6.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.6.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.7.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.7.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.7.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.7.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.8.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.8.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.8.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.8.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.query.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.query.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.key.weight              1      589824\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.key.bias                1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.value.weight            1      589824\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.value.bias              1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.dense.weight          1      589824\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.dense.bias            1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.LayerNorm.weight      1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.LayerNorm.bias        1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.intermediate.dense.weight              1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.9.intermediate.dense.bias                1        3072\n",
      "DEBUG:util: roberta.encoder.layer.9.output.dense.weight                    1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.9.output.dense.bias                      1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.output.LayerNorm.weight                1         768\n",
      "DEBUG:util: roberta.encoder.layer.9.output.LayerNorm.bias                  1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.query.weight           1      589824\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.query.bias             1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.key.weight             1      589824\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.key.bias               1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.value.weight           1      589824\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.value.bias             1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.dense.weight         1      589824\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.dense.bias           1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.LayerNorm.weight     1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.LayerNorm.bias       1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.intermediate.dense.weight             1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.10.intermediate.dense.bias               1        3072\n",
      "DEBUG:util: roberta.encoder.layer.10.output.dense.weight                   1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.10.output.dense.bias                     1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.output.LayerNorm.weight               1         768\n",
      "DEBUG:util: roberta.encoder.layer.10.output.LayerNorm.bias                 1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.query.weight           1      589824\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.query.bias             1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.key.weight             1      589824\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.key.bias               1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.value.weight           1      589824\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.value.bias             1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.dense.weight         1      589824\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.dense.bias           1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.LayerNorm.weight     1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.LayerNorm.bias       1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.intermediate.dense.weight             1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.11.intermediate.dense.bias               1        3072\n",
      "DEBUG:util: roberta.encoder.layer.11.output.dense.weight                   1     2359296\n",
      "DEBUG:util: roberta.encoder.layer.11.output.dense.bias                     1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.output.LayerNorm.weight               1         768\n",
      "DEBUG:util: roberta.encoder.layer.11.output.LayerNorm.bias                 1         768\n",
      "DEBUG:util: classifier.dense.weight                                        1      589824\n",
      "DEBUG:util: classifier.dense.bias                                          1         768\n",
      "DEBUG:util: classifier.out_proj.weight                                     1        1536\n",
      "DEBUG:util: classifier.out_proj.bias                                       1           2\n",
      "INFO:util:Total parameters: 124,647,170, thereof learnable: 124,647,170 (100.0000%)\n"
     ]
    }
   ],
   "source": [
    "from util import count_parameters\n",
    "util.logger.setLevel(logging.DEBUG)\n",
    "count_parameters(model, True);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "264bbd2e-d98f-4467-a75d-2278e8449e4d",
   "metadata": {},
   "source": [
    "**^^^ Also recognize the last line that the count of parameters is 124,647,170 and that 100% of the parameters are trained at this time.**\n",
    "\n",
    "### Picking A Linear Module\n",
    "As examples of Linear modules, let's pick the 8th layer (with the index starting at 0, it's layer 7). And here we look first at the linear projection of the query, then the feed forward layer with its up and down projection:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3118126f-46b8-4802-b11b-54b27c939f60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Linear(in_features=768, out_features=768, bias=True),\n",
       " Linear(in_features=768, out_features=3072, bias=True),\n",
       " Linear(in_features=3072, out_features=768, bias=True))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(model.roberta.encoder.layer[-1].attention.self.query,\n",
    " model.roberta.encoder.layer[-1].intermediate.dense,\n",
    " model.roberta.encoder.layer[-1].output.dense)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13f9bf2f-0ad9-4080-a5de-4809f1b022a9",
   "metadata": {},
   "source": [
    "^^^ Of those let's pick with the output Linear layer, as it is one of the two modules (the other being the query projection) that are supposed to have the strongest influence on the model's finetuning performance. \n",
    "\n",
    "Also for this module `fan_in` and `fan_out` are not equal, so that it's easier to follow along, when reviewing tensor values."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7aa6fa65-225c-4b41-b98b-1d30697cea2c",
   "metadata": {},
   "source": [
    "### Creating The Low Rank Matrices\n",
    "For that Linear module we now want to create the two low rank matrices that should approximate the whole weight matrix of the module, ie. `3072 x 768`. \n",
    " \n",
    "We create two Parameters, `lora_A` and `lora_B`, that individually will be `fan_in x r` `(3072x4)` and `r x fan_out` `(4x768)`, but their product's dimension will be `fan_in x fan_out` `(3072x768)`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "59b22de4-c095-43f7-b4b1-df9f40c396f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3051dfda-6f85-4ff6-b3a5-6644e26a4f7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=3072, out_features=768, bias=True)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adaptee = model.roberta.encoder.layer[-1].output.dense; adaptee # Get a reference to the module we want to adapt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "85d23ece-a5d5-4c32-922f-d8a9dd9f5b81",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = 4 # Let's start with a known, plausible value, for r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4474a5e4-f0e9-45ed-9940-ab5ed1cbda95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([3072, 4]), torch.Size([4, 768]), torch.Size([3072, 768]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lora_A = nn.Parameter(torch.randn(adaptee.in_features, r)/math.sqrt(adaptee.in_features))\n",
    "lora_B = nn.Parameter(torch.zeros(r, adaptee.out_features))\n",
    "lora_A.shape, lora_B.shape, (lora_A @ lora_B).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ebf0048-60b2-4aff-b9ec-b0186fc04ce1",
   "metadata": {},
   "source": [
    "So far so good. Let's also take a look at how much smaller the parameter count now is:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b0114d42-3a47-4342-b25d-a06faea61f7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2359296, 15360, '0.651%')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_size, low_rank_size = adaptee.weight.numel(), lora_A.numel()+lora_B.numel()\n",
    "full_size, low_rank_size, f'{100*(low_rank_size/full_size):4.3f}%'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "740f085d-3e9b-4c10-9eaa-90e7845b71cc",
   "metadata": {},
   "source": [
    "With a rank of `4` we are down from ~2.4M parameters to ~15K parameters, that is less than `1%` of the original parameters. So far this sounds good. But of course, we also need to review the resulting the performance. We will get to that.\n",
    "\n",
    "### Smoke Testing Of The Matrices\n",
    "\n",
    "For now we should already be able to see if we can use the product of the two low rank matrices in place of the full rank matrix.\n",
    "\n",
    "Also, remember from the article, that we initialized the two parameters in a way that their initial bias is to not change anything. Hence, we would expect that we can add our adaptation and it should work mechanically, but not change the original result.\n",
    "\n",
    "We do one forward pass through the original module, the `adaptee` and one forward pass through the product of the two small matrices, `lora_A` and `lora_B`. We then add the result. Given that we initialized one of the matrices with 0, the prodcut will be 0 too, and therefore the result should not be any different than just using the `adaptee` on its own as the addition of 0 does not change the result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c518ab5c-4fe1-41aa-9716-f321df1beaf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# our test data\n",
    "x = torch.randn((64, 3072)) # pretend bs 64\n",
    "\n",
    "# using the original path\n",
    "original_result = adaptee(x) # (bs, out_d)\n",
    "\n",
    "assert original_result.shape == (64, adaptee.out_features)\n",
    "\n",
    "# using the new adapter path\n",
    "lora_matrix = (lora_A@lora_B) \n",
    "adapter_result = x @ lora_matrix\n",
    "\n",
    "assert adapter_result.shape == (64, adaptee.out_features)\n",
    "\n",
    "# both together\n",
    "x_prime = adaptee(x) + x @ lora_A @ lora_B\n",
    "assert x_prime.shape == (64, adaptee.out_features)\n",
    "assert torch.allclose(original_result, x_prime)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b58f1352-c098-4c49-ad05-352f077a7106",
   "metadata": {},
   "source": [
    "Ok, this worked well. We don't know anything about the performance yet. We will see that when we traint the model. But let's consider how to integrate this? How do we integrate those two matrices into the forward pass of a module that we want to adapt? Also, how do we add and remove these adapters?\n",
    "\n",
    "### Creating a LoRA Adapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "20caf03a-3a02-4fa3-a21d-36f484316cde",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LoRAAdapter(nn.Module):\n",
    "    def __init__(self, \n",
    "                 adaptee, # <- module to be adapted\n",
    "                 r):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.r = r\n",
    "        self.adaptee = adaptee\n",
    "        \n",
    "        # Store a pointer to the original forward implementation \n",
    "        # of the module to be adapted.\n",
    "        # Then point its forward method to this adapter module.\n",
    "        self.orig_forward = adaptee.forward\n",
    "        adaptee.forward = self.forward\n",
    "    \n",
    "        # Adding the weight matrices directly to the adaptee,\n",
    "        # which makes is more practical to report the parameters,\n",
    "        # and to remove it later.\n",
    "        adaptee.lora_A = nn.Parameter(torch.randn(adaptee.in_features, r)/math.sqrt(adaptee.in_features))\n",
    "        adaptee.lora_B = nn.Parameter(torch.zeros(r, adaptee.out_features))\n",
    "        \n",
    "    def forward(self, x, *args, **kwargs):\n",
    "        return (\n",
    "            self.orig_forward(x, *args, **kwargs) +\n",
    "            x @ self.adaptee.lora_A @ self.adaptee.lora_B\n",
    "        )\n",
    "    \n",
    "    def extra_repr(self):\n",
    "        return f'LoRAAdapter (r={self.r}, dropout={self.dropout})'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e39015d6-8aa4-40ab-9c45-08cbf79795ba",
   "metadata": {},
   "source": [
    "^^^ Please have a look at the inline documentation in the code above. Please recognize that this is exactly what we tried before. Plus, we now hook into the forward pass. Here we call the original forward method of the module with our inputs and then take the same inputs and apply our product of `lora_A` and `lora_B` to it. We add both results.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2617e597-229f-4220-923f-ee82a5b182aa",
   "metadata": {},
   "source": [
    "Let's install such an adapter and take it for a spin.\n",
    "\n",
    "#### Before Installing Adapter\n",
    "\n",
    "How many parameters are tunable before installing the adapter?\n",
    "(We are also freezing the module to be adopted. We'll dive deeper into this a little later.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8632cb2e-a21d-4a21-aadc-ac607e2cd76e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DEBUG:util:Parameters (name, tunable, count):\n",
      "DEBUG:util: dense.weight         1     2359296\n",
      "DEBUG:util: dense.bias           1         768\n",
      "DEBUG:util: LayerNorm.weight     1         768\n",
      "DEBUG:util: LayerNorm.bias       1         768\n",
      "INFO:util:Total parameters: 2,361,600, thereof learnable: 2,361,600 (100.0000%)\n",
      "DEBUG:util:Parameters (name, tunable, count):\n",
      "DEBUG:util: dense.weight         0     2359296\n",
      "DEBUG:util: dense.bias           0         768\n",
      "DEBUG:util: LayerNorm.weight     0         768\n",
      "DEBUG:util: LayerNorm.bias       0         768\n",
      "INFO:util:Total parameters: 2,361,600, thereof learnable: 0 (0.0000%)\n"
     ]
    }
   ],
   "source": [
    "count_parameters(model.roberta.encoder.layer[-1].output);\n",
    "\n",
    "## Freezing the module to be adapted\n",
    "for p in model.roberta.encoder.layer[-1].output.parameters(): p.requires_grad_(False)\n",
    "\n",
    "count_parameters(model.roberta.encoder.layer[-1].output);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5ed78e64-1640-40b2-8594-85091f0a8f39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([64, 3072]), -943.4242553710938)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn((64, 3072))\n",
    "original_result = model.roberta.encoder.layer[-1].output.dense(x)\n",
    "\n",
    "adapter = LoRAAdapter(\n",
    "    adaptee=model.roberta.encoder.layer[-1].output.dense, \n",
    "    r=4)\n",
    "\n",
    "adapted_result = adapter(x)\n",
    "\n",
    "assert adapted_result.shape == (64, model.roberta.encoder.layer[-1].output.dense.out_features)\n",
    "assert torch.allclose(original_result, adapted_result)\n",
    "x.shape, x.sum().item()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d289737a-9172-4c18-84c5-b81294e6b459",
   "metadata": {},
   "source": [
    "#### After Installing Adapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d49e8934-9492-4fb8-ad52-17ab82c6ab3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DEBUG:util:Parameters (name, tunable, count):\n",
      "DEBUG:util: dense.weight         0     2359296\n",
      "DEBUG:util: dense.bias           0         768\n",
      "DEBUG:util: dense.lora_A         1       12288\n",
      "DEBUG:util: dense.lora_B         1        3072\n",
      "DEBUG:util: LayerNorm.weight     0         768\n",
      "DEBUG:util: LayerNorm.bias       0         768\n",
      "INFO:util:Total parameters: 2,376,960, thereof learnable: 15,360 (0.6462%)\n"
     ]
    }
   ],
   "source": [
    "count_parameters(model.roberta.encoder.layer[-1].output);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84000620-a286-4571-a8ce-d0384eed905f",
   "metadata": {},
   "source": [
    "### Next Steps\n",
    "\n",
    "The mechanics are working. That's great.\n",
    "\n",
    "But we don't know if using the LoRAAdapter is helping. We need to train it first.\n",
    "\n",
    "We'll get there, but:\n",
    "- First, we need to generalize the solution and make it configurable, so that we can adapt arbitrary modules; we'll cover Linear modules only to illustrate the concept.  \n",
    "- Second, currently all parameters of the models are trained (see the 100% above). For the finetuning to become efficient we need to make sure that only the adapters are trained, and the classifier head."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d16be1ce-9169-42b8-bbf6-2fd795bc80a4",
   "metadata": {},
   "source": [
    "#### Freezing The Model \n",
    "\n",
    "When finetuning we only want to finetune the modules that will contribute significantly to the overall finetuned performance. This is not new and not specific to Parameter Efficient Finetuning (PEFT) or LoRA. In the past we also tried to use our understanding of the data and the task to select which parts of a network to finetune. \n",
    "\n",
    "This was typically done on a rather coarse grained level. For example when finetuning a pre-trained LM for a high level downstream task, like sentiment analysis, we froze the embeddings and the lower layers of the network, assuming that the use of language and vocabulary does not change dramatically between the pre-training objective of predicting the next token/masked out token on one hand, and our sentiment analysis on the other. FOr all necessary changes the upper transformer layers and the classifier head were assumed to be proficient.\n",
    "\n",
    "In PEFT we can also chose our layers and modules with such care and foresight, and we'll define how to configure this in the next section, but just seeing above how this approach reduces the size makes it obvious that not as much care would be necessary.\n",
    "\n",
    "At any rate, we would always want the classifier's parameters to be tunable. It is initialized randomly and is totally task specific. For the remaining modules we can decide this.\n",
    "\n",
    "Here we start by freezing all parameters of the model. Then we unfreeze the classifier head's parameters. \n",
    "\n",
    "All adapters we will subsequently add are naturally also not frozen, but will be trained. That's the whole point, isn't it? :)\n",
    "\n",
    "#### Before Freezing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6cc5e6c4-10c4-4ede-a1ce-9c17627faeeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:util:Total parameters: 124,647,170, thereof learnable: 124,647,170 (100.0000%)\n"
     ]
    }
   ],
   "source": [
    "model = load_model_and_tokenizer_and_collator('roberta-base')[0];\n",
    "count_parameters(model, verbose=False);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e577214c-b427-440d-ad3d-9cf921d53e56",
   "metadata": {},
   "source": [
    "#### Freezing The Whole Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0c450b9b-02f0-4cc6-83ba-c204e055065d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now freezing all parameters\n",
    "for p in list(model.children()): p.requires_grad_(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4f33ff26-bacd-49fa-a70d-a47687a214ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:util:Total parameters: 124,647,170, thereof learnable: 0 (0.0000%)\n"
     ]
    }
   ],
   "source": [
    "count_parameters(model, verbose=False);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d037a389-de17-4480-9cd3-53997f622413",
   "metadata": {},
   "source": [
    "#### Unfreezing The Classifier / Head\n",
    "If you compare to the [RoBERTa-Modules](#RoBERTa-Modules) you'll see that the classifier is the last element, at index `-1`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e6047063-44c8-41f9-8473-2df4a4a7cfc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in list(model.children())[-1].parameters(): p.requires_grad_(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ba7468dc-885f-4ab7-bf83-6d0b75112f6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:util:Total parameters: 124,647,170, thereof learnable: 592,130 (0.4750%)\n"
     ]
    }
   ],
   "source": [
    "count_parameters(model, verbose=False);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f442a6c-471d-418f-8272-0c5d29560e16",
   "metadata": {},
   "source": [
    "#### Configuring The Modules\n",
    "\n",
    "We now look at an exemplary way how to select the modules we want to adapt. In the accompanying code something similar is implemented. Below we have a stripped down version for clarity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c54cd7d1-7229-453a-a66b-a71178c9a598",
   "metadata": {},
   "outputs": [],
   "source": [
    "lora_includes=['query', 'output']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "dd7011e1-d656-4386-9bea-8da94ca5d712",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "for name, module in model.named_modules():\n",
    "    \n",
    "    # We are only dealing with Linear modules at this time\n",
    "    if not isinstance(module, nn.Linear): \n",
    "        continue\n",
    "       \n",
    "    # Does the name of current module match any of the provided patterns?\n",
    "    for regex in lora_includes:\n",
    "        if re.match(f'.*{regex}', name):\n",
    "            LoRAAdapter(module, 4)\n",
    "            break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c1cdfed-4b30-485c-b65f-5af908d3f424",
   "metadata": {},
   "source": [
    "This was already it. We have adapted all Linear modules in the model that contain either `query` or `output`.\n",
    "\n",
    "Check now the `0`/`1` marking of our modules below. Verify that no parameters are trainable, except:\n",
    "- The classifier\n",
    "- The `lora_A` and `lora_B` parametes that are attached to the modules that contain either `output` or `query`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4cb04259-684f-4c0a-9adb-e23c12cb8268",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DEBUG:util:Parameters (name, tunable, count):\n",
      "DEBUG:util: roberta.embeddings.word_embeddings.weight                      0    38603520\n",
      "DEBUG:util: roberta.embeddings.position_embeddings.weight                  0      394752\n",
      "DEBUG:util: roberta.embeddings.token_type_embeddings.weight                0         768\n",
      "DEBUG:util: roberta.embeddings.LayerNorm.weight                            0         768\n",
      "DEBUG:util: roberta.embeddings.LayerNorm.bias                              0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.0.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.0.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.0.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.0.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.0.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.0.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.1.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.1.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.1.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.1.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.1.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.1.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.2.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.2.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.2.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.2.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.2.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.2.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.3.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.3.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.3.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.3.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.3.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.3.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.4.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.4.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.4.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.4.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.4.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.4.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.5.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.5.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.5.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.5.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.5.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.5.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.6.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.6.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.6.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.6.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.6.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.6.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.7.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.7.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.7.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.7.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.7.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.7.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.8.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.8.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.8.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.8.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.8.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.8.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.query.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.query.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.query.lora_A            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.query.lora_B            1        3072\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.key.weight              0      589824\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.key.bias                0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.value.weight            0      589824\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.self.value.bias              0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.dense.weight          0      589824\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.dense.bias            0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.dense.lora_A          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.dense.lora_B          1        3072\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.LayerNorm.weight      0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.attention.output.LayerNorm.bias        0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.intermediate.dense.weight              0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.9.intermediate.dense.bias                0        3072\n",
      "DEBUG:util: roberta.encoder.layer.9.output.dense.weight                    0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.9.output.dense.bias                      0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.output.dense.lora_A                    1       12288\n",
      "DEBUG:util: roberta.encoder.layer.9.output.dense.lora_B                    1        3072\n",
      "DEBUG:util: roberta.encoder.layer.9.output.LayerNorm.weight                0         768\n",
      "DEBUG:util: roberta.encoder.layer.9.output.LayerNorm.bias                  0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.query.weight           0      589824\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.query.bias             0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.query.lora_A           1        3072\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.query.lora_B           1        3072\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.key.weight             0      589824\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.key.bias               0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.value.weight           0      589824\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.self.value.bias             0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.dense.weight         0      589824\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.dense.bias           0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.dense.lora_A         1        3072\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.dense.lora_B         1        3072\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.LayerNorm.weight     0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.attention.output.LayerNorm.bias       0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.intermediate.dense.weight             0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.10.intermediate.dense.bias               0        3072\n",
      "DEBUG:util: roberta.encoder.layer.10.output.dense.weight                   0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.10.output.dense.bias                     0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.output.dense.lora_A                   1       12288\n",
      "DEBUG:util: roberta.encoder.layer.10.output.dense.lora_B                   1        3072\n",
      "DEBUG:util: roberta.encoder.layer.10.output.LayerNorm.weight               0         768\n",
      "DEBUG:util: roberta.encoder.layer.10.output.LayerNorm.bias                 0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.query.weight           0      589824\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.query.bias             0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.query.lora_A           1        3072\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.query.lora_B           1        3072\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.key.weight             0      589824\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.key.bias               0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.value.weight           0      589824\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.self.value.bias             0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.dense.weight         0      589824\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.dense.bias           0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.dense.lora_A         1        3072\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.dense.lora_B         1        3072\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.LayerNorm.weight     0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.attention.output.LayerNorm.bias       0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.intermediate.dense.weight             0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.11.intermediate.dense.bias               0        3072\n",
      "DEBUG:util: roberta.encoder.layer.11.output.dense.weight                   0     2359296\n",
      "DEBUG:util: roberta.encoder.layer.11.output.dense.bias                     0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.output.dense.lora_A                   1       12288\n",
      "DEBUG:util: roberta.encoder.layer.11.output.dense.lora_B                   1        3072\n",
      "DEBUG:util: roberta.encoder.layer.11.output.LayerNorm.weight               0         768\n",
      "DEBUG:util: roberta.encoder.layer.11.output.LayerNorm.bias                 0         768\n",
      "DEBUG:util: classifier.dense.weight                                        1      589824\n",
      "DEBUG:util: classifier.dense.bias                                          1         768\n",
      "DEBUG:util: classifier.out_proj.weight                                     1        1536\n",
      "DEBUG:util: classifier.out_proj.bias                                       1           2\n",
      "INFO:util:Total parameters: 124,978,946, thereof learnable: 923,906 (0.7392%)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(124978946, 923906)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_parameters(model, verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "387727c7-a244-4878-8df2-8a350e558bfc",
   "metadata": {},
   "source": [
    "**^^^Re-recognize that still the total number of parameters that are learnable are below `1%`.**\n",
    "\n",
    "Nice! \n",
    "\n",
    "See above that we now would train a 0.74% of all parameters.\n",
    "\n",
    "#### Sanity Check of The Dimensions\n",
    "\n",
    "Let's take a deeper look. We'll review the Linear module that does the up projection in the last layer's (index 11):\n",
    "\n",
    "```\n",
    "...\n",
    "roberta.encoder.layer.11.output.dense.weight                          0     2359296\n",
    "roberta.encoder.layer.11.output.dense.bias                            0         768\n",
    "roberta.encoder.layer.11.output.dense.lora_A                          1       12288\n",
    "roberta.encoder.layer.11.output.dense.lora_B                          1        3072\n",
    "...\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4be102ff-f048-407e-a446-9acb9b41d842",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=3072, out_features=768, bias=True)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.roberta.encoder.layer[-1].output.dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f91dc972-6c63-4319-ba0c-a4ec7ea70a1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12288"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.roberta.encoder.layer[-1].output.dense.lora_A.numel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ec611e89-868f-42f5-bbc0-c134b007868f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2359296, 12288, 3072)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that the matrix with all in multiplied with all out features is the same as the number of parameters of the weight matrix\n",
    "assert (model.roberta.encoder.layer[-1].output.dense.out_features * \n",
    "        model.roberta.encoder.layer[-1].output.dense.in_features == 2359296)\n",
    "\n",
    "# Check that the number of parameters of the low rank lora A matrix is the product of r (4) and in_features\n",
    "assert model.roberta.encoder.layer[-1].output.dense.in_features * 4 == model.roberta.encoder.layer[-1].output.dense.lora_A.numel()\n",
    "\n",
    "# Check that the number of parameters of the low rank lora B matrix is the product of r (4) and out_features\n",
    "assert model.roberta.encoder.layer[-1].output.dense.out_features * 4 == model.roberta.encoder.layer[-1].output.dense.lora_B.numel()\n",
    "\n",
    "3072 * 768, 3072 * 4, 4 * 768"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "504cf894-298d-4f90-8b29-9abe3973c40d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2359296"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.roberta.encoder.layer[-1].output.dense.out_features * model.roberta.encoder.layer[-1].output.dense.in_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "845a10da-403d-4f00-b896-688049c24b43",
   "metadata": {},
   "source": [
    "Cool, cool, cool. \n",
    "\n",
    "We still haven't seen the performance though.\n",
    "\n",
    "Let's check into that now and submit a Training Job. We'll validate that the mechanics are working and - ideally - get an initial feeling for the impact. If everything works, then we'll tune the hyperparameters. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec0b0f43-8901-43fb-9cda-9abd79156c76",
   "metadata": {},
   "source": [
    "## Validate Mechanics / Sneak Peek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a411218f-7d3f-42ab-afa7-3fc10f57a70b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:botocore.credentials:Found credentials in shared credentials file: ~/.aws/credentials\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Preferences/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/mkamp/Library/Preferences/sagemaker/config.yaml\n",
      "INFO:botocore.credentials:Found credentials in shared credentials file: ~/.aws/credentials\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:util:Total parameters: 124,647,170, thereof learnable: 124,647,170 (100.0000%)\n"
     ]
    }
   ],
   "source": [
    "import sagemaker\n",
    "from nb_helper import p\n",
    "from sagemaker.pytorch import PyTorch\n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.utils import name_from_base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5a01c029-291d-4dd5-b4a9-8a1f4fba1487",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Preferences/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/mkamp/Library/Preferences/sagemaker/config.yaml\n"
     ]
    }
   ],
   "source": [
    "estimator_parameters = p('estimator_parameters') | \\\n",
    "{\n",
    "    'role': get_execution_role(),\n",
    "    'metric_definitions': p('metric_definitions'),\n",
    "    'hyperparameters': p('hyperparameters')\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f707646-230f-402e-b132-c7264f4fd868",
   "metadata": {},
   "source": [
    "#### Submit Full-Finetuning Training Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "86701bcb-5fdf-4540-b1cc-0f65ead98ec5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Preferences/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/mkamp/Library/Preferences/sagemaker/config.yaml\n",
      "Using provided s3_resource\n",
      "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
      "INFO:sagemaker:Creating training-job with name: lora-2023-10-31-15-48-43-327\n"
     ]
    }
   ],
   "source": [
    "fullft_estimator = PyTorch(**estimator_parameters)\n",
    "fullft_estimator.set_hyperparameters(**{'sst2-lora-config': 'none', 'sst2-learning-rate': 5e-5})\n",
    "fullft_estimator.fit(wait=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d148b35-942f-4973-92f6-584d696bcc31",
   "metadata": {},
   "source": [
    "#### Submit LoRA Finetuning Training Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "530def08-4624-4cc2-9228-28145a5f2a30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Preferences/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/mkamp/Library/Preferences/sagemaker/config.yaml\n",
      "Using provided s3_resource\n",
      "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
      "INFO:sagemaker:Creating training-job with name: lora-2023-10-31-15-48-44-924\n"
     ]
    }
   ],
   "source": [
    "loraft_estimator = PyTorch(**estimator_parameters)\n",
    "loraft_estimator.set_hyperparameters(**{'sst2-lora-config': 'query|output', 'sst2-lora-r': 2, 'sst2-learning-rate': 4e-4})\n",
    "loraft_estimator.fit(wait=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2aba4abb-a351-4db7-89c3-0d453e5e270e",
   "metadata": {},
   "source": [
    "#### First Results\n",
    "We submitted both jobs with out `wait`, so that they can run in parallel. We are waiting for both here now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "557d98a6-f179-4be6-8911-15ab05dd37c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3 \n",
    "sm = boto3.client('sagemaker')\n",
    "sm.get_waiter('training_job_completed_or_stopped').wait(TrainingJobName=fullft_estimator.latest_training_job.job_name)\n",
    "sm.get_waiter('training_job_completed_or_stopped').wait(TrainingJobName=loraft_estimator.latest_training_job.job_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "166bfc34-c718-4ae0-9416-4f11b7d26cd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_job_metrics(training_job_name, metrics=['sst2_valid_acc']):\n",
    "    ms = sm.describe_training_job(TrainingJobName=training_job_name)['FinalMetricDataList']\n",
    "    return {m['MetricName']: m['Value'] for m in ms if m['MetricName'] in metrics}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "53530f8e-6482-4be0-80e1-838d4d215cc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('full-finetuning', {'sst2_valid_acc': 0.9346330165863037})"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'full-finetuning' , get_job_metrics(fullft_estimator.latest_training_job.job_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e640ba11-8f4b-4a2b-a0b0-57313867465a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('lora-finetuning', {'sst2_valid_acc': 0.9403669834136963})"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'lora-finetuning', get_job_metrics(loraft_estimator.latest_training_job.job_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3faeda2d-f922-4875-acb2-d4d3babf1eb6",
   "metadata": {},
   "source": [
    "Whoop, whoop! That looks pretty good. \n",
    "\n",
    "On the first try? Even though in this article we have not given much thought to the hyperparameters? What is the right `r`, which modules should be adapted? What is the right learning rate and regularization?\n",
    "\n",
    "No, I cheated. When writing, I already did some experimentation in parallel. Hence I already learned about some good hyperparameters for the full finetuning baseline and the LoRA adapted model. \n",
    "\n",
    "But in the next article we will run those experiments together. The numbers above are a first taste of what's to come. Maybe we can do better? At least we should be able to understand better what are impactful design decisions we can make. "
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}